Fix inventory cache interface (#50446)

* Replace InventoryFileCacheModule with a better developer-interface

Use new interface for inventory plugins with backwards compatibility

Auto-update the backing cache-plugin if the cache has changed after parsing the inventory plugin

* Update CacheModules to use the config system and add a deprecation warning if they are being imported directly rather than using cache_loader

* Fix foreman inventory caching

* Add tests

* Add integration test to check that fact caching works normally with cache plugins using ansible.constants and inventory caching provides a helpful error for non-compatible cache plugins

* Add some developer documentation for inventory and cache plugins

* Add user documentation for inventory caching

* Add deprecation docs

* Apply suggestions from docs review

* Add changelog

lib/ansible/plugins/cache/__init__.py

@@ -18,6 +18,7 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+import copy
 import os
 import time
 import errno
@@ -26,8 +27,9 @@ from abc import ABCMeta, abstractmethod
 from ansible import constants as C
 from ansible.errors import AnsibleError
 from ansible.module_utils.six import with_metaclass
-from ansible.module_utils._text import to_bytes
+from ansible.module_utils._text import to_bytes, to_text
 from ansible.module_utils.common._collections_compat import MutableMapping
+from ansible.plugins import AnsiblePlugin
 from ansible.plugins.loader import cache_loader
 from ansible.utils.display import Display
 from ansible.vars.fact_cache import FactCache as RealFactCache
@@ -51,11 +53,16 @@ class FactCache(RealFactCache):
         super(FactCache, self).__init__(*args, **kwargs)
 
 
-class BaseCacheModule(with_metaclass(ABCMeta, object)):
+class BaseCacheModule(AnsiblePlugin):
 
     # Backwards compat only.  Just import the global display instead
     _display = display
 
+    def __init__(self, *args, **kwargs):
+        self._load_name = self.__module__.split('.')[-1]
+        super(BaseCacheModule, self).__init__()
+        self.set_options(var_options=args, direct=kwargs)
+
     @abstractmethod
     def get(self, key):
         pass
@@ -91,11 +98,15 @@ class BaseFileCacheModule(BaseCacheModule):
     """
     def __init__(self, *args, **kwargs):
 
+        try:
+            super(BaseFileCacheModule, self).__init__(*args, **kwargs)
+            self._cache_dir = self._get_cache_connection(self.get_option('_uri'))
+            self._timeout = float(self.get_option('_timeout'))
+        except KeyError:
+            self._cache_dir = self._get_cache_connection(C.CACHE_PLUGIN_CONNECTION)
+            self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
         self.plugin_name = self.__module__.split('.')[-1]
-        self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
         self._cache = {}
-        self._cache_dir = self._get_cache_connection(C.CACHE_PLUGIN_CONNECTION)
-        self._set_inventory_cache_override(**kwargs)
         self.validate_cache_connection()
 
     def _get_cache_connection(self, source):
@@ -105,12 +116,6 @@ class BaseFileCacheModule(BaseCacheModule):
             except TypeError:
                 pass
 
-    def _set_inventory_cache_override(self, **kwargs):
-        if kwargs.get('cache_timeout'):
-            self._timeout = kwargs.get('cache_timeout')
-        if kwargs.get('cache_connection'):
-            self._cache_dir = self._get_cache_connection(kwargs.get('cache_connection'))
-
     def validate_cache_connection(self):
         if not self._cache_dir:
             raise AnsibleError("error, '%s' cache plugin requires the 'fact_caching_connection' config option "
@@ -262,50 +267,96 @@ class BaseFileCacheModule(BaseCacheModule):
         pass
 
 
-class InventoryFileCacheModule(BaseFileCacheModule):
+class CachePluginAdjudicator(MutableMapping):
     """
-    A caching module backed by file based storage.
+    Intermediary between a cache dictionary and a CacheModule
     """
-    def __init__(self, plugin_name, timeout, cache_dir):
-
-        self.plugin_name = plugin_name
-        self._timeout = timeout
+    def __init__(self, plugin_name='memory', **kwargs):
         self._cache = {}
-        self._cache_dir = self._get_cache_connection(cache_dir)
-        self.validate_cache_connection()
-        self._plugin = self.get_plugin(plugin_name)
+        self._retrieved = {}
 
-    def validate_cache_connection(self):
-        try:
-            super(InventoryFileCacheModule, self).validate_cache_connection()
-        except AnsibleError:
-            cache_connection_set = False
-        else:
-            cache_connection_set = True
+        self._plugin = cache_loader.get(plugin_name, **kwargs)
+        if not self._plugin:
+            raise AnsibleError('Unable to load the cache plugin (%s).' % plugin_name)
 
-        if not cache_connection_set:
-            raise AnsibleError("error, '%s' inventory cache plugin requires the one of the following to be set:\n"
-                               "ansible.cfg:\n[default]: fact_caching_connection,\n[inventory]: cache_connection;\n"
-                               "Environment:\nANSIBLE_INVENTORY_CACHE_CONNECTION,\nANSIBLE_CACHE_PLUGIN_CONNECTION."
-                               "to be set to a writeable directory path" % self.plugin_name)
+        self._plugin_name = plugin_name
 
-    def get(self, cache_key):
+    def update_cache_if_changed(self):
+        if self._retrieved != self._cache:
+            self.set_cache()
 
-        if not self.contains(cache_key):
-            # Check if cache file exists
-            raise KeyError
+    def set_cache(self):
+        for top_level_cache_key in self._cache.keys():
+            self._plugin.set(top_level_cache_key, self._cache[top_level_cache_key])
+        self._retrieved = copy.deepcopy(self._cache)
 
-        return super(InventoryFileCacheModule, self).get(cache_key)
+    def load_whole_cache(self):
+        for key in self._plugin.keys():
+            self._cache[key] = self._plugin.get(key)
 
-    def get_plugin(self, plugin_name):
-        plugin = cache_loader.get(plugin_name, cache_connection=self._cache_dir, cache_timeout=self._timeout)
-        if not plugin:
-            raise AnsibleError('Unable to load the facts cache plugin (%s).' % (plugin_name))
+    def __repr__(self):
+        return to_text(self._cache)
+
+    def __iter__(self):
+        return iter(self.keys())
+
+    def __len__(self):
+        return len(self.keys())
+
+    def _do_load_key(self, key):
+        load = False
+        if key not in self._cache and key not in self._retrieved and self._plugin_name != 'memory':
+            if isinstance(self._plugin, BaseFileCacheModule):
+                load = True
+            elif not isinstance(self._plugin, BaseFileCacheModule) and self._plugin.contains(key):
+                # Database-backed caches don't raise KeyError for expired keys, so only load if the key is valid by checking contains()
+                load = True
+        return load
+
+    def __getitem__(self, key):
+        if self._do_load_key(key):
+            try:
+                self._cache[key] = self._plugin.get(key)
+            except KeyError:
+                pass
+            else:
+                self._retrieved[key] = self._cache[key]
+        return self._cache[key]
+
+    def get(self, key, default=None):
+        if self._do_load_key(key):
+            try:
+                self._cache[key] = self._plugin.get(key)
+            except KeyError as e:
+                pass
+            else:
+                self._retrieved[key] = self._cache[key]
+        return self._cache.get(key, default)
+
+    def items(self):
+        return self._cache.items()
+
+    def values(self):
+        return self._cache.values()
+
+    def keys(self):
+        return self._cache.keys()
+
+    def pop(self, key, *args):
+        if args:
+            return self._cache.pop(key, args[0])
+        return self._cache.pop(key)
+
+    def __delitem__(self, key):
+        del self._cache[key]
+
+    def __setitem__(self, key, value):
+        self._cache[key] = value
+
+    def flush(self):
+        for key in self._cache.keys():
+            self._plugin.delete(key)
         self._cache = {}
-        return plugin
 
-    def _load(self, path):
-        return self._plugin._load(path)
-
-    def _dump(self, value, path):
-        return self._plugin._dump(value, path)
+    def update(self, value):
+        self._cache.update(value)