ansible-collections/community.general
1
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2025-07-22 12:50:22 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

Fix inventory cache interface (#50446)

Browse source 
* Replace InventoryFileCacheModule with a better developer-interface

Use new interface for inventory plugins with backwards compatibility

Auto-update the backing cache-plugin if the cache has changed after parsing the inventory plugin

* Update CacheModules to use the config system and add a deprecation warning if they are being imported directly rather than using cache_loader

* Fix foreman inventory caching

* Add tests

* Add integration test to check that fact caching works normally with cache plugins using ansible.constants and inventory caching provides a helpful error for non-compatible cache plugins

* Add some developer documentation for inventory and cache plugins

* Add user documentation for inventory caching

* Add deprecation docs

* Apply suggestions from docs review

* Add changelog
This commit is contained in:
Sloane Hertel 2019-03-06 12:12:35 -06:00 • committed by GitHub
commit 9687879840
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 831 additions and 86 deletions
Download patch file Download diff file Expand all files Collapse all files

145
lib/ansible/plugins/cache/__init__.py vendored
View file

@ -18,6 +18,7 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import copy
import os
import time
import errno
@ -26,8 +27,9 @@ from abc import ABCMeta, abstractmethod
from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.module_utils.six import with_metaclass
from ansible.module_utils._text import to_bytes
from ansible.module_utils._text import to_bytes, to_text
from ansible.module_utils.common._collections_compat import MutableMapping
from ansible.plugins import AnsiblePlugin
from ansible.plugins.loader import cache_loader
from ansible.utils.display import Display
from ansible.vars.fact_cache import FactCache as RealFactCache
@ -51,11 +53,16 @@ class FactCache(RealFactCache):
super(FactCache, self).__init__(*args, **kwargs)
class BaseCacheModule(with_metaclass(ABCMeta, object)):
class BaseCacheModule(AnsiblePlugin):
# Backwards compat only. Just import the global display instead
_display = display
def __init__(self, *args, **kwargs):
self._load_name = self.__module__.split('.')[-1]
super(BaseCacheModule, self).__init__()
self.set_options(var_options=args, direct=kwargs)
@abstractmethod
def get(self, key):
pass
@ -91,11 +98,15 @@ class BaseFileCacheModule(BaseCacheModule):
"""
def __init__(self, *args, **kwargs):
try:
super(BaseFileCacheModule, self).__init__(*args, **kwargs)
self._cache_dir = self._get_cache_connection(self.get_option('_uri'))
self._timeout = float(self.get_option('_timeout'))
except KeyError:
self._cache_dir = self._get_cache_connection(C.CACHE_PLUGIN_CONNECTION)
self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
self.plugin_name = self.__module__.split('.')[-1]
self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
self._cache = {}
self._cache_dir = self._get_cache_connection(C.CACHE_PLUGIN_CONNECTION)
self._set_inventory_cache_override(**kwargs)
self.validate_cache_connection()
def _get_cache_connection(self, source):
@ -105,12 +116,6 @@ class BaseFileCacheModule(BaseCacheModule):
except TypeError:
pass
def _set_inventory_cache_override(self, **kwargs):
if kwargs.get('cache_timeout'):
self._timeout = kwargs.get('cache_timeout')
if kwargs.get('cache_connection'):
self._cache_dir = self._get_cache_connection(kwargs.get('cache_connection'))
def validate_cache_connection(self):
if not self._cache_dir:
raise AnsibleError("error, '%s' cache plugin requires the 'fact_caching_connection' config option "
@ -262,50 +267,96 @@ class BaseFileCacheModule(BaseCacheModule):
pass
class InventoryFileCacheModule(BaseFileCacheModule):
class CachePluginAdjudicator(MutableMapping):
"""
A caching module backed by file based storage.
Intermediary between a cache dictionary and a CacheModule
"""
def __init__(self, plugin_name, timeout, cache_dir):
self.plugin_name = plugin_name
self._timeout = timeout
def __init__(self, plugin_name='memory', **kwargs):
self._cache = {}
self._cache_dir = self._get_cache_connection(cache_dir)
self.validate_cache_connection()
self._plugin = self.get_plugin(plugin_name)
self._retrieved = {}
def validate_cache_connection(self):
try:
super(InventoryFileCacheModule, self).validate_cache_connection()
except AnsibleError:
cache_connection_set = False
else:
cache_connection_set = True
self._plugin = cache_loader.get(plugin_name, **kwargs)
if not self._plugin:
raise AnsibleError('Unable to load the cache plugin (%s).' % plugin_name)
if not cache_connection_set:
raise AnsibleError("error, '%s' inventory cache plugin requires the one of the following to be set:\n"
"ansible.cfg:\n[default]: fact_caching_connection,\n[inventory]: cache_connection;\n"
"Environment:\nANSIBLE_INVENTORY_CACHE_CONNECTION,\nANSIBLE_CACHE_PLUGIN_CONNECTION."
"to be set to a writeable directory path" % self.plugin_name)
self._plugin_name = plugin_name
def get(self, cache_key):
def update_cache_if_changed(self):
if self._retrieved != self._cache:
self.set_cache()
if not self.contains(cache_key):
# Check if cache file exists
raise KeyError
def set_cache(self):
for top_level_cache_key in self._cache.keys():
self._plugin.set(top_level_cache_key, self._cache[top_level_cache_key])
self._retrieved = copy.deepcopy(self._cache)
return super(InventoryFileCacheModule, self).get(cache_key)
def load_whole_cache(self):
for key in self._plugin.keys():
self._cache[key] = self._plugin.get(key)
def get_plugin(self, plugin_name):
plugin = cache_loader.get(plugin_name, cache_connection=self._cache_dir, cache_timeout=self._timeout)
if not plugin:
raise AnsibleError('Unable to load the facts cache plugin (%s).' % (plugin_name))
def __repr__(self):
return to_text(self._cache)
def __iter__(self):
return iter(self.keys())
def __len__(self):
return len(self.keys())
def _do_load_key(self, key):
load = False
if key not in self._cache and key not in self._retrieved and self._plugin_name != 'memory':
if isinstance(self._plugin, BaseFileCacheModule):
load = True
elif not isinstance(self._plugin, BaseFileCacheModule) and self._plugin.contains(key):
# Database-backed caches don't raise KeyError for expired keys, so only load if the key is valid by checking contains()
load = True
return load
def __getitem__(self, key):
if self._do_load_key(key):
try:
self._cache[key] = self._plugin.get(key)
except KeyError:
pass
else:
self._retrieved[key] = self._cache[key]
return self._cache[key]
def get(self, key, default=None):
if self._do_load_key(key):
try:
self._cache[key] = self._plugin.get(key)
except KeyError as e:
pass
else:
self._retrieved[key] = self._cache[key]
return self._cache.get(key, default)
def items(self):
return self._cache.items()
def values(self):
return self._cache.values()
def keys(self):
return self._cache.keys()
def pop(self, key, *args):
if args:
return self._cache.pop(key, args[0])
return self._cache.pop(key)
def __delitem__(self, key):
del self._cache[key]
def __setitem__(self, key, value):
self._cache[key] = value
def flush(self):
for key in self._cache.keys():
self._plugin.delete(key)
self._cache = {}
return plugin
def _load(self, path):
return self._plugin._load(path)
def _dump(self, value, path):
return self._plugin._dump(value, path)
def update(self, value):
self._cache.update(value)

25
lib/ansible/plugins/cache/memcached.py vendored
View file

@ -26,6 +26,7 @@ DOCUMENTATION = '''
section: defaults
_prefix:
description: User defined prefix to use when creating the DB entries
default: ansible_facts
env:
- name: ANSIBLE_CACHE_PLUGIN_PREFIX
ini:
@ -52,12 +53,15 @@ from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.module_utils.common._collections_compat import MutableSet
from ansible.plugins.cache import BaseCacheModule
from ansible.utils.display import Display
try:
import memcache
except ImportError:
raise AnsibleError("python-memcached is required for the memcached fact cache")
display = Display()
class ProxyClientPool(object):
"""
@ -166,13 +170,22 @@ class CacheModuleKeys(MutableSet):
class CacheModule(BaseCacheModule):
def __init__(self, *args, **kwargs):
if C.CACHE_PLUGIN_CONNECTION:
connection = C.CACHE_PLUGIN_CONNECTION.split(',')
else:
connection = ['127.0.0.1:11211']
connection = ['127.0.0.1:11211']
try:
super(CacheModule, self).__init__(*args, **kwargs)
if self.get_option('_uri'):
connection = self.get_option('_uri')
self._timeout = self.get_option('_timeout')
self._prefix = self.get_option('_prefix')
except KeyError:
display.deprecated('Rather than importing CacheModules directly, '
'use ansible.plugins.loader.cache_loader', version='2.12')
if C.CACHE_PLUGIN_CONNECTION:
connection = C.CACHE_PLUGIN_CONNECTION.split(',')
self._timeout = C.CACHE_PLUGIN_TIMEOUT
self._prefix = C.CACHE_PLUGIN_PREFIX
self._timeout = C.CACHE_PLUGIN_TIMEOUT
self._prefix = C.CACHE_PLUGIN_PREFIX
self._cache = {}
self._db = ProxyClientPool(connection, debug=0)
self._keys = CacheModuleKeys(self._db, self._db.get(CacheModuleKeys.PREFIX) or [])

20
lib/ansible/plugins/cache/mongodb.py vendored
View file

@ -27,6 +27,7 @@ DOCUMENTATION = '''
section: defaults
_prefix:
description: User defined prefix to use when creating the DB entries
default: ansible_facts
env:
- name: ANSIBLE_CACHE_PLUGIN_PREFIX
ini:
@ -50,20 +51,33 @@ from contextlib import contextmanager
from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.plugins.cache import BaseCacheModule
from ansible.utils.display import Display
try:
import pymongo
except ImportError:
raise AnsibleError("The 'pymongo' python module is required for the mongodb fact cache, 'pip install pymongo>=3.0'")
display = Display()
class CacheModule(BaseCacheModule):
"""
A caching module backed by mongodb.
"""
def __init__(self, *args, **kwargs):
self._timeout = int(C.CACHE_PLUGIN_TIMEOUT)
self._prefix = C.CACHE_PLUGIN_PREFIX
try:
super(CacheModule, self).__init__(*args, **kwargs)
self._connection = self.get_option('_uri')
self._timeout = int(self.get_option('_timeout'))
self._prefix = self.get_option('_prefix')
except KeyError:
display.deprecated('Rather than importing CacheModules directly, '
'use ansible.plugins.loader.cache_loader', version='2.12')
self._connection = C.CACHE_PLUGIN_CONNECTION
self._timeout = int(C.CACHE_PLUGIN_TIMEOUT)
self._prefix = C.CACHE_PLUGIN_PREFIX
self._cache = {}
self._managed_indexes = False
@ -94,7 +108,7 @@ class CacheModule(BaseCacheModule):
This is a context manager for opening and closing mongo connections as needed. This exists as to not create a global
connection, due to pymongo not being fork safe (http://api.mongodb.com/python/current/faq.html#is-pymongo-fork-safe)
'''
mongo = pymongo.MongoClient(C.CACHE_PLUGIN_CONNECTION)
mongo = pymongo.MongoClient(self._connection)
try:
db = mongo.get_default_database()
except pymongo.errors.ConfigurationError:

30
lib/ansible/plugins/cache/redis.py vendored
View file

@ -24,6 +24,7 @@ DOCUMENTATION = '''
section: defaults
_prefix:
description: User defined prefix to use when creating the DB entries
default: ansible_facts
env:
- name: ANSIBLE_CACHE_PLUGIN_PREFIX
ini:
@ -45,13 +46,17 @@ import json
from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.parsing.ajson import AnsibleJSONEncoder, AnsibleJSONDecoder
from ansible.plugins.cache import BaseCacheModule
from ansible.utils.display import Display
try:
from redis import StrictRedis, VERSION
except ImportError:
raise AnsibleError("The 'redis' python module (version 2.4.5 or newer) is required for the redis fact cache, 'pip install redis'")
display = Display()
class CacheModule(BaseCacheModule):
"""
@ -63,13 +68,22 @@ class CacheModule(BaseCacheModule):
performance.
"""
def __init__(self, *args, **kwargs):
if C.CACHE_PLUGIN_CONNECTION:
connection = C.CACHE_PLUGIN_CONNECTION.split(':')
else:
connection = []
connection = []
try:
super(CacheModule, self).__init__(*args, **kwargs)
if self.get_option('_uri'):
connection = self.get_option('_uri').split(':')
self._timeout = float(self.get_option('_timeout'))
self._prefix = self.get_option('_prefix')
except KeyError:
display.deprecated('Rather than importing CacheModules directly, '
'use ansible.plugins.loader.cache_loader', version='2.12')
if C.CACHE_PLUGIN_CONNECTION:
connection = C.CACHE_PLUGIN_CONNECTION.split(':')
self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
self._prefix = C.CACHE_PLUGIN_PREFIX
self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
self._prefix = C.CACHE_PLUGIN_PREFIX
self._cache = {}
self._db = StrictRedis(*connection)
self._keys_set = 'ansible_cache_keys'
@ -87,13 +101,13 @@ class CacheModule(BaseCacheModule):
if value is None:
self.delete(key)
raise KeyError
self._cache[key] = json.loads(value)
self._cache[key] = json.loads(value, cls=AnsibleJSONDecoder)
return self._cache.get(key)
def set(self, key, value):
value2 = json.dumps(value)
value2 = json.dumps(value, cls=AnsibleJSONEncoder, sort_keys=True, indent=4)
if self._timeout > 0: # a timeout of 0 is handled as meaning 'never expire'
self._db.setex(self._make_key(key), int(self._timeout), value2)
else:

22
lib/ansible/plugins/doc_fragments/inventory_cache.py
View file

@ -23,9 +23,13 @@ options:
description:
- Cache plugin to use for the inventory's source data.
type: str
default: memory
env:
- name: ANSIBLE_CACHE_PLUGIN
- name: ANSIBLE_INVENTORY_CACHE_PLUGIN
ini:
- section: defaults
key: fact_caching
- section: inventory
key: cache_plugin
cache_timeout:
@ -34,8 +38,11 @@ options:
default: 3600
type: int
env:
- name: ANSIBLE_CACHE_PLUGIN_TIMEOUT
- name: ANSIBLE_INVENTORY_CACHE_TIMEOUT
ini:
- section: defaults
key: fact_caching_timeout
- section: inventory
key: cache_timeout
cache_connection:
@ -43,8 +50,23 @@ options:
- Cache connection data or path, read cache plugin documentation for specifics.
type: str
env:
- name: ANSIBLE_CACHE_PLUGIN_CONNECTION
- name: ANSIBLE_INVENTORY_CACHE_CONNECTION
ini:
- section: defaults
key: fact_caching_connection
- section: inventory
key: cache_connection
cache_prefix:
description:
- Prefix to use for cache plugin files/tables
default: ansible_inventory_
env:
- name: ANSIBLE_CACHE_PLUGIN_PREFIX
- name: ANSIBLE_INVENTORY_CACHE_PLUGIN_PREFIX
ini:
- section: default
key: fact_caching_prefix
- section: inventory
key: cache_prefix
'''

80
lib/ansible/plugins/inventory/__init__.py
View file

@ -27,7 +27,7 @@ from ansible.errors import AnsibleError, AnsibleParserError
from ansible.inventory.group import to_safe_group_name as original_safe
from ansible.parsing.utils.addresses import parse_address
from ansible.plugins import AnsiblePlugin
from ansible.plugins.cache import InventoryFileCacheModule
from ansible.plugins.cache import CachePluginAdjudicator as CacheObject
from ansible.module_utils._text import to_bytes, to_native
from ansible.module_utils.common._collections_compat import Mapping
from ansible.module_utils.parsing.convert_bool import boolean
@ -126,6 +126,25 @@ def expand_hostname_range(line=None):
return all_hosts
def get_cache_plugin(plugin_name, **kwargs):
try:
cache = CacheObject(plugin_name, **kwargs)
except AnsibleError as e:
if 'fact_caching_connection' in to_native(e):
raise AnsibleError("error, '%s' inventory cache plugin requires the one of the following to be set "
"to a writeable directory path:\nansible.cfg:\n[default]: fact_caching_connection,\n"
"[inventory]: cache_connection;\nEnvironment:\nANSIBLE_INVENTORY_CACHE_CONNECTION,\n"
"ANSIBLE_CACHE_PLUGIN_CONNECTION." % plugin_name)
else:
raise e
if plugin_name != 'memory' and kwargs and not getattr(cache._plugin, '_options', None):
raise AnsibleError('Unable to use cache plugin {0} for inventory. Cache options were provided but may not reconcile '
'correctly unless set via set_options. Refer to the porting guide if the plugin derives user settings '
'from ansible.constants.'.format(plugin_name))
return cache
class BaseInventoryPlugin(AnsiblePlugin):
""" Parses an Inventory Source"""
@ -138,7 +157,6 @@ class BaseInventoryPlugin(AnsiblePlugin):
self._options = {}
self.inventory = None
self.display = display
self.cache = None
def parse(self, inventory, loader, path, cache=True):
''' Populates inventory from the given data. Raises an error on any parse failure
@ -207,16 +225,13 @@ class BaseInventoryPlugin(AnsiblePlugin):
raise AnsibleParserError('inventory source has invalid structure, it should be a dictionary, got: %s' % type(config))
self.set_options(direct=config)
if self._options.get('cache'):
self._set_cache_options(self._options)
if 'cache' in self._options and self.get_option('cache'):
cache_option_keys = [('_uri', 'cache_connection'), ('_timeout', 'cache_timeout'), ('_prefix', 'cache_prefix')]
cache_options = dict((opt[0], self.get_option(opt[1])) for opt in cache_option_keys if self.get_option(opt[1]))
self._cache = get_cache_plugin(self.get_option('cache_plugin'), **cache_options)
return config
def _set_cache_options(self, options):
self.cache = InventoryFileCacheModule(plugin_name=options.get('cache_plugin'),
timeout=options.get('cache_timeout'),
cache_dir=options.get('cache_connection'))
def _consume_options(self, data):
''' update existing options from alternate configuration sources not normally used by Ansible.
Many API libraries already have existing configuration sources, this allows plugin author to leverage them.
@ -252,9 +267,6 @@ class BaseInventoryPlugin(AnsiblePlugin):
return (hostnames, port)
def clear_cache(self):
pass
class BaseFileInventoryPlugin(BaseInventoryPlugin):
""" Parses a File based Inventory Source"""
@ -266,9 +278,43 @@ class BaseFileInventoryPlugin(BaseInventoryPlugin):
super(BaseFileInventoryPlugin, self).__init__()
class DeprecatedCache(object):
def __init__(self, real_cacheable):
self.real_cacheable = real_cacheable
def get(self, key):
display.deprecated('InventoryModule should utilize self._cache as a dict instead of self.cache. '
'When expecting a KeyError, use self._cache[key] instead of using self.cache.get(key). '
'self._cache is a dictionary and will return a default value instead of raising a KeyError '
'when the key does not exist', version='2.12')
return self.real_cacheable._cache[key]
def set(self, key, value):
display.deprecated('InventoryModule should utilize self._cache as a dict instead of self.cache. '
'To set the self._cache dictionary, use self._cache[key] = value instead of self.cache.set(key, value). '
'To force update the underlying cache plugin with the contents of self._cache before parse() is complete, '
'call self.set_cache_plugin and it will use the self._cache dictionary to update the cache plugin', version='2.12')
self.real_cacheable._cache[key] = value
self.real_cacheable.set_cache_plugin()
def __getattr__(self, name):
display.deprecated('InventoryModule should utilize self._cache instead of self.cache', version='2.12')
return self.real_cacheable._cache.__getattribute__(name)
class Cacheable(object):
_cache = {}
_cache = CacheObject()
@property
def cache(self):
return DeprecatedCache(self)
def load_cache_plugin(self):
plugin_name = self.get_option('cache_plugin')
cache_option_keys = [('_uri', 'cache_connection'), ('_timeout', 'cache_timeout'), ('_prefix', 'cache_prefix')]
cache_options = dict((opt[0], self.get_option(opt[1])) for opt in cache_option_keys if self.get_option(opt[1]))
self._cache = get_cache_plugin(plugin_name, **cache_options)
def get_cache_key(self, path):
return "{0}_{1}".format(self.NAME, self._get_cache_prefix(path))
@ -287,7 +333,13 @@ class Cacheable(object):
return 's_'.join([d1[:5], d2[:5]])
def clear_cache(self):
self._cache = {}
self._cache.flush()
def update_cache_if_changed(self):
self._cache.update_cache_if_changed()
def set_cache_plugin(self):
self._cache.set_cache()
class Constructable(object):

2
lib/ansible/plugins/inventory/auto.py
View file

@ -53,3 +53,5 @@ class InventoryModule(BaseInventoryPlugin):
raise AnsibleParserError("inventory config '{0}' could not be verified by plugin '{1}'".format(path, plugin_name))
plugin.parse(inventory, loader, path, cache=cache)
if getattr(plugin, '_cache', None):
plugin.update_cache_if_changed()

6
lib/ansible/plugins/inventory/foreman.py
View file

@ -116,7 +116,7 @@ class InventoryModule(BaseInventoryPlugin, Cacheable):
if not self.use_cache or url not in self._cache.get(self.cache_key, {}):
if self.cache_key not in self._cache:
self._cache[self.cache_key] = {'url': ''}
self._cache[self.cache_key] = {url: ''}
results = []
s = self._get_session()
@ -153,7 +153,9 @@ class InventoryModule(BaseInventoryPlugin, Cacheable):
# get next page
params['page'] += 1
self._cache[self.cache_key][url] = results
# Set the cache if it is enabled or if the cache was refreshed
if self.use_cache or self.get_option('cache'):
self._cache[self.cache_key][url] = results
return self._cache[self.cache_key][url]