diff --git a/CHANGELOG.md b/CHANGELOG.md
index fef5615900..e91fb5ea83 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -267,6 +267,11 @@ Ansible Changes By Release
* keyring: allows getting password from system keyrings
+####New: cache
+
+* pickle (uses python's own serializer)
+* yaml
+
## 2.2.1 "The Battle of Evermore" - 2017-01-16
diff --git a/lib/ansible/plugins/cache/base.py b/lib/ansible/plugins/cache/base.py
index 0367d01534..47988a958f 100644
--- a/lib/ansible/plugins/cache/base.py
+++ b/lib/ansible/plugins/cache/base.py
@@ -18,9 +18,17 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
+import os
+import time
+import errno
+import codecs
+
from abc import ABCMeta, abstractmethod
+from ansible import constants as C
from ansible.compat.six import with_metaclass
+from ansible.errors import AnsibleError
+from ansible.module_utils._text import to_bytes
try:
from __main__ import display
@@ -28,7 +36,6 @@ except ImportError:
from ansible.utils.display import Display
display = Display()
-
class BaseCacheModule(with_metaclass(ABCMeta, object)):
# Backwards compat only. Just import the global display instead
@@ -62,3 +69,152 @@ class BaseCacheModule(with_metaclass(ABCMeta, object)):
def copy(self):
pass
+
+class BaseFileCacheModule(BaseCacheModule):
+ """
+ A caching module backed by file based storage.
+ """
+ plugin_name = None
+ read_mode = 'r'
+ write_mode = 'w'
+ encoding = 'utf-8'
+ def __init__(self, *args, **kwargs):
+
+ self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
+ self._cache = {}
+ self._cache_dir = None
+
+ if C.CACHE_PLUGIN_CONNECTION:
+ # expects a dir path
+ self._cache_dir = os.path.expanduser(os.path.expandvars(C.CACHE_PLUGIN_CONNECTION))
+
+ if not self._cache_dir:
+ raise AnsibleError("error, '%s' cache plugin requires the 'fact_caching_connection' config option to be set (to a writeable directory path)" % self.plugin_name)
+
+ if not os.path.exists(self._cache_dir):
+ try:
+ os.makedirs(self._cache_dir)
+ except (OSError,IOError) as e:
+ display.warning("error in '%s' cache plugin while trying to create cache dir %s : %s" % (self.plugin_name, self._cache_dir, to_bytes(e)))
+ return None
+
+ def get(self, key):
+ """ This checks the in memory cache first as the fact was not expired at 'gather time'
+ and it would be problematic if the key did expire after some long running tasks and
+ user gets 'undefined' error in the same play """
+
+ if key in self._cache:
+ return self._cache.get(key)
+
+ if self.has_expired(key) or key == "":
+ raise KeyError
+
+ cachefile = "%s/%s" % (self._cache_dir, key)
+ try:
+ with codecs.open(cachefile, self.read_mode, encoding=self.encoding) as f:
+ try:
+ value = self._load(f)
+ self._cache[key] = value
+ return value
+ except ValueError as e:
+ display.warning("error in '%s' cache plugin while trying to read %s : %s. Most likely a corrupt file, so erasing and failing." % (self.plugin_name, cachefile, to_bytes(e)))
+ self.delete(key)
+ raise AnsibleError("The cache file %s was corrupt, or did not otherwise contain valid data. It has been removed, so you can re-run your command now." % cachefile)
+ except (OSError,IOError) as e:
+ display.warning("error in '%s' cache plugin while trying to read %s : %s" % (self.plugin_name, cachefile, to_bytes(e)))
+ raise KeyError
+ except Exception as e:
+ raise AnsibleError("Error while decoding the cache file %s: %s" % (cachefile, to_bytes(e)))
+
+ def set(self, key, value):
+
+ self._cache[key] = value
+
+ cachefile = "%s/%s" % (self._cache_dir, key)
+ try:
+ f = codecs.open(cachefile, self.write_mode, encoding=self.encoding)
+ except (OSError,IOError) as e:
+ display.warning("error in '%s' cache plugin while trying to write to %s : %s" % (self.plugin_name, cachefile, to_bytes(e)))
+ pass
+ else:
+ self._dump(value, f)
+ finally:
+ try:
+ f.close()
+ except UnboundLocalError:
+ pass
+
+ def has_expired(self, key):
+
+ if self._timeout == 0:
+ return False
+
+ cachefile = "%s/%s" % (self._cache_dir, key)
+ try:
+ st = os.stat(cachefile)
+ except (OSError,IOError) as e:
+ if e.errno == errno.ENOENT:
+ return False
+ else:
+ display.warning("error in '%s' cache plugin while trying to stat %s : %s" % (self.plugin_name, cachefile, to_bytes(e)))
+ pass
+
+ if time.time() - st.st_mtime <= self._timeout:
+ return False
+
+ if key in self._cache:
+ del self._cache[key]
+ return True
+
+ def keys(self):
+ keys = []
+ for k in os.listdir(self._cache_dir):
+ if not (k.startswith('.') or self.has_expired(k)):
+ keys.append(k)
+ return keys
+
+ def contains(self, key):
+ cachefile = "%s/%s" % (self._cache_dir, key)
+
+ if key in self._cache:
+ return True
+
+ if self.has_expired(key):
+ return False
+ try:
+ os.stat(cachefile)
+ return True
+ except (OSError,IOError) as e:
+ if e.errno == errno.ENOENT:
+ return False
+ else:
+ display.warning("error in '%s' cache plugin while trying to stat %s : %s" % (self.plugin_name, cachefile, to_bytes(e)))
+ pass
+
+ def delete(self, key):
+ try:
+ del self._cache[key]
+ except KeyError:
+ pass
+ try:
+ os.remove("%s/%s" % (self._cache_dir, key))
+ except (OSError, IOError):
+ pass #TODO: only pass on non existing?
+
+ def flush(self):
+ self._cache = {}
+ for key in self.keys():
+ self.delete(key)
+
+ def copy(self):
+ ret = dict()
+ for key in self.keys():
+ ret[key] = self.get(key)
+ return ret
+
+ def _load(self, f):
+ raise AnsibleError("Plugin '%s' must implement _load method" % self.plugin_name)
+
+ def _dump(self, value, f):
+ raise AnsibleError("Plugin '%s' must implement _dump method" % self.plugin_name)
+
diff --git a/lib/ansible/plugins/cache/jsonfile.py b/lib/ansible/plugins/cache/jsonfile.py
index e1669068fc..9f10dfd1f2 100644
--- a/lib/ansible/plugins/cache/jsonfile.py
+++ b/lib/ansible/plugins/cache/jsonfile.py
@@ -19,162 +19,22 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
-import os
-import time
-import errno
-import codecs
-
try:
import simplejson as json
except ImportError:
import json
-from ansible import constants as C
-from ansible.errors import AnsibleError
-from ansible.module_utils._text import to_bytes
from ansible.parsing.utils.jsonify import jsonify
-from ansible.plugins.cache.base import BaseCacheModule
+from ansible.plugins.cache.base import BaseFileCacheModule
-try:
- from __main__ import display
-except ImportError:
- from ansible.utils.display import Display
- display = Display()
-
-
-class CacheModule(BaseCacheModule):
+class CacheModule(BaseFileCacheModule):
"""
A caching module backed by json files.
"""
- def __init__(self, *args, **kwargs):
+ plugin_name = 'jsonfile'
- self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
- self._cache = {}
- self._cache_dir = None
+ def _load(self, f):
+ return json.load(f)
- if C.CACHE_PLUGIN_CONNECTION:
- # expects a dir path
- self._cache_dir = os.path.expanduser(os.path.expandvars(C.CACHE_PLUGIN_CONNECTION))
-
- if not self._cache_dir:
- raise AnsibleError("error, 'jsonfile' cache plugin requires the 'fact_caching_connection' config option to be set (to a writeable directory path)")
-
- if not os.path.exists(self._cache_dir):
- try:
- os.makedirs(self._cache_dir)
- except (OSError,IOError) as e:
- display.warning("error in 'jsonfile' cache plugin while trying to create cache dir %s : %s" % (self._cache_dir, to_bytes(e)))
- return None
-
- def get(self, key):
- """ This checks the in memory cache first as the fact was not expired at 'gather time'
- and it would be problematic if the key did expire after some long running tasks and
- user gets 'undefined' error in the same play """
-
- if key in self._cache:
- return self._cache.get(key)
-
- if self.has_expired(key) or key == "":
- raise KeyError
-
- cachefile = "%s/%s" % (self._cache_dir, key)
- try:
- with codecs.open(cachefile, 'r', encoding='utf-8') as f:
- try:
- value = json.load(f)
- self._cache[key] = value
- return value
- except ValueError as e:
- display.warning("error in 'jsonfile' cache plugin while trying to read %s : %s. Most likely a corrupt file, so erasing and failing." % (cachefile, to_bytes(e)))
- self.delete(key)
- raise AnsibleError("The JSON cache file %s was corrupt, or did not otherwise contain valid JSON data."
- " It has been removed, so you can re-run your command now." % cachefile)
- except (OSError,IOError) as e:
- display.warning("error in 'jsonfile' cache plugin while trying to read %s : %s" % (cachefile, to_bytes(e)))
- raise KeyError
-
- def set(self, key, value):
-
- self._cache[key] = value
-
- cachefile = "%s/%s" % (self._cache_dir, key)
- try:
- f = codecs.open(cachefile, 'w', encoding='utf-8')
- except (OSError,IOError) as e:
- display.warning("error in 'jsonfile' cache plugin while trying to write to %s : %s" % (cachefile, to_bytes(e)))
- pass
- else:
- f.write(jsonify(value, format=True))
- finally:
- try:
- f.close()
- except UnboundLocalError:
- pass
-
- def has_expired(self, key):
-
- if self._timeout == 0:
- return False
-
- cachefile = "%s/%s" % (self._cache_dir, key)
- try:
- st = os.stat(cachefile)
- except (OSError,IOError) as e:
- if e.errno == errno.ENOENT:
- return False
- else:
- display.warning("error in 'jsonfile' cache plugin while trying to stat %s : %s" % (cachefile, to_bytes(e)))
- pass
-
- if time.time() - st.st_mtime <= self._timeout:
- return False
-
- if key in self._cache:
- del self._cache[key]
- return True
-
- def keys(self):
- keys = []
- for k in os.listdir(self._cache_dir):
- if not (k.startswith('.') or self.has_expired(k)):
- keys.append(k)
- return keys
-
- def contains(self, key):
- cachefile = "%s/%s" % (self._cache_dir, key)
-
- if key in self._cache:
- return True
-
- if self.has_expired(key):
- return False
- try:
- os.stat(cachefile)
- return True
- except (OSError,IOError) as e:
- if e.errno == errno.ENOENT:
- return False
- else:
- display.warning("error in 'jsonfile' cache plugin while trying to stat %s : %s" % (cachefile, to_bytes(e)))
- pass
-
- def delete(self, key):
- try:
- del self._cache[key]
- except KeyError:
- pass
- try:
- os.remove("%s/%s" % (self._cache_dir, key))
- except (OSError, IOError):
- pass #TODO: only pass on non existing?
-
- def flush(self):
- self._cache = {}
- for key in self.keys():
- self.delete(key)
-
- def copy(self):
- ret = dict()
- for key in self.keys():
- ret[key] = self.get(key)
- return ret
+ def _dump(self, value, f):
+ f.write(jsonify(value, format=True))
diff --git a/lib/ansible/plugins/cache/pickle.py b/lib/ansible/plugins/cache/pickle.py
new file mode 100644
index 0000000000..060c0381ec
--- /dev/null
+++ b/lib/ansible/plugins/cache/pickle.py
@@ -0,0 +1,42 @@
+# (c) 2017, Brian Coca
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible. If not, see .
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
+
+from ansible.plugins.cache.base import BaseFileCacheModule
+
+class CacheModule(BaseFileCacheModule):
+ """
+ A caching module backed by pickle files.
+ """
+ plugin_name = 'pickle'
+ read_mode = 'rb'
+ write_mode = 'wb'
+ encoding = None
+
+ def _load(self, f):
+ return pickle.load(f)
+
+ def _dump(self, value, f):
+ pickle.dump(value, f)
diff --git a/lib/ansible/plugins/cache/yaml.py b/lib/ansible/plugins/cache/yaml.py
new file mode 100644
index 0000000000..ecfb62fe69
--- /dev/null
+++ b/lib/ansible/plugins/cache/yaml.py
@@ -0,0 +1,38 @@
+# (c) 2017, Brian Coca
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible. If not, see .
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import yaml
+
+from ansible.parsing.yaml.loader import AnsibleLoader
+from ansible.parsing.yaml.dumper import AnsibleDumper
+from ansible.plugins.cache.base import BaseFileCacheModule
+
+class CacheModule(BaseFileCacheModule):
+ """
+ A caching module backed by yaml files.
+ """
+ plugin_name = 'yaml'
+
+ def _load(self, f):
+ return AnsibleLoader(f).get_single_data()
+
+ def _dump(self, value, f):
+ yaml.dump(value, f, Dumper=AnsibleDumper, default_flow_style=False)
diff --git a/test/sanity/pep8/legacy-files.txt b/test/sanity/pep8/legacy-files.txt
index 84813614ef..12c2f9a01c 100644
--- a/test/sanity/pep8/legacy-files.txt
+++ b/test/sanity/pep8/legacy-files.txt
@@ -243,7 +243,7 @@ lib/ansible/playbook/role/metadata.py
lib/ansible/plugins/action/set_fact.py
lib/ansible/plugins/action/set_stats.py
lib/ansible/plugins/action/synchronize.py
-lib/ansible/plugins/cache/jsonfile.py
+lib/ansible/plugins/cache/base.py
lib/ansible/plugins/callback/default.py
lib/ansible/plugins/callback/logentries.py
lib/ansible/plugins/callback/oneline.py