Adding VariableManager class for v2

v2/ansible/plugins/cache/__init__.py

@@ -0,0 +1,59 @@
+# (c) 2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+from collections import MutableMapping
+
+from ansible import constants as C
+from ansible.plugins import cache_loader
+
+class FactCache(MutableMapping):
+
+    def __init__(self, *args, **kwargs):
+        self._plugin = cache_loader.get(C.CACHE_PLUGIN)
+        if self._plugin is None:
+            return
+
+    def __getitem__(self, key):
+        if key not in self:
+            raise KeyError
+        return self._plugin.get(key)
+
+    def __setitem__(self, key, value):
+        self._plugin.set(key, value)
+
+    def __delitem__(self, key):
+        self._plugin.delete(key)
+
+    def __contains__(self, key):
+        return self._plugin.contains(key)
+
+    def __iter__(self):
+        return iter(self._plugin.keys())
+
+    def __len__(self):
+        return len(self._plugin.keys())
+
+    def copy(self):
+        """ Return a primitive copy of the keys and values from the cache. """
+        return dict([(k, v) for (k, v) in self.iteritems()])
+
+    def keys(self):
+        return self._plugin.keys()
+
+    def flush(self):
+        """ Flush the fact cache of all keys. """
+        self._plugin.flush()

v2/ansible/plugins/cache/base.py

@@ -0,0 +1,41 @@
+# (c) 2014, Brian Coca, Josh Drake, et al
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import exceptions
+
+class BaseCacheModule(object):
+
+    def get(self, key):
+        raise exceptions.NotImplementedError
+
+    def set(self, key, value):
+        raise exceptions.NotImplementedError
+
+    def keys(self):
+        raise exceptions.NotImplementedError
+
+    def contains(self, key):
+        raise exceptions.NotImplementedError
+
+    def delete(self, key):
+        raise exceptions.NotImplementedError
+
+    def flush(self):
+        raise exceptions.NotImplementedError
+
+    def copy(self):
+        raise exceptions.NotImplementedError

v2/ansible/plugins/cache/memcached.py

@@ -0,0 +1,191 @@
+# (c) 2014, Brian Coca, Josh Drake, et al
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import collections
+import os
+import sys
+import time
+import threading
+from itertools import chain
+
+from ansible import constants as C
+from ansible.plugins.cache.base import BaseCacheModule
+
+try:
+    import memcache
+except ImportError:
+    print 'python-memcached is required for the memcached fact cache'
+    sys.exit(1)
+
+
+class ProxyClientPool(object):
+    """
+    Memcached connection pooling for thread/fork safety. Inspired by py-redis
+    connection pool.
+
+    Available connections are maintained in a deque and released in a FIFO manner.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.max_connections = kwargs.pop('max_connections', 1024)
+        self.connection_args = args
+        self.connection_kwargs = kwargs
+        self.reset()
+
+    def reset(self):
+        self.pid = os.getpid()
+        self._num_connections = 0
+        self._available_connections = collections.deque(maxlen=self.max_connections)
+        self._locked_connections = set()
+        self._lock = threading.Lock()
+
+    def _check_safe(self):
+        if self.pid != os.getpid():
+            with self._lock:
+                if self.pid == os.getpid():
+                    # bail out - another thread already acquired the lock
+                    return
+                self.disconnect_all()
+                self.reset()
+
+    def get_connection(self):
+        self._check_safe()
+        try:
+            connection = self._available_connections.popleft()
+        except IndexError:
+            connection = self.create_connection()
+        self._locked_connections.add(connection)
+        return connection
+
+    def create_connection(self):
+        if self._num_connections >= self.max_connections:
+            raise RuntimeError("Too many memcached connections")
+        self._num_connections += 1
+        return memcache.Client(*self.connection_args, **self.connection_kwargs)
+
+    def release_connection(self, connection):
+        self._check_safe()
+        self._locked_connections.remove(connection)
+        self._available_connections.append(connection)
+
+    def disconnect_all(self):
+        for conn in chain(self._available_connections, self._locked_connections):
+            conn.disconnect_all()
+
+    def __getattr__(self, name):
+        def wrapped(*args, **kwargs):
+            return self._proxy_client(name, *args, **kwargs)
+        return wrapped
+
+    def _proxy_client(self, name, *args, **kwargs):
+        conn = self.get_connection()
+
+        try:
+            return getattr(conn, name)(*args, **kwargs)
+        finally:
+            self.release_connection(conn)
+
+
+class CacheModuleKeys(collections.MutableSet):
+    """
+    A set subclass that keeps track of insertion time and persists
+    the set in memcached.
+    """
+    PREFIX = 'ansible_cache_keys'
+
+    def __init__(self, cache, *args, **kwargs):
+        self._cache = cache
+        self._keyset = dict(*args, **kwargs)
+
+    def __contains__(self, key):
+        return key in self._keyset
+
+    def __iter__(self):
+        return iter(self._keyset)
+
+    def __len__(self):
+        return len(self._keyset)
+
+    def add(self, key):
+        self._keyset[key] = time.time()
+        self._cache.set(self.PREFIX, self._keyset)
+
+    def discard(self, key):
+        del self._keyset[key]
+        self._cache.set(self.PREFIX, self._keyset)
+
+    def remove_by_timerange(self, s_min, s_max):
+        for k in self._keyset.keys():
+            t = self._keyset[k]
+            if s_min < t < s_max:
+                del self._keyset[k]
+        self._cache.set(self.PREFIX, self._keyset)
+
+
+class CacheModule(BaseCacheModule):
+
+    def __init__(self, *args, **kwargs):
+        if C.CACHE_PLUGIN_CONNECTION:
+            connection = C.CACHE_PLUGIN_CONNECTION.split(',')
+        else:
+            connection = ['127.0.0.1:11211']
+
+        self._timeout = C.CACHE_PLUGIN_TIMEOUT
+        self._prefix = C.CACHE_PLUGIN_PREFIX
+        self._cache = ProxyClientPool(connection, debug=0)
+        self._keys = CacheModuleKeys(self._cache, self._cache.get(CacheModuleKeys.PREFIX) or [])
+
+    def _make_key(self, key):
+        return "{0}{1}".format(self._prefix, key)
+
+    def _expire_keys(self):
+        if self._timeout > 0:
+            expiry_age = time.time() - self._timeout
+        self._keys.remove_by_timerange(0, expiry_age)
+
+    def get(self, key):
+        value = self._cache.get(self._make_key(key))
+        # guard against the key not being removed from the keyset;
+        # this could happen in cases where the timeout value is changed
+        # between invocations
+        if value is None:
+            self.delete(key)
+            raise KeyError
+        return value
+
+    def set(self, key, value):
+        self._cache.set(self._make_key(key), value, time=self._timeout, min_compress_len=1)
+        self._keys.add(key)
+
+    def keys(self):
+        self._expire_keys()
+        return list(iter(self._keys))
+
+    def contains(self, key):
+        self._expire_keys()
+        return key in self._keys
+
+    def delete(self, key):
+        self._cache.delete(self._make_key(key))
+        self._keys.discard(key)
+
+    def flush(self):
+        for key in self.keys():
+            self.delete(key)
+
+    def copy(self):
+        return self._keys.copy()

v2/ansible/plugins/cache/memory.py

@@ -0,0 +1,44 @@
+# (c) 2014, Brian Coca, Josh Drake, et al
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+from ansible.plugins.cache.base import BaseCacheModule
+
+class CacheModule(BaseCacheModule):
+
+    def __init__(self, *args, **kwargs):
+        self._cache = {}
+
+    def get(self, key):
+        return self._cache.get(key)
+
+    def set(self, key, value):
+        self._cache[key] = value
+
+    def keys(self):
+        return self._cache.keys()
+
+    def contains(self, key):
+        return key in self._cache
+
+    def delete(self, key):
+        del self._cache[key]
+
+    def flush(self):
+        self._cache = {}
+
+    def copy(self):
+        return self._cache.copy()

v2/ansible/plugins/cache/redis.py

@@ -0,0 +1,102 @@
+# (c) 2014, Brian Coca, Josh Drake, et al
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import absolute_import
+import collections
+# FIXME: can we store these as something else before we ship it?
+import sys
+import time
+import json
+
+from ansible import constants as C
+from ansible.plugins.cache.base import BaseCacheModule
+
+try:
+    from redis import StrictRedis
+except ImportError:
+    print "The 'redis' python module is required, 'pip install redis'"
+    sys.exit(1)
+
+class CacheModule(BaseCacheModule):
+    """
+    A caching module backed by redis.
+
+    Keys are maintained in a zset with their score being the timestamp
+    when they are inserted. This allows for the usage of 'zremrangebyscore'
+    to expire keys. This mechanism is used or a pattern matched 'scan' for
+    performance.
+    """
+    def __init__(self, *args, **kwargs):
+        if C.CACHE_PLUGIN_CONNECTION:
+            connection = C.CACHE_PLUGIN_CONNECTION.split(':')
+        else:
+            connection = []
+
+        self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
+        self._prefix = C.CACHE_PLUGIN_PREFIX
+        self._cache = StrictRedis(*connection)
+        self._keys_set = 'ansible_cache_keys'
+
+    def _make_key(self, key):
+        return self._prefix + key
+
+    def get(self, key):
+        value = self._cache.get(self._make_key(key))
+        # guard against the key not being removed from the zset;
+        # this could happen in cases where the timeout value is changed
+        # between invocations
+        if value is None:
+            self.delete(key)
+            raise KeyError
+        return json.loads(value)
+
+    def set(self, key, value):
+        value2 = json.dumps(value)
+        if self._timeout > 0: # a timeout of 0 is handled as meaning 'never expire'
+            self._cache.setex(self._make_key(key), int(self._timeout), value2)
+        else:
+            self._cache.set(self._make_key(key), value2)
+
+        self._cache.zadd(self._keys_set, time.time(), key)
+
+    def _expire_keys(self):
+        if self._timeout > 0:
+            expiry_age = time.time() - self._timeout
+        self._cache.zremrangebyscore(self._keys_set, 0, expiry_age)
+
+    def keys(self):
+        self._expire_keys()
+        return self._cache.zrange(self._keys_set, 0, -1)
+
+    def contains(self, key):
+        self._expire_keys()
+        return (self._cache.zrank(self._keys_set, key) >= 0)
+
+    def delete(self, key):
+        self._cache.delete(self._make_key(key))
+        self._cache.zrem(self._keys_set, key)
+
+    def flush(self):
+        for key in self.keys():
+            self.delete(key)
+
+    def copy(self):
+        # FIXME: there is probably a better way to do this in redis
+        ret = dict()
+        for key in self.keys():
+            ret[key] = self.get(key)
+        return ret