Merge branch 'directory-inventory' of git://github.com/dhozac/ansible into devel

CHANGELOG.md

@@ -16,6 +16,7 @@ Core Features
 * a new chroot connection type
 * module common code now has basic type checking (and casting) capability 
 * module common now supports a 'no_log' attribute to mark a field as not to be syslogged
+* inventory can now point to a directory containing multiple scripts/hosts files
 
 Modules Added:
 

lib/ansible/inventory/__init__.py

@@ -25,6 +25,7 @@ import subprocess
 import ansible.constants as C
 from ansible.inventory.ini import InventoryParser
 from ansible.inventory.script import InventoryScript
+from ansible.inventory.dir import InventoryDirectory
 from ansible.inventory.group import Group
 from ansible.inventory.host import Host
 from ansible import errors
@@ -35,7 +36,7 @@ class Inventory(object):
     Host inventory for ansible.
     """
 
-    __slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset', '_is_script',
+    __slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset', 
                   'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list']
 
     def __init__(self, host_list=C.DEFAULT_HOST_LIST):
@@ -60,17 +61,11 @@ class Inventory(object):
         self._also_restriction = None
         self._subset = None
 
-        # whether the inventory file is a script
-        self._is_script = False
-
         if type(host_list) in [ str, unicode ]:
             if host_list.find(",") != -1:
                 host_list = host_list.split(",")
                 host_list = [ h for h in host_list if h and h.strip() ]
 
-        else:
-            utils.plugins.vars_loader.add_directory(self.basedir())
-
         if type(host_list) == list:
             all = Group('all')
             self.groups = [ all ]
@@ -81,8 +76,12 @@ class Inventory(object):
                 else:
                     all.add_host(Host(x))
         elif os.path.exists(host_list):
-            if utils.is_executable(host_list):
+            if os.path.isdir(host_list):
-                self._is_script = True
+                # Ensure basedir is inside the directory
+                self.host_list = os.path.join(self.host_list, "")
+                self.parser = InventoryDirectory(filename=host_list)
+                self.groups = self.parser.groups.values()
+            elif utils.is_executable(host_list):
                 self.parser = InventoryScript(filename=host_list)
                 self.groups = self.parser.groups.values()
             else:
@@ -92,6 +91,8 @@ class Inventory(object):
                     self.groups = self.parser.groups.values()
                 else:
                     raise errors.AnsibleError("YAML inventory support is deprecated in 0.6 and removed in 0.7, see the migration script in examples/scripts in the git checkout")
+
+            utils.plugins.vars_loader.add_directory(self.basedir(), with_subdir=True)
         else:
             raise errors.AnsibleError("Unable to find an inventory file, specify one with -i ?")
 
@@ -280,16 +281,7 @@ class Inventory(object):
                 vars.update(updated)
 
         vars.update(host.get_variables())
-        if self._is_script:
+        vars.update(self.parser.get_host_variables(host))
-            cmd = [self.host_list,"--host",hostname]
-            try:
-                sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            except OSError, e:
-                raise errors.AnsibleError("problem running %s (%s)" % (' '.join(cmd), e))
-            (out, err) = sp.communicate()
-            results = utils.parse_json(out)
-
-            vars.update(results)
         return vars
 
     def add_group(self, group):

lib/ansible/inventory/dir.py

@@ -0,0 +1,76 @@
+# (c) 2013, Daniel Hokka Zakrisson <daniel@hozac.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+#############################################
+
+import os
+import ansible.constants as C
+from ansible.inventory.host import Host
+from ansible.inventory.group import Group
+from ansible.inventory.ini import InventoryParser
+from ansible.inventory.script import InventoryScript
+from ansible import utils
+from ansible import errors
+
+class InventoryDirectory(object):
+    ''' Host inventory parser for ansible using a directory of inventories. '''
+
+    def __init__(self, filename=C.DEFAULT_HOST_LIST):
+        self.names = os.listdir(filename)
+        self.names.sort()
+        self.directory = filename
+        self.parsers = []
+        self.hosts = {}
+        self.groups = {}
+        for i in self.names:
+            if i.endswith("~") or i.endswith(".orig") or i.endswith(".bak"):
+                continue
+            # These are things inside of an inventory basedir
+            if i in ("host_vars", "group_vars", "vars_plugins"):
+                continue
+            fullpath = os.path.join(self.directory, i)
+            if os.path.isdir(fullpath):
+                parser = InventoryDirectory(filename=fullpath)
+            elif utils.is_executable(fullpath):
+                parser = InventoryScript(filename=fullpath)
+            else:
+                parser = InventoryParser(filename=fullpath)
+            self.parsers.append(parser)
+            # This takes a lot of code because we can't directly use any of the objects, as they have to blend
+            for name, group in parser.groups.iteritems():
+                if name not in self.groups:
+                    self.groups[name] = Group(name)
+                for k, v in group.get_variables().iteritems():
+                    self.groups[name].set_variable(k, v)
+                for host in group.get_hosts():
+                    if host.name not in self.hosts:
+                        self.hosts[host.name] = Host(host.name)
+                    for k, v in host.get_variables().iteritems():
+                        self.hosts[host.name].set_variable(k, v)
+                    self.groups[name].add_host(self.hosts[host.name])
+            # This needs to be a second loop to ensure all the parent groups exist
+            for name, group in parser.groups.iteritems():
+                for ancestor in group.get_ancestors():
+                    self.groups[ancestor.name].add_child_group(self.groups[name])
+
+    def get_host_variables(self, host):
+        """ Gets additional host variables from all inventories """
+        vars = {}
+        for i in self.parsers:
+            vars.update(i.get_host_variables(host))
+        return vars
+

lib/ansible/inventory/ini.py

@@ -173,3 +173,6 @@ class InventoryParser(object):
                         group.set_variable(k, re.sub(r"^['\"]|['\"]$", '', v))
                     else:
                         group.set_variable(k, v)
+
+    def get_host_variables(self, host):
+        return {}

lib/ansible/inventory/script.py

@@ -29,7 +29,8 @@ class InventoryScript(object):
 
     def __init__(self, filename=C.DEFAULT_HOST_LIST):
 
-        cmd = [ filename, "--list" ]
+        self.filename = filename
+        cmd = [ self.filename, "--list" ]
         try:
             sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         except OSError, e:
@@ -77,3 +78,13 @@ class InventoryScript(object):
                     if child_name in groups:
                         groups[group_name].add_child_group(groups[child_name])
         return groups
+
+    def get_host_variables(self, host):
+        """ Runs <script> --host <hostname> to determine additional host variables """
+        cmd = [self.filename, "--host", host.name]
+        try:
+            sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        except OSError, e:
+            raise errors.AnsibleError("problem running %s (%s)" % (' '.join(cmd), e))
+        (out, err) = sp.communicate()
+        return utils.parse_json(out)

lib/ansible/utils/plugins.py

@@ -68,9 +68,11 @@ class PluginLoader(object):
         ret += self._get_package_path()
         return ret
 
-    def add_directory(self, directory):
+    def add_directory(self, directory, with_subdir=False):
         """Adds an additional directory to the search path"""
         if directory is not None:
+            if with_subdir:
+                directory = os.path.join(directory, self.subdir)
             self._extra_dirs.append(directory)
 
     def print_paths(self):

test/TestInventory.py

@@ -13,6 +13,7 @@ class TestInventory(unittest.TestCase):
         self.large_range_inventory_file = os.path.join(self.test_dir, 'large_range')
         self.complex_inventory_file     = os.path.join(self.test_dir, 'complex_hosts')
         self.inventory_script           = os.path.join(self.test_dir, 'inventory_api.py')
+        self.inventory_dir              = os.path.join(self.test_dir, 'inventory_dir')
 
         os.chmod(self.inventory_script, 0755)
 
@@ -39,6 +40,9 @@ class TestInventory(unittest.TestCase):
     def complex_inventory(self):
         return Inventory(self.complex_inventory_file)
 
+    def dir_inventory(self):
+        return Inventory(self.inventory_dir)
+
     all_simple_hosts=['jupiter', 'saturn', 'zeus', 'hera',
             'cerberus001','cerberus002','cerberus003',
             'cottus99', 'cottus100',
@@ -288,3 +292,14 @@ class TestInventory(unittest.TestCase):
         assert vars == {'inventory_hostname': 'zeus',
                         'inventory_hostname_short': 'zeus',
                         'group_names': ['greek', 'major-god']}
+
+    def test_dir_inventory(self):
+        inventory = self.dir_inventory()
+        vars = inventory.get_variables('zeus')
+
+        print "VARS=%s" % vars
+
+        assert vars == {'inventory_hostname': 'zeus',
+                        'inventory_hostname_short': 'zeus',
+                        'group_names': ['greek', 'major-god', 'ungrouped'],
+                        'var_a': '1'}

test/inventory_dir/0hosts

@@ -0,0 +1,3 @@
+zeus var_a=2
+morpheus
+thor

test/inventory_dir/1mythology

@@ -0,0 +1,6 @@
+[greek]
+zeus
+morpheus
+
+[norse]
+thor

test/inventory_dir/2levels

@@ -0,0 +1,6 @@
+[major-god]
+zeus var_a=1
+thor
+
+[minor-god]
+morpheus