Move group/host_vars parsing into core inventory

modified:   lib/ansible/inventory/__init__.py
	deleted:    lib/ansible/inventory/vars_plugins/group_vars.py
	modified:   lib/ansible/utils/__init__.py

lib/ansible/inventory/__init__.py

@@ -57,7 +57,6 @@ class Inventory(object):
         self._groups_list    = {} 
         self._pattern_cache  = {}
 
-        self._inventory_basedir = inventory.basedir()
         # to be set by calling set_playbook_basedir by playbook code
         self._playbook_basedir = None
 
@@ -142,11 +141,11 @@ class Inventory(object):
 
         self._vars_plugins = [ x for x in utils.plugins.vars_loader.all(self) ]
 
-        # get group vars from vars plugins
+        # get group vars from group_vars/ files and vars plugins
         for group in self.groups:
             group.vars = utils.combine_vars(group.vars, self.get_group_variables(group.name, self._vault_password))
 
-        # get host vars from vars plugins
+        # get host vars from host_vars/ files and vars plugins
         for host in self.get_hosts():
             host.vars = utils.combine_vars(host.vars, self.get_variables(host.name, self._vault_password))
 
@@ -380,8 +379,8 @@ class Inventory(object):
                 return group
         return None
 
-    def get_group_variables(self, groupname, vault_password=None):
+    def get_group_variables(self, groupname, update_cached=False, vault_password=None):
-        if groupname not in self._vars_per_group:
+        if groupname not in self._vars_per_group or update_cached:
             self._vars_per_group[groupname] = self._get_group_variables(groupname, vault_password=vault_password)
         return self._vars_per_group[groupname]
 
@@ -392,16 +391,23 @@ class Inventory(object):
             raise Exception("group not found: %s" % groupname)
 
         vars = {}
+
+        # plugin.get_group_vars retrieves just vars for specific group
         vars_results = [ plugin.get_group_vars(group, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_group_vars')]
         for updated in vars_results:
             if updated is not None:
-                vars.update(updated)
+                vars = utils.combine_vars(vars, updated)
+
+        # get group variables set by Inventory Parsers
+        vars = utils.combine_vars(vars, group.get_variables())
+
+        # Read group_vars/ files
+        vars = utils.combine_vars(vars, self.get_group_vars(group))
 
-        vars.update(group.get_variables())
         return vars
 
-    def get_variables(self, hostname, vault_password=None):
+    def get_variables(self, hostname, update_cached=False, vault_password=None):
-        if hostname not in self._vars_per_host:
+        if hostname not in self._vars_per_host or update_cached:
             self._vars_per_host[hostname] = self._get_variables(hostname, vault_password=vault_password)
         return self._vars_per_host[hostname]
 
@@ -413,18 +419,19 @@ class Inventory(object):
 
         vars = {}
 
-        # plugin.get_host_vars retrieves just vars for specific host
-        vars_results = [ plugin.get_host_vars(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_host_vars')]
-        for updated in vars_results:
-            if updated is not None:
-                vars = utils.combine_vars(vars, updated)
-
         # plugin.run retrieves all vars (also from groups) for host
         vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'run')]
         for updated in vars_results:
             if updated is not None:
                 vars = utils.combine_vars(vars, updated)
 
+        # plugin.get_host_vars retrieves just vars for specific host
+        vars_results = [ plugin.get_host_vars(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_host_vars')]
+        for updated in vars_results:
+            if updated is not None:
+                vars = utils.combine_vars(vars, updated)
+
+        # get host variables set by Inventory Parsers
         vars = utils.combine_vars(vars, host.get_variables())
 
         # still need to check InventoryParser per host vars
@@ -433,6 +440,9 @@ class Inventory(object):
         if self.parser is not None:
             vars = utils.combine_vars(vars, self.parser.get_host_variables(host))
 
+        # Read host_vars/ files
+        vars = utils.combine_vars(vars, self.get_host_vars(host))
+
         return vars
 
     def add_group(self, group):
@@ -532,10 +542,73 @@ class Inventory(object):
         return self._playbook_basedir
 
     def set_playbook_basedir(self, dir):
-        """ 
-        sets the base directory of the playbook so inventory plugins can use it to find
-        variable files and other things. 
         """
-        self._playbook_basedir = dir
+        sets the base directory of the playbook so inventory can use it as a
+        basedir for host_ and group_vars, and other things.
+        """
+        # Only update things if dir is a different playbook basedir
+        if dir != self._playbook_basedir:
+            self._playbook_basedir = dir
+            # get group vars from group_vars/ files
+            for group in self.groups:
+                group.vars = utils.combine_vars(group.vars, self.get_group_vars(group, new_pb_basedir=True))
+            # get host vars from host_vars/ files
+            for host in self.get_hosts():
+                host.vars = utils.combine_vars(host.vars, self.get_host_vars(host, new_pb_basedir=True))
 
+    def get_host_vars(self, host, new_pb_basedir=False):
+        """ Read host_vars/ files """
+        return self._get_hostgroup_vars(host=host, group=None, new_pb_basedir=False)
+
+    def get_group_vars(self, group, new_pb_basedir=False):
+        """ Read group_vars/ files """
+        return self._get_hostgroup_vars(host=None, group=group, new_pb_basedir=False)
+
+    def _get_hostgroup_vars(self, host=None, group=None, new_pb_basedir=False):
+        """
+        Loads variables from group_vars/<groupname> and host_vars/<hostname> in directories parallel
+        to the inventory base directory or in the same directory as the playbook.  Variables in the playbook
+        dir will win over the inventory dir if files are in both.
+        """
+
+        results = {}
+        scan_pass = 0
+        _basedir = self.basedir()
+
+        # look in both the inventory base directory and the playbook base directory
+        # unless we do an update for a new playbook base dir
+        if not new_pb_basedir:
+            basedirs = [_basedir, self._playbook_basedir]
+        else:
+            basedirs = [self._playbook_basedir]
+
+        for basedir in basedirs:
+
+            # this can happen from particular API usages, particularly if not run
+            # from /usr/bin/ansible-playbook
+            if basedir is None:
+                continue
+
+            scan_pass = scan_pass + 1
+
+            # it's not an eror if the directory does not exist, keep moving
+            if not os.path.exists(basedir):
+                continue
+
+            # save work of second scan if the directories are the same
+            if _basedir == self._playbook_basedir and scan_pass != 1:
+                continue
+
+            if group and host is None:
+                # load vars in dir/group_vars/name_of_group
+                base_path = os.path.join(basedir, "group_vars/%s" % group.name)
+                results = utils.load_vars(base_path, results, vault_password=self._vault_password)
+
+            elif host and group is None:
+                # same for hostvars in dir/host_vars/name_of_host
+                base_path = os.path.join(basedir, "host_vars/%s" % host.name)
+                results = utils.load_vars(base_path, results, vault_password=self._vault_password)
+
+        # all done, results is a dictionary of variables for this particular host.
+        return results
 

lib/ansible/inventory/vars_plugins/group_vars.py

@@ -1,201 +0,0 @@
-# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-import os
-import stat
-import errno
-
-from ansible import errors
-from ansible import utils
-import ansible.constants as C
-
-def _load_vars(basepath, results, vault_password=None):
-    """
-    Load variables from any potential yaml filename combinations of basepath,
-    returning result.
-    """
-
-    paths_to_check = [ "".join([basepath, ext]) 
-                       for ext in C.YAML_FILENAME_EXTENSIONS ]
-
-    found_paths = []
-
-    for path in paths_to_check:
-        found, results = _load_vars_from_path(path, results, vault_password=vault_password)
-        if found:
-            found_paths.append(path)
-
-
-    # disallow the potentially confusing situation that there are multiple
-    # variable files for the same name. For example if both group_vars/all.yml
-    # and group_vars/all.yaml
-    if len(found_paths) > 1:
-        raise errors.AnsibleError("Multiple variable files found. "
-            "There should only be one. %s" % ( found_paths, ))
-
-    return results
-
-def _load_vars_from_path(path, results, vault_password=None):
-    """
-    Robustly access the file at path and load variables, carefully reporting
-    errors in a friendly/informative way.
-
-    Return the tuple (found, new_results, )
-    """
-
-    try:
-        # in the case of a symbolic link, we want the stat of the link itself,
-        # not its target
-        pathstat = os.lstat(path)
-    except os.error, err:
-        # most common case is that nothing exists at that path.
-        if err.errno == errno.ENOENT:
-            return False, results
-        # otherwise this is a condition we should report to the user
-        raise errors.AnsibleError(
-            "%s is not accessible: %s." 
-            " Please check its permissions." % ( path, err.strerror))
-
-    # symbolic link
-    if stat.S_ISLNK(pathstat.st_mode):
-        try:
-            target = os.path.realpath(path)
-        except os.error, err2:
-            raise errors.AnsibleError("The symbolic link at %s "
-                "is not readable: %s.  Please check its permissions."
-                % (path, err2.strerror, ))
-        # follow symbolic link chains by recursing, so we repeat the same
-        # permissions checks above and provide useful errors.
-        return _load_vars_from_path(target, results)
-
-    # directory
-    if stat.S_ISDIR(pathstat.st_mode):
-
-        # support organizing variables across multiple files in a directory
-        return True, _load_vars_from_folder(path, results, vault_password=vault_password)
-
-    # regular file
-    elif stat.S_ISREG(pathstat.st_mode):
-        data = utils.parse_yaml_from_file(path, vault_password=vault_password)
-        if type(data) != dict:
-            raise errors.AnsibleError(
-                "%s must be stored as a dictionary/hash" % path)
-
-        # combine vars overrides by default but can be configured to do a
-        # hash merge in settings
-        results = utils.combine_vars(results, data)
-        return True, results
-
-    # something else? could be a fifo, socket, device, etc.
-    else:
-        raise errors.AnsibleError("Expected a variable file or directory "
-            "but found a non-file object at path %s" % (path, ))
-
-def _load_vars_from_folder(folder_path, results, vault_password=None):
-    """
-    Load all variables within a folder recursively.
-    """
-
-    # this function and _load_vars_from_path are mutually recursive
-
-    try:
-        names = os.listdir(folder_path)
-    except os.error, err:
-        raise errors.AnsibleError(
-            "This folder cannot be listed: %s: %s." 
-             % ( folder_path, err.strerror))
-        
-    # evaluate files in a stable order rather than whatever order the
-    # filesystem lists them.
-    names.sort() 
-
-    # do not parse hidden files or dirs, e.g. .svn/
-    paths = [os.path.join(folder_path, name) for name in names if not name.startswith('.')]
-    for path in paths:
-        _found, results = _load_vars_from_path(path, results, vault_password=vault_password)
-    return results
-
-            
-class VarsModule(object):
-
-    """
-    Loads variables from group_vars/<groupname> and host_vars/<hostname> in directories parallel
-    to the inventory base directory or in the same directory as the playbook.  Variables in the playbook
-    dir will win over the inventory dir if files are in both.
-    """
-
-    def __init__(self, inventory):
-
-        """ constructor """
-
-        self.inventory = inventory
-        self.inventory_basedir = inventory.basedir()
-        # There's no playbook initialized yet:
-        self.pb_basedir = None
-
-
-    def get_host_vars(self, host, vault_password=None):
-        return self._get_vars(host=host, group=None, vault_password=vault_password)
-
-
-    def get_group_vars(self, group, vault_password=None):
-        return self._get_vars(host=None, group=group, vault_password=vault_password)
-
-
-    def _get_vars(self, host=None, group=None, vault_password=None):
-        """ main body of the plugin, does actual loading"""
-
-        if self.pb_basedir is None:
-            pb_basedir = self.inventory.playbook_basedir()
-            if pb_basedir is not None:
-                pb_basedir = os.path.abspath(pb_basedir)
-                self.pb_basedir = pb_basedir
-
-        results = {}
-        scan_pass = 0
-
-        # look in both the inventory base directory and the playbook base directory
-        for basedir in [self.inventory_basedir, self.pb_basedir ]:
-
-            # this can happen from particular API usages, particularly if not run
-            # from /usr/bin/ansible-playbook
-            if basedir is None:
-                continue
-
-            scan_pass = scan_pass + 1
-
-            # it's not an eror if the directory does not exist, keep moving
-            if not os.path.exists(basedir):
-                continue
-
-            # save work of second scan if the directories are the same
-            if self.inventory_basedir == self.pb_basedir and scan_pass != 1:
-                continue
-
-            if group and host is None:
-                # load vars in dir/group_vars/name_of_group
-                base_path = os.path.join(basedir, "group_vars/%s" % group.name)
-                results = _load_vars(base_path, results, vault_password=vault_password)
-
-            elif host and group is None:
-                # same for hostvars in dir/host_vars/name_of_host
-                base_path = os.path.join(basedir, "host_vars/%s" % host.name)
-                results = _load_vars(base_path, results, vault_password=vault_password)
-
-        # all done, results is a dictionary of variables for this particular host.
-        return results
-

lib/ansible/utils/__init__.py

@@ -1,4 +1,4 @@
-# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
 #
 # This file is part of Ansible
 #
@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
+import errno
 import sys
 import re
 import os
@@ -1108,5 +1109,112 @@ def before_comment(msg):
     msg = msg.replace("**NOT_A_COMMENT**","#")
     return msg
 
+def load_vars(basepath, results, vault_password=None):
+    """
+    Load variables from any potential yaml filename combinations of basepath,
+    returning result.
+    """
+
+    paths_to_check = [ "".join([basepath, ext])
+                       for ext in C.YAML_FILENAME_EXTENSIONS ]
+
+    found_paths = []
+
+    for path in paths_to_check:
+        found, results = _load_vars_from_path(path, results, vault_password=vault_password)
+        if found:
+            found_paths.append(path)
 
 
+    # disallow the potentially confusing situation that there are multiple
+    # variable files for the same name. For example if both group_vars/all.yml
+    # and group_vars/all.yaml
+    if len(found_paths) > 1:
+        raise errors.AnsibleError("Multiple variable files found. "
+            "There should only be one. %s" % ( found_paths, ))
+
+    return results
+
+## load variables from yaml files/dirs
+#  e.g. host/group_vars
+#
+def _load_vars_from_path(path, results, vault_password=None):
+    """
+    Robustly access the file at path and load variables, carefully reporting
+    errors in a friendly/informative way.
+
+    Return the tuple (found, new_results, )
+    """
+
+    try:
+        # in the case of a symbolic link, we want the stat of the link itself,
+        # not its target
+        pathstat = os.lstat(path)
+    except os.error, err:
+        # most common case is that nothing exists at that path.
+        if err.errno == errno.ENOENT:
+            return False, results
+        # otherwise this is a condition we should report to the user
+        raise errors.AnsibleError(
+            "%s is not accessible: %s."
+            " Please check its permissions." % ( path, err.strerror))
+
+    # symbolic link
+    if stat.S_ISLNK(pathstat.st_mode):
+        try:
+            target = os.path.realpath(path)
+        except os.error, err2:
+            raise errors.AnsibleError("The symbolic link at %s "
+                "is not readable: %s.  Please check its permissions."
+                % (path, err2.strerror, ))
+        # follow symbolic link chains by recursing, so we repeat the same
+        # permissions checks above and provide useful errors.
+        return _load_vars_from_path(target, results)
+
+    # directory
+    if stat.S_ISDIR(pathstat.st_mode):
+
+        # support organizing variables across multiple files in a directory
+        return True, _load_vars_from_folder(path, results, vault_password=vault_password)
+
+    # regular file
+    elif stat.S_ISREG(pathstat.st_mode):
+        data = parse_yaml_from_file(path, vault_password=vault_password)
+        if type(data) != dict:
+            raise errors.AnsibleError(
+                "%s must be stored as a dictionary/hash" % path)
+
+        # combine vars overrides by default but can be configured to do a
+        # hash merge in settings
+        results = combine_vars(results, data)
+        return True, results
+
+    # something else? could be a fifo, socket, device, etc.
+    else:
+        raise errors.AnsibleError("Expected a variable file or directory "
+            "but found a non-file object at path %s" % (path, ))
+
+def _load_vars_from_folder(folder_path, results, vault_password=None):
+    """
+    Load all variables within a folder recursively.
+    """
+
+    # this function and _load_vars_from_path are mutually recursive
+
+    try:
+        names = os.listdir(folder_path)
+    except os.error, err:
+        raise errors.AnsibleError(
+            "This folder cannot be listed: %s: %s."
+             % ( folder_path, err.strerror))
+
+    # evaluate files in a stable order rather than whatever order the
+    # filesystem lists them.
+    names.sort()
+
+    # do not parse hidden files or dirs, e.g. .svn/
+    paths = [os.path.join(folder_path, name) for name in names if not name.startswith('.')]
+    for path in paths:
+        _found, results = _load_vars_from_path(path, results, vault_password=vault_password)
+    return results
+