add inventory caching & use in virtualbox inventory plugin (#34510)

* Inventory caching

* Add inventory caching for virtualbox

* Don't populate cache for virtualbox with stdout, use a dict of inventory instead

* Fix error creating the cache dir if it doesn't exist

* Keep cache default False and set to True in VariableManager __init__

* Check all groups before determining if a host is ungrouped.

lib/ansible/plugins/inventory/virtualbox.py

@@ -14,6 +14,7 @@ DOCUMENTATION = '''
         - The inventory_hostname is always the 'Name' of the virtualbox instance.
     extends_documentation_fragment:
       - constructed
+      - inventory_cache
     options:
         running_only:
             description: toggles showing all vms vs only those currently running
@@ -91,7 +92,29 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
             # constructed groups based on conditionals
             self._add_host_to_composed_groups(self.get_option('groups'), hostvars, host)
 
-    def _populate_from_source(self, source_data):
+    def _populate_from_cache(self, source_data):
+        hostvars = source_data.pop('_meta', {}).get('hostvars', {})
+        for group in source_data:
+            if group == 'all':
+                continue
+            else:
+                self.inventory.add_group(group)
+                hosts = source_data[group].get('hosts', [])
+                for host in hosts:
+                    self._populate_host_vars([host], hostvars.get(host, {}), group)
+                self.inventory.add_child('all', group)
+        if not source_data:
+            for host in hostvars:
+                self.inventory.add_host(host)
+                self._populate_host_vars([host], hostvars.get(host, {}))
+
+    def _populate_from_source(self, source_data, using_current_cache=False):
+        if using_current_cache:
+            self._populate_from_cache(source_data)
+            return source_data
+
+        cacheable_results = {'_meta': {'hostvars': {}}}
+
         hostvars = {}
         prevkey = pref_k = ''
         current_host = None
@@ -100,6 +123,9 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
         netinfo = self.get_option('network_info_path')
 
         for line in source_data:
+            line = to_text(line)
+            if ':' not in line:
+                continue
             try:
                 k, v = line.split(':', 1)
             except:
@@ -127,8 +153,11 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
             elif k == 'Groups':
                 for group in v.split('/'):
                     if group:
+                        if group not in cacheable_results:
+                            cacheable_results[group] = {'hosts': []}
                         self.inventory.add_group(group)
                         self.inventory.add_child(group, current_host)
+                        cacheable_results[group]['hosts'].append(current_host)
                 continue
 
             else:
@@ -141,10 +170,32 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
                 else:
                     if v != '':
                         hostvars[current_host][pref_k] = v
+                if self._ungrouped_host(current_host, cacheable_results):
+                    if 'ungrouped' not in cacheable_results:
+                        cacheable_results['ungrouped'] = {'hosts': []}
+                    cacheable_results['ungrouped']['hosts'].append(current_host)
 
                 prevkey = pref_k
 
         self._set_variables(hostvars)
+        for host in hostvars:
+            h = self.inventory.get_host(host)
+            cacheable_results['_meta']['hostvars'][h.name] = h.vars
+
+        return cacheable_results
+
+    def _ungrouped_host(self, host, inventory):
+        def find_host(host, inventory):
+            for k, v in inventory.items():
+                if k == '_meta':
+                    continue
+                if isinstance(v, dict):
+                    yield self._ungrouped_host(host, v)
+                elif isinstance(v, list):
+                    yield host not in v
+            yield True
+
+        return all([found_host for found_host in find_host(host, inventory)])
 
     def verify_file(self, path):
 
@@ -158,7 +209,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
 
         super(InventoryModule, self).parse(inventory, loader, path)
 
-        cache_key = self._get_cache_prefix(path)
+        cache_key = self.get_cache_key(path)
 
         config_data = self._read_config_data(path)
 
@@ -166,11 +217,15 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
         self._consume_options(config_data)
 
         source_data = None
-        if cache and cache_key in self._cache:
+        if cache:
+            cache = self._options.get('cache')
+
+        update_cache = False
+        if cache:
             try:
-                source_data = self._cache[cache_key]
+                source_data = self.cache.get(cache_key)
             except KeyError:
-                pass
+                update_cache = True
 
         if not source_data:
             b_pwfile = to_bytes(self.get_option('settings_password_file'), errors='surrogate_or_strict')
@@ -192,7 +247,10 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
             except Exception as e:
                 AnsibleParserError(to_native(e))
 
-            source_data = p.stdout.read()
-            self._cache[cache_key] = to_text(source_data, errors='surrogate_or_strict')
+            source_data = p.stdout.read().splitlines()
 
-        self._populate_from_source(source_data.splitlines())
+        using_current_cache = cache and not update_cache
+        cacheable_results = self._populate_from_source(source_data, using_current_cache)
+
+        if update_cache:
+            self.cache.set(cache_key, cacheable_results)