Collection content loading (#52194)

* basic plugin loading working (with many hacks)

* task collections working

* play/block-level collection module/action working

* implement PEP302 loader

* implicit package support (no need for __init.py__ in collections)
* provides future options for secure loading of content that shouldn't execute inside controller (eg, actively ignore __init__.py on content/module paths)
* provide hook for synthetic collection setup (eg ansible.core pseudo-collection for specifying built-in plugins without legacy path, etc)

* synthetic package support

* ansible.core.plugins mapping works, others don't

* synthetic collections working for modules/actions

* fix direct-load legacy

* change base package name to ansible_collections

* note

* collection role loading

* expand paths from installed content root vars

* feature complete?

* rename ansible.core to ansible.builtin

* and various sanity fixes

* sanity tweaks

* unittest fixes

* less grabby error handler on has_plugin

* probably need to replace with a or harden callers

* fix win_ping test

* disable module test with explicit file extension; might be able to support in some scenarios, but can't see any other tests that verify that behavior...

* fix unicode conversion issues on py2

* attempt to keep things working-ish on py2.6

* python2.6 test fun round 2

* rename dirs/configs to "collections"

* add wrapper dir for content-adjacent

* fix pythoncheck to use localhost

* unicode tweaks, native/bytes string prefixing

* rename COLLECTION_PATHS to COLLECTIONS_PATHS

* switch to pathspec

* path handling cleanup

* change expensive `all` back to or chain

* unused import cleanup

* quotes tweak

* use wrapped iter/len in Jinja proxy

* var name expansion

* comment seemingly overcomplicated playbook_paths resolution

* drop unnecessary conditional nesting

* eliminate extraneous local

* zap superfluous validation function

* use slice for rolespec NS assembly

* misc naming/unicode fixes

* collection callback loader asks if valid FQ name instead of just '.'
* switch collection role resolution behavior to be internally `text` as much as possible

* misc fixmes

* to_native in exception constructor
* (slightly) detangle tuple accumulation mess in module_utils __init__ walker

* more misc fixmes

* tighten up action dispatch, add unqualified action test

* rename Collection mixin to CollectionSearch

* (attempt to) avoid potential confusion/conflict with builtin collections, etc

* stale fixmes

* tighten up pluginloader collections determination

* sanity test fixes

* ditch regex escape

* clarify comment

* update default collections paths config entry

* use PATH format instead of list

* skip integration tests on Python 2.6

ci_complete

lib/ansible/plugins/loader.py

@@ -11,6 +11,7 @@ import glob
 import imp
 import os
 import os.path
+import pkgutil
 import sys
 import warnings
 
@@ -23,9 +24,15 @@ from ansible.module_utils.six import string_types
 from ansible.parsing.utils.yaml import from_yaml
 from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible.plugins import get_plugin_class, MODULE_CACHE, PATH_CACHE, PLUGIN_PATH_CACHE
+from ansible.utils.collection_loader import AnsibleCollectionLoader, AnsibleFlatMapLoader, is_collection_ref
 from ansible.utils.display import Display
 from ansible.utils.plugin_docs import add_fragments
 
+# HACK: keep Python 2.6 controller tests happy in CI until they're properly split
+try:
+    from importlib import import_module
+except ImportError:
+    import_module = __import__
 
 display = Display()
 
@@ -298,7 +305,69 @@ class PluginLoader:
                 self._clear_caches()
                 display.debug('Added %s to loader search path' % (directory))
 
-    def _find_plugin(self, name, mod_type='', ignore_deprecated=False, check_aliases=False):
+    def _find_fq_plugin(self, fq_name, extension):
+        fq_name = to_native(fq_name)
+        # prefix our extension Python namespace if it isn't already there
+        if not fq_name.startswith('ansible_collections.'):
+            fq_name = 'ansible_collections.' + fq_name
+
+        splitname = fq_name.rsplit('.', 1)
+        if len(splitname) != 2:
+            raise ValueError('{0} is not a valid namespace-qualified plugin name'.format(to_native(fq_name)))
+
+        package = splitname[0]
+        resource = splitname[1]
+
+        append_plugin_type = self.class_name or self.subdir
+
+        if append_plugin_type:
+            # only current non-class special case, module_utils don't use this loader method
+            if append_plugin_type == 'library':
+                append_plugin_type = 'modules'
+            else:
+                append_plugin_type = get_plugin_class(append_plugin_type)
+            package += '.plugins.{0}'.format(append_plugin_type)
+
+        if extension:
+            resource += extension
+
+        pkg = sys.modules.get(package)
+        if not pkg:
+            # FIXME: there must be cheaper/safer way to do this
+            pkg = import_module(package)
+
+        # if the package is one of our flatmaps, we need to consult its loader to find the path, since the file could be
+        # anywhere in the tree
+        if hasattr(pkg, '__loader__') and isinstance(pkg.__loader__, AnsibleFlatMapLoader):
+            try:
+                file_path = pkg.__loader__.find_file(resource)
+                return to_text(file_path)
+            except IOError:
+                # this loader already takes care of extensionless files, so if we didn't find it, just bail
+                return None
+
+        pkg_path = os.path.dirname(pkg.__file__)
+
+        resource_path = os.path.join(pkg_path, resource)
+
+        # FIXME: and is file or file link or ...
+        if os.path.exists(resource_path):
+            return to_text(resource_path)
+
+        # look for any matching extension in the package location (sans filter)
+        ext_blacklist = ['.pyc', '.pyo']
+        found_files = [f for f in glob.iglob(os.path.join(pkg_path, resource) + '.*') if os.path.isfile(f) and os.path.splitext(f)[1] not in ext_blacklist]
+
+        if not found_files:
+            return None
+
+        if len(found_files) > 1:
+            # TODO: warn?
+            pass
+
+        return to_text(found_files[0])
+
+    def _find_plugin(self, name, mod_type='', ignore_deprecated=False, check_aliases=False, collection_list=None):
         ''' Find a plugin named name '''
 
         global _PLUGIN_FILTERS
@@ -315,6 +384,38 @@ class PluginLoader:
             # they can have any suffix
             suffix = ''
 
+        # HACK: need this right now so we can still load shipped PS module_utils
+        if (is_collection_ref(name) or collection_list) and not name.startswith('Ansible'):
+            if '.' in name or not collection_list:
+                candidates = [name]
+            else:
+                candidates = ['{0}.{1}'.format(c, name) for c in collection_list]
+            # TODO: keep actual errors, not just assembled messages
+            errors = []
+            for candidate_name in candidates:
+                try:
+                    # HACK: refactor this properly
+                    if candidate_name.startswith('ansible.legacy'):
+                        # just pass the raw name to the old lookup function to check in all the usual locations
+                        p = self._find_plugin_legacy(name.replace('ansible.legacy.', '', 1), ignore_deprecated, check_aliases, suffix)
+                    else:
+                        p = self._find_fq_plugin(candidate_name, suffix)
+                    if p:
+                        return p
+                except Exception as ex:
+                    errors.append(to_native(ex))
+
+            if errors:
+                display.debug(msg='plugin lookup for {0} failed; errors: {1}'.format(name, '; '.join(errors)))
+
+            return None
+
+        # if we got here, there's no collection list and it's not an FQ name, so do legacy lookup
+
+        return self._find_plugin_legacy(name, ignore_deprecated, check_aliases, suffix)
+
+    def _find_plugin_legacy(self, name, ignore_deprecated=False, check_aliases=False, suffix=None):
+
         if check_aliases:
             name = self.aliases.get(name, name)
 
@@ -388,13 +489,13 @@ class PluginLoader:
 
         return None
 
-    def find_plugin(self, name, mod_type='', ignore_deprecated=False, check_aliases=False):
+    def find_plugin(self, name, mod_type='', ignore_deprecated=False, check_aliases=False, collection_list=None):
         ''' Find a plugin named name '''
 
         # Import here to avoid circular import
         from ansible.vars.reserved import is_reserved_name
 
-        plugin = self._find_plugin(name, mod_type=mod_type, ignore_deprecated=ignore_deprecated, check_aliases=check_aliases)
+        plugin = self._find_plugin(name, mod_type=mod_type, ignore_deprecated=ignore_deprecated, check_aliases=check_aliases, collection_list=collection_list)
         if plugin and self.package == 'ansible.modules' and name not in ('gather_facts',) and is_reserved_name(name):
             raise AnsibleError(
                 'Module "%s" shadows the name of a reserved keyword. Please rename or remove this module. Found at %s' % (name, plugin)
@@ -402,10 +503,16 @@ class PluginLoader:
 
         return plugin
 
-    def has_plugin(self, name):
+    def has_plugin(self, name, collection_list=None):
         ''' Checks if a plugin named name exists '''
 
-        return self.find_plugin(name) is not None
+        try:
+            return self.find_plugin(name, collection_list=collection_list) is not None
+        except Exception as ex:
+            if isinstance(ex, AnsibleError):
+                raise
+            # log and continue, likely an innocuous type/package loading failure in collections import
+            display.debug('has_plugin error: {0}'.format(to_native(ex)))
 
     __contains__ = has_plugin
 
@@ -436,9 +543,10 @@ class PluginLoader:
 
         found_in_cache = True
         class_only = kwargs.pop('class_only', False)
+        collection_list = kwargs.pop('collection_list', None)
         if name in self.aliases:
             name = self.aliases[name]
-        path = self.find_plugin(name)
+        path = self.find_plugin(name, collection_list=collection_list)
         if path is None:
             return None
 
@@ -600,14 +708,20 @@ class Jinja2Loader(PluginLoader):
     The filter and test plugins are Jinja2 plugins encapsulated inside of our plugin format.
     The way the calling code is setup, we need to do a few things differently in the all() method
     """
-    def find_plugin(self, name):
+    def find_plugin(self, name, collection_list=None):
         # Nothing using Jinja2Loader use this method.  We can't use the base class version because
         # we deduplicate differently than the base class
+        if '.' in name:
+            return super(Jinja2Loader, self).find_plugin(name, collection_list=collection_list)
+
         raise AnsibleError('No code should call find_plugin for Jinja2Loaders (Not implemented)')
 
     def get(self, name, *args, **kwargs):
         # Nothing using Jinja2Loader use this method.  We can't use the base class version because
         # we deduplicate differently than the base class
+        if '.' in name:
+            return super(Jinja2Loader, self).get(name, *args, **kwargs)
+
         raise AnsibleError('No code should call find_plugin for Jinja2Loaders (Not implemented)')
 
     def all(self, *args, **kwargs):
@@ -695,11 +809,18 @@ def _load_plugin_filter():
     return filters
 
 
+def _configure_collection_loader():
+    if not any((isinstance(l, AnsibleCollectionLoader) for l in sys.meta_path)):
+        sys.meta_path.insert(0, AnsibleCollectionLoader())
+
+
 # TODO: All of the following is initialization code   It should be moved inside of an initialization
 # function which is called at some point early in the ansible and ansible-playbook CLI startup.
 
 _PLUGIN_FILTERS = _load_plugin_filter()
 
+_configure_collection_loader()
+
 # doc fragments first
 fragment_loader = PluginLoader(
     'ModuleDocFragment',