Combine jimi-c and bcoca's ideas and work on hooking module-utils into PluginLoader.

This version just gets the relevant paths from PluginLoader and then
uses the existing imp.find_plugin() calls in the AnsiballZ code to load
the proper module_utils.

Modify PluginLoader to optionally omit subdirectories (module_utils
needs to operate on top level dirs, not on subdirs because it has
a hierarchical namespace whereas all other plugins use a flat
namespace).

Rename snippet* variables to module_utils*

Add a small number of unittests for recursive_finder

Add a larger number of integration tests to demonstrate that
module_utils is working.

Whitelist module-style shebang in test target library dirs

Prefix module_data variable with b_ to be clear that it holds bytes data

test/integration/targets/module_utils/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/module_utils/library/test.py

@@ -0,0 +1,83 @@
+#!/usr/bin/python
+
+results = {}
+
+# Test import with no from
+import ansible.module_utils.foo0
+results['foo0'] = ansible.module_utils.foo0.data
+
+# Test depthful import with no from
+import ansible.module_utils.bar0.foo
+results['bar0'] = ansible.module_utils.bar0.foo.data
+
+# Test import of module_utils/foo1.py
+from ansible.module_utils import foo1
+results['foo1'] = foo1.data
+
+# Test import of an identifier inside of module_utils/foo2.py
+from ansible.module_utils.foo2 import data
+results['foo2'] = data
+
+# Test import of module_utils/bar1/__init__.py
+from ansible.module_utils import bar1
+results['bar1'] = bar1.data
+
+# Test import of an identifier inside of module_utils/bar2/__init__.py
+from ansible.module_utils.bar2 import data
+results['bar2'] = data
+
+# Test import of module_utils/baz1/one.py
+from ansible.module_utils.baz1 import one
+results['baz1'] = one.data
+
+# Test import of an identifier inside of module_utils/baz2/one.py
+from ansible.module_utils.baz2.one import data
+results['baz2'] = data
+
+# Test import of module_utils/spam1/ham/eggs/__init__.py
+from ansible.module_utils.spam1.ham import eggs
+results['spam1'] = eggs.data
+
+# Test import of an identifier inside module_utils/spam2/ham/eggs/__init__.py
+from ansible.module_utils.spam2.ham.eggs import data
+results['spam2'] = data
+
+# Test import of module_utils/spam3/ham/bacon.py
+from ansible.module_utils.spam3.ham import bacon
+results['spam3'] = bacon.data
+
+# Test import of an identifier inside of module_utils/spam4/ham/bacon.py
+from ansible.module_utils.spam4.ham.bacon import data
+results['spam4'] = data
+
+# Test import of module_utils.spam5.ham bacon and eggs (modules)
+from ansible.module_utils.spam5.ham import bacon, eggs
+results['spam5'] = (bacon.data, eggs.data)
+
+# Test import of module_utils.spam6.ham bacon and eggs (identifiers)
+from ansible.module_utils.spam6.ham import bacon, eggs
+results['spam6'] = (bacon, eggs)
+
+# Test import of module_utils.spam7.ham bacon and eggs (module and identifier)
+from ansible.module_utils.spam7.ham import bacon, eggs
+results['spam7'] = (bacon.data, eggs)
+
+# Test import of module_utils/spam8/ham/bacon.py and module_utils/spam8/ham/eggs.py separately
+from ansible.module_utils.spam8.ham import bacon
+from ansible.module_utils.spam8.ham import eggs
+results['spam8'] = (bacon.data, eggs)
+
+# Test that import of module_utils/qux1/quux.py using as works
+from ansible.module_utils.qux1 import quux as one
+results['qux1'] = one.data
+
+# Test that importing qux2/quux.py and qux2/quuz.py using as works
+from ansible.module_utils.qux2 import quux as one, quuz as two
+results['qux2'] = (one.data, two.data)
+
+# Test depth
+from ansible.module_utils.a.b.c.d.e.f.g.h import data
+
+results['abcdefgh'] = data
+from ansible.module_utils.basic import AnsibleModule
+AnsibleModule(argument_spec=dict()).exit_json(**results)

test/integration/targets/module_utils/library/test_failure.py

@@ -0,0 +1,14 @@
+#!/usr/bin/python
+
+results = {}
+# Test that we are rooted correctly
+# Following files:
+#   module_utils/yak/zebra/foo.py
+try:
+    from ansible.module_utils.zebra import foo
+    results['zebra'] = foo.data
+except ImportError:
+    results['zebra'] = 'Failed in module as expected but incorrectly did not fail in AnsiballZ construction'
+
+from ansible.module_utils.basic import AnsibleModule
+AnsibleModule(argument_spec=dict()).exit_json(**results)

test/integration/targets/module_utils/library/test_override.py

@@ -0,0 +1,7 @@
+#!/usr/bin/python
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.facts import data
+
+results = {"data": data}
+
+AnsibleModule(argument_spec=dict()).exit_json(**results)

test/integration/targets/module_utils/module_utils/a/b/c/d/e/f/g/h/__init__.py

@@ -0,0 +1 @@
+data = 'abcdefgh'

test/integration/targets/module_utils/module_utils/bar0/foo.py

@@ -0,0 +1 @@
+data = 'bar0'

test/integration/targets/module_utils/module_utils/bar1/__init__.py

@@ -0,0 +1 @@
+data = 'bar1'

test/integration/targets/module_utils/module_utils/bar2/__init__.py

@@ -0,0 +1 @@
+data = 'bar2'

test/integration/targets/module_utils/module_utils/baz1/one.py

@@ -0,0 +1 @@
+data = 'baz1'

test/integration/targets/module_utils/module_utils/baz2/one.py

@@ -0,0 +1 @@
+data = 'baz2'

test/integration/targets/module_utils/module_utils/facts.py

@@ -0,0 +1 @@
+data = 'overridden facts.py'

test/integration/targets/module_utils/module_utils/foo.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+foo = "FOO FROM foo.py"

test/integration/targets/module_utils/module_utils/foo0.py

@@ -0,0 +1 @@
+data = 'foo0'

test/integration/targets/module_utils/module_utils/foo1.py

@@ -0,0 +1 @@
+data = 'foo1'

test/integration/targets/module_utils/module_utils/foo2.py

@@ -0,0 +1 @@
+data = 'foo2'

test/integration/targets/module_utils/module_utils/qux1/quux.py

@@ -0,0 +1 @@
+data = 'qux1'

test/integration/targets/module_utils/module_utils/qux2/quux.py

@@ -0,0 +1 @@
+data = 'qux2:quux'

test/integration/targets/module_utils/module_utils/qux2/quuz.py

@@ -0,0 +1 @@
+data = 'qux2:quuz'

test/integration/targets/module_utils/module_utils/spam1/ham/eggs/__init__.py

@@ -0,0 +1 @@
+data = 'spam1'

test/integration/targets/module_utils/module_utils/spam2/ham/eggs/__init__.py

@@ -0,0 +1 @@
+data = 'spam2'

test/integration/targets/module_utils/module_utils/spam3/ham/bacon.py

@@ -0,0 +1 @@
+data = 'spam3'

test/integration/targets/module_utils/module_utils/spam4/ham/bacon.py

@@ -0,0 +1 @@
+data = 'spam4'

test/integration/targets/module_utils/module_utils/spam5/ham/bacon.py

@@ -0,0 +1 @@
+data = 'spam5:bacon'

test/integration/targets/module_utils/module_utils/spam5/ham/eggs.py

@@ -0,0 +1 @@
+data = 'spam5:eggs'

test/integration/targets/module_utils/module_utils/spam6/ham/__init__.py

@@ -0,0 +1,2 @@
+bacon = 'spam6:bacon'
+eggs = 'spam6:eggs'

test/integration/targets/module_utils/module_utils/spam7/ham/__init__.py

@@ -0,0 +1 @@
+eggs = 'spam7:eggs'

test/integration/targets/module_utils/module_utils/spam7/ham/bacon.py

@@ -0,0 +1 @@
+data = 'spam7:bacon'

test/integration/targets/module_utils/module_utils/spam8/ham/__init__.py

@@ -0,0 +1 @@
+eggs = 'spam8:eggs'

test/integration/targets/module_utils/module_utils/spam8/ham/bacon.py

@@ -0,0 +1 @@
+data = 'spam8:bacon'

test/integration/targets/module_utils/module_utils/sub/bam.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+bam = "BAM FROM sub/bam.py"

test/integration/targets/module_utils/module_utils/sub/bam/bam.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+bam = "BAM FROM sub/bam/bam.py"

test/integration/targets/module_utils/module_utils/sub/bar/bam.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+bam = "BAM FROM sub/bar/bam.py"

test/integration/targets/module_utils/module_utils/sub/bar/bar.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+bar = "BAR FROM sub/bar/bar.py"

test/integration/targets/module_utils/module_utils/yak/zebra/foo.py

@@ -0,0 +1 @@
+data = 'yak'

test/integration/targets/module_utils/module_utils_test.yml

@@ -0,0 +1,50 @@
+- hosts: localhost
+  gather_facts: no
+  tasks:
+  - name: Use a specially crafted module to see if things were imported correctly
+    test:
+    register: result
+
+  - name: Check that the module imported the correct version of each module_util
+    assert:
+      that:
+        - 'result["abcdefgh"] == "abcdefgh"'
+        - 'result["bar0"] == "bar0"'
+        - 'result["bar1"] == "bar1"'
+        - 'result["bar2"] == "bar2"'
+        - 'result["baz1"] == "baz1"'
+        - 'result["baz2"] == "baz2"'
+        - 'result["foo0"] == "foo0"'
+        - 'result["foo1"] == "foo1"'
+        - 'result["foo2"] == "foo2"'
+        - 'result["qux1"] == "qux1"'
+        - 'result["qux2"] == ["qux2:quux", "qux2:quuz"]'
+        - 'result["spam1"] == "spam1"'
+        - 'result["spam2"] == "spam2"'
+        - 'result["spam3"] == "spam3"'
+        - 'result["spam4"] == "spam4"'
+        - 'result["spam5"] == ["spam5:bacon", "spam5:eggs"]'
+        - 'result["spam6"] == ["spam6:bacon", "spam6:eggs"]'
+        - 'result["spam7"] == ["spam7:bacon", "spam7:eggs"]'
+        - 'result["spam8"] == ["spam8:bacon", "spam8:eggs"]'
+
+  # Test that overriding something in module_utils with something in the local library works
+  - name: Test that local module_utils overrides facts.py
+    test_override:
+    register: result
+
+  - name: Make sure the we used the local facts.py, not the one shipped with ansible
+    assert:
+      that:
+        - 'result["data"] == "overridden facts.py"'
+
+  - name: Test that importing a module that only exists inside of a submodule does not work
+    test_failure:
+    ignore_errors: True
+    register: result
+
+  - name: Make sure we failed in AnsiBallZ
+    assert:
+      that:
+        - 'result["failed"] == True'
+        - '"Could not find imported module support code for test_failure.  Looked for either foo or zebra" == result["msg"]'

test/integration/targets/module_utils/runme.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook module_utils_test.yml -i ../../inventory -v "$@"

test/sanity/code-smell/shebang.sh

@@ -6,6 +6,7 @@ grep '^#!' -rIn . \
     | grep ':1:' | sed 's/:1:/:/' | grep -v -E \
     -e '^\./lib/ansible/modules/' \
     -e '^\./test/integration/targets/[^/]*/library/[^/]*:#!powershell$' \
+    -e '^\./test/integration/targets/[^/]*/library/[^/]*:#!/usr/bin/python$' \
     -e '^\./test/sanity/validate-modules/validate-modules:#!/usr/bin/env python2$' \
     -e '^\./hacking/cherrypick.py:#!/usr/bin/env python3$' \
     -e ':#!/bin/sh$' \

test/units/executor/module_common/test_recursive_finder.py

@@ -0,0 +1,134 @@
+# (c) 2017, Toshio Kuratomi <tkuratomi@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import imp
+import zipfile
+from collections import namedtuple
+from functools import partial
+from io import BytesIO, StringIO
+
+import pytest
+
+import ansible.errors
+from ansible.compat.six import PY2
+from ansible.compat.six.moves import builtins
+
+from ansible.executor.module_common import recursive_finder
+
+
+original_find_module = imp.find_module
+
+
+@pytest.fixture
+def finder_containers():
+    FinderContainers = namedtuple('FinderContainers', ['py_module_names', 'py_module_cache', 'zf'])
+
+    py_module_names = set()
+    #py_module_cache = {('__init__',): b''}
+    py_module_cache = {}
+
+    zipoutput = BytesIO()
+    zf = zipfile.ZipFile(zipoutput, mode='w', compression=zipfile.ZIP_STORED)
+    #zf.writestr('ansible/__init__.py', b'')
+
+    return FinderContainers(py_module_names, py_module_cache, zf)
+
+
+def find_module_foo(module_utils_data, *args, **kwargs):
+    if args[0] == 'foo':
+        return (module_utils_data, '/usr/lib/python2.7/site-packages/ansible/module_utils/foo.py', ('.py', 'r', imp.PY_SOURCE))
+    return original_find_module(*args, **kwargs)
+
+
+def find_package_foo(module_utils_data, *args, **kwargs):
+    if args[0] == 'foo':
+        return (module_utils_data, '/usr/lib/python2.7/site-packages/ansible/module_utils/foo', ('', '', imp.PKG_DIRECTORY))
+    return original_find_module(*args, **kwargs)
+
+
+class TestRecursiveFinder(object):
+    def test_no_module_utils(self, finder_containers):
+        name = 'ping'
+        data = b'#!/usr/bin/python\nreturn \'{\"changed\": false}\''
+        recursive_finder(name, data, *finder_containers)
+        assert finder_containers.py_module_names == set(())
+        assert finder_containers.py_module_cache == {}
+        assert frozenset(finder_containers.zf.namelist()) == frozenset()
+
+    def test_from_import_toplevel_package(self, finder_containers, mocker):
+        if PY2:
+            module_utils_data = BytesIO(b'# License\ndef do_something():\n    pass\n')
+        else:
+            module_utils_data = StringIO(u'# License\ndef do_something():\n    pass\n')
+        mocker.patch('imp.find_module', side_effect=partial(find_package_foo, module_utils_data))
+        mocker.patch('ansible.executor.module_common._slurp', side_effect= lambda x: b'# License\ndef do_something():\n    pass\n')
+
+        name = 'ping'
+        data = b'#!/usr/bin/python\nfrom ansible.module_utils import foo'
+        recursive_finder(name, data, *finder_containers)
+        mocker.stopall()
+
+        assert finder_containers.py_module_names == set((('foo', '__init__'),))
+        assert finder_containers.py_module_cache == {}
+        assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/foo/__init__.py',))
+
+    def test_from_import_toplevel_module(self, finder_containers, mocker):
+        if PY2:
+            module_utils_data = BytesIO(b'# License\ndef do_something():\n    pass\n')
+        else:
+            module_utils_data = StringIO(u'# License\ndef do_something():\n    pass\n')
+        mocker.patch('imp.find_module', side_effect=partial(find_module_foo, module_utils_data))
+
+        name = 'ping'
+        data = b'#!/usr/bin/python\nfrom ansible.module_utils import foo'
+        recursive_finder(name, data, *finder_containers)
+        mocker.stopall()
+
+        assert finder_containers.py_module_names == set((('foo',),))
+        assert finder_containers.py_module_cache == {}
+        assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/foo.py',))
+
+    #
+    # Test importing six with many permutations because it is not a normal module
+    #
+    def test_from_import_six(self, finder_containers):
+        name = 'ping'
+        data = b'#!/usr/bin/python\nfrom ansible.module_utils import six'
+        recursive_finder(name, data, *finder_containers)
+        assert finder_containers.py_module_names == set((('six',),))
+        assert finder_containers.py_module_cache == {}
+        assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six.py',))
+
+    def test_import_six(self, finder_containers):
+        name = 'ping'
+        data = b'#!/usr/bin/python\nimport ansible.module_utils.six'
+        recursive_finder(name, data, *finder_containers)
+        assert finder_containers.py_module_names == set((('six',),))
+        assert finder_containers.py_module_cache == {}
+        assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six.py',))
+
+    def test_import_six_from_many_submodules(self, finder_containers):
+        name = 'ping'
+        data = b'#!/usr/bin/python\nfrom ansible.module_utils.six.moves.urllib.parse import urlparse'
+        recursive_finder(name, data, *finder_containers)
+        assert finder_containers.py_module_names == set((('six',),))
+        assert finder_containers.py_module_cache == {}
+        assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six.py',))