ansible-collections/community.general
1
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2025-04-24 03:11:24 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Combine jimi-c and bcoca's ideas and work on hooking module-utils into PluginLoader.

Browse source 
This version just gets the relevant paths from PluginLoader and then
uses the existing imp.find_plugin() calls in the AnsiballZ code to load
the proper module_utils.

Modify PluginLoader to optionally omit subdirectories (module_utils
needs to operate on top level dirs, not on subdirs because it has
a hierarchical namespace whereas all other plugins use a flat
namespace).

Rename snippet* variables to module_utils*

Add a small number of unittests for recursive_finder

Add a larger number of integration tests to demonstrate that
module_utils is working.

Whitelist module-style shebang in test target library dirs

Prefix module_data variable with b_ to be clear that it holds bytes data
This commit is contained in:
Toshio Kuratomi 2017-01-27 11:53:02 -08:00
parent b89f222028
commit 5c38f3cea2
71 changed files with 410 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

79
lib/ansible/executor/module_common.py
View file

@ -34,6 +34,7 @@ from ansible.release import __version__, __author__
from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_bytes, to_text
from ansible.plugins import module_utils_loader
# Must import strategy and use write_locks from there
# If we import write_locks directly then we end up binding a
# variable to the object and then it never gets updated.
@ -57,8 +58,8 @@ REPLACER_SELINUX = b"<<SELINUX_SPECIAL_FILESYSTEMS>>"
# specify an encoding for the python source file
ENCODING_STRING = u'# -*- coding: utf-8 -*-'
# we've moved the module_common relative to the snippets, so fix the path
_SNIPPET_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils')
# module_common is relative to module_utils, so fix the path
_MODULE_UTILS_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils')
# ******************************************************************************
@ -468,13 +469,16 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
# Loop through the imports that we've found to normalize them
# Exclude paths that match with paths we've already processed
# (Have to exclude them a second time once the paths are processed)
module_utils_paths = [p for p in module_utils_loader._get_paths(subdirs=False) if os.path.isdir(p)]
module_utils_paths.append(_MODULE_UTILS_PATH)
for py_module_name in finder.submodules.difference(py_module_names):
module_info = None
if py_module_name[0] == 'six':
# Special case the python six library because it messes up the
# import process in an incompatible way
module_info = imp.find_module('six', [_SNIPPET_PATH])
module_info = imp.find_module('six', module_utils_paths)
py_module_name = ('six',)
idx = 0
else:
@ -485,7 +489,7 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
break
try:
module_info = imp.find_module(py_module_name[-idx],
[os.path.join(_SNIPPET_PATH, *py_module_name[:-idx])])
[os.path.join(p, *py_module_name[:-idx]) for p in module_utils_paths])
break
except ImportError:
continue
@ -513,7 +517,7 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
if module_info[2][2] == imp.PKG_DIRECTORY:
# Read the __init__.py instead of the module file as this is
# a python package
py_module_cache[py_module_name + ('__init__',)] = _slurp(os.path.join(os.path.join(_SNIPPET_PATH, *py_module_name), '__init__.py'))
py_module_cache[py_module_name + ('__init__',)] = _slurp(os.path.join(os.path.join(module_info[1], '__init__.py')))
normalized_modules.add(py_module_name + ('__init__',))
else:
py_module_cache[py_module_name] = module_info[0].read()
@ -525,8 +529,10 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
for i in range(1, len(py_module_name)):
py_pkg_name = py_module_name[:-i] + ('__init__',)
if py_pkg_name not in py_module_names:
pkg_dir_info = imp.find_module(py_pkg_name[-1],
[os.path.join(p, *py_pkg_name[:-1]) for p in module_utils_paths])
normalized_modules.add(py_pkg_name)
py_module_cache[py_pkg_name] = _slurp('%s.py' % os.path.join(_SNIPPET_PATH, *py_pkg_name))
py_module_cache[py_pkg_name] = _slurp(pkg_dir_info[1])
#
# iterate through all of the ansible.module_utils* imports that we haven't
@ -554,13 +560,13 @@ def recursive_finder(name, data, py_module_names, py_module_cache, zf):
del py_module_cache[py_module_file]
def _is_binary(module_data):
def _is_binary(b_module_data):
textchars = bytearray(set([7, 8, 9, 10, 12, 13, 27]) | set(range(0x20, 0x100)) - set([0x7f]))
start = module_data[:1024]
start = b_module_data[:1024]
return bool(start.translate(None, textchars))
def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression):
def _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, module_compression):
"""
Given the source of the module, convert it to a Jinja2 template to insert
module code and return whether it's a new or old style module.
@ -574,31 +580,31 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
# module_substyle is extra information that's useful internally. It tells
# us what we have to look to substitute in the module files and whether
# we're using module replacer or ansiballz to format the module itself.
if _is_binary(module_data):
if _is_binary(b_module_data):
module_substyle = module_style = 'binary'
elif REPLACER in module_data:
elif REPLACER in b_module_data:
# Do REPLACER before from ansible.module_utils because we need make sure
# we substitute "from ansible.module_utils basic" for REPLACER
module_style = 'new'
module_substyle = 'python'
module_data = module_data.replace(REPLACER, b'from ansible.module_utils.basic import *')
elif b'from ansible.module_utils.' in module_data:
b_module_data = b_module_data.replace(REPLACER, b'from ansible.module_utils.basic import *')
elif b'from ansible.module_utils.' in b_module_data:
module_style = 'new'
module_substyle = 'python'
elif REPLACER_WINDOWS in module_data:
elif REPLACER_WINDOWS in b_module_data:
module_style = 'new'
module_substyle = 'powershell'
elif REPLACER_JSONARGS in module_data:
elif REPLACER_JSONARGS in b_module_data:
module_style = 'new'
module_substyle = 'jsonargs'
elif b'WANT_JSON' in module_data:
elif b'WANT_JSON' in b_module_data:
module_substyle = module_style = 'non_native_want_json'
shebang = None
# Neither old-style, non_native_want_json nor binary modules should be modified
# except for the shebang line (Done by modify_module)
if module_style in ('old', 'non_native_want_json', 'binary'):
return module_data, module_style, shebang
return b_module_data, module_style, shebang
output = BytesIO()
py_module_names = set()
@ -651,10 +657,10 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
to_bytes(__author__) + b'"\n')
zf.writestr('ansible/module_utils/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n')
zf.writestr('ansible_module_%s.py' % module_name, module_data)
zf.writestr('ansible_module_%s.py' % module_name, b_module_data)
py_module_cache = { ('__init__',): b'' }
recursive_finder(module_name, module_data, py_module_names, py_module_cache, zf)
recursive_finder(module_name, b_module_data, py_module_names, py_module_cache, zf)
zf.close()
zipdata = base64.b64encode(zipoutput.getvalue())
@ -712,22 +718,23 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
minute=now.minute,
second=now.second,
)))
module_data = output.getvalue()
b_module_data = output.getvalue()
elif module_substyle == 'powershell':
# Module replacer for jsonargs and windows
lines = module_data.split(b'\n')
lines = b_module_data.split(b'\n')
for line in lines:
if REPLACER_WINDOWS in line:
ps_data = _slurp(os.path.join(_SNIPPET_PATH, "powershell.ps1"))
# FIXME: Need to make a module_utils loader for powershell at some point
ps_data = _slurp(os.path.join(_MODULE_UTILS_PATH, "powershell.ps1"))
output.write(ps_data)
py_module_names.add((b'powershell',))
continue
output.write(line + b'\n')
module_data = output.getvalue()
b_module_data = output.getvalue()
module_args_json = to_bytes(json.dumps(module_args))
module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)
b_module_data = b_module_data.replace(REPLACER_JSONARGS, module_args_json)
# Powershell/winrm don't actually make use of shebang so we can
# safely set this here. If we let the fallback code handle this
@ -751,17 +758,17 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
# ansiballz) If we remove them from jsonargs-style module replacer
# then we can remove them everywhere.
python_repred_args = to_bytes(repr(module_args_json))
module_data = module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__)))
module_data = module_data.replace(REPLACER_COMPLEX, python_repred_args)
module_data = module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS)))
b_module_data = b_module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__)))
b_module_data = b_module_data.replace(REPLACER_COMPLEX, python_repred_args)
b_module_data = b_module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS)))
# The main event -- substitute the JSON args string into the module
module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)
b_module_data = b_module_data.replace(REPLACER_JSONARGS, module_args_json)
facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='surrogate_or_strict')
module_data = module_data.replace(b'syslog.LOG_USER', facility)
b_module_data = b_module_data.replace(b'syslog.LOG_USER', facility)
return (module_data, module_style, shebang)
return (b_module_data, module_style, shebang)
def modify_module(module_name, module_path, module_args, task_vars=dict(), module_compression='ZIP_STORED'):
@ -791,14 +798,14 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul
with open(module_path, 'rb') as f:
# read in the module source
module_data = f.read()
b_module_data = f.read()
(module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression)
(b_module_data, module_style, shebang) = _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, module_compression)
if module_style == 'binary':
return (module_data, module_style, to_text(shebang, nonstring='passthru'))
return (b_module_data, module_style, to_text(shebang, nonstring='passthru'))
elif shebang is None:
lines = module_data.split(b"\n", 1)
lines = b_module_data.split(b"\n", 1)
if lines[0].startswith(b"#!"):
shebang = lines[0].strip()
args = shlex.split(str(shebang[2:]))
@ -815,8 +822,8 @@ def modify_module(module_name, module_path, module_args, task_vars=dict(), modul
# No shebang, assume a binary module?
pass
module_data = b"\n".join(lines)
b_module_data = b"\n".join(lines)
else:
shebang = to_bytes(shebang, errors='surrogate_or_strict')
return (module_data, module_style, to_text(shebang, nonstring='passthru'))
return (b_module_data, module_style, to_text(shebang, nonstring='passthru'))