ansible-collections/community.general
1
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2025-04-23 19:01:26 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

Ziploader

Browse source 
* Ziploader proof of concept (jimi-c)

* Cleanups to proof of concept ziploader branch:

* python3 compatible base64 encoding
* zipfile compression (still need to enable toggling this off for
  systems without zlib support in python)
* Allow non-wildcard imports (still need to make this recusrsive so that
  we can have module_utils code that imports other module_utils code.)
* Better tracebacks: module filename is kept and module_utils directory
  is kept so that tracebacks show the real filenames that the errors
  appear in.

* Make sure we import modules that are used into the module_utils files that they are used in.

* Set ansible version in a more pythonic way for ziploader than we were doing in module replacer

* Make it possible to set the module compression as an inventory var

This may be necessary on systems where python has been compiled without
zlib compression.

* Refactoring of module_common code:

* module replacer only replaces values that make sense for that type of
  file (example: don't attempt to replace python imports if we're in
  a powershell module).
* Implement configurable shebang support for ziploader wrapper
* Implement client-side constants (for SELINUX_SPECIAL_FS and SYSLOG)
  via environment variable.
* Remove strip_comments param as we're never going to use it (ruins line
  numbering)

* Don't repeat ourselves about detecting REPLACER

* Add an easy way to debug

* Port test-module to the ziploader-aware modify_module()

* strip comments and blank lines from the wrapper so we send less over the wire.

* Comments cleanup

* Remember to output write the module line itself in powershell modules

* for line in lines strips the newlines so we have to add them back in
This commit is contained in:
Toshio Kuratomi 2016-04-05 11:06:17 -07:00
parent 6a3670b1f0
commit 4b0aa1214c
32 changed files with 438 additions and 125 deletions
Download patch file Download diff file Expand all files Collapse all files

328
lib/ansible/executor/module_common.py
View file

@ -20,11 +20,12 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
# from python and deps
from io import BytesIO
import base64
import json
import os
import shlex
import zipfile
from io import BytesIO
# from Ansible
from ansible import __version__
@ -32,13 +33,17 @@ from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes, to_unicode
try:
from __main__ import display
except ImportError:
from ansible.utils.display import Display
display = Display()
REPLACER = b"#<<INCLUDE_ANSIBLE_MODULE_COMMON>>"
REPLACER_ARGS = b"\"<<INCLUDE_ANSIBLE_MODULE_ARGS>>\""
REPLACER_VERSION = b"\"<<ANSIBLE_VERSION>>\""
REPLACER_COMPLEX = b"\"<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>\""
REPLACER_WINDOWS = b"# POWERSHELL_COMMON"
REPLACER_WINARGS = b"<<INCLUDE_ANSIBLE_MODULE_WINDOWS_ARGS>>"
REPLACER_JSONARGS = b"<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"
REPLACER_VERSION = b"\"<<ANSIBLE_VERSION>>\""
REPLACER_SELINUX = b"<<SELINUX_SPECIAL_FILESYSTEMS>>"
# We could end up writing out parameters with unicode characters so we need to
@ -50,6 +55,82 @@ _SNIPPET_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils')
# ******************************************************************************
ZIPLOADER_TEMPLATE = u'''%(shebang)s
# -*- coding: utf-8 -*-'
import os
import sys
import base64
import tempfile
ZIPDATA = """%(zipdata)s"""
def debug(command, zipped_mod):
# The code here normally doesn't run. It's only used for debugging on the
# remote machine. Run with ANSIBLE_KEEP_REMOTE_FILES=1 envvar and -vvv
# to save the module file remotely. Login to the remote machine and use
# /path/to/module explode to extract the ZIPDATA payload into source
# files. Edit the source files to instrument the code or experiment with
# different values. Then use /path/to/module execute to run the extracted
# files you've edited instead of the actual zipped module.
#
# Okay to use __file__ here because we're running from a kept file
basedir = os.path.dirname(__file__)
if command == 'explode':
import zipfile
z = zipfile.ZipFile(zipped_mod)
for filename in z.namelist():
if filename.startswith('/'):
raise Exception('Something wrong with this module zip file: should not contain absolute paths')
dest_filename = os.path.join(basedir, filename)
if dest_filename.endswith(os.path.sep) and not os.path.exists(dest_filename):
os.makedirs(dest_filename)
else:
directory = os.path.dirname(dest_filename)
if not os.path.exists(directory):
os.makedirs(directory)
f = open(dest_filename, 'w')
f.write(z.read(filename))
f.close()
print('Module expanded into: %%s' %% os.path.join(basedir, 'ansible'))
elif command == 'execute':
sys.path.insert(0, basedir)
from ansible.module_exec.%(ansible_module)s.__main__ import main
main()
os.environ['ANSIBLE_MODULE_ARGS'] = %(args)s
os.environ['ANSIBLE_MODULE_CONSTANTS'] = %(constants)s
try:
temp_fd, temp_path = tempfile.mkstemp(prefix='ansible_')
os.write(temp_fd, base64.b64decode(ZIPDATA))
if len(sys.argv) == 2:
debug(sys.argv[1], temp_path)
else:
sys.path.insert(0, temp_path)
from ansible.module_exec.%(ansible_module)s.__main__ import main
main()
finally:
try:
os.close(temp_fd)
os.remove(temp_path)
except NameError:
# mkstemp failed
pass
'''
def _strip_comments(source):
# Strip comments and blank lines from the wrapper
buf = []
for line in source.splitlines():
l = line.strip()
if not l or l.startswith(u'#'):
continue
buf.append(line)
return u'\n'.join(buf)
# ZIPLOADER_TEMPLATE stripped of comments for smaller over the wire size
STRIPPED_ZIPLOADER_TEMPLATE = _strip_comments(ZIPLOADER_TEMPLATE)
def _slurp(path):
if not os.path.exists(path):
raise AnsibleError("imported module support code does not exist at %s" % path)
@ -58,69 +139,171 @@ def _slurp(path):
fd.close()
return data
def _find_snippet_imports(module_data, module_path, strip_comments):
def _get_shebang(interpreter, task_vars, args=tuple()):
"""
Note not stellar API:
Returns None instead of always returning a shebang line. Doing it this
way allows the caller to decide to use the shebang it read from the
file rather than trust that we reformatted what they already have
correctly.
"""
interpreter_config = u'ansible_%s_interpreter' % os.path.basename(interpreter)
if interpreter_config not in task_vars:
return None
interpreter = task_vars[interpreter_config]
shebang = u'#!' + interpreter
if args:
shebang = shebang + u' ' + u' '.join(args)
return shebang
def _get_facility(task_vars):
facility = C.DEFAULT_SYSLOG_FACILITY
if 'ansible_syslog_facility' in task_vars:
facility = task_vars['ansible_syslog_facility']
return facility
def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression):
"""
Given the source of the module, convert it to a Jinja2 template to insert
module code and return whether it's a new or old style module.
"""
module_style = 'old'
module_substyle = module_style = 'old'
# module_style is something important to calling code (ActionBase). It
# determines how arguments are formatted (json vs k=v) and whether
# a separate arguments file needs to be sent over the wire.
# module_substyle is extra information that's useful internally. It tells
# us what we have to look to substitute in the module files and whether
# we're using module replacer or ziploader to format the module itself.
if REPLACER in module_data:
# Do REPLACER before from ansible.module_utils because we need make sure
# we substitute "from ansible.module_utils basic" for REPLACER
module_style = 'new'
elif REPLACER_WINDOWS in module_data:
module_style = 'new'
elif REPLACER_JSONARGS in module_data:
module_style = 'new'
module_substyle = 'python'
module_data = module_data.replace(REPLACER, b'from ansible.module_utils.basic import *')
elif b'from ansible.module_utils.' in module_data:
module_style = 'new'
module_substyle = 'python'
elif REPLACER_WINDOWS in module_data:
module_style = 'new'
module_substyle = 'powershell'
elif REPLACER_JSONARGS in module_data:
module_style = 'new'
module_substyle = 'jsonargs'
elif b'WANT_JSON' in module_data:
module_style = 'non_native_want_json'
module_substyle = module_style = 'non_native_want_json'
shebang = None
# Neither old-style nor non_native_want_json modules should be modified
# except for the shebang line (Done by modify_module)
if module_style in ('old', 'non_native_want_json'):
return module_data, module_style, shebang
module_args_json = to_bytes(json.dumps(module_args))
output = BytesIO()
lines = module_data.split(b'\n')
snippet_names = []
for line in lines:
snippet_names = set()
if REPLACER in line:
output.write(_slurp(os.path.join(_SNIPPET_PATH, "basic.py")))
snippet_names.append(b'basic')
if REPLACER_WINDOWS in line:
ps_data = _slurp(os.path.join(_SNIPPET_PATH, "powershell.ps1"))
output.write(ps_data)
snippet_names.append(b'powershell')
elif line.startswith(b'from ansible.module_utils.'):
tokens=line.split(b".")
import_error = False
if len(tokens) != 3:
import_error = True
if b" import *" not in line:
import_error = True
if import_error:
raise AnsibleError("error importing module in %s, expecting format like 'from ansible.module_utils.<lib name> import *'" % module_path)
snippet_name = tokens[2].split()[0]
snippet_names.append(snippet_name)
output.write(_slurp(os.path.join(_SNIPPET_PATH, to_unicode(snippet_name) + ".py")))
else:
if strip_comments and line.startswith(b"#") or line == b'':
pass
output.write(line)
output.write(b"\n")
if module_substyle == 'python':
# ziploader for new-style python classes
python_repred_args = to_bytes(repr(module_args_json))
constants = dict(
SELINUX_SPECIAL_FS=C.DEFAULT_SELINUX_SPECIAL_FS,
SYSLOG_FACILITY=_get_facility(task_vars),
)
python_repred_constants = to_bytes(repr(json.dumps(constants)), errors='strict')
if not module_path.endswith(".ps1"):
# Unixy modules
if len(snippet_names) > 0 and not b'basic' in snippet_names:
raise AnsibleError("missing required import in %s: from ansible.module_utils.basic import *" % module_path)
else:
# Windows modules
if len(snippet_names) > 0 and not b'powershell' in snippet_names:
try:
compression_method = getattr(zipfile, module_compression)
except AttributeError:
display.warning(u'Bad module compression string specified: %s. Using ZIP_STORED (no compression)' % module_compression)
compression_method = zipfile.ZIP_STORED
zipoutput = BytesIO()
zf = zipfile.ZipFile(zipoutput, mode='w', compression=compression_method)
zf.writestr('ansible/__init__.py', b''.join((b"__version__ = '", to_bytes(__version__), b"'\n")))
zf.writestr('ansible/module_utils/__init__.py', b'')
zf.writestr('ansible/module_exec/__init__.py', b'')
zf.writestr('ansible/module_exec/%s/__init__.py' % module_name, b"")
final_data = []
for line in lines:
if line.startswith(b'from ansible.module_utils.'):
tokens=line.split(b".")
snippet_name = tokens[2].split()[0]
snippet_names.add(snippet_name)
fname = to_unicode(snippet_name + b".py")
zf.writestr(os.path.join("ansible/module_utils", fname), _slurp(os.path.join(_SNIPPET_PATH, fname)))
final_data.append(line)
else:
final_data.append(line)
zf.writestr('ansible/module_exec/%s/__main__.py' % module_name, b"\n".join(final_data))
zf.close()
shebang = _get_shebang(u'/usr/bin/python', task_vars) or u'#!/usr/bin/python'
output.write(to_bytes(STRIPPED_ZIPLOADER_TEMPLATE % dict(
zipdata=base64.b64encode(zipoutput.getvalue()),
ansible_module=module_name,
args=python_repred_args,
constants=python_repred_constants,
shebang=shebang,
)))
module_data = output.getvalue()
# Sanity check from 1.x days. Maybe too strict. Some custom python
# modules that use ziploader may implement their own helpers and not
# need basic.py. All the constants that we substituted into basic.py
# for module_replacer are now available in other, better ways.
if b'basic' not in snippet_names:
raise AnsibleError("missing required import in %s: Did not import ansible.module_utils.basic for boilerplate helper code" % module_path)
elif module_substyle == 'powershell':
# Module replacer for jsonargs and windows
for line in lines:
if REPLACER_WINDOWS in line:
ps_data = _slurp(os.path.join(_SNIPPET_PATH, "powershell.ps1"))
output.write(ps_data)
snippet_names.add(b'powershell')
continue
output.write(line + b'\n')
module_data = output.getvalue()
module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)
# Sanity check from 1.x days. This is currently useless as we only
# get here if we are going to substitute powershell.ps1 into the
# module anyway. Leaving it for when/if we add other powershell
# module_utils files.
if b'powershell' not in snippet_names:
raise AnsibleError("missing required import in %s: # POWERSHELL_COMMON" % module_path)
return (output.getvalue(), module_style)
elif module_substyle == 'jsonargs':
# these strings could be included in a third-party module but
# officially they were included in the 'basic' snippet for new-style
# python modules (which has been replaced with something else in
# ziploader) If we remove them from jsonargs-style module replacer
# then we can remove them everywhere.
module_data = module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__)))
module_data = module_data.replace(REPLACER_COMPLEX, python_repred_args)
module_data = module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS)))
# The main event -- substitute the JSON args string into the module
module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)
facility = b'syslog.' + to_bytes(_get_facility(task_vars), errors='strict')
module_data = module_data.replace(b'syslog.LOG_USER', facility)
return (module_data, module_style, shebang)
# ******************************************************************************
def modify_module(module_path, module_args, task_vars=dict(), strip_comments=False):
def modify_module(module_name, module_path, module_args, task_vars=dict(), module_compression='ZIP_STORED'):
"""
Used to insert chunks of code into modules before transfer rather than
doing regular python imports. This allows for more efficient transfer in
@ -163,43 +346,28 @@ def modify_module(module_path, module_args, task_vars=dict(), strip_comments=Fal
# read in the module source
module_data = f.read()
(module_data, module_style) = _find_snippet_imports(module_data, module_path, strip_comments)
(module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression)
module_args_json = to_bytes(json.dumps(module_args))
python_repred_args = to_bytes(repr(module_args_json))
if shebang is None:
lines = module_data.split(b"\n", 1)
if lines[0].startswith(b"#!"):
shebang = lines[0].strip()
args = shlex.split(str(shebang[2:]))
interpreter = args[0]
interpreter = to_bytes(interpreter)
# these strings should be part of the 'basic' snippet which is required to be included
module_data = module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__)))
module_data = module_data.replace(REPLACER_COMPLEX, python_repred_args)
module_data = module_data.replace(REPLACER_WINARGS, module_args_json)
module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)
module_data = module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS)))
new_shebang = to_bytes(_get_shebang(interpreter, task_vars, args[1:]), errors='strict', nonstring='passthru')
if new_shebang:
lines[0] = shebang = new_shebang
if module_style == 'new':
facility = C.DEFAULT_SYSLOG_FACILITY
if 'ansible_syslog_facility' in task_vars:
facility = task_vars['ansible_syslog_facility']
module_data = module_data.replace(b'syslog.LOG_USER', to_bytes("syslog.%s" % facility))
if os.path.basename(interpreter).startswith(b'python'):
lines.insert(1, ENCODING_STRING)
else:
# No shebang, assume a binary module?
pass
lines = module_data.split(b"\n", 1)
shebang = None
if lines[0].startswith(b"#!"):
shebang = lines[0].strip()
args = shlex.split(str(shebang[2:]))
interpreter = args[0]
interpreter_config = 'ansible_%s_interpreter' % os.path.basename(interpreter)
interpreter = to_bytes(interpreter)
if interpreter_config in task_vars:
interpreter = to_bytes(task_vars[interpreter_config], errors='strict')
lines[0] = shebang = b"#!{0} {1}".format(interpreter, b" ".join(args[1:]))
if os.path.basename(interpreter).startswith(b'python'):
lines.insert(1, ENCODING_STRING)
module_data = b"\n".join(lines)
else:
# No shebang, assume a binary module?
pass
module_data = b"\n".join(lines)
shebang = to_bytes(shebang, errors='strict')
return (module_data, module_style, shebang)