Ziploader

* Ziploader proof of concept (jimi-c)

* Cleanups to proof of concept ziploader branch:

* python3 compatible base64 encoding
* zipfile compression (still need to enable toggling this off for
  systems without zlib support in python)
* Allow non-wildcard imports (still need to make this recusrsive so that
  we can have module_utils code that imports other module_utils code.)
* Better tracebacks: module filename is kept and module_utils directory
  is kept so that tracebacks show the real filenames that the errors
  appear in.

* Make sure we import modules that are used into the module_utils files that they are used in.

* Set ansible version in a more pythonic way for ziploader than we were doing in module replacer

* Make it possible to set the module compression as an inventory var

This may be necessary on systems where python has been compiled without
zlib compression.

* Refactoring of module_common code:

* module replacer only replaces values that make sense for that type of
  file (example: don't attempt to replace python imports if we're in
  a powershell module).
* Implement configurable shebang support for ziploader wrapper
* Implement client-side constants (for SELINUX_SPECIAL_FS and SYSLOG)
  via environment variable.
* Remove strip_comments param as we're never going to use it (ruins line
  numbering)

* Don't repeat ourselves about detecting REPLACER

* Add an easy way to debug

* Port test-module to the ziploader-aware modify_module()

* strip comments and blank lines from the wrapper so we send less over the wire.

* Comments cleanup

* Remember to output write the module line itself in powershell modules

* for line in lines strips the newlines so we have to add them back in

test/units/module_utils/basic/test__log_invocation.py

@@ -33,6 +33,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
 
         # test basic log invocation
         basic.MODULE_COMPLEX_ARGS = json.dumps(dict(foo=False, bar=[1,2,3], bam="bam", baz=u'baz'))
+        basic.MODULE_CONSTANTS = '{}'
         am = basic.AnsibleModule(
             argument_spec=dict(
                 foo = dict(default=True, type='bool'),

test/units/module_utils/basic/test_exit_json.py

@@ -39,6 +39,7 @@ class TestAnsibleModuleExitJson(unittest.TestCase):
     def setUp(self):
         self.COMPLEX_ARGS = basic.MODULE_COMPLEX_ARGS
         basic.MODULE_COMPLEX_ARGS = '{}'
+        basic.MODULE_CONSTANTS = '{}'
 
         self.old_stdout = sys.stdout
         self.fake_stream = BytesIO()
@@ -129,6 +130,7 @@ class TestAnsibleModuleExitValuesRemoved(unittest.TestCase):
         for args, return_val, expected in self.dataset:
             sys.stdout = BytesIO()
             basic.MODULE_COMPLEX_ARGS = json.dumps(args)
+            basic.MODULE_CONSTANTS = '{}'
             module = basic.AnsibleModule(
                 argument_spec = dict(
                     username=dict(),
@@ -148,6 +150,7 @@ class TestAnsibleModuleExitValuesRemoved(unittest.TestCase):
             expected['failed'] = True
             sys.stdout = BytesIO()
             basic.MODULE_COMPLEX_ARGS = json.dumps(args)
+            basic.MODULE_CONSTANTS = '{}'
             module = basic.AnsibleModule(
                 argument_spec = dict(
                     username=dict(),

test/units/module_utils/basic/test_log.py

@@ -42,7 +42,9 @@ class TestAnsibleModuleSysLogSmokeTest(unittest.TestCase):
 
     def setUp(self):
         self.complex_args_token = basic.MODULE_COMPLEX_ARGS
+        self.constants_sentinel = basic.MODULE_CONSTANTS
         basic.MODULE_COMPLEX_ARGS = '{}'
+        basic.MODULE_CONSTANTS = '{}'
         self.am = basic.AnsibleModule(
             argument_spec = dict(),
         )
@@ -54,6 +56,7 @@ class TestAnsibleModuleSysLogSmokeTest(unittest.TestCase):
 
     def tearDown(self):
         basic.MODULE_COMPLEX_ARGS = self.complex_args_token
+        basic.MODULE_CONSTANTS = self.constants_sentinel
         basic.has_journal = self.has_journal
 
     def test_smoketest_syslog(self):
@@ -73,13 +76,16 @@ class TestAnsibleModuleJournaldSmokeTest(unittest.TestCase):
 
     def setUp(self):
         self.complex_args_token = basic.MODULE_COMPLEX_ARGS
+        self.constants_sentinel = basic.MODULE_CONSTANTS
         basic.MODULE_COMPLEX_ARGS = '{}'
+        basic.MODULE_CONSTANTS = '{}'
         self.am = basic.AnsibleModule(
             argument_spec = dict(),
         )
 
     def tearDown(self):
         basic.MODULE_COMPLEX_ARGS = self.complex_args_token
+        basic.MODULE_CONSTANTS = self.constants_sentinel
 
     @unittest.skipUnless(basic.has_journal, 'python systemd bindings not installed')
     def test_smoketest_journal(self):
@@ -116,7 +122,9 @@ class TestAnsibleModuleLogSyslog(unittest.TestCase):
 
     def setUp(self):
         self.complex_args_token = basic.MODULE_COMPLEX_ARGS
+        self.constants_sentinel = basic.MODULE_CONSTANTS
         basic.MODULE_COMPLEX_ARGS = '{}'
+        basic.MODULE_CONSTANTS = '{}'
         self.am = basic.AnsibleModule(
             argument_spec = dict(),
         )
@@ -127,6 +135,7 @@ class TestAnsibleModuleLogSyslog(unittest.TestCase):
 
     def tearDown(self):
         basic.MODULE_COMPLEX_ARGS = self.complex_args_token
+        basic.MODULE_CONSTANTS = self.constants_sentinel
         basic.has_journal = self.has_journal
 
     @patch('syslog.syslog', autospec=True)
@@ -168,7 +177,9 @@ class TestAnsibleModuleLogJournal(unittest.TestCase):
 
     def setUp(self):
         self.complex_args_token = basic.MODULE_COMPLEX_ARGS
+        self.constants_sentinel = basic.MODULE_CONSTANTS
         basic.MODULE_COMPLEX_ARGS = '{}'
+        basic.MODULE_CONSTANTS = '{}'
         self.am = basic.AnsibleModule(
             argument_spec = dict(),
         )
@@ -188,6 +199,7 @@ class TestAnsibleModuleLogJournal(unittest.TestCase):
 
     def tearDown(self):
         basic.MODULE_COMPLEX_ARGS = self.complex_args_token
+        basic.MODULE_CONSTANTS = self.constants_sentinel
         basic.has_journal = self.has_journal
         if self.module_patcher:
             self.module_patcher.stop()

test/units/module_utils/basic/test_run_command.py

@@ -62,6 +62,7 @@ class TestAnsibleModuleRunCommand(unittest.TestCase):
                 raise OSError(errno.EPERM, "Permission denied: '/inaccessible'")
 
         basic.MODULE_COMPLEX_ARGS = '{}'
+        basic.MODULE_CONSTANTS = '{}'
         self.module = AnsibleModule(argument_spec=dict())
         self.module.fail_json = MagicMock(side_effect=SystemExit)
 

test/units/module_utils/basic/test_safe_eval.py

@@ -29,6 +29,7 @@ class TestAnsibleModuleExitJson(unittest.TestCase):
         from ansible.module_utils import basic
 
         basic.MODULE_COMPLEX_ARGS = '{}'
+        basic.MODULE_CONSTANTS = '{}'
         am = basic.AnsibleModule(
             argument_spec=dict(),
         )