Fix circular import with unsafe_proxy, template, and vars

template/__init__.py imported unsafe_proxy from vars which caused
vars/__init__.py to load.  vars/__init__.py needed template/__init__.py
which caused issues.  Loading unsafe_proxy from another location fixes
that.

CHANGELOG.md

@@ -21,6 +21,9 @@ Ansible Changes By Release
 * The fetch module's validate_md5 parameter has been deprecated and will be
   removed in 2.8.  If you wish to disable post-validation of the downloaded
   file, use validate_checksum instead.
+* Those using ansible as a library should note that the ansible.vars.unsafe_proxy
+  module is deprecated and slated to go away in 2.8.  The functionality has been
+  moved to ansible.utils.unsafe_proxy to avoid a circular import.
 
 ### Minor Changes
 * removed previously deprecated config option 'hostfile' and env var 'ANSIBLE_HOSTS'

lib/ansible/executor/task_executor.py

@@ -36,7 +36,7 @@ from ansible.template import Templar
 from ansible.utils.encrypt import key_for_hostname
 from ansible.utils.listify import listify_lookup_plugin_terms
 from ansible.utils.ssh_functions import check_for_controlpersist
-from ansible.vars.unsafe_proxy import UnsafeProxy, wrap_var
+from ansible.utils.unsafe_proxy import UnsafeProxy, wrap_var
 
 try:
     from __main__ import display
@@ -229,7 +229,6 @@ class TaskExecutor:
                 del self._job_vars[k]
 
         if items:
-            from ansible.vars.unsafe_proxy import UnsafeProxy
             for idx, item in enumerate(items):
                 if item is not None and not isinstance(item, UnsafeProxy):
                     items[idx] = UnsafeProxy(item)

lib/ansible/parsing/yaml/constructor.py

@@ -26,7 +26,7 @@ from ansible.module_utils._text import to_bytes
 from ansible.parsing.vault import VaultLib
 from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleSequence, AnsibleUnicode
 from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
-from ansible.vars.unsafe_proxy import wrap_var
+from ansible.utils.unsafe_proxy import wrap_var
 
 try:
     from __main__ import display

lib/ansible/parsing/yaml/dumper.py

@@ -24,8 +24,8 @@ import yaml
 from ansible.module_utils.six import PY3
 from ansible.parsing.yaml.objects import AnsibleUnicode, AnsibleSequence, AnsibleMapping
 from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
+from ansible.utils.unsafe_proxy import AnsibleUnsafeText
 from ansible.vars.hostvars import HostVars
-from ansible.vars.unsafe_proxy import AnsibleUnsafeText
 
 
 class AnsibleDumper(yaml.SafeDumper):

lib/ansible/plugins/action/__init__.py

@@ -39,7 +39,7 @@ from ansible.module_utils._text import to_bytes, to_native, to_text
 from ansible.parsing.utils.jsonify import jsonify
 from ansible.playbook.play_context import MAGIC_VARIABLE_MAPPING
 from ansible.release import __version__
-from ansible.vars.unsafe_proxy import wrap_var
+from ansible.utils.unsafe_proxy import wrap_var
 
 
 try:

lib/ansible/plugins/callback/oneline.py

@@ -35,9 +35,9 @@ class CallbackModule(CallbackBase):
     CALLBACK_NAME = 'oneline'
 
     def _command_generic_msg(self, hostname, result,  caption):
-        stdout = result.get('stdout','').replace('\n', '\\n')
+        stdout = result.get('stdout','').replace('\n', '\\n').replace('\r', '\\r')
         if 'stderr' in result and result['stderr']:
-            stderr = result.get('stderr','').replace('\n', '\\n')
+            stderr = result.get('stderr','').replace('\n', '\\n').replace('\r', '\\r')
             return "%s | %s | rc=%s | (stdout) %s (stderr) %s" % (hostname, caption, result.get('rc', -1), stdout, stderr)
         else:
             return "%s | %s | rc=%s | (stdout) %s" % (hostname, caption, result.get('rc', -1), stdout)

lib/ansible/template/__init__.py

@@ -49,7 +49,7 @@ from ansible.plugins import filter_loader, lookup_loader, test_loader
 from ansible.template.safe_eval import safe_eval
 from ansible.template.template import AnsibleJ2Template
 from ansible.template.vars import AnsibleJ2Vars
-from ansible.vars.unsafe_proxy import UnsafeProxy, wrap_var
+from ansible.utils.unsafe_proxy import UnsafeProxy, wrap_var
 
 try:
     from __main__ import display

lib/ansible/utils/unsafe_proxy.py

@@ -0,0 +1,124 @@
+# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+# --------------------------------------------
+#
+# 1. This LICENSE AGREEMENT is between the Python Software Foundation
+# ("PSF"), and the Individual or Organization ("Licensee") accessing and
+# otherwise using this software ("Python") in source or binary form and
+# its associated documentation.
+#
+# 2. Subject to the terms and conditions of this License Agreement, PSF hereby
+# grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+# analyze, test, perform and/or display publicly, prepare derivative works,
+# distribute, and otherwise use Python alone or in any derivative version,
+# provided, however, that PSF's License Agreement and PSF's notice of copyright,
+# i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+# 2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved" are
+# retained in Python alone or in any derivative version prepared by Licensee.
+#
+# 3. In the event Licensee prepares a derivative work that is based on
+# or incorporates Python or any part thereof, and wants to make
+# the derivative work available to others as provided herein, then
+# Licensee hereby agrees to include in any such work a brief summary of
+# the changes made to Python.
+#
+# 4. PSF is making Python available to Licensee on an "AS IS"
+# basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+# IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+# INFRINGE ANY THIRD PARTY RIGHTS.
+#
+# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+# FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+# A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+# OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+#
+# 6. This License Agreement will automatically terminate upon a material
+# breach of its terms and conditions.
+#
+# 7. Nothing in this License Agreement shall be deemed to create any
+# relationship of agency, partnership, or joint venture between PSF and
+# Licensee.  This License Agreement does not grant permission to use PSF
+# trademarks or trade name in a trademark sense to endorse or promote
+# products or services of Licensee, or any third party.
+#
+# 8. By copying, installing or otherwise using Python, Licensee
+# agrees to be bound by the terms and conditions of this License
+# Agreement.
+#
+# Original Python Recipe for Proxy:
+# http://code.activestate.com/recipes/496741-object-proxying/
+# Author: Tomer Filiba
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+
+from ansible.module_utils.six import string_types, text_type
+from ansible.module_utils._text import to_text
+
+
+__all__ = ['UnsafeProxy', 'AnsibleUnsafe', 'AnsibleJSONUnsafeEncoder', 'AnsibleJSONUnsafeDecoder', 'wrap_var']
+
+
+class AnsibleUnsafe(object):
+    __UNSAFE__ = True
+
+
+class AnsibleUnsafeText(text_type, AnsibleUnsafe):
+    pass
+
+
+class UnsafeProxy(object):
+    def __new__(cls, obj, *args, **kwargs):
+        # In our usage we should only receive unicode strings.
+        # This conditional and conversion exists to sanity check the values
+        # we're given but we may want to take it out for testing and sanitize
+        # our input instead.
+        if isinstance(obj, string_types):
+            obj = to_text(obj, errors='surrogate_or_strict')
+            return AnsibleUnsafeText(obj)
+        return obj
+
+
+class AnsibleJSONUnsafeEncoder(json.JSONEncoder):
+    def encode(self, obj):
+        if isinstance(obj, AnsibleUnsafe):
+            return super(AnsibleJSONUnsafeEncoder, self).encode(dict(__ansible_unsafe=True, value=unicode(obj)))
+        else:
+            return super(AnsibleJSONUnsafeEncoder, self).encode(obj)
+
+
+class AnsibleJSONUnsafeDecoder(json.JSONDecoder):
+    def decode(self, obj):
+        value = super(AnsibleJSONUnsafeDecoder, self).decode(obj)
+        if isinstance(value, dict) and '__ansible_unsafe' in value:
+            return UnsafeProxy(value.get('value', ''))
+        else:
+            return value
+
+
+def _wrap_dict(v):
+    for k in v.keys():
+        if v[k] is not None:
+            v[wrap_var(k)] = wrap_var(v[k])
+    return v
+
+
+def _wrap_list(v):
+    for idx, item in enumerate(v):
+        if item is not None:
+            v[idx] = wrap_var(item)
+    return v
+
+
+def wrap_var(v):
+    if isinstance(v, dict):
+        v = _wrap_dict(v)
+    elif isinstance(v, list):
+        v = _wrap_list(v)
+    else:
+        if v is not None and not isinstance(v, AnsibleUnsafe):
+            v = UnsafeProxy(v)
+    return v

lib/ansible/vars/__init__.py

@@ -41,7 +41,7 @@ from ansible.plugins.cache import FactCache
 from ansible.template import Templar
 from ansible.utils.listify import listify_lookup_plugin_terms
 from ansible.utils.vars import combine_vars
-from ansible.vars.unsafe_proxy import wrap_var
+from ansible.utils.unsafe_proxy import wrap_var
 from ansible.module_utils._text import to_native
 
 try:

lib/ansible/vars/unsafe_proxy.py

@@ -1,122 +1,31 @@
-# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+# (c) 2017, Toshio Kuratomi <tkuratomi@ansible.com>
-# --------------------------------------------
 #
-# 1. This LICENSE AGREEMENT is between the Python Software Foundation
+# This file is part of Ansible
-# ("PSF"), and the Individual or Organization ("Licensee") accessing and
-# otherwise using this software ("Python") in source or binary form and
-# its associated documentation.
 #
-# 2. Subject to the terms and conditions of this License Agreement, PSF hereby
+# Ansible is free software: you can redistribute it and/or modify
-# grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+# it under the terms of the GNU General Public License as published by
-# analyze, test, perform and/or display publicly, prepare derivative works,
+# the Free Software Foundation, either version 3 of the License, or
-# distribute, and otherwise use Python alone or in any derivative version,
+# (at your option) any later version.
-# provided, however, that PSF's License Agreement and PSF's notice of copyright,
-# i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-# 2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved" are
-# retained in Python alone or in any derivative version prepared by Licensee.
 #
-# 3. In the event Licensee prepares a derivative work that is based on
+# Ansible is distributed in the hope that it will be useful,
-# or incorporates Python or any part thereof, and wants to make
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# the derivative work available to others as provided herein, then
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# Licensee hereby agrees to include in any such work a brief summary of
+# GNU General Public License for more details.
-# the changes made to Python.
 #
-# 4. PSF is making Python available to Licensee on an "AS IS"
+# You should have received a copy of the GNU General Public License
-# basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-# IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
-# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
-# INFRINGE ANY THIRD PARTY RIGHTS.
-#
-# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
-# FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
-# A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
-# OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-#
-# 6. This License Agreement will automatically terminate upon a material
-# breach of its terms and conditions.
-#
-# 7. Nothing in this License Agreement shall be deemed to create any
-# relationship of agency, partnership, or joint venture between PSF and
-# Licensee.  This License Agreement does not grant permission to use PSF
-# trademarks or trade name in a trademark sense to endorse or promote
-# products or services of Licensee, or any third party.
-#
-# 8. By copying, installing or otherwise using Python, Licensee
-# agrees to be bound by the terms and conditions of this License
-# Agreement.
-#
-# Original Python Recipe for Proxy:
-# http://code.activestate.com/recipes/496741-object-proxying/
-# Author: Tomer Filiba
 
+# Make coding more python3-ish
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import json
+# This is backwards compat.  unsafe_proxy was moved to avoid circular imports.
-from ansible.module_utils.six import string_types, text_type
+from ansible.utils.unsafe_proxy import *
-from ansible.module_utils._text import to_text
 
+try:
+    from __main__ import display
+except:
+    from ansible.utils.display import Display
+    display = Display()
 
-__all__ = ['UnsafeProxy', 'AnsibleUnsafe', 'AnsibleJSONUnsafeEncoder', 'AnsibleJSONUnsafeDecoder', 'wrap_var']
+display.deprecated('ansible.vars.unsafe_proxy is deprecated.  Use ansible.utils.unsafe_proxy instead.', version='2.8')
-
-
-class AnsibleUnsafe(object):
-    __UNSAFE__ = True
-
-class AnsibleUnsafeText(text_type, AnsibleUnsafe):
-    pass
-
-
-class UnsafeProxy(object):
-    def __new__(cls, obj, *args, **kwargs):
-        # In our usage we should only receive unicode strings.
-        # This conditional and conversion exists to sanity check the values
-        # we're given but we may want to take it out for testing and sanitize
-        # our input instead.
-        if isinstance(obj, string_types):
-            obj = to_text(obj, errors='surrogate_or_strict')
-            return AnsibleUnsafeText(obj)
-        return obj
-
-
-class AnsibleJSONUnsafeEncoder(json.JSONEncoder):
-    def encode(self, obj):
-        if isinstance(obj, AnsibleUnsafe):
-            return super(AnsibleJSONUnsafeEncoder, self).encode(dict(__ansible_unsafe=True, value=unicode(obj)))
-        else:
-            return super(AnsibleJSONUnsafeEncoder, self).encode(obj)
-
-
-class AnsibleJSONUnsafeDecoder(json.JSONDecoder):
-    def decode(self, obj):
-        value = super(AnsibleJSONUnsafeDecoder, self).decode(obj)
-        if isinstance(value, dict) and '__ansible_unsafe' in value:
-            return UnsafeProxy(value.get('value', ''))
-        else:
-            return value
-
-
-def _wrap_dict(v):
-    for k in v.keys():
-        if v[k] is not None:
-            v[wrap_var(k)] = wrap_var(v[k])
-    return v
-
-
-def _wrap_list(v):
-    for idx, item in enumerate(v):
-        if item is not None:
-            v[idx] = wrap_var(item)
-    return v
-
-
-def wrap_var(v):
-    if isinstance(v, dict):
-        v = _wrap_dict(v)
-    elif isinstance(v, list):
-        v = _wrap_list(v)
-    else:
-        if v is not None and not isinstance(v, AnsibleUnsafe):
-            v = UnsafeProxy(v)
-    return v

test/sanity/pep8/legacy-files.txt

@@ -975,7 +975,6 @@ lib/ansible/utils/vars.py
 lib/ansible/vars/__init__.py
 lib/ansible/vars/hostvars.py
 lib/ansible/vars/reserved.py
-lib/ansible/vars/unsafe_proxy.py
 setup.py
 test/integration/cleanup_azure.py
 test/integration/cleanup_ec2.py

test/units/template/test_templar.py

@@ -28,8 +28,7 @@ from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleUndefinedVariable
 from ansible.module_utils.six import string_types
 from ansible.template import Templar, AnsibleContext, AnsibleEnvironment
-from ansible.vars.unsafe_proxy import AnsibleUnsafe, wrap_var
+from ansible.utils.unsafe_proxy import AnsibleUnsafe, wrap_var
-#from ansible.unsafe_proxy import AnsibleUnsafe, wrap_var
 from units.mock.loader import DictDataLoader