Move uses of to_bytes, to_text, to_native to use the module_utils version (#17423)

We couldn't copy to_unicode, to_bytes, to_str into module_utils because
of licensing.  So once created it we had two sets of functions that did
the same things but had different implementations.  To remedy that, this
change removes the ansible.utils.unicode versions of those functions.

lib/ansible/plugins/lookup/__init__.py

@@ -98,8 +98,8 @@ class LookupBase(with_metaclass(ABCMeta, object)):
         must be converted into python's unicode type as the strings will be run
         through jinja2 which has this requirement.  You can use::
 
-            from ansible.utils.unicode import to_unicode
-            result_string = to_unicode(result_string)
+            from ansible.module_utils._text import to_text
+            result_string = to_text(result_string)
         """
         pass
 

lib/ansible/plugins/lookup/csvfile.py

@@ -22,7 +22,8 @@ import csv
 
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
-from ansible.utils.unicode import to_bytes, to_str, to_unicode
+from ansible.module_utils._text import to_bytes, to_native, to_text
+
 
 class CSVRecoder:
     """
@@ -49,7 +50,7 @@ class CSVReader:
 
     def next(self):
         row = self.reader.next()
-        return [to_unicode(s) for s in row]
+        return [to_text(s) for s in row]
 
     def __iter__(self):
         return self
@@ -66,7 +67,7 @@ class LookupModule(LookupBase):
                 if row[0] == key:
                     return row[int(col)]
         except Exception as e:
-            raise AnsibleError("csvfile: %s" % to_str(e))
+            raise AnsibleError("csvfile: %s" % to_native(e))
 
         return dflt
 

lib/ansible/plugins/lookup/fileglob.py

@@ -22,7 +22,8 @@ import glob
 
 from ansible.plugins.lookup import LookupBase
 from ansible.errors import AnsibleFileNotFound
-from ansible.utils.unicode import to_bytes, to_unicode
+from ansible.module_utils._text import to_bytes, to_text
+
 
 class LookupModule(LookupBase):
 
@@ -36,6 +37,6 @@ class LookupModule(LookupBase):
             except AnsibleFileNotFound:
                 dwimmed_path = None
             if dwimmed_path:
-                globbed = glob.glob(to_bytes(os.path.join(dwimmed_path, term_file), errors='strict'))
-                ret.extend(to_unicode(g, errors='strict') for g in globbed if os.path.isfile(g))
+                globbed = glob.glob(to_bytes(os.path.join(dwimmed_path, term_file), errors='surrogate_or_strict'))
+                ret.extend(to_text(g, errors='surrogate_or_strict') for g in globbed if os.path.isfile(g))
         return ret

lib/ansible/plugins/lookup/filetree.py

@@ -22,12 +22,6 @@ import pwd
 import grp
 import stat
 
-from ansible.plugins.lookup import LookupBase
-from ansible.utils.unicode import to_str
-
-from __main__ import display
-warning = display.warning
-
 HAVE_SELINUX=False
 try:
     import selinux
@@ -35,6 +29,11 @@ try:
 except ImportError:
     pass
 
+from ansible.plugins.lookup import LookupBase
+from ansible.module_utils._text import to_native
+
+from __main__ import display
+
 
 # If selinux fails to find a default, return an array of None
 def selinux_context(path):
@@ -43,7 +42,7 @@ def selinux_context(path):
         try:
             # note: the selinux module uses byte strings on python2 and text
             # strings on python3
-            ret = selinux.lgetfilecon_raw(to_str(path))
+            ret = selinux.lgetfilecon_raw(to_native(path))
         except OSError:
             return context
         if ret[0] != -1:
@@ -60,7 +59,7 @@ def file_props(root, path):
     try:
         st = os.lstat(abspath)
     except OSError as e:
-        warning('filetree: Error using stat() on path %s (%s)' % (abspath, e))
+        display.warning('filetree: Error using stat() on path %s (%s)' % (abspath, e))
         return None
 
     ret = dict(root=root, path=path)
@@ -74,7 +73,7 @@ def file_props(root, path):
         ret['state'] = 'file'
         ret['src'] = abspath
     else:
-        warning('filetree: Error file type of %s is not supported' % abspath)
+        display.warning('filetree: Error file type of %s is not supported' % abspath)
         return None
 
     ret['uid'] = st.st_uid

lib/ansible/plugins/lookup/ini.py

@@ -30,7 +30,7 @@ except ImportError:
 
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
-from ansible.utils.unicode import to_bytes
+from ansible.module_utils._text import to_bytes, to_text
 
 
 def _parse_params(term):
@@ -59,13 +59,15 @@ class LookupModule(LookupBase):
 
     def read_properties(self, filename, key, dflt, is_regexp):
         config = StringIO()
-        config.write(u'[java_properties]\n' + open(to_bytes(filename, errors='strict')).read())
+        current_cfg_file = open(to_bytes(filename, errors='surrogate_or_strict'), 'rb')
+
+        config.write(u'[java_properties]\n' + to_text(current_cfg_file.read(), errors='surrogate_or_strict'))
         config.seek(0, os.SEEK_SET)
         self.cp.readfp(config)
         return self.get_value(key, 'java_properties', dflt, is_regexp)
 
     def read_ini(self, filename, key, section, dflt, is_regexp):
-        self.cp.readfp(open(to_bytes(filename, errors='strict')))
+        self.cp.readfp(open(to_bytes(filename, errors='surrogate_or_strict')))
         return self.get_value(key, section, dflt, is_regexp)
 
     def get_value(self, key, section, dflt, is_regexp):

lib/ansible/plugins/lookup/shelvefile.py

@@ -21,11 +21,11 @@ import shelve
 
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
-from ansible.utils.unicode import to_bytes, to_unicode
+from ansible.module_utils._text import to_bytes, to_text
+
 
 class LookupModule(LookupBase):
 
-
     def read_shelve(self, shelve_filename, key):
         """
         Read the value of "key" from a shelve file
@@ -66,7 +66,7 @@ class LookupModule(LookupBase):
                 if res is None:
                     raise AnsibleError("Key %s not found in shelve file %s" % (key, file))
                 # Convert the value read to string
-                ret.append(to_unicode(res))
+                ret.append(to_text(res))
                 break
             else:
                 raise AnsibleError("Could not locate shelve file in lookup: %s" % file)

lib/ansible/plugins/lookup/template.py

@@ -21,7 +21,7 @@ import os
 
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
-from ansible.utils.unicode import to_unicode, to_bytes
+from ansible.module_utils._text import to_bytes, to_text
 
 try:
     from __main__ import display
@@ -43,8 +43,8 @@ class LookupModule(LookupBase):
             lookupfile = self.find_file_in_search_path(variables, 'templates', term)
             display.vvvv("File lookup using %s as file" % lookupfile)
             if lookupfile:
-                with open(to_bytes(lookupfile, errors='strict'), 'r') as f:
-                    template_data = to_unicode(f.read())
+                with open(to_bytes(lookupfile, errors='surrogate_or_strict'), 'rb') as f:
+                    template_data = to_text(f.read(), errors='surrogate_or_strict')
 
                     # set jinja2 internal search path for includes
                     if 'ansible_search_path' in variables:

lib/ansible/plugins/lookup/url.py

@@ -18,11 +18,11 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 
-from ansible.errors import AnsibleError
-from ansible.plugins.lookup import LookupBase
-from ansible.module_utils.urls import open_url, ConnectionError, SSLValidationError
-from ansible.utils.unicode import to_unicode
 from ansible.compat.six.moves.urllib.error import HTTPError, URLError
+from ansible.errors import AnsibleError
+from ansible.module_utils._text import to_text
+from ansible.module_utils.urls import open_url, ConnectionError, SSLValidationError
+from ansible.plugins.lookup import LookupBase
 
 try:
     from __main__ import display
@@ -52,5 +52,5 @@ class LookupModule(LookupBase):
                 raise AnsibleError("Error connecting to %s: %s" % (term, str(e)))
 
             for line in response.read().splitlines():
-                ret.append(to_unicode(line))
+                ret.append(to_text(line))
         return ret