Update module_utils.six to latest (#22855)

* Update module_utils.six to latest

We've been held back on the version of six we could use on the module
side to 1.4.x because of python-2.4 compatibility.  Now that our minimum
is Python-2.6, we can update to the latest version of six in
module_utils and get rid of the second copy in lib/ansible/compat.

test/sanity/code-smell/boilerplate.sh

@@ -6,7 +6,7 @@ future1=$(find ./bin -type f -exec grep -HL 'from __future__ import (absolute_im
 metaclass2=$(find ./lib/ansible -path ./lib/ansible/modules -prune \
         -o -path ./lib/ansible/modules/__init__.py \
         -o -path ./lib/ansible/module_utils -prune \
-        -o -path ./lib/ansible/compat/six/_six.py -prune \
+        -o -path ./lib/ansible/module_utils/six/_six.py -prune \
         -o -path ./lib/ansible/compat/selectors/_selectors2.py -prune \
         -o -path ./lib/ansible/utils/module_docs_fragments -prune \
         -o -name '*.py' -exec grep -HL '__metaclass__ = type' '{}' '+')
@@ -14,7 +14,7 @@ metaclass2=$(find ./lib/ansible -path ./lib/ansible/modules -prune \
 future2=$(find ./lib/ansible -path ./lib/ansible/modules -prune \
         -o -path ./lib/ansible/modules/__init__.py \
         -o -path ./lib/ansible/module_utils -prune \
-        -o -path ./lib/ansible/compat/six/_six.py -prune \
+        -o -path ./lib/ansible/module_utils/six/_six.py -prune \
         -o -path ./lib/ansible/compat/selectors/_selectors2.py -prune \
         -o -path ./lib/ansible/utils/module_docs_fragments -prune \
         -o -name '*.py' -exec grep -HL 'from __future__ import (absolute_import, division, print_function)' '{}' '+')

test/sanity/code-smell/no-basestring.sh

@@ -6,8 +6,7 @@ BASESTRING_USERS=$(grep -r basestring . \
     | grep isinstance \
     | grep -v \
     -e test/results/ \
-    -e lib/ansible/compat/six/_six.py \
-    -e lib/ansible/module_utils/six.py \
+    -e lib/ansible/module_utils/six/_six.py \
     -e lib/ansible/modules/ \
     -e '^[^:]*:#'
     )

test/sanity/code-smell/no-dict-iteritems.sh

@@ -6,8 +6,7 @@ ITERITEMS_USERS=$(grep -rI '\.iteritems' . \
     --exclude-dir docsite \
     | grep -v \
     -e 'six\.iteritems' \
-    -e lib/ansible/compat/six/_six.py \
-    -e lib/ansible/module_utils/six.py \
+    -e lib/ansible/module_utils/six/_six.py \
     -e test/sanity/code-smell/no-dict-iteritems.sh \
     )
 
@@ -15,7 +14,6 @@ if [ "${ITERITEMS_USERS}" ]; then
     echo 'iteritems has been removed in python3.  Alternatives:'
     echo '    for KEY, VALUE in DICT.items():'
     echo '    from ansible.module_utils.six import iteritems ; for KEY, VALUE in iteritems(DICT):'
-    echo '    from ansible.compat.six import iteritems ; for KEY, VALUE in iteritems(DICT):'
     echo "${ITERITEMS_USERS}"
     exit 1
 fi

test/sanity/code-smell/no-dict-itervalues.sh

@@ -6,8 +6,7 @@ ITERVALUES_USERS=$(grep -rI '\.itervalues' . \
     --exclude-dir docsite \
     | grep -v \
     -e 'six\.itervalues' \
-    -e lib/ansible/compat/six/_six.py \
-    -e lib/ansible/module_utils/six.py \
+    -e lib/ansible/module_utils/six/_six.py \
     -e test/sanity/code-smell/no-dict-itervalues.sh \
     )
 
@@ -15,7 +14,6 @@ if [ "${ITERVALUES_USERS}" ]; then
     echo 'itervalues has been removed in python3.  Alternatives:'
     echo '    for VALUE in DICT.values():'
     echo '    from ansible.module_utils.six import itervalues ; for VALUE in itervalues(DICT):'
-    echo '    from ansible.compat.six import itervalues ; for VALUE in itervalues(DICT):'
     echo "${ITERVALUES_USERS}"
     exit 1
 fi

test/sanity/code-smell/no-iterkeys.sh

@@ -7,8 +7,7 @@ ITERKEYS_USERS=$(grep -r -I iterkeys . \
     --exclude-dir docsite \
     --exclude-dir results \
     | grep -v \
-    -e lib/ansible/compat/six/_six.py \
-    -e lib/ansible/module_utils/six.py \
+    -e lib/ansible/module_utils/six/_six.py \
     -e test/sanity/code-smell/no-iterkeys.sh \
     -e '^[^:]*:#'
     )

test/sanity/code-smell/replace-urlopen.sh

@@ -3,8 +3,7 @@
 urllib_users=$(find . -name '*.py' -exec grep -H urlopen '{}' '+' | grep -v \
     -e '^[^:]*/.tox/' \
     -e '^\./lib/ansible/module_utils/urls.py:' \
-    -e '^\./lib/ansible/module_utils/six.py:' \
-    -e '^\./lib/ansible/compat/six/_six.py:' \
+    -e '^\./lib/ansible/module_utils/six/_six.py:' \
     -e '^[^:]*:#'
 )
 

test/sanity/code-smell/use-compat-six.sh

@@ -10,8 +10,8 @@ WHITELIST='(lib/ansible/modules/cloud/digital_ocean/digital_ocean.py)'
 
 SIX_USERS=$(find "$BASEDIR" -name '*.py' -exec grep -wH six '{}' '+' \
     | grep import \
-    | grep -v ansible.compat \
     | grep -v ansible.module_utils.six \
+    | grep -v 'ansible.module_utils import six' \
     | egrep -v "^$WHITELIST:"
 )
 

test/sanity/pep8/legacy-files.txt

@@ -57,8 +57,6 @@ lib/ansible/cli/pull.py
 lib/ansible/cli/vault.py
 lib/ansible/compat/__init__.py
 lib/ansible/compat/selectors/__init__.py
-lib/ansible/compat/six/__init__.py
-lib/ansible/compat/six/_six.py
 lib/ansible/compat/tests/__init__.py
 lib/ansible/constants.py
 lib/ansible/errors/__init__.py
@@ -135,7 +133,7 @@ lib/ansible/module_utils/pycompat24.py
 lib/ansible/module_utils/redhat.py
 lib/ansible/module_utils/service.py
 lib/ansible/module_utils/shell.py
-lib/ansible/module_utils/six.py
+lib/ansible/module_utils/six/_six.py
 lib/ansible/module_utils/splitter.py
 lib/ansible/module_utils/sros.py
 lib/ansible/module_utils/univention_umc.py
@@ -887,12 +885,9 @@ lib/ansible/plugins/action/copy.py
 lib/ansible/plugins/action/dellos10_config.py
 lib/ansible/plugins/action/dellos6_config.py
 lib/ansible/plugins/action/dellos9_config.py
-lib/ansible/plugins/action/eos.py
 lib/ansible/plugins/action/eos_template.py
 lib/ansible/plugins/action/fetch.py
 lib/ansible/plugins/action/group_by.py
-lib/ansible/plugins/action/include_vars.py
-lib/ansible/plugins/action/ios.py
 lib/ansible/plugins/action/ios_template.py
 lib/ansible/plugins/action/iosxr_template.py
 lib/ansible/plugins/action/junos.py
@@ -908,13 +903,11 @@ lib/ansible/plugins/action/patch.py
 lib/ansible/plugins/action/pause.py
 lib/ansible/plugins/action/script.py
 lib/ansible/plugins/action/service.py
-lib/ansible/plugins/action/set_fact.py
 lib/ansible/plugins/action/set_stats.py
 lib/ansible/plugins/action/sros_config.py
 lib/ansible/plugins/action/synchronize.py
 lib/ansible/plugins/action/template.py
 lib/ansible/plugins/action/unarchive.py
-lib/ansible/plugins/action/vyos.py
 lib/ansible/plugins/cache/__init__.py
 lib/ansible/plugins/cache/base.py
 lib/ansible/plugins/cache/jsonfile.py
@@ -1056,7 +1049,6 @@ test/units/executor/test_task_result.py
 test/units/inventory/test_inventory.py
 test/units/mock/generator.py
 test/units/mock/loader.py
-test/units/mock/yaml_helper.py
 test/units/module_utils/basic/test__log_invocation.py
 test/units/module_utils/basic/test_deprecate_warn.py
 test/units/module_utils/basic/test_exit_json.py

test/units/cli/test_galaxy.py

@@ -25,9 +25,9 @@ import tarfile
 import tempfile
 import yaml
 
-from ansible.compat.six import PY3
 from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import call, patch
+from ansible.module_utils.six import PY3
 
 import ansible
 from ansible.errors import AnsibleError, AnsibleOptionsError

test/units/executor/module_common/test_module_common.py

@@ -22,9 +22,9 @@ __metaclass__ = type
 import pytest
 
 import ansible.errors
-from ansible.compat.six import PY2
 
 from ansible.executor import module_common as amc
+from ansible.module_utils.six import PY2
 
 
 class TestStripComments(object):

test/units/executor/module_common/test_recursive_finder.py

@@ -28,10 +28,10 @@ from io import BytesIO, StringIO
 import pytest
 
 import ansible.errors
-from ansible.compat.six import PY2
-from ansible.compat.six.moves import builtins
 
 from ansible.executor.module_common import recursive_finder
+from ansible.module_utils.six import PY2
+from ansible.module_utils.six.moves import builtins
 
 
 original_find_module = imp.find_module
@@ -113,22 +113,22 @@ class TestRecursiveFinder(object):
         name = 'ping'
         data = b'#!/usr/bin/python\nfrom ansible.module_utils import six'
         recursive_finder(name, data, *finder_containers)
-        assert finder_containers.py_module_names == set((('six',),))
+        assert finder_containers.py_module_names == set((('six', '__init__'), ('six', '_six')))
         assert finder_containers.py_module_cache == {}
-        assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six.py',))
+        assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six/__init__.py', 'ansible/module_utils/six/_six.py'))
 
     def test_import_six(self, finder_containers):
         name = 'ping'
         data = b'#!/usr/bin/python\nimport ansible.module_utils.six'
         recursive_finder(name, data, *finder_containers)
-        assert finder_containers.py_module_names == set((('six',),))
+        assert finder_containers.py_module_names == set((('six', '__init__'), ('six', '_six')))
         assert finder_containers.py_module_cache == {}
-        assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six.py',))
+        assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six/__init__.py', 'ansible/module_utils/six/_six.py'))
 
     def test_import_six_from_many_submodules(self, finder_containers):
         name = 'ping'
         data = b'#!/usr/bin/python\nfrom ansible.module_utils.six.moves.urllib.parse import urlparse'
         recursive_finder(name, data, *finder_containers)
-        assert finder_containers.py_module_names == set((('six',),))
+        assert finder_containers.py_module_names == set((('six', '__init__'), ('six', '_six')))
         assert finder_containers.py_module_cache == {}
-        assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six.py',))
+        assert frozenset(finder_containers.zf.namelist()) == frozenset(('ansible/module_utils/six/__init__.py', 'ansible/module_utils/six/_six.py'))

test/units/inventory/test_host.py

@@ -18,11 +18,11 @@
 # for __setstate__/__getstate__ tests
 import pickle
 
-from ansible.compat.six import string_types
 from ansible.compat.tests import unittest
 
 from ansible.inventory.group import Group
 from ansible.inventory.host import Host
+from ansible.module_utils.six import string_types
 
 
 class TestHost(unittest.TestCase):

test/units/mock/procenv.py

@@ -25,8 +25,8 @@ import json
 
 from contextlib import contextmanager
 from io import BytesIO, StringIO
-from ansible.compat.six import PY3
 from ansible.compat.tests import unittest
+from ansible.module_utils.six import PY3
 from ansible.module_utils._text import to_bytes
 
 

test/units/mock/yaml_helper.py

@@ -1,10 +1,10 @@
 import io
 import yaml
 
+from ansible.module_utils.six import PY3
 from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible.parsing.yaml.dumper import AnsibleDumper
 
-from ansible.compat.six import PY3
 
 class YamlTestUtils(object):
     """Mixin class to combine with a unittest.TestCase subclass."""

test/units/module_utils/basic/test_run_command.py

@@ -27,7 +27,7 @@ from io import BytesIO, StringIO
 
 import pytest
 
-from ansible.compat.six import PY3
+from ansible.module_utils.six import PY3
 from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import call, MagicMock, Mock, patch, sentinel
 

test/units/module_utils/test_basic.py

@@ -29,9 +29,9 @@ from io import BytesIO, StringIO
 
 from units.mock.procenv import ModuleTestCase, swap_stdin_and_argv
 
-from ansible.compat.six.moves import builtins
 from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import patch, MagicMock, mock_open, Mock, call
+from ansible.module_utils.six.moves import builtins
 
 realimport = builtins.__import__
 

test/units/module_utils/test_postgresql.py

@@ -3,10 +3,10 @@ import sys
 
 from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import patch, MagicMock
-from ansible.compat.six.moves import builtins
 
-from ansible.module_utils._text import to_native
 from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.six.moves import builtins
+from ansible.module_utils._text import to_native
 from units.mock.procenv import swap_stdin_and_argv
 
 

test/units/module_utils/test_text.py

@@ -20,8 +20,8 @@
 from __future__ import (absolute_import, division)
 __metaclass__ = type
 
-from ansible.compat.six import PY3
 from ansible.compat.tests import unittest
+from ansible.module_utils.six import PY3
 from units.mock.generator import add_method
 
 

test/units/parsing/test_dataloader.py

@@ -19,12 +19,11 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from six import PY3
-
 from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import patch, mock_open
 from ansible.errors import AnsibleParserError
 from ansible.errors import yaml_strings
+from ansible.module_utils.six import PY3
 
 from ansible.parsing.dataloader import DataLoader
 

test/units/parsing/vault/test_vault.py

@@ -21,8 +21,6 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import six
-
 import binascii
 import io
 import os
@@ -33,9 +31,10 @@ from nose.plugins.skip import SkipTest
 from ansible.compat.tests import unittest
 
 from ansible import errors
+from ansible.module_utils import six
+from ansible.module_utils._text import to_bytes, to_text
 from ansible.parsing.vault import VaultLib
 from ansible.parsing import vault
-from ansible.module_utils._text import to_bytes, to_text
 
 
 # Counter import fails for 2.0.1, requires >= 2.6.1 from pip

test/units/parsing/yaml/test_loader.py

@@ -22,12 +22,12 @@ __metaclass__ = type
 
 from io import StringIO
 
-from six import text_type, binary_type
 from collections import Sequence, Set, Mapping
 
 from ansible.compat.tests import unittest
 
 from ansible import errors
+from ansible.module_utils.six import text_type, binary_type
 from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible.parsing import vault
 from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode

test/units/playbook/test_base.py

@@ -21,8 +21,8 @@ __metaclass__ = type
 
 from ansible.compat.tests import unittest
 
-from ansible.compat.six import string_types
 from ansible.errors import AnsibleParserError
+from ansible.module_utils.six import string_types
 from ansible.playbook.attribute import FieldAttribute
 from ansible.template import Templar
 from ansible.playbook import base

test/units/playbook/test_play_context.py

@@ -25,9 +25,9 @@ from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import patch, MagicMock
 
 from ansible import constants as C
-from ansible.compat.six.moves import shlex_quote
 from ansible.cli import CLI
 from ansible.errors import AnsibleError, AnsibleParserError
+from ansible.module_utils.six.moves import shlex_quote
 from ansible.playbook.play_context import PlayContext
 
 from units.mock.loader import DictDataLoader

test/units/plugins/action/test_action.py

@@ -23,18 +23,19 @@ __metaclass__ = type
 import os
 
 from ansible import constants as C
-from ansible.compat.six import text_type
-from ansible.compat.six.moves import shlex_quote, builtins
 from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import patch, MagicMock, mock_open
 
 from ansible.errors import AnsibleError
+from ansible.module_utils.six import text_type
+from ansible.module_utils.six.moves import shlex_quote, builtins
+from ansible.module_utils._text import to_bytes
 from ansible.playbook.play_context import PlayContext
 from ansible.plugins.action import ActionBase
 from ansible.template import Templar
 
 from units.mock.loader import DictDataLoader
-from ansible.module_utils._text import to_bytes
+
 
 python_module_replacers = b"""
 #!/usr/bin/python

test/units/plugins/connection/test_ssh.py

@@ -29,8 +29,8 @@ from ansible.compat.tests.mock import patch, MagicMock, PropertyMock
 
 from ansible import constants as C
 from ansible.compat.selectors import SelectorKey, EVENT_READ
-from ansible.compat.six.moves import shlex_quote
 from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
+from ansible.module_utils.six.moves import shlex_quote
 from ansible.playbook.play_context import PlayContext
 from ansible.plugins.connection import ssh
 from ansible.module_utils._text import to_bytes

test/units/plugins/lookup/test_lastpass.py

@@ -19,15 +19,16 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from ansible.compat.tests import unittest
-
-from ansible.plugins.lookup.lastpass import LookupModule, LPass, LPassException
-from ansible.errors import AnsibleError
 from argparse import ArgumentParser
-import six
 
+from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import patch
 
+from ansible.errors import AnsibleError
+from ansible.module_utils import six
+from ansible.plugins.lookup.lastpass import LookupModule, LPass, LPassException
+
+
 MOCK_ENTRIES = [{'username': 'user',
                  'name': 'Mock Entry',
                  'password': 't0pS3cret passphrase entry!',

test/units/plugins/lookup/test_password.py

@@ -23,11 +23,11 @@ __metaclass__ = type
 import passlib
 from passlib.handlers import pbkdf2
 
-from ansible.compat.six import text_type
-from ansible.compat.six.moves import builtins
 from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import mock_open, patch
 from ansible.errors import AnsibleError
+from ansible.module_utils.six import text_type
+from ansible.module_utils.six.moves import builtins
 from ansible.plugins import PluginLoader
 from ansible.utils import encrypt
 

test/units/plugins/strategy/test_strategy_base.py

@@ -25,15 +25,15 @@ from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import patch, MagicMock
 
 from ansible.errors import AnsibleError, AnsibleParserError
-from ansible.plugins.strategy import StrategyBase
 from ansible.executor.process.worker import WorkerProcess
 from ansible.executor.task_queue_manager import TaskQueueManager
 from ansible.executor.task_result import TaskResult
+from ansible.inventory.host import Host
+from ansible.module_utils.six.moves import queue as Queue
 from ansible.playbook.block import Block
 from ansible.playbook.handler import Handler
-from ansible.inventory.host import Host
+from ansible.plugins.strategy import StrategyBase
 
-from six.moves import queue as Queue
 from units.mock.loader import DictDataLoader
 
 class TestStrategyBase(unittest.TestCase):

test/units/template/test_templar.py

@@ -23,10 +23,10 @@ from jinja2.runtime import Context
 
 from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import patch
-from ansible.compat.six import string_types
 
 from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleUndefinedVariable
+from ansible.module_utils.six import string_types
 from ansible.template import Templar, AnsibleContext, AnsibleEnvironment
 from ansible.vars.unsafe_proxy import AnsibleUnsafe, wrap_var
 #from ansible.unsafe_proxy import AnsibleUnsafe, wrap_var

test/units/test_constants.py

@@ -26,8 +26,8 @@ import os
 import pytest
 
 from ansible import constants
-from ansible.compat.six import StringIO
-from ansible.compat.six.moves import configparser
+from ansible.module_utils.six import StringIO
+from ansible.module_utils.six.moves import configparser
 from ansible.module_utils._text import to_text
 
 

test/units/vars/test_variable_manager.py

@@ -21,11 +21,11 @@ __metaclass__ = type
 
 from collections import defaultdict
 
-from ansible.compat.six import iteritems
-from ansible.compat.six.moves import builtins
 from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import MagicMock, mock_open, patch
 from ansible.inventory import Inventory
+from ansible.module_utils.six import iteritems
+from ansible.module_utils.six.moves import builtins
 from ansible.playbook.play import Play
 
 from units.mock.loader import DictDataLoader