Docker common consolidation (#49707)

* [docker] Consolidating Python Boolean conversion for Docker API (#49563)

* [docker] Consolidating docker option min version checks (#49564)

* [docker] Moving option min version checks out of docker_swarm (#49564)

Also renaming Boolean cleanup function and fixing docker_container minimum
version check for network interfaces.

* Cleanup from PR feedback

lib/ansible/module_utils/docker_common.py

@@ -163,7 +163,8 @@ class AnsibleDockerClient(Client):
 
     def __init__(self, argument_spec=None, supports_check_mode=False, mutually_exclusive=None,
                  required_together=None, required_if=None, min_docker_version=MIN_DOCKER_VERSION,
-                 min_docker_api_version=None):
+                 min_docker_api_version=None, option_minimal_versions=None,
+                 option_minimal_versions_ignore_params=None):
 
         merged_arg_spec = dict()
         merged_arg_spec.update(DOCKER_COMMON_ARGS)
@@ -235,6 +236,9 @@ class AnsibleDockerClient(Client):
             if self.docker_api_version < LooseVersion(min_docker_api_version):
                 self.fail('docker API version is %s. Minimum version required is %s.' % (self.docker_api_version_str, min_docker_api_version))
 
+        if option_minimal_versions is not None:
+            self._get_minimal_versions(option_minimal_versions, option_minimal_versions_ignore_params)
+
     def log(self, msg, pretty_print=False):
         pass
         # if self.debug:
@@ -416,6 +420,58 @@ class AnsibleDockerClient(Client):
                       % (self.auth_params['tls_hostname'], match.group(1), match.group(1)))
         self.fail("SSL Exception: %s" % (error))
 
+    def _get_minimal_versions(self, option_minimal_versions, ignore_params=None):
+        self.option_minimal_versions = dict()
+        for option in self.module.argument_spec:
+            if ignore_params is not None:
+                if option in ignore_params:
+                    continue
+            self.option_minimal_versions[option] = dict()
+        self.option_minimal_versions.update(option_minimal_versions)
+
+        for option, data in self.option_minimal_versions.items():
+            # Test whether option is supported, and store result
+            support_docker_py = True
+            support_docker_api = True
+            if 'docker_py_version' in data:
+                support_docker_py = self.docker_py_version >= LooseVersion(data['docker_py_version'])
+            if 'docker_api_version' in data:
+                support_docker_api = self.docker_api_version >= LooseVersion(data['docker_api_version'])
+            data['supported'] = support_docker_py and support_docker_api
+            # Fail if option is not supported but used
+            if not data['supported']:
+                # Test whether option is specified
+                if 'detect_usage' in data:
+                    used = data['detect_usage']()
+                else:
+                    used = self.module.params.get(option) is not None
+                    if used and 'default' in self.module.argument_spec[option]:
+                        used = self.module.params[option] != self.module.argument_spec[option]['default']
+                if used:
+                    # If the option is used, compose error message.
+                    if 'usage_msg' in data:
+                        usg = data['usage_msg']
+                    else:
+                        usg = 'set %s option' % (option, )
+                    if not support_docker_api:
+                        msg = 'docker API version is %s. Minimum version required is %s to %s.'
+                        msg = msg % (self.docker_api_version_str, data['docker_api_version'], usg)
+                    elif not support_docker_py:
+                        if LooseVersion(data['docker_py_version']) < LooseVersion('2.0.0'):
+                            msg = ("docker-py version is %s. Minimum version required is %s to %s. "
+                                   "Consider switching to the 'docker' package if you do not require Python 2.6 support.")
+                        elif self.docker_py_version < LooseVersion('2.0.0'):
+                            msg = ("docker-py version is %s. Minimum version required is %s to %s. "
+                                   "You have to switch to the Python 'docker' package. First uninstall 'docker-py' before "
+                                   "installing 'docker' to avoid a broken installation.")
+                        else:
+                            msg = "docker version is %s. Minimum version required is %s to %s."
+                        msg = msg % (docker_version, data['docker_py_version'], usg)
+                    else:
+                        # should not happen
+                        msg = 'Cannot %s with your configuration.' % (usg, )
+                    self.fail(msg)
+
     def get_container(self, name=None):
         '''
         Lookup a container and return the inspection results.
@@ -741,3 +797,23 @@ class DifferenceTracker(object):
         '''
         result = [entry['name'] for entry in self._diff]
         return result
+
+
+def clean_dict_booleans_for_docker_api(data):
+    '''
+    Go doesn't like Python booleans 'True' or 'False', while Ansible is just
+    fine with them in YAML. As such, they need to be converted in cases where
+    we pass dictionaries to the Docker API (e.g. docker_network's
+    driver_options and docker_prune's filters).
+    '''
+    result = dict()
+    if data is not None:
+        for k, v in data.items():
+            if v is True:
+                v = 'true'
+            elif v is False:
+                v = 'false'
+            else:
+                v = str(v)
+            result[str(k)] = v
+    return result

lib/ansible/modules/cloud/docker/docker_container.py

@@ -2745,27 +2745,43 @@ class AnsibleDockerClientContainer(AnsibleDockerClient):
             self.module.warn('The ignore_image option has been overridden by the comparisons option!')
         self.comparisons = comparisons
 
-    def _get_minimal_versions(self):
+    def _get_additional_minimal_versions(self):
-        # Helper function to detect whether any specified network uses ipv4_address or ipv6_address
+        stop_timeout_supported = self.docker_api_version >= LooseVersion('1.25')
+        stop_timeout_needed_for_update = self.module.params.get("stop_timeout") is not None and self.module.params.get('state') != 'absent'
+        if stop_timeout_supported:
+            stop_timeout_supported = self.docker_py_version >= LooseVersion('2.1')
+            if stop_timeout_needed_for_update and not stop_timeout_supported:
+                # We warn (instead of fail) since in older versions, stop_timeout was not used
+                # to update the container's configuration, but only when stopping a container.
+                self.module.warn("docker or docker-py version is %s. Minimum version required is 2.1 to update "
+                                 "the container's stop_timeout configuration. "
+                                 "If you use the 'docker-py' module, you have to switch to the docker 'Python' package." % (docker_version,))
+        else:
+            if stop_timeout_needed_for_update and not stop_timeout_supported:
+                # We warn (instead of fail) since in older versions, stop_timeout was not used
+                # to update the container's configuration, but only when stopping a container.
+                self.module.warn("docker API version is %s. Minimum version required is 1.25 to set or "
+                                 "update the container's stop_timeout configuration." % (self.docker_api_version_str,))
+        self.option_minimal_versions['stop_timeout']['supported'] = stop_timeout_supported
+
+    def __init__(self, **kwargs):
         def detect_ipvX_address_usage():
+            '''
+            Helper function to detect whether any specified network uses ipv4_address or ipv6_address
+            '''
             for network in self.module.params.get("networks") or []:
                 if network.get('ipv4_address') is not None or network.get('ipv6_address') is not None:
                     return True
             return False
 
-        self.option_minimal_versions = dict(
+        option_minimal_versions = dict(
             # internal options
             log_config=dict(),
             publish_all_ports=dict(),
             ports=dict(),
             volume_binds=dict(),
             name=dict(),
-        )
+            # normal options
-        for option, data in self.module.argument_spec.items():
-            if option in self.__NON_CONTAINER_PROPERTY_OPTIONS:
-                continue
-            self.option_minimal_versions[option] = dict()
-        self.option_minimal_versions.update(dict(
             device_read_bps=dict(docker_py_version='1.9.0', docker_api_version='1.22'),
             device_read_iops=dict(docker_py_version='1.9.0', docker_api_version='1.22'),
             device_write_bps=dict(docker_py_version='1.9.0', docker_api_version='1.22'),
@@ -2791,74 +2807,16 @@ class AnsibleDockerClientContainer(AnsibleDockerClient):
             pids_limit=dict(docker_py_version='1.10.0', docker_api_version='1.23'),
             # specials
             ipvX_address_supported=dict(docker_py_version='1.9.0', detect_usage=detect_ipvX_address_usage,
-                                        usage_msg='ipv4_address or ipv6_address in networks'),
+                                        usage_msg='ipv4_address or ipv6_address in networks'),  # see above
-            stop_timeout=dict(),  # see below!
+            stop_timeout=dict(),  # see _get_additional_minimal_versions()
-        ))
+        )
 
-        for option, data in self.option_minimal_versions.items():
+        super(AnsibleDockerClientContainer, self).__init__(
-            # Test whether option is supported, and store result
+            option_minimal_versions=option_minimal_versions,
-            support_docker_py = True
+            option_minimal_versions_ignore_params=self.__NON_CONTAINER_PROPERTY_OPTIONS,
-            support_docker_api = True
+            **kwargs
-            if 'docker_py_version' in data:
+        )
-                support_docker_py = self.docker_py_version >= LooseVersion(data['docker_py_version'])
+        self._get_additional_minimal_versions()
-            if 'docker_api_version' in data:
-                support_docker_api = self.docker_api_version >= LooseVersion(data['docker_api_version'])
-            data['supported'] = support_docker_py and support_docker_api
-            # Fail if option is not supported but used
-            if not data['supported']:
-                # Test whether option is specified
-                if 'detect_usage' in data:
-                    used = data['detect_usage']()
-                else:
-                    used = self.module.params.get(option) is not None
-                    if used and 'default' in self.module.argument_spec[option]:
-                        used = self.module.params[option] != self.module.argument_spec[option]['default']
-                if used:
-                    # If the option is used, compose error message.
-                    if 'usage_msg' in data:
-                        usg = data['usage_msg']
-                    else:
-                        usg = 'set %s option' % (option, )
-                    if not support_docker_api:
-                        msg = 'docker API version is %s. Minimum version required is %s to %s.'
-                        msg = msg % (self.docker_api_version_str, data['docker_api_version'], usg)
-                    elif not support_docker_py:
-                        if LooseVersion(data['docker_py_version']) < LooseVersion('2.0.0'):
-                            msg = ("docker-py version is %s. Minimum version required is %s to %s. "
-                                   "Consider switching to the 'docker' package if you do not require Python 2.6 support.")
-                        elif self.docker_py_version < LooseVersion('2.0.0'):
-                            msg = ("docker-py version is %s. Minimum version required is %s to %s. "
-                                   "You have to switch to the Python 'docker' package. First uninstall 'docker-py' before "
-                                   "installing 'docker' to avoid a broken installation.")
-                        else:
-                            msg = "docker version is %s. Minimum version required is %s to %s."
-                        msg = msg % (docker_version, data['docker_py_version'], usg)
-                    else:
-                        # should not happen
-                        msg = 'Cannot %s with your configuration.' % (usg, )
-                    self.fail(msg)
-
-        stop_timeout_supported = self.docker_api_version >= LooseVersion('1.25')
-        stop_timeout_needed_for_update = self.module.params.get("stop_timeout") is not None and self.module.params.get('state') != 'absent'
-        if stop_timeout_supported:
-            stop_timeout_supported = self.docker_py_version >= LooseVersion('2.1')
-            if stop_timeout_needed_for_update and not stop_timeout_supported:
-                # We warn (instead of fail) since in older versions, stop_timeout was not used
-                # to update the container's configuration, but only when stopping a container.
-                self.module.warn("docker or docker-py version is %s. Minimum version required is 2.1 to update "
-                                 "the container's stop_timeout configuration. "
-                                 "If you use the 'docker-py' module, you have to switch to the docker 'Python' package." % (docker_version,))
-        else:
-            if stop_timeout_needed_for_update and not stop_timeout_supported:
-                # We warn (instead of fail) since in older versions, stop_timeout was not used
-                # to update the container's configuration, but only when stopping a container.
-                self.module.warn("docker API version is %s. Minimum version required is 1.25 to set or "
-                                 "update the container's stop_timeout configuration." % (self.docker_api_version_str,))
-        self.option_minimal_versions['stop_timeout']['supported'] = stop_timeout_supported
-
-    def __init__(self, **kwargs):
-        super(AnsibleDockerClientContainer, self).__init__(**kwargs)
-        self._get_minimal_versions()
         self._parse_comparisons()
 
 

lib/ansible/modules/cloud/docker/docker_network.py

@@ -255,6 +255,7 @@ from ansible.module_utils.docker_common import (
     DockerBaseClass,
     docker_version,
     DifferenceTracker,
+    clean_dict_booleans_for_docker_api,
 )
 
 try:
@@ -315,77 +316,8 @@ def get_ip_version(cidr):
     raise ValueError('"{0}" is not a valid CIDR'.format(cidr))
 
 
-def get_driver_options(driver_options):
-    # TODO: Move this and the same from docker_prune.py to docker_common.py
-    result = dict()
-    if driver_options is not None:
-        for k, v in driver_options.items():
-            # Go doesn't like 'True' or 'False'
-            if v is True:
-                v = 'true'
-            elif v is False:
-                v = 'false'
-            else:
-                v = str(v)
-            result[str(k)] = v
-    return result
-
-
 class DockerNetworkManager(object):
 
-    def _get_minimal_versions(self):
-        # TODO: Move this and the same from docker_container.py to docker_common.py
-        self.option_minimal_versions = dict()
-        for option, data in self.client.module.argument_spec.items():
-            self.option_minimal_versions[option] = dict()
-        self.option_minimal_versions.update(dict(
-            scope=dict(docker_py_version='2.6.0', docker_api_version='1.30'),
-            attachable=dict(docker_py_version='2.0.0', docker_api_version='1.26'),
-        ))
-
-        for option, data in self.option_minimal_versions.items():
-            # Test whether option is supported, and store result
-            support_docker_py = True
-            support_docker_api = True
-            if 'docker_py_version' in data:
-                support_docker_py = self.client.docker_py_version >= LooseVersion(data['docker_py_version'])
-            if 'docker_api_version' in data:
-                support_docker_api = self.client.docker_api_version >= LooseVersion(data['docker_api_version'])
-            data['supported'] = support_docker_py and support_docker_api
-            # Fail if option is not supported but used
-            if not data['supported']:
-                # Test whether option is specified
-                if 'detect_usage' in data:
-                    used = data['detect_usage']()
-                else:
-                    used = self.client.module.params.get(option) is not None
-                    if used and 'default' in self.client.module.argument_spec[option]:
-                        used = self.client.module.params[option] != self.client.module.argument_spec[option]['default']
-                if used:
-                    # If the option is used, compose error message.
-                    if 'usage_msg' in data:
-                        usg = data['usage_msg']
-                    else:
-                        usg = 'set %s option' % (option, )
-                    if not support_docker_api:
-                        msg = 'docker API version is %s. Minimum version required is %s to %s.'
-                        msg = msg % (self.client.docker_api_version_str, data['docker_api_version'], usg)
-                    elif not support_docker_py:
-                        if LooseVersion(data['docker_py_version']) < LooseVersion('2.0.0'):
-                            msg = ("docker-py version is %s. Minimum version required is %s to %s. "
-                                   "Consider switching to the 'docker' package if you do not require Python 2.6 support.")
-                        elif self.client.docker_py_version < LooseVersion('2.0.0'):
-                            msg = ("docker-py version is %s. Minimum version required is %s to %s. "
-                                   "You have to switch to the Python 'docker' package. First uninstall 'docker-py' before "
-                                   "installing 'docker' to avoid a broken installation.")
-                        else:
-                            msg = "docker version is %s. Minimum version required is %s to %s."
-                        msg = msg % (docker_version, data['docker_py_version'], usg)
-                    else:
-                        # should not happen
-                        msg = 'Cannot %s with your configuration.' % (usg, )
-                    self.client.fail(msg)
-
     def __init__(self, client):
         self.client = client
         self.parameters = TaskParameters(client)
@@ -398,8 +330,6 @@ class DockerNetworkManager(object):
         self.diff_tracker = DifferenceTracker()
         self.diff_result = dict()
 
-        self._get_minimal_versions()
-
         self.existing_network = self.get_existing_network()
 
         if not self.parameters.connected and self.existing_network:
@@ -410,7 +340,7 @@ class DockerNetworkManager(object):
             self.parameters.ipam_config = [self.parameters.ipam_options]
 
         if self.parameters.driver_options:
-            self.parameters.driver_options = get_driver_options(self.parameters.driver_options)
+            self.parameters.driver_options = clean_dict_booleans_for_docker_api(self.parameters.driver_options)
 
         state = self.parameters.state
         if state == 'present':
@@ -665,13 +595,19 @@ def main():
         ('ipam_config', 'ipam_options')
     ]
 
+    option_minimal_versions = dict(
+        scope=dict(docker_py_version='2.6.0', docker_api_version='1.30'),
+        attachable=dict(docker_py_version='2.0.0', docker_api_version='1.26'),
+    )
+
     client = AnsibleDockerClient(
         argument_spec=argument_spec,
         mutually_exclusive=mutually_exclusive,
         supports_check_mode=True,
         min_docker_version='1.10.0',
-        min_docker_api_version='1.22'
+        min_docker_api_version='1.22',
         # "The docker server >= 1.10.0"
+        option_minimal_versions=option_minimal_versions,
     )
 
     cm = DockerNetworkManager(client)

lib/ansible/modules/cloud/docker/docker_prune.py

@@ -176,28 +176,12 @@ from distutils.version import LooseVersion
 from ansible.module_utils.docker_common import AnsibleDockerClient
 
 try:
-    from ansible.module_utils.docker_common import docker_version
+    from ansible.module_utils.docker_common import docker_version, clean_dict_booleans_for_docker_api
 except Exception as dummy:
     # missing docker-py handled in ansible.module_utils.docker
     pass
 
 
-def get_filters(module, name):
-    result = dict()
-    filters = module.params.get(name)
-    if filters is not None:
-        for k, v in filters.items():
-            # Go doesn't like 'True' or 'False'
-            if v is True:
-                v = 'true'
-            elif v is False:
-                v = 'false'
-            else:
-                v = str(v)
-            result[str(k)] = v
-    return result
-
-
 def main():
     argument_spec = dict(
         containers=dict(type='bool', default=False),
@@ -227,24 +211,24 @@ def main():
     result = dict()
 
     if client.module.params['containers']:
-        filters = get_filters(client.module, 'containers_filters')
+        filters = clean_dict_booleans_for_docker_api(client.module.params.get('containers_filters'))
         res = client.prune_containers(filters=filters)
         result['containers'] = res.get('ContainersDeleted') or []
         result['containers_space_reclaimed'] = res['SpaceReclaimed']
 
     if client.module.params['images']:
-        filters = get_filters(client.module, 'images_filters')
+        filters = clean_dict_booleans_for_docker_api(client.module.params.get('images_filters'))
         res = client.prune_images(filters=filters)
         result['images'] = res.get('ImagesDeleted') or []
         result['images_space_reclaimed'] = res['SpaceReclaimed']
 
     if client.module.params['networks']:
-        filters = get_filters(client.module, 'networks_filters')
+        filters = clean_dict_booleans_for_docker_api(client.module.params.get('networks_filters'))
         res = client.prune_networks(filters=filters)
         result['networks'] = res.get('NetworksDeleted') or []
 
     if client.module.params['volumes']:
-        filters = get_filters(client.module, 'volumes_filters')
+        filters = clean_dict_booleans_for_docker_api(client.module.params.get('volumes_filters'))
         res = client.prune_volumes(filters=filters)
         result['volumes'] = res.get('VolumesDeleted') or []
         result['volumes_space_reclaimed'] = res['SpaceReclaimed']

lib/ansible/modules/cloud/docker/docker_swarm.py

@@ -288,60 +288,6 @@ class TaskParameters(DockerBaseClass):
 
 class SwarmManager(DockerBaseClass):
 
-    def _get_minimal_versions(self):
-        # TODO: Move this and the same from docker_container.py to docker_common.py
-        self.option_minimal_versions = dict()
-        for option, data in self.client.module.argument_spec.items():
-            self.option_minimal_versions[option] = dict()
-        self.option_minimal_versions.update(dict(
-            signing_ca_cert=dict(docker_api_version='1.30'),
-            signing_ca_key=dict(docker_api_version='1.30'),
-            ca_force_rotate=dict(docker_api_version='1.30'),
-        ))
-
-        for option, data in self.option_minimal_versions.items():
-            # Test whether option is supported, and store result
-            support_docker_py = True
-            support_docker_api = True
-            if 'docker_py_version' in data:
-                support_docker_py = self.client.docker_py_version >= LooseVersion(data['docker_py_version'])
-            if 'docker_api_version' in data:
-                support_docker_api = self.client.docker_api_version >= LooseVersion(data['docker_api_version'])
-            data['supported'] = support_docker_py and support_docker_api
-            # Fail if option is not supported but used
-            if not data['supported']:
-                # Test whether option is specified
-                if 'detect_usage' in data:
-                    used = data['detect_usage']()
-                else:
-                    used = self.client.module.params.get(option) is not None
-                    if used and 'default' in self.client.module.argument_spec[option]:
-                        used = self.client.module.params[option] != self.client.module.argument_spec[option]['default']
-                if used:
-                    # If the option is used, compose error message.
-                    if 'usage_msg' in data:
-                        usg = data['usage_msg']
-                    else:
-                        usg = 'set %s option' % (option, )
-                    if not support_docker_api:
-                        msg = 'docker API version is %s. Minimum version required is %s to %s.'
-                        msg = msg % (self.client.docker_api_version_str, data['docker_api_version'], usg)
-                    elif not support_docker_py:
-                        if LooseVersion(data['docker_py_version']) < LooseVersion('2.0.0'):
-                            msg = ("docker-py version is %s. Minimum version required is %s to %s. "
-                                   "Consider switching to the 'docker' package if you do not require Python 2.6 support.")
-                        elif self.client.docker_py_version < LooseVersion('2.0.0'):
-                            msg = ("docker-py version is %s. Minimum version required is %s to %s. "
-                                   "You have to switch to the Python 'docker' package. First uninstall 'docker-py' before "
-                                   "installing 'docker' to avoid a broken installation.")
-                        else:
-                            msg = "docker version is %s. Minimum version required is %s to %s."
-                        msg = msg % (docker_version, data['docker_py_version'], usg)
-                    else:
-                        # should not happen
-                        msg = 'Cannot %s with your configuration.' % (usg, )
-                    self.client.fail(msg)
-
     def __init__(self, client, results):
 
         super(SwarmManager, self).__init__()
@@ -350,8 +296,6 @@ class SwarmManager(DockerBaseClass):
         self.results = results
         self.check_mode = self.client.check_mode
 
-        self._get_minimal_versions()
-
         self.parameters = TaskParameters(client)
 
     def __call__(self):
@@ -562,12 +506,19 @@ def main():
         ('state', 'remove', ['node_id'])
     ]
 
+    option_minimal_versions = dict(
+        signing_ca_cert=dict(docker_api_version='1.30'),
+        signing_ca_key=dict(docker_api_version='1.30'),
+        ca_force_rotate=dict(docker_api_version='1.30'),
+    )
+
     client = AnsibleDockerClient(
         argument_spec=argument_spec,
         supports_check_mode=True,
         required_if=required_if,
         min_docker_version='2.6.0',
         min_docker_api_version='1.25',
+        option_minimal_versions=option_minimal_versions,
     )
 
     results = dict(