ec2_vpc_igw: migrate to boto3 (#45346)

* ec2_vpc_igw: draft migrate to boto3

* ec2_vpc_igw: migrate to boto3

* ec2_vpc_igw: fix error and implement code review

* ec2_vpc_igw: always catch BotoCoreError with ClientError, remove unused fail method

* ec2_vpc_igw: fix pep error

lib/ansible/modules/cloud/amazon/ec2_vpc_igw.py

@@ -37,6 +37,9 @@ options:
 extends_documentation_fragment:
     - aws
     - ec2
+requirements:
+  - botocore
+  - boto3
 '''
 
 EXAMPLES = '''
@@ -80,128 +83,175 @@ vpc_id:
 '''
 
 try:
-    import boto.ec2
+    import botocore
-    import boto.vpc
-    from boto.exception import EC2ResponseError
-    HAS_BOTO = True
 except ImportError:
-    HAS_BOTO = False
+    pass  # Handled by AnsibleAWSModule
 
-from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.aws.core import AnsibleAWSModule
-from ansible.module_utils.ec2 import AnsibleAWSError, connect_to_aws, ec2_argument_spec, get_aws_connection_info
+from ansible.module_utils.ec2 import (
+    AWSRetry,
+    boto3_conn,
+    ec2_argument_spec,
+    get_aws_connection_info,
+    camel_dict_to_snake_dict,
+    boto3_tag_list_to_ansible_dict,
+    ansible_dict_to_boto3_filter_list,
+    ansible_dict_to_boto3_tag_list,
+    compare_aws_tags
+)
 from ansible.module_utils.six import string_types
 
 
-class AnsibleIGWException(Exception):
+class AnsibleEc2Igw(object):
-    pass
 
+    def __init__(self, module, results):
+        self._module = module
+        self._results = results
+        self._connection = self._module.client('ec2')
+        self._check_mode = self._module.check_mode
 
-def get_igw_info(igw):
+    def process(self):
-    return {'gateway_id': igw.id,
+        vpc_id = self._module.params.get('vpc_id')
-            'tags': igw.tags,
+        state = self._module.params.get('state', 'present')
-            'vpc_id': igw.vpc_id
+        tags = self._module.params.get('tags')
-            }
 
+        if state == 'present':
+            self.ensure_igw_present(vpc_id, tags)
+        elif state == 'absent':
+            self.ensure_igw_absent(vpc_id)
 
-def get_resource_tags(vpc_conn, resource_id):
+    def get_matching_igw(self, vpc_id):
-    return dict((t.name, t.value) for t in
+        filters = ansible_dict_to_boto3_filter_list({'attachment.vpc-id': vpc_id})
-                vpc_conn.get_all_tags(filters={'resource-id': resource_id}))
+        igws = []
-
-
-def ensure_tags(vpc_conn, resource_id, tags, add_only, check_mode):
         try:
-        cur_tags = get_resource_tags(vpc_conn, resource_id)
+            response = self._connection.describe_internet_gateways(Filters=filters)
-        if cur_tags == tags:
+            igws = response.get('InternetGateways', [])
-            return {'changed': False, 'tags': cur_tags}
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+            self._module.fail_json_aws(e)
 
-        if check_mode:
+        igw = None
-            latest_check_mode_tags = cur_tags
-
-        to_delete = dict((k, cur_tags[k]) for k in cur_tags if k not in tags)
-        if to_delete and not add_only:
-            if check_mode:
-                # just overwriting latest_check_mode_tags instead of deleting keys
-                latest_check_mode_tags = dict((k, cur_tags[k]) for k in cur_tags if k not in to_delete)
-            else:
-                vpc_conn.delete_tags(resource_id, to_delete)
-
-        to_add = dict((k, tags[k]) for k in tags if k not in cur_tags or cur_tags[k] != tags[k])
-        if to_add:
-            if check_mode:
-                latest_check_mode_tags.update(to_add)
-            else:
-                vpc_conn.create_tags(resource_id, to_add)
-
-        if check_mode:
-            return {'changed': True, 'tags': latest_check_mode_tags}
-        latest_tags = get_resource_tags(vpc_conn, resource_id)
-        return {'changed': True, 'tags': latest_tags}
-    except EC2ResponseError as e:
-        raise AnsibleIGWException(
-            'Unable to update tags for {0}, error: {1}'.format(resource_id, e))
-
-
-def get_matching_igw(vpc_conn, vpc_id):
-    igws = vpc_conn.get_all_internet_gateways(filters={'attachment.vpc-id': vpc_id})
         if len(igws) > 1:
-        raise AnsibleIGWException(
+            self._module.fail_json(
-            'EC2 returned more than one Internet Gateway for VPC {0}, aborting'
+                msg='EC2 returned more than one Internet Gateway for VPC {0}, aborting'.format(vpc_id))
-            .format(vpc_id))
+        elif igws:
-    return igws[0] if igws else None
+            igw = camel_dict_to_snake_dict(igws[0])
 
+        return igw
 
-def ensure_igw_absent(vpc_conn, vpc_id, check_mode):
+    def check_input_tags(self, tags):
-    igw = get_matching_igw(vpc_conn, vpc_id)
+        nonstring_tags = [k for k, v in tags.items() if not isinstance(v, string_types)]
-    if igw is None:
+        if nonstring_tags:
-        return {'changed': False}
+            self._module.fail_json(msg='One or more tags contain non-string values: {0}'.format(nonstring_tags))
 
-    if check_mode:
+    def ensure_tags(self, igw_id, tags, add_only):
-        return {'changed': True}
+        final_tags = []
 
+        filters = ansible_dict_to_boto3_filter_list({'resource-id': igw_id, 'resource-type': 'internet-gateway'})
+        cur_tags = None
         try:
-        vpc_conn.detach_internet_gateway(igw.id, vpc_id)
+            cur_tags = self._connection.describe_tags(Filters=filters)
-        vpc_conn.delete_internet_gateway(igw.id)
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-    except EC2ResponseError as e:
+            self._module.fail_json_aws(e, msg="Couldn't describe tags")
-        raise AnsibleIGWException(
-            'Unable to delete Internet Gateway, error: {0}'.format(e))
 
-    return {'changed': True}
+        purge_tags = bool(not add_only)
-
+        to_update, to_delete = compare_aws_tags(boto3_tag_list_to_ansible_dict(cur_tags.get('Tags')), tags, purge_tags)
-
+        final_tags = boto3_tag_list_to_ansible_dict(cur_tags.get('Tags'))
-def ensure_igw_present(vpc_conn, vpc_id, tags, check_mode):
-    igw = get_matching_igw(vpc_conn, vpc_id)
-    changed = False
-    if igw is None:
-        if check_mode:
-            return {'changed': True, 'gateway_id': None}
 
+        if to_update:
             try:
-            igw = vpc_conn.create_internet_gateway()
+                if self._check_mode:
-            vpc_conn.attach_internet_gateway(igw.id, vpc_id)
+                    # update tags
-            changed = True
+                    final_tags.update(to_update)
-        except EC2ResponseError as e:
+                else:
-            raise AnsibleIGWException(
+                    AWSRetry.exponential_backoff()(self._connection.create_tags)(
-                'Unable to create Internet Gateway, error: {0}'.format(e))
+                        Resources=[igw_id],
+                        Tags=ansible_dict_to_boto3_tag_list(to_update)
+                    )
 
-    igw.vpc_id = vpc_id
+                self._results['changed'] = True
+            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+                self._module.fail_json_aws(e, msg="Couldn't create tags")
 
-    if tags != igw.tags:
+        if to_delete:
-        if check_mode:
+            try:
-            check_mode_tags = ensure_tags(vpc_conn, igw.id, tags, False, check_mode)
+                if self._check_mode:
-            igw_info = get_igw_info(igw)
+                    # update tags
-            igw_info.get('tags', {}).update(check_mode_tags.get('tags', {}))
+                    for key in to_delete:
-            return {'changed': True, 'gateway': igw_info}
+                        del final_tags[key]
-        ensure_tags(vpc_conn, igw.id, tags, False, check_mode)
+                else:
-        igw.tags = tags
+                    tags_list = []
-        changed = True
+                    for key in to_delete:
+                        tags_list.append({'Key': key})
 
-    igw_info = get_igw_info(igw)
+                    AWSRetry.exponential_backoff()(self._connection.delete_tags)(Resources=[igw_id], Tags=tags_list)
 
+                self._results['changed'] = True
+            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+                self._module.fail_json_aws(e, msg="Couldn't delete tags")
+
+        if not self._check_mode and (to_update or to_delete):
+            try:
+                response = self._connection.describe_tags(Filters=filters)
+                final_tags = boto3_tag_list_to_ansible_dict(response.get('Tags'))
+            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+                self._module.fail_json_aws(e, msg="Couldn't describe tags")
+
+        return final_tags
+
+    @staticmethod
+    def get_igw_info(igw):
         return {
-        'changed': changed,
+            'gateway_id': igw['internet_gateway_id'],
-        'gateway': igw_info
+            'tags': igw['tags'],
+            'vpc_id': igw['vpc_id']
         }
 
+    def ensure_igw_absent(self, vpc_id):
+        igw = self.get_matching_igw(vpc_id)
+        if igw is None:
+            return self._results
+
+        if self._check_mode:
+            self._results['changed'] = True
+            return self._results
+
+        try:
+            self._results['changed'] = True
+            self._connection.detach_internet_gateway(InternetGatewayId=igw['internet_gateway_id'], VpcId=vpc_id)
+            self._connection.delete_internet_gateway(InternetGatewayId=igw['internet_gateway_id'])
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+            self._module.fail_json_aws(e, msg="Unable to delete Internet Gateway")
+
+        return self._results
+
+    def ensure_igw_present(self, vpc_id, tags):
+        self.check_input_tags(tags)
+
+        igw = self.get_matching_igw(vpc_id)
+
+        if igw is None:
+            if self._check_mode:
+                self._results['changed'] = True
+                self._results['gateway_id'] = None
+                return self._results
+
+            try:
+                response = self._connection.create_internet_gateway()
+                igw = camel_dict_to_snake_dict(response['InternetGateway'])
+                self._connection.attach_internet_gateway(InternetGatewayId=igw['internet_gateway_id'], VpcId=vpc_id)
+                self._results['changed'] = True
+            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+                self._module.fail_json_aws(e, msg='Unable to create Internet Gateway')
+
+        igw['vpc_id'] = vpc_id
+
+        igw['tags'] = self.ensure_tags(igw_id=igw['internet_gateway_id'], tags=tags, add_only=False)
+
+        igw_info = self.get_igw_info(igw)
+        self._results.update(igw_info)
+
+        return self._results
+
 
 def main():
     argument_spec = ec2_argument_spec()
@@ -213,41 +263,17 @@ def main():
         )
     )
 
-    module = AnsibleModule(
+    module = AnsibleAWSModule(
         argument_spec=argument_spec,
         supports_check_mode=True,
     )
+    results = dict(
+        changed=False
+    )
+    igw_manager = AnsibleEc2Igw(module=module, results=results)
+    igw_manager.process()
 
-    if not HAS_BOTO:
+    module.exit_json(**results)
-        module.fail_json(msg='boto is required for this module')
-
-    region, ec2_url, aws_connect_params = get_aws_connection_info(module)
-
-    if region:
-        try:
-            connection = connect_to_aws(boto.vpc, region, **aws_connect_params)
-        except (boto.exception.NoAuthHandlerFound, AnsibleAWSError) as e:
-            module.fail_json(msg=str(e))
-    else:
-        module.fail_json(msg="region must be specified")
-
-    vpc_id = module.params.get('vpc_id')
-    state = module.params.get('state', 'present')
-    tags = module.params.get('tags')
-
-    nonstring_tags = [k for k, v in tags.items() if not isinstance(v, string_types)]
-    if nonstring_tags:
-        module.fail_json(msg='One or more tags contain non-string values: {0}'.format(nonstring_tags))
-
-    try:
-        if state == 'present':
-            result = ensure_igw_present(connection, vpc_id, tags, check_mode=module.check_mode)
-        elif state == 'absent':
-            result = ensure_igw_absent(connection, vpc_id, check_mode=module.check_mode)
-    except AnsibleIGWException as e:
-        module.fail_json(msg=str(e))
-
-    module.exit_json(changed=result['changed'], **result.get('gateway', {}))
 
 
 if __name__ == '__main__':