Add AWS boto3 error code exception function is_boto3_error_code (#41202)

* Add aws/core.py function to check for specific AWS error codes

* Use sys.exc_info to get exception object if it isn't passed in

* Allow catching exceptions with is_boto3_error_code

* Replace from_code with is_boto3_error_code

* Return a type that will never be raised to support stricter type comparisons in Python 3+

* Use is_boto3_error_code in aws_eks_cluster

* Add duplicate-except to ignores when using is_boto3_error_code

* Add is_boto3_error_code to module development guideline docs

changelogs/fragments/aws_core_is_boto3_error_code.yml

@@ -0,0 +1,4 @@
+---
+minor_changes:
+  - Add `is_boto3_error_code` function to `module_utils/aws/core.py` to make it
+    easier for modules to handle special AWS error codes.

lib/ansible/module_utils/aws/core.py

@@ -254,3 +254,25 @@ class _RetryingBotoClientWrapper(object):
             return wrapped
         else:
             return unwrapped
+
+
+def is_boto3_error_code(code, e=None):
+    """Check if the botocore exception is raised by a specific error code.
+
+    Returns ClientError if the error code matches, a dummy exception if it does not have an error code or does not match
+
+    Example:
+    try:
+        ec2.describe_instances(InstanceIds=['potato'])
+    except is_boto3_error_code('InvalidInstanceID.Malformed'):
+        # handle the error for that code case
+    except botocore.exceptions.ClientError as e:
+        # handle the generic error case for all other codes
+    """
+    from botocore.exceptions import ClientError
+    if e is None:
+        import sys
+        dummy, e, dummy = sys.exc_info()
+    if isinstance(e, ClientError) and e.response['Error']['Code'] == code:
+        return ClientError
+    return type('NeverEverRaisedException', (Exception,), {})

lib/ansible/modules/cloud/amazon/GUIDELINES.md

@@ -207,14 +207,30 @@ extends_documentation_fragment:
 You should wrap any boto3 or botocore call in a try block. If an exception is thrown, then there
 are a number of possibilities for handling it.
 
-* use aws_module.fail_json_aws() to report the module failure in a standard way
+* Catch the general `ClientError` or look for a specific error code with
-* retry using AWSRetry
+    `is_boto3_error_code`.
-* use fail_json() to report the failure without using `ansible.module_utils.aws.core`
+* Use aws_module.fail_json_aws() to report the module failure in a standard way
-* do something custom in the case where you know how to handle the exception
+* Retry using AWSRetry
+* Use fail_json() to report the failure without using `ansible.module_utils.aws.core`
+* Do something custom in the case where you know how to handle the exception
 
 For more information on botocore exception handling see [the botocore error documentation](http://botocore.readthedocs.org/en/latest/client_upgrades.html#error-handling).
 
-#### using fail_json_aws()
+### Using is_boto3_error_code
+
+To use `ansible.module_utils.aws.core.is_boto3_error_code` to catch a single
+AWS error code, call it in place of `ClientError` in your except clauses. In
+this case, *only* the `InvalidGroup.NotFound` error code will be caught here,
+and any other error will be raised for handling elsewhere in the program.
+
+```python
+try:
+    return connection.describe_security_groups(**kwargs)
+except is_boto3_error_code('InvalidGroup.NotFound'):
+    return {'SecurityGroups': []}
+```
+
+#### Using fail_json_aws()
 
 In the AnsibleAWSModule there is a special method, `module.fail_json_aws()` for nice reporting of
 exceptions.  Call this on your exception and it will report the error together with a traceback for

lib/ansible/modules/cloud/amazon/aws_config_aggregator.py

@@ -85,7 +85,7 @@ try:
 except ImportError:
     pass  # handled by AnsibleAWSModule
 
-from ansible.module_utils.aws.core import AnsibleAWSModule
+from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code
 from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, AWSRetry
 from ansible.module_utils.ec2 import camel_dict_to_snake_dict, boto3_tag_list_to_ansible_dict
 
@@ -96,9 +96,9 @@ def resource_exists(client, module, resource_type, params):
             ConfigurationAggregatorNames=[params['name']]
         )
         return aggregator['ConfigurationAggregators'][0]
-    except client.exceptions.from_code('NoSuchConfigurationAggregatorException'):
+    except is_boto3_error_code('NoSuchConfigurationAggregatorException'):
         return
-    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e)
 
 

lib/ansible/modules/cloud/amazon/aws_config_delivery_channel.py

@@ -69,7 +69,7 @@ try:
 except ImportError:
     pass  # handled by AnsibleAWSModule
 
-from ansible.module_utils.aws.core import AnsibleAWSModule
+from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code
 from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, AWSRetry
 from ansible.module_utils.ec2 import camel_dict_to_snake_dict, boto3_tag_list_to_ansible_dict
 
@@ -88,9 +88,9 @@ def resource_exists(client, module, params):
             aws_retry=True,
         )
         return channel['DeliveryChannels'][0]
-    except client.exceptions.from_code('NoSuchDeliveryChannelException'):
+    except is_boto3_error_code('NoSuchDeliveryChannelException'):
         return
-    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e)
 
 
@@ -104,12 +104,12 @@ def create_resource(client, module, params, result):
         result['changed'] = True
         result['channel'] = camel_dict_to_snake_dict(resource_exists(client, module, params))
         return result
-    except client.exceptions.from_code('InvalidS3KeyPrefixException') as e:
+    except is_boto3_error_code('InvalidS3KeyPrefixException') as e:
         module.fail_json_aws(e, msg="The `s3_prefix` parameter was invalid. Try '/' for no prefix")
-    except client.exceptions.from_code('InsufficientDeliveryPolicyException') as e:
+    except is_boto3_error_code('InsufficientDeliveryPolicyException') as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e, msg="The `s3_prefix` or `s3_bucket` parameter is invalid. "
                              "Make sure the bucket exists and is available")
-    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e, msg="Couldn't create AWS Config delivery channel")
 
 
@@ -129,12 +129,12 @@ def update_resource(client, module, params, result):
             result['changed'] = True
             result['channel'] = camel_dict_to_snake_dict(resource_exists(client, module, params))
             return result
-        except client.exceptions.from_code('InvalidS3KeyPrefixException') as e:
+        except is_boto3_error_code('InvalidS3KeyPrefixException') as e:
             module.fail_json_aws(e, msg="The `s3_prefix` parameter was invalid. Try '/' for no prefix")
-        except client.exceptions.from_code('InsufficientDeliveryPolicyException') as e:
+        except is_boto3_error_code('InsufficientDeliveryPolicyException') as e:  # pylint: disable=duplicate-except
             module.fail_json_aws(e, msg="The `s3_prefix` or `s3_bucket` parameter is invalid. "
                                  "Make sure the bucket exists and is available")
-        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
             module.fail_json_aws(e, msg="Couldn't create AWS Config delivery channel")
 
 

lib/ansible/modules/cloud/amazon/aws_config_recorder.py

@@ -86,7 +86,7 @@ try:
 except ImportError:
     pass  # handled by AnsibleAWSModule
 
-from ansible.module_utils.aws.core import AnsibleAWSModule
+from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code
 from ansible.module_utils.ec2 import boto3_conn, get_aws_connection_info, AWSRetry
 from ansible.module_utils.ec2 import camel_dict_to_snake_dict, boto3_tag_list_to_ansible_dict
 
@@ -97,9 +97,9 @@ def resource_exists(client, module, params):
             ConfigurationRecorderNames=[params['name']]
         )
         return recorder['ConfigurationRecorders'][0]
-    except client.exceptions.from_code('NoSuchConfigurationRecorderException'):
+    except is_boto3_error_code('NoSuchConfigurationRecorderException'):
         return
-    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e)
 
 

lib/ansible/modules/cloud/amazon/aws_config_rule.py

@@ -110,7 +110,7 @@ try:
 except ImportError:
     pass  # handled by AnsibleAWSModule
 
-from ansible.module_utils.aws.core import AnsibleAWSModule
+from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code
 from ansible.module_utils.ec2 import AWSRetry, camel_dict_to_snake_dict
 
 
@@ -121,9 +121,9 @@ def rule_exists(client, module, params):
             aws_retry=True,
         )
         return rule['ConfigRules'][0]
-    except client.exceptions.from_code('NoSuchConfigRuleException'):
+    except is_boto3_error_code('NoSuchConfigRuleException'):
         return
-    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e)
 
 

lib/ansible/modules/cloud/amazon/aws_eks_cluster.py

@@ -129,7 +129,7 @@ version:
 '''
 
 
-from ansible.module_utils.aws.core import AnsibleAWSModule
+from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code
 from ansible.module_utils.ec2 import camel_dict_to_snake_dict, get_ec2_security_group_ids_from_names
 
 try:
@@ -197,11 +197,11 @@ def get_cluster(client, module):
     name = module.params.get('name')
     try:
         return client.describe_cluster(name=name)['cluster']
-    except client.exceptions.from_code('ResourceNotFoundException'):
+    except is_boto3_error_code('ResourceNotFoundException'):
         return None
-    except botocore.exceptions.EndpointConnectionError as e:
+    except botocore.exceptions.EndpointConnectionError as e:  # pylint: disable=duplicate-except
         module.fail_json(msg="Region %s is not supported by EKS" % client.meta.region_name)
-    except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+    except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:  # pylint: disable=duplicate-except
         module.fail_json(e, msg="Couldn't get cluster %s" % name)
 
 

lib/ansible/modules/cloud/amazon/ec2_group.py

@@ -292,7 +292,7 @@ import json
 import re
 from time import sleep
 from collections import namedtuple
-from ansible.module_utils.aws.core import AnsibleAWSModule
+from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code
 from ansible.module_utils.aws.iam import get_aws_account_id
 from ansible.module_utils.aws.waiters import get_waiter
 from ansible.module_utils.ec2 import AWSRetry, camel_dict_to_snake_dict, compare_aws_tags
@@ -430,7 +430,7 @@ def get_security_groups_with_backoff(connection, **kwargs):
 def sg_exists_with_backoff(connection, **kwargs):
     try:
         return connection.describe_security_groups(**kwargs)
-    except connection.exceptions.from_code('InvalidGroup.NotFound') as e:
+    except is_boto3_error_code('InvalidGroup.NotFound'):
         return {'SecurityGroups': []}
 
 
@@ -519,10 +519,10 @@ def get_target_from_rule(module, client, rule, name, group, groups, vpc_id):
                 # retry describing the group once
                 try:
                     auto_group = get_security_groups_with_backoff(client, Filters=ansible_dict_to_boto3_filter_list(filters)).get('SecurityGroups', [])[0]
-                except (client.exceptions.from_code('InvalidGroup.NotFound'), IndexError) as e:
+                except (is_boto3_error_code('InvalidGroup.NotFound'), IndexError):
                     module.fail_json(msg="group %s will be automatically created by rule %s but "
                                          "no description was provided" % (group_name, rule))
-                except ClientError as e:
+                except ClientError as e:  # pylint: disable=duplicate-except
                     module.fail_json_aws(e)
             elif not module.check_mode:
                 params = dict(GroupName=group_name, Description=rule['group_desc'])
@@ -535,7 +535,7 @@ def get_target_from_rule(module, client, rule, name, group, groups, vpc_id):
                     ).wait(
                         GroupIds=[auto_group['GroupId']],
                     )
-                except client.exceptions.from_code('InvalidGroup.Duplicate') as e:
+                except is_boto3_error_code('InvalidGroup.Duplicate'):
                     # The group exists, but didn't show up in any of our describe-security-groups calls
                     # Try searching on a filter for the name, and allow a retry window for AWS to update
                     # the model on their end.
@@ -829,7 +829,7 @@ def group_exists(client, module, vpc_id, group_id, name):
     try:
         security_groups = sg_exists_with_backoff(client, **params).get('SecurityGroups', [])
         all_groups = get_security_groups_with_backoff(client).get('SecurityGroups', [])
-    except (BotoCoreError, ClientError) as e:
+    except (BotoCoreError, ClientError) as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e, msg="Error in describe_security_groups")
 
     if security_groups:

lib/ansible/modules/cloud/amazon/ec2_vpc_vgw.py

@@ -116,6 +116,7 @@ try:
 except ImportError:
     HAS_BOTO3 = False
 
+from ansible.module_utils.aws.core import is_boto3_error_code
 from ansible.module_utils.aws.waiters import get_waiter
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.ec2 import HAS_BOTO3, boto3_conn, ec2_argument_spec, get_aws_connection_info, AWSRetry
@@ -220,9 +221,9 @@ def create_vgw(client, module):
     except botocore.exceptions.WaiterError as e:
         module.fail_json(msg="Failed to wait for Vpn Gateway {0} to be available".format(response['VpnGateway']['VpnGatewayId']),
                          exception=traceback.format_exc())
-    except client.exceptions.from_code('VpnGatewayLimitExceeded') as e:
+    except is_boto3_error_code('VpnGatewayLimitExceeded'):
         module.fail_json(msg="Too many VPN gateways exist in this account.", exception=traceback.format_exc())
-    except botocore.exceptions.ClientError as e:
+    except botocore.exceptions.ClientError as e:  # pylint: disable=duplicate-except
         module.fail_json(msg=to_native(e), exception=traceback.format_exc())
 
     result = response

lib/ansible/modules/cloud/amazon/rds_instance_facts.py

@@ -340,7 +340,7 @@ instances:
           sample: sg-abcd1234
 '''
 
-from ansible.module_utils.aws.core import AnsibleAWSModule
+from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code
 from ansible.module_utils.ec2 import ansible_dict_to_boto3_filter_list, boto3_tag_list_to_ansible_dict, AWSRetry, camel_dict_to_snake_dict
 
 
@@ -363,9 +363,9 @@ def instance_facts(module, conn):
     paginator = conn.get_paginator('describe_db_instances')
     try:
         results = paginator.paginate(**params).build_full_result()['DBInstances']
-    except conn.exceptions.from_code('DBInstanceNotFound'):
+    except is_boto3_error_code('DBInstanceNotFound'):
         results = []
-    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e, "Couldn't get instance information")
 
     for instance in results:

lib/ansible/modules/cloud/amazon/rds_snapshot_facts.py

@@ -284,7 +284,7 @@ cluster_snapshots:
       sample: vpc-abcd1234
 '''
 
-from ansible.module_utils.aws.core import AnsibleAWSModule
+from ansible.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code
 from ansible.module_utils.ec2 import AWSRetry, boto3_tag_list_to_ansible_dict, camel_dict_to_snake_dict
 
 try:
@@ -297,9 +297,9 @@ def common_snapshot_facts(module, conn, method, prefix, params):
     paginator = conn.get_paginator(method)
     try:
         results = paginator.paginate(**params).build_full_result()['%ss' % prefix]
-    except conn.exceptions.from_code('%sNotFound' % prefix):
+    except is_boto3_error_code('%sNotFound' % prefix):
         results = []
-    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e, "trying to get snapshot information")
 
     for snapshot in results: