parse botocore.endpoint logs into a list of AWS actions (#49312)

* Add an option to parse botocore.endpoint logs for the AWS actions performed during a task

Add a callback to consolidate all AWS actions used by modules

Added some documentation to the AWS guidelines

* Enable aws_resource_actions callback only for AWS tests

* Add script to help generate policies

* Set debug_botocore_endpoint_logs via environment variable for all AWS integration tests

Ensure AWS tests inherit environment

(also remove AWS CLI in aws_rds inventory tests and use the module)

lib/ansible/module_utils/aws/core.py

@@ -60,10 +60,18 @@ don't need to be wrapped in the backoff decorator.
 
 """
 
+import re
+import logging
 import traceback
 from functools import wraps
 from distutils.version import LooseVersion
 
+try:
+    from cStringIO import StringIO
+except ImportError:
+    # Python 3
+    from io import StringIO
+
 from ansible.module_utils.basic import AnsibleModule, missing_required_lib
 from ansible.module_utils._text import to_native
 from ansible.module_utils.ec2 import HAS_BOTO3, camel_dict_to_snake_dict, ec2_argument_spec, boto3_conn, get_aws_connection_info
@@ -120,14 +128,38 @@ class AnsibleAWSModule(object):
         self._diff = self._module._diff
         self._name = self._module._name
 
+        self._botocore_endpoint_log_stream = StringIO()
+        self.logger = None
+        if self.params.get('debug_botocore_endpoint_logs'):
+            self.logger = logging.getLogger('botocore.endpoint')
+            self.logger.setLevel(logging.DEBUG)
+            self.logger.addHandler(logging.StreamHandler(self._botocore_endpoint_log_stream))
+
     @property
     def params(self):
         return self._module.params
 
+    def _get_resource_action_list(self):
+        actions = []
+        for ln in self._botocore_endpoint_log_stream.getvalue().split('\n'):
+            ln = ln.strip()
+            if not ln:
+                continue
+            found_operational_request = re.search(r"OperationModel\(name=.*?\)", ln)
+            if found_operational_request:
+                operation_request = found_operational_request.group(0)[20:-1]
+                resource = re.search(r"https://.*?\.", ln).group(0)[8:-1]
+                actions.append("{0}:{1}".format(resource, operation_request))
+        return list(set(actions))
+
     def exit_json(self, *args, **kwargs):
+        if self.params.get('debug_botocore_endpoint_logs'):
+            kwargs['resource_actions'] = self._get_resource_action_list()
         return self._module.exit_json(*args, **kwargs)
 
     def fail_json(self, *args, **kwargs):
+        if self.params.get('debug_botocore_endpoint_logs'):
+            kwargs['resource_actions'] = self._get_resource_action_list()
         return self._module.fail_json(*args, **kwargs)
 
     def debug(self, *args, **kwargs):
@@ -190,7 +222,7 @@ class AnsibleAWSModule(object):
         if response is not None:
             failure.update(**camel_dict_to_snake_dict(response))
 
-        self._module.fail_json(**failure)
+        self.fail_json(**failure)
 
     def _gather_versions(self):
         """Gather AWS SDK (boto3 and botocore) dependency versions

lib/ansible/module_utils/ec2.py

@@ -31,7 +31,7 @@ import re
 import traceback
 
 from ansible.module_utils.ansible_release import __version__
-from ansible.module_utils.basic import missing_required_lib
+from ansible.module_utils.basic import missing_required_lib, env_fallback
 from ansible.module_utils._text import to_native, to_text
 from ansible.module_utils.cloud import CloudRetry
 from ansible.module_utils.six import string_types, binary_type, text_type
@@ -177,6 +177,7 @@ def boto_exception(err):
 
 def aws_common_argument_spec():
     return dict(
+        debug_botocore_endpoint_logs=dict(fallback=(env_fallback, ['ANSIBLE_DEBUG_BOTOCORE_LOGS']), default=False, type='bool'),
         ec2_url=dict(),
         aws_secret_key=dict(aliases=['ec2_secret_key', 'secret_key'], no_log=True),
         aws_access_key=dict(aliases=['ec2_access_key', 'access_key']),

lib/ansible/modules/cloud/amazon/GUIDELINES.md

@@ -633,14 +633,66 @@ for every call, it's preferrable to use [YAML Anchors](http://blog.daemonl.com/2
 
 As explained in the [Integration Test guide](https://docs.ansible.com/ansible/latest/dev_guide/testing_integration.html#iam-policies-for-aws)
 there are defined IAM policies in `hacking/aws_config/testing_policies/` that contain the necessary permissions
-to run the AWS integration test.
+to run the AWS integration test. The permissions used by CI are more restrictive than those in `hacking/aws_config/testing_policies`; for CI we want
+the most restrictive policy possible that still allows the given tests to pass.
 
-If your module is interacting with a new service or otherwise requires new permissions you must update the
-appropriate policy file to grant the permissions needed to run your integration test.
+If your module interacts with a new service or otherwise requires new permissions, tests will fail when you submit a pull request and the
+[Ansibullbot](https://github.com/ansible/ansibullbot/blob/master/ISSUE_HELP.md) will tag your PR as needing revision.
+We do not automatically grant additional permissions to the roles used by the continuous integration builds. You must provide the minimum IAM permissions required to run your integration test.
 
-There is no process for automatically granting additional permissions to the roles used by the continuous
-integration builds, so the tests will initially fail when you submit a pull request and the
-[Ansibullbot](https://github.com/ansible/ansibullbot/blob/master/ISSUE_HELP.md) will tag it as needing revision.
-
-Once you're certain the failure is only due to the missing permissions, add a comment with the `ready_for_review`
+If your PR has test failures, check carefully to be certain the failure is only due to the missing permissions. If you've ruled out other sources of failure, add a comment with the `ready_for_review`
 tag and explain that it's due to missing permissions.
+
+Your pull request cannot be merged until the tests are passing. If your pull request is failing due to missing permissions,
+you must collect the minimum IAM permissions required to
+run the tests.
+
+There are two ways to figure out which IAM permissions you need for your PR to pass:
+
+* Start with the most permissive IAM policy, run the tests to collect information about which resources your tests actually use, then construct a policy based on that output. This approach only works on modules that use `AnsibleAWSModule`.
+* Start with the least permissive IAM policy, run the tests to discover a failure, add permissions for the resource that addresses that failure, then repeat. If your module uses `AnsibleModule` instead of `AnsibleAWSModule`, you must use this approach.
+
+To start with the most permissive IAM policy:
+
+1) [Create an IAM policy](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html#access_policies_create-start) that allows all actions (set `Action` and `Resource` to `*`).
+2) Run your tests locally with this policy. On `AnsibleAWSModule`-based modules, the `debug_botocore_endpoint_logs` option is automatically set to `yes`, so you
+should see a list of `AWS ACTIONS` after the `PLAY RECAP` showing all the permissions used. If your tests use a `boto`/`AnsibleModule` module, you must start with the least permissive policy (see below).
+3) Modify your policy to allow only the actions your tests use. Restrict account, region, and prefix where possible. Wait a few minutes for your policy to update.
+4) Run the tests again with a user or role that allows only the new policy.
+5) If the tests fail, troubleshoot (see tips below), modify the policy, run the tests again, and repeat the process until the tests pass with a restrictive policy.
+6) Share the minimum policy in a comment on your PR.
+
+To start from the least permissive IAM policy:
+
+1) Run the integration tests locally with no IAM permissions.
+2) Examine the error when the tests reach a failure.
+    a) If the error message indicates the action used in the request, add the action to your policy.
+    b) If the error message does not indicate the action used in the request:
+        - Usually the action is a CamelCase version of the method name - for example, for an ec2 client the method `describe_security_groups` correlates to the action `ec2:DescribeSecurityGroups`.
+        - Refer to the documentation to identify the action.
+    c) If the error message indicates the resource ARN used in the request, limit the action to that resource.
+    d) If the error message does not indicate the resource ARN used:
+        - Determine if the action can be restricted to a resource by examining the documentation.
+        - If the action can be restricted, use the documentation to construct the ARN and add it to the policy.
+3) Add the action or resource that caused the failure to [an IAM policy](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html#access_policies_create-start). Wait a few minutes for your policy to update.
+4) Run the tests again with this policy attached to your user or role.
+5) If the tests still fail at the same place with the same error you will need to troubleshoot (see tips below). If the first test passes, repeat steps 2 and 3 for the next error. Repeat the process until the tests pass with a restrictive policy.
+6) Share the minimum policy in a comment on your PR.
+
+Troubleshooting IAM policies:
+
+- When you make changes to a policy, wait a few minutes for the policy to update before re-running the tests.
+- Use the [policy simulator](https://policysim.aws.amazon.com/) to verify that each action (limited by resource when applicable) in your policy is allowed.
+- If you're restricting actions to certain resources, replace resources temporarily with `*`. If the tests pass with wildcard resources, there is a problem with the resource definition in your policy.
+- If the initial troubleshooting above doesn't provide any more insight, AWS may be using additional undisclosed resources and actions.
+- Examine the AWS FullAccess policy for the service for clues.
+- Re-read the AWS documentation, especially the [list of Actions, Resources and Condition Keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_actions-resources-contextkeys.html) for the various AWS services.
+- Look at the [cloudonaut](https://iam.cloudonaut.io) documentation as a troubleshooting cross-reference.
+- Use a search engine.
+- Ask in the Ansible IRC channel #ansible-aws.
+
+
+Some cases where tests should be marked as unsupported:
+1) The tests take longer than 10 or 15 minutes to complete
+2) The tests create expensive resources
+3) The tests create inline policies

lib/ansible/plugins/callback/aws_resource_actions.py

@@ -0,0 +1,72 @@
+# (C) 2018 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+    callback: aws_resource_actions
+    type: aggregate
+    short_description: summarizes all "resource:actions" completed
+    version_added: "2.8"
+    description:
+      - Ansible callback plugin for collecting the AWS actions completed by all boto3 modules using
+        AnsibleAWSModule in a playbook. Botocore endpoint logs need to be enabled for those modules, which can
+        be done easily by setting debug_botocore_endpoint_logs to True for group/aws using module_defaults.
+    requirements:
+      - whitelisting in configuration - see examples section below for details.
+'''
+
+EXAMPLES = '''
+example: >
+  To enable, add this to your ansible.cfg file in the defaults block
+    [defaults]
+    callback_whitelist = aws_resource_actions
+sample output: >
+#
+# AWS ACTIONS: ['s3:PutBucketAcl', 's3:HeadObject', 's3:DeleteObject', 's3:PutObjectAcl', 's3:CreateMultipartUpload',
+#               's3:DeleteBucket', 's3:GetObject', 's3:DeleteObjects', 's3:CreateBucket', 's3:CompleteMultipartUpload',
+#               's3:ListObjectsV2', 's3:HeadBucket', 's3:UploadPart', 's3:PutObject']
+#
+sample output: >
+#
+# AWS ACTIONS: ['ec2:DescribeVpcAttribute', 'ec2:DescribeVpcClassicLink', 'ec2:ModifyVpcAttribute', 'ec2:CreateTags',
+#               'sts:GetCallerIdentity', 'ec2:DescribeSecurityGroups', 'ec2:DescribeTags', 'ec2:DescribeVpcs', 'ec2:CreateVpc']
+#
+'''
+
+from ansible.plugins.callback import CallbackBase
+from ansible.module_utils._text import to_native
+
+
+class CallbackModule(CallbackBase):
+    CALLBACK_VERSION = 2.8
+    CALLBACK_TYPE = 'aggregate'
+    CALLBACK_NAME = 'aws_resource_actions'
+    CALLBACK_NEEDS_WHITELIST = True
+
+    def __init__(self):
+        self.aws_resource_actions = []
+        super(CallbackModule, self).__init__()
+
+    def extend_aws_resource_actions(self, result):
+        if result.get('resource_actions'):
+            self.aws_resource_actions.extend(result['resource_actions'])
+
+    def runner_on_ok(self, host, res):
+        self.extend_aws_resource_actions(res)
+
+    def runner_on_failed(self, host, res, ignore_errors=False):
+        self.extend_aws_resource_actions(res)
+
+    def v2_runner_item_on_ok(self, result):
+        self.extend_aws_resource_actions(result._result)
+
+    def v2_runner_item_on_failed(self, result):
+        self.extend_aws_resource_actions(result._result)
+
+    def playbook_on_stats(self, stats):
+        if self.aws_resource_actions:
+            self.aws_resource_actions = sorted(list(to_native(action) for action in set(self.aws_resource_actions)))
+            self._display.display("AWS ACTIONS: {0}".format(self.aws_resource_actions))

lib/ansible/plugins/doc_fragments/aws.py

@@ -9,6 +9,14 @@ class ModuleDocFragment(object):
     # AWS only documentation fragment
     DOCUMENTATION = r'''
 options:
+  debug_botocore_endpoint_logs:
+    description:
+      - Use a botocore.endpoint logger to parse the unique (rather than total) "resource:action" API calls made during a task, outputing
+        the set to the resource_actions key in the task results. Use the aws_resource_action callback to output to total list made during
+        a playbook. The ANSIBLE_DEBUG_BOTOCORE_LOGS environment variable may also be used.
+    type: bool
+    default: 'no'
+    version_added: "2.8"
   ec2_url:
     description:
       - Url to use to connect to EC2 or your Eucalyptus cloud (by default the module will use EC2 endpoints).