[cloud] new module lambda_policy (PR #24951)

- Fixes to lambda
- reformatting + tests for lambda_facts
- lambda module integration test
- switch lambda and lambda_facts to AnsibleAwsModule
- Get the account ID from STS, GetUser, and finally error message

test/integration/targets/aws_lambda/aliases

@@ -0,0 +1,4 @@
+cloud/aws
+posix/ci/cloud/aws
+execute_lambda
+lambda

test/integration/targets/aws_lambda/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+# defaults file for aws_lambda test
+lambda_function_name: '{{resource_prefix}}'

test/integration/targets/aws_lambda/files/mini_lambda.py

@@ -0,0 +1,34 @@
+from __future__ import print_function
+import json
+
+
+def handler(event, context):
+    """
+    The handler function is the function which gets called each time
+    the lambda is run.
+    """
+    # printing goes to the cloudwatch log allowing us to simply debug the lambda if we can find
+    # the log entry.
+    print("got event:\n" + json.dumps(event))
+
+    # if the name parameter isn't present this can throw an exception
+    # which will result in an amazon chosen failure from the lambda
+    # which can be completely fine.
+
+    name = event["name"]
+
+    return {"message": "hello " + name}
+
+
+def main():
+    """
+    This main function will normally never be called during normal
+    lambda use.  It is here for testing the lambda program only.
+    """
+    event = {"name": "james"}
+    context = None
+    print(handler(event, context))
+
+
+if __name__ == '__main__':
+    main()

test/integration/targets/aws_lambda/tasks/main.yml

@@ -0,0 +1,409 @@
+---
+#
+#  Author: Michael De La Rue
+#  based on ec2_key.yml + lambda.py
+
+- block:
+
+    # ============================================================
+    - name: test with no parameters
+      lambda:
+      register: result
+      ignore_errors: true
+
+    - name: assert failure when called with no parameters
+      assert:
+        that:
+           - 'result.failed'
+           - 'result.msg.startswith("missing required arguments: name")'
+
+    # ============================================================
+    - name: test with no parameters except state absent
+      lambda:
+        state=absent
+      register: result
+      ignore_errors: true
+
+    - name: assert failure when called with no parameters
+      assert:
+        that:
+           - 'result.failed'
+           - 'result.msg.startswith("missing required arguments: name")'
+
+    # ============================================================
+    - name: test with no role or handler
+      lambda:
+        name=ansible-testing-fake-should-not-be-created
+        runtime="python2.7"
+      register: result
+      ignore_errors: true
+
+    - name: assert failure when called with no parameters
+      assert:
+        that:
+           - 'result.failed'
+           - 'result.msg.startswith("state is present but the following are missing: handler")'
+
+    # ============================================================
+    - name: test with all module required variables but no region
+      lambda:
+        name=ansible-testing-fake-should-not-be-created
+        runtime="python2.7"
+        handler="no-handler"
+        role=arn:fake-role-doesnt-exist
+      register: result
+      ignore_errors: true
+
+    - name: assert failure when called with only 'name'
+      assert:
+        that:
+           - 'result.failed'
+           - 'result.msg == "region must be specified"'
+
+    # ============================================================
+    - name: test with all module required variables, no region and all possible variables set to blank
+      lambda:
+        name: ansible-testing-fake-should-not-be-created
+        state: present
+        runtime: "python2.7"
+        role: arn:fake-role-doesnt-exist
+        handler:
+        s3_bucket:
+        s3_key:
+        s3_object_version:
+        description:
+        vpc_subnet_ids:
+        vpc_security_group_ids:
+        environment_variables:
+        dead_letter_arn:
+      register: result
+      ignore_errors: true
+
+    - name: assert failure when called with only 'name'
+      assert:
+        that:
+           - 'result.failed'
+           - 'result.msg == "region must be specified"'
+
+    # ============================================================
+    # direct zip file upload
+    - name: move lambda into place for archive module
+      copy:
+        src: "mini_lambda.py"
+        dest: "{{output_dir}}/mini_lambda.py"
+
+    - name: bundle lambda into a zip
+      archive:
+        format: zip
+        path: "{{output_dir}}/mini_lambda.py"
+        dest: "{{output_dir}}/mini_lambda.zip"
+      register: zip_res
+
+    - name: test state=present - upload the lambda
+      lambda:
+        name="{{lambda_function_name}}"
+        runtime="python2.7"
+        handler="mini_lambda.handler"
+        role="ansible_lambda_role"
+        ec2_region='{{ec2_region}}'
+        ec2_access_key='{{ec2_access_key}}'
+        ec2_secret_key='{{ec2_secret_key}}'
+        security_token='{{security_token}}'
+        zip_file="{{zip_res.dest}}"
+      register: result
+
+    - name: assert lambda upload succeeded
+      assert:
+        that:
+           - 'not result|failed'
+
+    - name: test lambda works
+      execute_lambda:
+        name: "{{lambda_function_name}}"
+        payload:
+          name: "Mr Ansible Tests"
+        ec2_region: '{{ec2_region}}'
+        ec2_access_key: '{{ec2_access_key}}'
+        ec2_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      register: result
+
+    - name: assert lambda manages to respond as expected
+      assert:
+        that:
+           - 'not result|failed'
+           - 'result.result.output.message == "hello Mr Ansible Tests"'
+
+    # ============================================================
+    - name: test state=present with security group but no vpc
+      lambda:
+        name: "{{lambda_function_name}}"
+        runtime: "python2.7"
+        role: "ansible_lambda_role"
+        ec2_region: '{{ec2_region}}'
+        ec2_access_key: '{{ec2_access_key}}'
+        ec2_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+        zip_file: "{{zip_res.dest}}"
+        handler:
+        description:
+        vpc_subnet_ids:
+        vpc_security_group_ids: sg-FA6E
+        environment_variables:
+        dead_letter_arn:
+      register: result
+      ignore_errors: true
+
+    - name: assert lambda fails with proper message
+      assert:
+        that:
+           - 'result|failed'
+           - 'result.msg != "MODULE FAILURE"'
+           - 'result.changed == False'
+           - '"requires at least one security group and one subnet" in result.msg'
+
+    # ============================================================
+    - name: test state=present with all nullable variables explicitly set to null
+      lambda:
+        name: "{{lambda_function_name}}"
+        runtime: "python2.7"
+        role: "ansible_lambda_role"
+        ec2_region: '{{ec2_region}}'
+        ec2_access_key: '{{ec2_access_key}}'
+        ec2_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+        zip_file: "{{zip_res.dest}}"
+        handler: "mini_lambda.handler"
+# These are not allowed because of mutually exclusive.
+#        s3_bucket:
+#        s3_key:
+#        s3_object_version:
+        description:
+        vpc_subnet_ids:
+        vpc_security_group_ids:
+        environment_variables:
+        dead_letter_arn:
+      register: result
+
+    - name: assert lambda was updated as expected
+      assert:
+        that:
+           - 'not result|failed'
+           - 'result.changed == False'
+
+    # ============================================================
+    - name: test state=present triggering a network exception due to bad url
+      lambda:
+        name: "{{lambda_function_name}}"
+        runtime: "python2.7"
+        role: "ansible_lambda_role"
+        ec2_url: https://noexist.example.com
+        ec2_region: '{{ec2_region}}'
+        ec2_access_key: 'iamnotreallyanaccesskey'
+        ec2_secret_key: 'thisisabadsecretkey'
+        security_token: 'andthisisabadsecuritytoken'
+        zip_file: "{{zip_res.dest}}"
+      register: result
+      ignore_errors: true
+
+    - name: assert lambda manages to respond as expected
+      assert:
+        that:
+           - 'result|failed'
+           - 'result.changed == False'
+
+    # ============================================================
+    - name: test state=absent (expect changed=False)
+      lambda:
+        name="{{lambda_function_name}}"
+        ec2_region='{{ec2_region}}'
+        ec2_access_key='{{ec2_access_key}}'
+        ec2_secret_key='{{ec2_secret_key}}'
+        security_token='{{security_token}}'
+        state=absent
+      register: result
+
+    - name: assert state=absent
+      assert:
+        that:
+           - 'not result|failed'
+           - 'result.changed == True'
+
+    # ============================================================
+    # parallel lambda creation
+
+    - name: parallel lambda creation 1/4
+      lambda:
+        name: "{{lambda_function_name}}_1"
+        runtime: "python2.7"
+        handler: "mini_lambda.handler"
+        role: "ansible_lambda_role"
+        ec2_region: '{{ec2_region}}'
+        ec2_access_key: '{{ec2_access_key}}'
+        ec2_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+        zip_file: "{{zip_res.dest}}"
+      async: 1000
+      register: async_1
+
+    - name: parallel lambda creation 2/4
+      lambda:
+        name: "{{lambda_function_name}}_2"
+        runtime: "python2.7"
+        handler: "mini_lambda.handler"
+        role: "ansible_lambda_role"
+        ec2_region: '{{ec2_region}}'
+        ec2_access_key: '{{ec2_access_key}}'
+        ec2_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+        zip_file: "{{zip_res.dest}}"
+      async: 1000
+      register: async_2
+
+    - name: parallel lambda creation 3/4
+      lambda:
+        name: "{{lambda_function_name}}_3"
+        runtime: "python2.7"
+        handler: "mini_lambda.handler"
+        role: "ansible_lambda_role"
+        ec2_region: '{{ec2_region}}'
+        ec2_access_key: '{{ec2_access_key}}'
+        ec2_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+        zip_file: "{{zip_res.dest}}"
+      async: 1000
+      register: async_3
+
+    - name: parallel lambda creation 4/4
+      lambda:
+        name: "{{lambda_function_name}}_4"
+        runtime: "python2.7"
+        handler: "mini_lambda.handler"
+        role: "ansible_lambda_role"
+        ec2_region: '{{ec2_region}}'
+        ec2_access_key: '{{ec2_access_key}}'
+        ec2_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+        zip_file: "{{zip_res.dest}}"
+      register: result
+
+    - name: assert lambda manages to respond as expected
+      assert:
+        that:
+           - 'not result|failed'
+
+    - name: wait for async job 1
+      async_status: jid={{ async_1.ansible_job_id }}
+      register: job_result
+      until: job_result.finished
+      retries: 30
+
+    - name: wait for async job 2
+      async_status: jid={{ async_1.ansible_job_id }}
+      register: job_result
+      until: job_result.finished
+      retries: 30
+
+    - name: wait for async job 3
+      async_status: jid={{ async_3.ansible_job_id }}
+      register: job_result
+      until: job_result.finished
+      retries: 30
+
+
+    - name: parallel lambda deletion 1/4
+      lambda:
+        name: "{{lambda_function_name}}_1"
+        state: absent
+        ec2_region: '{{ec2_region}}'
+        ec2_access_key: '{{ec2_access_key}}'
+        ec2_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+        zip_file: "{{zip_res.dest}}"
+      async: 1000
+      register: async_1
+
+    - name: parallel lambda deletion 2/4
+      lambda:
+        name: "{{lambda_function_name}}_2"
+        state: absent
+        ec2_region: '{{ec2_region}}'
+        ec2_access_key: '{{ec2_access_key}}'
+        ec2_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+        zip_file: "{{zip_res.dest}}"
+      async: 1000
+      register: async_2
+
+    - name: parallel lambda deletion 3/4
+      lambda:
+        name: "{{lambda_function_name}}_3"
+        state: absent
+        ec2_region: '{{ec2_region}}'
+        ec2_access_key: '{{ec2_access_key}}'
+        ec2_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+        zip_file: "{{zip_res.dest}}"
+      async: 1000
+      register: async_3
+
+    - name: parallel lambda deletion 4/4
+      lambda:
+        name: "{{lambda_function_name}}_4"
+        state: absent
+        ec2_region: '{{ec2_region}}'
+        ec2_access_key: '{{ec2_access_key}}'
+        ec2_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+        zip_file: "{{zip_res.dest}}"
+      register: result
+
+    - name: assert lambda creation has succeeded
+      assert:
+        that:
+           - 'not result|failed'
+
+    - name: wait for async job 1
+      async_status: jid={{ async_1.ansible_job_id }}
+      register: job_result
+      until: job_result.finished
+      retries: 30
+
+    - name: wait for async job 2
+      async_status: jid={{ async_1.ansible_job_id }}
+      register: job_result
+      until: job_result.finished
+      retries: 30
+
+    - name: wait for async job 3
+      async_status: jid={{ async_3.ansible_job_id }}
+      register: job_result
+      until: job_result.finished
+      retries: 30
+
+
+    # ============================================================
+    # upload via s3 bucket - multi function
+
+    # ============================================================
+    # update already existing function
+
+
+  always:
+
+    # ============================================================
+    - name: test state=absent (expect changed=False)
+      lambda:
+        name="{{lambda_function_name}}"
+        ec2_region='{{ec2_region}}'
+        ec2_access_key='{{ec2_access_key}}'
+        ec2_secret_key='{{ec2_secret_key}}'
+        security_token='{{security_token}}'
+        state=absent
+      register: result
+
+    - name: assert state=absent
+      assert:
+        that:
+           - 'not result|failed'
+           - 'result.changed == False'

test/integration/targets/aws_lambda_policy/aliases

@@ -0,0 +1,2 @@
+cloud/aws
+posix/ci/cloud/aws

test/integration/targets/aws_lambda_policy/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+# defaults file for aws_lambda test
+lambda_function_name: '{{resource_prefix}}-api-endpoint'

test/integration/targets/aws_lambda_policy/files/mini_http_lambda.py

@@ -0,0 +1,36 @@
+from __future__ import print_function
+import json
+
+
+def handler(event, context):
+    """
+    The handler function is the function which gets called each time
+    the lambda is run.
+    """
+    # printing goes to the cloudwatch log allowing us to simply debug the lambda if we can find
+    # the log entry.
+    print("got event:\n" + json.dumps(event))
+
+    # if the name parameter isn't present this can throw an exception
+    # which will result in an amazon chosen failure from the lambda
+    # which can be completely fine.
+
+    name = event["pathParameters"]["greet_name"]
+
+    return {"statusCode": 200,
+            "body": 'hello: "' + name + '"',
+            "headers": {}}
+
+
+def main():
+    """
+    This main function will normally never be called during normal
+    lambda use.  It is here for testing the lambda program only.
+    """
+    event = {"name": "james"}
+    context = None
+    print(handler(event, context))
+
+
+if __name__ == '__main__':
+    main()

test/integration/targets/aws_lambda_policy/tasks/main.yml

@@ -0,0 +1,218 @@
+---
+#
+#  Author: Michael De La Rue
+#  based on ec2_key.yml + lambda.py
+
+- block:
+
+    # ============================================================
+    - name: test with no parameters
+      lambda_policy:
+      register: result
+      ignore_errors: true
+
+    - name: assert failure when called with no parameters
+      assert:
+        that:
+           - 'result.failed'
+           - 'result.msg.startswith("missing required arguments: ")'
+
+    # ============================================================
+    - name: test with all required dummy parameters but no region
+      lambda_policy:
+        statement_id: dummy
+        principal: api_fakeway
+        action: fake:do_something_fake
+        function_name: dummy_fake_function
+      ignore_errors: true
+      register: result
+
+    - name: assert failure and appropriate message when called without region
+      assert:
+        that:
+           - 'result.failed'
+           - '"region must be specified" in result.msg'
+
+    # ============================================================
+    - name: test with all required dummy parameters but no region
+      lambda_policy:
+        statement_id: dummy
+        principal: api_fakeway
+        action: fake:do_something_fake
+        function_name: dummy_fake_function
+        region: null
+      ignore_errors: true
+      register: result
+
+    - name: assert failure and appropriate message when called false region region
+      assert:
+        that:
+           - 'result.failed'
+           - '"region must be specified" in result.msg'
+
+    # ============================================================
+    - name: test exceptions generated by forcing bad ec2 url
+      lambda_policy:
+        function_name: "{{ lambda_function_name }}"
+        region: "{{ec2_region}}"
+        state: present
+        statement_id: api-gateway-invoke-lambdas
+        action: lambda:InvokeFunction
+        principal: apigateway.amazonaws.com
+        source_arn: "arn:aws:execute-api:no-north-0:1234567:*/*"
+        ec2_url: https://noexist.example.com
+        ec2_region: 'no-north-0'
+        ec2_access_key: 'iamnotreallyanaccesskey'
+        ec2_secret_key: 'thisisabadsecretkey'
+        security_token: 'andthisisabadsecuritytoken'
+      register: result
+      ignore_errors: true
+
+    - name: assert lambda manages to respond as expected
+      assert:
+        that:
+           - 'result|failed'
+           - 'result.msg != "MODULE FAILURE"'
+           - 'result.changed == False'
+
+    # ============================================================
+    # direct zip file upload
+    - name: move lambda into place for archive module
+      copy:
+        src: "mini_http_lambda.py"
+        dest: "{{output_dir}}/mini_http_lambda.py"
+
+    - name: bundle lambda into a zip
+      archive:
+        format: zip
+        path: "{{output_dir}}/mini_http_lambda.py"
+        dest: "{{output_dir}}/mini_http_lambda.zip"
+      register: zip_res
+
+    - name: test state=present - upload the lambda
+      lambda:
+        name="{{lambda_function_name}}"
+        runtime="python2.7"
+        handler="mini_http_lambda.handler"
+        role="ansible_lambda_role"
+        ec2_region='{{ec2_region}}'
+        aws_access_key='{{aws_access_key}}'
+        aws_secret_key='{{aws_secret_key}}'
+        security_token='{{security_token}}'
+        zip_file="{{zip_res.dest}}"
+      register: lambda_result
+
+    - name: install aws cli - FIXME temporary this should go for a lighterweight solution
+      command: pip install awscli
+      register: result
+
+    - name: get the aws account ID for use in future commands
+      command: aws sts get-caller-identity --output text --query 'Account'
+      environment:
+          AWS_ACCESS_KEY_ID: '{{aws_access_key}}'
+          AWS_SECRET_ACCESS_KEY: '{{aws_secret_key}}'
+          AWS_SESSION_TOKEN: '{{security_token}}'
+      register: result
+
+    - name: register account id
+      set_fact:
+        aws_account_id: "{{ result.stdout | replace('\n', '') }}"
+
+    - name: register lambda uri for use in template
+      set_fact:
+        mini_lambda_uri: "arn:aws:apigateway:{{ec2_region}}:lambda:path/2015-03-31/functions/arn:aws:lambda:{{ec2_region}}:{{aws_account_id}}:function:{{ lambda_result.configuration.function_name }}/invocations"
+
+    - name: build API file
+      template:
+        src: endpoint-test-swagger-api.yml.j2
+        dest: "{{output_dir}}/endpoint-test-swagger-api.yml.j2"
+
+    - name: deploy new API
+      aws_api_gateway:
+        api_file: "{{output_dir}}/endpoint-test-swagger-api.yml.j2"
+        stage: "lambdabased"
+        region: '{{ec2_region}}'
+        aws_access_key: '{{aws_access_key}}'
+        aws_secret_key: '{{aws_secret_key}}'
+        security_token: '{{security_token}}'
+      register: create_result
+
+
+    - name: register api id for later
+      set_fact:
+        api_id: "{{ create_result.api_id }}"
+
+    - name: check API fails with permissions failure
+      uri: url="https://{{create_result.api_id}}.execute-api.{{ec2_region}}.amazonaws.com/lambdabased/mini/Mr_Ansible_Tester"
+      register: unauth_uri_result
+      ignore_errors: true
+
+    - name: assert internal server error due to permissions
+      assert:
+        that:
+          - unauth_uri_result|failed
+          - 'unauth_uri_result.status == 500'
+
+    - name: give api gateway execute permissions on lambda
+      lambda_policy:
+        function_name: "{{ lambda_function_name }}"
+        region: "{{ec2_region}}"
+        state: present
+        statement_id: api-gateway-invoke-lambdas
+        action: lambda:InvokeFunction
+        principal: apigateway.amazonaws.com
+        source_arn: "arn:aws:execute-api:{{ ec2_region }}:{{ aws_account_id }}:*/*"
+        aws_access_key: '{{aws_access_key}}'
+        aws_secret_key: '{{aws_secret_key}}'
+        security_token: '{{security_token}}'
+
+    - name: check API works with execute permissions
+      uri: url="https://{{create_result.api_id}}.execute-api.{{ec2_region}}.amazonaws.com/lambdabased/mini/Mr_Ansible_Tester"
+      register: uri_result
+
+    - name: assert API works success
+      assert:
+        that:
+           - 'uri_result'
+
+
+    - name: deploy new API
+      aws_api_gateway:
+        api_file: "{{output_dir}}/endpoint-test-swagger-api.yml.j2"
+        stage: "lambdabased"
+        region: '{{ec2_region}}'
+        aws_access_key: '{{aws_access_key}}'
+        aws_secret_key: '{{aws_secret_key}}'
+        security_token: '{{security_token}}'
+      register: create_result
+      ignore_errors: true
+
+
+  always:
+
+    # ============================================================
+    - name: destroy lambda for test cleanup if created
+      lambda:
+        name="{{lambda_function_name}}"
+        ec2_region='{{ec2_region}}'
+        ec2_access_key='{{ec2_access_key}}'
+        ec2_secret_key='{{ec2_secret_key}}'
+        security_token='{{security_token}}'
+        state=absent
+      register: result
+
+    - name: destroy API for test cleanup if created
+      aws_api_gateway:
+        state: absent
+        api_id: '{{api_id}}'
+        region: '{{ec2_region}}'
+        aws_access_key: '{{ec2_access_key}}'
+        aws_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      register: destroy_result
+
+    - name: assert destroy statements succeeded
+      assert:
+        that:
+           - 'destroy_result.changed == True'
+           - 'not result|failed'

test/integration/targets/aws_lambda_policy/templates/endpoint-test-swagger-api.yml.j2

@@ -0,0 +1,39 @@
+---
+swagger: "2.0"
+info:
+  version: "2017-05-11T12:14:59Z"
+  title: "{{resource_prefix}}LambdaBased_API"
+host: "fakeexample.execute-api.us-east-1.amazonaws.com"
+basePath: "/lambdabased"
+schemes:
+- "https"
+paths:
+  /mini/{greet_name}:
+    get:
+      produces:
+      - "application/json"
+      parameters:
+      - name: "greet_name"
+        in: "path"
+        required: true
+        type: "string"
+      responses:
+        200:
+          description: "200 response"
+          schema:
+            $ref: "#/definitions/Empty"
+      x-amazon-apigateway-integration:
+        responses:
+          default:
+            statusCode: "200"
+        uri: "{{mini_lambda_uri}}"
+        requestTemplates:
+          application/json: "{\"statusCode\": 200}"
+        passthroughBehavior: "when_no_match"
+        httpMethod: "POST"
+        contentHandling: "CONVERT_TO_TEXT"
+        type: "aws_proxy"
+definitions:
+  Empty:
+    type: "object"
+    title: "Empty Schema"