[cloud] New module: AWS API Gageway module (#20230)

* Ultra basic api-gateway module based of lambda.py

* Ultra basic deployment added to api-gateway module

* ApiGateway module Allow creation of APIs, more documentation and better return value

* ApiGateway module incorporate review feedback

* ApiGateway module flake8 cleanup

* APIGateway module - more review fixes.

* slightly better messages in api_gateway module

* AWS api_gateway module - try to improve messages in case of exceptions

* rename api_gateway module to aws_api_gateway as discussed in PR 20230

* aws_api_gateway - Allow delivery of swagger either as text or dictionary.

* aws_api_gateway module - introduce 'unit' tests, improve imports using them and small fixes

* aws_api_gateway module - move path expand_user to avoid early typecheck

* aws_api_gateway - version means version of metadata not module - fix to 1.0

* aws_api_gateway module - Rely on module_utils.ec2 for imports & path type for expanduser / cleanups

* aws_api_gateway module - heavy cleanup and refactor of code + cloud retry functionality.

* api_gateway_module - failing test case for handling more than one deployment in succession and API deletion

* add TooManyRequestsException to AWSRetry exception list - makes API deployment work.

* api_gateway_module - Fixes for various review comments + errors from various linters

* api_gateway_module - Fixes for more review comments + linter error

* api_gateway_module - Major refactor into sensible functions - create_response becomes configure_response

* api_gateway_module - should be working under python3; remove test exclusion

* api_gateway_module - finish off remaining review fixes - use ansible defaults and fix mutually exclusive

* api_gateway_module - attempt to improve handling of botocore errors in python3

* api_gateway_module - implement state=absent / API deletion

test/integration/targets/aws_api_gateway/aliases

@@ -0,0 +1,2 @@
+cloud/aws
+posix/ci/cloud/aws

test/integration/targets/aws_api_gateway/meta/main.yml

@@ -0,0 +1,3 @@
+dependencies:
+  - prepare_tests
+  - setup_ec2

test/integration/targets/aws_api_gateway/tasks/main.yml

@@ -0,0 +1,182 @@
+- block:
+
+    # ============================================================
+    - name: test with no parameters
+      aws_api_gateway:
+      register: result
+      ignore_errors: true
+
+    - name: assert failure when called with no parameters
+      assert:
+        that:
+           - 'result.failed'
+           - 'result.msg.startswith("Region must be specified")'
+
+    # ============================================================
+    - name: test with minimal parameters but no region
+      aws_api_gateway:
+        api_id: 'fake-api-doesnt-exist'
+      register: result
+      ignore_errors: true
+
+    - name: assert failure when called with with minimal parameters but no region
+      assert:
+        that:
+           - 'result.failed'
+           - 'result.msg.startswith("Region must be specified")'
+
+    # ============================================================
+    - name: test disallow multiple swagger sources
+      aws_api_gateway:
+        api_id: 'fake-api-doesnt-exist'
+        region: 'fake_region'
+        swagger_file: foo.yml
+        swagger_text: "this is not really an API"
+      register: result
+      ignore_errors: true
+
+    - name: assert failure when called with with minimal parameters but no region
+      assert:
+        that:
+           - 'result.failed'
+           - 'result.msg.startswith("parameters are mutually exclusive")'
+
+    # This fails with
+
+    # msg": "There is an issue in the code of the module. You must
+    # specify either both, resource or client to the conn_type
+    # parameter in the boto3_conn function call"
+
+    # even though the call appears to include conn_type='client'
+
+    # # ============================================================
+    # - name: test invalid region parameter
+    #   aws_api_gateway:
+    #     api_id: 'fake-api-doesnt-exist'
+    #     region: 'asdf querty 1234'
+    #   register: result
+    #   ignore_errors: true
+
+    # - name: assert invalid region parameter
+    #   assert:
+    #     that:
+    #        - 'result.failed'
+    #        - 'result.msg.startswith("Region asdf querty 1234 does not seem to be available ")'
+
+    # ============================================================
+
+    - name: build API file 
+      template:
+        src: minimal-swagger-api.yml.j2
+        dest: "{{output_dir}}/minimal-swagger-api.yml"
+      tags: new_api,api,api_file
+
+    - name: deploy new API
+      aws_api_gateway: 
+        api_file: "{{output_dir}}/minimal-swagger-api.yml"
+        stage: "minimal"
+        region: '{{ec2_region}}'
+        aws_access_key: '{{ec2_access_key}}'
+        aws_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      register: create_result
+
+    - name: assert deploy new API worked
+      assert:
+        that:
+           - 'create_result.changed == True'
+           - '"api_id" in create_result'
+#           - '"created_response.created_date" in create_result'
+#           - '"deploy_response.created_date" in create_result'
+
+    - name: check API works
+      uri: url="https://{{create_result.api_id}}.execute-api.{{ec2_region}}.amazonaws.com/minimal"
+      register: uri_result
+
+    - name: assert API works success
+      assert:
+        that:
+           - 'uri_result'
+
+    - name: check nonexistent endpoints cause errors
+      uri: url="https://{{create_result.api_id}}.execute-api.{{ec2_region}}.amazonaws.com/nominal"
+      register: bad_uri_result
+      ignore_errors: true
+
+    - name: assert 
+      assert:
+        that:
+          - bad_uri_result|failed
+
+    # ============================================================
+
+    - name: deploy first API
+      aws_api_gateway:
+        api_file: "{{output_dir}}/minimal-swagger-api.yml"
+        stage: "minimal"
+        region: '{{ec2_region}}'
+        aws_access_key: '{{ec2_access_key}}'
+        aws_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      register: create_result_1
+
+    - name: deploy second API rapidly after first
+      aws_api_gateway:
+        api_file: "{{output_dir}}/minimal-swagger-api.yml"
+        stage: "minimal"
+        region: '{{ec2_region}}'
+        aws_access_key: '{{ec2_access_key}}'
+        aws_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      register: create_result_2
+
+    - name: assert both APIs deployed successfully
+      assert:
+        that:
+           - 'create_result_1.changed == True'
+           - 'create_result_2.changed == True'
+           - '"api_id" in create_result_1'
+           - '"api_id" in create_result_1'
+#           - '"created_response.created_date" in create_result'
+#           - '"deploy_response.created_date" in create_result'
+
+    - name: destroy first API
+      aws_api_gateway:
+        state: absent
+        api_id: '{{create_result_1.api_id}}'
+        region: '{{ec2_region}}'
+        aws_access_key: '{{ec2_access_key}}'
+        aws_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      register: destroy_result_1
+
+    - name: destroy second API rapidly after first
+      aws_api_gateway:
+        state: absent
+        api_id: '{{create_result_2.api_id}}'
+        region: '{{ec2_region}}'
+        aws_access_key: '{{ec2_access_key}}'
+        aws_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      register: destroy_result_2
+
+    - name: assert both APIs deployed successfully
+      assert:
+        that:
+           - 'destroy_result_1.changed == True'
+           - 'destroy_result_2.changed == True'
+#           - '"created_response.created_date" in create_result'
+#           - '"deploy_response.created_date" in create_result'
+
+  always:
+
+    # ============================================================
+    - name: test state=absent (expect changed=false)
+      aws_api_gateway:
+        state: absent
+        api_id: '{{create_result.api_id}}'        
+        ec2_region: '{{ec2_region}}'
+        aws_access_key: '{{ec2_access_key}}'
+        aws_secret_key: '{{ec2_secret_key}}'
+        security_token: '{{security_token}}'
+      register: destroy_result

test/integration/targets/aws_api_gateway/templates/minimal-swagger-api.yml.j2

@@ -0,0 +1,33 @@
+---
+swagger: "2.0"
+info:
+  version: "2017-05-11T12:14:59Z"
+  title: "{{resource_prefix}}Empty_API"
+host: "fakeexample.execute-api.us-east-1.amazonaws.com"
+basePath: "/minimal"
+schemes:
+- "https"
+paths:
+  /:
+    get:
+      consumes:
+      - "application/json"
+      produces:
+      - "application/json"
+      responses:
+        200:
+          description: "200 response"
+          schema:
+            $ref: "#/definitions/Empty"
+      x-amazon-apigateway-integration:
+        responses:
+          default:
+            statusCode: "200"
+        requestTemplates:
+          application/json: "{\"statusCode\": 200}"
+        passthroughBehavior: "when_no_match"
+        type: "mock"
+definitions:
+  Empty:
+    type: "object"
+    title: "Empty Schema"