[aws]Add VPC configuration to ECS modules (#34381)

Enable awsvpc network mode for ECS services and tasks and
their underlying task definitions

Improve test suite to thoroughly test the changes

Use runme.sh technique to run old and new versions of botocore to
ensure that the modules work with older botocore and older network modes
and fail gracefully if awsvpc network mode is used with older botocore

test/integration/targets/ecs_cluster/playbooks/full_test.yml

@@ -0,0 +1,5 @@
+- hosts: localhost
+  connection: local
+
+  roles:
+    - ecs_cluster

test/integration/targets/ecs_cluster/playbooks/network_fail.yml

@@ -0,0 +1,146 @@
+- hosts: localhost
+  connection: local
+
+  tasks:
+    - block:
+        - name: set up aws connection info
+          set_fact:
+            aws_connection_info: &aws_connection_info
+              aws_access_key: "{{ aws_access_key }}"
+              aws_secret_key: "{{ aws_secret_key }}"
+              security_token: "{{ security_token }}"
+              region: "{{ aws_region }}"
+          no_log: True
+
+        - name: create ecs cluster
+          ecs_cluster:
+            name: "{{ resource_prefix }}"
+            state: present
+            <<: *aws_connection_info
+
+        - name: create ecs_taskdefinition with bridged network
+          ecs_taskdefinition:
+            containers:
+              - name: my_container
+                image: ubuntu
+                memory: 128
+            family: "{{ resource_prefix }}"
+            state: present
+            network_mode: bridge
+            <<: *aws_connection_info
+          register: ecs_taskdefinition_creation
+
+        - name: create ecs_taskdefinition with awsvpc network
+          ecs_taskdefinition:
+            containers:
+              - name: my_container
+                image: ubuntu
+                memory: 128
+            family: "{{ resource_prefix }}-vpc"
+            state: present
+            network_mode: awsvpc
+            <<: *aws_connection_info
+          register: ecs_taskdefinition_creation_vpc
+
+        - name: ecs_taskdefinition works fine even when older botocore is used
+          assert:
+            that:
+              - ecs_taskdefinition_creation_vpc.changed
+
+        - name: create ecs_service using bridged network
+          ecs_service:
+            name: "{{ resource_prefix }}"
+            cluster: "{{ resource_prefix }}"
+            task_definition: "{{ resource_prefix }}"
+            desired_count: 1
+            state: present
+            <<: *aws_connection_info
+          register: ecs_service_creation
+
+        - name: create ecs_service using awsvpc network_configuration
+          ecs_service:
+            name: "{{ resource_prefix }}-vpc"
+            cluster: "{{ resource_prefix }}"
+            task_definition: "{{ resource_prefix }}"
+            desired_count: 1
+            network_configuration:
+              subnets:
+                - subnet-abcd1234
+              groups:
+                - sg-abcd1234
+            state: present
+            <<: *aws_connection_info
+          register: ecs_service_creation_vpc
+          ignore_errors: yes
+
+        - name: check that graceful failure message is returned from ecs_service
+          assert:
+            that:
+              - ecs_service_creation_vpc.failed
+              - 'ecs_service_creation_vpc.msg == "botocore needs to be version 1.7.44 or higher to use network configuration"'
+
+        - name: create ecs_task using awsvpc network_configuration
+          ecs_task:
+            cluster: "{{ resource_prefix }}-vpc"
+            task_definition: "{{ resource_prefix }}"
+            operation: run
+            count: 1
+            started_by: me
+            network_configuration:
+              subnets:
+                - subnet-abcd1234
+              groups:
+                - sg-abcd1234
+            <<: *aws_connection_info
+          register: ecs_task_creation_vpc
+          ignore_errors: yes
+
+        - name: check that graceful failure message is returned from ecs_task
+          assert:
+            that:
+              - ecs_task_creation_vpc.failed
+              - 'ecs_task_creation_vpc.msg == "botocore needs to be version 1.7.44 or higher to use network configuration"'
+
+      always:
+        - name: scale down ecs service
+          ecs_service:
+            name: "{{ resource_prefix }}"
+            cluster: "{{ resource_prefix }}"
+            task_definition: "{{ resource_prefix }}"
+            desired_count: 0
+            state: present
+            <<: *aws_connection_info
+          ignore_errors: yes
+
+        - name: pause to wait for scale down
+          pause:
+            seconds: 30
+
+        - name: remove ecs service
+          ecs_service:
+            name: "{{ resource_prefix }}"
+            cluster: "{{ resource_prefix }}"
+            task_definition: "{{ resource_prefix }}"
+            desired_count: 1
+            state: absent
+            <<: *aws_connection_info
+          ignore_errors: yes
+
+        - name: remove ecs task definition
+          ecs_taskdefinition:
+            containers:
+              - name: my_container
+                image: ubuntu
+                memory: 128
+            family: "{{ resource_prefix }}"
+            revision: "{{ ecs_taskdefinition_creation.taskdefinition.revision }}"
+            state: absent
+            <<: *aws_connection_info
+          ignore_errors: yes
+
+        - name: remove ecs cluster
+          ecs_cluster:
+            name: "{{ resource_prefix }}"
+            state: absent
+            <<: *aws_connection_info
+          ignore_errors: yes

test/integration/targets/ecs_cluster/playbooks/roles/ecs_cluster/defaults/main.yml

@@ -31,5 +31,5 @@ ecs_service_placement_strategy:
   - type: spread
     field: attribute:ecs.availability-zone
 ecs_task_container_port: 8080
-ecs_target_group_name: "{{ resource_prefix[:29] }}-tg"
+ecs_target_group_name: "{{ resource_prefix[:28] }}-tg"
 ecs_load_balancer_name: "{{ resource_prefix[:29] }}-lb"

test/integration/targets/ecs_cluster/playbooks/roles/ecs_cluster/tasks/main.yml

@@ -17,7 +17,7 @@
         name: ecsInstanceRole
         assume_role_policy_document: "{{ lookup('file','ec2-trust-policy.json') }}"
         state: present
-        create_instance_profile: no
+        create_instance_profile: yes
         managed_policy:
         - AmazonEC2ContainerServiceforEC2Role
         <<: *aws_connection_info
@@ -32,6 +32,20 @@
         - AmazonEC2ContainerServiceRole
         <<: *aws_connection_info
 
+    - name: ensure AWSServiceRoleForECS role exists
+      iam_role_facts:
+        name: AWSServiceRoleForECS
+        <<: *aws_connection_info
+      register: iam_role_result
+
+      # FIXME: come up with a way to automate this
+    - name: fail if AWSServiceRoleForECS role does not exist
+      fail:
+        msg: >
+          Run `aws iam create-service-linked-role --aws-service-name=ecs.amazonaws.com ` to create
+          a linked role for AWS VPC load balancer management
+      when: not iam_role_result.iam_roles
+
     - name: create an ECS cluster
       ecs_cluster:
         name: "{{ ecs_cluster_name }}"
@@ -122,20 +136,32 @@
           Name: '{{ resource_prefix }}_ecs_agent'
         group_id: '{{ setup_sg.group_id }}'
         vpc_subnet_id: '{{ setup_subnet.results[0].subnet.id }}'
-        assign_public_ip: yes  # public IP address assigned to avoid need for NAT GW.
         <<: *aws_connection_info
       register: setup_instance
 
     - name: create target group
       elb_target_group:
-        name: "{{ ecs_target_group_name }}"
+        name: "{{ ecs_target_group_name }}1"
         state: present
         protocol: HTTP
         port: 8080
         modify_targets: no
         vpc_id: '{{ setup_vpc.vpc.id }}'
+        target_type: instance
         <<: *aws_connection_info
-      register: elb_target_group
+      register: elb_target_group_instance
+
+    - name: create second target group to use ip target_type
+      elb_target_group:
+        name: "{{ ecs_target_group_name }}2"
+        state: present
+        protocol: HTTP
+        port: 8080
+        modify_targets: no
+        vpc_id: '{{ setup_vpc.vpc.id }}'
+        target_type: ip
+        <<: *aws_connection_info
+      register: elb_target_group_ip
 
     - name: create load balancer
       elb_application_lb:
@@ -149,7 +175,12 @@
             Port: 80
             DefaultActions:
               - Type: forward
-                TargetGroupName: "{{ ecs_target_group_name }}"
+                TargetGroupName: "{{ ecs_target_group_name }}1"
+          - Protocol: HTTP
+            Port: 81
+            DefaultActions:
+              - Type: forward
+                TargetGroupName: "{{ ecs_target_group_name }}2"
         <<: *aws_connection_info
 
     - name: create task definition
@@ -190,7 +221,7 @@
         deployment_configuration: "{{ ecs_service_deployment_configuration }}"
         placement_strategy: "{{ ecs_service_placement_strategy }}"
         load_balancers:
-          - targetGroupArn: "{{ elb_target_group.target_group_arn }}"
+          - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}"
             containerName: "{{ ecs_task_name }}"
             containerPort: "{{ ecs_task_container_port }}"
         role: "ecsServiceRole"
@@ -212,7 +243,7 @@
         deployment_configuration: "{{ ecs_service_deployment_configuration }}"
         placement_strategy: "{{ ecs_service_placement_strategy }}"
         load_balancers:
-          - targetGroupArn: "{{ elb_target_group.target_group_arn }}"
+          - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}"
             containerName: "{{ ecs_task_name }}"
             containerPort: "{{ ecs_task_container_port }}"
         role: "ecsServiceRole"
@@ -237,7 +268,7 @@
         deployment_configuration: "{{ ecs_service_deployment_configuration }}"
         placement_strategy: "{{ ecs_service_placement_strategy }}"
         load_balancers:
-          - targetGroupArn: "{{ elb_target_group.target_group_arn }}"
+          - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}"
             containerName: "{{ ecs_task_name }}"
             containerPort: "{{ ecs_task_container_port|int + 1 }}"
         role: "ecsServiceRole"
@@ -248,11 +279,176 @@
     - name: assert that updating ECS load balancer failed with helpful message
       assert:
         that:
-          - update_ecs_service.failed
+          - update_ecs_service is failed
+          - "'error' not in update_ecs_service"
           - "'msg' in update_ecs_service"
-      # FIXME: fixed in #32876
+
+
+    - name: attempt to use ECS network configuration on task definition without awsvpc network_mode
+      ecs_service:
+        state: present
+        name: "{{ ecs_service_name }}3"
+        cluster: "{{ ecs_cluster_name }}"
+        task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}"
+        desired_count: 1
+        deployment_configuration: "{{ ecs_service_deployment_configuration }}"
+        placement_strategy: "{{ ecs_service_placement_strategy }}"
+        load_balancers:
+          - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}"
+            containerName: "{{ ecs_task_name }}"
+            containerPort: "{{ ecs_task_container_port }}"
+        network_configuration:
+          subnets: "{{ setup_subnet.results | json_query('[].subnet.id') }}"
+          security_groups:
+            - '{{ setup_sg.group_id }}'
+        <<: *aws_connection_info
+      register: ecs_service_network_without_awsvpc_task
       ignore_errors: yes
 
+    - name: assert that using ECS network configuration with non AWSVPC task definition fails
+      assert:
+        that:
+          - ecs_service_network_without_awsvpc_task is failed
+
+    - name: scale down ECS service
+      ecs_service:
+        state: present
+        name: "{{ ecs_service_name }}"
+        cluster: "{{ ecs_cluster_name }}"
+        task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}"
+        desired_count: 0
+        deployment_configuration: "{{ ecs_service_deployment_configuration }}"
+        placement_strategy: "{{ ecs_service_placement_strategy }}"
+        load_balancers:
+          - targetGroupArn: "{{ elb_target_group_instance.target_group_arn }}"
+            containerName: "{{ ecs_task_name }}"
+            containerPort: "{{ ecs_task_container_port }}"
+        role: "ecsServiceRole"
+        <<: *aws_connection_info
+      register: ecs_service_scale_down
+
+    - name: pause to allow service to scale down
+      pause:
+        seconds: 60
+
+    - name: delete ECS service definition
+      ecs_service:
+        state: absent
+        name: "{{ ecs_service_name }}"
+        cluster: "{{ ecs_cluster_name }}"
+        <<: *aws_connection_info
+      register: delete_ecs_service
+
+    - name: assert that deleting ECS service worked
+      assert:
+        that:
+          - delete_ecs_service.changed
+
+    - name: create VPC-networked task definition with host port set to 0 (expected to fail)
+      ecs_taskdefinition:
+        containers: "{{ ecs_task_containers }}"
+        family: "{{ ecs_task_name }}-vpc"
+        state: present
+        network_mode: awsvpc
+        <<: *aws_connection_info
+      register: ecs_task_definition_vpc_no_host_port
+      ignore_errors: yes
+
+    - name: check that awsvpc task definition with host port 0 fails gracefully
+      assert:
+        that:
+          - ecs_task_definition_vpc_no_host_port is failed
+          - "'error' not in ecs_task_definition_vpc_no_host_port"
+
+    - name: create VPC-networked task definition with host port set to 8080
+      ecs_taskdefinition:
+        containers: "{{ ecs_task_containers }}"
+        family: "{{ ecs_task_name }}-vpc"
+        network_mode: awsvpc
+        state: present
+        <<: *aws_connection_info
+      vars:
+        ecs_task_host_port: 8080
+      register: ecs_task_definition_vpc_with_host_port
+
+    - name: obtain ECS task definition facts
+      ecs_taskdefinition_facts:
+        task_definition: "{{ ecs_task_name }}-vpc:{{ ecs_task_definition_vpc_with_host_port.taskdefinition.revision }}"
+        <<: *aws_connection_info
+      register: ecs_taskdefinition_facts
+
+    - name: assert that network mode is awsvpc
+      assert:
+        that:
+          - "ecs_taskdefinition_facts.network_mode == 'awsvpc'"
+
+
+    - name: create ECS service definition with network configuration
+      ecs_service:
+        state: present
+        name: "{{ ecs_service_name }}2"
+        cluster: "{{ ecs_cluster_name }}"
+        task_definition: "{{ ecs_task_name }}-vpc:{{ ecs_task_definition_vpc_with_host_port.taskdefinition.revision }}"
+        desired_count: 1
+        deployment_configuration: "{{ ecs_service_deployment_configuration }}"
+        placement_strategy: "{{ ecs_service_placement_strategy }}"
+        load_balancers:
+          - targetGroupArn: "{{ elb_target_group_ip.target_group_arn }}"
+            containerName: "{{ ecs_task_name }}"
+            containerPort: "{{ ecs_task_container_port }}"
+        network_configuration:
+          subnets: "{{ setup_subnet.results | json_query('[].subnet.id') }}"
+          security_groups:
+            - '{{ setup_sg.group_id }}'
+        <<: *aws_connection_info
+      register: create_ecs_service_with_vpc
+
+    - name: assert that network configuration is correct
+      assert:
+        that:
+          - "'networkConfiguration' in create_ecs_service_with_vpc.service"
+          - "'awsvpcConfiguration' in create_ecs_service_with_vpc.service.networkConfiguration"
+          - "create_ecs_service_with_vpc.service.networkConfiguration.awsvpcConfiguration.subnets|length == 2"
+          - "create_ecs_service_with_vpc.service.networkConfiguration.awsvpcConfiguration.securityGroups|length == 1"
+
+    - name: create dummy group to update ECS service with
+      ec2_group:
+        name: "{{ resource_prefix }}-ecs-vpc-test-sg"
+        description: "Test security group for ECS with VPC"
+        vpc_id: '{{ setup_vpc.vpc.id }}'
+        state: present
+        <<: *aws_connection_info
+
+    - name: update ECS service definition with new network configuration
+      ecs_service:
+        state: present
+        name: "{{ ecs_service_name }}2"
+        cluster: "{{ ecs_cluster_name }}"
+        task_definition: "{{ ecs_task_name }}-vpc:{{ ecs_task_definition_vpc_with_host_port.taskdefinition.revision }}"
+        desired_count: 1
+        deployment_configuration: "{{ ecs_service_deployment_configuration }}"
+        placement_strategy: "{{ ecs_service_placement_strategy }}"
+        load_balancers:
+          - targetGroupArn: "{{ elb_target_group_ip.target_group_arn }}"
+            containerName: "{{ ecs_task_name }}"
+            containerPort: "{{ ecs_task_container_port }}"
+        network_configuration:
+          subnets: "{{ setup_subnet.results | json_query('[].subnet.id') }}"
+          security_groups:
+            - "{{ resource_prefix }}-ecs-vpc-test-sg"
+        <<: *aws_connection_info
+      register: update_ecs_service_with_vpc
+
+    - name: check that ECS service changed
+      assert:
+        that:
+          - update_ecs_service_with_vpc.changed
+          - "'networkConfiguration' in update_ecs_service_with_vpc.service"
+          - "'awsvpcConfiguration' in update_ecs_service_with_vpc.service.networkConfiguration"
+          - "update_ecs_service_with_vpc.service.networkConfiguration.awsvpcConfiguration.subnets|length == 2"
+          - "update_ecs_service_with_vpc.service.networkConfiguration.awsvpcConfiguration.securityGroups|length == 1"
+
+
     - name: obtain facts for all ECS services in the cluster
       ecs_service_facts:
         cluster: "{{ ecs_cluster_name }}"
@@ -297,38 +493,85 @@
         that:
           - "ecs_service_facts.services_not_running[0].reason == 'MISSING'"
 
+    - name: obtain specific ECS service facts
+      ecs_service_facts:
+        service: "{{ ecs_service_name }}2"
+        cluster: "{{ ecs_cluster_name }}"
+        details: yes
+        <<: *aws_connection_info
+      register: ecs_service_facts
+
+    - name: check that facts contain network configuration
+      assert:
+        that:
+          - "'networkConfiguration' in ecs_service_facts.ansible_facts.services[0]"
+
     - name: attempt to get facts from missing task definition
       ecs_taskdefinition_facts:
         task_definition: "{{ ecs_task_name }}-vpc:{{ ecs_task_definition.taskdefinition.revision + 1}}"
         <<: *aws_connection_info
 
   always:
-
     # TEAR DOWN: snapshot, ec2 instance, ec2 key pair, security group, vpc
     - name: Announce teardown start
       debug:
         msg: "***** TESTING COMPLETE. COMMENCE TEARDOWN *****"
 
+    - name: obtain ECS service facts
+      ecs_service_facts:
+        service: "{{ ecs_service_name }}"
+        cluster: "{{ ecs_cluster_name }}"
+        details: yes
+        <<: *aws_connection_info
+      register: ecs_service_facts
+
     - name: scale down ECS service
       ecs_service:
         state: present
         name: "{{ ecs_service_name }}"
         cluster: "{{ ecs_cluster_name }}"
-        task_definition: "{{ ecs_task_name }}:{{ ecs_task_definition.taskdefinition.revision }}"
+        task_definition: "{{ ecs_service_facts.ansible_facts.services[0].taskDefinition }}"
         desired_count: 0
         deployment_configuration: "{{ ecs_service_deployment_configuration }}"
         placement_strategy: "{{ ecs_service_placement_strategy }}"
         load_balancers:
-          - targetGroupArn: "{{ elb_target_group.target_group_arn }}"
+          - targetGroupArn: "{{ ecs_service_facts.ansible_facts.services[0].loadBalancers[0].targetGroupArn }}"
             containerName: "{{ ecs_task_name }}"
             containerPort: "{{ ecs_task_container_port }}"
-        role: "ecsServiceRole"
         <<: *aws_connection_info
       ignore_errors: yes
+      register: ecs_service_scale_down
 
-    - name: pause to allow service to scale down
+    - name: obtain second ECS service facts
+      ecs_service_facts:
+        service: "{{ ecs_service_name }}2"
+        cluster: "{{ ecs_cluster_name }}"
+        details: yes
+        <<: *aws_connection_info
+      ignore_errors: yes
+      register: ecs_service_facts
+
+    - name: scale down second ECS service
+      ecs_service:
+        state: present
+        name: "{{ ecs_service_name }}2"
+        cluster: "{{ ecs_cluster_name }}"
+        task_definition: "{{ ecs_service_facts.ansible_facts.services[0].taskDefinition }}"
+        desired_count: 0
+        deployment_configuration: "{{ ecs_service_deployment_configuration }}"
+        placement_strategy: "{{ ecs_service_placement_strategy }}"
+        load_balancers:
+          - targetGroupArn: "{{ ecs_service_facts.ansible_facts.services[0].loadBalancers[0].targetGroupArn }}"
+            containerName: "{{ ecs_task_name }}"
+            containerPort: "{{ ecs_task_container_port }}"
+        <<: *aws_connection_info
+      ignore_errors: yes
+      register: ecs_service_scale_down
+
+    - name: pause to allow services to scale down
       pause:
         seconds: 60
+      when: ecs_service_scale_down is not failed
 
     - name: remove ecs service
       ecs_service:
@@ -338,6 +581,14 @@
         <<: *aws_connection_info
       ignore_errors: yes
 
+    - name: remove second ecs service
+      ecs_service:
+        state: absent
+        cluster: "{{ ecs_cluster_name }}"
+        name: "{{ ecs_service_name }}2"
+        <<: *aws_connection_info
+      ignore_errors: yes
+
     - name: remove ecs task definition
       ecs_taskdefinition:
         containers: "{{ ecs_task_containers }}"
@@ -345,6 +596,19 @@
         revision: "{{ ecs_task_definition.taskdefinition.revision }}"
         state: absent
         <<: *aws_connection_info
+      vars:
+        ecs_task_host_port: 8080
+      ignore_errors: yes
+
+    - name: remove second ecs task definition
+      ecs_taskdefinition:
+        containers: "{{ ecs_task_containers }}"
+        family: "{{ ecs_task_name }}-vpc"
+        revision: "{{ ecs_task_definition_vpc_with_host_port.taskdefinition.revision }}"
+        state: absent
+        <<: *aws_connection_info
+      vars:
+        ecs_task_host_port: 8080
       ignore_errors: yes
 
     - name: remove load balancer
@@ -354,16 +618,21 @@
         wait: yes
         <<: *aws_connection_info
       ignore_errors: yes
+      register: elb_application_lb_remove
 
     - name: pause to allow target group to be disassociated
       pause:
         seconds: 30
+      when: not elb_application_lb_remove is failed
 
-    - name: remove target group
+    - name: remove target groups
       elb_target_group:
-        name: "{{ ecs_target_group_name }}"
+        name: "{{ item }}"
         state: absent
         <<: *aws_connection_info
+      with_items:
+        - "{{ ecs_target_group_name }}1"
+        - "{{ ecs_target_group_name }}2"
       ignore_errors: yes
 
     - name: remove setup ec2 instance
@@ -381,13 +650,16 @@
         <<: *aws_connection_info
       ignore_errors: yes
 
-    - name: remove setup security group
+    - name: remove security groups
       ec2_group:
-        name: '{{ resource_prefix }}_ecs_cluster-sg'
+        name: '{{ item }}'
         description: 'created by Ansible integration tests'
         state: absent
         vpc_id: '{{ setup_vpc.vpc.id }}'
         <<: *aws_connection_info
+      with_items:
+        - "{{ resource_prefix }}-ecs-vpc-test-sg"
+        - '{{ resource_prefix }}_ecs_cluster-sg'
       ignore_errors: yes
 
     - name: remove IGW

test/integration/targets/ecs_cluster/runme.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# We don't set -u here, due to pypa/virtualenv#150
+set -ex
+
+MYTMPDIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
+
+trap 'rm -rf "${MYTMPDIR}"' EXIT
+
+# This is needed for the ubuntu1604py3 tests
+# Ubuntu patches virtualenv to make the default python2
+# but for the python3 tests we need virtualenv to use python3
+PYTHON=${ANSIBLE_TEST_PYTHON_INTERPRETER:-python}
+
+# Test graceful failure for older versions of botocore
+virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/botocore-1.7.40"
+source "${MYTMPDIR}/botocore-1.7.40/bin/activate"
+$PYTHON -m pip install 'botocore<=1.7.40' boto3
+ansible-playbook -i ../../inventory -e @../../integration_config.yml -e @../../cloud-config-aws.yml -v playbooks/network_fail.yml "$@"
+
+# Run full test suite
+virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/botocore-recent"
+source "${MYTMPDIR}/botocore-recent/bin/activate"
+$PYTHON -m pip install 'botocore>=1.8.0' boto3
+ansible-playbook -i ../../inventory -e @../../integration_config.yml -e @../../cloud-config-aws.yml -v playbooks/full_test.yml "$@"