[aws] Create classes for Application Load Balancer (#33769)

* Create classes for Application Load Balancer
* Add unsupported CI alias
* Add AWSRetry
* Add integration tests using the ALB

test/integration/targets/elb_application_lb/aliases

@@ -0,0 +1,2 @@
+cloud/aws
+unsupported

test/integration/targets/elb_application_lb/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# load balancer and target group names have to be less than 32 characters
+# the 8 digit identifier at the end of resource_prefix helps determine during which test something
+# was created and allows tests to be run in parallel
+alb_name: "my-alb-{{ resource_prefix | regex_search('([0-9]+)$') }}"
+tg_name: "my-tg-{{ resource_prefix | regex_search('([0-9]+)$') }}"

test/integration/targets/elb_application_lb/meta/main.yml

@@ -0,0 +1,3 @@
+dependencies:
+  - prepare_tests
+  - setup_ec2

test/integration/targets/elb_application_lb/tasks/main.yml

@@ -0,0 +1,204 @@
+- block:
+
+  - name: set connection information for all tasks
+    set_fact:
+      aws_connection_info: &aws_connection_info
+        aws_access_key: "{{ aws_access_key }}"
+        aws_secret_key: "{{ aws_secret_key }}"
+        security_token: "{{ security_token }}"
+        region: "{{ aws_region }}"
+    no_log: yes
+
+  - name: create VPC
+    ec2_vpc_net:
+      cidr_block: 10.228.228.0/22
+      name: "{{ resource_prefix }}_vpc"
+      state: present
+      <<: *aws_connection_info
+    register: vpc
+
+  - name: create internet gateway
+    ec2_vpc_igw:
+      vpc_id: "{{ vpc.vpc.id }}"
+      state: present
+      tags:
+        Name: "{{ resource_prefix }}"
+      <<: *aws_connection_info
+    register: igw
+
+  - name: create public subnet
+    ec2_vpc_subnet:
+      cidr: "{{ item.cidr }}"
+      az: "{{ aws_region}}{{ item.az }}"
+      vpc_id: "{{ vpc.vpc.id }}"
+      state: present
+      tags:
+        Public: "{{ item.public|string }}"
+        Name: "{{ item.public|ternary('public', 'private') }}-{{ item.az }}"
+      <<: *aws_connection_info
+    with_items:
+      - cidr: 10.228.228.0/24
+        az: "a"
+        public: "True"
+      - cidr: 10.228.229.0/24
+        az: "b"
+        public: "True"
+      - cidr: 10.228.230.0/24
+        az: "a"
+        public: "False"
+      - cidr: 10.228.231.0/24
+        az: "b"
+        public: "False"
+    register: subnets
+
+  - ec2_vpc_subnet_facts:
+      filters:
+        vpc-id: "{{ vpc.vpc.id }}"
+      <<: *aws_connection_info
+    register: vpc_subnets
+
+  - name: create list of subnet ids
+    set_fact:
+      alb_subnets: "{{ vpc_subnets|json_query('subnets[?tags.Public == `True`].id') }}"
+      private_subnets: "{{ vpc_subnets|json_query('subnets[?tags.Public != `True`].id') }}"
+
+  - name: create a route table
+    ec2_vpc_route_table:
+      vpc_id: "{{ vpc.vpc.id }}"
+      <<: *aws_connection_info
+      tags:
+        Name: igw-route
+        Created: "{{ resource_prefix }}"
+      subnets: "{{ alb_subnets + private_subnets }}"
+      routes:
+        - dest: 0.0.0.0/0
+          gateway_id: "{{ igw.gateway_id }}"
+    register: route_table
+
+  - ec2_group:
+      name: "{{ resource_prefix }}"
+      description: "security group for Ansible ALB integration tests"
+      state: present
+      vpc_id: "{{ vpc.vpc.id }}"
+      rules:
+        - proto: tcp
+          from_port: 1
+          to_port: 65535
+          cidr_ip: 0.0.0.0/0
+      <<: *aws_connection_info
+    register: sec_group
+
+  - name: create a target group for testing
+    elb_target_group:
+      name: "{{ tg_name }}"
+      protocol: http
+      port: 80
+      vpc_id: "{{ vpc.vpc.id }}"
+      state: present
+      <<: *aws_connection_info
+    register: tg
+
+  - include_tasks: test_alb_bad_listener_options.yml
+  - include_tasks: test_alb_tags.yml
+  - include_tasks: test_creating_alb.yml
+  - include_tasks: test_alb_with_asg.yml
+  - include_tasks: test_modifying_alb_listeners.yml
+  - include_tasks: test_deleting_alb.yml
+
+  always:
+  #############################################################################
+  # TEAR DOWN STARTS HERE
+  #############################################################################
+    - name: destroy ALB
+      elb_application_lb:
+        name: "{{ alb_name }}"
+        state: absent
+        wait: yes
+        wait_timeout: 600
+        <<: *aws_connection_info
+      ignore_errors: yes
+
+    - name: destroy target group if it was created
+      elb_target_group:
+        name: "{{ tg_name }}"
+        protocol: http
+        port: 80
+        vpc_id: "{{ vpc.vpc.id }}"
+        state: absent
+        wait: yes
+        wait_timeout: 600
+        <<: *aws_connection_info
+      register: remove_tg
+      retries: 5
+      delay: 3
+      until: remove_tg is success
+      when: tg is defined
+      ignore_errors: yes
+
+    - name: destroy sec group
+      ec2_group:
+        name: "{{ sec_group.group_name }}"
+        description: "security group for Ansible ALB integration tests"
+        state: absent
+        vpc_id: "{{ vpc.vpc.id }}"
+        <<: *aws_connection_info
+      register: remove_sg
+      retries: 10
+      delay: 5
+      until: remove_sg is success
+      ignore_errors: yes
+
+    - name: remove route table
+      ec2_vpc_route_table:
+        vpc_id: "{{ vpc.vpc.id }}"
+        route_table_id: "{{ route_table.route_table.route_table_id }}"
+        lookup: id
+        state: absent
+        <<: *aws_connection_info
+      register: remove_rt
+      retries: 10
+      delay: 5
+      until: remove_rt is success
+      ignore_errors: yes
+
+    - name: destroy subnets
+      ec2_vpc_subnet:
+        cidr: "{{ item.cidr }}"
+        vpc_id: "{{ vpc.vpc.id }}"
+        state: absent
+        <<: *aws_connection_info
+      register: remove_subnet
+      retries: 10
+      delay: 5
+      until: remove_subnet is success
+      with_items:
+        - cidr: 10.228.228.0/24
+        - cidr: 10.228.229.0/24
+        - cidr: 10.228.230.0/24
+        - cidr: 10.228.231.0/24
+      ignore_errors: yes
+
+    - name: destroy internet gateway
+      ec2_vpc_igw:
+        vpc_id: "{{ vpc.vpc.id }}"
+        tags:
+          Name: "{{ resource_prefix }}"
+        state: absent
+        <<: *aws_connection_info
+      register: remove_igw
+      retries: 10
+      delay: 5
+      until: remove_igw is success
+      ignore_errors: yes
+
+    - name: destroy VPC
+      ec2_vpc_net:
+        cidr_block: 10.228.228.0/22
+        name: "{{ resource_prefix }}_vpc"
+        state: absent
+        <<: *aws_connection_info
+      register: remove_vpc
+      retries: 10
+      delay: 5
+      until: remove_vpc is success
+      ignore_errors: yes

test/integration/targets/elb_application_lb/tasks/test_alb_bad_listener_options.yml

@@ -0,0 +1,71 @@
+- block:
+
+  - name: set connection information for all tasks
+    set_fact:
+      aws_connection_info: &aws_connection_info
+        aws_access_key: "{{ aws_access_key }}"
+        aws_secret_key: "{{ aws_secret_key }}"
+        security_token: "{{ security_token }}"
+        region: "{{ aws_region }}"
+    no_log: yes
+
+  - name: test creating an ALB with invalid listener options
+    elb_application_lb:
+      name: "{{ alb_name }}"
+      subnets: "{{ alb_subnets }}"
+      security_groups: "{{ sec_group.group_id }}"
+      state: present
+      listeners:
+        - Protocol: HTTPS
+          Port: 80
+          DefaultActions:
+            - Type: forward
+              TargetGroupName: "{{ tg_name }}"
+      <<: *aws_connection_info
+    ignore_errors: yes
+    register: alb
+
+  - assert:
+      that:
+        - alb is failed
+        - alb.msg.startswith("'SslPolicy' is a required listener dict key when Protocol = HTTPS")
+
+  - name: test creating an ALB without providing required listener options
+    elb_application_lb:
+      name: "{{ alb_name }}"
+      subnets: "{{ alb_subnets }}"
+      security_groups: "{{ sec_group.group_id }}"
+      state: present
+      listeners:
+        - Port: 80
+      <<: *aws_connection_info
+    ignore_errors: yes
+    register: alb
+
+  - assert:
+      that:
+        - alb is failed
+        - '"missing required arguments" in alb.msg'
+        - '"Protocol" in alb.msg'
+        - '"DefaultActions" in alb.msg'
+
+  - name: test creating an ALB providing an invalid listener option type
+    elb_application_lb:
+      name: "{{ alb_name }}"
+      subnets: "{{ alb_subnets }}"
+      security_groups: "{{ sec_group.group_id }}"
+      state: present
+      listeners:
+        - Protocol: HTTP
+          Port: "bad type"
+          DefaultActions:
+            - Type: forward
+              TargetGroupName: "{{ tg_name }}"
+      <<: *aws_connection_info
+    ignore_errors: yes
+    register: alb
+
+  - assert:
+      that:
+        - alb is failed
+        - "'unable to convert to int' in alb.msg"

test/integration/targets/elb_application_lb/tasks/test_alb_tags.yml

@@ -0,0 +1,93 @@
+- block:
+
+  - name: set connection information for all tasks
+    set_fact:
+      aws_connection_info: &aws_connection_info
+        aws_access_key: "{{ aws_access_key }}"
+        aws_secret_key: "{{ aws_secret_key }}"
+        security_token: "{{ security_token }}"
+        region: "{{ aws_region }}"
+    no_log: yes
+
+  - name: create ALB with no listeners
+    elb_application_lb:
+      name: "{{ alb_name }}"
+      subnets: "{{ alb_subnets }}"
+      security_groups: "{{ sec_group.group_id }}"
+      state: present
+      <<: *aws_connection_info
+    register: alb
+
+  - assert:
+      that:
+        - alb.changed
+
+  - name: re-create ALB with no listeners
+    elb_application_lb:
+      name: "{{ alb_name }}"
+      subnets: "{{ alb_subnets }}"
+      security_groups: "{{ sec_group.group_id }}"
+      state: present
+      <<: *aws_connection_info
+    register: alb
+
+  - assert:
+      that:
+        - not alb.changed
+
+  - name: add tags to ALB
+    elb_application_lb:
+      name: "{{ alb_name }}"
+      subnets: "{{ alb_subnets }}"
+      security_groups: "{{ sec_group.group_id }}"
+      state: present
+      tags:
+        created_by: "ALB test {{ resource_prefix }}"
+      <<: *aws_connection_info
+    register: alb
+
+  - assert:
+      that:
+        - alb.changed
+        - 'alb.tags == {"created_by": "ALB test {{ resource_prefix }}"}'
+
+  - name: remove tags from ALB
+    elb_application_lb:
+      name: "{{ alb_name }}"
+      subnets: "{{ alb_subnets }}"
+      security_groups: "{{ sec_group.group_id }}"
+      state: present
+      tags: {}
+      <<: *aws_connection_info
+    register: alb
+
+  - assert:
+      that:
+        - alb.changed
+        - not alb.tags
+
+  - name: test idempotence
+    elb_application_lb:
+      name: "{{ alb_name }}"
+      subnets: "{{ alb_subnets }}"
+      security_groups: "{{ sec_group.group_id }}"
+      state: present
+      tags: {}
+      <<: *aws_connection_info
+    register: alb
+
+  - assert:
+      that:
+        - not alb.changed
+        - not alb.tags
+
+  - name: destroy ALB with no listeners
+    elb_application_lb:
+      name: "{{ alb_name }}"
+      state: absent
+      <<: *aws_connection_info
+    register: alb
+
+  - assert:
+      that:
+        - alb.changed

test/integration/targets/elb_application_lb/tasks/test_alb_with_asg.yml

@@ -0,0 +1,88 @@
+- block:
+
+    - name: set connection information for all tasks
+      set_fact:
+        aws_connection_info: &aws_connection_info
+          aws_access_key: "{{ aws_access_key }}"
+          aws_secret_key: "{{ aws_secret_key }}"
+          security_token: "{{ security_token }}"
+          region: "{{ aws_region }}"
+      no_log: yes
+
+    - ec2_ami_facts:
+        <<: *aws_connection_info
+        filters:
+          architecture: x86_64
+          virtualization-type: hvm
+          root-device-type: ebs
+          name: "amzn-ami-hvm*"
+      register: amis
+
+    - set_fact:
+        latest_amazon_linux: "{{ amis.images | sort(attribute='creation_date') | last }}"
+
+    - ec2_asg:
+        <<: *aws_connection_info
+        state: absent
+        name: "{{ resource_prefix }}-webservers"
+        wait_timeout: 900
+
+    - ec2_lc:
+        <<: *aws_connection_info
+        name: "{{ resource_prefix }}-web-lcfg"
+        state: absent
+
+    - name: Create launch config for testing
+      ec2_lc:
+        <<: *aws_connection_info
+        name: "{{ resource_prefix }}-web-lcfg"
+        assign_public_ip: true
+        image_id: "{{ latest_amazon_linux.image_id }}"
+        security_groups: "{{ sec_group.group_id }}"
+        instance_type: t2.medium
+        user_data: |
+          #!/bin/bash
+          set -x
+          yum update -y --nogpgcheck
+          yum install -y --nogpgcheck httpd
+          echo "Hello Ansiblings!" >> /var/www/html/index.html
+          service httpd start
+        volumes:
+          - device_name: /dev/xvda
+            volume_size: 10
+            volume_type: gp2
+            delete_on_termination: true
+
+    - name: Create autoscaling group for app server fleet
+      ec2_asg:
+        <<: *aws_connection_info
+        name: "{{ resource_prefix }}-webservers"
+        vpc_zone_identifier: "{{ alb_subnets }}"
+        launch_config_name: "{{ resource_prefix }}-web-lcfg"
+        termination_policies:
+        - OldestLaunchConfiguration
+        - Default
+        health_check_period: 600
+        health_check_type: EC2
+        replace_all_instances: true
+        min_size: 0
+        max_size: 2
+        desired_capacity: 1
+        wait_for_instances: true
+        target_group_arns:
+        - "{{ tg.target_group_arn }}"
+
+  always:
+
+    - ec2_asg:
+        <<: *aws_connection_info
+        state: absent
+        name: "{{ resource_prefix }}-webservers"
+        wait_timeout: 900
+      ignore_errors: yes
+
+    - ec2_lc:
+        <<: *aws_connection_info
+        name: "{{ resource_prefix }}-web-lcfg"
+        state: absent
+      ignore_errors: yes

test/integration/targets/elb_application_lb/tasks/test_creating_alb.yml

@@ -0,0 +1,52 @@
+- block:
+
+  - name: set connection information for all tasks
+    set_fact:
+      aws_connection_info: &aws_connection_info
+        aws_access_key: "{{ aws_access_key }}"
+        aws_secret_key: "{{ aws_secret_key }}"
+        security_token: "{{ security_token }}"
+        region: "{{ aws_region }}"
+    no_log: yes
+
+  - name: create ALB with a listener
+    elb_application_lb:
+      name: "{{ alb_name }}"
+      subnets: "{{ alb_subnets }}"
+      security_groups: "{{ sec_group.group_id }}"
+      state: present
+      listeners:
+        - Protocol: HTTP
+          Port: 80
+          DefaultActions:
+            - Type: forward
+              TargetGroupName: "{{ tg_name }}"
+      <<: *aws_connection_info
+    register: alb
+
+  - assert:
+      that:
+        - alb.changed
+        - alb.listeners|length == 1
+        - alb.listeners[0].rules|length == 1
+
+  - name: test idempotence creating ALB with a listener
+    elb_application_lb:
+      name: "{{ alb_name }}"
+      subnets: "{{ alb_subnets }}"
+      security_groups: "{{ sec_group.group_id }}"
+      state: present
+      listeners:
+        - Protocol: HTTP
+          Port: 80
+          DefaultActions:
+            - Type: forward
+              TargetGroupName: "{{ tg_name }}"
+      <<: *aws_connection_info
+    register: alb
+
+  - assert:
+      that:
+        - not alb.changed
+        - alb.listeners|length == 1
+        - alb.listeners[0].rules|length == 1

test/integration/targets/elb_application_lb/tasks/test_deleting_alb.yml

@@ -0,0 +1,52 @@
+- block:
+
+  - name: set connection information for all tasks
+    set_fact:
+      aws_connection_info: &aws_connection_info
+        aws_access_key: "{{ aws_access_key }}"
+        aws_secret_key: "{{ aws_secret_key }}"
+        security_token: "{{ security_token }}"
+        region: "{{ aws_region }}"
+    no_log: yes
+
+  - name: destroy ALB with listener
+    elb_application_lb:
+      name: "{{ alb_name }}"
+      subnets: "{{ alb_subnets }}"
+      security_groups: "{{ sec_group.group_id }}"
+      state: absent
+      listeners:
+        - Protocol: HTTP
+          Port: 80
+          DefaultActions:
+            - Type: forward
+              TargetGroupName: "{{ tg_name }}"
+      <<: *aws_connection_info
+      wait: yes
+      wait_timeout: 300
+    register: alb
+
+  - assert:
+      that:
+        - alb.changed
+
+  - name: test idempotence
+    elb_application_lb:
+      name: "{{ alb_name }}"
+      subnets: "{{ alb_subnets }}"
+      security_groups: "{{ sec_group.group_id }}"
+      state: absent
+      listeners:
+        - Protocol: HTTP
+          Port: 80
+          DefaultActions:
+            - Type: forward
+              TargetGroupName: "{{ tg_name }}"
+      <<: *aws_connection_info
+      wait: yes
+      wait_timeout: 300
+    register: alb
+
+  - assert:
+      that:
+        - not alb.changed

test/sanity/validate-modules/ignore.txt

@@ -92,9 +92,7 @@ lib/ansible/modules/cloud/amazon/elasticache.py E324
 lib/ansible/modules/cloud/amazon/elasticache.py E326
 lib/ansible/modules/cloud/amazon/elasticache_parameter_group.py E326
 lib/ansible/modules/cloud/amazon/elasticache_subnet_group.py E324
-lib/ansible/modules/cloud/amazon/elb_application_lb.py E322
 lib/ansible/modules/cloud/amazon/elb_application_lb.py E324
-lib/ansible/modules/cloud/amazon/elb_application_lb.py E325
 lib/ansible/modules/cloud/amazon/elb_classic_lb_facts.py E323
 lib/ansible/modules/cloud/amazon/elb_instance.py E326
 lib/ansible/modules/cloud/amazon/elb_target.py E327

test/units/modules/cloud/amazon/test_elb_application_lb.py

@@ -1,154 +0,0 @@
-#
-# (c) 2017 Michael Tinning
-#
-# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-
-from __future__ import (absolute_import, division, print_function)
-
-import json
-from copy import deepcopy
-
-import pytest
-
-from ansible.module_utils._text import to_bytes
-from ansible.module_utils import basic
-from ansible.module_utils.ec2 import HAS_BOTO3
-
-if not HAS_BOTO3:
-    pytestmark = pytest.mark.skip("test_elb_application_lb.py requires the `boto3` and `botocore` modules")
-
-import ansible.modules.cloud.amazon.elb_application_lb as elb_module
-
-
-@pytest.fixture
-def listener():
-    return {
-        'Protocol': 'HTTP',
-        'Port': 80,
-        'DefaultActions': [{
-            'Type': 'forward',
-            'TargetGroupName': 'target-group'
-        }],
-        'Rules': [{
-            'Conditions': [{
-                'Field': 'host-header',
-                'Values': [
-                    'www.example.com'
-                ]
-            }],
-            'Priority': 1,
-            'Actions': [{
-                'TargetGroupName': 'other-target-group',
-                'Type': 'forward'
-            }]
-        }]
-    }
-
-
-@pytest.fixture
-def compare_listeners(mocker):
-    return mocker.Mock()
-
-
-@pytest.fixture
-def ensure_listeners(mocker):
-    ensure_listeners_mock = mocker.Mock()
-    ensure_listeners_mock.return_value = []
-    return ensure_listeners_mock
-
-
-@pytest.fixture
-def compare_rules(mocker):
-    compare_rules_mock = mocker.Mock()
-    compare_rules_mock.return_value = ([], [], [])
-    return compare_rules_mock
-
-
-@pytest.fixture
-def get_elb_listeners(mocker):
-    get_elb_listeners_mock = mocker.Mock()
-    get_elb_listeners_mock.return_value = []
-    return get_elb_listeners_mock
-
-
-@pytest.fixture
-def elb(mocker, monkeypatch, compare_listeners, ensure_listeners, compare_rules, get_elb_listeners):
-    monkeypatch.setattr(elb_module, "ensure_listeners_default_action_has_arn", ensure_listeners)
-    monkeypatch.setattr(elb_module, "get_elb_listeners", get_elb_listeners)
-    monkeypatch.setattr(elb_module, "ensure_rules_action_has_arn", mocker.Mock())
-    monkeypatch.setattr(elb_module, "get_listener", mocker.Mock())
-    monkeypatch.setattr(elb_module, "compare_rules", compare_rules)
-    monkeypatch.setattr(elb_module, "compare_listeners", compare_listeners)
-    return elb_module
-
-
-@pytest.fixture
-def created_listener(mocker, listener):
-    return {
-        'Port': listener['Port'],
-        'ListenerArn': 'new-listener-arn'
-    }
-
-
-@pytest.fixture
-def connection(mocker, created_listener):
-    connection_mock = mocker.Mock()
-    connection_mock.create_listener.return_value = {
-        'Listeners': [created_listener]
-    }
-    return connection_mock
-
-
-@pytest.fixture
-def existing_elb():
-    return {'LoadBalancerArn': 'fake'}
-
-
-def test_create_listeners_called_with_correct_args(mocker, connection, listener, elb, compare_listeners, existing_elb):
-    compare_listeners.return_value = ([listener], [], [])
-
-    elb.create_or_update_elb_listeners(connection, mocker.Mock(), existing_elb)
-
-    connection.create_listener.assert_called_once_with(
-        Protocol=listener['Protocol'],
-        Port=listener['Port'],
-        DefaultActions=listener['DefaultActions'],
-        LoadBalancerArn=existing_elb['LoadBalancerArn']
-    )
-
-
-def test_modify_listeners_called_with_correct_args(mocker, connection, listener, elb, compare_listeners, existing_elb):
-    # In the case of modify listener, LoadBalancerArn is set in compare_listeners
-    listener['LoadBalancerArn'] = existing_elb['LoadBalancerArn']
-    compare_listeners.return_value = ([], [listener], [])
-
-    elb.create_or_update_elb_listeners(connection, mocker.Mock(), existing_elb)
-
-    connection.modify_listener.assert_called_once_with(
-        Protocol=listener['Protocol'],
-        Port=listener['Port'],
-        DefaultActions=listener['DefaultActions'],
-        LoadBalancerArn=existing_elb['LoadBalancerArn']
-    )
-
-
-def test_compare_rules_called_with_new_listener(
-    mocker,
-    connection,
-    listener,
-    elb,
-    compare_listeners,
-    ensure_listeners,
-    compare_rules,
-    existing_elb,
-    created_listener
-):
-    compare_listeners.return_value = ([listener], [], [])
-    listener_from_ensure_listeners = deepcopy(listener)
-    ensure_listeners.return_value = [listener_from_ensure_listeners]
-
-    elb.create_or_update_elb_listeners(connection, mocker.Mock(), existing_elb)
-
-    (_conn, _module, current_listeners, _listener), _kwargs = compare_rules.call_args
-
-    assert created_listener in current_listeners