[aws] New module: elb_target_facts (#43565)

* initial source of the elb_target_facts module, which gathers all
registered ELBv2 target groups for a given instance id

* updating

* initial dump of tests

* updating docs, adding AWSRetry decorators

* updating tests

* More brush up, some refactoring

* updating for sanity tests

* removing extra space

* updating elb_target_facts to use a new name for the return value, not return a fact, and use instance variables for the AWS connections.  updating tests to go along with that

* updating classes to be 'new-style' classes

test/integration/targets/elb_target_facts/aliases

@@ -0,0 +1,2 @@
+cloud/aws
+unsupported

test/integration/targets/elb_target_facts/playbooks/full_test.yml

@@ -0,0 +1,5 @@
+- hosts: localhost
+  connection: local
+
+  roles:
+    - elb_target_facts

test/integration/targets/elb_target_facts/playbooks/roles/elb_target_facts/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+ec2_ami_image:
+  us-east-1: ami-8c1be5f6
+  us-east-2: ami-c5062ba0
+
+tg_name: "ansible-test-{{ resource_prefix | regex_search('([0-9]+)$') }}-tg"
+lb_name: "ansible-test-{{ resource_prefix | regex_search('([0-9]+)$') }}-lb"

test/integration/targets/elb_target_facts/playbooks/roles/elb_target_facts/tasks/main.yml

@@ -0,0 +1,532 @@
+---
+  - name: set up elb_target_facts test prerequisites
+
+    block:
+
+    - name:
+      debug: msg="********** Setting up elb_target_facts test dependencies **********"
+    # ============================================================
+
+    - name: set up aws connection info
+      set_fact:
+        aws_connection_info: &aws_connection_info
+          aws_access_key: "{{ aws_access_key }}"
+          aws_secret_key: "{{ aws_secret_key }}"
+          security_token: "{{ security_token }}"
+          region: "{{ aws_region }}"
+      no_log: yes
+
+    # ============================================================
+
+    - name: set up testing VPC
+      ec2_vpc_net:
+        name: "{{ resource_prefix }}-vpc"
+        state: present
+        cidr_block: 20.0.0.0/16
+        <<: *aws_connection_info
+        tags:
+          Name: "{{ resource_prefix }}-vpc"
+          Description: "Created by ansible-test"
+      register: vpc
+
+    - name: set up testing internet gateway
+      ec2_vpc_igw:
+        vpc_id: "{{ vpc.vpc.id }}"
+        state: present
+        <<: *aws_connection_info
+      register: igw
+
+    - name: set up testing subnet
+      ec2_vpc_subnet:
+        state: present
+        vpc_id: "{{ vpc.vpc.id }}"
+        cidr: 20.0.0.0/18
+        az: "{{ aws_region }}a"
+        resource_tags:
+          Name: "{{ resource_prefix }}-subnet"
+        <<: *aws_connection_info
+      register: subnet_1
+
+    - name: set up testing subnet
+      ec2_vpc_subnet:
+        state: present
+        vpc_id: "{{ vpc.vpc.id }}"
+        cidr: 20.0.64.0/18
+        az: "{{ aws_region }}b"
+        resource_tags:
+          Name: "{{ resource_prefix }}-subnet"
+        <<: *aws_connection_info
+      register: subnet_2
+
+    - name: create routing rules
+      ec2_vpc_route_table:
+        vpc_id: "{{ vpc.vpc.id }}"
+        tags:
+          created: "{{ resource_prefix }}-route"
+        routes:
+          - dest: 0.0.0.0/0
+            gateway_id: "{{ igw.gateway_id }}"
+        subnets:
+          - "{{ subnet_1.subnet.id }}"
+          - "{{ subnet_2.subnet.id }}"
+        <<: *aws_connection_info
+      register: route_table
+
+    - name: create testing security group
+      ec2_group:
+        name: "{{ resource_prefix }}-sg"
+        description: a security group for ansible tests
+        vpc_id: "{{ vpc.vpc.id }}"
+        rules:
+          - proto: tcp
+            from_port: 80
+            to_port: 80
+            cidr_ip: 0.0.0.0/0
+          - proto: tcp
+            from_port: 22
+            to_port: 22
+            cidr_ip: 0.0.0.0/0
+        <<: *aws_connection_info
+      register: sg
+
+    - name: set up testing target group (type=instance)
+      register: alb_target_group
+      elb_target_group:
+        name: "{{ tg_name }}-inst"
+        health_check_port: 80
+        protocol: http
+        port: 80
+        vpc_id: '{{ vpc.vpc.id }}'
+        state: present
+        target_type: instance
+        # set this to 30 to test polling for changes, instead of having everything go out immediately
+        deregistration_delay_timeout: 30
+        tags:
+          Description: "Created by {{ resource_prefix }}"
+        <<: *aws_connection_info
+
+    - name: set up testing target group (type=ip)
+      register: nlb_target_group
+      elb_target_group:
+        name: "{{ tg_name }}-ip"
+        health_check_port: 80
+        protocol: tcp
+        port: 80
+        vpc_id: '{{ vpc.vpc.id }}'
+        state: present
+        # set this to 30 to test polling for changes, instead of having everything go out immediately
+        deregistration_delay_timeout: 30
+        target_type: ip 
+        tags:
+          Description: "Created by {{ resource_prefix }}"
+        <<: *aws_connection_info
+
+    - name: set up testing target group which will not be associated with any load balancers
+      register: idle_target_group
+      elb_target_group:
+        name: "{{ tg_name }}-idle"
+        health_check_port: 80
+        protocol: tcp
+        port: 80
+        vpc_id: '{{ vpc.vpc.id }}'
+        state: present
+        target_type: instance 
+        tags:
+          Description: "Created by {{ resource_prefix }}"
+        <<: *aws_connection_info
+
+    - name: set up ec2 instance to use as a target
+      ec2:
+        group_id: "{{ sg.group_id }}"
+        instance_type: t2.micro
+        image: "{{ ec2_ami_image[aws_region] }}"
+        vpc_subnet_id: "{{ subnet_2.subnet.id }}"
+        instance_tags:
+          Name: "{{ resource_prefix }}-inst"
+        exact_count: 1
+        count_tag:
+          Name: "{{ resource_prefix }}-inst"
+        assign_public_ip: true
+        volumes: []
+        wait: true
+        ebs_optimized: false
+        user_data: |
+          #cloud-config
+          package_upgrade: true
+          package_update: true
+          packages:
+            - httpd
+          runcmd:
+            - "service httpd start"
+            - echo "HELLO ANSIBLE" > /var/www/html/index.html
+        <<: *aws_connection_info
+      register: ec2
+
+    - name: create an application load balancer
+      elb_application_lb:
+        name: "{{ lb_name }}-alb"
+        security_groups:
+          - "{{ sg.group_id }}"
+        subnets:
+          - "{{ subnet_1.subnet.id }}"
+          - "{{ subnet_2.subnet.id }}"
+        listeners:
+          - Protocol: HTTP
+            Port: 80
+            DefaultActions:
+              - Type: forward
+                TargetGroupName: "{{ tg_name }}-inst"
+        state: present
+        <<: *aws_connection_info
+
+
+    - name: create a network load balancer
+      elb_network_lb:
+        name: "{{ lb_name }}-nlb"
+        subnets:
+          - "{{ subnet_1.subnet.id }}"
+          - "{{ subnet_2.subnet.id }}"
+        listeners:
+          - Protocol: TCP
+            Port: 80
+            DefaultActions:
+              - Type: forward
+                TargetGroupName: "{{ tg_name }}-ip"
+        state: present
+        <<: *aws_connection_info
+ 
+    - name: register with the ALB
+      elb_target:
+        target_group_name: "{{ tg_name }}-inst"
+        target_id: "{{ ec2.instance_ids[0] }}"
+        state: present
+        target_status: "initial" 
+        <<: *aws_connection_info
+
+    - name: register with the NLB IP target group
+      elb_target:
+        target_group_name: "{{ tg_name }}-ip"
+        target_id: "{{ ec2.instances[0].private_ip }}" 
+        state: present
+        target_status: "initial"
+        <<: *aws_connection_info
+
+    # ============================================================
+
+    - debug: msg="********** Running elb_target_facts integration tests **********"
+
+    # ============================================================
+    - name: gather facts
+      elb_target_facts:
+        instance_id: "{{ ec2.instance_ids[0]}}"
+        <<: *aws_connection_info
+      register: target_facts
+
+    - assert:
+        that:
+          - "{{ alb_target_group.target_group_arn in (target_facts.instance_target_groups | map(attribute='target_group_arn')) }}"
+          - "{{ nlb_target_group.target_group_arn in (target_facts.instance_target_groups | map(attribute='target_group_arn')) }}"
+          - "{{ idle_target_group.target_group_arn not in (target_facts.instance_target_groups | map(attribute='target_group_arn')) }}"
+          - (target_facts.instance_target_groups | length) == 2
+        msg: "target facts showed the target in the right target groups"
+
+
+    - name: register with unused target group
+      elb_target:
+        target_group_name: "{{ tg_name }}-idle"
+        target_id: "{{ ec2.instance_ids[0]}}"
+        state: present
+        target_status: "unused"
+        <<: *aws_connection_info
+
+    - name: gather facts again, including the idle group
+      elb_target_facts:
+        instance_id: "{{ ec2.instance_ids[0]}}"
+        <<: *aws_connection_info
+      register: target_facts
+
+    - assert:
+        that:
+          - "{{ alb_target_group.target_group_arn in (target_facts.instance_target_groups | map(attribute='target_group_arn')) }}"
+          - "{{ nlb_target_group.target_group_arn in (target_facts.instance_target_groups | map(attribute='target_group_arn')) }}"
+          - "{{ idle_target_group.target_group_arn in (target_facts.instance_target_groups | map(attribute='target_group_arn')) }}"
+          - (target_facts.instance_target_groups | length) == 3
+        msg: "target facts reflected the addition of the target to the idle group"
+
+    - name: gather facts again, this time excluding the idle group
+      elb_target_facts:
+        instance_id: "{{ ec2.instance_ids[0]}}"
+        get_unused_target_groups: false
+        <<: *aws_connection_info
+      register: target_facts
+
+    - assert:
+        that:
+          - "{{ alb_target_group.target_group_arn in (target_facts.instance_target_groups | map(attribute='target_group_arn')) }}"
+          - "{{ nlb_target_group.target_group_arn in (target_facts.instance_target_groups | map(attribute='target_group_arn')) }}"
+          - "{{ idle_target_group.target_group_arn not in (target_facts.instance_target_groups | map(attribute='target_group_arn')) }}"
+          - (target_facts.instance_target_groups | length) == 2
+        msg: "target_facts.instance_target_groups did not gather unused target groups when variable was set"
+        
+    - name: register twice in the same target group
+      elb_target:
+        target_group_name: "{{ tg_name }}-ip"
+        target_port: 22
+        target_id: "{{ ec2.instances[0].private_ip }}" 
+        state: present
+        target_status: "healthy"
+        target_status_timeout: 300
+        <<: *aws_connection_info
+
+    - name: gather facts
+      elb_target_facts:
+        instance_id: "{{ ec2.instance_ids[0] }}"
+        get_unused_target_groups: false
+        <<: *aws_connection_info
+      register: target_facts
+
+    - assert:
+        that:
+          - alb_target_group.target_group_arn in (target_facts.instance_target_groups | map(attribute='target_group_arn'))
+          - nlb_target_group.target_group_arn in (target_facts.instance_target_groups | map(attribute='target_group_arn'))
+          - (target_facts.instance_target_groups | length) == 2 
+          - (target_facts.instance_target_groups |
+             selectattr('target_group_arn', 'equalto', nlb_target_group.target_group_arn) |
+             map(attribute='targets') |
+             flatten |
+             list |
+             length) == 2
+        msg: "registering a target twice didn't affect the overall target group count, increased target count"
+
+    - set_fact:
+        original_target_groups: "{{ target_facts.instance_target_groups }}"
+
+    - name: Deregister instance from all target groups
+      elb_target:
+          target_group_arn: "{{ item.0.target_group_arn }}"
+          target_port: "{{ item.1.target_port }}"
+          target_az: "{{ item.1.target_az }}"
+          target_id: "{{ item.1.target_id }}"
+          state: absent
+          target_status: "draining"
+          <<: *aws_connection_info
+      with_subelements:
+        - "{{ original_target_groups }}"
+        - "targets"
+
+    - name: wait for all targets to deregister simultaneously
+      elb_target_facts:
+        get_unused_target_groups: false
+        instance_id: "{{ ec2.instance_ids[0] }}"
+        <<: *aws_connection_info
+      register: target_facts
+      until: (target_facts.instance_target_groups | length) == 0
+      retries: 60
+      delay: 10
+
+    - name: reregister in elbv2s
+      elb_target:
+        target_group_arn: "{{ item.0.target_group_arn }}"
+        target_port: "{{ item.1.target_port }}"
+        target_az: "{{ item.1.target_az }}"
+        target_id: "{{ item.1.target_id }}"
+        state: present
+        target_status: "initial"
+        <<: *aws_connection_info
+      with_subelements:
+        - "{{ original_target_groups }}"
+        - "targets"
+
+    # wait until all groups associated with this instance are 'healthy' or
+    # 'unused'
+    - name: wait for registration
+      elb_target_facts:
+        get_unused_target_groups: false
+        instance_id: "{{ ec2.instance_ids[0] }}"
+        <<: *aws_connection_info
+      register: target_facts
+      until: >
+                (target_facts.instance_target_groups |
+                 map(attribute='targets') |
+                 flatten |
+                 map(attribute='target_health') |
+                 rejectattr('state', 'equalto', 'healthy') |
+                 rejectattr('state', 'equalto', 'unused') |
+                 list |
+                 length) == 0
+      retries: 61
+      delay: 10
+
+    - assert:
+        that:
+          - alb_target_group.target_group_arn in (target_facts.instance_target_groups | map(attribute='target_group_arn'))
+          - nlb_target_group.target_group_arn in (target_facts.instance_target_groups | map(attribute='target_group_arn'))
+          - (target_facts.instance_target_groups | length) == 2 
+          - (target_facts.instance_target_groups |
+             selectattr('target_group_arn', 'equalto', nlb_target_group.target_group_arn) |
+             map(attribute='targets') |
+             flatten |
+             list |
+             length) == 2
+      msg: "reregistration completed successfully"
+
+    always:
+
+    - name:
+      debug: msg="********** Tearing down elb_target_facts test dependencies **********"
+
+    - name: remove ec2 instance
+      ec2:
+        group_id: "{{ sg.group_id }}"
+        instance_type: t2.micro
+        image: "{{ ec2_ami_image[aws_region] }}"
+        vpc_subnet_id: "{{ subnet_2.subnet.id }}"
+        instance_tags:
+          Name: "{{ resource_prefix }}-inst"
+        exact_count: 0
+        count_tag:
+          Name: "{{ resource_prefix }}-inst"
+        assign_public_ip: true
+        volumes: []
+        wait: true
+        ebs_optimized: false
+        <<: *aws_connection_info
+      ignore_errors: true
+
+    - name: remove application load balancer
+      elb_application_lb:
+        name: "{{ lb_name }}-alb"
+        security_groups:
+          - "{{ sg.group_id }}"
+        subnets:
+          - "{{ subnet_1.subnet.id }}"
+          - "{{ subnet_2.subnet.id }}"
+        listeners:
+          - Protocol: HTTP
+            Port: 80
+            DefaultActions:
+              - Type: forward
+                TargetGroupName: "{{ tg_name }}-inst"
+        state: absent
+        wait: true
+        wait_timeout: 200
+        <<: *aws_connection_info
+      register: removed
+      retries: 10
+      until: removed is not failed
+      ignore_errors: true
+
+    - name: remove NLB
+      ignore_errors: true
+      elb_network_lb:
+        name: "{{ lb_name }}-nlb"
+        state: absent
+        <<: *aws_connection_info
+
+    - name: remove testing target groups
+      elb_target_group:
+        name: "{{ item }}"
+        health_check_port: 80
+        protocol: http
+        port: 80
+        vpc_id: '{{ vpc.vpc.id }}'
+        state: absent
+        target_type: instance
+        tags:
+          Description: "Created by {{ resource_prefix }}"
+        wait: true
+        wait_timeout: 200
+        <<: *aws_connection_info
+      register: removed
+      retries: 10
+      until: removed is not failed
+      with_items:
+        - "{{ tg_name }}-idle"
+        - "{{ tg_name }}-ip"
+        - "{{ tg_name }}-inst"
+      ignore_errors: true
+
+    - name: remove testing security group
+      ec2_group:
+        state: absent
+        name: "{{ resource_prefix }}-sg"
+        description: a security group for ansible tests
+        vpc_id: "{{ vpc.vpc.id }}"
+        rules:
+          - proto: tcp
+            from_port: 80
+            to_port: 80
+            cidr_ip: 0.0.0.0/0
+          - proto: tcp
+            from_port: 22
+            to_port: 22
+            cidr_ip: 0.0.0.0/0
+        <<: *aws_connection_info
+      register: removed
+      retries: 10
+      until: removed is not failed
+      ignore_errors: true
+
+    - name: remove routing rules
+      ec2_vpc_route_table:
+        state: absent
+        lookup: id
+        route_table_id: "{{ route_table.route_table.id }}"
+        <<: *aws_connection_info
+      register: removed
+      retries: 10
+      until: removed is not failed
+      ignore_errors: true
+
+    - name: remove testing subnet
+      ec2_vpc_subnet:
+        state: absent
+        vpc_id: "{{ vpc.vpc.id }}"
+        cidr: 20.0.0.0/18
+        az: "{{ aws_region }}a"
+        resource_tags:
+          Name: "{{ resource_prefix }}-subnet"
+        <<: *aws_connection_info
+      register: removed
+      retries: 10
+      until: removed is not failed
+      ignore_errors: true
+
+    - name: remove testing subnet
+      ec2_vpc_subnet:
+        state: absent
+        vpc_id: "{{ vpc.vpc.id }}"
+        cidr: 20.0.64.0/18
+        az: "{{ aws_region }}b"
+        resource_tags:
+          Name: "{{ resource_prefix }}-subnet"
+        <<: *aws_connection_info
+      register: removed
+      retries: 10
+      until: removed is not failed
+      ignore_errors: true
+
+    - name: remove testing internet gateway
+      ec2_vpc_igw:
+        vpc_id: "{{ vpc.vpc.id }}"
+        state: absent
+        <<: *aws_connection_info
+      register: removed
+      retries: 10
+      until: removed is not failed
+      ignore_errors: true
+
+    - name: remove testing VPC
+      ec2_vpc_net:
+        name: "{{ resource_prefix }}-vpc"
+        state: absent
+        cidr_block: 20.0.0.0/16
+        tags:
+          Name: "{{ resource_prefix }}-vpc"
+          Description: "Created by ansible-test"
+        <<: *aws_connection_info
+      register: removed
+      retries: 10
+      until: removed is not failed
+
+    # ============================================================

test/integration/targets/elb_target_facts/runme.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# We don't set -u here, due to pypa/virtualenv#150
+set -ex
+
+ansible-playbook -i ../../inventory -e @../../integration_config.yml -e @../../cloud-config-aws.yml -v playbooks/full_test.yml "$@"