ansible-collections/community.general
1
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2025-07-02 06:30:19 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

[aws]Add VPC configuration to ECS modules (#34381)

Browse source 
Enable awsvpc network mode for ECS services and tasks and
their underlying task definitions

Improve test suite to thoroughly test the changes

Use runme.sh technique to run old and new versions of botocore to
ensure that the modules work with older botocore and older network modes
and fail gracefully if awsvpc network mode is used with older botocore
This commit is contained in:
Will Thames 2018-04-26 05:41:04 +10:00 • committed by Ryan Brown
parent 58bf4ae611
commit 12f2b9506d
12 changed files with 639 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

122
lib/ansible/modules/cloud/amazon/ecs_service.py
View file

@ -70,7 +70,7 @@ options:
role:
description:
- The name or full Amazon Resource Name (ARN) of the IAM role that allows your Amazon ECS container agent to make calls to your load balancer
on your behalf. This parameter is only required if you are using a load balancer with your service.
on your behalf. This parameter is only required if you are using a load balancer with your service, in a network mode other than `awsvpc`.
required: false
delay:
description:
@ -97,6 +97,12 @@ options:
- The placement strategy objects to use for tasks in your service. You can specify a maximum of 5 strategy rules per service
required: false
version_added: 2.4
network_configuration:
description:
- network configuration of the service. Only applicable for task definitions created with C(awsvpc) I(network_mode).
- I(network_configuration) has two keys, I(subnets), a list of subnet IDs to which the task is attached and I(security_groups),
a list of group names or group IDs for the task
version_added: 2.6
extends_documentation_fragment:
- aws
- ec2
@ -117,6 +123,20 @@ EXAMPLES = '''
state: present
cluster: new_cluster
- name: create ECS service on VPC network
ecs_service:
state: present
name: console-test-service
cluster: new_cluster
task_definition: 'new_cluster-task:1'
desired_count: 0
network_configuration:
subnets:
- subnet-abcd1234
security_groups:
- sg-aaaa1111
- my_security_group
# Simple example to delete
- ecs_service:
name: default
@ -265,15 +285,14 @@ DEPLOYMENT_CONFIGURATION_TYPE_MAP = {
'minimum_healthy_percent': 'int'
}
from ansible.module_utils.aws.core import AnsibleAWSModule
from ansible.module_utils.ec2 import ec2_argument_spec
from ansible.module_utils.ec2 import snake_dict_to_camel_dict, map_complex_type, get_ec2_security_group_ids_from_names
try:
import botocore
HAS_BOTO3 = True
except ImportError:
HAS_BOTO3 = False
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.ec2 import boto3_conn, ec2_argument_spec, get_aws_connection_info, snake_dict_to_camel_dict, map_complex_type
pass # handled by AnsibleAWSModule
class EcsServiceManager:
@ -281,9 +300,25 @@ class EcsServiceManager:
def __init__(self, module):
self.module = module
self.ecs = module.client('ecs')
self.ec2 = module.client('ec2')
region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True)
self.ecs = boto3_conn(module, conn_type='client', resource='ecs', region=region, endpoint=ec2_url, **aws_connect_kwargs)
def format_network_configuration(self, network_config):
result = dict()
if 'subnets' in network_config:
result['subnets'] = network_config['subnets']
else:
self.module.fail_json(msg="Network configuration must include subnets")
if 'security_groups' in network_config:
groups = network_config['security_groups']
if any(not sg.startswith('sg-') for sg in groups):
try:
vpc_id = self.ec2.describe_subnets(SubnetIds=[result['subnets'][0]])['Subnets'][0]['VpcId']
groups = get_ec2_security_group_ids_from_names(groups, self.ec2, vpc_id)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg="Couldn't look up security groups")
result['securityGroups'] = groups
return dict(awsvpcConfiguration=result)
def find_in_array(self, array_of_services, service_name, field_name='serviceArn'):
for c in array_of_services:
@ -322,8 +357,8 @@ class EcsServiceManager:
def create_service(self, service_name, cluster_name, task_definition, load_balancers,
desired_count, client_token, role, deployment_configuration,
placement_constraints, placement_strategy):
response = self.ecs.create_service(
placement_constraints, placement_strategy, network_configuration):
params = dict(
cluster=cluster_name,
serviceName=service_name,
taskDefinition=task_definition,
@ -334,21 +369,51 @@ class EcsServiceManager:
deploymentConfiguration=deployment_configuration,
placementConstraints=placement_constraints,
placementStrategy=placement_strategy)
return response['service']
if network_configuration:
params['networkConfiguration'] = network_configuration
response = self.ecs.create_service(**params)
return self.jsonize(response['service'])
def update_service(self, service_name, cluster_name, task_definition,
desired_count, deployment_configuration):
response = self.ecs.update_service(
desired_count, deployment_configuration, network_configuration):
params = dict(
cluster=cluster_name,
service=service_name,
taskDefinition=task_definition,
desiredCount=desired_count,
deploymentConfiguration=deployment_configuration)
return response['service']
if network_configuration:
params['networkConfiguration'] = network_configuration
response = self.ecs.update_service(**params)
return self.jsonize(response['service'])
def jsonize(self, service):
# some fields are datetime which is not JSON serializable
# make them strings
if 'createdAt' in service:
service['createdAt'] = str(service['createdAt'])
if 'deployments' in service:
for d in service['deployments']:
if 'createdAt' in d:
d['createdAt'] = str(d['createdAt'])
if 'updatedAt' in d:
d['updatedAt'] = str(d['updatedAt'])
if 'events' in service:
for e in service['events']:
if 'createdAt' in e:
e['createdAt'] = str(e['createdAt'])
return service
def delete_service(self, service, cluster=None):
return self.ecs.delete_service(cluster=cluster, service=service)
def ecs_api_handles_network_configuration(self):
from distutils.version import LooseVersion
# There doesn't seem to be a nice way to inspect botocore to look
# for attributes (and networkConfiguration is not an explicit argument
# to e.g. ecs.run_task, it's just passed as a keyword argument)
return LooseVersion(botocore.__version__) >= LooseVersion('1.7.44')
def main():
argument_spec = ec2_argument_spec()
@ -365,21 +430,22 @@ def main():
repeat=dict(required=False, type='int', default=10),
deployment_configuration=dict(required=False, default={}, type='dict'),
placement_constraints=dict(required=False, default=[], type='list'),
placement_strategy=dict(required=False, default=[], type='list')
placement_strategy=dict(required=False, default=[], type='list'),
network_configuration=dict(required=False, type='dict')
))
module = AnsibleModule(argument_spec=argument_spec,
supports_check_mode=True,
required_if=[
('state', 'present', ['task_definition', 'desired_count'])
],
required_together=[['load_balancers', 'role']]
)
if not HAS_BOTO3:
module.fail_json(msg='boto3 is required.')
module = AnsibleAWSModule(argument_spec=argument_spec,
supports_check_mode=True,
required_if=[('state', 'present', ['task_definition', 'desired_count'])],
required_together=[['load_balancers', 'role']])
service_mgr = EcsServiceManager(module)
if module.params['network_configuration']:
if not service_mgr.ecs_api_handles_network_configuration():
module.fail_json(msg='botocore needs to be version 1.7.44 or higher to use network configuration')
network_configuration = service_mgr.format_network_configuration(module.params['network_configuration'])
else:
network_configuration = None
deployment_configuration = map_complex_type(module.params['deployment_configuration'],
DEPLOYMENT_CONFIGURATION_TYPE_MAP)
@ -418,7 +484,8 @@ def main():
module.params['cluster'],
module.params['task_definition'],
module.params['desired_count'],
deploymentConfiguration)
deploymentConfiguration,
network_configuration)
else:
for loadBalancer in loadBalancers:
if 'containerPort' in loadBalancer:
@ -433,7 +500,8 @@ def main():
role,
deploymentConfiguration,
module.params['placement_constraints'],
module.params['placement_strategy'])
module.params['placement_strategy'],
network_configuration)
results['service'] = response

80
lib/ansible/modules/cloud/amazon/ecs_task.py
View file

@ -54,6 +54,12 @@ options:
description:
- A value showing who or what started the task (for informational purposes)
required: False
network_configuration:
description:
- network configuration of the service. Only applicable for task definitions created with C(awsvpc) I(network_mode).
- I(network_configuration) has two keys, I(subnets), a list of subnet IDs to which the task is attached and I(security_groups),
a list of group names or group IDs for the task
version_added: 2.6
extends_documentation_fragment:
- aws
- ec2
@ -81,6 +87,12 @@ EXAMPLES = '''
container_instances:
- arn:aws:ecs:us-west-2:172139249013:container-instance/79c23f22-876c-438a-bddf-55c98a3538a8
started_by: ansible_user
network_configuration:
subnets:
- subnet-abcd1234
security_groups:
- sg-aaaa1111
- my_security_group
register: task_output
- name: Stop a task
@ -150,14 +162,13 @@ task:
type: string
'''
from ansible.module_utils.aws.core import AnsibleAWSModule
from ansible.module_utils.ec2 import ec2_argument_spec, get_ec2_security_group_ids_from_names
try:
import botocore
HAS_BOTO3 = True
except ImportError:
HAS_BOTO3 = False
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.ec2 import boto3_conn, ec2_argument_spec, get_aws_connection_info
pass # handled by AnsibleAWSModule
class EcsExecManager:
@ -165,9 +176,25 @@ class EcsExecManager:
def __init__(self, module):
self.module = module
self.ecs = module.client('ecs')
self.ec2 = module.client('ec2')
region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True)
self.ecs = boto3_conn(module, conn_type='client', resource='ecs', region=region, endpoint=ec2_url, **aws_connect_kwargs)
def format_network_configuration(self, network_config):
result = dict()
if 'subnets' in network_config:
result['subnets'] = network_config['subnets']
else:
self.module.fail_json(msg="Network configuration must include subnets")
if 'security_groups' in network_config:
groups = network_config['security_groups']
if any(not sg.startswith('sg-') for sg in groups):
try:
vpc_id = self.ec2.describe_subnets(SubnetIds=[result['subnets'][0]])['Subnets'][0]['VpcId']
groups = get_ec2_security_group_ids_from_names(groups, self.ec2, vpc_id)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg="Couldn't look up security groups")
result['securityGroups'] = groups
return dict(awsvpcConfiguration=result)
def list_tasks(self, cluster_name, service_name, status):
response = self.ecs.list_tasks(
@ -184,12 +211,14 @@ class EcsExecManager:
def run_task(self, cluster, task_definition, overrides, count, startedBy):
if overrides is None:
overrides = dict()
response = self.ecs.run_task(
cluster=cluster,
taskDefinition=task_definition,
overrides=overrides,
count=count,
startedBy=startedBy)
params = dict(cluster=cluster, taskDefinition=task_definition,
overrides=overrides, count=count, startedBy=startedBy)
if self.module.params['network_configuration']:
params['networkConfiguration'] = self.format_network_configuration(self.module.params['network_configuration'])
try:
response = self.ecs.run_task(**params)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg="Couldn't run task")
# include tasks and failures
return response['tasks']
@ -205,7 +234,12 @@ class EcsExecManager:
args['containerInstances'] = container_instances
if startedBy:
args['startedBy'] = startedBy
response = self.ecs.start_task(**args)
if self.module.params['network_configuration']:
args['networkConfiguration'] = self.format_network_configuration(self.module.params['network_configuration'])
try:
response = self.ecs.start_task(**args)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg="Couldn't start task")
# include tasks and failures
return response['tasks']
@ -213,6 +247,13 @@ class EcsExecManager:
response = self.ecs.stop_task(cluster=cluster, task=task)
return response['task']
def ecs_api_handles_network_configuration(self):
from distutils.version import LooseVersion
# There doesn't seem to be a nice way to inspect botocore to look
# for attributes (and networkConfiguration is not an explicit argument
# to e.g. ecs.run_task, it's just passed as a keyword argument)
return LooseVersion(botocore.__version__) >= LooseVersion('1.7.44')
def main():
argument_spec = ec2_argument_spec()
@ -224,14 +265,11 @@ def main():
count=dict(required=False, type='int'), # R
task=dict(required=False, type='str'), # P*
container_instances=dict(required=False, type='list'), # S*
started_by=dict(required=False, type='str') # R S
started_by=dict(required=False, type='str'), # R S
network_configuration=dict(required=False, type='dict')
))
module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=True)
# Validate Requirements
if not HAS_BOTO3:
module.fail_json(msg='boto3 is required.')
module = AnsibleAWSModule(argument_spec=argument_spec, supports_check_mode=True)
# Validate Inputs
if module.params['operation'] == 'run':
@ -257,6 +295,8 @@ def main():
status_type = "STOPPED"
service_mgr = EcsExecManager(module)
if module.params['network_configuration'] and not service_mgr.ecs_api_handles_network_configuration():
module.fail_json(msg='botocore needs to be version 1.7.44 or higher to use network configuration')
existing = service_mgr.list_tasks(module.params['cluster'], task_to_list, status_type)
results = dict(changed=False)

10
lib/ansible/modules/cloud/amazon/ecs_taskdefinition.py
View file

@ -58,9 +58,10 @@ options:
network_mode:
description:
- The Docker networking mode to use for the containers in the task.
- C(awsvpc) mode was added in Ansible 2.5
required: false
default: bridge
choices: [ 'bridge', 'host', 'none' ]
choices: [ 'bridge', 'host', 'none', 'awsvpc' ]
version_added: 2.3
task_role_arn:
description:
@ -166,6 +167,10 @@ class EcsTaskManager:
for port in ('hostPort', 'containerPort'):
if port in port_mapping:
port_mapping[port] = int(port_mapping[port])
if network_mode == 'awsvpc' and 'hostPort' in port_mapping:
if port_mapping['hostPort'] != port_mapping.get('containerPort'):
self.module.fail_json(msg="In awsvpc network mode, host port must be set to the same as "
"container port or not be set")
validated_containers.append(container)
@ -227,7 +232,7 @@ def main():
revision=dict(required=False, type='int'),
force_create=dict(required=False, default=False, type='bool'),
containers=dict(required=False, type='list'),
network_mode=dict(required=False, default='bridge', choices=['bridge', 'host', 'none'], type='str'),
network_mode=dict(required=False, default='bridge', choices=['bridge', 'host', 'none', 'awsvpc'], type='str'),
task_role_arn=dict(required=False, default='', type='str'),
volumes=dict(required=False, type='list')))
@ -390,5 +395,6 @@ def main():
module.exit_json(**results)
if __name__ == '__main__':
main()