sns_topic boto3 port (#39292)

* Port sns_topic to boto3 and add tests
This commit is contained in:
Will Thames 2018-08-24 11:04:18 +10:00 committed by Sloane Hertel
commit 60e3af42d5
13 changed files with 676 additions and 154 deletions

View file

@ -16,15 +16,17 @@ DOCUMENTATION = """
module: sns_topic
short_description: Manages AWS SNS topics and subscriptions
description:
- The C(sns_topic) module allows you to create, delete, and manage subscriptions for AWS SNS topics.
- The C(sns_topic) module allows you to create, delete, and manage subscriptions for AWS SNS topics. As of 2.6,
this module can be use to subscribe and unsubscribe to topics outside of your AWS account.
version_added: 2.0
author:
- "Joel Thompson (@joelthompson)"
- "Fernando Jose Pando (@nand0p)"
- "Will Thames (@willthames)"
options:
name:
description:
- The name or ARN of the SNS topic to converge
- The name or ARN of the SNS topic to manage
required: True
state:
description:
@ -45,6 +47,13 @@ options:
- List of subscriptions to apply to the topic. Note that AWS requires
subscriptions to be confirmed, so you will need to confirm any new
subscriptions.
suboptions:
endpoint:
description: Endpoint of subscription
required: yes
protocol:
description: Protocol of subscription
required: yes
default: []
purge_subscriptions:
description:
@ -91,42 +100,123 @@ sns_arn:
description: The ARN of the topic you are modifying
type: string
returned: always
sample: "arn:aws:sns:us-east-1:123456789012:my_topic_name"
sample: "arn:aws:sns:us-east-2:111111111111:my_topic_name"
sns_topic:
description: Dict of sns topic details
type: dict
returned: always
sample:
name: sns-topic-name
state: present
display_name: default
policy: {}
delivery_policy: {}
subscriptions_new: []
subscriptions_existing: []
subscriptions_deleted: []
subscriptions_added: []
subscriptions_purge': false
check_mode: false
topic_created: false
topic_deleted: false
attributes_set: []
description: Dict of sns topic details
type: complex
returned: always
contains:
attributes_set:
description: list of attributes set during this run
returned: always
type: list
sample: []
check_mode:
description: whether check mode was on
returned: always
type: bool
sample: false
delivery_policy:
description: Delivery policy for the SNS topic
returned: when topic is owned by this AWS account
type: string
sample: >
{"http":{"defaultHealthyRetryPolicy":{"minDelayTarget":20,"maxDelayTarget":20,"numRetries":3,"numMaxDelayRetries":0,
"numNoDelayRetries":0,"numMinDelayRetries":0,"backoffFunction":"linear"},"disableSubscriptionOverrides":false}}
display_name:
description: Display name for SNS topic
returned: when topic is owned by this AWS account
type: string
sample: My topic name
name:
description: Topic name
returned: always
type: string
sample: ansible-test-dummy-topic
owner:
description: AWS account that owns the topic
returned: when topic is owned by this AWS account
type: string
sample: '111111111111'
policy:
description: Policy for the SNS topic
returned: when topic is owned by this AWS account
type: string
sample: >
{"Version":"2012-10-17","Id":"SomePolicyId","Statement":[{"Sid":"ANewSid","Effect":"Allow","Principal":{"AWS":"arn:aws:iam::111111111111:root"},
"Action":"sns:Subscribe","Resource":"arn:aws:sns:us-east-2:111111111111:ansible-test-dummy-topic","Condition":{"StringEquals":{"sns:Protocol":"email"}}}]}
state:
description: whether the topic is present or absent
returned: always
type: string
sample: present
subscriptions:
description: List of subscribers to the topic in this AWS account
returned: always
type: list
sample: []
subscriptions_added:
description: List of subscribers added in this run
returned: always
type: list
sample: []
subscriptions_confirmed:
description: Count of confirmed subscriptions
returned: when topic is owned by this AWS account
type: list
sample: []
subscriptions_deleted:
description: Count of deleted subscriptions
returned: when topic is owned by this AWS account
type: list
sample: []
subscriptions_existing:
description: List of existing subscriptions
returned: always
type: list
sample: []
subscriptions_new:
description: List of new subscriptions
returned: always
type: list
sample: []
subscriptions_pending:
description: Count of pending subscriptions
returned: when topic is owned by this AWS account
type: string
sample: '0'
subscriptions_purge:
description: Whether or not purge_subscriptions was set
returned: always
type: bool
sample: true
topic_arn:
description: ARN of the SNS topic (equivalent to sns_arn)
returned: when topic is owned by this AWS account
type: string
sample: arn:aws:sns:us-east-2:111111111111:ansible-test-dummy-topic
topic_created:
description: Whether the topic was created
returned: always
type: bool
sample: false
topic_deleted:
description: Whether the topic was deleted
returned: always
type: bool
sample: false
'''
import json
import re
import time
try:
import boto.sns
from boto.exception import BotoServerError
HAS_BOTO = True
import botocore
except ImportError:
HAS_BOTO = False
pass # handled by AnsibleAWSModule
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.ec2 import connect_to_aws, ec2_argument_spec, get_aws_connection_info
from ansible.module_utils.aws.core import AnsibleAWSModule
from ansible.module_utils.ec2 import compare_policies, AWSRetry, camel_dict_to_snake_dict
class SnsTopicManager(object):
@ -141,14 +231,9 @@ class SnsTopicManager(object):
delivery_policy,
subscriptions,
purge_subscriptions,
check_mode,
region,
**aws_connect_params):
check_mode):
self.region = region
self.aws_connect_params = aws_connect_params
self.connection = self._get_boto_connection()
self.changed = False
self.connection = module.client('sns')
self.module = module
self.name = name
self.state = state
@ -163,151 +248,193 @@ class SnsTopicManager(object):
self.check_mode = check_mode
self.topic_created = False
self.topic_deleted = False
self.arn_topic = None
self.topic_arn = None
self.attributes_set = []
def _get_boto_connection(self):
try:
return connect_to_aws(boto.sns, self.region,
**self.aws_connect_params)
except BotoServerError as err:
self.module.fail_json(msg=err.message)
@AWSRetry.jittered_backoff()
def _list_topics_with_backoff(self):
paginator = self.connection.get_paginator('list_topics')
return paginator.paginate().build_full_result()['Topics']
def _get_all_topics(self):
next_token = None
topics = []
while True:
try:
response = self.connection.get_all_topics(next_token)
except BotoServerError as err:
self.module.fail_json(msg=err.message)
topics.extend(response['ListTopicsResponse']['ListTopicsResult']['Topics'])
next_token = response['ListTopicsResponse']['ListTopicsResult']['NextToken']
if not next_token:
break
@AWSRetry.jittered_backoff()
def _list_topic_subscriptions_with_backoff(self):
paginator = self.connection.get_paginator('list_subscriptions_by_topic')
return paginator.paginate(TopicArn=self.topic_arn).build_full_result()['Subscriptions']
@AWSRetry.jittered_backoff()
def _list_subscriptions_with_backoff(self):
paginator = self.connection.get_paginator('list_subscriptions')
return paginator.paginate().build_full_result()['Subscriptions']
def _list_topics(self):
try:
topics = self._list_topics_with_backoff()
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg="Couldn't get topic list")
return [t['TopicArn'] for t in topics]
def _arn_topic_lookup(self):
def _topic_arn_lookup(self):
# topic names cannot have colons, so this captures the full topic name
all_topics = self._get_all_topics()
all_topics = self._list_topics()
lookup_topic = ':%s' % self.name
for topic in all_topics:
if topic.endswith(lookup_topic):
return topic
def _create_topic(self):
self.changed = True
self.topic_created = True
if not self.check_mode:
self.connection.create_topic(self.name)
self.arn_topic = self._arn_topic_lookup()
while not self.arn_topic:
time.sleep(3)
self.arn_topic = self._arn_topic_lookup()
try:
response = self.connection.create_topic(Name=self.name)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg="Couldn't create topic %s" % self.name)
self.topic_arn = response['TopicArn']
return True
def _set_topic_attrs(self):
topic_attributes = self.connection.get_topic_attributes(self.arn_topic)['GetTopicAttributesResponse']['GetTopicAttributesResult']['Attributes']
changed = False
try:
topic_attributes = self.connection.get_topic_attributes(TopicArn=self.topic_arn)['Attributes']
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg="Couldn't get topic attributes for topic %s" % self.topic_arn)
if self.display_name and self.display_name != topic_attributes['DisplayName']:
self.changed = True
changed = True
self.attributes_set.append('display_name')
if not self.check_mode:
self.connection.set_topic_attributes(self.arn_topic, 'DisplayName',
self.display_name)
try:
self.connection.set_topic_attributes(TopicArn=self.topic_arn, AttributeName='DisplayName',
AttributeValue=self.display_name)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg="Couldn't set display name")
if self.policy and self.policy != json.loads(topic_attributes['Policy']):
self.changed = True
if self.policy and compare_policies(self.policy, json.loads(topic_attributes['Policy'])):
changed = True
self.attributes_set.append('policy')
if not self.check_mode:
self.connection.set_topic_attributes(self.arn_topic, 'Policy',
json.dumps(self.policy))
try:
self.connection.set_topic_attributes(TopicArn=self.topic_arn, AttributeName='Policy',
AttributeValue=json.dumps(self.policy))
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg="Couldn't set topic policy")
if self.delivery_policy and ('DeliveryPolicy' not in topic_attributes or
self.delivery_policy != json.loads(topic_attributes['DeliveryPolicy'])):
self.changed = True
compare_policies(self.delivery_policy, json.loads(topic_attributes['DeliveryPolicy']))):
changed = True
self.attributes_set.append('delivery_policy')
if not self.check_mode:
self.connection.set_topic_attributes(self.arn_topic, 'DeliveryPolicy',
json.dumps(self.delivery_policy))
try:
self.connection.set_topic_attributes(TopicArn=self.topic_arn, AttributeName='DeliveryPolicy',
AttributeValue=json.dumps(self.delivery_policy))
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg="Couldn't set topic delivery policy")
return changed
def _canonicalize_endpoint(self, protocol, endpoint):
if protocol == 'sms':
return re.sub('[^0-9]*', '', endpoint)
return endpoint
def _get_topic_subs(self):
next_token = None
while True:
response = self.connection.get_all_subscriptions_by_topic(self.arn_topic, next_token)
self.subscriptions_existing.extend(response['ListSubscriptionsByTopicResponse']
['ListSubscriptionsByTopicResult']['Subscriptions'])
next_token = response['ListSubscriptionsByTopicResponse']['ListSubscriptionsByTopicResult']['NextToken']
if not next_token:
break
def _set_topic_subs(self):
subscriptions_existing_list = []
changed = False
subscriptions_existing_list = set()
desired_subscriptions = [(sub['protocol'],
self._canonicalize_endpoint(sub['protocol'], sub['endpoint'])) for sub in
self.subscriptions]
if self.subscriptions_existing:
for sub in self.subscriptions_existing:
sub_key = (sub['Protocol'], sub['Endpoint'])
subscriptions_existing_list.append(sub_key)
if (self.purge_subscriptions and sub_key not in desired_subscriptions and
sub['SubscriptionArn'] not in ('PendingConfirmation', 'Deleted')):
self.changed = True
self.subscriptions_deleted.append(sub_key)
if not self.check_mode:
self.connection.unsubscribe(sub['SubscriptionArn'])
for (protocol, endpoint) in desired_subscriptions:
if (protocol, endpoint) not in subscriptions_existing_list:
self.changed = True
self.subscriptions_added.append((protocol, endpoint))
for sub in self._list_topic_subscriptions():
sub_key = (sub['Protocol'], sub['Endpoint'])
subscriptions_existing_list.add(sub_key)
if (self.purge_subscriptions and sub_key not in desired_subscriptions and
sub['SubscriptionArn'] not in ('PendingConfirmation', 'Deleted')):
changed = True
self.subscriptions_deleted.append(sub_key)
if not self.check_mode:
self.connection.subscribe(self.arn_topic, protocol, endpoint)
try:
self.connection.unsubscribe(SubscriptionArn=sub['SubscriptionArn'])
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg="Couldn't unsubscribe from topic")
for protocol, endpoint in set(desired_subscriptions).difference(subscriptions_existing_list):
changed = True
self.subscriptions_added.append((protocol, endpoint))
if not self.check_mode:
try:
self.connection.subscribe(TopicArn=self.topic_arn, Protocol=protocol, Endpoint=endpoint)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg="Couldn't subscribe to topic %s" % self.topic_arn)
return changed
def _list_topic_subscriptions(self):
try:
return self._list_topic_subscriptions_with_backoff()
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
try:
# potentially AuthorizationError when listing subscriptions for third party topic
return [sub for sub in self._list_subscriptions_with_backoff()
if sub['TopicArn'] == self.topic_arn]
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg="Couldn't get subscriptions list for topic %s" % self.topic_arn)
def _delete_subscriptions(self):
# NOTE: subscriptions in 'PendingConfirmation' timeout in 3 days
# https://forums.aws.amazon.com/thread.jspa?threadID=85993
for sub in self.subscriptions_existing:
subscriptions = self._list_topic_subscriptions()
if not subscriptions:
return False
for sub in subscriptions:
if sub['SubscriptionArn'] not in ('PendingConfirmation', 'Deleted'):
self.subscriptions_deleted.append(sub['SubscriptionArn'])
self.changed = True
if not self.check_mode:
self.connection.unsubscribe(sub['SubscriptionArn'])
try:
self.connection.unsubscribe(SubscriptionArn=sub['SubscriptionArn'])
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg="Couldn't unsubscribe from topic")
return True
def _delete_topic(self):
self.topic_deleted = True
self.changed = True
if not self.check_mode:
self.connection.delete_topic(self.arn_topic)
try:
self.connection.delete_topic(TopicArn=self.topic_arn)
except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
self.module.fail_json_aws(e, msg="Couldn't delete topic %s" % self.topic_arn)
return True
def _name_is_arn(self):
return self.name.startswith('arn:')
def ensure_ok(self):
self.arn_topic = self._arn_topic_lookup()
if not self.arn_topic:
self._create_topic()
self._set_topic_attrs()
self._get_topic_subs()
self._set_topic_subs()
changed = False
if self._name_is_arn():
self.topic_arn = self.name
else:
self.topic_arn = self._topic_arn_lookup()
if not self.topic_arn:
changed = self._create_topic()
if self.topic_arn in self._list_topics():
changed |= self._set_topic_attrs()
elif self.display_name or self.policy or self.delivery_policy:
self.module.fail_json(msg="Cannot set display name, policy or delivery policy for SNS topics not owned by this account")
changed |= self._set_topic_subs()
return changed
def ensure_gone(self):
self.arn_topic = self._arn_topic_lookup()
if self.arn_topic:
self._get_topic_subs()
if self.subscriptions_existing:
self._delete_subscriptions()
self._delete_topic()
changed = False
if self._name_is_arn():
self.topic_arn = self.name
else:
self.topic_arn = self._topic_arn_lookup()
if self.topic_arn:
if self.topic_arn not in self._list_topics():
self.module.fail_json(msg="Cannot use state=absent with third party ARN. Use subscribers=[] to unsubscribe")
changed = self._delete_subscriptions()
changed |= self._delete_topic()
return changed
def get_info(self):
info = {
'name': self.name,
'state': self.state,
'display_name': self.display_name,
'policy': self.policy,
'delivery_policy': self.delivery_policy,
'subscriptions_new': self.subscriptions,
'subscriptions_existing': self.subscriptions_existing,
'subscriptions_deleted': self.subscriptions_deleted,
@ -316,32 +443,30 @@ class SnsTopicManager(object):
'check_mode': self.check_mode,
'topic_created': self.topic_created,
'topic_deleted': self.topic_deleted,
'attributes_set': self.attributes_set
'attributes_set': self.attributes_set,
}
if self.state != 'absent':
if self.topic_arn in self._list_topics():
info.update(camel_dict_to_snake_dict(self.connection.get_topic_attributes(TopicArn=self.topic_arn)['Attributes']))
info['delivery_policy'] = info.pop('effective_delivery_policy')
info['subscriptions'] = [camel_dict_to_snake_dict(sub) for sub in self._list_topic_subscriptions()]
return info
def main():
argument_spec = ec2_argument_spec()
argument_spec.update(
dict(
name=dict(type='str', required=True),
state=dict(type='str', default='present', choices=['present',
'absent']),
display_name=dict(type='str', required=False),
policy=dict(type='dict', required=False),
delivery_policy=dict(type='dict', required=False),
subscriptions=dict(default=[], type='list', required=False),
purge_subscriptions=dict(type='bool', default=True),
)
argument_spec = dict(
name=dict(required=True),
state=dict(default='present', choices=['present', 'absent']),
display_name=dict(),
policy=dict(type='dict'),
delivery_policy=dict(type='dict'),
subscriptions=dict(default=[], type='list'),
purge_subscriptions=dict(type='bool', default=True),
)
module = AnsibleModule(argument_spec=argument_spec,
supports_check_mode=True)
if not HAS_BOTO:
module.fail_json(msg='boto required for this module')
module = AnsibleAWSModule(argument_spec=argument_spec,
supports_check_mode=True)
name = module.params.get('name')
state = module.params.get('state')
@ -352,10 +477,6 @@ def main():
purge_subscriptions = module.params.get('purge_subscriptions')
check_mode = module.check_mode
region, ec2_url, aws_connect_params = get_aws_connection_info(module)
if not region:
module.fail_json(msg="region must be specified")
sns_topic = SnsTopicManager(module,
name,
state,
@ -364,18 +485,16 @@ def main():
delivery_policy,
subscriptions,
purge_subscriptions,
check_mode,
region,
**aws_connect_params)
check_mode)
if state == 'present':
sns_topic.ensure_ok()
changed = sns_topic.ensure_ok()
elif state == 'absent':
sns_topic.ensure_gone()
changed = sns_topic.ensure_gone()
sns_facts = dict(changed=sns_topic.changed,
sns_arn=sns_topic.arn_topic,
sns_facts = dict(changed=changed,
sns_arn=sns_topic.topic_arn,
sns_topic=sns_topic.get_info())
module.exit_json(**sns_facts)