[cloud] new module lambda_policy (PR #24951)

- Fixes to lambda
- reformatting + tests for lambda_facts
- lambda module integration test
- switch lambda and lambda_facts to AnsibleAwsModule
- Get the account ID from STS, GetUser, and finally error message
This commit is contained in:
Michael De La Rue 2017-05-16 14:26:55 +01:00 committed by Ryan S. Brown
commit fbec5ab12d
16 changed files with 1481 additions and 123 deletions

View file

@ -0,0 +1,4 @@
cloud/aws
posix/ci/cloud/aws
execute_lambda
lambda

View file

@ -0,0 +1,3 @@
---
# defaults file for aws_lambda test
lambda_function_name: '{{resource_prefix}}'

View file

@ -0,0 +1,34 @@
from __future__ import print_function
import json
def handler(event, context):
"""
The handler function is the function which gets called each time
the lambda is run.
"""
# printing goes to the cloudwatch log allowing us to simply debug the lambda if we can find
# the log entry.
print("got event:\n" + json.dumps(event))
# if the name parameter isn't present this can throw an exception
# which will result in an amazon chosen failure from the lambda
# which can be completely fine.
name = event["name"]
return {"message": "hello " + name}
def main():
"""
This main function will normally never be called during normal
lambda use. It is here for testing the lambda program only.
"""
event = {"name": "james"}
context = None
print(handler(event, context))
if __name__ == '__main__':
main()

View file

@ -0,0 +1,409 @@
---
#
# Author: Michael De La Rue
# based on ec2_key.yml + lambda.py
- block:
# ============================================================
- name: test with no parameters
lambda:
register: result
ignore_errors: true
- name: assert failure when called with no parameters
assert:
that:
- 'result.failed'
- 'result.msg.startswith("missing required arguments: name")'
# ============================================================
- name: test with no parameters except state absent
lambda:
state=absent
register: result
ignore_errors: true
- name: assert failure when called with no parameters
assert:
that:
- 'result.failed'
- 'result.msg.startswith("missing required arguments: name")'
# ============================================================
- name: test with no role or handler
lambda:
name=ansible-testing-fake-should-not-be-created
runtime="python2.7"
register: result
ignore_errors: true
- name: assert failure when called with no parameters
assert:
that:
- 'result.failed'
- 'result.msg.startswith("state is present but the following are missing: handler")'
# ============================================================
- name: test with all module required variables but no region
lambda:
name=ansible-testing-fake-should-not-be-created
runtime="python2.7"
handler="no-handler"
role=arn:fake-role-doesnt-exist
register: result
ignore_errors: true
- name: assert failure when called with only 'name'
assert:
that:
- 'result.failed'
- 'result.msg == "region must be specified"'
# ============================================================
- name: test with all module required variables, no region and all possible variables set to blank
lambda:
name: ansible-testing-fake-should-not-be-created
state: present
runtime: "python2.7"
role: arn:fake-role-doesnt-exist
handler:
s3_bucket:
s3_key:
s3_object_version:
description:
vpc_subnet_ids:
vpc_security_group_ids:
environment_variables:
dead_letter_arn:
register: result
ignore_errors: true
- name: assert failure when called with only 'name'
assert:
that:
- 'result.failed'
- 'result.msg == "region must be specified"'
# ============================================================
# direct zip file upload
- name: move lambda into place for archive module
copy:
src: "mini_lambda.py"
dest: "{{output_dir}}/mini_lambda.py"
- name: bundle lambda into a zip
archive:
format: zip
path: "{{output_dir}}/mini_lambda.py"
dest: "{{output_dir}}/mini_lambda.zip"
register: zip_res
- name: test state=present - upload the lambda
lambda:
name="{{lambda_function_name}}"
runtime="python2.7"
handler="mini_lambda.handler"
role="ansible_lambda_role"
ec2_region='{{ec2_region}}'
ec2_access_key='{{ec2_access_key}}'
ec2_secret_key='{{ec2_secret_key}}'
security_token='{{security_token}}'
zip_file="{{zip_res.dest}}"
register: result
- name: assert lambda upload succeeded
assert:
that:
- 'not result|failed'
- name: test lambda works
execute_lambda:
name: "{{lambda_function_name}}"
payload:
name: "Mr Ansible Tests"
ec2_region: '{{ec2_region}}'
ec2_access_key: '{{ec2_access_key}}'
ec2_secret_key: '{{ec2_secret_key}}'
security_token: '{{security_token}}'
register: result
- name: assert lambda manages to respond as expected
assert:
that:
- 'not result|failed'
- 'result.result.output.message == "hello Mr Ansible Tests"'
# ============================================================
- name: test state=present with security group but no vpc
lambda:
name: "{{lambda_function_name}}"
runtime: "python2.7"
role: "ansible_lambda_role"
ec2_region: '{{ec2_region}}'
ec2_access_key: '{{ec2_access_key}}'
ec2_secret_key: '{{ec2_secret_key}}'
security_token: '{{security_token}}'
zip_file: "{{zip_res.dest}}"
handler:
description:
vpc_subnet_ids:
vpc_security_group_ids: sg-FA6E
environment_variables:
dead_letter_arn:
register: result
ignore_errors: true
- name: assert lambda fails with proper message
assert:
that:
- 'result|failed'
- 'result.msg != "MODULE FAILURE"'
- 'result.changed == False'
- '"requires at least one security group and one subnet" in result.msg'
# ============================================================
- name: test state=present with all nullable variables explicitly set to null
lambda:
name: "{{lambda_function_name}}"
runtime: "python2.7"
role: "ansible_lambda_role"
ec2_region: '{{ec2_region}}'
ec2_access_key: '{{ec2_access_key}}'
ec2_secret_key: '{{ec2_secret_key}}'
security_token: '{{security_token}}'
zip_file: "{{zip_res.dest}}"
handler: "mini_lambda.handler"
# These are not allowed because of mutually exclusive.
# s3_bucket:
# s3_key:
# s3_object_version:
description:
vpc_subnet_ids:
vpc_security_group_ids:
environment_variables:
dead_letter_arn:
register: result
- name: assert lambda was updated as expected
assert:
that:
- 'not result|failed'
- 'result.changed == False'
# ============================================================
- name: test state=present triggering a network exception due to bad url
lambda:
name: "{{lambda_function_name}}"
runtime: "python2.7"
role: "ansible_lambda_role"
ec2_url: https://noexist.example.com
ec2_region: '{{ec2_region}}'
ec2_access_key: 'iamnotreallyanaccesskey'
ec2_secret_key: 'thisisabadsecretkey'
security_token: 'andthisisabadsecuritytoken'
zip_file: "{{zip_res.dest}}"
register: result
ignore_errors: true
- name: assert lambda manages to respond as expected
assert:
that:
- 'result|failed'
- 'result.changed == False'
# ============================================================
- name: test state=absent (expect changed=False)
lambda:
name="{{lambda_function_name}}"
ec2_region='{{ec2_region}}'
ec2_access_key='{{ec2_access_key}}'
ec2_secret_key='{{ec2_secret_key}}'
security_token='{{security_token}}'
state=absent
register: result
- name: assert state=absent
assert:
that:
- 'not result|failed'
- 'result.changed == True'
# ============================================================
# parallel lambda creation
- name: parallel lambda creation 1/4
lambda:
name: "{{lambda_function_name}}_1"
runtime: "python2.7"
handler: "mini_lambda.handler"
role: "ansible_lambda_role"
ec2_region: '{{ec2_region}}'
ec2_access_key: '{{ec2_access_key}}'
ec2_secret_key: '{{ec2_secret_key}}'
security_token: '{{security_token}}'
zip_file: "{{zip_res.dest}}"
async: 1000
register: async_1
- name: parallel lambda creation 2/4
lambda:
name: "{{lambda_function_name}}_2"
runtime: "python2.7"
handler: "mini_lambda.handler"
role: "ansible_lambda_role"
ec2_region: '{{ec2_region}}'
ec2_access_key: '{{ec2_access_key}}'
ec2_secret_key: '{{ec2_secret_key}}'
security_token: '{{security_token}}'
zip_file: "{{zip_res.dest}}"
async: 1000
register: async_2
- name: parallel lambda creation 3/4
lambda:
name: "{{lambda_function_name}}_3"
runtime: "python2.7"
handler: "mini_lambda.handler"
role: "ansible_lambda_role"
ec2_region: '{{ec2_region}}'
ec2_access_key: '{{ec2_access_key}}'
ec2_secret_key: '{{ec2_secret_key}}'
security_token: '{{security_token}}'
zip_file: "{{zip_res.dest}}"
async: 1000
register: async_3
- name: parallel lambda creation 4/4
lambda:
name: "{{lambda_function_name}}_4"
runtime: "python2.7"
handler: "mini_lambda.handler"
role: "ansible_lambda_role"
ec2_region: '{{ec2_region}}'
ec2_access_key: '{{ec2_access_key}}'
ec2_secret_key: '{{ec2_secret_key}}'
security_token: '{{security_token}}'
zip_file: "{{zip_res.dest}}"
register: result
- name: assert lambda manages to respond as expected
assert:
that:
- 'not result|failed'
- name: wait for async job 1
async_status: jid={{ async_1.ansible_job_id }}
register: job_result
until: job_result.finished
retries: 30
- name: wait for async job 2
async_status: jid={{ async_1.ansible_job_id }}
register: job_result
until: job_result.finished
retries: 30
- name: wait for async job 3
async_status: jid={{ async_3.ansible_job_id }}
register: job_result
until: job_result.finished
retries: 30
- name: parallel lambda deletion 1/4
lambda:
name: "{{lambda_function_name}}_1"
state: absent
ec2_region: '{{ec2_region}}'
ec2_access_key: '{{ec2_access_key}}'
ec2_secret_key: '{{ec2_secret_key}}'
security_token: '{{security_token}}'
zip_file: "{{zip_res.dest}}"
async: 1000
register: async_1
- name: parallel lambda deletion 2/4
lambda:
name: "{{lambda_function_name}}_2"
state: absent
ec2_region: '{{ec2_region}}'
ec2_access_key: '{{ec2_access_key}}'
ec2_secret_key: '{{ec2_secret_key}}'
security_token: '{{security_token}}'
zip_file: "{{zip_res.dest}}"
async: 1000
register: async_2
- name: parallel lambda deletion 3/4
lambda:
name: "{{lambda_function_name}}_3"
state: absent
ec2_region: '{{ec2_region}}'
ec2_access_key: '{{ec2_access_key}}'
ec2_secret_key: '{{ec2_secret_key}}'
security_token: '{{security_token}}'
zip_file: "{{zip_res.dest}}"
async: 1000
register: async_3
- name: parallel lambda deletion 4/4
lambda:
name: "{{lambda_function_name}}_4"
state: absent
ec2_region: '{{ec2_region}}'
ec2_access_key: '{{ec2_access_key}}'
ec2_secret_key: '{{ec2_secret_key}}'
security_token: '{{security_token}}'
zip_file: "{{zip_res.dest}}"
register: result
- name: assert lambda creation has succeeded
assert:
that:
- 'not result|failed'
- name: wait for async job 1
async_status: jid={{ async_1.ansible_job_id }}
register: job_result
until: job_result.finished
retries: 30
- name: wait for async job 2
async_status: jid={{ async_1.ansible_job_id }}
register: job_result
until: job_result.finished
retries: 30
- name: wait for async job 3
async_status: jid={{ async_3.ansible_job_id }}
register: job_result
until: job_result.finished
retries: 30
# ============================================================
# upload via s3 bucket - multi function
# ============================================================
# update already existing function
always:
# ============================================================
- name: test state=absent (expect changed=False)
lambda:
name="{{lambda_function_name}}"
ec2_region='{{ec2_region}}'
ec2_access_key='{{ec2_access_key}}'
ec2_secret_key='{{ec2_secret_key}}'
security_token='{{security_token}}'
state=absent
register: result
- name: assert state=absent
assert:
that:
- 'not result|failed'
- 'result.changed == False'

View file

@ -0,0 +1,2 @@
cloud/aws
posix/ci/cloud/aws

View file

@ -0,0 +1,3 @@
---
# defaults file for aws_lambda test
lambda_function_name: '{{resource_prefix}}-api-endpoint'

View file

@ -0,0 +1,36 @@
from __future__ import print_function
import json
def handler(event, context):
"""
The handler function is the function which gets called each time
the lambda is run.
"""
# printing goes to the cloudwatch log allowing us to simply debug the lambda if we can find
# the log entry.
print("got event:\n" + json.dumps(event))
# if the name parameter isn't present this can throw an exception
# which will result in an amazon chosen failure from the lambda
# which can be completely fine.
name = event["pathParameters"]["greet_name"]
return {"statusCode": 200,
"body": 'hello: "' + name + '"',
"headers": {}}
def main():
"""
This main function will normally never be called during normal
lambda use. It is here for testing the lambda program only.
"""
event = {"name": "james"}
context = None
print(handler(event, context))
if __name__ == '__main__':
main()

View file

@ -0,0 +1,218 @@
---
#
# Author: Michael De La Rue
# based on ec2_key.yml + lambda.py
- block:
# ============================================================
- name: test with no parameters
lambda_policy:
register: result
ignore_errors: true
- name: assert failure when called with no parameters
assert:
that:
- 'result.failed'
- 'result.msg.startswith("missing required arguments: ")'
# ============================================================
- name: test with all required dummy parameters but no region
lambda_policy:
statement_id: dummy
principal: api_fakeway
action: fake:do_something_fake
function_name: dummy_fake_function
ignore_errors: true
register: result
- name: assert failure and appropriate message when called without region
assert:
that:
- 'result.failed'
- '"region must be specified" in result.msg'
# ============================================================
- name: test with all required dummy parameters but no region
lambda_policy:
statement_id: dummy
principal: api_fakeway
action: fake:do_something_fake
function_name: dummy_fake_function
region: null
ignore_errors: true
register: result
- name: assert failure and appropriate message when called false region region
assert:
that:
- 'result.failed'
- '"region must be specified" in result.msg'
# ============================================================
- name: test exceptions generated by forcing bad ec2 url
lambda_policy:
function_name: "{{ lambda_function_name }}"
region: "{{ec2_region}}"
state: present
statement_id: api-gateway-invoke-lambdas
action: lambda:InvokeFunction
principal: apigateway.amazonaws.com
source_arn: "arn:aws:execute-api:no-north-0:1234567:*/*"
ec2_url: https://noexist.example.com
ec2_region: 'no-north-0'
ec2_access_key: 'iamnotreallyanaccesskey'
ec2_secret_key: 'thisisabadsecretkey'
security_token: 'andthisisabadsecuritytoken'
register: result
ignore_errors: true
- name: assert lambda manages to respond as expected
assert:
that:
- 'result|failed'
- 'result.msg != "MODULE FAILURE"'
- 'result.changed == False'
# ============================================================
# direct zip file upload
- name: move lambda into place for archive module
copy:
src: "mini_http_lambda.py"
dest: "{{output_dir}}/mini_http_lambda.py"
- name: bundle lambda into a zip
archive:
format: zip
path: "{{output_dir}}/mini_http_lambda.py"
dest: "{{output_dir}}/mini_http_lambda.zip"
register: zip_res
- name: test state=present - upload the lambda
lambda:
name="{{lambda_function_name}}"
runtime="python2.7"
handler="mini_http_lambda.handler"
role="ansible_lambda_role"
ec2_region='{{ec2_region}}'
aws_access_key='{{aws_access_key}}'
aws_secret_key='{{aws_secret_key}}'
security_token='{{security_token}}'
zip_file="{{zip_res.dest}}"
register: lambda_result
- name: install aws cli - FIXME temporary this should go for a lighterweight solution
command: pip install awscli
register: result
- name: get the aws account ID for use in future commands
command: aws sts get-caller-identity --output text --query 'Account'
environment:
AWS_ACCESS_KEY_ID: '{{aws_access_key}}'
AWS_SECRET_ACCESS_KEY: '{{aws_secret_key}}'
AWS_SESSION_TOKEN: '{{security_token}}'
register: result
- name: register account id
set_fact:
aws_account_id: "{{ result.stdout | replace('\n', '') }}"
- name: register lambda uri for use in template
set_fact:
mini_lambda_uri: "arn:aws:apigateway:{{ec2_region}}:lambda:path/2015-03-31/functions/arn:aws:lambda:{{ec2_region}}:{{aws_account_id}}:function:{{ lambda_result.configuration.function_name }}/invocations"
- name: build API file
template:
src: endpoint-test-swagger-api.yml.j2
dest: "{{output_dir}}/endpoint-test-swagger-api.yml.j2"
- name: deploy new API
aws_api_gateway:
api_file: "{{output_dir}}/endpoint-test-swagger-api.yml.j2"
stage: "lambdabased"
region: '{{ec2_region}}'
aws_access_key: '{{aws_access_key}}'
aws_secret_key: '{{aws_secret_key}}'
security_token: '{{security_token}}'
register: create_result
- name: register api id for later
set_fact:
api_id: "{{ create_result.api_id }}"
- name: check API fails with permissions failure
uri: url="https://{{create_result.api_id}}.execute-api.{{ec2_region}}.amazonaws.com/lambdabased/mini/Mr_Ansible_Tester"
register: unauth_uri_result
ignore_errors: true
- name: assert internal server error due to permissions
assert:
that:
- unauth_uri_result|failed
- 'unauth_uri_result.status == 500'
- name: give api gateway execute permissions on lambda
lambda_policy:
function_name: "{{ lambda_function_name }}"
region: "{{ec2_region}}"
state: present
statement_id: api-gateway-invoke-lambdas
action: lambda:InvokeFunction
principal: apigateway.amazonaws.com
source_arn: "arn:aws:execute-api:{{ ec2_region }}:{{ aws_account_id }}:*/*"
aws_access_key: '{{aws_access_key}}'
aws_secret_key: '{{aws_secret_key}}'
security_token: '{{security_token}}'
- name: check API works with execute permissions
uri: url="https://{{create_result.api_id}}.execute-api.{{ec2_region}}.amazonaws.com/lambdabased/mini/Mr_Ansible_Tester"
register: uri_result
- name: assert API works success
assert:
that:
- 'uri_result'
- name: deploy new API
aws_api_gateway:
api_file: "{{output_dir}}/endpoint-test-swagger-api.yml.j2"
stage: "lambdabased"
region: '{{ec2_region}}'
aws_access_key: '{{aws_access_key}}'
aws_secret_key: '{{aws_secret_key}}'
security_token: '{{security_token}}'
register: create_result
ignore_errors: true
always:
# ============================================================
- name: destroy lambda for test cleanup if created
lambda:
name="{{lambda_function_name}}"
ec2_region='{{ec2_region}}'
ec2_access_key='{{ec2_access_key}}'
ec2_secret_key='{{ec2_secret_key}}'
security_token='{{security_token}}'
state=absent
register: result
- name: destroy API for test cleanup if created
aws_api_gateway:
state: absent
api_id: '{{api_id}}'
region: '{{ec2_region}}'
aws_access_key: '{{ec2_access_key}}'
aws_secret_key: '{{ec2_secret_key}}'
security_token: '{{security_token}}'
register: destroy_result
- name: assert destroy statements succeeded
assert:
that:
- 'destroy_result.changed == True'
- 'not result|failed'

View file

@ -0,0 +1,39 @@
---
swagger: "2.0"
info:
version: "2017-05-11T12:14:59Z"
title: "{{resource_prefix}}LambdaBased_API"
host: "fakeexample.execute-api.us-east-1.amazonaws.com"
basePath: "/lambdabased"
schemes:
- "https"
paths:
/mini/{greet_name}:
get:
produces:
- "application/json"
parameters:
- name: "greet_name"
in: "path"
required: true
type: "string"
responses:
200:
description: "200 response"
schema:
$ref: "#/definitions/Empty"
x-amazon-apigateway-integration:
responses:
default:
statusCode: "200"
uri: "{{mini_lambda_uri}}"
requestTemplates:
application/json: "{\"statusCode\": 200}"
passthroughBehavior: "when_no_match"
httpMethod: "POST"
contentHandling: "CONVERT_TO_TEXT"
type: "aws_proxy"
definitions:
Empty:
type: "object"
title: "Empty Schema"