ansible-collections/community.general
1
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2025-07-22 12:50:22 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Initial commit

Browse source
This commit is contained in:
Ansible Core Team 2020-03-09 09:11:07 +00:00
commit aebc1b03fd
4861 changed files with 812621 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

575
plugins/modules/clustering/consul/consul.py Normal file
View file

@ -0,0 +1,575 @@
#!/usr/bin/python
#
# (c) 2015, Steve Gargan <steve.gargan@gmail.com>
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import absolute_import, division, print_function
__metaclass__ = type
ANSIBLE_METADATA = {'metadata_version': '1.1',
'status': ['preview'],
'supported_by': 'community'}
DOCUMENTATION = '''
module: consul
short_description: "Add, modify & delete services within a consul cluster."
description:
- Registers services and checks for an agent with a consul cluster.
A service is some process running on the agent node that should be advertised by
consul's discovery mechanism. It may optionally supply a check definition,
a periodic service test to notify the consul cluster of service's health.
- "Checks may also be registered per node e.g. disk usage, or cpu usage and
notify the health of the entire node to the cluster.
Service level checks do not require a check name or id as these are derived
by Consul from the Service name and id respectively by appending 'service:'
Node level checks require a I(check_name) and optionally a I(check_id)."
- Currently, there is no complete way to retrieve the script, interval or ttl
metadata for a registered check. Without this metadata it is not possible to
tell if the data supplied with ansible represents a change to a check. As a
result this does not attempt to determine changes and will always report a
changed occurred. An API method is planned to supply this metadata so at that
stage change management will be added.
- "See U(http://consul.io) for more details."
requirements:
- python-consul
- requests
author: "Steve Gargan (@sgargan)"
options:
state:
description:
- register or deregister the consul service, defaults to present
default: present
choices: ['present', 'absent']
service_name:
type: str
description:
- Unique name for the service on a node, must be unique per node,
required if registering a service. May be omitted if registering
a node level check
service_id:
type: str
description:
- the ID for the service, must be unique per node. If I(state=absent),
defaults to the service name if supplied.
host:
type: str
description:
- host of the consul agent defaults to localhost
default: localhost
port:
type: int
description:
- the port on which the consul agent is running
default: 8500
scheme:
type: str
description:
- the protocol scheme on which the consul agent is running
default: http
validate_certs:
description:
- whether to verify the TLS certificate of the consul agent
type: bool
default: 'yes'
notes:
type: str
description:
- Notes to attach to check when registering it.
service_port:
type: int
description:
- the port on which the service is listening. Can optionally be supplied for
registration of a service, i.e. if I(service_name) or I(service_id) is set
service_address:
type: str
description:
- the address to advertise that the service will be listening on.
This value will be passed as the I(address) parameter to Consul's
U(/v1/agent/service/register) API method, so refer to the Consul API
documentation for further details.
tags:
type: list
description:
- tags that will be attached to the service registration.
script:
type: str
description:
- the script/command that will be run periodically to check the health
of the service. Scripts require I(interval) and vice versa.
interval:
type: str
description:
- the interval at which the service check will be run. This is a number
with a s or m suffix to signify the units of seconds or minutes e.g
C(15s) or C(1m). If no suffix is supplied, m will be used by default e.g.
C(1) will be C(1m). Required if the I(script) parameter is specified.
check_id:
type: str
description:
- an ID for the service check. If I(state=absent), defaults to
I(check_name). Ignored if part of a service definition.
check_name:
type: str
description:
- a name for the service check. Required if standalone, ignored if
part of service definition.
ttl:
type: str
description:
- checks can be registered with a ttl instead of a I(script) and I(interval)
this means that the service will check in with the agent before the
ttl expires. If it doesn't the check will be considered failed.
Required if registering a check and the script an interval are missing
Similar to the interval this is a number with a s or m suffix to
signify the units of seconds or minutes e.g C(15s) or C(1m). If no suffix
is supplied, C(m) will be used by default e.g. C(1) will be C(1m)
http:
type: str
description:
- checks can be registered with an HTTP endpoint. This means that consul
will check that the http endpoint returns a successful HTTP status.
I(interval) must also be provided with this option.
timeout:
type: str
description:
- A custom HTTP check timeout. The consul default is 10 seconds.
Similar to the interval this is a number with a C(s) or C(m) suffix to
signify the units of seconds or minutes, e.g. C(15s) or C(1m).
token:
type: str
description:
- the token key identifying an ACL rule set. May be required to register services.
'''
EXAMPLES = '''
- name: register nginx service with the local consul agent
consul:
service_name: nginx
service_port: 80
- name: register nginx service with curl check
consul:
service_name: nginx
service_port: 80
script: curl http://localhost
interval: 60s
- name: register nginx with an http check
consul:
service_name: nginx
service_port: 80
interval: 60s
http: http://localhost:80/status
- name: register external service nginx available at 10.1.5.23
consul:
service_name: nginx
service_port: 80
service_address: 10.1.5.23
- name: register nginx with some service tags
consul:
service_name: nginx
service_port: 80
tags:
- prod
- webservers
- name: remove nginx service
consul:
service_name: nginx
state: absent
- name: register celery worker service
consul:
service_name: celery-worker
tags:
- prod
- worker
- name: create a node level check to test disk usage
consul:
check_name: Disk usage
check_id: disk_usage
script: /opt/disk_usage.py
interval: 5m
- name: register an http check against a service that's already registered
consul:
check_name: nginx-check2
check_id: nginx-check2
service_id: nginx
interval: 60s
http: http://localhost:80/morestatus
'''
try:
import consul
from requests.exceptions import ConnectionError
class PatchedConsulAgentService(consul.Consul.Agent.Service):
def deregister(self, service_id, token=None):
params = {}
if token:
params['token'] = token
return self.agent.http.put(consul.base.CB.bool(),
'/v1/agent/service/deregister/%s' % service_id,
params=params)
python_consul_installed = True
except ImportError:
python_consul_installed = False
from ansible.module_utils.basic import AnsibleModule
def register_with_consul(module):
state = module.params.get('state')
if state == 'present':
add(module)
else:
remove(module)
def add(module):
''' adds a service or a check depending on supplied configuration'''
check = parse_check(module)
service = parse_service(module)
if not service and not check:
module.fail_json(msg='a name and port are required to register a service')
if service:
if check:
service.add_check(check)
add_service(module, service)
elif check:
add_check(module, check)
def remove(module):
''' removes a service or a check '''
service_id = module.params.get('service_id') or module.params.get('service_name')
check_id = module.params.get('check_id') or module.params.get('check_name')
if not (service_id or check_id):
module.fail_json(msg='services and checks are removed by id or name. please supply a service id/name or a check id/name')
if service_id:
remove_service(module, service_id)
else:
remove_check(module, check_id)
def add_check(module, check):
''' registers a check with the given agent. currently there is no way
retrieve the full metadata of an existing check through the consul api.
Without this we can't compare to the supplied check and so we must assume
a change. '''
if not check.name and not check.service_id:
module.fail_json(msg='a check name is required for a node level check, one not attached to a service')
consul_api = get_consul_api(module)
check.register(consul_api)
module.exit_json(changed=True,
check_id=check.check_id,
check_name=check.name,
script=check.script,
interval=check.interval,
ttl=check.ttl,
http=check.http,
timeout=check.timeout,
service_id=check.service_id)
def remove_check(module, check_id):
''' removes a check using its id '''
consul_api = get_consul_api(module)
if check_id in consul_api.agent.checks():
consul_api.agent.check.deregister(check_id)
module.exit_json(changed=True, id=check_id)
module.exit_json(changed=False, id=check_id)
def add_service(module, service):
''' registers a service with the current agent '''
result = service
changed = False
consul_api = get_consul_api(module)
existing = get_service_by_id_or_name(consul_api, service.id)
# there is no way to retrieve the details of checks so if a check is present
# in the service it must be re-registered
if service.has_checks() or not existing or not existing == service:
service.register(consul_api)
# check that it registered correctly
registered = get_service_by_id_or_name(consul_api, service.id)
if registered:
result = registered
changed = True
module.exit_json(changed=changed,
service_id=result.id,
service_name=result.name,
service_port=result.port,
checks=[check.to_dict() for check in service.checks],
tags=result.tags)
def remove_service(module, service_id):
''' deregister a service from the given agent using its service id '''
consul_api = get_consul_api(module)
service = get_service_by_id_or_name(consul_api, service_id)
if service:
consul_api.agent.service.deregister(service_id, token=module.params.get('token'))
module.exit_json(changed=True, id=service_id)
module.exit_json(changed=False, id=service_id)
def get_consul_api(module, token=None):
consulClient = consul.Consul(host=module.params.get('host'),
port=module.params.get('port'),
scheme=module.params.get('scheme'),
verify=module.params.get('validate_certs'),
token=module.params.get('token'))
consulClient.agent.service = PatchedConsulAgentService(consulClient)
return consulClient
def get_service_by_id_or_name(consul_api, service_id_or_name):
''' iterate the registered services and find one with the given id '''
for name, service in consul_api.agent.services().items():
if service['ID'] == service_id_or_name or service['Service'] == service_id_or_name:
return ConsulService(loaded=service)
def parse_check(module):
if len([p for p in (module.params.get('script'), module.params.get('ttl'), module.params.get('http')) if p]) > 1:
module.fail_json(
msg='checks are either script, http or ttl driven, supplying more than one does not make sense')
if module.params.get('check_id') or module.params.get('script') or module.params.get('ttl') or module.params.get('http'):
return ConsulCheck(
module.params.get('check_id'),
module.params.get('check_name'),
module.params.get('check_node'),
module.params.get('check_host'),
module.params.get('script'),
module.params.get('interval'),
module.params.get('ttl'),
module.params.get('notes'),
module.params.get('http'),
module.params.get('timeout'),
module.params.get('service_id'),
)
def parse_service(module):
if module.params.get('service_name'):
return ConsulService(
module.params.get('service_id'),
module.params.get('service_name'),
module.params.get('service_address'),
module.params.get('service_port'),
module.params.get('tags'),
)
elif not module.params.get('service_name'):
module.fail_json(msg="service_name is required to configure a service.")
class ConsulService():
def __init__(self, service_id=None, name=None, address=None, port=-1,
tags=None, loaded=None):
self.id = self.name = name
if service_id:
self.id = service_id
self.address = address
self.port = port
self.tags = tags
self.checks = []
if loaded:
self.id = loaded['ID']
self.name = loaded['Service']
self.port = loaded['Port']
self.tags = loaded['Tags']
def register(self, consul_api):
optional = {}
if self.port:
optional['port'] = self.port
if len(self.checks) > 0:
optional['check'] = self.checks[0].check
consul_api.agent.service.register(
self.name,
service_id=self.id,
address=self.address,
tags=self.tags,
**optional)
def add_check(self, check):
self.checks.append(check)
def checks(self):
return self.checks
def has_checks(self):
return len(self.checks) > 0
def __eq__(self, other):
return (isinstance(other, self.__class__) and
self.id == other.id and
self.name == other.name and
self.port == other.port and
self.tags == other.tags)
def __ne__(self, other):
return not self.__eq__(other)
def to_dict(self):
data = {'id': self.id, "name": self.name}
if self.port:
data['port'] = self.port
if self.tags and len(self.tags) > 0:
data['tags'] = self.tags
if len(self.checks) > 0:
data['check'] = self.checks[0].to_dict()
return data
class ConsulCheck(object):
def __init__(self, check_id, name, node=None, host='localhost',
script=None, interval=None, ttl=None, notes=None, http=None, timeout=None, service_id=None):
self.check_id = self.name = name
if check_id:
self.check_id = check_id
self.service_id = service_id
self.notes = notes
self.node = node
self.host = host
self.interval = self.validate_duration('interval', interval)
self.ttl = self.validate_duration('ttl', ttl)
self.script = script
self.http = http
self.timeout = self.validate_duration('timeout', timeout)
self.check = None
if script:
self.check = consul.Check.script(script, self.interval)
if ttl:
self.check = consul.Check.ttl(self.ttl)
if http:
if interval is None:
raise Exception('http check must specify interval')
self.check = consul.Check.http(http, self.interval, self.timeout)
def validate_duration(self, name, duration):
if duration:
duration_units = ['ns', 'us', 'ms', 's', 'm', 'h']
if not any((duration.endswith(suffix) for suffix in duration_units)):
duration = "{0}s".format(duration)
return duration
def register(self, consul_api):
consul_api.agent.check.register(self.name, check_id=self.check_id, service_id=self.service_id,
notes=self.notes,
check=self.check)
def __eq__(self, other):
return (isinstance(other, self.__class__) and
self.check_id == other.check_id and
self.service_id == other.service_id and
self.name == other.name and
self.script == other.script and
self.interval == other.interval)
def __ne__(self, other):
return not self.__eq__(other)
def to_dict(self):
data = {}
self._add(data, 'id', attr='check_id')
self._add(data, 'name', attr='check_name')
self._add(data, 'script')
self._add(data, 'node')
self._add(data, 'notes')
self._add(data, 'host')
self._add(data, 'interval')
self._add(data, 'ttl')
self._add(data, 'http')
self._add(data, 'timeout')
self._add(data, 'service_id')
return data
def _add(self, data, key, attr=None):
try:
if attr is None:
attr = key
data[key] = getattr(self, attr)
except Exception:
pass
def test_dependencies(module):
if not python_consul_installed:
module.fail_json(msg="python-consul required for this module. see https://python-consul.readthedocs.io/en/latest/#installation")
def main():
module = AnsibleModule(
argument_spec=dict(
host=dict(default='localhost'),
port=dict(default=8500, type='int'),
scheme=dict(required=False, default='http'),
validate_certs=dict(required=False, default=True, type='bool'),
check_id=dict(required=False),
check_name=dict(required=False),
check_node=dict(required=False),
check_host=dict(required=False),
notes=dict(required=False),
script=dict(required=False),
service_id=dict(required=False),
service_name=dict(required=False),
service_address=dict(required=False, type='str', default=None),
service_port=dict(required=False, type='int', default=None),
state=dict(default='present', choices=['present', 'absent']),
interval=dict(required=False, type='str'),
ttl=dict(required=False, type='str'),
http=dict(required=False, type='str'),
timeout=dict(required=False, type='str'),
tags=dict(required=False, type='list'),
token=dict(required=False, no_log=True)
),
supports_check_mode=False,
)
test_dependencies(module)
try:
register_with_consul(module)
except ConnectionError as e:
module.fail_json(msg='Could not connect to consul agent at %s:%s, error was %s' % (
module.params.get('host'), module.params.get('port'), str(e)))
except Exception as e:
module.fail_json(msg=str(e))
if __name__ == '__main__':
main()

660
plugins/modules/clustering/consul/consul_acl.py Normal file
View file

@ -0,0 +1,660 @@
#!/usr/bin/python
#
# (c) 2015, Steve Gargan <steve.gargan@gmail.com>
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import absolute_import, division, print_function
__metaclass__ = type
ANSIBLE_METADATA = {'metadata_version': '1.1',
'status': ['preview'],
'supported_by': 'community'}
DOCUMENTATION = '''
module: consul_acl
short_description: Manipulate Consul ACL keys and rules
description:
- Allows the addition, modification and deletion of ACL keys and associated
rules in a consul cluster via the agent. For more details on using and
configuring ACLs, see https://www.consul.io/docs/guides/acl.html.
author:
- Steve Gargan (@sgargan)
- Colin Nolan (@colin-nolan)
options:
mgmt_token:
description:
- a management token is required to manipulate the acl lists
state:
description:
- whether the ACL pair should be present or absent
required: false
choices: ['present', 'absent']
default: present
token_type:
description:
- the type of token that should be created
choices: ['client', 'management']
default: client
name:
description:
- the name that should be associated with the acl key, this is opaque
to Consul
required: false
token:
description:
- the token key identifying an ACL rule set. If generated by consul
this will be a UUID
required: false
rules:
type: list
description:
- rules that should be associated with a given token
required: false
host:
description:
- host of the consul agent defaults to localhost
required: false
default: localhost
port:
type: int
description:
- the port on which the consul agent is running
required: false
default: 8500
scheme:
description:
- the protocol scheme on which the consul agent is running
required: false
default: http
validate_certs:
type: bool
description:
- whether to verify the tls certificate of the consul agent
required: false
default: True
requirements:
- python-consul
- pyhcl
- requests
'''
EXAMPLES = """
- name: create an ACL with rules
consul_acl:
host: consul1.example.com
mgmt_token: some_management_acl
name: Foo access
rules:
- key: "foo"
policy: read
- key: "private/foo"
policy: deny
- name: create an ACL with a specific token
consul_acl:
host: consul1.example.com
mgmt_token: some_management_acl
name: Foo access
token: my-token
rules:
- key: "foo"
policy: read
- name: update the rules associated to an ACL token
consul_acl:
host: consul1.example.com
mgmt_token: some_management_acl
name: Foo access
token: some_client_token
rules:
- event: "bbq"
policy: write
- key: "foo"
policy: read
- key: "private"
policy: deny
- keyring: write
- node: "hgs4"
policy: write
- operator: read
- query: ""
policy: write
- service: "consul"
policy: write
- session: "standup"
policy: write
- name: remove a token
consul_acl:
host: consul1.example.com
mgmt_token: some_management_acl
token: 172bd5c8-9fe9-11e4-b1b0-3c15c2c9fd5e
state: absent
"""
RETURN = """
token:
description: the token associated to the ACL (the ACL's ID)
returned: success
type: str
sample: a2ec332f-04cf-6fba-e8b8-acf62444d3da
rules:
description: the HCL JSON representation of the rules associated to the ACL, in the format described in the
Consul documentation (https://www.consul.io/docs/guides/acl.html#rule-specification).
returned: I(status) == "present"
type: str
sample: {
"key": {
"foo": {
"policy": "write"
},
"bar": {
"policy": "deny"
}
}
}
operation:
description: the operation performed on the ACL
returned: changed
type: str
sample: update
"""
try:
import consul
python_consul_installed = True
except ImportError:
python_consul_installed = False
try:
import hcl
pyhcl_installed = True
except ImportError:
pyhcl_installed = False
try:
from requests.exceptions import ConnectionError
has_requests = True
except ImportError:
has_requests = False
from collections import defaultdict
from ansible.module_utils.basic import to_text, AnsibleModule
RULE_SCOPES = ["agent", "event", "key", "keyring", "node", "operator", "query", "service", "session"]
MANAGEMENT_PARAMETER_NAME = "mgmt_token"
HOST_PARAMETER_NAME = "host"
SCHEME_PARAMETER_NAME = "scheme"
VALIDATE_CERTS_PARAMETER_NAME = "validate_certs"
NAME_PARAMETER_NAME = "name"
PORT_PARAMETER_NAME = "port"
RULES_PARAMETER_NAME = "rules"
STATE_PARAMETER_NAME = "state"
TOKEN_PARAMETER_NAME = "token"
TOKEN_TYPE_PARAMETER_NAME = "token_type"
PRESENT_STATE_VALUE = "present"
ABSENT_STATE_VALUE = "absent"
CLIENT_TOKEN_TYPE_VALUE = "client"
MANAGEMENT_TOKEN_TYPE_VALUE = "management"
REMOVE_OPERATION = "remove"
UPDATE_OPERATION = "update"
CREATE_OPERATION = "create"
_POLICY_JSON_PROPERTY = "policy"
_RULES_JSON_PROPERTY = "Rules"
_TOKEN_JSON_PROPERTY = "ID"
_TOKEN_TYPE_JSON_PROPERTY = "Type"
_NAME_JSON_PROPERTY = "Name"
_POLICY_YML_PROPERTY = "policy"
_POLICY_HCL_PROPERTY = "policy"
_ARGUMENT_SPEC = {
MANAGEMENT_PARAMETER_NAME: dict(required=True, no_log=True),
HOST_PARAMETER_NAME: dict(default='localhost'),
SCHEME_PARAMETER_NAME: dict(required=False, default='http'),
VALIDATE_CERTS_PARAMETER_NAME: dict(required=False, type='bool', default=True),
NAME_PARAMETER_NAME: dict(required=False),
PORT_PARAMETER_NAME: dict(default=8500, type='int'),
RULES_PARAMETER_NAME: dict(default=None, required=False, type='list'),
STATE_PARAMETER_NAME: dict(default=PRESENT_STATE_VALUE, choices=[PRESENT_STATE_VALUE, ABSENT_STATE_VALUE]),
TOKEN_PARAMETER_NAME: dict(required=False),
TOKEN_TYPE_PARAMETER_NAME: dict(required=False, choices=[CLIENT_TOKEN_TYPE_VALUE, MANAGEMENT_TOKEN_TYPE_VALUE],
default=CLIENT_TOKEN_TYPE_VALUE)
}
def set_acl(consul_client, configuration):
"""
Sets an ACL based on the given configuration.
:param consul_client: the consul client
:param configuration: the run configuration
:return: the output of setting the ACL
"""
acls_as_json = decode_acls_as_json(consul_client.acl.list())
existing_acls_mapped_by_name = dict((acl.name, acl) for acl in acls_as_json if acl.name is not None)
existing_acls_mapped_by_token = dict((acl.token, acl) for acl in acls_as_json)
if None in existing_acls_mapped_by_token:
raise AssertionError("expecting ACL list to be associated to a token: %s" %
existing_acls_mapped_by_token[None])
if configuration.token is None and configuration.name and configuration.name in existing_acls_mapped_by_name:
# No token but name given so can get token from name
configuration.token = existing_acls_mapped_by_name[configuration.name].token
if configuration.token and configuration.token in existing_acls_mapped_by_token:
return update_acl(consul_client, configuration)
else:
if configuration.token in existing_acls_mapped_by_token:
raise AssertionError()
if configuration.name in existing_acls_mapped_by_name:
raise AssertionError()
return create_acl(consul_client, configuration)
def update_acl(consul_client, configuration):
"""
Updates an ACL.
:param consul_client: the consul client
:param configuration: the run configuration
:return: the output of the update
"""
existing_acl = load_acl_with_token(consul_client, configuration.token)
changed = existing_acl.rules != configuration.rules
if changed:
name = configuration.name if configuration.name is not None else existing_acl.name
rules_as_hcl = encode_rules_as_hcl_string(configuration.rules)
updated_token = consul_client.acl.update(
configuration.token, name=name, type=configuration.token_type, rules=rules_as_hcl)
if updated_token != configuration.token:
raise AssertionError()
return Output(changed=changed, token=configuration.token, rules=configuration.rules, operation=UPDATE_OPERATION)
def create_acl(consul_client, configuration):
"""
Creates an ACL.
:param consul_client: the consul client
:param configuration: the run configuration
:return: the output of the creation
"""
rules_as_hcl = encode_rules_as_hcl_string(configuration.rules) if len(configuration.rules) > 0 else None
token = consul_client.acl.create(
name=configuration.name, type=configuration.token_type, rules=rules_as_hcl, acl_id=configuration.token)
rules = configuration.rules
return Output(changed=True, token=token, rules=rules, operation=CREATE_OPERATION)
def remove_acl(consul, configuration):
"""
Removes an ACL.
:param consul: the consul client
:param configuration: the run configuration
:return: the output of the removal
"""
token = configuration.token
changed = consul.acl.info(token) is not None
if changed:
consul.acl.destroy(token)
return Output(changed=changed, token=token, operation=REMOVE_OPERATION)
def load_acl_with_token(consul, token):
"""
Loads the ACL with the given token (token == rule ID).
:param consul: the consul client
:param token: the ACL "token"/ID (not name)
:return: the ACL associated to the given token
:exception ConsulACLTokenNotFoundException: raised if the given token does not exist
"""
acl_as_json = consul.acl.info(token)
if acl_as_json is None:
raise ConsulACLNotFoundException(token)
return decode_acl_as_json(acl_as_json)
def encode_rules_as_hcl_string(rules):
"""
Converts the given rules into the equivalent HCL (string) representation.
:param rules: the rules
:return: the equivalent HCL (string) representation of the rules. Will be None if there is no rules (see internal
note for justification)
"""
if len(rules) == 0:
# Note: empty string is not valid HCL according to `hcl.load` however, the ACL `Rule` property will be an empty
# string if there is no rules...
return None
rules_as_hcl = ""
for rule in rules:
rules_as_hcl += encode_rule_as_hcl_string(rule)
return rules_as_hcl
def encode_rule_as_hcl_string(rule):
"""
Converts the given rule into the equivalent HCL (string) representation.
:param rule: the rule
:return: the equivalent HCL (string) representation of the rule
"""
if rule.pattern is not None:
return '%s "%s" {\n %s = "%s"\n}\n' % (rule.scope, rule.pattern, _POLICY_HCL_PROPERTY, rule.policy)
else:
return '%s = "%s"\n' % (rule.scope, rule.policy)
def decode_rules_as_hcl_string(rules_as_hcl):
"""
Converts the given HCL (string) representation of rules into a list of rule domain models.
:param rules_as_hcl: the HCL (string) representation of a collection of rules
:return: the equivalent domain model to the given rules
"""
rules_as_hcl = to_text(rules_as_hcl)
rules_as_json = hcl.loads(rules_as_hcl)
return decode_rules_as_json(rules_as_json)
def decode_rules_as_json(rules_as_json):
"""
Converts the given JSON representation of rules into a list of rule domain models.
:param rules_as_json: the JSON representation of a collection of rules
:return: the equivalent domain model to the given rules
"""
rules = RuleCollection()
for scope in rules_as_json:
if not isinstance(rules_as_json[scope], dict):
rules.add(Rule(scope, rules_as_json[scope]))
else:
for pattern, policy in rules_as_json[scope].items():
rules.add(Rule(scope, policy[_POLICY_JSON_PROPERTY], pattern))
return rules
def encode_rules_as_json(rules):
"""
Converts the given rules into the equivalent JSON representation according to the documentation:
https://www.consul.io/docs/guides/acl.html#rule-specification.
:param rules: the rules
:return: JSON representation of the given rules
"""
rules_as_json = defaultdict(dict)
for rule in rules:
if rule.pattern is not None:
if rule.pattern in rules_as_json[rule.scope]:
raise AssertionError()
rules_as_json[rule.scope][rule.pattern] = {
_POLICY_JSON_PROPERTY: rule.policy
}
else:
if rule.scope in rules_as_json:
raise AssertionError()
rules_as_json[rule.scope] = rule.policy
return rules_as_json
def decode_rules_as_yml(rules_as_yml):
"""
Converts the given YAML representation of rules into a list of rule domain models.
:param rules_as_yml: the YAML representation of a collection of rules
:return: the equivalent domain model to the given rules
"""
rules = RuleCollection()
if rules_as_yml:
for rule_as_yml in rules_as_yml:
rule_added = False
for scope in RULE_SCOPES:
if scope in rule_as_yml:
if rule_as_yml[scope] is None:
raise ValueError("Rule for '%s' does not have a value associated to the scope" % scope)
policy = rule_as_yml[_POLICY_YML_PROPERTY] if _POLICY_YML_PROPERTY in rule_as_yml \
else rule_as_yml[scope]
pattern = rule_as_yml[scope] if _POLICY_YML_PROPERTY in rule_as_yml else None
rules.add(Rule(scope, policy, pattern))
rule_added = True
break
if not rule_added:
raise ValueError("A rule requires one of %s and a policy." % ('/'.join(RULE_SCOPES)))
return rules
def decode_acl_as_json(acl_as_json):
"""
Converts the given JSON representation of an ACL into the equivalent domain model.
:param acl_as_json: the JSON representation of an ACL
:return: the equivalent domain model to the given ACL
"""
rules_as_hcl = acl_as_json[_RULES_JSON_PROPERTY]
rules = decode_rules_as_hcl_string(acl_as_json[_RULES_JSON_PROPERTY]) if rules_as_hcl.strip() != "" \
else RuleCollection()
return ACL(
rules=rules,
token_type=acl_as_json[_TOKEN_TYPE_JSON_PROPERTY],
token=acl_as_json[_TOKEN_JSON_PROPERTY],
name=acl_as_json[_NAME_JSON_PROPERTY]
)
def decode_acls_as_json(acls_as_json):
"""
Converts the given JSON representation of ACLs into a list of ACL domain models.
:param acls_as_json: the JSON representation of a collection of ACLs
:return: list of equivalent domain models for the given ACLs (order not guaranteed to be the same)
"""
return [decode_acl_as_json(acl_as_json) for acl_as_json in acls_as_json]
class ConsulACLNotFoundException(Exception):
"""
Exception raised if an ACL with is not found.
"""
class Configuration:
"""
Configuration for this module.
"""
def __init__(self, management_token=None, host=None, scheme=None, validate_certs=None, name=None, port=None,
rules=None, state=None, token=None, token_type=None):
self.management_token = management_token # type: str
self.host = host # type: str
self.scheme = scheme # type: str
self.validate_certs = validate_certs # type: bool
self.name = name # type: str
self.port = port # type: int
self.rules = rules # type: RuleCollection
self.state = state # type: str
self.token = token # type: str
self.token_type = token_type # type: str
class Output:
"""
Output of an action of this module.
"""
def __init__(self, changed=None, token=None, rules=None, operation=None):
self.changed = changed # type: bool
self.token = token # type: str
self.rules = rules # type: RuleCollection
self.operation = operation # type: str
class ACL:
"""
Consul ACL. See: https://www.consul.io/docs/guides/acl.html.
"""
def __init__(self, rules, token_type, token, name):
self.rules = rules
self.token_type = token_type
self.token = token
self.name = name
def __eq__(self, other):
return other \
and isinstance(other, self.__class__) \
and self.rules == other.rules \
and self.token_type == other.token_type \
and self.token == other.token \
and self.name == other.name
def __hash__(self):
return hash(self.rules) ^ hash(self.token_type) ^ hash(self.token) ^ hash(self.name)
class Rule:
"""
ACL rule. See: https://www.consul.io/docs/guides/acl.html#acl-rules-and-scope.
"""
def __init__(self, scope, policy, pattern=None):
self.scope = scope
self.policy = policy
self.pattern = pattern
def __eq__(self, other):
return isinstance(other, self.__class__) \
and self.scope == other.scope \
and self.policy == other.policy \
and self.pattern == other.pattern
def __ne__(self, other):
return not self.__eq__(other)
def __hash__(self):
return (hash(self.scope) ^ hash(self.policy)) ^ hash(self.pattern)
def __str__(self):
return encode_rule_as_hcl_string(self)
class RuleCollection:
"""
Collection of ACL rules, which are part of a Consul ACL.
"""
def __init__(self):
self._rules = {}
for scope in RULE_SCOPES:
self._rules[scope] = {}
def __iter__(self):
all_rules = []
for scope, pattern_keyed_rules in self._rules.items():
for pattern, rule in pattern_keyed_rules.items():
all_rules.append(rule)
return iter(all_rules)
def __len__(self):
count = 0
for scope in RULE_SCOPES:
count += len(self._rules[scope])
return count
def __eq__(self, other):
return isinstance(other, self.__class__) \
and set(self) == set(other)
def __ne__(self, other):
return not self.__eq__(other)
def __str__(self):
return encode_rules_as_hcl_string(self)
def add(self, rule):
"""
Adds the given rule to this collection.
:param rule: model of a rule
:raises ValueError: raised if there already exists a rule for a given scope and pattern
"""
if rule.pattern in self._rules[rule.scope]:
patten_info = " and pattern '%s'" % rule.pattern if rule.pattern is not None else ""
raise ValueError("Duplicate rule for scope '%s'%s" % (rule.scope, patten_info))
self._rules[rule.scope][rule.pattern] = rule
def get_consul_client(configuration):
"""
Gets a Consul client for the given configuration.
Does not check if the Consul client can connect.
:param configuration: the run configuration
:return: Consul client
"""
token = configuration.management_token
if token is None:
token = configuration.token
if token is None:
raise AssertionError("Expecting the management token to always be set")
return consul.Consul(host=configuration.host, port=configuration.port, scheme=configuration.scheme,
verify=configuration.validate_certs, token=token)
def check_dependencies():
"""
Checks that the required dependencies have been imported.
:exception ImportError: if it is detected that any of the required dependencies have not been imported
"""
if not python_consul_installed:
raise ImportError("python-consul required for this module. "
"See: https://python-consul.readthedocs.io/en/latest/#installation")
if not pyhcl_installed:
raise ImportError("pyhcl required for this module. "
"See: https://pypi.org/project/pyhcl/")
if not has_requests:
raise ImportError("requests required for this module. See https://pypi.org/project/requests/")
def main():
"""
Main method.
"""
module = AnsibleModule(_ARGUMENT_SPEC, supports_check_mode=False)
try:
check_dependencies()
except ImportError as e:
module.fail_json(msg=str(e))
configuration = Configuration(
management_token=module.params.get(MANAGEMENT_PARAMETER_NAME),
host=module.params.get(HOST_PARAMETER_NAME),
scheme=module.params.get(SCHEME_PARAMETER_NAME),
validate_certs=module.params.get(VALIDATE_CERTS_PARAMETER_NAME),
name=module.params.get(NAME_PARAMETER_NAME),
port=module.params.get(PORT_PARAMETER_NAME),
rules=decode_rules_as_yml(module.params.get(RULES_PARAMETER_NAME)),
state=module.params.get(STATE_PARAMETER_NAME),
token=module.params.get(TOKEN_PARAMETER_NAME),
token_type=module.params.get(TOKEN_TYPE_PARAMETER_NAME)
)
consul_client = get_consul_client(configuration)
try:
if configuration.state == PRESENT_STATE_VALUE:
output = set_acl(consul_client, configuration)
else:
output = remove_acl(consul_client, configuration)
except ConnectionError as e:
module.fail_json(msg='Could not connect to consul agent at %s:%s, error was %s' % (
configuration.host, configuration.port, str(e)))
raise
return_values = dict(changed=output.changed, token=output.token, operation=output.operation)
if output.rules is not None:
return_values["rules"] = encode_rules_as_json(output.rules)
module.exit_json(**return_values)
if __name__ == "__main__":
main()

333
plugins/modules/clustering/consul/consul_kv.py Normal file
View file

@ -0,0 +1,333 @@
#!/usr/bin/python
#
# (c) 2015, Steve Gargan <steve.gargan@gmail.com>
# (c) 2018 Genome Research Ltd.
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import absolute_import, division, print_function
__metaclass__ = type
ANSIBLE_METADATA = {'metadata_version': '1.1',
'status': ['preview'],
'supported_by': 'community'}
DOCUMENTATION = '''
module: consul_kv
short_description: Manipulate entries in the key/value store of a consul cluster
description:
- Allows the retrieval, addition, modification and deletion of key/value entries in a
consul cluster via the agent. The entire contents of the record, including
the indices, flags and session are returned as C(value).
- If the C(key) represents a prefix then note that when a value is removed, the existing
value if any is returned as part of the results.
- See http://www.consul.io/docs/agent/http.html#kv for more details.
requirements:
- python-consul
- requests
author:
- Steve Gargan (@sgargan)
- Colin Nolan (@colin-nolan)
options:
state:
description:
- The action to take with the supplied key and value. If the state is 'present' and `value` is set, the key
contents will be set to the value supplied and `changed` will be set to `true` only if the value was
different to the current contents. If the state is 'present' and `value` is not set, the existing value
associated to the key will be returned. The state 'absent' will remove the key/value pair,
again 'changed' will be set to true only if the key actually existed
prior to the removal. An attempt can be made to obtain or free the
lock associated with a key/value pair with the states 'acquire' or
'release' respectively. a valid session must be supplied to make the
attempt changed will be true if the attempt is successful, false
otherwise.
choices: [ absent, acquire, present, release ]
default: present
key:
description:
- The key at which the value should be stored.
type: str
required: yes
value:
description:
- The value should be associated with the given key, required if C(state)
is C(present).
type: str
required: yes
recurse:
description:
- If the key represents a prefix, each entry with the prefix can be
retrieved by setting this to C(yes).
type: bool
default: 'no'
retrieve:
description:
- If the I(state) is C(present) and I(value) is set, perform a
read after setting the value and return this value.
default: True
type: bool
session:
description:
- The session that should be used to acquire or release a lock
associated with a key/value pair.
type: str
token:
description:
- The token key identifying an ACL rule set that controls access to
the key value pair
type: str
cas:
description:
- Used when acquiring a lock with a session. If the C(cas) is C(0), then
Consul will only put the key if it does not already exist. If the
C(cas) value is non-zero, then the key is only set if the index matches
the ModifyIndex of that key.
type: str
flags:
description:
- Opaque positive integer value that can be passed when setting a value.
type: str
host:
description:
- Host of the consul agent.
type: str
default: localhost
port:
description:
- The port on which the consul agent is running.
type: int
default: 8500
scheme:
description:
- The protocol scheme on which the consul agent is running.
type: str
default: http
validate_certs:
description:
- Whether to verify the tls certificate of the consul agent.
type: bool
default: 'yes'
'''
EXAMPLES = '''
# If the key does not exist, the value associated to the "data" property in `retrieved_key` will be `None`
# If the key value is empty string, `retrieved_key["data"]["Value"]` will be `None`
- name: retrieve a value from the key/value store
consul_kv:
key: somekey
register: retrieved_key
- name: Add or update the value associated with a key in the key/value store
consul_kv:
key: somekey
value: somevalue
- name: Remove a key from the store
consul_kv:
key: somekey
state: absent
- name: Add a node to an arbitrary group via consul inventory (see consul.ini)
consul_kv:
key: ansible/groups/dc1/somenode
value: top_secret
- name: Register a key/value pair with an associated session
consul_kv:
key: stg/node/server_birthday
value: 20160509
session: "{{ sessionid }}"
state: acquire
'''
from ansible.module_utils._text import to_text
try:
import consul
from requests.exceptions import ConnectionError
python_consul_installed = True
except ImportError:
python_consul_installed = False
from ansible.module_utils.basic import AnsibleModule
# Note: although the python-consul documentation implies that using a key with a value of `None` with `put` has a
# special meaning (https://python-consul.readthedocs.io/en/latest/#consul-kv), if not set in the subsequently API call,
# the value just defaults to an empty string (https://www.consul.io/api/kv.html#create-update-key)
NOT_SET = None
def _has_value_changed(consul_client, key, target_value):
"""
Uses the given Consul client to determine if the value associated to the given key is different to the given target
value.
:param consul_client: Consul connected client
:param key: key in Consul
:param target_value: value to be associated to the key
:return: tuple where the first element is the value of the "X-Consul-Index" header and the second is `True` if the
value has changed (i.e. the stored value is not the target value)
"""
index, existing = consul_client.kv.get(key)
if not existing:
return index, True
try:
changed = to_text(existing['Value'], errors='surrogate_or_strict') != target_value
return index, changed
except UnicodeError:
# Existing value was not decodable but all values we set are valid utf-8
return index, True
def execute(module):
state = module.params.get('state')
if state == 'acquire' or state == 'release':
lock(module, state)
elif state == 'present':
if module.params.get('value') is NOT_SET:
get_value(module)
else:
set_value(module)
elif state == 'absent':
remove_value(module)
else:
module.exit_json(msg="Unsupported state: %s" % (state, ))
def lock(module, state):
consul_api = get_consul_api(module)
session = module.params.get('session')
key = module.params.get('key')
value = module.params.get('value')
if not session:
module.fail(
msg='%s of lock for %s requested but no session supplied' %
(state, key))
index, changed = _has_value_changed(consul_api, key, value)
if changed and not module.check_mode:
if state == 'acquire':
changed = consul_api.kv.put(key, value,
cas=module.params.get('cas'),
acquire=session,
flags=module.params.get('flags'))
else:
changed = consul_api.kv.put(key, value,
cas=module.params.get('cas'),
release=session,
flags=module.params.get('flags'))
module.exit_json(changed=changed,
index=index,
key=key)
def get_value(module):
consul_api = get_consul_api(module)
key = module.params.get('key')
index, existing_value = consul_api.kv.get(key, recurse=module.params.get('recurse'))
module.exit_json(changed=False, index=index, data=existing_value)
def set_value(module):
consul_api = get_consul_api(module)
key = module.params.get('key')
value = module.params.get('value')
if value is NOT_SET:
raise AssertionError('Cannot set value of "%s" to `NOT_SET`' % key)
index, changed = _has_value_changed(consul_api, key, value)
if changed and not module.check_mode:
changed = consul_api.kv.put(key, value,
cas=module.params.get('cas'),
flags=module.params.get('flags'))
stored = None
if module.params.get('retrieve'):
index, stored = consul_api.kv.get(key)
module.exit_json(changed=changed,
index=index,
key=key,
data=stored)
def remove_value(module):
''' remove the value associated with the given key. if the recurse parameter
is set then any key prefixed with the given key will be removed. '''
consul_api = get_consul_api(module)
key = module.params.get('key')
index, existing = consul_api.kv.get(
key, recurse=module.params.get('recurse'))
changed = existing is not None
if changed and not module.check_mode:
consul_api.kv.delete(key, module.params.get('recurse'))
module.exit_json(changed=changed,
index=index,
key=key,
data=existing)
def get_consul_api(module, token=None):
return consul.Consul(host=module.params.get('host'),
port=module.params.get('port'),
scheme=module.params.get('scheme'),
verify=module.params.get('validate_certs'),
token=module.params.get('token'))
def test_dependencies(module):
if not python_consul_installed:
module.fail_json(msg="python-consul required for this module. "
"see https://python-consul.readthedocs.io/en/latest/#installation")
def main():
module = AnsibleModule(
argument_spec=dict(
cas=dict(type='str'),
flags=dict(type='str'),
key=dict(type='str', required=True),
host=dict(type='str', default='localhost'),
scheme=dict(type='str', default='http'),
validate_certs=dict(type='bool', default=True),
port=dict(type='int', default=8500),
recurse=dict(type='bool'),
retrieve=dict(type='bool', default=True),
state=dict(type='str', default='present', choices=['absent', 'acquire', 'present', 'release']),
token=dict(type='str', no_log=True),
value=dict(type='str', default=NOT_SET),
session=dict(type='str'),
),
supports_check_mode=True
)
test_dependencies(module)
try:
execute(module)
except ConnectionError as e:
module.fail_json(msg='Could not connect to consul agent at %s:%s, error was %s' % (
module.params.get('host'), module.params.get('port'), e))
except Exception as e:
module.fail_json(msg=str(e))
if __name__ == '__main__':
main()

280
plugins/modules/clustering/consul/consul_session.py Normal file
View file

@ -0,0 +1,280 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright: (c) 2015, Steve Gargan <steve.gargan@gmail.com>
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import absolute_import, division, print_function
__metaclass__ = type
ANSIBLE_METADATA = {'metadata_version': '1.1',
'status': ['preview'],
'supported_by': 'community'}
DOCUMENTATION = '''
module: consul_session
short_description: Manipulate consul sessions
description:
- Allows the addition, modification and deletion of sessions in a consul
cluster. These sessions can then be used in conjunction with key value pairs
to implement distributed locks. In depth documentation for working with
sessions can be found at http://www.consul.io/docs/internals/sessions.html
requirements:
- python-consul
- requests
author:
- Steve Gargan (@sgargan)
options:
id:
description:
- ID of the session, required when I(state) is either C(info) or
C(remove).
type: str
state:
description:
- Whether the session should be present i.e. created if it doesn't
exist, or absent, removed if present. If created, the I(id) for the
session is returned in the output. If C(absent), I(id) is
required to remove the session. Info for a single session, all the
sessions for a node or all available sessions can be retrieved by
specifying C(info), C(node) or C(list) for the I(state); for C(node)
or C(info), the node I(name) or session I(id) is required as parameter.
choices: [ absent, info, list, node, present ]
type: str
default: present
name:
description:
- The name that should be associated with the session. Required when
I(state=node) is used.
type: str
delay:
description:
- The optional lock delay that can be attached to the session when it
is created. Locks for invalidated sessions ar blocked from being
acquired until this delay has expired. Durations are in seconds.
type: int
default: 15
node:
description:
- The name of the node that with which the session will be associated.
by default this is the name of the agent.
type: str
datacenter:
description:
- The name of the datacenter in which the session exists or should be
created.
type: str
checks:
description:
- Checks that will be used to verify the session health. If
all the checks fail, the session will be invalidated and any locks
associated with the session will be release and can be acquired once
the associated lock delay has expired.
type: list
host:
description:
- The host of the consul agent defaults to localhost.
type: str
default: localhost
port:
description:
- The port on which the consul agent is running.
type: int
default: 8500
scheme:
description:
- The protocol scheme on which the consul agent is running.
type: str
default: http
validate_certs:
description:
- Whether to verify the TLS certificate of the consul agent.
type: bool
default: True
behavior:
description:
- The optional behavior that can be attached to the session when it
is created. This controls the behavior when a session is invalidated.
choices: [ delete, release ]
type: str
default: release
'''
EXAMPLES = '''
- name: register basic session with consul
consul_session:
name: session1
- name: register a session with an existing check
consul_session:
name: session_with_check
checks:
- existing_check_name
- name: register a session with lock_delay
consul_session:
name: session_with_delay
delay: 20s
- name: retrieve info about session by id
consul_session:
id: session_id
state: info
- name: retrieve active sessions
consul_session:
state: list
'''
try:
import consul
from requests.exceptions import ConnectionError
python_consul_installed = True
except ImportError:
python_consul_installed = False
from ansible.module_utils.basic import AnsibleModule
def execute(module):
state = module.params.get('state')
if state in ['info', 'list', 'node']:
lookup_sessions(module)
elif state == 'present':
update_session(module)
else:
remove_session(module)
def lookup_sessions(module):
datacenter = module.params.get('datacenter')
state = module.params.get('state')
consul_client = get_consul_api(module)
try:
if state == 'list':
sessions_list = consul_client.session.list(dc=datacenter)
# Ditch the index, this can be grabbed from the results
if sessions_list and len(sessions_list) >= 2:
sessions_list = sessions_list[1]
module.exit_json(changed=True,
sessions=sessions_list)
elif state == 'node':
node = module.params.get('node')
sessions = consul_client.session.node(node, dc=datacenter)
module.exit_json(changed=True,
node=node,
sessions=sessions)
elif state == 'info':
session_id = module.params.get('id')
session_by_id = consul_client.session.info(session_id, dc=datacenter)
module.exit_json(changed=True,
session_id=session_id,
sessions=session_by_id)
except Exception as e:
module.fail_json(msg="Could not retrieve session info %s" % e)
def update_session(module):
name = module.params.get('name')
delay = module.params.get('delay')
checks = module.params.get('checks')
datacenter = module.params.get('datacenter')
node = module.params.get('node')
behavior = module.params.get('behavior')
consul_client = get_consul_api(module)
try:
session = consul_client.session.create(
name=name,
behavior=behavior,
node=node,
lock_delay=delay,
dc=datacenter,
checks=checks
)
module.exit_json(changed=True,
session_id=session,
name=name,
behavior=behavior,
delay=delay,
checks=checks,
node=node)
except Exception as e:
module.fail_json(msg="Could not create/update session %s" % e)
def remove_session(module):
session_id = module.params.get('id')
consul_client = get_consul_api(module)
try:
consul_client.session.destroy(session_id)
module.exit_json(changed=True,
session_id=session_id)
except Exception as e:
module.fail_json(msg="Could not remove session with id '%s' %s" % (
session_id, e))
def get_consul_api(module):
return consul.Consul(host=module.params.get('host'),
port=module.params.get('port'),
scheme=module.params.get('scheme'),
verify=module.params.get('validate_certs'))
def test_dependencies(module):
if not python_consul_installed:
module.fail_json(msg="python-consul required for this module. "
"see https://python-consul.readthedocs.io/en/latest/#installation")
def main():
argument_spec = dict(
checks=dict(type='list'),
delay=dict(type='int', default='15'),
behavior=dict(type='str', default='release', choices=['release', 'delete']),
host=dict(type='str', default='localhost'),
port=dict(type='int', default=8500),
scheme=dict(type='str', default='http'),
validate_certs=dict(type='bool', default=True),
id=dict(type='str'),
name=dict(type='str'),
node=dict(type='str'),
state=dict(type='str', default='present', choices=['absent', 'info', 'list', 'node', 'present']),
datacenter=dict(type='str'),
)
module = AnsibleModule(
argument_spec=argument_spec,
required_if=[
('state', 'node', ['name']),
('state', 'info', ['id']),
('state', 'remove', ['id']),
],
supports_check_mode=False
)
test_dependencies(module)
try:
execute(module)
except ConnectionError as e:
module.fail_json(msg='Could not connect to consul agent at %s:%s, error was %s' % (
module.params.get('host'), module.params.get('port'), e))
except Exception as e:
module.fail_json(msg=str(e))
if __name__ == '__main__':
main()