Config continued (#31024)

* included inventory and callback in new config

allow inventory to be configurable
updated connection options settings
also updated winrm to work with new configs
removed now obsolete set_host_overrides
added notes for future bcoca, current one is just punting, it's future's problem
updated docs per feedback
added remove group/host methods to inv data
moved fact cache from data to constructed
cleaner/better options
fix when vars are added
extended ignore list to config dicts
updated paramiko connection docs
removed options from base that paramiko already handles
left the look option as it is used by other plugin types
resolve delegation
updated cache doc options
fixed test_script
better fragment merge for options
fixed proxy command
restore ini for proxy
normalized options
moved pipelining to class
updates for host_key_checking
restructured mixins

* fix typo
This commit is contained in:
Brian Coca 2017-11-16 13:49:57 -05:00 committed by GitHub
commit 23b1dbacaf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
32 changed files with 667 additions and 366 deletions

View file

@ -19,13 +19,15 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from collections import MutableMapping
import hashlib
import os
import re
import string
from collections import MutableMapping
from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError
from ansible.plugins import AnsiblePlugin
from ansible.module_utils._text import to_bytes, to_native
from ansible.module_utils.parsing.convert_bool import boolean
from ansible.module_utils.six import string_types
@ -40,131 +42,6 @@ except ImportError:
_SAFE_GROUP = re.compile("[^A-Za-z0-9\_]")
class BaseInventoryPlugin(object):
""" Parses an Inventory Source"""
TYPE = 'generator'
def __init__(self):
self.inventory = None
self.display = display
self._cache = {}
def parse(self, inventory, loader, path, cache=True):
''' Populates self.groups from the given data. Raises an error on any parse failure. '''
self.loader = loader
self.inventory = inventory
self.templar = Templar(loader=loader)
def verify_file(self, path):
''' Verify if file is usable by this plugin, base does minimal accessability check '''
b_path = to_bytes(path, errors='surrogate_or_strict')
return (os.path.exists(b_path) and os.access(b_path, os.R_OK))
def get_cache_prefix(self, path):
''' create predictable unique prefix for plugin/inventory '''
m = hashlib.sha1()
m.update(to_bytes(self.NAME, errors='surrogate_or_strict'))
d1 = m.hexdigest()
n = hashlib.sha1()
n.update(to_bytes(path, errors='surrogate_or_strict'))
d2 = n.hexdigest()
return 's_'.join([d1[:5], d2[:5]])
def clear_cache(self):
pass
def populate_host_vars(self, hosts, variables, group=None, port=None):
if not isinstance(variables, MutableMapping):
raise AnsibleParserError("Invalid data from file, expected dictionary and got:\n\n%s" % to_native(variables))
for host in hosts:
self.inventory.add_host(host, group=group, port=port)
for k in variables:
self.inventory.set_variable(host, k, variables[k])
def _compose(self, template, variables):
''' helper method for pluigns to compose variables for Ansible based on jinja2 expression and inventory vars'''
t = self.templar
t.set_available_variables(variables)
return t.do_template('%s%s%s' % (t.environment.variable_start_string, template, t.environment.variable_end_string), disable_lookups=True)
def _set_composite_vars(self, compose, variables, host, strict=False):
''' loops over compose entries to create vars for hosts '''
if compose and isinstance(compose, dict):
for varname in compose:
try:
composite = self._compose(compose[varname], variables)
except Exception as e:
if strict:
raise AnsibleOptionsError("Could set %s: %s" % (varname, to_native(e)))
continue
self.inventory.set_variable(host, varname, composite)
def _add_host_to_composed_groups(self, groups, variables, host, strict=False):
''' helper to create complex groups for plugins based on jinaj2 conditionals, hosts that meet the conditional are added to group'''
# process each 'group entry'
if groups and isinstance(groups, dict):
self.templar.set_available_variables(variables)
for group_name in groups:
conditional = "{%% if %s %%} True {%% else %%} False {%% endif %%}" % groups[group_name]
try:
result = boolean(self.templar.template(conditional))
except Exception as e:
if strict:
raise AnsibleOptionsError("Could not add to group %s: %s" % (group_name, to_native(e)))
continue
if result:
# ensure group exists
self.inventory.add_group(group_name)
# add host to group
self.inventory.add_child(group_name, host)
def _add_host_to_keyed_groups(self, keys, variables, host, strict=False):
''' helper to create groups for plugins based on variable values and add the corresponding hosts to it'''
if keys and isinstance(keys, list):
for keyed in keys:
if keyed and isinstance(keyed, dict):
prefix = keyed.get('prefix', '')
key = keyed.get('key')
if key is not None:
try:
groups = to_safe_group_name('%s_%s' % (prefix, self._compose(key, variables)))
except Exception as e:
if strict:
raise AnsibleOptionsError("Could not generate group on %s: %s" % (key, to_native(e)))
continue
if isinstance(groups, string_types):
groups = [groups]
if isinstance(groups, list):
for group_name in groups:
if group_name not in self.inventory.groups:
self.inventory.add_group(group_name)
self.inventory.add_child(group_name, host)
else:
raise AnsibleOptionsError("Invalid group name format, expected string or list of strings, got: %s" % type(groups))
else:
raise AnsibleOptionsError("No key supplied, invalid entry")
else:
raise AnsibleOptionsError("Invalid keyed group entry, it must be a dictionary: %s " % keyed)
class BaseFileInventoryPlugin(BaseInventoryPlugin):
""" Parses a File based Inventory Source"""
TYPE = 'storage'
def __init__(self):
super(BaseFileInventoryPlugin, self).__init__()
# Helper methods
def to_safe_group_name(name):
''' Converts 'bad' characters in a string to underscores so they can be used as Ansible hosts or groups '''
@ -251,3 +128,171 @@ def expand_hostname_range(line=None):
all_hosts.append(hname)
return all_hosts
class BaseInventoryPlugin(AnsiblePlugin):
""" Parses an Inventory Source"""
TYPE = 'generator'
def __init__(self):
super(BaseInventoryPlugin, self).__init__()
self._options = {}
self.inventory = None
self.display = display
def parse(self, inventory, loader, path, cache=True):
''' Populates self.groups from the given data. Raises an error on any parse failure. '''
self.loader = loader
self.inventory = inventory
self.templar = Templar(loader=loader)
def verify_file(self, path):
''' Verify if file is usable by this plugin, base does minimal accessability check '''
b_path = to_bytes(path, errors='surrogate_or_strict')
return (os.path.exists(b_path) and os.access(b_path, os.R_OK))
def _populate_host_vars(self, hosts, variables, group=None, port=None):
if not isinstance(variables, MutableMapping):
raise AnsibleParserError("Invalid data from file, expected dictionary and got:\n\n%s" % to_native(variables))
for host in hosts:
self.inventory.add_host(host, group=group, port=port)
for k in variables:
self.inventory.set_variable(host, k, variables[k])
def _read_config_data(self, path):
''' validate config and set options as appropriate '''
config = {}
try:
config = self.loader.load_from_file(path)
except Exception as e:
raise AnsibleParserError(to_native(e))
if not config:
# no data
raise AnsibleParserError("%s is empty" % (to_native(path)))
elif config.get('plugin') != self.NAME:
# this is not my config file
raise AnsibleParserError("Incorrect plugin name in file: %s" % config.get('plugin', 'none found'))
elif not isinstance(config, MutableMapping):
# configs are dictionaries
raise AnsibleParserError('inventory source has invalid structure, it should be a dictionary, got: %s' % type(config))
self.set_options(direct=config)
return config
def _consume_options(self, data):
''' update existing options from file data'''
for k in self._options:
if k in data:
self._options[k] = data.pop(k)
def clear_cache(self):
pass
class BaseFileInventoryPlugin(BaseInventoryPlugin):
""" Parses a File based Inventory Source"""
TYPE = 'storage'
def __init__(self):
super(BaseFileInventoryPlugin, self).__init__()
class Cacheable(object):
_cache = {}
def _get_cache_prefix(self, path):
''' create predictable unique prefix for plugin/inventory '''
m = hashlib.sha1()
m.update(to_bytes(self.NAME, errors='surrogate_or_strict'))
d1 = m.hexdigest()
n = hashlib.sha1()
n.update(to_bytes(path, errors='surrogate_or_strict'))
d2 = n.hexdigest()
return 's_'.join([d1[:5], d2[:5]])
def clear_cache(self):
self._cache = {}
class Constructable(object):
def _compose(self, template, variables):
''' helper method for pluigns to compose variables for Ansible based on jinja2 expression and inventory vars'''
t = self.templar
t.set_available_variables(variables)
return t.do_template('%s%s%s' % (t.environment.variable_start_string, template, t.environment.variable_end_string), disable_lookups=True)
def _set_composite_vars(self, compose, variables, host, strict=False):
''' loops over compose entries to create vars for hosts '''
if compose and isinstance(compose, dict):
for varname in compose:
try:
composite = self._compose(compose[varname], variables)
except Exception as e:
if strict:
raise AnsibleOptionsError("Could set %s: %s" % (varname, to_native(e)))
continue
self.inventory.set_variable(host, varname, composite)
def _add_host_to_composed_groups(self, groups, variables, host, strict=False):
''' helper to create complex groups for plugins based on jinaj2 conditionals, hosts that meet the conditional are added to group'''
# process each 'group entry'
if groups and isinstance(groups, dict):
self.templar.set_available_variables(variables)
for group_name in groups:
conditional = "{%% if %s %%} True {%% else %%} False {%% endif %%}" % groups[group_name]
try:
result = boolean(self.templar.template(conditional))
except Exception as e:
if strict:
raise AnsibleOptionsError("Could not add to group %s: %s" % (group_name, to_native(e)))
continue
if result:
# ensure group exists
self.inventory.add_group(group_name)
# add host to group
self.inventory.add_child(group_name, host)
def _add_host_to_keyed_groups(self, keys, variables, host, strict=False):
''' helper to create groups for plugins based on variable values and add the corresponding hosts to it'''
if keys and isinstance(keys, list):
for keyed in keys:
if keyed and isinstance(keyed, dict):
prefix = keyed.get('prefix', '')
key = keyed.get('key')
if key is not None:
try:
groups = to_safe_group_name('%s_%s' % (prefix, self._compose(key, variables)))
except Exception as e:
if strict:
raise AnsibleOptionsError("Could not generate group on %s: %s" % (key, to_native(e)))
continue
if isinstance(groups, string_types):
groups = [groups]
if isinstance(groups, list):
for group_name in groups:
if group_name not in self.inventory.groups:
self.inventory.add_group(group_name)
self.inventory.add_child(group_name, host)
else:
raise AnsibleOptionsError("Invalid group name format, expected string or list of strings, got: %s" % type(groups))
else:
raise AnsibleOptionsError("No key supplied, invalid entry")
else:
raise AnsibleOptionsError("Invalid keyed group entry, it must be a dictionary: %s " % keyed)

View file

@ -54,17 +54,15 @@ EXAMPLES = '''
import os
from collections import MutableMapping
from ansible import constants as C
from ansible.errors import AnsibleParserError
from ansible.plugins.cache import FactCache
from ansible.plugins.inventory import BaseInventoryPlugin
from ansible.plugins.inventory import BaseInventoryPlugin, Constructable
from ansible.module_utils._text import to_native
from ansible.utils.vars import combine_vars
class InventoryModule(BaseInventoryPlugin):
class InventoryModule(BaseInventoryPlugin, Constructable):
""" constructs groups and vars using Jinaj2 template expressions """
NAME = 'constructed'
@ -91,30 +89,21 @@ class InventoryModule(BaseInventoryPlugin):
super(InventoryModule, self).parse(inventory, loader, path, cache=cache)
try:
data = self.loader.load_from_file(path)
except Exception as e:
raise AnsibleParserError("Unable to parse %s: %s" % (to_native(path), to_native(e)))
self._read_config_data(path)
if not data:
raise AnsibleParserError("%s is empty" % (to_native(path)))
elif not isinstance(data, MutableMapping):
raise AnsibleParserError('inventory source has invalid structure, it should be a dictionary, got: %s' % type(data))
elif data.get('plugin') != self.NAME:
raise AnsibleParserError("%s is not a constructed groups config file, plugin entry must be 'constructed'" % (to_native(path)))
strict = data.get('strict', False)
strict = self._options['strict']
fact_cache = FactCache()
try:
# Go over hosts (less var copies)
for host in inventory.hosts:
# get available variables to templar
hostvars = inventory.hosts[host].get_vars()
if host in self._cache: # adds facts if cache is active
hostvars = combine_vars(hostvars, self._cache[host])
if host in fact_cache: # adds facts if cache is active
hostvars = combine_vars(hostvars, fact_cache[host])
# create composite vars
self._set_composite_vars(data.get('compose'), hostvars, host, strict=strict)
self._set_composite_vars(self._options['compose'], hostvars, host, strict=strict)
# refetch host vars in case new ones have been created above
hostvars = inventory.hosts[host].get_vars()
@ -122,10 +111,10 @@ class InventoryModule(BaseInventoryPlugin):
hostvars = combine_vars(hostvars, self._cache[host])
# constructed groups based on conditionals
self._add_host_to_composed_groups(data.get('groups'), hostvars, host, strict=strict)
self._add_host_to_composed_groups(self._options['groups'], hostvars, host, strict=strict)
# constructed groups based variable values
self._add_host_to_keyed_groups(data.get('keyed_groups'), hostvars, host, strict=strict)
self._add_host_to_keyed_groups(self._options['keyed_groups'], hostvars, host, strict=strict)
except Exception as e:
raise AnsibleParserError("failed to parse %s: %s " % (to_native(path), to_native(e)))

View file

@ -208,7 +208,7 @@ class InventoryModule(BaseFileInventoryPlugin):
# the current group.
if state == 'hosts':
hosts, port, variables = self._parse_host_definition(line)
self.populate_host_vars(hosts, variables, groupname, port)
self._populate_host_vars(hosts, variables, groupname, port)
# [groupname:vars] contains variable definitions that must be
# applied to the current group.

View file

@ -104,7 +104,7 @@ simple_config_file:
import collections
from ansible.errors import AnsibleParserError
from ansible.plugins.inventory import BaseInventoryPlugin
from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable
try:
import os_client_config
@ -115,7 +115,7 @@ except ImportError:
HAS_SHADE = False
class InventoryModule(BaseInventoryPlugin):
class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
''' Host inventory provider for ansible using OpenStack clouds. '''
NAME = 'openstack'
@ -124,13 +124,10 @@ class InventoryModule(BaseInventoryPlugin):
super(InventoryModule, self).parse(inventory, loader, path)
cache_key = self.get_cache_prefix(path)
cache_key = self._get_cache_prefix(path)
# file is config file
try:
self._config_data = self.loader.load_from_file(path)
except Exception as e:
raise AnsibleParserError(e)
self._config_data = self._read_config_data(path)
msg = ''
if not self._config_data:

View file

@ -9,10 +9,21 @@ DOCUMENTATION = '''
inventory: script
version_added: "2.4"
short_description: Executes an inventory script that returns JSON
options:
cache:
description: Toggle the usage of the configured Cache plugin.
default: False
type: boolean
ini:
- section: inventory_plugin_script
key: cache
env:
- name: ANSIBLE_INVENTORY_PLUGIN_SCRIPT_CACHE
description:
- The source provided must an executable that returns Ansible inventory JSON
- The source must accept C(--list) and C(--host <hostname>) as arguments.
C(--host) will only be used if no C(_meta) key is present (performance optimization)
C(--host) will only be used if no C(_meta) key is present.
This is a performance optimization as the script would be called per host otherwise.
notes:
- It takes the place of the previously hardcoded script inventory.
- To function it requires being whitelisted in configuration, which is true by default.
@ -26,10 +37,10 @@ from ansible.errors import AnsibleError, AnsibleParserError
from ansible.module_utils.basic import json_dict_bytes_to_unicode
from ansible.module_utils.six import iteritems
from ansible.module_utils._text import to_native, to_text
from ansible.plugins.inventory import BaseInventoryPlugin
from ansible.plugins.inventory import BaseInventoryPlugin, Cacheable
class InventoryModule(BaseInventoryPlugin):
class InventoryModule(BaseInventoryPlugin, Cacheable):
''' Host inventory parser for ansible using external inventory scripts. '''
NAME = 'script'
@ -61,17 +72,20 @@ class InventoryModule(BaseInventoryPlugin):
return valid
def parse(self, inventory, loader, path, cache=True):
def parse(self, inventory, loader, path, cache=None):
super(InventoryModule, self).parse(inventory, loader, path)
if cache is None:
cache = self._options['cache']
# Support inventory scripts that are not prefixed with some
# path information but happen to be in the current working
# directory when '.' is not in PATH.
cmd = [path, "--list"]
try:
cache_key = self.get_cache_prefix(path)
cache_key = self._get_cache_prefix(path)
if not cache or cache_key not in self._cache:
try:
sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -125,7 +139,7 @@ class InventoryModule(BaseInventoryPlugin):
except AttributeError as e:
raise AnsibleError("Improperly formatted host information for %s: %s" % (host, to_native(e)))
self.populate_host_vars([host], got)
self._populate_host_vars([host], got)
except Exception as e:
raise AnsibleParserError(to_native(e))

View file

@ -12,6 +12,8 @@ DOCUMENTATION = '''
- Get inventory hosts from the local virtualbox installation.
- Uses a <name>.vbox.yaml (or .vbox.yml) YAML configuration file.
- The inventory_hostname is always the 'Name' of the virtualbox instance.
extends_documentation_fragment:
- constructed
options:
running_only:
description: toggles showing all vms vs only those currently running
@ -26,14 +28,6 @@ DOCUMENTATION = '''
description: create vars from virtualbox properties
type: dictionary
default: {}
compose:
description: create vars from jinja2 expressions, these are created AFTER the query block
type: dictionary
default: {}
groups:
description: add hosts to group based on Jinja2 conditionals, these also run after query block
type: dictionary
default: {}
'''
EXAMPLES = '''
@ -54,10 +48,10 @@ from subprocess import Popen, PIPE
from ansible.errors import AnsibleParserError
from ansible.module_utils._text import to_bytes, to_native, to_text
from ansible.plugins.inventory import BaseInventoryPlugin
from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable
class InventoryModule(BaseInventoryPlugin):
class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
''' Host inventory parser for ansible using local virtualbox. '''
NAME = 'virtualbox'
@ -76,33 +70,34 @@ class InventoryModule(BaseInventoryPlugin):
pass
return ret
def _set_variables(self, hostvars, data):
def _set_variables(self, hostvars):
# set vars in inventory from hostvars
for host in hostvars:
query = self._options['query']
# create vars from vbox properties
if data.get('query') and isinstance(data['query'], MutableMapping):
for varname in data['query']:
hostvars[host][varname] = self._query_vbox_data(host, data['query'][varname])
if query and isinstance(query, MutableMapping):
for varname in query:
hostvars[host][varname] = self._query_vbox_data(host, query[varname])
# create composite vars
self._set_composite_vars(data.get('compose'), hostvars, host)
self._set_composite_vars(self._options['compose'], hostvars, host)
# actually update inventory
for key in hostvars[host]:
self.inventory.set_variable(host, key, hostvars[host][key])
# constructed groups based on conditionals
self._add_host_to_composed_groups(data.get('groups'), hostvars, host)
self._add_host_to_composed_groups(self._options['groups'], hostvars, host)
def _populate_from_source(self, source_data, config_data):
def _populate_from_source(self, source_data):
hostvars = {}
prevkey = pref_k = ''
current_host = None
# needed to possibly set ansible_host
netinfo = config_data.get('network_info_path', "/VirtualBox/GuestInfo/Net/0/V4/IP")
netinfo = self._options['network_info_path']
for line in source_data:
try:
@ -149,7 +144,7 @@ class InventoryModule(BaseInventoryPlugin):
prevkey = pref_k
self._set_variables(hostvars, config_data)
self._set_variables(hostvars)
def verify_file(self, path):
@ -163,17 +158,12 @@ class InventoryModule(BaseInventoryPlugin):
super(InventoryModule, self).parse(inventory, loader, path)
cache_key = self.get_cache_prefix(path)
cache_key = self._get_cache_prefix(path)
# file is config file
try:
config_data = self.loader.load_from_file(path)
except Exception as e:
raise AnsibleParserError(to_native(e))
config_data = self._read_config_data(path)
if not config_data or config_data.get('plugin') != self.NAME:
# this is not my config file
raise AnsibleParserError("Incorrect plugin name in file: %s" % config_data.get('plugin', 'none found'))
# set _options from config data
self._consume_options(config_data)
source_data = None
if cache and cache_key in self._cache:
@ -183,8 +173,8 @@ class InventoryModule(BaseInventoryPlugin):
pass
if not source_data:
b_pwfile = to_bytes(config_data.get('settings_password_file'), errors='surrogate_or_strict')
running = config_data.get('running_only', False)
b_pwfile = to_bytes(self._options['settings_password_file'], errors='surrogate_or_strict')
running = self._options['running_only']
# start getting data
cmd = [self.VBOX, b'list', b'-l']
@ -205,4 +195,4 @@ class InventoryModule(BaseInventoryPlugin):
source_data = p.stdout.read()
self._cache[cache_key] = to_text(source_data, errors='surrogate_or_strict')
self._populate_from_source(source_data.splitlines(), config_data)
self._populate_from_source(source_data.splitlines())

View file

@ -18,10 +18,19 @@ DOCUMENTATION = '''
- It takes the place of the previously hardcoded YAML inventory.
- To function it requires being whitelisted in configuration.
options:
yaml_extensions:
description: list of 'valid' extensions for files containing YAML
type: list
default: ['.yaml', '.yml', '.json']
yaml_extensions:
description: list of 'valid' extensions for files containing YAML
type: list
default: ['.yaml', '.yml', '.json']
env:
- name: ANSIBLE_YAML_FILENAME_EXT
- name: ANSIBLE_INVENTORY_PLUGIN_EXTS
ini:
- key: yaml_valid_extensions
section: defaults
- section: inventory_plugin_yaml
key: yaml_valid_extensions
'''
EXAMPLES = '''
all: # keys must be unique, i.e. only one 'hosts' per group
@ -52,7 +61,6 @@ all: # keys must be unique, i.e. only one 'hosts' per group
import os
from collections import MutableMapping
from ansible import constants as C
from ansible.errors import AnsibleParserError
from ansible.module_utils.six import string_types
from ansible.module_utils._text import to_native
@ -73,7 +81,7 @@ class InventoryModule(BaseFileInventoryPlugin):
valid = False
if super(InventoryModule, self).verify_file(path):
file_name, ext = os.path.splitext(path)
if not ext or ext in C.YAML_FILENAME_EXTENSIONS:
if not ext or ext in self._options['yaml_extensions']:
valid = True
return valid
@ -131,7 +139,7 @@ class InventoryModule(BaseFileInventoryPlugin):
elif key == 'hosts':
for host_pattern in group_data['hosts']:
hosts, port = self._parse_host(host_pattern)
self.populate_host_vars(hosts, group_data['hosts'][host_pattern] or {}, group, port)
self._populate_host_vars(hosts, group_data['hosts'][host_pattern] or {}, group, port)
else:
self.display.warning('Skipping unexpected key (%s) in group (%s), only "vars", "children" and "hosts" are valid' % (key, group))