Fact collector ordering deps (#31362)

Add deps/requires for fact collectors

Fact collectors can now set a required_facts
class attribute that will be a set of the names
of fact collectors they require to be run first.

ie, if a collector needs to know the ansible_distribution,
it should set it's required_facts to include 'distribution'

        required_facts = set(['distribution'])

If a collector requires another collector, it gets added
to the selected collector names.

We then topological sort the ordering of the collectors
so that deps work out (ie, 'distribution' will run before
'service_mgr')

required_facts were added to the collectors for:

        - network (requires 'distribution', 'platform')
        - hardware (requires 'platform')
        - service_mgr (requires 'distribution', 'platform')

Fix name references for facts (need 'ansible_' prefix)
is service_mgr

Fixes #30753
This commit is contained in:
Adrian Likins 2018-01-22 18:23:40 -05:00 committed by GitHub
commit 6e585bdf24
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 439 additions and 51 deletions

View file

@ -36,11 +36,31 @@ import platform
from ansible.module_utils.facts import timeout
class CycleFoundInFactDeps(Exception):
'''Indicates there is a cycle in fact collector deps
If collector-B requires collector-A, and collector-A requires
collector-B, that is a cycle. In that case, there is no ordering
that will satisfy B before A and A and before B. That will cause this
error to be raised.
'''
pass
class UnresolvedFactDep(ValueError):
pass
class CollectorNotFoundError(KeyError):
pass
class BaseFactCollector:
_fact_ids = set()
_platform = 'Generic'
name = None
required_facts = set()
def __init__(self, collectors=None, namespace=None):
'''Base class for things that collect facts.
@ -216,28 +236,112 @@ def build_fact_id_to_collector_map(collectors_for_platform):
return fact_id_to_collector_map, aliases_map
def select_collector_classes(collector_names, all_fact_subsets, all_collector_classes):
# TODO: can be a set()
seen_collector_classes = []
def select_collector_classes(collector_names, all_fact_subsets):
seen_collector_classes = set()
selected_collector_classes = []
for candidate_collector_class in all_collector_classes:
candidate_collector_name = candidate_collector_class.name
if candidate_collector_name not in collector_names:
continue
collector_classes = all_fact_subsets.get(candidate_collector_name, [])
for collector_name in collector_names:
collector_classes = all_fact_subsets.get(collector_name, [])
for collector_class in collector_classes:
if collector_class not in seen_collector_classes:
selected_collector_classes.append(collector_class)
seen_collector_classes.append(collector_class)
seen_collector_classes.add(collector_class)
return selected_collector_classes
def _get_requires_by_collector_name(collector_name, all_fact_subsets):
required_facts = set()
try:
collector_classes = all_fact_subsets[collector_name]
except KeyError:
raise CollectorNotFoundError('Fact collector "%s" not found' % collector_name)
for collector_class in collector_classes:
required_facts.update(collector_class.required_facts)
return required_facts
def find_unresolved_requires(collector_names, all_fact_subsets):
'''Find any collector names that have unresolved requires
Returns a list of collector names that correspond to collector
classes whose .requires_facts() are not in collector_names.
'''
unresolved = set()
for collector_name in collector_names:
required_facts = _get_requires_by_collector_name(collector_name, all_fact_subsets)
for required_fact in required_facts:
if required_fact not in collector_names:
unresolved.add(required_fact)
return unresolved
def resolve_requires(unresolved_requires, all_fact_subsets):
new_names = set()
failed = []
for unresolved in unresolved_requires:
if unresolved in all_fact_subsets:
new_names.add(unresolved)
else:
failed.append(unresolved)
if failed:
raise UnresolvedFactDep('unresolved fact dep %s' % ','.join(failed))
return new_names
def build_dep_data(collector_names, all_fact_subsets):
dep_map = defaultdict(set)
for collector_name in collector_names:
collector_deps = set()
for collector in all_fact_subsets[collector_name]:
for dep in collector.required_facts:
collector_deps.add(dep)
dep_map[collector_name] = collector_deps
return dep_map
def tsort(dep_map):
sorted_list = []
unsorted_map = dep_map.copy()
while unsorted_map:
acyclic = False
for node, edges in list(unsorted_map.items()):
for edge in edges:
if edge in unsorted_map:
break
else:
acyclic = True
del unsorted_map[node]
sorted_list.append((node, edges))
if not acyclic:
raise CycleFoundInFactDeps('Unable to tsort deps, there was a cycle in the graph. sorted=%s' % sorted_list)
return sorted_list
def _solve_deps(collector_names, all_fact_subsets):
unresolved = collector_names.copy()
solutions = collector_names.copy()
while True:
unresolved = find_unresolved_requires(solutions, all_fact_subsets)
if unresolved == set():
break
new_names = resolve_requires(unresolved, all_fact_subsets)
solutions.update(new_names)
return solutions
def collector_classes_from_gather_subset(all_collector_classes=None,
valid_subsets=None,
minimal_gather_subset=None,
@ -283,8 +387,14 @@ def collector_classes_from_gather_subset(all_collector_classes=None,
aliases_map=aliases_map,
platform_info=platform_info)
selected_collector_classes = select_collector_classes(collector_names,
all_fact_subsets,
all_collector_classes)
complete_collector_names = _solve_deps(collector_names, all_fact_subsets)
dep_map = build_dep_data(complete_collector_names, all_fact_subsets)
ordered_deps = tsort(dep_map)
ordered_collector_names = [x[0] for x in ordered_deps]
selected_collector_classes = select_collector_classes(ordered_collector_names,
all_fact_subsets)
return selected_collector_classes

View file

@ -161,3 +161,5 @@ class HPUXHardware(Hardware):
class HPUXHardwareCollector(HardwareCollector):
_fact_class = HPUXHardware
_platform = 'HP-UX'
required_facts = set(['platform', 'distribution'])

View file

@ -710,3 +710,5 @@ class LinuxHardware(Hardware):
class LinuxHardwareCollector(HardwareCollector):
_platform = 'Linux'
_fact_class = LinuxHardware
required_facts = set(['platform'])

View file

@ -263,3 +263,5 @@ class SunOSHardware(Hardware):
class SunOSHardwareCollector(HardwareCollector):
_fact_class = SunOSHardware
_platform = 'SunOS'
required_facts = set(['platform'])

View file

@ -310,3 +310,4 @@ class LinuxNetwork(Network):
class LinuxNetworkCollector(NetworkCollector):
_platform = 'Linux'
_fact_class = LinuxNetwork
required_facts = set(['distribution', 'platform'])

View file

@ -579,7 +579,8 @@ class DistributionFactCollector(BaseFactCollector):
name = 'distribution'
_fact_ids = set(['distribution_version',
'distribution_release',
'distribution_major_version'])
'distribution_major_version',
'os_family'])
def collect(self, module=None, collected_facts=None):
collected_facts = collected_facts or {}

View file

@ -35,6 +35,7 @@ class PlatformFactCollector(BaseFactCollector):
'kernel',
'machine',
'python_version',
'architecture',
'machine_id'])
def collect(self, module=None, collected_facts=None):

View file

@ -38,6 +38,7 @@ if platform.system() != 'SunOS':
class ServiceMgrFactCollector(BaseFactCollector):
name = 'service_mgr'
_fact_ids = set()
required_facts = set(['platform', 'distribution'])
@staticmethod
def is_systemd_managed(module):