Fact collector ordering deps (#31362)

Add deps/requires for fact collectors

Fact collectors can now set a required_facts
class attribute that will be a set of the names
of fact collectors they require to be run first.

ie, if a collector needs to know the ansible_distribution,
it should set it's required_facts to include 'distribution'

        required_facts = set(['distribution'])

If a collector requires another collector, it gets added
to the selected collector names.

We then topological sort the ordering of the collectors
so that deps work out (ie, 'distribution' will run before
'service_mgr')

required_facts were added to the collectors for:

        - network (requires 'distribution', 'platform')
        - hardware (requires 'platform')
        - service_mgr (requires 'distribution', 'platform')

Fix name references for facts (need 'ansible_' prefix)
is service_mgr

Fixes #30753

lib/ansible/module_utils/facts/collector.py

@@ -36,11 +36,31 @@ import platform
 from ansible.module_utils.facts import timeout
 
 
+class CycleFoundInFactDeps(Exception):
+    '''Indicates there is a cycle in fact collector deps
+
+    If collector-B requires collector-A, and collector-A requires
+    collector-B, that is a cycle. In that case, there is no ordering
+    that will satisfy B before A and A and before B. That will cause this
+    error to be raised.
+    '''
+    pass
+
+
+class UnresolvedFactDep(ValueError):
+    pass
+
+
+class CollectorNotFoundError(KeyError):
+    pass
+
+
 class BaseFactCollector:
     _fact_ids = set()
 
     _platform = 'Generic'
     name = None
+    required_facts = set()
 
     def __init__(self, collectors=None, namespace=None):
         '''Base class for things that collect facts.
@@ -216,28 +236,112 @@ def build_fact_id_to_collector_map(collectors_for_platform):
     return fact_id_to_collector_map, aliases_map
 
 
-def select_collector_classes(collector_names, all_fact_subsets, all_collector_classes):
-    # TODO: can be a set()
-    seen_collector_classes = []
+def select_collector_classes(collector_names, all_fact_subsets):
+    seen_collector_classes = set()
 
     selected_collector_classes = []
 
-    for candidate_collector_class in all_collector_classes:
-        candidate_collector_name = candidate_collector_class.name
-
-        if candidate_collector_name not in collector_names:
-            continue
-
-        collector_classes = all_fact_subsets.get(candidate_collector_name, [])
-
+    for collector_name in collector_names:
+        collector_classes = all_fact_subsets.get(collector_name, [])
         for collector_class in collector_classes:
             if collector_class not in seen_collector_classes:
                 selected_collector_classes.append(collector_class)
-                seen_collector_classes.append(collector_class)
+                seen_collector_classes.add(collector_class)
 
     return selected_collector_classes
 
 
+def _get_requires_by_collector_name(collector_name, all_fact_subsets):
+    required_facts = set()
+
+    try:
+        collector_classes = all_fact_subsets[collector_name]
+    except KeyError:
+        raise CollectorNotFoundError('Fact collector "%s" not found' % collector_name)
+    for collector_class in collector_classes:
+        required_facts.update(collector_class.required_facts)
+    return required_facts
+
+
+def find_unresolved_requires(collector_names, all_fact_subsets):
+    '''Find any collector names that have unresolved requires
+
+    Returns a list of collector names that correspond to collector
+    classes whose .requires_facts() are not in collector_names.
+    '''
+    unresolved = set()
+
+    for collector_name in collector_names:
+        required_facts = _get_requires_by_collector_name(collector_name, all_fact_subsets)
+        for required_fact in required_facts:
+            if required_fact not in collector_names:
+                unresolved.add(required_fact)
+
+    return unresolved
+
+
+def resolve_requires(unresolved_requires, all_fact_subsets):
+    new_names = set()
+    failed = []
+    for unresolved in unresolved_requires:
+        if unresolved in all_fact_subsets:
+            new_names.add(unresolved)
+        else:
+            failed.append(unresolved)
+
+    if failed:
+        raise UnresolvedFactDep('unresolved fact dep %s' % ','.join(failed))
+    return new_names
+
+
+def build_dep_data(collector_names, all_fact_subsets):
+    dep_map = defaultdict(set)
+    for collector_name in collector_names:
+        collector_deps = set()
+        for collector in all_fact_subsets[collector_name]:
+            for dep in collector.required_facts:
+                collector_deps.add(dep)
+        dep_map[collector_name] = collector_deps
+    return dep_map
+
+
+def tsort(dep_map):
+    sorted_list = []
+
+    unsorted_map = dep_map.copy()
+
+    while unsorted_map:
+        acyclic = False
+        for node, edges in list(unsorted_map.items()):
+            for edge in edges:
+                if edge in unsorted_map:
+                    break
+            else:
+                acyclic = True
+                del unsorted_map[node]
+                sorted_list.append((node, edges))
+
+        if not acyclic:
+            raise CycleFoundInFactDeps('Unable to tsort deps, there was a cycle in the graph. sorted=%s' % sorted_list)
+
+    return sorted_list
+
+
+def _solve_deps(collector_names, all_fact_subsets):
+    unresolved = collector_names.copy()
+    solutions = collector_names.copy()
+
+    while True:
+        unresolved = find_unresolved_requires(solutions, all_fact_subsets)
+        if unresolved == set():
+            break
+
+        new_names = resolve_requires(unresolved, all_fact_subsets)
+        solutions.update(new_names)
+
+    return solutions
+
+
 def collector_classes_from_gather_subset(all_collector_classes=None,
                                          valid_subsets=None,
                                          minimal_gather_subset=None,
@@ -283,8 +387,14 @@ def collector_classes_from_gather_subset(all_collector_classes=None,
                                           aliases_map=aliases_map,
                                           platform_info=platform_info)
 
-    selected_collector_classes = select_collector_classes(collector_names,
-                                                          all_fact_subsets,
-                                                          all_collector_classes)
+    complete_collector_names = _solve_deps(collector_names, all_fact_subsets)
+
+    dep_map = build_dep_data(complete_collector_names, all_fact_subsets)
+
+    ordered_deps = tsort(dep_map)
+    ordered_collector_names = [x[0] for x in ordered_deps]
+
+    selected_collector_classes = select_collector_classes(ordered_collector_names,
+                                                          all_fact_subsets)
 
     return selected_collector_classes

lib/ansible/module_utils/facts/hardware/hpux.py

@@ -161,3 +161,5 @@ class HPUXHardware(Hardware):
 class HPUXHardwareCollector(HardwareCollector):
     _fact_class = HPUXHardware
     _platform = 'HP-UX'
+
+    required_facts = set(['platform', 'distribution'])

lib/ansible/module_utils/facts/hardware/linux.py

@@ -710,3 +710,5 @@ class LinuxHardware(Hardware):
 class LinuxHardwareCollector(HardwareCollector):
     _platform = 'Linux'
     _fact_class = LinuxHardware
+
+    required_facts = set(['platform'])

lib/ansible/module_utils/facts/hardware/sunos.py

@@ -263,3 +263,5 @@ class SunOSHardware(Hardware):
 class SunOSHardwareCollector(HardwareCollector):
     _fact_class = SunOSHardware
     _platform = 'SunOS'
+
+    required_facts = set(['platform'])

lib/ansible/module_utils/facts/network/linux.py

@@ -310,3 +310,4 @@ class LinuxNetwork(Network):
 class LinuxNetworkCollector(NetworkCollector):
     _platform = 'Linux'
     _fact_class = LinuxNetwork
+    required_facts = set(['distribution', 'platform'])

lib/ansible/module_utils/facts/system/distribution.py

@@ -579,7 +579,8 @@ class DistributionFactCollector(BaseFactCollector):
     name = 'distribution'
     _fact_ids = set(['distribution_version',
                      'distribution_release',
-                     'distribution_major_version'])
+                     'distribution_major_version',
+                     'os_family'])
 
     def collect(self, module=None, collected_facts=None):
         collected_facts = collected_facts or {}

lib/ansible/module_utils/facts/system/platform.py

@@ -35,6 +35,7 @@ class PlatformFactCollector(BaseFactCollector):
                      'kernel',
                      'machine',
                      'python_version',
+                     'architecture',
                      'machine_id'])
 
     def collect(self, module=None, collected_facts=None):

lib/ansible/module_utils/facts/system/service_mgr.py

@@ -38,6 +38,7 @@ if platform.system() != 'SunOS':
 class ServiceMgrFactCollector(BaseFactCollector):
     name = 'service_mgr'
     _fact_ids = set()
+    required_facts = set(['platform', 'distribution'])
 
     @staticmethod
     def is_systemd_managed(module):