Allow loading dirs from role defaults/vars (#36357)

This commit moves code to look for vars files/dirs to a common place and
uses it for loading role defaults/vars. This allows things such as
'defaults/main' or 'vars/main' being a directory in a role, allowing
splitting of defaults/vars into multiple files. This commit also fixes
the role loading unit tests for py3 when bytestrings are used for paths
instead of utf8 strings.

This fixes #14248 and #11639.

lib/ansible/parsing/dataloader.py

@@ -12,6 +12,7 @@ import os.path
 import re
 import tempfile
 
+from ansible import constants as C
 from ansible.errors import AnsibleFileNotFound, AnsibleParserError
 from ansible.module_utils.basic import is_executable
 from ansible.module_utils.six import binary_type, text_type
@@ -393,3 +394,53 @@ class DataLoader:
                 self.cleanup_tmp_file(f)
             except Exception as e:
                 display.warning("Unable to cleanup temp files: %s" % to_native(e))
+
+    def find_vars_files(self, path, name, extensions=None, allow_dir=True):
+        """
+        Find vars files in a given path with specified name. This will find
+        files in a dir named <name>/ or a file called <name> ending in known
+        extensions.
+        """
+
+        b_path = to_bytes(os.path.join(path, name))
+        found = []
+
+        if extensions is None:
+            # Look for file with no extension first to find dir before file
+            extensions = [''] + C.YAML_FILENAME_EXTENSIONS
+        # add valid extensions to name
+        for ext in extensions:
+
+            if '.' in ext:
+                full_path = b_path + to_bytes(ext)
+            elif ext:
+                full_path = b'.'.join([b_path, to_bytes(ext)])
+            else:
+                full_path = b_path
+
+            if self.path_exists(full_path):
+                if self.is_directory(full_path):
+                    if allow_dir:
+                        found.extend(self._get_dir_vars_files(to_text(full_path), extensions))
+                    else:
+                        next
+                else:
+                    found.append(full_path)
+                break
+        return found
+
+    def _get_dir_vars_files(self, path, extensions):
+        found = []
+        for spath in sorted(self.list_directory(path)):
+            if not spath.startswith(u'.') and not spath.endswith(u'~'):  # skip hidden and backups
+
+                ext = os.path.splitext(spath)[-1]
+                full_spath = os.path.join(path, spath)
+
+                if self.is_directory(full_spath) and not ext:  # recursive search if dir
+                    found.extend(self._get_dir_vars_files(full_spath, extensions))
+                elif self.is_file(full_spath) and (not ext or to_text(ext) in extensions):
+                    # only consider files with valid extensions or no extension
+                    found.append(full_spath)
+
+        return found