Allow loading dirs from role defaults/vars (#36357)

This commit moves code to look for vars files/dirs to a common place and
uses it for loading role defaults/vars. This allows things such as
'defaults/main' or 'vars/main' being a directory in a role, allowing
splitting of defaults/vars into multiple files. This commit also fixes
the role loading unit tests for py3 when bytestrings are used for paths
instead of utf8 strings.

This fixes #14248 and #11639.

test/units/mock/loader.py

@@ -23,7 +23,7 @@ import os
 
 from ansible.errors import AnsibleParserError
 from ansible.parsing.dataloader import DataLoader
-from ansible.module_utils._text import to_bytes
+from ansible.module_utils._text import to_bytes, to_text
 
 
 class DictDataLoader(DataLoader):
@@ -39,6 +39,7 @@ class DictDataLoader(DataLoader):
         self._vault_secrets = None
 
     def load_from_file(self, path, unsafe=False):
+        path = to_text(path)
         if path in self._file_mapping:
             return self.load(self._file_mapping[path], path)
         return None
@@ -46,22 +47,32 @@ class DictDataLoader(DataLoader):
     # TODO: the real _get_file_contents returns a bytestring, so we actually convert the
     #       unicode/text it's created with to utf-8
     def _get_file_contents(self, path):
+        path = to_text(path)
         if path in self._file_mapping:
             return (to_bytes(self._file_mapping[path]), False)
         else:
             raise AnsibleParserError("file not found: %s" % path)
 
     def path_exists(self, path):
+        path = to_text(path)
         return path in self._file_mapping or path in self._known_directories
 
     def is_file(self, path):
+        path = to_text(path)
         return path in self._file_mapping
 
     def is_directory(self, path):
+        path = to_text(path)
         return path in self._known_directories
 
     def list_directory(self, path):
-        return [x for x in self._known_directories]
+        ret = []
+        path = to_text(path)
+        for x in (list(self._file_mapping.keys()) + self._known_directories):
+            if x.startswith(path):
+                if os.path.dirname(x) == path:
+                    ret.append(os.path.basename(x))
+        return ret
 
     def is_executable(self, path):
         # FIXME: figure out a way to make paths return true for this

test/units/playbook/role/test_role.py

@@ -187,6 +187,68 @@ class TestRole(unittest.TestCase):
         self.assertEqual(r._default_vars, dict(foo='bar'))
         self.assertEqual(r._role_vars, dict(foo='bam'))
 
+    @patch('ansible.playbook.role.definition.unfrackpath', mock_unfrackpath_noop)
+    def test_load_role_with_vars_dirs(self):
+
+        fake_loader = DictDataLoader({
+            "/etc/ansible/roles/foo_vars/defaults/main/foo.yml": """
+            foo: bar
+            """,
+            "/etc/ansible/roles/foo_vars/vars/main/bar.yml": """
+            foo: bam
+            """,
+        })
+
+        mock_play = MagicMock()
+        mock_play.ROLE_CACHE = {}
+
+        i = RoleInclude.load('foo_vars', play=mock_play, loader=fake_loader)
+        r = Role.load(i, play=mock_play)
+
+        self.assertEqual(r._default_vars, dict(foo='bar'))
+        self.assertEqual(r._role_vars, dict(foo='bam'))
+
+    @patch('ansible.playbook.role.definition.unfrackpath', mock_unfrackpath_noop)
+    def test_load_role_with_vars_nested_dirs(self):
+
+        fake_loader = DictDataLoader({
+            "/etc/ansible/roles/foo_vars/defaults/main/foo/bar.yml": """
+            foo: bar
+            """,
+            "/etc/ansible/roles/foo_vars/vars/main/bar/foo.yml": """
+            foo: bam
+            """,
+        })
+
+        mock_play = MagicMock()
+        mock_play.ROLE_CACHE = {}
+
+        i = RoleInclude.load('foo_vars', play=mock_play, loader=fake_loader)
+        r = Role.load(i, play=mock_play)
+
+        self.assertEqual(r._default_vars, dict(foo='bar'))
+        self.assertEqual(r._role_vars, dict(foo='bam'))
+
+    @patch('ansible.playbook.role.definition.unfrackpath', mock_unfrackpath_noop)
+    def test_load_role_with_vars_dir_vs_file(self):
+
+        fake_loader = DictDataLoader({
+            "/etc/ansible/roles/foo_vars/vars/main/foo.yml": """
+            foo: bar
+            """,
+            "/etc/ansible/roles/foo_vars/vars/main.yml": """
+            foo: bam
+            """,
+        })
+
+        mock_play = MagicMock()
+        mock_play.ROLE_CACHE = {}
+
+        i = RoleInclude.load('foo_vars', play=mock_play, loader=fake_loader)
+        r = Role.load(i, play=mock_play)
+
+        self.assertEqual(r._role_vars, dict(foo='bam'))
+
     @patch('ansible.playbook.role.definition.unfrackpath', mock_unfrackpath_noop)
     def test_load_role_with_metadata(self):