YAML Config for Downloadable Assets (#1252)

* starting yaml config load work

* working version for downloadable file

* yaml file update

* checkpoint

* sort if needed

* refactoring

* moving config

* checkpoint

* old files

* skipping downloadble tests for now

* more modularization

* more refactor, new excel yml

* pylint

* completed tabs

* Update excel.yml

* remvoing obsolete tests

* addressing PR feedback

* addressing changes

* confirmed change in yaml breaks tests

* safety bump

* PR review

* adding tests back

* pylint

* Incorporating latest score fields from Emma

* incorporating newest fields from Emma

* passing tests

* adding shapefile aws sync

* missing test

* passing tests

data/data-pipeline/data_pipeline/utils.py

@@ -9,6 +9,7 @@ import zipfile
 from pathlib import Path
 import urllib3
 import requests
+import yaml
 
 from data_pipeline.config import settings
 
@@ -322,6 +323,57 @@ def zip_directory(
     )
 
 
+def load_yaml_dict_from_file(yaml_file_path: Path) -> dict:
+    """Load a YAML file specified in path into a Python dictionary.
+
+    Args:
+        yaml_file_path (int): the path to the YAML file
+
+    Returns:
+        dict: the parsed YAML object as a Python dictionary
+    """
+    with open(yaml_file_path, encoding="UTF-8") as file:
+        yaml_dict = yaml.load(file, Loader=yaml.FullLoader)
+    return yaml_dict
+
+
+def column_list_from_yaml_object_fields(
+    yaml_object: dict, target_field: str
+) -> list:
+    """Creates a list of the columns from a YAML score configuration file fields list.
+
+    Args:
+        yaml_object (dict): raw dictionary returned from reading the YAML score configuration file
+        target_field (str): the dict field to extract
+
+    Returns:
+        list: a list of all the fields that match the target field
+    """
+    yaml_list = []
+    for field in yaml_object:
+        yaml_list.append(field[target_field])
+    return yaml_list
+
+
+def load_dict_from_yaml_object_fields(
+    yaml_object: dict, object_key: str, object_value: str
+) -> dict:
+    """Creates a dictionary with a configurable key and value from a YAML score configuration file fields list.
+
+    Args:
+        yaml_object (dict): raw dictionary returned from reading the YAML score configuratio nfile
+        object_key (str): key for the dictionary
+        object_value (str): value for the dictionary
+
+    Returns:
+        dict: a dict with the specified keys and values
+    """
+    yaml_dict = {}
+    for field in yaml_object:
+        yaml_dict[field[object_key]] = field[object_value]
+    return yaml_dict
+
+
 def get_excel_column_name(index: int) -> str:
     """Map a numeric index to the appropriate column in Excel. E.g., column #95 is "CR".
     Only works for the first 1000 columns.