mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-07-25 01:20:17 -07:00
Refactor DOE Energy Burden and COI to use YAML (#1796)
* added tribalId for Supplemental dataset (#1804) * Setting zoom levels for tribal map (#1810) * NRI dataset and initial score YAML configuration (#1534) * update be staging gha * NRI dataset and initial score YAML configuration * checkpoint * adding data checks for release branch * passing tests * adding INPUT_EXTRACTED_FILE_NAME to base class * lint * columns to keep and tests * update be staging gha * checkpoint * update be staging gha * NRI dataset and initial score YAML configuration * checkpoint * adding data checks for release branch * passing tests * adding INPUT_EXTRACTED_FILE_NAME to base class * lint * columns to keep and tests * checkpoint * PR Review * renoving source url * tests * stop execution of ETL if there's a YAML schema issue * update be staging gha * adding source url as class var again * clean up * force cache bust * gha cache bust * dynamically set score vars from YAML * docsctrings * removing last updated year - optional reverse percentile * passing tests * sort order * column ordening * PR review * class level vars * Updating DatasetsConfig * fix pylint errors * moving metadata hint back to code Co-authored-by: lucasmbrown-usds <lucas.m.brown@omb.eop.gov> * Correct copy typo (#1809) * Add basic test suite for COI (#1518) * Update COI to use new yaml (#1518) * Add tests for DOE energy budren (1518 * Add dataset config for energy budren (1518) * Refactor ETL to use datasets.yml (#1518) * Add fake GEOIDs to COI tests (#1518) * Refactor _setup_etl_instance_and_run_extract to base (#1518) For the three classes we've done so far, a generic _setup_etl_instance_and_run_extract will work fine, for the moment we can reuse the same setup method until we decide future classes need more flexibility --- but they can also always subclass so... * Add output-path tests (#1518) * Update YAML to match constant (#1518) * Don't blindly set float format (#1518) * Add defaults for extract (#1518) * Run YAML load on all subclasses (#1518) * Update description fields (#1518) * Update YAML per final format (#1518) * Update fixture tract IDs (#1518) * Update base class refactor (#1518) Now that NRI is final I needed to make a small number of updates to my refactored code. * Remove old comment (#1518) * Fix type signature and return (#1518) * Update per code review (#1518) Co-authored-by: Jorge Escobar <83969469+esfoobar-usds@users.noreply.github.com> Co-authored-by: lucasmbrown-usds <lucas.m.brown@omb.eop.gov> Co-authored-by: Vim <86254807+vim-usds@users.noreply.github.com>
This commit is contained in:
Matt Bowen
Emma Nechamkin
parent
baa591a6c6
commit
97e17546cc
28 changed files with 455 additions and 189 deletions
|
|
@ -46,7 +46,8 @@ class ExtractTransformLoad:
|
|||
DATA_PATH: pathlib.Path = APP_ROOT / "data"
|
||||
TMP_PATH: pathlib.Path = DATA_PATH / "tmp"
|
||||
CONTENT_CONFIG: pathlib.Path = APP_ROOT / "content" / "config"
|
||||
DATASET_CONFIG: pathlib.Path = APP_ROOT / "etl" / "score" / "config"
|
||||
DATASET_CONFIG_PATH: pathlib.Path = APP_ROOT / "etl" / "score" / "config"
|
||||
DATASET_CONFIG: Optional[dict] = None
|
||||
|
||||
# Parameters
|
||||
GEOID_FIELD_NAME: str = "GEOID10"
|
||||
|
|
@ -98,48 +99,51 @@ class ExtractTransformLoad:
|
|||
# It is used on the "load" base class method
|
||||
output_df: pd.DataFrame = None
|
||||
|
||||
def __init_subclass__(cls) -> None:
|
||||
cls.DATASET_CONFIG = cls.yaml_config_load()
|
||||
|
||||
@classmethod
|
||||
def yaml_config_load(cls) -> dict:
|
||||
def yaml_config_load(cls) -> Optional[dict]:
|
||||
"""Generate config dictionary and set instance variables from YAML dataset."""
|
||||
|
||||
# check if the class instance has score YAML definitions
|
||||
datasets_config = load_yaml_dict_from_file(
|
||||
cls.DATASET_CONFIG / "datasets.yml",
|
||||
DatasetsConfig,
|
||||
)
|
||||
|
||||
# get the config for this dataset
|
||||
try:
|
||||
dataset_config = next(
|
||||
item
|
||||
for item in datasets_config.get("datasets")
|
||||
if item["module_name"] == cls.NAME
|
||||
if cls.NAME is not None:
|
||||
# check if the class instance has score YAML definitions
|
||||
datasets_config = load_yaml_dict_from_file(
|
||||
cls.DATASET_CONFIG_PATH / "datasets.yml",
|
||||
DatasetsConfig,
|
||||
)
|
||||
except StopIteration:
|
||||
# Note: it'd be nice to log the name of the dataframe, but that's not accessible in this scope.
|
||||
logger.error(
|
||||
f"Exception encountered while extracting dataset config for dataset {cls.NAME}"
|
||||
)
|
||||
sys.exit()
|
||||
|
||||
# set some of the basic fields
|
||||
cls.INPUT_GEOID_TRACT_FIELD_NAME = dataset_config[
|
||||
"input_geoid_tract_field_name"
|
||||
]
|
||||
# get the config for this dataset
|
||||
try:
|
||||
dataset_config = next(
|
||||
item
|
||||
for item in datasets_config.get("datasets")
|
||||
if item["module_name"] == cls.NAME
|
||||
)
|
||||
except StopIteration:
|
||||
# Note: it'd be nice to log the name of the dataframe, but that's not accessible in this scope.
|
||||
logger.error(
|
||||
f"Exception encountered while extracting dataset config for dataset {cls.NAME}"
|
||||
)
|
||||
sys.exit()
|
||||
|
||||
# get the columns to write on the CSV
|
||||
# and set the constants
|
||||
cls.COLUMNS_TO_KEEP = [
|
||||
cls.GEOID_TRACT_FIELD_NAME, # always index with geoid tract id
|
||||
]
|
||||
for field in dataset_config["load_fields"]:
|
||||
cls.COLUMNS_TO_KEEP.append(field["long_name"])
|
||||
# set some of the basic fields
|
||||
cls.INPUT_GEOID_TRACT_FIELD_NAME = dataset_config[
|
||||
"input_geoid_tract_field_name"
|
||||
]
|
||||
|
||||
# set the constants for the class
|
||||
setattr(cls, field["df_field_name"], field["long_name"])
|
||||
# get the columns to write on the CSV
|
||||
# and set the constants
|
||||
cls.COLUMNS_TO_KEEP = [
|
||||
cls.GEOID_TRACT_FIELD_NAME, # always index with geoid tract id
|
||||
]
|
||||
for field in dataset_config["load_fields"]:
|
||||
cls.COLUMNS_TO_KEEP.append(field["long_name"])
|
||||
setattr(cls, field["df_field_name"], field["long_name"])
|
||||
|
||||
# return the config dict
|
||||
return dataset_config
|
||||
# set the constants for the class
|
||||
setattr(cls, field["df_field_name"], field["long_name"])
|
||||
return dataset_config
|
||||
return None
|
||||
|
||||
# This is a classmethod so it can be used by `get_data_frame` without
|
||||
# needing to create an instance of the class. This is a use case in `etl_score`.
|
||||
|
|
@ -176,14 +180,18 @@ class ExtractTransformLoad:
|
|||
to get the file from a source url, unzips it and stores it on an
|
||||
extract_path."""
|
||||
|
||||
# this can be accessed via super().extract()
|
||||
if source_url and extract_path:
|
||||
unzip_file_from_url(
|
||||
file_url=source_url,
|
||||
download_path=self.get_tmp_path(),
|
||||
unzipped_file_path=extract_path,
|
||||
verify=verify,
|
||||
)
|
||||
if source_url is None:
|
||||
source_url = self.SOURCE_URL
|
||||
|
||||
if extract_path is None:
|
||||
extract_path = self.get_tmp_path()
|
||||
|
||||
unzip_file_from_url(
|
||||
file_url=source_url,
|
||||
download_path=self.get_tmp_path(),
|
||||
unzipped_file_path=extract_path,
|
||||
verify=verify,
|
||||
)
|
||||
|
||||
def transform(self) -> None:
|
||||
"""Transform the data extracted into a format that can be consumed by the
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue