publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-02-23 01:54:18 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update downloadable zip file (#659)

Browse source 
* Update downloadable zip file

* Don't use spaces in the name, as per #620
* Add the score D columns, as per #596

* fix paths and directories in etl_score_post

while the tests seemed to  be passing, I encountered an error when
running poetry run score, which was caused by us creating a directory
called <name>.csv, instead of creating the parent directory.

Co-authored-by: Shelby Switzer <shelby.switzer@cms.hhs.gov>
This commit is contained in:
Shelby Switzer 2021-09-10 16:06:47 -04:00 • committed by GitHub
parent dc8e82b8e2
commit d7274888b6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 7 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
data/data-pipeline/data_pipeline/etl/score/constants.py
View file

@ -39,7 +39,7 @@ DATA_SCORE_TILES_FILE_PATH = DATA_SCORE_TILES_DIR / "usa.csv"
SCORE_DOWNLOADABLE_DIR = DATA_SCORE_DIR / "downloadable" SCORE_DOWNLOADABLE_DIR = DATA_SCORE_DIR / "downloadable"
SCORE_DOWNLOADABLE_CSV_FILE_PATH = SCORE_DOWNLOADABLE_DIR / "usa.csv" SCORE_DOWNLOADABLE_CSV_FILE_PATH = SCORE_DOWNLOADABLE_DIR / "usa.csv"
SCORE_DOWNLOADABLE_EXCEL_FILE_PATH = SCORE_DOWNLOADABLE_DIR / "usa.xlsx" SCORE_DOWNLOADABLE_EXCEL_FILE_PATH = SCORE_DOWNLOADABLE_DIR / "usa.xlsx"
SCORE_DOWNLOADABLE_ZIP_FILE_PATH = SCORE_DOWNLOADABLE_DIR / "Screening Tool Data.zip" SCORE_DOWNLOADABLE_ZIP_FILE_PATH = SCORE_DOWNLOADABLE_DIR / "Screening_Tool_Data.zip"
# Column subsets # Column subsets
CENSUS_COUNTIES_COLUMNS = ["USPS", "GEOID", "NAME"] CENSUS_COUNTIES_COLUMNS = ["USPS", "GEOID", "NAME"]
@ -104,5 +104,7 @@ DOWNLOADABLE_SCORE_COLUMNS = [
"GEOID10", "GEOID10",
"County Name", "County Name",
"State Name", "State Name",
"Score D (percentile)",
"Score D (top 25th percentile)",
*DOWNLOADABLE_SCORE_INDICATOR_COLUMNS_FULL, *DOWNLOADABLE_SCORE_INDICATOR_COLUMNS_FULL,
] ]

10
data/data-pipeline/data_pipeline/etl/score/etl_score_post.py
View file

@ -219,9 +219,7 @@ class PostScoreETL(ExtractTransformLoad):
self, score_tiles_df: pd.DataFrame, tile_score_path: Path self, score_tiles_df: pd.DataFrame, tile_score_path: Path
) -> None: ) -> None:
logger.info("Saving Tile Score CSV") logger.info("Saving Tile Score CSV")
# TODO: check which are the columns we'll use tile_score_path.parent.mkdir(parents=True, exist_ok=True)
# Related to: https://github.com/usds/justice40-tool/issues/302
tile_score_path.mkdir(parents=True, exist_ok=True)
score_tiles_df.to_csv(tile_score_path, index=False) score_tiles_df.to_csv(tile_score_path, index=False)
def _load_downloadable_zip( def _load_downloadable_zip(
@ -230,9 +228,9 @@ class PostScoreETL(ExtractTransformLoad):
logger.info("Saving Downloadable CSV") logger.info("Saving Downloadable CSV")
downloadable_info_path.mkdir(parents=True, exist_ok=True) downloadable_info_path.mkdir(parents=True, exist_ok=True)
csv_path = downloadable_info_path / "usa.csv" csv_path = constants.SCORE_DOWNLOADABLE_CSV_FILE_PATH
excel_path = downloadable_info_path / "usa.xlsx" excel_path = constants.SCORE_DOWNLOADABLE_EXCEL_FILE_PATH
zip_path = downloadable_info_path / "Screening Tool Data.zip" zip_path = constants.SCORE_DOWNLOADABLE_ZIP_FILE_PATH
logger.info("Writing downloadable csv") logger.info("Writing downloadable csv")
downloadable_df.to_csv(csv_path, index=False) downloadable_df.to_csv(csv_path, index=False)

BIN
data/data-pipeline/data_pipeline/etl/score/tests/snapshots/downloadable_data_expected.pkl
View file

Binary file not shown.