From fb8be1a8cedddf7eb3fcb97fb2be1e450735bee3 Mon Sep 17 00:00:00 2001 From: Jorge Escobar Date: Fri, 17 Dec 2021 13:48:31 -0500 Subject: [PATCH] Download column order completed --- .../data_pipeline/etl/score/constants.py | 98 +++++++++---------- .../data_pipeline/etl/score/etl_score_post.py | 8 +- 2 files changed, 52 insertions(+), 54 deletions(-) diff --git a/data/data-pipeline/data_pipeline/etl/score/constants.py b/data/data-pipeline/data_pipeline/etl/score/constants.py index e1c15c77..bc8076ab 100644 --- a/data/data-pipeline/data_pipeline/etl/score/constants.py +++ b/data/data-pipeline/data_pipeline/etl/score/constants.py @@ -194,88 +194,84 @@ DOWNLOADABLE_SCORE_COLUMNS = [ field_names.SCORE_L_COMMUNITIES, field_names.TOTAL_POP_FIELD, field_names.FPL_200_SERIES, - field_names.POVERTY_LESS_THAN_200_FPL_FIELD, - field_names.POVERTY_LESS_THAN_200_FPL_FIELD - + field_names.PERCENTILE_FIELD_SUFFIX, - field_names.EXPECTED_AGRICULTURE_LOSS_RATE_FIELD, + field_names.EXPECTED_AGRICULTURE_LOSS_RATE_LOW_INCOME_FIELD, field_names.EXPECTED_AGRICULTURE_LOSS_RATE_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, - field_names.EXPECTED_AGRICULTURE_LOSS_RATE_LOW_INCOME_FIELD, - field_names.EXPECTED_BUILDING_LOSS_RATE_FIELD, + field_names.EXPECTED_AGRICULTURE_LOSS_RATE_FIELD, + field_names.EXPECTED_BUILDING_LOSS_RATE_LOW_INCOME_FIELD, field_names.EXPECTED_BUILDING_LOSS_RATE_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, - field_names.EXPECTED_BUILDING_LOSS_RATE_LOW_INCOME_FIELD, - field_names.EXPECTED_POPULATION_LOSS_RATE_FIELD, + field_names.EXPECTED_POPULATION_LOSS_RATE_LOW_INCOME_FIELD, field_names.EXPECTED_POPULATION_LOSS_RATE_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, - field_names.EXPECTED_POPULATION_LOSS_RATE_LOW_INCOME_FIELD, - field_names.ENERGY_BURDEN_FIELD, - field_names.ENERGY_BURDEN_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.EXPECTED_BUILDING_LOSS_RATE_FIELD, field_names.ENERGY_BURDEN_LOW_INCOME_FIELD, - field_names.PM25_FIELD, - field_names.PM25_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.ENERGY_BURDEN_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.ENERGY_BURDEN_FIELD, field_names.PM25_EXPOSURE_LOW_INCOME_FIELD, - field_names.DIESEL_FIELD, - field_names.DIESEL_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.PM25_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.PM25_FIELD, field_names.DIESEL_PARTICULATE_MATTER_LOW_INCOME_FIELD, - field_names.TRAFFIC_FIELD, - field_names.TRAFFIC_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.DIESEL_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.DIESEL_FIELD, field_names.TRAFFIC_PROXIMITY_LOW_INCOME_FIELD, - field_names.HOUSING_BURDEN_FIELD, - field_names.HOUSING_BURDEN_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.TRAFFIC_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.TRAFFIC_FIELD, field_names.HOUSING_BURDEN_LOW_INCOME_FIELD, - field_names.LEAD_PAINT_FIELD, - field_names.LEAD_PAINT_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.HOUSING_BURDEN_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.HOUSING_BURDEN_FIELD, field_names.LEAD_PAINT_MEDIAN_HOUSE_VALUE_LOW_INCOME_FIELD, - field_names.MEDIAN_HOUSE_VALUE_FIELD, + field_names.LEAD_PAINT_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.LEAD_PAINT_FIELD, field_names.MEDIAN_HOUSE_VALUE_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, - field_names.TSDF_FIELD, - field_names.TSDF_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.MEDIAN_HOUSE_VALUE_FIELD, field_names.HAZARDOUS_WASTE_LOW_INCOME_FIELD, - field_names.NPL_FIELD, - field_names.NPL_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.TSDF_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.TSDF_FIELD, field_names.SUPERFUND_LOW_INCOME_FIELD, - field_names.RMP_FIELD, - field_names.RMP_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.NPL_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.NPL_FIELD, field_names.RMP_LOW_INCOME_FIELD, - field_names.WASTEWATER_FIELD, - field_names.WASTEWATER_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.RMP_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.RMP_FIELD, field_names.WASTEWATER_DISCHARGE_LOW_INCOME_FIELD, - field_names.ASTHMA_FIELD, - field_names.ASTHMA_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.WASTEWATER_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.WASTEWATER_FIELD, field_names.ASTHMA_LOW_INCOME_FIELD, - field_names.DIABETES_FIELD, - field_names.DIABETES_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.ASTHMA_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.ASTHMA_FIELD, field_names.DIABETES_LOW_INCOME_FIELD, - field_names.HEART_DISEASE_FIELD, - field_names.HEART_DISEASE_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.DIABETES_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.DIABETES_FIELD, field_names.HEART_DISEASE_LOW_INCOME_FIELD, - field_names.LIFE_EXPECTANCY_FIELD, - field_names.LOW_LIFE_EXPECTANCY_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.HEART_DISEASE_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.HEART_DISEASE_FIELD, field_names.LOW_LIFE_EXPECTANCY_LOW_INCOME_FIELD, - field_names.MEDIAN_INCOME_AS_PERCENT_OF_AMI_FIELD, + field_names.LOW_LIFE_EXPECTANCY_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.LIFE_EXPECTANCY_FIELD, + field_names.LOW_MEDIAN_INCOME_LOW_HS_EDUCATION_FIELD, field_names.LOW_MEDIAN_INCOME_AS_PERCENT_OF_AMI_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, - field_names.LOW_MEDIAN_INCOME_LOW_HS_EDUCATION_FIELD, - field_names.LINGUISTIC_ISO_FIELD, + field_names.MEDIAN_INCOME_AS_PERCENT_OF_AMI_FIELD, field_names.LINGUISTIC_ISOLATION_LOW_HS_EDUCATION_FIELD, - field_names.UNEMPLOYMENT_FIELD, - field_names.UNEMPLOYMENT_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, field_names.LINGUISTIC_ISO_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, - field_names.POVERTY_LESS_THAN_100_FPL_FIELD, + field_names.LINGUISTIC_ISO_FIELD, + field_names.UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD, + field_names.UNEMPLOYMENT_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.UNEMPLOYMENT_FIELD, + field_names.POVERTY_LOW_HS_EDUCATION_FIELD, + field_names.POVERTY_LESS_THAN_200_FPL_FIELD + + field_names.PERCENTILE_FIELD_SUFFIX, field_names.POVERTY_LESS_THAN_100_FPL_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, - field_names.POVERTY_LOW_HS_EDUCATION_FIELD, - field_names.HIGH_SCHOOL_ED_FIELD, + field_names.POVERTY_LESS_THAN_200_FPL_FIELD, + field_names.POVERTY_LESS_THAN_100_FPL_FIELD, field_names.HIGH_SCHOOL_ED_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, - field_names.LOW_HS_EDUCATION_FIELD, - field_names.THRESHOLD_COUNT, - field_names.UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD, + field_names.HIGH_SCHOOL_ED_FIELD, field_names.COMBINED_UNEMPLOYMENT_2010, - field_names.CENSUS_DECENNIAL_UNEMPLOYMENT_FIELD_2009, field_names.COMBINED_POVERTY_LESS_THAN_100_FPL_FIELD_2010, field_names.ISLAND_AREAS_UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD, field_names.ISLAND_AREAS_POVERTY_LOW_HS_EDUCATION_FIELD, field_names.ISLAND_AREAS_LOW_MEDIAN_INCOME_LOW_HS_EDUCATION_FIELD, - field_names.ISLAND_AREAS_LOW_HS_EDUCATION_FIELD, + field_names.THRESHOLD_COUNT, ] diff --git a/data/data-pipeline/data_pipeline/etl/score/etl_score_post.py b/data/data-pipeline/data_pipeline/etl/score/etl_score_post.py index 8f527c7a..048b0bce 100644 --- a/data/data-pipeline/data_pipeline/etl/score/etl_score_post.py +++ b/data/data-pipeline/data_pipeline/etl/score/etl_score_post.py @@ -235,9 +235,11 @@ class PostScoreETL(ExtractTransformLoad): def _create_downloadable_data( self, score_county_state_merged_df: pd.DataFrame ) -> pd.DataFrame: - return score_county_state_merged_df[ - constants.DOWNLOADABLE_SCORE_COLUMNS - ] + df = score_county_state_merged_df[constants.DOWNLOADABLE_SCORE_COLUMNS] + + # rename fields + + return df def transform(self) -> None: logger.info("Transforming data sources for Score + County CSVs")