diff --git a/data/data-pipeline/data_pipeline/etl/score/etl_score.py b/data/data-pipeline/data_pipeline/etl/score/etl_score.py index e5674518..c736e3fc 100644 --- a/data/data-pipeline/data_pipeline/etl/score/etl_score.py +++ b/data/data-pipeline/data_pipeline/etl/score/etl_score.py @@ -43,9 +43,7 @@ class ScoreETL(ExtractTransformLoad): logger.info("Loading data sets from disk.") # EJSCreen csv Load - ejscreen_csv = ( - constants.DATA_PATH / "dataset" / "ejscreen_2019" / "usa.csv" - ) + ejscreen_csv = constants.DATA_PATH / "dataset" / "ejscreen" / "usa.csv" self.ejscreen_df = pd.read_csv( ejscreen_csv, dtype={self.GEOID_TRACT_FIELD_NAME: "string"}, diff --git a/data/data-pipeline/data_pipeline/etl/sources/ejscreen/etl.py b/data/data-pipeline/data_pipeline/etl/sources/ejscreen/etl.py index 683dbcd2..0ba5db88 100644 --- a/data/data-pipeline/data_pipeline/etl/sources/ejscreen/etl.py +++ b/data/data-pipeline/data_pipeline/etl/sources/ejscreen/etl.py @@ -8,16 +8,14 @@ logger = get_module_logger(__name__) class EJSCREENETL(ExtractTransformLoad): - """Load EJSCREEN data. - - Data dictionary: - https://gaftp.epa.gov/EJSCREEN/2019/2019_EJSCREEN_columns_explained.csv - """ + """Load updated EJSCREEN data.""" def __init__(self): - self.EJSCREEN_FTP_URL = "https://edap-arcgiscloud-data-commons.s3.amazonaws.com/EJSCREEN2020/EJSCREEN_Tract_2020_USPR.csv.zip" - self.EJSCREEN_CSV = self.get_tmp_path() / "EJSCREEN_Tract_2020_USPR.csv" - self.CSV_PATH = self.DATA_PATH / "dataset" / "ejscreen_2019" + self.EJSCREEN_FTP_URL = "https://gaftp.epa.gov/EJSCREEN/2021/EJSCREEN_2021_USPR_Tracts.csv.zip" + self.EJSCREEN_CSV = ( + self.get_tmp_path() / "EJSCREEN_2021_USPR_Tracts.csv" + ) + self.CSV_PATH = self.DATA_PATH / "dataset" / "ejscreen" self.df: pd.DataFrame self.COLUMNS_TO_KEEP = [ @@ -39,6 +37,7 @@ class EJSCREENETL(ExtractTransformLoad): field_names.OVER_64_FIELD, field_names.UNDER_5_FIELD, field_names.LEAD_PAINT_FIELD, + field_names.UST_FIELD, ] def extract(self) -> None: @@ -53,7 +52,7 @@ class EJSCREENETL(ExtractTransformLoad): logger.info("Transforming EJScreen Data") self.df = pd.read_csv( self.EJSCREEN_CSV, - dtype={"ID": "string"}, + dtype={"ID": str}, # EJSCREEN writes the word "None" for NA data. na_values=["None"], low_memory=False, @@ -63,8 +62,6 @@ class EJSCREENETL(ExtractTransformLoad): self.df.rename( columns={ "ID": self.GEOID_TRACT_FIELD_NAME, - # Note: it is currently unorthodox to use `field_names` in an ETL class, - # but I think that's the direction we'd like to move all ETL classes. - LMB "ACSTOTPOP": field_names.TOTAL_POP_FIELD, "CANCER": field_names.AIR_TOXICS_CANCER_RISK_FIELD, "RESP": field_names.RESPIRATORY_HAZARD_FIELD, @@ -81,6 +78,7 @@ class EJSCREENETL(ExtractTransformLoad): "OVER64PCT": field_names.OVER_64_FIELD, "UNDER5PCT": field_names.UNDER_5_FIELD, "PRE1960PCT": field_names.LEAD_PAINT_FIELD, + "UST": field_names.UST_FIELD, # added for 2021 update }, inplace=True, ) diff --git a/data/data-pipeline/data_pipeline/score/field_names.py b/data/data-pipeline/data_pipeline/score/field_names.py index 3254f08e..04bc62e4 100644 --- a/data/data-pipeline/data_pipeline/score/field_names.py +++ b/data/data-pipeline/data_pipeline/score/field_names.py @@ -170,6 +170,7 @@ TSDF_FIELD = "Proximity to hazardous waste sites" NPL_FIELD = "Proximity to NPL sites" AIR_TOXICS_CANCER_RISK_FIELD = "Air toxics cancer risk" RESPIRATORY_HAZARD_FIELD = "Respiratory hazard index" +UST_FIELD = "Underground storage tanks" LOW_INCOME_THRESHOLD = "Exceeds FPL200 threshold" diff --git a/docs/decisions/0007-files/MapComparison.mp4 b/docs/decisions/0007-files/MapComparison.mp4 deleted file mode 100644 index 273c9392..00000000 Binary files a/docs/decisions/0007-files/MapComparison.mp4 and /dev/null differ