publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-07-28 14:21:16 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

updating ejscreen data, try two (#1747)

Browse source
This commit is contained in:
Emma Nechamkin 2022-07-18 09:59:10 -04:00 • committed by Emma Nechamkin
commit 2ab24c60fa
4 changed files with 11 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

20
data/data-pipeline/data_pipeline/etl/sources/ejscreen/etl.py
View file

@ -8,16 +8,14 @@ logger = get_module_logger(__name__)
class EJSCREENETL(ExtractTransformLoad):
"""Load EJSCREEN data.
Data dictionary:
https://gaftp.epa.gov/EJSCREEN/2019/2019_EJSCREEN_columns_explained.csv
"""
"""Load updated EJSCREEN data."""
def __init__(self):
self.EJSCREEN_FTP_URL = "https://edap-arcgiscloud-data-commons.s3.amazonaws.com/EJSCREEN2020/EJSCREEN_Tract_2020_USPR.csv.zip"
self.EJSCREEN_CSV = self.get_tmp_path() / "EJSCREEN_Tract_2020_USPR.csv"
self.CSV_PATH = self.DATA_PATH / "dataset" / "ejscreen_2019"
self.EJSCREEN_FTP_URL = "https://gaftp.epa.gov/EJSCREEN/2021/EJSCREEN_2021_USPR_Tracts.csv.zip"
self.EJSCREEN_CSV = (
self.get_tmp_path() / "EJSCREEN_2021_USPR_Tracts.csv"
)
self.CSV_PATH = self.DATA_PATH / "dataset" / "ejscreen"
self.df: pd.DataFrame
self.COLUMNS_TO_KEEP = [
@ -39,6 +37,7 @@ class EJSCREENETL(ExtractTransformLoad):
field_names.OVER_64_FIELD,
field_names.UNDER_5_FIELD,
field_names.LEAD_PAINT_FIELD,
field_names.UST_FIELD,
]
def extract(self) -> None:
@ -53,7 +52,7 @@ class EJSCREENETL(ExtractTransformLoad):
logger.info("Transforming EJScreen Data")
self.df = pd.read_csv(
self.EJSCREEN_CSV,
dtype={"ID": "string"},
dtype={"ID": str},
# EJSCREEN writes the word "None" for NA data.
na_values=["None"],
low_memory=False,
@ -63,8 +62,6 @@ class EJSCREENETL(ExtractTransformLoad):
self.df.rename(
columns={
"ID": self.GEOID_TRACT_FIELD_NAME,
# Note: it is currently unorthodox to use `field_names` in an ETL class,
# but I think that's the direction we'd like to move all ETL classes. - LMB
"ACSTOTPOP": field_names.TOTAL_POP_FIELD,
"CANCER": field_names.AIR_TOXICS_CANCER_RISK_FIELD,
"RESP": field_names.RESPIRATORY_HAZARD_FIELD,
@ -81,6 +78,7 @@ class EJSCREENETL(ExtractTransformLoad):
"OVER64PCT": field_names.OVER_64_FIELD,
"UNDER5PCT": field_names.UNDER_5_FIELD,
"PRE1960PCT": field_names.LEAD_PAINT_FIELD,
"UST": field_names.UST_FIELD, # added for 2021 update
},
inplace=True,
)