publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-02-23 01:54:18 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

Adding ETL script

Browse source
This commit is contained in:
VincentLaUSDS 2021-09-22 14:23:20 -04:00
parent c3fb80fbdc
commit 0d7a9161a8
1 changed files with 21 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
data/data-pipeline/data_pipeline/etl/sources/ejscreen_areas_of_concern/etl.py
View file

@ -8,31 +8,34 @@ logger = get_module_logger(__name__)
class EJScreenAreasOfConcernETL(ExtractTransformLoad):
def __init__(self):
# self.EJSCREEN_FTP_URL = "https://gaftp.epa.gov/EJSCREEN/2019/EJSCREEN_2019_StatePctile.csv.zip"
# self.EJSCREEN_CSV = self.TMP_PATH / "EJSCREEN_2019_StatePctiles.csv"
self.CSV_PATH = self.DATA_PATH / "dataset" / "ejscreen_2019"
self.OUTPUT_PATH = self.DATA_PATH / "dataset" / "ejscreen_areas_of_concern"
# TO DO: Load from actual source; the issue is that this dataset is not public for now
self.LOCAL_CSV_PATH = self.DATA_PATH / "local"
self.GEOID_CBG_FIELD_NAME = "GEOID10_CBG"
self.df: pd.DataFrame
def extract(self) -> None:
logger.info("Downloading EJScreen Data")
super().extract(
self.EJSCREEN_FTP_URL,
self.TMP_PATH,
verify=False, # EPA EJScreen end point has certificate issues often
)
logger.info("Loading EJScreen Areas of Concern Data Locally")
def transform(self) -> None:
logger.info("Transforming EJScreen Data")
self.df = pd.read_csv(
self.EJSCREEN_CSV,
dtype={"ID": "string"},
# EJSCREEN writes the word "None" for NA data.
na_values=["None"],
filepath_or_buffer=self.LOCAL_CSV_PATH
/ "ejscreen_areas_of_concerns_indicators.csv",
dtype={
self.GEOID_CBG_FIELD_NAME: "string",
},
low_memory=False,
)
def transform(self) -> None:
logger.info("Transforming EJScreen Areas of Concern Data")
# TO DO: As a one off we did all the processing in a separate Notebook
# Can add here later for a future PR
pass
def load(self) -> None:
logger.info("Saving EJScreen CSV")
logger.info("Saving EJScreen Areas of Concern Data")
# write nationwide csv
self.CSV_PATH.mkdir(parents=True, exist_ok=True)
self.df.to_csv(self.CSV_PATH / "usa.csv", index=False)
self.OUTPUT_PATH.mkdir(parents=True, exist_ok=True)
self.df.to_csv(self.OUTPUT_PATH / "usa.csv", index=False)