publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-07-28 08:01:17 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Issue 1141: Definition M (#1151)

Browse source
This commit is contained in:
Lucas Merrill Brown 2022-01-18 14:56:55 -05:00 • committed by GitHub
commit 18f299c5f8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 1000 additions and 143 deletions
Download patch file Download diff file Expand all files Collapse all files

2
data/data-pipeline/data_pipeline/etl/sources/cdc_svi_index/etl.py
View file

@ -42,7 +42,7 @@ class CDCSVIIndex(ExtractTransformLoad):
self.df: pd.DataFrame
def extract(self) -> None:
logger.info("Extracting 43 MB CDC SVI INDEX")
logger.info("Downloading 43 MB CDC SVI INDEX")
self.df = pd.read_csv(
filepath_or_buffer=self.CDC_SVI_INDEX_URL,
dtype={self.CDC_SVI_INDEX_TRACTS_FIPS_CODE: "string"},

0
data/data-pipeline/data_pipeline/etl/sources/epa_rsei_aggregate/README.md → data/data-pipeline/data_pipeline/etl/sources/epa_rsei/README.md
View file

0
data/data-pipeline/data_pipeline/etl/sources/epa_rsei_aggregate/__init__.py → data/data-pipeline/data_pipeline/etl/sources/epa_rsei/__init__.py
View file

8
data/data-pipeline/data_pipeline/etl/sources/epa_rsei_aggregate/etl.py → data/data-pipeline/data_pipeline/etl/sources/epa_rsei/etl.py
View file

@ -22,9 +22,7 @@ class EPARiskScreeningEnvironmentalIndicatorsETL(ExtractTransformLoad):
def __init__(self):
self.AGGREGATED_RSEI_SCORE_FILE_URL = "http://abt-rsei.s3.amazonaws.com/microdata2019/census_agg/CensusMicroTracts2019_2019_aggregated.zip"
self.OUTPUT_PATH: Path = (
self.DATA_PATH / "dataset" / "epa_rsei_aggregated"
)
self.OUTPUT_PATH: Path = self.DATA_PATH / "dataset" / "epa_rsei"
self.EPA_RSEI_SCORE_THRESHOLD_CUTOFF = 0.75
self.TRACT_INPUT_COLUMN_NAME = "GEOID10"
self.NUMBER_FACILITIES_INPUT_FIELD = "NUMFACS"
@ -74,12 +72,12 @@ class EPARiskScreeningEnvironmentalIndicatorsETL(ExtractTransformLoad):
unzip_file_from_url(
file_url=self.AGGREGATED_RSEI_SCORE_FILE_URL,
download_path=self.TMP_PATH,
unzipped_file_path=self.TMP_PATH / "epa_rsei_aggregated",
unzipped_file_path=self.TMP_PATH / "epa_rsei",
)
self.df = pd.read_csv(
filepath_or_buffer=self.TMP_PATH
/ "epa_rsei_aggregated"
/ "epa_rsei"
/ "CensusMicroTracts2019_2019_aggregated.csv",
# The following need to remain as strings for all of their digits, not get
# converted to numbers.

2
data/data-pipeline/data_pipeline/etl/sources/maryland_ejscreen/etl.py
View file

@ -33,7 +33,7 @@ class MarylandEJScreenETL(ExtractTransformLoad):
self.df: pd.DataFrame
def extract(self) -> None:
logger.info("Downloading Maryland EJSCREEN Data")
logger.info("Downloading 207MB Maryland EJSCREEN Data")
super().extract(
self.MARYLAND_EJSCREEN_URL,
self.TMP_PATH,