publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-08-06 14:44:18 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Hotfix for fips zip download location + added full-score-run command (#465)

Browse source 
* Hotfix for S3 locations of data sources

* updated README

* lint failures

Co-authored-by: Nat Hillard <Nathaniel.K.Hillard@omb.eop.gov>
This commit is contained in:
Jorge Escobar 2021-08-05 12:55:21 -04:00 • committed by GitHub
commit 4d7465c833
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 26 additions and 74 deletions
Download patch file Download diff file Expand all files Collapse all files

5
data/data-pipeline/etl/sources/calenviroscreen/etl.py
View file

@ -2,13 +2,16 @@ import pandas as pd
from etl.base import ExtractTransformLoad
from utils import get_module_logger
from config import settings
logger = get_module_logger(__name__)
class CalEnviroScreenETL(ExtractTransformLoad):
def __init__(self):
self.CALENVIROSCREEN_FTP_URL = "https://justice40-data.s3.amazonaws.com/data-sources/CalEnviroScreen_4.0_2021.zip"
self.CALENVIROSCREEN_FTP_URL = (
settings.AWS_JUSTICE40_DATASOURCES_URL + "/CalEnviroScreen_4.0_2021.zip"
)
self.CALENVIROSCREEN_CSV = self.TMP_PATH / "CalEnviroScreen_4.0_2021.csv"
self.CSV_PATH = self.DATA_PATH / "dataset" / "calenviroscreen4"

2
data/data-pipeline/etl/sources/census/etl_utils.py
View file

@ -38,7 +38,7 @@ def get_state_fips_codes(data_path: Path) -> list:
if not os.path.isfile(fips_csv_path):
logger.info("Downloading fips from S3 repository")
unzip_file_from_url(
settings.AWS_JUSTICE40_DATA_URL + "/Census/fips_states_2010.zip",
settings.AWS_JUSTICE40_DATASOURCES_URL + "/fips_states_2010.zip",
data_path / "tmp",
data_path / "census" / "csv",
)