publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-02-23 10:04:18 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

Stop swallowing Census API errors (#1051)

Browse source
This commit is contained in:
Lucas Merrill Brown 2021-12-16 10:54:41 -05:00 • committed by GitHub
parent d90e028c1b
commit 0d57dd572b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 23 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
data/data-pipeline/data_pipeline/etl/sources/census_acs/etl_utils.py
View file

@ -8,6 +8,8 @@ from data_pipeline.utils import get_module_logger
logger = get_module_logger(__name__) logger = get_module_logger(__name__)
CENSUS_ACS_FIPS_CODES_TO_SKIP = ["60", "66", "69", "78"]
def _fips_from_censusdata_censusgeo(censusgeo: censusdata.censusgeo) -> str: def _fips_from_censusdata_censusgeo(censusgeo: censusdata.censusgeo) -> str:
"""Create a FIPS code from the proprietary censusgeo index.""" """Create a FIPS code from the proprietary censusgeo index."""
@ -22,11 +24,15 @@ def retrieve_census_acs_data(
tract_output_field_name: str, tract_output_field_name: str,
data_path_for_fips_codes: Path, data_path_for_fips_codes: Path,
acs_type="acs5", acs_type="acs5",
raise_errors: bool = False,
) -> pd.DataFrame: ) -> pd.DataFrame:
"""Retrieves and combines census ACS data for a given year.""" """Retrieves and combines census ACS data for a given year."""
dfs = [] dfs = []
for fips in get_state_fips_codes(data_path_for_fips_codes): for fips in get_state_fips_codes(data_path_for_fips_codes):
if fips in CENSUS_ACS_FIPS_CODES_TO_SKIP:
logger.info(
f"Skipping download for state/territory with FIPS code {fips}"
)
else:
logger.info( logger.info(
f"Downloading data for state/territory with FIPS code {fips}" f"Downloading data for state/territory with FIPS code {fips}"
) )
@ -46,8 +52,6 @@ def retrieve_census_acs_data(
logger.error( logger.error(
f"Could not download data for state/territory with FIPS code {fips}" f"Could not download data for state/territory with FIPS code {fips}"
) )
if raise_errors:
raise e raise e
df = pd.concat(dfs) df = pd.concat(dfs)

2
data/data-pipeline/data_pipeline/etl/sources/census_acs_2010/etl.py
View file

@ -50,7 +50,6 @@ class CensusACS2010ETL(ExtractTransformLoad):
] ]
self.EMPLOYMENT_LESS_THAN_HS_IN_LABOR_FORCE = ( self.EMPLOYMENT_LESS_THAN_HS_IN_LABOR_FORCE = (
# TODO: FIX!!!!!!
"B23006_005E" "B23006_005E"
# Estimate!!Total!!Less than high school graduate!!In labor force!!Civilian # Estimate!!Total!!Less than high school graduate!!In labor force!!Civilian
) )
@ -115,7 +114,6 @@ class CensusACS2010ETL(ExtractTransformLoad):
tract_output_field_name=self.GEOID_TRACT_FIELD_NAME, tract_output_field_name=self.GEOID_TRACT_FIELD_NAME,
data_path_for_fips_codes=self.DATA_PATH, data_path_for_fips_codes=self.DATA_PATH,
acs_type=self.ACS_TYPE, acs_type=self.ACS_TYPE,
raise_errors=False,
) )
def transform(self) -> None: def transform(self) -> None: