publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-02-22 17:44:20 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

Stop swallowing Census API errors (#1051)

Browse source
This commit is contained in:
Lucas Merrill Brown 2021-12-16 10:54:41 -05:00 • committed by GitHub
parent d90e028c1b
commit 0d57dd572b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 23 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
data/data-pipeline/data_pipeline/etl/sources/census_acs/etl_utils.py
View file

@ -8,6 +8,8 @@ from data_pipeline.utils import get_module_logger
logger = get_module_logger(__name__)
CENSUS_ACS_FIPS_CODES_TO_SKIP = ["60", "66", "69", "78"]
def _fips_from_censusdata_censusgeo(censusgeo: censusdata.censusgeo) -> str:
"""Create a FIPS code from the proprietary censusgeo index."""
@ -22,32 +24,34 @@ def retrieve_census_acs_data(
tract_output_field_name: str,
data_path_for_fips_codes: Path,
acs_type="acs5",
raise_errors: bool = False,
) -> pd.DataFrame:
"""Retrieves and combines census ACS data for a given year."""
dfs = []
for fips in get_state_fips_codes(data_path_for_fips_codes):
logger.info(
f"Downloading data for state/territory with FIPS code {fips}"
)
try:
response = censusdata.download(
src=acs_type,
year=acs_year,
geo=censusdata.censusgeo(
[("state", fips), ("county", "*"), ("tract", "*")]
),
var=variables,
if fips in CENSUS_ACS_FIPS_CODES_TO_SKIP:
logger.info(
f"Skipping download for state/territory with FIPS code {fips}"
)
dfs.append(response)
except ValueError as e:
logger.error(
f"Could not download data for state/territory with FIPS code {fips}"
else:
logger.info(
f"Downloading data for state/territory with FIPS code {fips}"
)
if raise_errors:
try:
response = censusdata.download(
src=acs_type,
year=acs_year,
geo=censusdata.censusgeo(
[("state", fips), ("county", "*"), ("tract", "*")]
),
var=variables,
)
dfs.append(response)
except ValueError as e:
logger.error(
f"Could not download data for state/territory with FIPS code {fips}"
)
raise e
df = pd.concat(dfs)

2
data/data-pipeline/data_pipeline/etl/sources/census_acs_2010/etl.py
View file

@ -50,7 +50,6 @@ class CensusACS2010ETL(ExtractTransformLoad):
]
self.EMPLOYMENT_LESS_THAN_HS_IN_LABOR_FORCE = (
# TODO: FIX!!!!!!
"B23006_005E"
# Estimate!!Total!!Less than high school graduate!!In labor force!!Civilian
)
@ -115,7 +114,6 @@ class CensusACS2010ETL(ExtractTransformLoad):
tract_output_field_name=self.GEOID_TRACT_FIELD_NAME,
data_path_for_fips_codes=self.DATA_PATH,
acs_type=self.ACS_TYPE,
raise_errors=False,
)
def transform(self) -> None: