publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-08-04 04:34:18 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

undo acs

Browse source
This commit is contained in:
Saran Ahluwalia 2022-01-12 18:53:13 -05:00
commit 1856034fe1
1 changed files with 4 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

20
data/data-pipeline/data_pipeline/etl/sources/census_acs/etl.py
View file

@ -5,7 +5,6 @@ from data_pipeline.etl.sources.census_acs.etl_utils import (
retrieve_census_acs_data, retrieve_census_acs_data,
) )
from data_pipeline.utils import get_module_logger from data_pipeline.utils import get_module_logger
from data_pipeline.score import field_names
logger = get_module_logger(__name__) logger = get_module_logger(__name__)
@ -23,7 +22,7 @@ class CensusACSETL(ExtractTransformLoad):
self.TOTAL_UNEMPLOYED_FIELD, self.TOTAL_UNEMPLOYED_FIELD,
self.TOTAL_IN_LABOR_FORCE, self.TOTAL_IN_LABOR_FORCE,
] ]
self.UNEMPLOYED_FIELD_NAME = "Unemployment (percent)" self.UNEMPLOYED_FIELD_NAME = "Unemployed civilians (percent)"
self.LINGUISTIC_ISOLATION_FIELD_NAME = "Linguistic isolation (percent)" self.LINGUISTIC_ISOLATION_FIELD_NAME = "Linguistic isolation (percent)"
self.LINGUISTIC_ISOLATION_TOTAL_FIELD_NAME = ( self.LINGUISTIC_ISOLATION_TOTAL_FIELD_NAME = (
@ -354,29 +353,18 @@ class CensusACSETL(ExtractTransformLoad):
+ df[self.COLLEGE_ATTENDANCE_FEMALE_ENROLLED_PRIVATE] + df[self.COLLEGE_ATTENDANCE_FEMALE_ENROLLED_PRIVATE]
) / df[self.COLLEGE_ATTENDANCE_TOTAL_POPULATION_ASKED] ) / df[self.COLLEGE_ATTENDANCE_TOTAL_POPULATION_ASKED]
# strip columns
df = df[self.COLUMNS_TO_KEEP]
# Save results to self. # Save results to self.
self.df = df self.df = df
# rename columns to be used in score
rename_fields = {
"Percent of individuals < 200% Federal Poverty Line": field_names.POVERTY_LESS_THAN_200_FPL_FIELD,
}
self.df.rename(
columns=rename_fields,
inplace=True,
errors="raise",
)
def load(self) -> None: def load(self) -> None:
logger.info("Saving Census ACS Data") logger.info("Saving Census ACS Data")
# mkdir census # mkdir census
self.OUTPUT_PATH.mkdir(parents=True, exist_ok=True) self.OUTPUT_PATH.mkdir(parents=True, exist_ok=True)
self.df.to_csv(path_or_buf=self.OUTPUT_PATH / "usa.csv", index=False) self.df[self.COLUMNS_TO_KEEP].to_csv(
path_or_buf=self.OUTPUT_PATH / "usa.csv", index=False
)
def validate(self) -> None: def validate(self) -> None:
logger.info("Validating Census ACS Data") logger.info("Validating Census ACS Data")