mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-07-28 22:01:16 -07:00
Adding HOLC indicator (#1579)
Added HOLC indicator (Historic Redlining Score) from NCRC work; included 3.25 cutoff and low income as part of the housing burden category.
This commit is contained in:
Emma Nechamkin
Emma Nechamkin
parent
f047ca9d83
commit
1782d022a9
10 changed files with 202 additions and 40 deletions
|
|
@ -0,0 +1,72 @@
|
|||
import pandas as pd
|
||||
|
||||
from data_pipeline.etl.base import ExtractTransformLoad
|
||||
from data_pipeline.utils import get_module_logger
|
||||
from data_pipeline.config import settings
|
||||
|
||||
logger = get_module_logger(__name__)
|
||||
|
||||
|
||||
class HistoricRedliningETL(ExtractTransformLoad):
|
||||
def __init__(self):
|
||||
self.CSV_PATH = self.DATA_PATH / "dataset" / "historic_redlining"
|
||||
self.HISTORIC_REDLINING_URL = (
|
||||
settings.AWS_JUSTICE40_DATASOURCES_URL + "/HRS_2010.zip"
|
||||
)
|
||||
self.HISTORIC_REDLINING_FILE_PATH = (
|
||||
self.get_tmp_path() / "HRS_2010.xlsx"
|
||||
)
|
||||
|
||||
self.REDLINING_SCALAR = "Tract-level redlining score"
|
||||
|
||||
self.COLUMNS_TO_KEEP = [
|
||||
self.GEOID_TRACT_FIELD_NAME,
|
||||
self.REDLINING_SCALAR,
|
||||
]
|
||||
self.df: pd.DataFrame
|
||||
|
||||
def extract(self) -> None:
|
||||
logger.info("Downloading Historic Redlining Data")
|
||||
super().extract(
|
||||
self.HISTORIC_REDLINING_URL,
|
||||
self.get_tmp_path(),
|
||||
)
|
||||
|
||||
def transform(self) -> None:
|
||||
logger.info("Transforming Historic Redlining Data")
|
||||
# this is obviously temporary
|
||||
historic_redlining_data = pd.read_excel(
|
||||
self.HISTORIC_REDLINING_FILE_PATH
|
||||
)
|
||||
historic_redlining_data[self.GEOID_TRACT_FIELD_NAME] = (
|
||||
historic_redlining_data["GEOID10"].astype(str).str.zfill(11)
|
||||
)
|
||||
historic_redlining_data = historic_redlining_data.rename(
|
||||
columns={"HRS2010": self.REDLINING_SCALAR}
|
||||
)
|
||||
|
||||
logger.info(f"{historic_redlining_data.columns}")
|
||||
|
||||
# Calculate lots of different score thresholds for convenience
|
||||
for threshold in [3.25, 3.5, 3.75]:
|
||||
historic_redlining_data[
|
||||
f"{self.REDLINING_SCALAR} meets or exceeds {round(threshold, 2)}"
|
||||
] = (historic_redlining_data[self.REDLINING_SCALAR] >= threshold)
|
||||
## NOTE We add to columns to keep here
|
||||
self.COLUMNS_TO_KEEP.append(
|
||||
f"{self.REDLINING_SCALAR} meets or exceeds {round(threshold, 2)}"
|
||||
)
|
||||
|
||||
self.df = historic_redlining_data
|
||||
|
||||
def load(self) -> None:
|
||||
logger.info("Saving Historic Redlining CSV")
|
||||
# write selected states csv
|
||||
self.CSV_PATH.mkdir(parents=True, exist_ok=True)
|
||||
self.df[self.COLUMNS_TO_KEEP].to_csv(
|
||||
self.CSV_PATH / "usa.csv", index=False
|
||||
)
|
||||
|
||||
def validate(self) -> None:
|
||||
logger.info("Validating Historic Redlining Data")
|
||||
pass
|
||||
|
|
@ -47,16 +47,21 @@ class MappingInequalityETL(ExtractTransformLoad):
|
|||
self.HOLC_GRADE_AND_ID_FIELD: str = "holc_id"
|
||||
self.CITY_INPUT_FIELD: str = "city"
|
||||
|
||||
self.HOLC_GRADE_D_FIELD: str = "HOLC Grade D"
|
||||
self.HOLC_GRADE_D_FIELD: str = "HOLC Grade D (hazardous)"
|
||||
self.HOLC_GRADE_C_FIELD: str = "HOLC Grade C (declining)"
|
||||
self.HOLC_GRADE_MANUAL_FIELD: str = "HOLC Grade (manually mapped)"
|
||||
self.HOLC_GRADE_DERIVED_FIELD: str = "HOLC Grade (derived)"
|
||||
|
||||
self.COLUMNS_TO_KEEP = [
|
||||
self.GEOID_TRACT_FIELD_NAME,
|
||||
field_names.HOLC_GRADE_C_TRACT_PERCENT_FIELD,
|
||||
field_names.HOLC_GRADE_C_OR_D_TRACT_PERCENT_FIELD,
|
||||
field_names.HOLC_GRADE_C_OR_D_TRACT_50_PERCENT_FIELD,
|
||||
field_names.HOLC_GRADE_D_TRACT_PERCENT_FIELD,
|
||||
field_names.HOLC_GRADE_D_TRACT_20_PERCENT_FIELD,
|
||||
field_names.HOLC_GRADE_D_TRACT_50_PERCENT_FIELD,
|
||||
field_names.HOLC_GRADE_D_TRACT_75_PERCENT_FIELD,
|
||||
field_names.REDLINED_SHARE,
|
||||
]
|
||||
|
||||
self.df: pd.DataFrame
|
||||
|
|
@ -113,34 +118,58 @@ class MappingInequalityETL(ExtractTransformLoad):
|
|||
how="left",
|
||||
)
|
||||
|
||||
# Create a single field that combines the 'derived' grade D field with the
|
||||
# manually mapped grade D field into a single grade D field.
|
||||
merged_df[self.HOLC_GRADE_D_FIELD] = np.where(
|
||||
(merged_df[self.HOLC_GRADE_DERIVED_FIELD] == "D")
|
||||
| (merged_df[self.HOLC_GRADE_MANUAL_FIELD] == "D"),
|
||||
True,
|
||||
None,
|
||||
)
|
||||
# Create a single field that combines the 'derived' grade C and D fields with the
|
||||
# manually mapped grade C and D field into a single grade C and D field.
|
||||
## Note: there are no manually derived C tracts at the moment
|
||||
|
||||
# Start grouping by, to sum all of the grade D parts of each tract.
|
||||
grouped_df = (
|
||||
merged_df.groupby(
|
||||
by=[
|
||||
self.GEOID_TRACT_FIELD_NAME,
|
||||
self.HOLC_GRADE_D_FIELD,
|
||||
],
|
||||
# Keep the nulls, so we know the non-D proportion.
|
||||
dropna=False,
|
||||
)[self.TRACT_PROPORTION_FIELD]
|
||||
for grade, field_name in [
|
||||
("C", self.HOLC_GRADE_C_FIELD),
|
||||
("D", self.HOLC_GRADE_D_FIELD),
|
||||
]:
|
||||
merged_df[field_name] = np.where(
|
||||
(merged_df[self.HOLC_GRADE_DERIVED_FIELD] == grade)
|
||||
| (merged_df[self.HOLC_GRADE_MANUAL_FIELD] == grade),
|
||||
True,
|
||||
None,
|
||||
)
|
||||
|
||||
redlined_dataframes_list = [
|
||||
merged_df[merged_df[field].fillna(False)]
|
||||
.groupby(self.GEOID_TRACT_FIELD_NAME)[self.TRACT_PROPORTION_FIELD]
|
||||
.sum()
|
||||
.rename(new_name)
|
||||
for field, new_name in [
|
||||
(
|
||||
self.HOLC_GRADE_D_FIELD,
|
||||
field_names.HOLC_GRADE_D_TRACT_PERCENT_FIELD,
|
||||
),
|
||||
(
|
||||
self.HOLC_GRADE_C_FIELD,
|
||||
field_names.HOLC_GRADE_C_TRACT_PERCENT_FIELD,
|
||||
),
|
||||
]
|
||||
]
|
||||
|
||||
# Group by tract ID to get tract proportions of just C or just D
|
||||
# This produces a single row per tract
|
||||
grouped_df = (
|
||||
pd.concat(
|
||||
redlined_dataframes_list,
|
||||
axis=1,
|
||||
)
|
||||
.fillna(0)
|
||||
.reset_index()
|
||||
)
|
||||
|
||||
# Create a field that is only the percent that is grade D.
|
||||
grouped_df[field_names.HOLC_GRADE_D_TRACT_PERCENT_FIELD] = np.where(
|
||||
grouped_df[self.HOLC_GRADE_D_FIELD],
|
||||
grouped_df[self.TRACT_PROPORTION_FIELD],
|
||||
0,
|
||||
grouped_df[
|
||||
field_names.HOLC_GRADE_C_OR_D_TRACT_PERCENT_FIELD
|
||||
] = grouped_df[
|
||||
[
|
||||
field_names.HOLC_GRADE_C_TRACT_PERCENT_FIELD,
|
||||
field_names.HOLC_GRADE_D_TRACT_PERCENT_FIELD,
|
||||
]
|
||||
].sum(
|
||||
axis=1
|
||||
)
|
||||
|
||||
# Calculate some specific threshold cutoffs, for convenience.
|
||||
|
|
@ -154,15 +183,14 @@ class MappingInequalityETL(ExtractTransformLoad):
|
|||
grouped_df[field_names.HOLC_GRADE_D_TRACT_PERCENT_FIELD] > 0.75
|
||||
)
|
||||
|
||||
# Drop the non-True values of `self.HOLC_GRADE_D_FIELD` -- we only
|
||||
# want one row per tract for future joins.
|
||||
# Note this means not all tracts will be in this data.
|
||||
# Note: this singleton comparison warning may be a pylint bug:
|
||||
# https://stackoverflow.com/questions/51657715/pylint-pandas-comparison-to-true-should-be-just-expr-or-expr-is-true-sin#comment90876517_51657715
|
||||
# pylint: disable=singleton-comparison
|
||||
grouped_df = grouped_df[
|
||||
grouped_df[self.HOLC_GRADE_D_FIELD] == True # noqa: E712
|
||||
]
|
||||
grouped_df[field_names.HOLC_GRADE_C_OR_D_TRACT_50_PERCENT_FIELD] = (
|
||||
grouped_df[field_names.HOLC_GRADE_C_OR_D_TRACT_PERCENT_FIELD] > 0.5
|
||||
)
|
||||
|
||||
# Create the indicator we will use
|
||||
grouped_df[field_names.REDLINED_SHARE] = (
|
||||
grouped_df[field_names.HOLC_GRADE_C_OR_D_TRACT_PERCENT_FIELD] > 0.5
|
||||
) & (grouped_df[field_names.HOLC_GRADE_D_TRACT_PERCENT_FIELD] > 0)
|
||||
|
||||
# Sort for convenience.
|
||||
grouped_df.sort_values(by=self.GEOID_TRACT_FIELD_NAME, inplace=True)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue