mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-08-13 10:24:19 -07:00
Issue 308 python linting (#443)
* Adds flake8, pylint, liccheck, flake8 to dependencies for data-pipeline * Sets up and runs black autoformatting * Adds flake8 to tox linting * Fixes flake8 error F541 f string missing placeholders * Fixes flake8 E501 line too long * Fixes flake8 F401 imported but not used * Adds pylint to tox and disables the following pylint errors: - C0114: module docstrings - R0201: method could have been a function - R0903: too few public methods - C0103: name case styling - W0511: fix me - W1203: f-string interpolation in logging * Adds utils.py to tox.ini linting, runs black on utils.py * Fixes import related pylint errors: C0411 and C0412 * Fixes or ignores remaining pylint errors (for discussion later) * Adds safety and liccheck to tox.ini
This commit is contained in:
Billy Daly
GitHub
parent
51f7666062
commit
5504528fdf
22 changed files with 709 additions and 228 deletions
|
|
@ -9,16 +9,12 @@ logger = get_module_logger(__name__)
|
|||
class CalEnviroScreenETL(ExtractTransformLoad):
|
||||
def __init__(self):
|
||||
self.CALENVIROSCREEN_FTP_URL = "https://justice40-data.s3.amazonaws.com/data-sources/CalEnviroScreen_4.0_2021.zip"
|
||||
self.CALENVIROSCREEN_CSV = (
|
||||
self.TMP_PATH / "CalEnviroScreen_4.0_2021.csv"
|
||||
)
|
||||
self.CALENVIROSCREEN_CSV = self.TMP_PATH / "CalEnviroScreen_4.0_2021.csv"
|
||||
self.CSV_PATH = self.DATA_PATH / "dataset" / "calenviroscreen4"
|
||||
|
||||
# Definining some variable names
|
||||
self.CALENVIROSCREEN_SCORE_FIELD_NAME = "calenviroscreen_score"
|
||||
self.CALENVIROSCREEN_PERCENTILE_FIELD_NAME = (
|
||||
"calenviroscreen_percentile"
|
||||
)
|
||||
self.CALENVIROSCREEN_PERCENTILE_FIELD_NAME = "calenviroscreen_percentile"
|
||||
self.CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD_NAME = (
|
||||
"calenviroscreen_priority_community"
|
||||
)
|
||||
|
|
@ -30,14 +26,14 @@ class CalEnviroScreenETL(ExtractTransformLoad):
|
|||
self.df: pd.DataFrame
|
||||
|
||||
def extract(self) -> None:
|
||||
logger.info(f"Downloading CalEnviroScreen Data")
|
||||
logger.info("Downloading CalEnviroScreen Data")
|
||||
super().extract(
|
||||
self.CALENVIROSCREEN_FTP_URL,
|
||||
self.TMP_PATH,
|
||||
)
|
||||
|
||||
def transform(self) -> None:
|
||||
logger.info(f"Transforming CalEnviroScreen Data")
|
||||
logger.info("Transforming CalEnviroScreen Data")
|
||||
|
||||
# Data from https://calenviroscreen-oehha.hub.arcgis.com/#Data, specifically:
|
||||
# https://oehha.ca.gov/media/downloads/calenviroscreen/document/calenviroscreen40resultsdatadictionaryd12021.zip
|
||||
|
|
@ -67,7 +63,7 @@ class CalEnviroScreenETL(ExtractTransformLoad):
|
|||
)
|
||||
|
||||
def load(self) -> None:
|
||||
logger.info(f"Saving CalEnviroScreen CSV")
|
||||
logger.info("Saving CalEnviroScreen CSV")
|
||||
# write nationwide csv
|
||||
self.CSV_PATH.mkdir(parents=True, exist_ok=True)
|
||||
self.df.to_csv(self.CSV_PATH / f"data06.csv", index=False)
|
||||
self.df.to_csv(self.CSV_PATH / "data06.csv", index=False)
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
import csv
|
||||
import os
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import geopandas as gpd
|
||||
|
||||
from .etl_utils import get_state_fips_codes
|
||||
from utils import unzip_file_from_url, get_module_logger
|
||||
from .etl_utils import get_state_fips_codes
|
||||
|
||||
logger = get_module_logger(__name__)
|
||||
|
||||
|
|
@ -29,9 +30,7 @@ def download_census_csvs(data_path: Path) -> None:
|
|||
|
||||
for fips in state_fips_codes:
|
||||
# check if file exists
|
||||
shp_file_path = (
|
||||
data_path / "census" / "shp" / fips / f"tl_2010_{fips}_bg10.shp"
|
||||
)
|
||||
shp_file_path = data_path / "census" / "shp" / fips / f"tl_2010_{fips}_bg10.shp"
|
||||
|
||||
logger.info(f"Checking if {fips} file exists")
|
||||
if not os.path.isfile(shp_file_path):
|
||||
|
|
@ -110,7 +109,7 @@ def download_census_csvs(data_path: Path) -> None:
|
|||
)
|
||||
|
||||
## create national geojson
|
||||
logger.info(f"Generating national geojson file")
|
||||
logger.info("Generating national geojson file")
|
||||
usa_df = gpd.GeoDataFrame()
|
||||
|
||||
for file_name in geojson_dir_path.rglob("*.json"):
|
||||
|
|
@ -119,7 +118,7 @@ def download_census_csvs(data_path: Path) -> None:
|
|||
usa_df = usa_df.append(state_gdf)
|
||||
|
||||
usa_df = usa_df.to_crs("+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs")
|
||||
logger.info(f"Writing national geojson file")
|
||||
logger.info("Writing national geojson file")
|
||||
usa_df.to_file(geojson_dir_path / "us.json", driver="GeoJSON")
|
||||
|
||||
logger.info("Census block groups downloading complete")
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
from pathlib import Path
|
||||
import csv
|
||||
import pandas as pd
|
||||
import os
|
||||
import csv
|
||||
|
||||
from pathlib import Path
|
||||
import pandas as pd
|
||||
from config import settings
|
||||
|
||||
from utils import (
|
||||
|
|
@ -35,7 +36,7 @@ def get_state_fips_codes(data_path: Path) -> list:
|
|||
|
||||
# check if file exists
|
||||
if not os.path.isfile(fips_csv_path):
|
||||
logger.info(f"Downloading fips from S3 repository")
|
||||
logger.info("Downloading fips from S3 repository")
|
||||
unzip_file_from_url(
|
||||
settings.AWS_JUSTICE40_DATA_URL + "/Census/fips_states_2010.zip",
|
||||
data_path / "tmp",
|
||||
|
|
|
|||
|
|
@ -11,14 +11,10 @@ logger = get_module_logger(__name__)
|
|||
class CensusACSETL(ExtractTransformLoad):
|
||||
def __init__(self):
|
||||
self.ACS_YEAR = 2019
|
||||
self.OUTPUT_PATH = (
|
||||
self.DATA_PATH / "dataset" / f"census_acs_{self.ACS_YEAR}"
|
||||
)
|
||||
self.OUTPUT_PATH = self.DATA_PATH / "dataset" / f"census_acs_{self.ACS_YEAR}"
|
||||
self.UNEMPLOYED_FIELD_NAME = "Unemployed civilians (percent)"
|
||||
self.LINGUISTIC_ISOLATION_FIELD_NAME = "Linguistic isolation (percent)"
|
||||
self.LINGUISTIC_ISOLATION_TOTAL_FIELD_NAME = (
|
||||
"Linguistic isolation (total)"
|
||||
)
|
||||
self.LINGUISTIC_ISOLATION_TOTAL_FIELD_NAME = "Linguistic isolation (total)"
|
||||
self.LINGUISTIC_ISOLATION_FIELDS = [
|
||||
"C16002_001E",
|
||||
"C16002_004E",
|
||||
|
|
@ -28,9 +24,7 @@ class CensusACSETL(ExtractTransformLoad):
|
|||
]
|
||||
self.df: pd.DataFrame
|
||||
|
||||
def _fips_from_censusdata_censusgeo(
|
||||
self, censusgeo: censusdata.censusgeo
|
||||
) -> str:
|
||||
def _fips_from_censusdata_censusgeo(self, censusgeo: censusdata.censusgeo) -> str:
|
||||
"""Create a FIPS code from the proprietary censusgeo index."""
|
||||
fips = "".join([value for (key, value) in censusgeo.params()])
|
||||
return fips
|
||||
|
|
@ -38,9 +32,7 @@ class CensusACSETL(ExtractTransformLoad):
|
|||
def extract(self) -> None:
|
||||
dfs = []
|
||||
for fips in get_state_fips_codes(self.DATA_PATH):
|
||||
logger.info(
|
||||
f"Downloading data for state/territory with FIPS code {fips}"
|
||||
)
|
||||
logger.info(f"Downloading data for state/territory with FIPS code {fips}")
|
||||
|
||||
dfs.append(
|
||||
censusdata.download(
|
||||
|
|
@ -65,13 +57,11 @@ class CensusACSETL(ExtractTransformLoad):
|
|||
)
|
||||
|
||||
def transform(self) -> None:
|
||||
logger.info(f"Starting Census ACS Transform")
|
||||
logger.info("Starting Census ACS Transform")
|
||||
|
||||
# Calculate percent unemployment.
|
||||
# TODO: remove small-sample data that should be `None` instead of a high-variance fraction.
|
||||
self.df[self.UNEMPLOYED_FIELD_NAME] = (
|
||||
self.df.B23025_005E / self.df.B23025_003E
|
||||
)
|
||||
self.df[self.UNEMPLOYED_FIELD_NAME] = self.df.B23025_005E / self.df.B23025_003E
|
||||
|
||||
# Calculate linguistic isolation.
|
||||
individual_limited_english_fields = [
|
||||
|
|
@ -92,7 +82,7 @@ class CensusACSETL(ExtractTransformLoad):
|
|||
self.df[self.LINGUISTIC_ISOLATION_FIELD_NAME].describe()
|
||||
|
||||
def load(self) -> None:
|
||||
logger.info(f"Saving Census ACS Data")
|
||||
logger.info("Saving Census ACS Data")
|
||||
|
||||
# mkdir census
|
||||
self.OUTPUT_PATH.mkdir(parents=True, exist_ok=True)
|
||||
|
|
@ -108,6 +98,6 @@ class CensusACSETL(ExtractTransformLoad):
|
|||
)
|
||||
|
||||
def validate(self) -> None:
|
||||
logger.info(f"Validating Census ACS Data")
|
||||
logger.info("Validating Census ACS Data")
|
||||
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -8,20 +8,22 @@ logger = get_module_logger(__name__)
|
|||
|
||||
class EJScreenETL(ExtractTransformLoad):
|
||||
def __init__(self):
|
||||
self.EJSCREEN_FTP_URL = "https://gaftp.epa.gov/EJSCREEN/2019/EJSCREEN_2019_StatePctile.csv.zip"
|
||||
self.EJSCREEN_FTP_URL = (
|
||||
"https://gaftp.epa.gov/EJSCREEN/2019/EJSCREEN_2019_StatePctile.csv.zip"
|
||||
)
|
||||
self.EJSCREEN_CSV = self.TMP_PATH / "EJSCREEN_2019_StatePctiles.csv"
|
||||
self.CSV_PATH = self.DATA_PATH / "dataset" / "ejscreen_2019"
|
||||
self.df: pd.DataFrame
|
||||
|
||||
def extract(self) -> None:
|
||||
logger.info(f"Downloading EJScreen Data")
|
||||
logger.info("Downloading EJScreen Data")
|
||||
super().extract(
|
||||
self.EJSCREEN_FTP_URL,
|
||||
self.TMP_PATH,
|
||||
)
|
||||
|
||||
def transform(self) -> None:
|
||||
logger.info(f"Transforming EJScreen Data")
|
||||
logger.info("Transforming EJScreen Data")
|
||||
self.df = pd.read_csv(
|
||||
self.EJSCREEN_CSV,
|
||||
dtype={"ID": "string"},
|
||||
|
|
@ -31,7 +33,7 @@ class EJScreenETL(ExtractTransformLoad):
|
|||
)
|
||||
|
||||
def load(self) -> None:
|
||||
logger.info(f"Saving EJScreen CSV")
|
||||
logger.info("Saving EJScreen CSV")
|
||||
# write nationwide csv
|
||||
self.CSV_PATH.mkdir(parents=True, exist_ok=True)
|
||||
self.df.to_csv(self.CSV_PATH / f"usa.csv", index=False)
|
||||
self.df.to_csv(self.CSV_PATH / "usa.csv", index=False)
|
||||
|
|
|
|||
|
|
@ -35,9 +35,7 @@ class HousingTransportationETL(ExtractTransformLoad):
|
|||
)
|
||||
|
||||
# New file name:
|
||||
tmp_csv_file_path = (
|
||||
zip_file_dir / f"htaindex_data_blkgrps_{fips}.csv"
|
||||
)
|
||||
tmp_csv_file_path = zip_file_dir / f"htaindex_data_blkgrps_{fips}.csv"
|
||||
tmp_df = pd.read_csv(filepath_or_buffer=tmp_csv_file_path)
|
||||
|
||||
dfs.append(tmp_df)
|
||||
|
|
@ -45,16 +43,16 @@ class HousingTransportationETL(ExtractTransformLoad):
|
|||
self.df = pd.concat(dfs)
|
||||
|
||||
def transform(self) -> None:
|
||||
logger.info(f"Transforming Housing and Transportation Data")
|
||||
logger.info("Transforming Housing and Transportation Data")
|
||||
|
||||
# Rename and reformat block group ID
|
||||
self.df.rename(columns={"blkgrp": self.GEOID_FIELD_NAME}, inplace=True)
|
||||
self.df[self.GEOID_FIELD_NAME] = self.df[
|
||||
self.GEOID_FIELD_NAME
|
||||
].str.replace('"', "")
|
||||
self.df[self.GEOID_FIELD_NAME] = self.df[self.GEOID_FIELD_NAME].str.replace(
|
||||
'"', ""
|
||||
)
|
||||
|
||||
def load(self) -> None:
|
||||
logger.info(f"Saving Housing and Transportation Data")
|
||||
logger.info("Saving Housing and Transportation Data")
|
||||
|
||||
self.OUTPUT_PATH.mkdir(parents=True, exist_ok=True)
|
||||
self.df.to_csv(path_or_buf=self.OUTPUT_PATH / "usa.csv", index=False)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
import pandas as pd
|
||||
|
||||
from etl.base import ExtractTransformLoad
|
||||
from etl.sources.census.etl_utils import get_state_fips_codes
|
||||
from utils import get_module_logger, unzip_file_from_url, remove_all_from_dir
|
||||
from utils import get_module_logger
|
||||
|
||||
logger = get_module_logger(__name__)
|
||||
|
||||
|
|
@ -11,33 +10,37 @@ class HudHousingETL(ExtractTransformLoad):
|
|||
def __init__(self):
|
||||
self.OUTPUT_PATH = self.DATA_PATH / "dataset" / "hud_housing"
|
||||
self.GEOID_TRACT_FIELD_NAME = "GEOID10_TRACT"
|
||||
self.HOUSING_FTP_URL = "https://www.huduser.gov/portal/datasets/cp/2012thru2016-140-csv.zip"
|
||||
self.HOUSING_FTP_URL = (
|
||||
"https://www.huduser.gov/portal/datasets/cp/2012thru2016-140-csv.zip"
|
||||
)
|
||||
self.HOUSING_ZIP_FILE_DIR = self.TMP_PATH / "hud_housing"
|
||||
|
||||
# We measure households earning less than 80% of HUD Area Median Family Income by county
|
||||
# and paying greater than 30% of their income to housing costs.
|
||||
self.HOUSING_BURDEN_FIELD_NAME = "Housing burden (percent)"
|
||||
self.HOUSING_BURDEN_NUMERATOR_FIELD_NAME = "HOUSING_BURDEN_NUMERATOR"
|
||||
self.HOUSING_BURDEN_DENOMINATOR_FIELD_NAME = (
|
||||
"HOUSING_BURDEN_DENOMINATOR"
|
||||
)
|
||||
self.HOUSING_BURDEN_DENOMINATOR_FIELD_NAME = "HOUSING_BURDEN_DENOMINATOR"
|
||||
|
||||
# Note: some variable definitions.
|
||||
# HUD-adjusted median family income (HAMFI).
|
||||
# The four housing problems are: incomplete kitchen facilities, incomplete plumbing facilities, more than 1 person per room, and cost burden greater than 30%.
|
||||
# The four housing problems are:
|
||||
# - incomplete kitchen facilities,
|
||||
# - incomplete plumbing facilities,
|
||||
# - more than 1 person per room,
|
||||
# - cost burden greater than 30%.
|
||||
# Table 8 is the desired table.
|
||||
|
||||
self.df: pd.DataFrame
|
||||
|
||||
def extract(self) -> None:
|
||||
logger.info(f"Extracting HUD Housing Data")
|
||||
logger.info("Extracting HUD Housing Data")
|
||||
super().extract(
|
||||
self.HOUSING_FTP_URL,
|
||||
self.HOUSING_ZIP_FILE_DIR,
|
||||
)
|
||||
|
||||
def transform(self) -> None:
|
||||
logger.info(f"Transforming HUD Housing Data")
|
||||
logger.info("Transforming HUD Housing Data")
|
||||
|
||||
# New file name:
|
||||
tmp_csv_file_path = (
|
||||
|
|
@ -53,9 +56,7 @@ class HudHousingETL(ExtractTransformLoad):
|
|||
)
|
||||
|
||||
# Rename and reformat block group ID
|
||||
self.df.rename(
|
||||
columns={"geoid": self.GEOID_TRACT_FIELD_NAME}, inplace=True
|
||||
)
|
||||
self.df.rename(columns={"geoid": self.GEOID_TRACT_FIELD_NAME}, inplace=True)
|
||||
|
||||
# The CHAS data has census tract ids such as `14000US01001020100`
|
||||
# Whereas the rest of our data uses, for the same tract, `01001020100`.
|
||||
|
|
@ -70,69 +71,177 @@ class HudHousingETL(ExtractTransformLoad):
|
|||
|
||||
# Owner occupied numerator fields
|
||||
OWNER_OCCUPIED_NUMERATOR_FIELDS = [
|
||||
# Key: Column Name Line_Type Tenure Household income Cost burden Facilities
|
||||
# T8_est7 Subtotal Owner occupied less than or equal to 30% of HAMFI greater than 30% but less than or equal to 50% All
|
||||
# Column Name
|
||||
# Line_Type
|
||||
# Tenure
|
||||
# Household income
|
||||
# Cost burden
|
||||
# Facilities
|
||||
"T8_est7",
|
||||
# T8_est10 Subtotal Owner occupied less than or equal to 30% of HAMFI greater than 50% All
|
||||
# Subtotal
|
||||
# Owner occupied
|
||||
# less than or equal to 30% of HAMFI
|
||||
# greater than 30% but less than or equal to 50%
|
||||
# All
|
||||
"T8_est10",
|
||||
# T8_est20 Subtotal Owner occupied greater than 30% but less than or equal to 50% of HAMFI greater than 30% but less than or equal to 50% All
|
||||
# Subtotal
|
||||
# Owner occupied
|
||||
# less than or equal to 30% of HAMFI
|
||||
# greater than 50%
|
||||
# All
|
||||
"T8_est20",
|
||||
# T8_est23 Subtotal Owner occupied greater than 30% but less than or equal to 50% of HAMFI greater than 50% All
|
||||
# Subtotal
|
||||
# Owner occupied
|
||||
# greater than 30% but less than or equal to 50% of HAMFI
|
||||
# greater than 30% but less than or equal to 50%
|
||||
# All
|
||||
"T8_est23",
|
||||
# T8_est33 Subtotal Owner occupied greater than 50% but less than or equal to 80% of HAMFI greater than 30% but less than or equal to 50% All
|
||||
# Subtotal
|
||||
# Owner occupied
|
||||
# greater than 30% but less than or equal to 50% of HAMFI
|
||||
# greater than 50%
|
||||
# All
|
||||
"T8_est33",
|
||||
# T8_est36 Subtotal Owner occupied greater than 50% but less than or equal to 80% of HAMFI greater than 50% All
|
||||
# Subtotal
|
||||
# Owner occupied
|
||||
# greater than 50% but less than or equal to 80% of HAMFI
|
||||
# greater than 30% but less than or equal to 50%
|
||||
# All
|
||||
"T8_est36",
|
||||
# Subtotal
|
||||
# Owner occupied
|
||||
# greater than 50% but less than or equal to 80% of HAMFI
|
||||
# greater than 50%
|
||||
# All
|
||||
]
|
||||
|
||||
# These rows have the values where HAMFI was not computed, b/c of no or negative income.
|
||||
OWNER_OCCUPIED_NOT_COMPUTED_FIELDS = [
|
||||
# Key: Column Name Line_Type Tenure Household income Cost burden Facilities
|
||||
# T8_est13 Subtotal Owner occupied less than or equal to 30% of HAMFI not computed (no/negative income) All
|
||||
# Column Name
|
||||
# Line_Type
|
||||
# Tenure
|
||||
# Household income
|
||||
# Cost burden
|
||||
# Facilities
|
||||
"T8_est13",
|
||||
# T8_est26 Subtotal Owner occupied greater than 30% but less than or equal to 50% of HAMFI not computed (no/negative income) All
|
||||
# Subtotal
|
||||
# Owner occupied
|
||||
# less than or equal to 30% of HAMFI
|
||||
# not computed (no/negative income)
|
||||
# All
|
||||
"T8_est26",
|
||||
# T8_est39 Subtotal Owner occupied greater than 50% but less than or equal to 80% of HAMFI not computed (no/negative income) All
|
||||
# Subtotal
|
||||
# Owner occupied
|
||||
# greater than 30% but less than or equal to 50% of HAMFI
|
||||
# not computed (no/negative income)
|
||||
# All
|
||||
"T8_est39",
|
||||
# T8_est52 Subtotal Owner occupied greater than 80% but less than or equal to 100% of HAMFI not computed (no/negative income) All
|
||||
# Subtotal
|
||||
# Owner occupied
|
||||
# greater than 50% but less than or equal to 80% of HAMFI
|
||||
# not computed (no/negative income)
|
||||
# All
|
||||
"T8_est52",
|
||||
# T8_est65 Subtotal Owner occupied greater than 100% of HAMFI not computed (no/negative income) All
|
||||
# Subtotal
|
||||
# Owner occupied
|
||||
# greater than 80% but less than or equal to 100% of HAMFI
|
||||
# not computed (no/negative income)
|
||||
# All
|
||||
"T8_est65",
|
||||
# Subtotal
|
||||
# Owner occupied
|
||||
# greater than 100% of HAMFI
|
||||
# not computed (no/negative income)
|
||||
# All
|
||||
]
|
||||
|
||||
# T8_est2 Subtotal Owner occupied All All All
|
||||
OWNER_OCCUPIED_POPULATION_FIELD = "T8_est2"
|
||||
# Subtotal
|
||||
# Owner occupied
|
||||
# All
|
||||
# All
|
||||
# All
|
||||
|
||||
# Renter occupied numerator fields
|
||||
RENTER_OCCUPIED_NUMERATOR_FIELDS = [
|
||||
# Key: Column Name Line_Type Tenure Household income Cost burden Facilities
|
||||
# T8_est73 Subtotal Renter occupied less than or equal to 30% of HAMFI greater than 30% but less than or equal to 50% All
|
||||
# Column Name
|
||||
# Line_Type
|
||||
# Tenure
|
||||
# Household income
|
||||
# Cost burden
|
||||
# Facilities
|
||||
"T8_est73",
|
||||
# T8_est76 Subtotal Renter occupied less than or equal to 30% of HAMFI greater than 50% All
|
||||
# Subtotal
|
||||
# Renter occupied
|
||||
# less than or equal to 30% of HAMFI
|
||||
# greater than 30% but less than or equal to 50%
|
||||
# All
|
||||
"T8_est76",
|
||||
# T8_est86 Subtotal Renter occupied greater than 30% but less than or equal to 50% of HAMFI greater than 30% but less than or equal to 50% All
|
||||
# Subtotal
|
||||
# Renter occupied
|
||||
# less than or equal to 30% of HAMFI
|
||||
# greater than 50%
|
||||
# All
|
||||
"T8_est86",
|
||||
# T8_est89 Subtotal Renter occupied greater than 30% but less than or equal to 50% of HAMFI greater than 50% All
|
||||
# Subtotal
|
||||
# Renter occupied
|
||||
# greater than 30% but less than or equal to 50% of HAMFI
|
||||
# greater than 30% but less than or equal to 50%
|
||||
# All
|
||||
"T8_est89",
|
||||
# T8_est99 Subtotal Renter occupied greater than 50% but less than or equal to 80% of HAMFI greater than 30% but less than or equal to 50% All
|
||||
# Subtotal
|
||||
# Renter occupied
|
||||
# greater than 30% but less than or equal to 50% of HAMFI
|
||||
# greater than 50%
|
||||
# All
|
||||
"T8_est99",
|
||||
# T8_est102 Subtotal Renter occupied greater than 50% but less than or equal to 80% of HAMFI greater than 50% All
|
||||
# Subtotal
|
||||
# Renter occupied greater than 50% but less than or equal to 80% of HAMFI
|
||||
# greater than 30% but less than or equal to 50%
|
||||
# All
|
||||
"T8_est102",
|
||||
# Subtotal
|
||||
# Renter occupied
|
||||
# greater than 50% but less than or equal to 80% of HAMFI
|
||||
# greater than 50%
|
||||
# All
|
||||
]
|
||||
|
||||
# These rows have the values where HAMFI was not computed, b/c of no or negative income.
|
||||
RENTER_OCCUPIED_NOT_COMPUTED_FIELDS = [
|
||||
# Key: Column Name Line_Type Tenure Household income Cost burden Facilities
|
||||
# T8_est79 Subtotal Renter occupied less than or equal to 30% of HAMFI not computed (no/negative income) All
|
||||
# Column Name
|
||||
# Line_Type
|
||||
# Tenure
|
||||
# Household income
|
||||
# Cost burden
|
||||
# Facilities
|
||||
"T8_est79",
|
||||
# T8_est92 Subtotal Renter occupied greater than 30% but less than or equal to 50% of HAMFI not computed (no/negative income) All
|
||||
# Subtotal
|
||||
# Renter occupied less than or equal to 30% of HAMFI
|
||||
# not computed (no/negative income)
|
||||
# All
|
||||
"T8_est92",
|
||||
# T8_est105 Subtotal Renter occupied greater than 50% but less than or equal to 80% of HAMFI not computed (no/negative income) All
|
||||
# Subtotal
|
||||
# Renter occupied greater than 30% but less than or equal to 50% of HAMFI
|
||||
# not computed (no/negative income)
|
||||
# All
|
||||
"T8_est105",
|
||||
# T8_est118 Subtotal Renter occupied greater than 80% but less than or equal to 100% of HAMFI not computed (no/negative income) All
|
||||
# Subtotal
|
||||
# Renter occupied
|
||||
# greater than 50% but less than or equal to 80% of HAMFI
|
||||
# not computed (no/negative income)
|
||||
# All
|
||||
"T8_est118",
|
||||
# T8_est131 Subtotal Renter occupied greater than 100% of HAMFI not computed (no/negative income) All
|
||||
# Subtotal
|
||||
# Renter occupied greater than 80% but less than or equal to 100% of HAMFI
|
||||
# not computed (no/negative income)
|
||||
# All
|
||||
"T8_est131",
|
||||
# Subtotal
|
||||
# Renter occupied
|
||||
# greater than 100% of HAMFI
|
||||
# not computed (no/negative income)
|
||||
# All
|
||||
]
|
||||
|
||||
# T8_est68 Subtotal Renter occupied All All All
|
||||
|
|
@ -165,14 +274,12 @@ class HudHousingETL(ExtractTransformLoad):
|
|||
# TODO: add small sample size checks
|
||||
self.df[self.HOUSING_BURDEN_FIELD_NAME] = self.df[
|
||||
self.HOUSING_BURDEN_NUMERATOR_FIELD_NAME
|
||||
].astype(float) / self.df[
|
||||
self.HOUSING_BURDEN_DENOMINATOR_FIELD_NAME
|
||||
].astype(
|
||||
].astype(float) / self.df[self.HOUSING_BURDEN_DENOMINATOR_FIELD_NAME].astype(
|
||||
float
|
||||
)
|
||||
|
||||
def load(self) -> None:
|
||||
logger.info(f"Saving HUD Housing Data")
|
||||
logger.info("Saving HUD Housing Data")
|
||||
|
||||
self.OUTPUT_PATH.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ logger = get_module_logger(__name__)
|
|||
|
||||
class HudRecapETL(ExtractTransformLoad):
|
||||
def __init__(self):
|
||||
self.HUD_RECAP_CSV_URL = "https://opendata.arcgis.com/api/v3/datasets/56de4edea8264fe5a344da9811ef5d6e_0/downloads/data?format=csv&spatialRefId=4326"
|
||||
# pylint: disable=line-too-long
|
||||
self.HUD_RECAP_CSV_URL = "https://opendata.arcgis.com/api/v3/datasets/56de4edea8264fe5a344da9811ef5d6e_0/downloads/data?format=csv&spatialRefId=4326" # noqa: E501
|
||||
self.HUD_RECAP_CSV = (
|
||||
self.TMP_PATH
|
||||
/ "Racially_or_Ethnically_Concentrated_Areas_of_Poverty__R_ECAPs_.csv"
|
||||
|
|
@ -22,7 +23,7 @@ class HudRecapETL(ExtractTransformLoad):
|
|||
self.df: pd.DataFrame
|
||||
|
||||
def extract(self) -> None:
|
||||
logger.info(f"Downloading HUD Recap Data")
|
||||
logger.info("Downloading HUD Recap Data")
|
||||
download = requests.get(self.HUD_RECAP_CSV_URL, verify=None)
|
||||
file_contents = download.content
|
||||
csv_file = open(self.HUD_RECAP_CSV, "wb")
|
||||
|
|
@ -30,7 +31,7 @@ class HudRecapETL(ExtractTransformLoad):
|
|||
csv_file.close()
|
||||
|
||||
def transform(self) -> None:
|
||||
logger.info(f"Transforming HUD Recap Data")
|
||||
logger.info("Transforming HUD Recap Data")
|
||||
|
||||
# Load comparison index (CalEnviroScreen 4)
|
||||
self.df = pd.read_csv(self.HUD_RECAP_CSV, dtype={"GEOID": "string"})
|
||||
|
|
@ -57,7 +58,7 @@ class HudRecapETL(ExtractTransformLoad):
|
|||
self.df.sort_values(by=self.GEOID_TRACT_FIELD_NAME, inplace=True)
|
||||
|
||||
def load(self) -> None:
|
||||
logger.info(f"Saving HUD Recap CSV")
|
||||
logger.info("Saving HUD Recap CSV")
|
||||
# write nationwide csv
|
||||
self.CSV_PATH.mkdir(parents=True, exist_ok=True)
|
||||
self.df.to_csv(self.CSV_PATH / f"usa.csv", index=False)
|
||||
self.df.to_csv(self.CSV_PATH / "usa.csv", index=False)
|
||||
|
|
|
|||
|
|
@ -3,25 +3,72 @@ import geopandas as gpd
|
|||
|
||||
from etl.base import ExtractTransformLoad
|
||||
from utils import get_module_logger
|
||||
import os
|
||||
|
||||
logger = get_module_logger(__name__)
|
||||
|
||||
|
||||
class TreeEquityScoreETL(ExtractTransformLoad):
|
||||
def __init__(self):
|
||||
self.TES_URL = "https://national-tes-data-share.s3.amazonaws.com/national_tes_share/"
|
||||
self.TES_URL = (
|
||||
"https://national-tes-data-share.s3.amazonaws.com/national_tes_share/"
|
||||
)
|
||||
self.TES_CSV = self.TMP_PATH / "tes_2021_data.csv"
|
||||
self.CSV_PATH = self.DATA_PATH / "dataset" / "tree_equity_score"
|
||||
self.df: gpd.GeoDataFrame
|
||||
self.states = ["al", "az", "ar", "ca", "co", "ct", "de", "dc", "fl",
|
||||
"ga", "id", "il", "in", "ia", "ks", "ky", "la", "me",
|
||||
"md", "ma", "mi", "mn", "ms", "mo", "mt", "ne", "nv", "nh",
|
||||
"nj", "nm", "ny", "nc", "nd", "oh", "ok", "or", "pa",
|
||||
"ri", "sc", "sd", "tn", "tx", "ut", "vt", "va", "wa", "wv", "wi", "wy"]
|
||||
self.states = [
|
||||
"al",
|
||||
"az",
|
||||
"ar",
|
||||
"ca",
|
||||
"co",
|
||||
"ct",
|
||||
"de",
|
||||
"dc",
|
||||
"fl",
|
||||
"ga",
|
||||
"id",
|
||||
"il",
|
||||
"in",
|
||||
"ia",
|
||||
"ks",
|
||||
"ky",
|
||||
"la",
|
||||
"me",
|
||||
"md",
|
||||
"ma",
|
||||
"mi",
|
||||
"mn",
|
||||
"ms",
|
||||
"mo",
|
||||
"mt",
|
||||
"ne",
|
||||
"nv",
|
||||
"nh",
|
||||
"nj",
|
||||
"nm",
|
||||
"ny",
|
||||
"nc",
|
||||
"nd",
|
||||
"oh",
|
||||
"ok",
|
||||
"or",
|
||||
"pa",
|
||||
"ri",
|
||||
"sc",
|
||||
"sd",
|
||||
"tn",
|
||||
"tx",
|
||||
"ut",
|
||||
"vt",
|
||||
"va",
|
||||
"wa",
|
||||
"wv",
|
||||
"wi",
|
||||
"wy",
|
||||
]
|
||||
|
||||
def extract(self) -> None:
|
||||
logger.info(f"Downloading Tree Equity Score Data")
|
||||
logger.info("Downloading Tree Equity Score Data")
|
||||
for state in self.states:
|
||||
super().extract(
|
||||
f"{self.TES_URL}{state}.zip.zip",
|
||||
|
|
@ -29,14 +76,14 @@ class TreeEquityScoreETL(ExtractTransformLoad):
|
|||
)
|
||||
|
||||
def transform(self) -> None:
|
||||
logger.info(f"Transforming Tree Equity Score Data")
|
||||
logger.info("Transforming Tree Equity Score Data")
|
||||
tes_state_dfs = []
|
||||
for state in self.states:
|
||||
tes_state_dfs.append(gpd.read_file(f"{self.TMP_PATH}/{state}/{state}.shp"))
|
||||
self.df = gpd.GeoDataFrame(pd.concat(tes_state_dfs), crs=tes_state_dfs[0].crs)
|
||||
|
||||
def load(self) -> None:
|
||||
logger.info(f"Saving Tree Equity Score GeoJSON")
|
||||
logger.info("Saving Tree Equity Score GeoJSON")
|
||||
# write nationwide csv
|
||||
self.CSV_PATH.mkdir(parents=True, exist_ok=True)
|
||||
self.df.to_file(self.CSV_PATH / "tes_conus.geojson", driver='GeoJSON')
|
||||
self.df.to_file(self.CSV_PATH / "tes_conus.geojson", driver="GeoJSON")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue