mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-02-23 10:04:18 -08:00
41e394972c
* running black throughout * adding housing * hud housing etl working * got score d and e working * updating scoring comparison * minor fixes * small changes * small comments
274 lines
10 KiB
Text
274 lines
10 KiB
Text
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c21b63a3",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"import censusdata\n",
|
|
"import csv\n",
|
|
"from pathlib import Path\n",
|
|
"import os\n",
|
|
"import re\n",
|
|
"import sys\n",
|
|
"\n",
|
|
"module_path = os.path.abspath(os.path.join(\"..\"))\n",
|
|
"if module_path not in sys.path:\n",
|
|
" sys.path.append(module_path)\n",
|
|
"\n",
|
|
"from etl.sources.census.etl_utils import get_state_fips_codes\n",
|
|
"from utils import unzip_file_from_url, remove_all_from_dir\n",
|
|
"\n",
|
|
"DATA_PATH = Path.cwd().parent / \"data\"\n",
|
|
"TMP_PATH = DATA_PATH / \"tmp\"\n",
|
|
"OUTPUT_PATH = DATA_PATH / \"dataset\" / \"hud_housing\"\n",
|
|
"\n",
|
|
"GEOID_TRACT_FIELD_NAME = \"GEOID10_TRACT\"\n",
|
|
"\n",
|
|
"# We measure households earning less than 80% of HUD Area Median Family Income by county\n",
|
|
"# and paying greater than 30% of their income to housing costs.\n",
|
|
"HOUSING_BURDEN_FIELD_NAME = \"Housing burden (percent)\"\n",
|
|
"HOUSING_BURDEN_NUMERATOR_FIELD_NAME = \"HOUSING_BURDEN_NUMERATOR\"\n",
|
|
"HOUSING_BURDEN_DENOMINATOR_FIELD_NAME = \"HOUSING_BURDEN_DENOMINATOR\"\n",
|
|
"\n",
|
|
"# Note: some variable definitions.\n",
|
|
"# HUD-adjusted median family income (HAMFI).\n",
|
|
"# The four housing problems are: incomplete kitchen facilities, incomplete plumbing facilities, more than 1 person per room, and cost burden greater than 30%.\n",
|
|
"# Table 8 is the desired table."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6696bc66",
|
|
"metadata": {
|
|
"scrolled": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Download the data.\n",
|
|
"dfs = []\n",
|
|
"zip_file_dir = TMP_PATH / \"hud_housing\"\n",
|
|
"\n",
|
|
"print(f\"Downloading 225MB housing data\")\n",
|
|
"unzip_file_from_url(\n",
|
|
" \"https://www.huduser.gov/portal/datasets/cp/2012thru2016-140-csv.zip\",\n",
|
|
" TMP_PATH,\n",
|
|
" zip_file_dir,\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3e954589",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# New file name:\n",
|
|
"tmp_csv_file_path = (\n",
|
|
" zip_file_dir\n",
|
|
" / \"2012thru2016-140-csv\"\n",
|
|
" / \"2012thru2016-140-csv\"\n",
|
|
" / \"140\"\n",
|
|
" / \"Table8.csv\"\n",
|
|
")\n",
|
|
"df = pd.read_csv(filepath_or_buffer=tmp_csv_file_path)\n",
|
|
"\n",
|
|
"df.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "244e0d03",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Rename and reformat block group ID\n",
|
|
"df.rename(columns={\"geoid\": GEOID_TRACT_FIELD_NAME}, inplace=True)\n",
|
|
"\n",
|
|
"# The CHAS data has census tract ids such as `14000US01001020100`\n",
|
|
"# Whereas the rest of our data uses, for the same tract, `01001020100`.\n",
|
|
"# the characters before `US`:\n",
|
|
"df[GEOID_TRACT_FIELD_NAME] = df[GEOID_TRACT_FIELD_NAME].str.replace(\n",
|
|
" r\"^.*?US\", \"\", regex=True\n",
|
|
")\n",
|
|
"\n",
|
|
"df[GEOID_TRACT_FIELD_NAME].head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "03250026",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Calculate housing burden\n",
|
|
"# This is quite a number of steps. It does not appear to be accessible nationally in a simpler format, though.\n",
|
|
"# See \"CHAS data dictionary 12-16.xlsx\"\n",
|
|
"\n",
|
|
"# Owner occupied numerator fields\n",
|
|
"OWNER_OCCUPIED_NUMERATOR_FIELDS = [\n",
|
|
" # Key: Column Name\tLine_Type\tTenure\tHousehold income\tCost burden\tFacilities\n",
|
|
" # T8_est7\tSubtotal\tOwner occupied\tless than or equal to 30% of HAMFI\tgreater than 30% but less than or equal to 50%\tAll\n",
|
|
" \"T8_est7\",\n",
|
|
" # T8_est10\tSubtotal\tOwner occupied\tless than or equal to 30% of HAMFI\tgreater than 50%\tAll\n",
|
|
" \"T8_est10\",\n",
|
|
" # T8_est20\tSubtotal\tOwner occupied\tgreater than 30% but less than or equal to 50% of HAMFI\tgreater than 30% but less than or equal to 50%\tAll\n",
|
|
" \"T8_est20\",\n",
|
|
" # T8_est23\tSubtotal\tOwner occupied\tgreater than 30% but less than or equal to 50% of HAMFI\tgreater than 50%\tAll\n",
|
|
" \"T8_est23\",\n",
|
|
" # T8_est33\tSubtotal\tOwner occupied\tgreater than 50% but less than or equal to 80% of HAMFI\tgreater than 30% but less than or equal to 50%\tAll\n",
|
|
" \"T8_est33\",\n",
|
|
" # T8_est36\tSubtotal\tOwner occupied\tgreater than 50% but less than or equal to 80% of HAMFI\tgreater than 50%\tAll\n",
|
|
" \"T8_est36\",\n",
|
|
"]\n",
|
|
"\n",
|
|
"# These rows have the values where HAMFI was not computed, b/c of no or negative income.\n",
|
|
"OWNER_OCCUPIED_NOT_COMPUTED_FIELDS = [\n",
|
|
" # Key: Column Name\tLine_Type\tTenure\tHousehold income\tCost burden\tFacilities\n",
|
|
" # T8_est13\tSubtotal\tOwner occupied\tless than or equal to 30% of HAMFI\tnot computed (no/negative income)\tAll\n",
|
|
" \"T8_est13\",\n",
|
|
" # T8_est26\tSubtotal\tOwner occupied\tgreater than 30% but less than or equal to 50% of HAMFI\tnot computed (no/negative income)\tAll\n",
|
|
" \"T8_est26\",\n",
|
|
" # T8_est39\tSubtotal\tOwner occupied\tgreater than 50% but less than or equal to 80% of HAMFI\tnot computed (no/negative income)\tAll\n",
|
|
" \"T8_est39\",\n",
|
|
" # T8_est52\tSubtotal\tOwner occupied\tgreater than 80% but less than or equal to 100% of HAMFI\tnot computed (no/negative income)\tAll\n",
|
|
" \"T8_est52\",\n",
|
|
" # T8_est65\tSubtotal\tOwner occupied\tgreater than 100% of HAMFI\tnot computed (no/negative income)\tAll\n",
|
|
" \"T8_est65\",\n",
|
|
"]\n",
|
|
"\n",
|
|
"# T8_est2\tSubtotal\tOwner occupied\tAll\tAll\tAll\n",
|
|
"OWNER_OCCUPIED_POPULATION_FIELD = \"T8_est2\"\n",
|
|
"\n",
|
|
"# Renter occupied numerator fields\n",
|
|
"RENTER_OCCUPIED_NUMERATOR_FIELDS = [\n",
|
|
" # Key: Column Name\tLine_Type\tTenure\tHousehold income\tCost burden\tFacilities\n",
|
|
" # T8_est73\tSubtotal\tRenter occupied\tless than or equal to 30% of HAMFI\tgreater than 30% but less than or equal to 50%\tAll\n",
|
|
" \"T8_est73\",\n",
|
|
" # T8_est76\tSubtotal\tRenter occupied\tless than or equal to 30% of HAMFI\tgreater than 50%\tAll\n",
|
|
" \"T8_est76\",\n",
|
|
" # T8_est86\tSubtotal\tRenter occupied\tgreater than 30% but less than or equal to 50% of HAMFI\tgreater than 30% but less than or equal to 50%\tAll\n",
|
|
" \"T8_est86\",\n",
|
|
" # T8_est89\tSubtotal\tRenter occupied\tgreater than 30% but less than or equal to 50% of HAMFI\tgreater than 50%\tAll\n",
|
|
" \"T8_est89\",\n",
|
|
" # T8_est99\tSubtotal\tRenter occupied\tgreater than 50% but less than or equal to 80% of HAMFI\tgreater than 30% but less than or equal to 50%\tAll\n",
|
|
" \"T8_est99\",\n",
|
|
" # T8_est102\tSubtotal\tRenter occupied\tgreater than 50% but less than or equal to 80% of HAMFI\tgreater than 50%\tAll\n",
|
|
" \"T8_est102\",\n",
|
|
"]\n",
|
|
"\n",
|
|
"# These rows have the values where HAMFI was not computed, b/c of no or negative income.\n",
|
|
"RENTER_OCCUPIED_NOT_COMPUTED_FIELDS = [\n",
|
|
" # Key: Column Name\tLine_Type\tTenure\tHousehold income\tCost burden\tFacilities\n",
|
|
" # T8_est79\tSubtotal\tRenter occupied\tless than or equal to 30% of HAMFI\tnot computed (no/negative income)\tAll\n",
|
|
" \"T8_est79\",\n",
|
|
" # T8_est92\tSubtotal\tRenter occupied\tgreater than 30% but less than or equal to 50% of HAMFI\tnot computed (no/negative income)\tAll\n",
|
|
" \"T8_est92\",\n",
|
|
" # T8_est105\tSubtotal\tRenter occupied\tgreater than 50% but less than or equal to 80% of HAMFI\tnot computed (no/negative income)\tAll\n",
|
|
" \"T8_est105\",\n",
|
|
" # T8_est118\tSubtotal\tRenter occupied\tgreater than 80% but less than or equal to 100% of HAMFI\tnot computed (no/negative income)\tAll\n",
|
|
" \"T8_est118\",\n",
|
|
" # T8_est131\tSubtotal\tRenter occupied\tgreater than 100% of HAMFI\tnot computed (no/negative income)\tAll\n",
|
|
" \"T8_est131\",\n",
|
|
"]\n",
|
|
"\n",
|
|
"\n",
|
|
"# T8_est68\tSubtotal\tRenter occupied\tAll\tAll\tAll\n",
|
|
"RENTER_OCCUPIED_POPULATION_FIELD = \"T8_est68\"\n",
|
|
"\n",
|
|
"\n",
|
|
"# Math:\n",
|
|
"# (\n",
|
|
"# # of Owner Occupied Units Meeting Criteria\n",
|
|
"# + # of Renter Occupied Units Meeting Criteria\n",
|
|
"# )\n",
|
|
"# divided by\n",
|
|
"# (\n",
|
|
"# Total # of Owner Occupied Units\n",
|
|
"# + Total # of Renter Occupied Units\n",
|
|
"# - # of Owner Occupied Units with HAMFI Not Computed\n",
|
|
"# - # of Renter Occupied Units with HAMFI Not Computed\n",
|
|
"# )\n",
|
|
"\n",
|
|
"df[HOUSING_BURDEN_NUMERATOR_FIELD_NAME] = df[OWNER_OCCUPIED_NUMERATOR_FIELDS].sum(\n",
|
|
" axis=1\n",
|
|
") + df[RENTER_OCCUPIED_NUMERATOR_FIELDS].sum(axis=1)\n",
|
|
"\n",
|
|
"df[HOUSING_BURDEN_DENOMINATOR_FIELD_NAME] = (\n",
|
|
" df[OWNER_OCCUPIED_POPULATION_FIELD]\n",
|
|
" + df[RENTER_OCCUPIED_POPULATION_FIELD]\n",
|
|
" - df[OWNER_OCCUPIED_NOT_COMPUTED_FIELDS].sum(axis=1)\n",
|
|
" - df[RENTER_OCCUPIED_NOT_COMPUTED_FIELDS].sum(axis=1)\n",
|
|
")\n",
|
|
"\n",
|
|
"# TODO: add small sample size checks\n",
|
|
"df[HOUSING_BURDEN_FIELD_NAME] = df[HOUSING_BURDEN_NUMERATOR_FIELD_NAME].astype(\n",
|
|
" float\n",
|
|
") / df[HOUSING_BURDEN_DENOMINATOR_FIELD_NAME].astype(float)\n",
|
|
"\n",
|
|
"df.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "8275c1ef",
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"OUTPUT_PATH.mkdir(parents=True, exist_ok=True)\n",
|
|
"\n",
|
|
"# Drop unnecessary fields\n",
|
|
"df[\n",
|
|
" [\n",
|
|
" GEOID_TRACT_FIELD_NAME,\n",
|
|
" HOUSING_BURDEN_NUMERATOR_FIELD_NAME,\n",
|
|
" HOUSING_BURDEN_DENOMINATOR_FIELD_NAME,\n",
|
|
" HOUSING_BURDEN_FIELD_NAME,\n",
|
|
" ]\n",
|
|
"].to_csv(path_or_buf=OUTPUT_PATH / \"usa.csv\", index=False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "ef5bb862",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# cleanup\n",
|
|
"remove_all_from_dir(TMP_PATH)"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.7.1"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|