mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-02-23 10:04:18 -08:00
41e394972c
* running black throughout * adding housing * hud housing etl working * got score d and e working * updating scoring comparison * minor fixes * small changes * small comments
423 lines
17 KiB
Text
423 lines
17 KiB
Text
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "54615cef",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Before running this script as it currently stands, you'll need to run two notebooks:\n",
|
|
"# 1. ejscreen_etl.ipynb\n",
|
|
"# 2. score_calc_0.1.ipynb\n",
|
|
"\n",
|
|
"import numpy as np\n",
|
|
"import pandas as pd\n",
|
|
"from pathlib import Path\n",
|
|
"import requests\n",
|
|
"import zipfile\n",
|
|
"from datetime import datetime\n",
|
|
"from tqdm.notebook import tqdm_notebook\n",
|
|
"\n",
|
|
"# Turn on TQDM for pandas so that we can have progress bars when running `apply`.\n",
|
|
"tqdm_notebook.pandas()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "49a63129",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Suppress scientific notation in pandas (this shows up for census tract IDs)\n",
|
|
"pd.options.display.float_format = \"{:.2f}\".format\n",
|
|
"\n",
|
|
"# Set some global parameters\n",
|
|
"DATA_DIR = Path.cwd().parent / \"data\"\n",
|
|
"TEMP_DATA_DIR = Path.cwd().parent / \"data\" / \"tmp\"\n",
|
|
"# None of these numbers are final, but just for the purposes of comparison.\n",
|
|
"CALENVIROSCREEN_PRIORITY_COMMUNITY_THRESHOLD = 75\n",
|
|
"CEJST_PRIORITY_COMMUNITY_THRESHOLD = 0.75\n",
|
|
"\n",
|
|
"# Name fields using variables. (This makes it easy to reference the same fields frequently without using strings\n",
|
|
"# and introducing the risk of misspelling the field name.)\n",
|
|
"CENSUS_BLOCK_GROUP_ID_FIELD = \"census_block_group_id\"\n",
|
|
"CENSUS_BLOCK_GROUP_POPULATION_FIELD = \"census_block_group_population\"\n",
|
|
"CENSUS_TRACT_ID_FIELD = \"census_tract_id\"\n",
|
|
"CALENVIROSCREEN_SCORE_FIELD = \"calenviroscreen_score\"\n",
|
|
"CALENVIROSCREEN_PERCENTILE_FIELD = \"calenviroscreen_percentile\"\n",
|
|
"CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD = \"calenviroscreen_priority_community\"\n",
|
|
"\n",
|
|
"# Note: we are pretending the EJSCREEN's low income percent is the actual score for now as a placeholder.\n",
|
|
"CEJST_SCORE_FIELD = \"cejst_score\"\n",
|
|
"CEJST_PERCENTILE_FIELD = \"cejst_percentile\"\n",
|
|
"CEJST_PRIORITY_COMMUNITY_FIELD = \"cejst_priority_community\"\n",
|
|
"\n",
|
|
"# Comparison field names\n",
|
|
"any_tract_has_at_least_one_cbg = \"Tract has at least one CEJST CBG?\"\n",
|
|
"tract_has_at_least_one_cbg = \"CES Tract has at least one CEJST CBG?\"\n",
|
|
"tract_has_100_percent_cbg = \"CES Tract has 100% CEJST CBGs?\"\n",
|
|
"non_ces_tract_has_at_least_one_cbg = \"Non-CES Tract has at least one CEJST CBG?\"\n",
|
|
"non_ces_tract_has_100_percent_cbg = \"Non-CES Tract has 100% CEJST CBGs?\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2b26dccf",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Load CEJST score data\n",
|
|
"cejst_data_path = DATA_DIR / \"score\" / \"csv\" / \"usa.csv\"\n",
|
|
"\n",
|
|
"cejst_df = pd.read_csv(cejst_data_path)\n",
|
|
"\n",
|
|
"cejst_df.head()\n",
|
|
"\n",
|
|
"# Rename unclear name \"id\" to \"census_block_group_id\", as well as other renamings.\n",
|
|
"\n",
|
|
"score_used = \"Score A\"\n",
|
|
"\n",
|
|
"cejst_df.rename(\n",
|
|
" columns={\n",
|
|
" \"GEOID10\": CENSUS_BLOCK_GROUP_ID_FIELD,\n",
|
|
" \"Total population\": CENSUS_BLOCK_GROUP_POPULATION_FIELD,\n",
|
|
" score_used: CEJST_SCORE_FIELD,\n",
|
|
" f\"{score_used} (percentile)\": CEJST_PERCENTILE_FIELD,\n",
|
|
" },\n",
|
|
" inplace=True,\n",
|
|
" errors=\"raise\",\n",
|
|
")\n",
|
|
"\n",
|
|
"# Calculate the top K% of prioritized communities\n",
|
|
"cejst_df[CEJST_PRIORITY_COMMUNITY_FIELD] = (\n",
|
|
" cejst_df[CEJST_PERCENTILE_FIELD] >= CEJST_PRIORITY_COMMUNITY_THRESHOLD\n",
|
|
")\n",
|
|
"\n",
|
|
"# Create the CBG's Census Tract ID by dropping the last number from the FIPS CODE of the CBG.\n",
|
|
"# The CBG ID is the last one character.\n",
|
|
"# For more information, see https://www.census.gov/programs-surveys/geography/guidance/geo-identifiers.html.\n",
|
|
"cejst_df.loc[:, CENSUS_TRACT_ID_FIELD] = (\n",
|
|
" cejst_df.loc[:, CENSUS_BLOCK_GROUP_ID_FIELD].astype(str).str[:-1].astype(np.int64)\n",
|
|
")\n",
|
|
"\n",
|
|
"# Remove all non-California data\n",
|
|
"cejst_df = cejst_df.loc[\n",
|
|
" cejst_df[CENSUS_BLOCK_GROUP_ID_FIELD].astype(str).str[0] == \"6\", :\n",
|
|
"]\n",
|
|
"\n",
|
|
"cejst_df.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "ec6b27e3",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Data from https://calenviroscreen-oehha.hub.arcgis.com/#Data, specifically:\n",
|
|
"# https://oehha.ca.gov/media/downloads/calenviroscreen/document/calenviroscreen40resultsdatadictionaryd12021.zip\n",
|
|
"\n",
|
|
"download = requests.get(\n",
|
|
" \"https://justice40-data.s3.amazonaws.com/CalEnviroScreen/CalEnviroScreen_4.0_2021.zip\",\n",
|
|
" verify=False,\n",
|
|
")\n",
|
|
"file_contents = download.content\n",
|
|
"zip_file_path = TEMP_DATA_DIR\n",
|
|
"zip_file = open(zip_file_path / \"downloaded.zip\", \"wb\")\n",
|
|
"zip_file.write(file_contents)\n",
|
|
"zip_file.close()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "bdf08971",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Extract zip\n",
|
|
"print(zip_file_path)\n",
|
|
"with zipfile.ZipFile(zip_file_path / \"downloaded.zip\", \"r\") as zip_ref:\n",
|
|
" zip_ref.extractall(zip_file_path)\n",
|
|
"calenviroscreen_4_csv_name = \"CalEnviroScreen_4.0_2021.csv\"\n",
|
|
"calenviroscreen_data_path = TEMP_DATA_DIR.joinpath(calenviroscreen_4_csv_name)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "29c14b29",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Load comparison index (CalEnviroScreen 4)\n",
|
|
"\n",
|
|
"calenviroscreen_df = pd.read_csv(calenviroscreen_data_path)\n",
|
|
"\n",
|
|
"calenviroscreen_df.rename(\n",
|
|
" columns={\n",
|
|
" \"Census Tract\": CENSUS_TRACT_ID_FIELD,\n",
|
|
" \"DRAFT CES 4.0 Score\": CALENVIROSCREEN_SCORE_FIELD,\n",
|
|
" \"DRAFT CES 4.0 Percentile\": CALENVIROSCREEN_PERCENTILE_FIELD,\n",
|
|
" },\n",
|
|
" inplace=True,\n",
|
|
")\n",
|
|
"\n",
|
|
"\n",
|
|
"# Calculate the top K% of prioritized communities\n",
|
|
"calenviroscreen_df[CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD] = (\n",
|
|
" calenviroscreen_df[CALENVIROSCREEN_PERCENTILE_FIELD]\n",
|
|
" >= CALENVIROSCREEN_PRIORITY_COMMUNITY_THRESHOLD\n",
|
|
")\n",
|
|
"\n",
|
|
"calenviroscreen_df.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "813e5656",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Join CalEnviroScreen and CEJST data.\n",
|
|
"# Note: we're joining on the census *tract*, so there will be multiple CBG entries joined to the same census tract row from CES,\n",
|
|
"# creating multiple rows of the same CES data.\n",
|
|
"\n",
|
|
"# For simplicity, we'll only keep certain columns from each data frame.\n",
|
|
"cejst_columns_to_keep = [\n",
|
|
" CENSUS_BLOCK_GROUP_ID_FIELD,\n",
|
|
" CENSUS_TRACT_ID_FIELD,\n",
|
|
" CENSUS_BLOCK_GROUP_POPULATION_FIELD,\n",
|
|
" CEJST_SCORE_FIELD,\n",
|
|
" CEJST_PERCENTILE_FIELD,\n",
|
|
" CEJST_PRIORITY_COMMUNITY_FIELD,\n",
|
|
"]\n",
|
|
"\n",
|
|
"calenviroscreen_columns_to_keep = [\n",
|
|
" CENSUS_TRACT_ID_FIELD,\n",
|
|
" CALENVIROSCREEN_SCORE_FIELD,\n",
|
|
" CALENVIROSCREEN_PERCENTILE_FIELD,\n",
|
|
" CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD,\n",
|
|
"]\n",
|
|
"\n",
|
|
"merged_df = cejst_df.loc[:, cejst_columns_to_keep].merge(\n",
|
|
" calenviroscreen_df.loc[:, calenviroscreen_columns_to_keep],\n",
|
|
" how=\"left\",\n",
|
|
" on=CENSUS_TRACT_ID_FIELD,\n",
|
|
")\n",
|
|
"\n",
|
|
"merged_df.head()\n",
|
|
"\n",
|
|
"# merged_df.to_csv(\n",
|
|
"# path_or_buf=TEMP_DATA_DIR / \"merged.csv\",\n",
|
|
"# na_rep=\"\",\n",
|
|
"# index=False\n",
|
|
"# )"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "939baea4",
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Create analysis\n",
|
|
"def calculate_comparison(frame):\n",
|
|
" # Keep all the CES values at the Census Tract Level\n",
|
|
" df = frame.loc[\n",
|
|
" frame.index[0],\n",
|
|
" [\n",
|
|
" CENSUS_TRACT_ID_FIELD,\n",
|
|
" CALENVIROSCREEN_SCORE_FIELD,\n",
|
|
" CALENVIROSCREEN_PERCENTILE_FIELD,\n",
|
|
" CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD,\n",
|
|
" ],\n",
|
|
" ]\n",
|
|
"\n",
|
|
" # Convenience constant for whether the tract is or is not a CalEnviroScreen priority community.\n",
|
|
" is_a_ces_priority_tract = frame.loc[\n",
|
|
" frame.index[0], [CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD]\n",
|
|
" ][0]\n",
|
|
"\n",
|
|
" # Recall that NaN values are not falsy, so we need to check if `is_a_ces_priority_tract` is True.\n",
|
|
" is_a_ces_priority_tract = is_a_ces_priority_tract is True\n",
|
|
"\n",
|
|
" # Calculate whether the tract (whether or not it is a CES priority tract) includes CBGs that are priority\n",
|
|
" # according to the current CEJST score.\n",
|
|
" df[any_tract_has_at_least_one_cbg] = (\n",
|
|
" frame.loc[:, CEJST_PRIORITY_COMMUNITY_FIELD].sum() > 0\n",
|
|
" )\n",
|
|
"\n",
|
|
" # Calculate comparison\n",
|
|
" # A CES priority tract has at least one CEJST priority CBG.\n",
|
|
" df[tract_has_at_least_one_cbg] = (\n",
|
|
" frame.loc[:, CEJST_PRIORITY_COMMUNITY_FIELD].sum() > 0\n",
|
|
" if is_a_ces_priority_tract\n",
|
|
" else None\n",
|
|
" )\n",
|
|
"\n",
|
|
" # A CES priority tract has all of its contained CBGs as CEJST priority CBGs.\n",
|
|
" df[tract_has_100_percent_cbg] = (\n",
|
|
" frame.loc[:, CEJST_PRIORITY_COMMUNITY_FIELD].mean() == 1\n",
|
|
" if is_a_ces_priority_tract\n",
|
|
" else None\n",
|
|
" )\n",
|
|
"\n",
|
|
" # Calculate the inverse\n",
|
|
" # A tract that is _not_ a CES priority has at least one CEJST priority CBG.\n",
|
|
" df[non_ces_tract_has_at_least_one_cbg] = (\n",
|
|
" frame.loc[:, CEJST_PRIORITY_COMMUNITY_FIELD].sum() > 0\n",
|
|
" if not is_a_ces_priority_tract\n",
|
|
" else None\n",
|
|
" )\n",
|
|
"\n",
|
|
" # A tract that is _not_ a CES priority has all of its contained CBGs as CEJST priority CBGs.\n",
|
|
" df[non_ces_tract_has_100_percent_cbg] = (\n",
|
|
" frame.loc[:, CEJST_PRIORITY_COMMUNITY_FIELD].mean() == 1\n",
|
|
" if not is_a_ces_priority_tract\n",
|
|
" else None\n",
|
|
" )\n",
|
|
"\n",
|
|
" return df\n",
|
|
"\n",
|
|
"\n",
|
|
"# Group all data by the census tract.\n",
|
|
"grouped_df = merged_df.groupby(CENSUS_TRACT_ID_FIELD)\n",
|
|
"\n",
|
|
"# Run the comparison function on the groups.\n",
|
|
"comparison_df = grouped_df.progress_apply(calculate_comparison)\n",
|
|
"\n",
|
|
"# Sort descending by highest CES Score for convenience when viewing output file\n",
|
|
"comparison_df.sort_values(\n",
|
|
" by=[CALENVIROSCREEN_PERCENTILE_FIELD], ascending=False, inplace=True\n",
|
|
")\n",
|
|
"\n",
|
|
"# Write comparison to CSV.\n",
|
|
"comparison_df.to_csv(\n",
|
|
" path_or_buf=TEMP_DATA_DIR / \"Comparison Output.csv\", na_rep=\"\", index=False\n",
|
|
")\n",
|
|
"\n",
|
|
"print(comparison_df.head())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "85709225",
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Prepare some constants for use in the following Markdown cell.\n",
|
|
"total_cbgs_ca_only = len(cejst_df)\n",
|
|
"cejst_cbgs_ca_only = cejst_df.loc[:, CEJST_PRIORITY_COMMUNITY_FIELD].sum()\n",
|
|
"cejst_cbgs_ca_only_percent = f\"{cejst_cbgs_ca_only / total_cbgs_ca_only:.0%}\"\n",
|
|
"\n",
|
|
"total_tracts_count = len(comparison_df)\n",
|
|
"ces_tracts_count = comparison_df.loc[:, CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD].sum()\n",
|
|
"ces_tracts_count_percent = f\"{ces_tracts_count / total_tracts_count:.0%}\"\n",
|
|
"non_ces_tracts_count = total_tracts_count - ces_tracts_count\n",
|
|
"\n",
|
|
"total_tracts_count = len(comparison_df[CENSUS_TRACT_ID_FIELD])\n",
|
|
"cejst_tracts_count = comparison_df.loc[:, any_tract_has_at_least_one_cbg].sum()\n",
|
|
"cejst_tracts_count_percent = f\"{cejst_tracts_count / total_tracts_count:.0%}\"\n",
|
|
"\n",
|
|
"# CES stats\n",
|
|
"at_least_one_sum = comparison_df.loc[:, tract_has_at_least_one_cbg].sum()\n",
|
|
"at_least_one_sum_percent = f\"{at_least_one_sum / ces_tracts_count:.0%}\"\n",
|
|
"\n",
|
|
"all_100_sum = comparison_df.loc[:, tract_has_100_percent_cbg].sum()\n",
|
|
"all_100_sum_percent = f\"{all_100_sum / ces_tracts_count:.0%}\"\n",
|
|
"\n",
|
|
"# Non-CES stats:\n",
|
|
"non_ces_at_least_one_sum = comparison_df.loc[\n",
|
|
" :, non_ces_tract_has_at_least_one_cbg\n",
|
|
"].sum()\n",
|
|
"non_ces_at_least_one_sum_percent = (\n",
|
|
" f\"{non_ces_at_least_one_sum / non_ces_tracts_count:.0%}\"\n",
|
|
")\n",
|
|
"\n",
|
|
"non_ces_all_100_sum = comparison_df.loc[:, non_ces_tract_has_100_percent_cbg].sum()\n",
|
|
"non_ces_all_100_sum_percent = f\"{non_ces_all_100_sum / non_ces_tracts_count:.0%}\"\n",
|
|
"\n",
|
|
"# Note, for the following Markdown cell to render the variables properly, follow the steps at\n",
|
|
"# \"Activating variable-enabled Markdown for Jupyter notebooks\" within `score/README.md`."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "0c534966",
|
|
"metadata": {
|
|
"variables": {
|
|
" total_tracts_count": "8057",
|
|
"all_100_sum": "1168",
|
|
"all_100_sum_percent": "59%",
|
|
"at_least_one_sum": "1817",
|
|
"at_least_one_sum_percent": "92%",
|
|
"cejst_cbgs_ca_only": "6987",
|
|
"cejst_cbgs_ca_only_percent": "30%",
|
|
"cejst_tracts_count": "3516",
|
|
"cejst_tracts_count_percent": "44%",
|
|
"ces_tracts_count": "1983",
|
|
"ces_tracts_count_percent": "25%",
|
|
"datetime.today().strftime('%Y-%m-%d')": "2021-06-28",
|
|
"non_ces_all_100_sum": "438",
|
|
"non_ces_all_100_sum_percent": "7%",
|
|
"non_ces_at_least_one_sum": "1699",
|
|
"non_ces_at_least_one_sum_percent": "28%",
|
|
"score_used": "Score A",
|
|
"total_cbgs_ca_only": "23212"
|
|
}
|
|
},
|
|
"source": [
|
|
"# Summary of findings for {{score_used}}\n",
|
|
"\n",
|
|
"(Calculated on {{datetime.today().strftime('%Y-%m-%d')}})\n",
|
|
"\n",
|
|
"Recall that census tracts contain one or more census block groups, with up to nine census block groups per tract.\n",
|
|
"\n",
|
|
"There are {{ces_tracts_count}} census tracts designated as Disadvantaged Communities by CalEnviroScreen 4.0, out of {{total_tracts_count}} total tracts ({{ces_tracts_count_percent}}). \n",
|
|
"\n",
|
|
"Within California, there are {{cejst_cbgs_ca_only}} census block groups considered as priority communities by the current version of the CEJST score used in this analysis, out of {{total_cbgs_ca_only}} CBGs in the state ({{cejst_cbgs_ca_only_percent}}). They occupy {{cejst_tracts_count}} ({{cejst_tracts_count_percent}}) of all the census tracts in California.\n",
|
|
"\n",
|
|
"Out of every CalEnviroScreen Disadvantaged Community census tract, {{at_least_one_sum}} ({{at_least_one_sum_percent}}) of these census tracts have at least one census block group within them that is considered a priority community by the current version of the CEJST score.\n",
|
|
"\n",
|
|
"Out of every CalEnviroScreen Disadvantaged Community census tract, {{all_100_sum}} ({{all_100_sum_percent}}) of these census tracts have 100% of the included census block groups within them considered priority communities by the current version of the CEJST score.\n",
|
|
"\n",
|
|
"Out of every census tract in California that is __not__ marked as a CalEnviroScreen Disadvantaged Community, {{non_ces_at_least_one_sum}} ({{non_ces_at_least_one_sum_percent}}) of these census tracts have at least one census block group within them that is considered a priority community by the current version of the CEJST score.\n",
|
|
"\n",
|
|
"Out of every census tract in California that is __not__ marked as a CalEnviroScreen Disadvantaged Community, {{non_ces_all_100_sum}} ({{non_ces_all_100_sum_percent}}) of these census tracts have 100% of the included census block groups within them considered priority communities by the current version of the CEJST score."
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.7.1"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|