mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-02-23 10:04:18 -08:00
589ec483e3
* Ingest two data sources and add to score Co-authored-by: Jorge Escobar <jorge.e.escobar@omb.eop.gov>
348 lines
12 KiB
Text
348 lines
12 KiB
Text
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "54615cef",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Before running this script as it currently stands, you'll need to run two notebooks:\n",
|
|
"# 1. ejscreen_etl.ipynb\n",
|
|
"# 2. score_calc_0.1.ipynb\n",
|
|
"\n",
|
|
"import numpy as np\n",
|
|
"import pandas as pd\n",
|
|
"from pathlib import Path\n",
|
|
"import requests\n",
|
|
"import zipfile"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "49a63129",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Suppress scientific notation in pandas (this shows up for census tract IDs)\n",
|
|
"pd.options.display.float_format = \"{:.2f}\".format\n",
|
|
"\n",
|
|
"# Set some global parameters\n",
|
|
"DATA_DIR = Path.cwd().parent / \"data\"\n",
|
|
"TEMP_DATA_DIR = Path.cwd().parent / \"data\" / \"tmp\"\n",
|
|
"# None of these numbers are final, but just for the purposes of comparison.\n",
|
|
"CALENVIROSCREEN_PRIORITY_COMMUNITY_THRESHOLD = 75\n",
|
|
"CEJST_PRIORITY_COMMUNITY_THRESHOLD = 0.75\n",
|
|
"\n",
|
|
"# Name fields using variables. (This makes it easy to reference the same fields frequently without using strings\n",
|
|
"# and introducing the risk of misspelling the field name.)\n",
|
|
"CENSUS_BLOCK_GROUP_ID_FIELD = \"census_block_group_id\"\n",
|
|
"CENSUS_BLOCK_GROUP_POPULATION_FIELD = \"census_block_group_population\"\n",
|
|
"CENSUS_TRACT_ID_FIELD = \"census_tract_id\"\n",
|
|
"CALENVIROSCREEN_SCORE_FIELD = \"calenviroscreen_score\"\n",
|
|
"CALENVIROSCREEN_PERCENTILE_FIELD = \"calenviroscreen_percentile\"\n",
|
|
"CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD = \"calenviroscreen_priority_community\"\n",
|
|
"\n",
|
|
"# Note: we are pretending the EJSCREEN's low income percent is the actual score for now as a placeholder.\n",
|
|
"CEJST_SCORE_FIELD = \"cejst_score\"\n",
|
|
"CEJST_PERCENTILE_FIELD = \"cejst_percentile\"\n",
|
|
"CEJST_PRIORITY_COMMUNITY_FIELD = \"cejst_priority_community\"\n",
|
|
"\n",
|
|
"# Comparison field names\n",
|
|
"tract_has_at_least_one_cbg = \"CES Tract has at least one CEJST CBG?\"\n",
|
|
"tract_has_100_percent_cbg = \"CES Tract has 100% CEJST CBGs?\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2b26dccf",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Load CEJST score data\n",
|
|
"cejst_data_path = DATA_DIR / \"score\" / \"csv\" / \"usa.csv\"\n",
|
|
"\n",
|
|
"cejst_df = pd.read_csv(cejst_data_path)\n",
|
|
"\n",
|
|
"cejst_df.head()\n",
|
|
"\n",
|
|
"# Rename unclear name \"id\" to \"census_block_group_id\", as well as other renamings.\n",
|
|
"cejst_df.rename(\n",
|
|
" columns={\n",
|
|
" \"GEOID10\": CENSUS_BLOCK_GROUP_ID_FIELD,\n",
|
|
" \"Total population\": CENSUS_BLOCK_GROUP_POPULATION_FIELD,\n",
|
|
" \"Score C\": CEJST_SCORE_FIELD,\n",
|
|
" \"Score C (percentile)\": CEJST_PERCENTILE_FIELD,\n",
|
|
" },\n",
|
|
" inplace=True,\n",
|
|
" errors=\"raise\",\n",
|
|
")\n",
|
|
"\n",
|
|
"# Calculate the top K% of prioritized communities\n",
|
|
"cejst_df[CEJST_PRIORITY_COMMUNITY_FIELD] = (\n",
|
|
" cejst_df[CEJST_PERCENTILE_FIELD] >= CEJST_PRIORITY_COMMUNITY_THRESHOLD\n",
|
|
")\n",
|
|
"\n",
|
|
"# Create the CBG's Census Tract ID by dropping the last number from the FIPS CODE of the CBG.\n",
|
|
"# The CBG ID is the last one character.\n",
|
|
"# For more information, see https://www.census.gov/programs-surveys/geography/guidance/geo-identifiers.html.\n",
|
|
"cejst_df.loc[:, CENSUS_TRACT_ID_FIELD] = (\n",
|
|
" cejst_df.loc[:, CENSUS_BLOCK_GROUP_ID_FIELD].astype(str).str[:-1].astype(np.int64)\n",
|
|
")\n",
|
|
"\n",
|
|
"# Remove all non-California data\n",
|
|
"cejst_df = cejst_df.loc[\n",
|
|
" cejst_df[CENSUS_BLOCK_GROUP_ID_FIELD].astype(str).str[0] == \"6\", :\n",
|
|
"]\n",
|
|
"\n",
|
|
"cejst_df.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "ec6b27e3",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Data from https://calenviroscreen-oehha.hub.arcgis.com/#Data, specifically:\n",
|
|
"# https://oehha.ca.gov/media/downloads/calenviroscreen/document/calenviroscreen40resultsdatadictionaryd12021.zip\n",
|
|
"\n",
|
|
"download = requests.get(\n",
|
|
" \"https://justice40-data.s3.amazonaws.com/CalEnviroScreen/CalEnviroScreen_4.0_2021.zip\",\n",
|
|
" verify=False,\n",
|
|
")\n",
|
|
"file_contents = download.content\n",
|
|
"zip_file_path = TEMP_DATA_DIR\n",
|
|
"zip_file = open(zip_file_path / \"downloaded.zip\", \"wb\")\n",
|
|
"zip_file.write(file_contents)\n",
|
|
"zip_file.close()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "bdf08971",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Extract zip\n",
|
|
"print(zip_file_path)\n",
|
|
"with zipfile.ZipFile(zip_file_path / \"downloaded.zip\", \"r\") as zip_ref:\n",
|
|
" zip_ref.extractall(zip_file_path)\n",
|
|
"calenviroscreen_4_csv_name = \"CalEnviroScreen_4.0_2021.csv\"\n",
|
|
"calenviroscreen_data_path = TEMP_DATA_DIR.joinpath(calenviroscreen_4_csv_name)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "29c14b29",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Load comparison index (CalEnviroScreen 4)\n",
|
|
"\n",
|
|
"calenviroscreen_df = pd.read_csv(calenviroscreen_data_path)\n",
|
|
"\n",
|
|
"calenviroscreen_df.rename(\n",
|
|
" columns={\n",
|
|
" \"Census Tract\": CENSUS_TRACT_ID_FIELD,\n",
|
|
" \"DRAFT CES 4.0 Score\": CALENVIROSCREEN_SCORE_FIELD,\n",
|
|
" \"DRAFT CES 4.0 Percentile\": CALENVIROSCREEN_PERCENTILE_FIELD,\n",
|
|
" },\n",
|
|
" inplace=True,\n",
|
|
")\n",
|
|
"\n",
|
|
"\n",
|
|
"# Calculate the top K% of prioritized communities\n",
|
|
"calenviroscreen_df[CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD] = (\n",
|
|
" calenviroscreen_df[CALENVIROSCREEN_PERCENTILE_FIELD]\n",
|
|
" >= CALENVIROSCREEN_PRIORITY_COMMUNITY_THRESHOLD\n",
|
|
")\n",
|
|
"\n",
|
|
"calenviroscreen_df.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "813e5656",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Join CalEnviroScreen and CEJST data.\n",
|
|
"# Note: we're joining on the census *tract*, so there will be multiple CBG entries joined to the same census tract row from CES,\n",
|
|
"# creating multiple rows of the same CES data.\n",
|
|
"\n",
|
|
"# For simplicity, we'll only keep certain columns from each data frame.\n",
|
|
"cejst_columns_to_keep = [\n",
|
|
" CENSUS_BLOCK_GROUP_ID_FIELD,\n",
|
|
" CENSUS_TRACT_ID_FIELD,\n",
|
|
" CENSUS_BLOCK_GROUP_POPULATION_FIELD,\n",
|
|
" CEJST_SCORE_FIELD,\n",
|
|
" CEJST_PERCENTILE_FIELD,\n",
|
|
" CEJST_PRIORITY_COMMUNITY_FIELD,\n",
|
|
"]\n",
|
|
"\n",
|
|
"calenviroscreen_columns_to_keep = [\n",
|
|
" CENSUS_TRACT_ID_FIELD,\n",
|
|
" CALENVIROSCREEN_SCORE_FIELD,\n",
|
|
" CALENVIROSCREEN_PERCENTILE_FIELD,\n",
|
|
" CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD,\n",
|
|
"]\n",
|
|
"\n",
|
|
"merged_df = cejst_df.loc[:, cejst_columns_to_keep].merge(\n",
|
|
" calenviroscreen_df.loc[:, calenviroscreen_columns_to_keep],\n",
|
|
" how=\"left\",\n",
|
|
" on=CENSUS_TRACT_ID_FIELD,\n",
|
|
")\n",
|
|
"\n",
|
|
"merged_df.head()\n",
|
|
"\n",
|
|
"# merged_df.to_csv(\n",
|
|
"# path_or_buf=TEMP_DATA_DIR / \"merged.csv\",\n",
|
|
"# na_rep=\"\",\n",
|
|
"# index=False\n",
|
|
"# )"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "939baea4",
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Create analysis\n",
|
|
"def calculate_comparison(frame):\n",
|
|
" # Keep all the CES values at the Census Tract Level\n",
|
|
" df = frame.loc[\n",
|
|
" frame.index[0],\n",
|
|
" [\n",
|
|
" CENSUS_TRACT_ID_FIELD,\n",
|
|
" CALENVIROSCREEN_SCORE_FIELD,\n",
|
|
" CALENVIROSCREEN_PERCENTILE_FIELD,\n",
|
|
" CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD,\n",
|
|
" ],\n",
|
|
" ]\n",
|
|
"\n",
|
|
" # Convenience constant for whether the tract is or is not a CalEnviroScreen priority community.\n",
|
|
" is_a_ces_priority_tract = frame.loc[\n",
|
|
" frame.index[0], [CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD]\n",
|
|
" ][0]\n",
|
|
"\n",
|
|
" # Recall that NaN values are not falsy, so we need to check if `is_a_ces_priority_tract` is True.\n",
|
|
" is_a_ces_priority_tract = is_a_ces_priority_tract is True\n",
|
|
"\n",
|
|
" # Calculate comparison\n",
|
|
" df[tract_has_at_least_one_cbg] = (\n",
|
|
" frame.loc[:, CEJST_PRIORITY_COMMUNITY_FIELD].sum() > 0\n",
|
|
" if is_a_ces_priority_tract\n",
|
|
" else None\n",
|
|
" )\n",
|
|
" df[tract_has_100_percent_cbg] = (\n",
|
|
" frame.loc[:, CEJST_PRIORITY_COMMUNITY_FIELD].mean() == 1\n",
|
|
" if is_a_ces_priority_tract\n",
|
|
" else None\n",
|
|
" )\n",
|
|
"\n",
|
|
" return df\n",
|
|
"\n",
|
|
"\n",
|
|
"# Group all data by the census tract.\n",
|
|
"grouped_df = merged_df.groupby(CENSUS_TRACT_ID_FIELD)\n",
|
|
"\n",
|
|
"# Run the comparison function on the groups.\n",
|
|
"comparison_df = grouped_df.apply(calculate_comparison)\n",
|
|
"\n",
|
|
"# Sort descending by highest CES Score for convenience when viewing output file\n",
|
|
"comparison_df.sort_values(\n",
|
|
" by=[CALENVIROSCREEN_PERCENTILE_FIELD], ascending=False, inplace=True\n",
|
|
")\n",
|
|
"\n",
|
|
"# Write comparison to CSV.\n",
|
|
"comparison_df.to_csv(\n",
|
|
" path_or_buf=TEMP_DATA_DIR / \"Comparison Output.csv\", na_rep=\"\", index=False\n",
|
|
")\n",
|
|
"\n",
|
|
"print(comparison_df.head())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "85709225",
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Prepare some constants for use in the following Markdown cell.\n",
|
|
"\n",
|
|
"cejst_cbgs_ca_only = cejst_df.loc[:, CEJST_PRIORITY_COMMUNITY_FIELD].sum()\n",
|
|
"ces_tracts_count = comparison_df.loc[:, CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD].sum()\n",
|
|
"at_least_one_sum = comparison_df.loc[:, tract_has_at_least_one_cbg].sum()\n",
|
|
"at_least_one_sum_percent = f\"{at_least_one_sum / ces_tracts_count:.0%}\"\n",
|
|
"\n",
|
|
"all_100_sum = comparison_df.loc[:, tract_has_100_percent_cbg].sum()\n",
|
|
"all_100_sum_percent = f\"{all_100_sum / ces_tracts_count:.0%}\"\n",
|
|
"\n",
|
|
"# Note, for the following Markdown cell to render the variables properly, follow the steps at\n",
|
|
"# \"Activating variable-enabled Markdown for Jupyter notebooks\" within `score/README.md`."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "0c534966",
|
|
"metadata": {
|
|
"variables": {
|
|
"all_100_sum": "1373",
|
|
"all_100_sum_percent": "69%",
|
|
"at_least_one_sum": "1866",
|
|
"at_least_one_sum_percent": "94%",
|
|
"cejst_cbgs_ca_only": "10849",
|
|
"ces_tracts_count": "1983"
|
|
}
|
|
},
|
|
"source": [
|
|
"# Summary of findings\n",
|
|
"\n",
|
|
"Recall that census tracts contain one or more census block groups, with up to nine census block groups per tract.\n",
|
|
"\n",
|
|
"There are {{ces_tracts_count}} census tracts designated as Disadvantaged Communities by CalEnviroScreen 4.0. \n",
|
|
"\n",
|
|
"Within California, there are {{cejst_cbgs_ca_only}} census block groups considered as priority communities by the current version of the CEJST score used in this analysis.\n",
|
|
"\n",
|
|
"Out of every CalEnviroScreen Disadvantaged Community census tract, {{at_least_one_sum}} ({{at_least_one_sum_percent}}) of these census tracts have at least one census block group within them that is considered a priority community by the current version of the CEJST score.\n",
|
|
"\n",
|
|
"Out of every CalEnviroScreen Disadvantaged Community census tract, {{all_100_sum}} ({{all_100_sum_percent}}) of these census tracts have 100% of the included census block groups within them considered priority communities by the current version of the CEJST score."
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.7.1"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|