{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54615cef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Before running this script as it currently stands, you'll need to run two notebooks:\n",
    "# 1. ejscreen_etl.ipynb\n",
    "# 2. score_calc_0.1.ipynb\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import requests\n",
    "import zipfile\n",
    "from datetime import datetime\n",
    "from tqdm.notebook import tqdm_notebook\n",
    "\n",
    "# Turn on TQDM for pandas so that we can have progress bars when running `apply`.\n",
    "tqdm_notebook.pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49a63129",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Suppress scientific notation in pandas (this shows up for census tract IDs)\n",
    "pd.options.display.float_format = \"{:.2f}\".format\n",
    "\n",
    "# Set some global parameters\n",
    "DATA_DIR = Path.cwd().parent / \"data\"\n",
    "TEMP_DATA_DIR = Path.cwd().parent / \"data\" / \"tmp\"\n",
    "# None of these numbers are final, but just for the purposes of comparison.\n",
    "CALENVIROSCREEN_PRIORITY_COMMUNITY_THRESHOLD = 75\n",
    "CEJST_PRIORITY_COMMUNITY_THRESHOLD = 0.75\n",
    "\n",
    "# Name fields using variables. (This makes it easy to reference the same fields frequently without using strings\n",
    "# and introducing the risk of misspelling the field name.)\n",
    "CENSUS_BLOCK_GROUP_ID_FIELD = \"census_block_group_id\"\n",
    "CENSUS_BLOCK_GROUP_POPULATION_FIELD = \"census_block_group_population\"\n",
    "CENSUS_TRACT_ID_FIELD = \"census_tract_id\"\n",
    "CALENVIROSCREEN_SCORE_FIELD = \"calenviroscreen_score\"\n",
    "CALENVIROSCREEN_PERCENTILE_FIELD = \"calenviroscreen_percentile\"\n",
    "CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD = \"calenviroscreen_priority_community\"\n",
    "\n",
    "# Note: we are pretending the EJSCREEN's low income percent is the actual score for now as a placeholder.\n",
    "CEJST_SCORE_FIELD = \"cejst_score\"\n",
    "CEJST_PERCENTILE_FIELD = \"cejst_percentile\"\n",
    "CEJST_PRIORITY_COMMUNITY_FIELD = \"cejst_priority_community\"\n",
    "\n",
    "# Comparison field names\n",
    "any_tract_has_at_least_one_cbg = \"Tract has at least one CEJST CBG?\"\n",
    "tract_has_at_least_one_cbg = \"CES Tract has at least one CEJST CBG?\"\n",
    "tract_has_100_percent_cbg = \"CES Tract has 100% CEJST CBGs?\"\n",
    "non_ces_tract_has_at_least_one_cbg = \"Non-CES Tract has at least one CEJST CBG?\"\n",
    "non_ces_tract_has_100_percent_cbg = \"Non-CES Tract has 100% CEJST CBGs?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b26dccf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load CEJST score data\n",
    "cejst_data_path = DATA_DIR / \"score\" / \"csv\" / \"usa.csv\"\n",
    "\n",
    "cejst_df = pd.read_csv(cejst_data_path)\n",
    "\n",
    "cejst_df.head()\n",
    "\n",
    "# Rename unclear name \"id\" to \"census_block_group_id\", as well as other renamings.\n",
    "\n",
    "score_used = \"Score A\"\n",
    "\n",
    "cejst_df.rename(\n",
    "    columns={\n",
    "        \"GEOID10\": CENSUS_BLOCK_GROUP_ID_FIELD,\n",
    "        \"Total population\": CENSUS_BLOCK_GROUP_POPULATION_FIELD,\n",
    "        score_used: CEJST_SCORE_FIELD,\n",
    "        f\"{score_used} (percentile)\": CEJST_PERCENTILE_FIELD,\n",
    "    },\n",
    "    inplace=True,\n",
    "    errors=\"raise\",\n",
    ")\n",
    "\n",
    "# Calculate the top K% of prioritized communities\n",
    "cejst_df[CEJST_PRIORITY_COMMUNITY_FIELD] = (\n",
    "    cejst_df[CEJST_PERCENTILE_FIELD] >= CEJST_PRIORITY_COMMUNITY_THRESHOLD\n",
    ")\n",
    "\n",
    "# Create the CBG's Census Tract ID by dropping the last number from the FIPS CODE of the CBG.\n",
    "# The CBG ID is the last one character.\n",
    "# For more information, see https://www.census.gov/programs-surveys/geography/guidance/geo-identifiers.html.\n",
    "cejst_df.loc[:, CENSUS_TRACT_ID_FIELD] = (\n",
    "    cejst_df.loc[:, CENSUS_BLOCK_GROUP_ID_FIELD].astype(str).str[:-1].astype(np.int64)\n",
    ")\n",
    "\n",
    "# Remove all non-California data\n",
    "cejst_df = cejst_df.loc[\n",
    "    cejst_df[CENSUS_BLOCK_GROUP_ID_FIELD].astype(str).str[0] == \"6\", :\n",
    "]\n",
    "\n",
    "cejst_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec6b27e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data from https://calenviroscreen-oehha.hub.arcgis.com/#Data, specifically:\n",
    "# https://oehha.ca.gov/media/downloads/calenviroscreen/document/calenviroscreen40resultsdatadictionaryd12021.zip\n",
    "\n",
    "download = requests.get(\n",
    "    \"https://justice40-data.s3.amazonaws.com/CalEnviroScreen/CalEnviroScreen_4.0_2021.zip\",\n",
    "    verify=False,\n",
    ")\n",
    "file_contents = download.content\n",
    "zip_file_path = TEMP_DATA_DIR\n",
    "zip_file = open(zip_file_path / \"downloaded.zip\", \"wb\")\n",
    "zip_file.write(file_contents)\n",
    "zip_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bdf08971",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract zip\n",
    "print(zip_file_path)\n",
    "with zipfile.ZipFile(zip_file_path / \"downloaded.zip\", \"r\") as zip_ref:\n",
    "    zip_ref.extractall(zip_file_path)\n",
    "calenviroscreen_4_csv_name = \"CalEnviroScreen_4.0_2021.csv\"\n",
    "calenviroscreen_data_path = TEMP_DATA_DIR.joinpath(calenviroscreen_4_csv_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29c14b29",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load comparison index (CalEnviroScreen 4)\n",
    "\n",
    "calenviroscreen_df = pd.read_csv(calenviroscreen_data_path)\n",
    "\n",
    "calenviroscreen_df.rename(\n",
    "    columns={\n",
    "        \"Census Tract\": CENSUS_TRACT_ID_FIELD,\n",
    "        \"DRAFT CES 4.0 Score\": CALENVIROSCREEN_SCORE_FIELD,\n",
    "        \"DRAFT CES 4.0 Percentile\": CALENVIROSCREEN_PERCENTILE_FIELD,\n",
    "    },\n",
    "    inplace=True,\n",
    ")\n",
    "\n",
    "\n",
    "# Calculate the top K% of prioritized communities\n",
    "calenviroscreen_df[CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD] = (\n",
    "    calenviroscreen_df[CALENVIROSCREEN_PERCENTILE_FIELD]\n",
    "    >= CALENVIROSCREEN_PRIORITY_COMMUNITY_THRESHOLD\n",
    ")\n",
    "\n",
    "calenviroscreen_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "813e5656",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Join CalEnviroScreen and CEJST data.\n",
    "# Note: we're joining on the census *tract*, so there will be multiple CBG entries joined to the same census tract row from CES,\n",
    "# creating multiple rows of the same CES data.\n",
    "\n",
    "# For simplicity, we'll only keep certain columns from each data frame.\n",
    "cejst_columns_to_keep = [\n",
    "    CENSUS_BLOCK_GROUP_ID_FIELD,\n",
    "    CENSUS_TRACT_ID_FIELD,\n",
    "    CENSUS_BLOCK_GROUP_POPULATION_FIELD,\n",
    "    CEJST_SCORE_FIELD,\n",
    "    CEJST_PERCENTILE_FIELD,\n",
    "    CEJST_PRIORITY_COMMUNITY_FIELD,\n",
    "]\n",
    "\n",
    "calenviroscreen_columns_to_keep = [\n",
    "    CENSUS_TRACT_ID_FIELD,\n",
    "    CALENVIROSCREEN_SCORE_FIELD,\n",
    "    CALENVIROSCREEN_PERCENTILE_FIELD,\n",
    "    CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD,\n",
    "]\n",
    "\n",
    "merged_df = cejst_df.loc[:, cejst_columns_to_keep].merge(\n",
    "    calenviroscreen_df.loc[:, calenviroscreen_columns_to_keep],\n",
    "    how=\"left\",\n",
    "    on=CENSUS_TRACT_ID_FIELD,\n",
    ")\n",
    "\n",
    "merged_df.head()\n",
    "\n",
    "# merged_df.to_csv(\n",
    "#     path_or_buf=TEMP_DATA_DIR / \"merged.csv\",\n",
    "#     na_rep=\"\",\n",
    "#     index=False\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "939baea4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Create analysis\n",
    "def calculate_comparison(frame):\n",
    "    # Keep all the CES values at the Census Tract Level\n",
    "    df = frame.loc[\n",
    "        frame.index[0],\n",
    "        [\n",
    "            CENSUS_TRACT_ID_FIELD,\n",
    "            CALENVIROSCREEN_SCORE_FIELD,\n",
    "            CALENVIROSCREEN_PERCENTILE_FIELD,\n",
    "            CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD,\n",
    "        ],\n",
    "    ]\n",
    "\n",
    "    # Convenience constant for whether the tract is or is not a CalEnviroScreen priority community.\n",
    "    is_a_ces_priority_tract = frame.loc[\n",
    "        frame.index[0], [CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD]\n",
    "    ][0]\n",
    "\n",
    "    # Recall that NaN values are not falsy, so we need to check if `is_a_ces_priority_tract` is True.\n",
    "    is_a_ces_priority_tract = is_a_ces_priority_tract is True\n",
    "\n",
    "    # Calculate whether the tract (whether or not it is a CES priority tract) includes CBGs that are priority\n",
    "    # according to the current CEJST score.\n",
    "    df[any_tract_has_at_least_one_cbg] = (\n",
    "        frame.loc[:, CEJST_PRIORITY_COMMUNITY_FIELD].sum() > 0\n",
    "    )\n",
    "\n",
    "    # Calculate comparison\n",
    "    # A CES priority tract has at least one CEJST priority CBG.\n",
    "    df[tract_has_at_least_one_cbg] = (\n",
    "        frame.loc[:, CEJST_PRIORITY_COMMUNITY_FIELD].sum() > 0\n",
    "        if is_a_ces_priority_tract\n",
    "        else None\n",
    "    )\n",
    "\n",
    "    # A CES priority tract has all of its contained CBGs as CEJST priority CBGs.\n",
    "    df[tract_has_100_percent_cbg] = (\n",
    "        frame.loc[:, CEJST_PRIORITY_COMMUNITY_FIELD].mean() == 1\n",
    "        if is_a_ces_priority_tract\n",
    "        else None\n",
    "    )\n",
    "\n",
    "    # Calculate the inverse\n",
    "    # A tract that is _not_ a CES priority has at least one CEJST priority CBG.\n",
    "    df[non_ces_tract_has_at_least_one_cbg] = (\n",
    "        frame.loc[:, CEJST_PRIORITY_COMMUNITY_FIELD].sum() > 0\n",
    "        if not is_a_ces_priority_tract\n",
    "        else None\n",
    "    )\n",
    "\n",
    "    # A tract that is _not_ a CES priority has all of its contained CBGs as CEJST priority CBGs.\n",
    "    df[non_ces_tract_has_100_percent_cbg] = (\n",
    "        frame.loc[:, CEJST_PRIORITY_COMMUNITY_FIELD].mean() == 1\n",
    "        if not is_a_ces_priority_tract\n",
    "        else None\n",
    "    )\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "# Group all data by the census tract.\n",
    "grouped_df = merged_df.groupby(CENSUS_TRACT_ID_FIELD)\n",
    "\n",
    "# Run the comparison function on the groups.\n",
    "comparison_df = grouped_df.progress_apply(calculate_comparison)\n",
    "\n",
    "# Sort descending by highest CES Score for convenience when viewing output file\n",
    "comparison_df.sort_values(\n",
    "    by=[CALENVIROSCREEN_PERCENTILE_FIELD], ascending=False, inplace=True\n",
    ")\n",
    "\n",
    "# Write comparison to CSV.\n",
    "comparison_df.to_csv(\n",
    "    path_or_buf=TEMP_DATA_DIR / \"Comparison Output.csv\", na_rep=\"\", index=False\n",
    ")\n",
    "\n",
    "print(comparison_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85709225",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Prepare some constants for use in the following Markdown cell.\n",
    "total_cbgs_ca_only = len(cejst_df)\n",
    "cejst_cbgs_ca_only = cejst_df.loc[:, CEJST_PRIORITY_COMMUNITY_FIELD].sum()\n",
    "cejst_cbgs_ca_only_percent = f\"{cejst_cbgs_ca_only / total_cbgs_ca_only:.0%}\"\n",
    "\n",
    "total_tracts_count = len(comparison_df)\n",
    "ces_tracts_count = comparison_df.loc[:, CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD].sum()\n",
    "ces_tracts_count_percent = f\"{ces_tracts_count / total_tracts_count:.0%}\"\n",
    "non_ces_tracts_count = total_tracts_count - ces_tracts_count\n",
    "\n",
    "total_tracts_count = len(comparison_df[CENSUS_TRACT_ID_FIELD])\n",
    "cejst_tracts_count = comparison_df.loc[:, any_tract_has_at_least_one_cbg].sum()\n",
    "cejst_tracts_count_percent = f\"{cejst_tracts_count / total_tracts_count:.0%}\"\n",
    "\n",
    "# CES stats\n",
    "at_least_one_sum = comparison_df.loc[:, tract_has_at_least_one_cbg].sum()\n",
    "at_least_one_sum_percent = f\"{at_least_one_sum / ces_tracts_count:.0%}\"\n",
    "\n",
    "all_100_sum = comparison_df.loc[:, tract_has_100_percent_cbg].sum()\n",
    "all_100_sum_percent = f\"{all_100_sum / ces_tracts_count:.0%}\"\n",
    "\n",
    "# Non-CES stats:\n",
    "non_ces_at_least_one_sum = comparison_df.loc[\n",
    "    :, non_ces_tract_has_at_least_one_cbg\n",
    "].sum()\n",
    "non_ces_at_least_one_sum_percent = (\n",
    "    f\"{non_ces_at_least_one_sum / non_ces_tracts_count:.0%}\"\n",
    ")\n",
    "\n",
    "non_ces_all_100_sum = comparison_df.loc[:, non_ces_tract_has_100_percent_cbg].sum()\n",
    "non_ces_all_100_sum_percent = f\"{non_ces_all_100_sum / non_ces_tracts_count:.0%}\"\n",
    "\n",
    "# Note, for the following Markdown cell to render the variables properly, follow the steps at\n",
    "# \"Activating variable-enabled Markdown for Jupyter notebooks\" within `score/README.md`."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c534966",
   "metadata": {
    "variables": {
     " total_tracts_count": "8057",
     "all_100_sum": "1168",
     "all_100_sum_percent": "59%",
     "at_least_one_sum": "1817",
     "at_least_one_sum_percent": "92%",
     "cejst_cbgs_ca_only": "6987",
     "cejst_cbgs_ca_only_percent": "30%",
     "cejst_tracts_count": "3516",
     "cejst_tracts_count_percent": "44%",
     "ces_tracts_count": "1983",
     "ces_tracts_count_percent": "25%",
     "datetime.today().strftime('%Y-%m-%d')": "2021-06-28",
     "non_ces_all_100_sum": "438",
     "non_ces_all_100_sum_percent": "7%",
     "non_ces_at_least_one_sum": "1699",
     "non_ces_at_least_one_sum_percent": "28%",
     "score_used": "Score A",
     "total_cbgs_ca_only": "23212"
    }
   },
   "source": [
    "# Summary of findings for {{score_used}}\n",
    "\n",
    "(Calculated on {{datetime.today().strftime('%Y-%m-%d')}})\n",
    "\n",
    "Recall that census tracts contain one or more census block groups, with up to nine census block groups per tract.\n",
    "\n",
    "There are {{ces_tracts_count}} census tracts designated as Disadvantaged Communities by CalEnviroScreen 4.0, out of {{total_tracts_count}} total tracts ({{ces_tracts_count_percent}}). \n",
    "\n",
    "Within California, there are {{cejst_cbgs_ca_only}} census block groups considered as priority communities by the current version of the CEJST score used in this analysis, out of {{total_cbgs_ca_only}} CBGs in the state ({{cejst_cbgs_ca_only_percent}}). They occupy {{cejst_tracts_count}} ({{cejst_tracts_count_percent}}) of all the census tracts in California.\n",
    "\n",
    "Out of every CalEnviroScreen Disadvantaged Community census tract, {{at_least_one_sum}} ({{at_least_one_sum_percent}}) of these census tracts have at least one census block group within them that is considered a priority community by the current version of the CEJST score.\n",
    "\n",
    "Out of every CalEnviroScreen Disadvantaged Community census tract, {{all_100_sum}} ({{all_100_sum_percent}}) of these census tracts have 100% of the included census block groups within them considered priority communities by the current version of the CEJST score.\n",
    "\n",
    "Out of every census tract in California that is __not__ marked as a CalEnviroScreen Disadvantaged Community, {{non_ces_at_least_one_sum}} ({{non_ces_at_least_one_sum_percent}}) of these census tracts have at least one census block group within them that is considered a priority community by the current version of the CEJST score.\n",
    "\n",
    "Out of every census tract in California that is __not__ marked as a CalEnviroScreen Disadvantaged Community, {{non_ces_all_100_sum}} ({{non_ces_all_100_sum_percent}}) of these census tracts have 100% of the included census block groups within them considered priority communities by the current version of the CEJST score."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}