mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-02-24 10:34:18 -08:00
383 lines
11 KiB
Text
383 lines
11 KiB
Text
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "51412a14",
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import collections\n",
|
|
"from datetime import datetime\n",
|
|
"import functools\n",
|
|
"import glob\n",
|
|
"import itertools\n",
|
|
"import os\n",
|
|
"import pathlib\n",
|
|
"import requests\n",
|
|
"import string\n",
|
|
"import sys\n",
|
|
"import typing\n",
|
|
"import zipfile\n",
|
|
"\n",
|
|
"import IPython\n",
|
|
"import numpy as np\n",
|
|
"import pandas as pd\n",
|
|
"import pypandoc\n",
|
|
"\n",
|
|
"from tqdm.notebook import tqdm_notebook\n",
|
|
"\n",
|
|
"module_path = os.path.abspath(os.path.join(\"../..\"))\n",
|
|
"if module_path not in sys.path:\n",
|
|
" sys.path.append(module_path)\n",
|
|
"\n",
|
|
"from data_pipeline.utils import remove_all_from_dir, get_excel_column_name\n",
|
|
"from data_pipeline.etl.sources.census.etl_utils import get_state_information\n",
|
|
"\n",
|
|
"# Turn on TQDM for pandas so that we can have progress bars when running `apply`.\n",
|
|
"tqdm_notebook.pandas()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e3234c61",
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Suppress scientific notation in pandas (this shows up for census tract IDs)\n",
|
|
"pd.options.display.float_format = \"{:.2f}\".format\n",
|
|
"pd.set_option('max_columns', None)\n",
|
|
"\n",
|
|
"# Set some global parameters\n",
|
|
"DATA_DIR = pathlib.Path.cwd().parent / \"data\"\n",
|
|
"TEMP_DATA_DIR = DATA_DIR / \"tmp\"\n",
|
|
"COMPARISON_OUTPUTS_DIR = DATA_DIR / \"comparison_outputs\"\n",
|
|
"\n",
|
|
"## I (Vincent) created this manually locally. Will need to change potentially when putting into official ETL scripts\n",
|
|
"EJSCREEN_DATA_DIR = DATA_DIR / \"ejscreen\"\n",
|
|
"EJSCREEN_CEQ_NAT_DIR = EJSCREEN_DATA_DIR / \"CEQ_NationalExports\"\n",
|
|
"EJSCREEN_CEQ_STA_DIR = EJSCREEN_DATA_DIR / \"CEQ_StateExports\"\n",
|
|
"\n",
|
|
"# Make the dirs if they don't exist\n",
|
|
"TEMP_DATA_DIR.mkdir(parents=True, exist_ok=True)\n",
|
|
"COMPARISON_OUTPUTS_DIR.mkdir(parents=True, exist_ok=True)\n",
|
|
"\n",
|
|
"# Name fields using variables. (This makes it easy to reference the same fields frequently without using strings\n",
|
|
"# and introducing the risk of misspelling the field name.)\n",
|
|
"\n",
|
|
"GEOID_FIELD_NAME = \"GEOID10\"\n",
|
|
"GEOID_TRACT_FIELD_NAME = \"GEOID10_TRACT\"\n",
|
|
"GEOID_STATE_FIELD_NAME = \"GEOID10_STATE\"\n",
|
|
"GEOID_CBG_FIELD_NAME = \"GEOID10_CBG\"\n",
|
|
"COUNTRY_FIELD_NAME = \"Country\"\n",
|
|
"CENSUS_BLOCK_GROUP_POPULATION_FIELD = \"Total population\"\n",
|
|
"\n",
|
|
"CEJST_SCORE_FIELD = \"cejst_score\"\n",
|
|
"CEJST_PERCENTILE_FIELD = \"cejst_percentile\"\n",
|
|
"CEJST_PRIORITY_COMMUNITY_FIELD = \"cejst_priority_community\"\n",
|
|
"\n",
|
|
"# Define some suffixes\n",
|
|
"POPULATION_SUFFIX = \" (priority population)\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "376f5b2e",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Loading EJ Screen CEQ Data"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "886e3f8c",
|
|
"metadata": {},
|
|
"source": [
|
|
"### National"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0c1b8c66",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Replace this with something like glob when you have internet\n",
|
|
"filenames = [\n",
|
|
" 'CEQ_EJSCREEN_National_70.csv',\n",
|
|
" 'CEQ_EJSCREEN_National_75.csv',\n",
|
|
" 'CEQ_EJSCREEN_National_80.csv',\n",
|
|
" 'CEQ_EJSCREEN_National_85.csv',\n",
|
|
" 'CEQ_EJSCREEN_National_90.csv',\n",
|
|
" 'CEQ_EJSCREEN_National_95.csv',\n",
|
|
"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "a75dadc5",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"dfs = []\n",
|
|
"for f in filenames:\n",
|
|
" percentile = f[-6:][:-4]\n",
|
|
" print(percentile)\n",
|
|
"\n",
|
|
" df = pd.read_csv(\n",
|
|
" os.path.join(EJSCREEN_CEQ_NAT_DIR, 'CEQ_EJSCREEN_National_{}.csv'.format(percentile)),\n",
|
|
" encoding = \"ISO-8859-1\",\n",
|
|
" dtype='str',\n",
|
|
" )\n",
|
|
" df['EXCEED_COUNT'] = pd.to_numeric(df['EXCEED_COUNT'])\n",
|
|
"\n",
|
|
" df.rename(columns={'ID': GEOID_CBG_FIELD_NAME}, inplace=True)\n",
|
|
" df['percentile'] = percentile\n",
|
|
" df = df[[GEOID_CBG_FIELD_NAME, 'percentile', 'EXCEED_COUNT']]\n",
|
|
" dfs.append(df)\n",
|
|
" \n",
|
|
"df = pd.concat(dfs)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "823510c2",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "751edb42",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_reshaped_nat = df.pivot(index=GEOID_CBG_FIELD_NAME, columns='percentile', values='EXCEED_COUNT')\n",
|
|
"df_reshaped_nat.columns = \\\n",
|
|
" ['EJSCREEN Areas of Concern, National, {}th percentile'.format(p) for p in df_reshaped_nat.columns]\n",
|
|
"df_reshaped_nat.fillna(0, inplace=True)\n",
|
|
"\n",
|
|
"for c in df_reshaped_nat.columns:\n",
|
|
" df_reshaped_nat[c + ', (communities)'] = (df_reshaped_nat[c] > 0) * 1\n",
|
|
"df_reshaped_nat.reset_index(inplace=True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "78276a83",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_reshaped_nat.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d4cf5fd5",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_reshaped_nat.describe()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d5c8ae49",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"pd.isnull(df_reshaped_nat).describe()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "8636cad6",
|
|
"metadata": {},
|
|
"source": [
|
|
"### State"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "fcd3a812",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Replace this with something like glob when you have internet\n",
|
|
"filenames = [\n",
|
|
" 'CEQ_EJSCREEN_State_70.csv',\n",
|
|
" 'CEQ_EJSCREEN_State_75.csv',\n",
|
|
" 'CEQ_EJSCREEN_State_80.csv',\n",
|
|
" 'CEQ_EJSCREEN_State_85.csv',\n",
|
|
" 'CEQ_EJSCREEN_State_90.csv',\n",
|
|
" 'CEQ_EJSCREEN_State_95.csv',\n",
|
|
"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "a9c86190",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"dfs = []\n",
|
|
"for f in filenames:\n",
|
|
" percentile = f[-6:][:-4]\n",
|
|
" print(percentile)\n",
|
|
"\n",
|
|
" df = pd.read_csv(\n",
|
|
" os.path.join(EJSCREEN_CEQ_STA_DIR, 'CEQ_EJSCREEN_State_{}.csv'.format(percentile)),\n",
|
|
" encoding = \"ISO-8859-1\",\n",
|
|
" dtype='str',\n",
|
|
" )\n",
|
|
" df['EXCEED_COUNT'] = pd.to_numeric(df['EXCEED_COUNT'])\n",
|
|
"\n",
|
|
" df.rename(columns={'ID': GEOID_CBG_FIELD_NAME}, inplace=True)\n",
|
|
" df['percentile'] = percentile\n",
|
|
" df = df[[GEOID_CBG_FIELD_NAME, 'percentile', 'EXCEED_COUNT']]\n",
|
|
" dfs.append(df)\n",
|
|
" \n",
|
|
"df = pd.concat(dfs)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "4ddbb936",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6536820e",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_reshaped_sta = df.pivot(index=GEOID_CBG_FIELD_NAME, columns='percentile', values='EXCEED_COUNT')\n",
|
|
"df_reshaped_sta.columns = ['EJSCREEN Areas of Concern, State, {}th percentile'.format(p) for p in df_reshaped_sta.columns]\n",
|
|
"df_reshaped_sta.fillna(0, inplace=True)\n",
|
|
"\n",
|
|
"for c in df_reshaped_sta.columns:\n",
|
|
" df_reshaped_sta[c + ', (communities)'] = (df_reshaped_sta[c] > 0) * 1\n",
|
|
"df_reshaped_sta.reset_index(inplace=True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "40fd9e8c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_reshaped_sta.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "a290bfab",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_reshaped_nat.describe()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "9d141fb8",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"pd.isnull(df_reshaped_sta).describe()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "5a1538e7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_reshaped = df_reshaped_nat.merge(\n",
|
|
" df_reshaped_sta,\n",
|
|
" on=GEOID_CBG_FIELD_NAME)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "354eef0c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_reshaped.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "56098d7b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_reshaped.to_csv(\n",
|
|
" path_or_buf=EJSCREEN_DATA_DIR / \"ejscreen_areas_of_concerns_indicators.csv\", na_rep=\"\", index=False\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "c5a8db56",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Next Steps / Questions\n",
|
|
"Lucas, here's what the output file looks like. For each CBG I have new columns corresponding to the different percentiles for both State and National. For each percentile there are two columns: one for the number of `EXCEED_COUNT` and a boolean indicator for whether `EXCEED_COUNT > 0` for that percentile. I think that's what we wanted right?\n",
|
|
"\n",
|
|
"1. Do we have a list of all CBGs? The reason for asking is I created a CSV that lists each CBG and the number of EJSCREEN Areas of Concerns for each percentile. It's not going to have all the CBGs in them since if the CBG doesn't have an area concern at least at the 70th percentile, then the CBG wouldn't have appeared in the source data set. Do we want to make sure to add all the remaining CBGs with 0's across the board? \n",
|
|
"1. Definitely need to clean up the code, at least not make it so duplicatous across national and state"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.8.12"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|