diff --git a/score/ipython/census_etl.ipynb b/score/ipython/census_etl.ipynb index 1e43b94b..2adfa1c0 100644 --- a/score/ipython/census_etl.ipynb +++ b/score/ipython/census_etl.ipynb @@ -16,6 +16,7 @@ "\n", "DATA_PATH = Path.cwd().parent / \"data\"\n", "FIPS_CSV_PATH = DATA_PATH / \"fips_states_2010.csv\"\n", + "OUTPUT_PATH = DATA_PATH / \"dataset\" / f\"census_acs_{ACS_YEAR}\"\n", "\n", "GEOID_FIELD_NAME = \"GEOID10\"\n", "UNEMPLOYED_FIELD_NAME = \"Unemployed Civilians (fraction)\"\n", @@ -64,7 +65,7 @@ " line_count += 1\n", " else:\n", " fips = row[0].strip()\n", - " print(f\"Downloading data for state with FIPS code {fips}\")\n", + " print(f\"Downloading data for state/territory with FIPS code {fips}\")\n", "\n", " dfs.append(\n", " censusdata.download(\n", @@ -89,7 +90,7 @@ "execution_count": null, "id": "803cce31", "metadata": { - "scrolled": true + "scrolled": false }, "outputs": [], "source": [ @@ -105,41 +106,15 @@ "execution_count": null, "id": "2a269bb1", "metadata": { - "scrolled": false + "scrolled": true }, "outputs": [], "source": [ "# mkdir census\n", "columns_to_include = [GEOID_FIELD_NAME, UNEMPLOYED_FIELD_NAME]\n", "\n", - "df[columns_to_include].to_csv(\n", - " path_or_buf=DATA_PATH / \"tmp\" / \"census\" / \"census.csv\", index=False\n", - ")" + "df[columns_to_include].to_csv(path_or_buf=OUTPUT_PATH / \"usa.csv\", index=False)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "05b93deb", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "43784bc1", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1abc16a5", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/score/ipython/score_calc.ipynb b/score/ipython/score_calc.ipynb index f2f032c8..277c9646 100644 --- a/score/ipython/score_calc.ipynb +++ b/score/ipython/score_calc.ipynb @@ -7,7 +7,9 @@ "metadata": {}, "outputs": [], "source": [ - "# Before running this notebook, you must run the notebook `ejscreen_etl.ipynb`.\n", + "# Before running this notebook, you must run the following notebooks:\n", + "# 1. `ejscreen_etl.ipynb`\n", + "# 2. `census_etl.ipynb`\n", "\n", "import collections\n", "from pathlib import Path\n", @@ -15,6 +17,7 @@ "import csv\n", "\n", "# Define some global parameters\n", + "GEOID_FIELD_NAME = \"GEOID10\"\n", "BUCKET_SOCIOECONOMIC = \"Socioeconomic Factors\"\n", "BUCKET_SENSITIVE = \"Sensitive populations\"\n", "BUCKET_ENVIRONMENTAL = \"Environmental effects\"\n", @@ -51,7 +54,40 @@ "source": [ "# EJSCreen csv Load\n", "ejscreen_csv = data_path / \"dataset\" / \"ejscreen_2020\" / \"usa.csv\"\n", - "df = pd.read_csv(ejscreen_csv, dtype={\"ID\": \"string\"}, low_memory=False)\n", + "ejscreen_df = pd.read_csv(ejscreen_csv, dtype={\"ID\": \"string\"}, low_memory=False)\n", + "ejscreen_df.rename(columns={\"ID\": GEOID_FIELD_NAME}, inplace=True)\n", + "ejscreen_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "daba69fb", + "metadata": {}, + "outputs": [], + "source": [ + "# Load census data\n", + "census_csv = data_path / \"dataset\" / \"census_acs_2019\" / \"usa.csv\"\n", + "census_df = pd.read_csv(\n", + " census_csv, dtype={GEOID_FIELD_NAME: \"string\"}, low_memory=False\n", + ")\n", + "census_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf89efd8", + "metadata": {}, + "outputs": [], + "source": [ + "# Join the two datasets\n", + "df = ejscreen_df.merge(\n", + " census_df,\n", + " how=\"left\",\n", + " on=GEOID_FIELD_NAME,\n", + ")\n", + "\n", "df.head()" ] }, @@ -70,9 +106,10 @@ "data_sets = [\n", " # The following data sets have `bucket=None`, because it's not used in the score.\n", " DataSet(\n", - " input_field=\"ID\", \n", + " input_field=GEOID_FIELD_NAME,\n", " # Use the name `GEOID10` to enable geoplatform.gov's workflow.\n", - " renamed_field=\"GEOID10\", bucket=None\n", + " renamed_field=GEOID_FIELD_NAME,\n", + " bucket=None,\n", " ),\n", " DataSet(input_field=\"ACSTOTPOP\", renamed_field=\"Total population\", bucket=None),\n", " # The following data sets have buckets, because they're used in the score\n", @@ -152,6 +189,12 @@ " renamed_field=\"Percent individuals age 25 or over with less than high school degree\",\n", " bucket=BUCKET_SOCIOECONOMIC,\n", " ),\n", + " DataSet(\n", + " input_field=\"Unemployed Civilians (fraction)\",\n", + " # Following EJSCREEN conventions, where fractional data is named as a percent.\n", + " renamed_field=\"Unemployed Civilians (percent)\",\n", + " bucket=BUCKET_SOCIOECONOMIC,\n", + " ),\n", "]" ] }, diff --git a/score/ipython/scoring_comparison.ipynb b/score/ipython/scoring_comparison.ipynb index 6bd965c2..1b3f2683 100644 --- a/score/ipython/scoring_comparison.ipynb +++ b/score/ipython/scoring_comparison.ipynb @@ -71,10 +71,10 @@ "# Rename unclear name \"id\" to \"census_block_group_id\", as well as other renamings.\n", "cejst_df.rename(\n", " columns={\n", - " \"ID\": CENSUS_BLOCK_GROUP_ID_FIELD,\n", - " \"ACSTOTPOP\": CENSUS_BLOCK_GROUP_POPULATION_FIELD,\n", - " \"score_a\": CEJST_SCORE_FIELD,\n", - " \"score_a_percentile\": CEJST_PERCENTILE_FIELD,\n", + " \"GEOID10\": CENSUS_BLOCK_GROUP_ID_FIELD,\n", + " \"Total population\": CENSUS_BLOCK_GROUP_POPULATION_FIELD,\n", + " \"Score C\": CEJST_SCORE_FIELD,\n", + " \"Score C (percentile)\": CEJST_PERCENTILE_FIELD,\n", " },\n", " inplace=True,\n", " errors=\"raise\",\n", @@ -110,10 +110,13 @@ "# Data from https://calenviroscreen-oehha.hub.arcgis.com/#Data, specifically:\n", "# https://oehha.ca.gov/media/downloads/calenviroscreen/document/calenviroscreen40resultsdatadictionaryd12021.zip\n", "\n", - "download = requests.get(\"https://justice40-data.s3.amazonaws.com/CalEnviroScreen/CalEnviroScreen_4.0_2021.zip\", verify=False)\n", + "download = requests.get(\n", + " \"https://justice40-data.s3.amazonaws.com/CalEnviroScreen/CalEnviroScreen_4.0_2021.zip\",\n", + " verify=False,\n", + ")\n", "file_contents = download.content\n", "zip_file_path = TEMP_DATA_DIR\n", - "zip_file = open(zip_file_path / \"downloaded.zip\", \"wb\")\n", + "zip_file = open(zip_file_path / \"downloaded.zip\", \"wb\")\n", "zip_file.write(file_contents)\n", "zip_file.close()" ] @@ -298,12 +301,12 @@ "id": "0c534966", "metadata": { "variables": { - "all_100_sum": "1168", - "all_100_sum_percent": "59%", - "at_least_one_sum": "1817", - "at_least_one_sum_percent": "92%", - "cejst_cbgs_ca_only": "6987", - "ces_tracts_count": "1983" + "all_100_sum": {}, + "all_100_sum_percent": {}, + "at_least_one_sum": {}, + "at_least_one_sum_percent": {}, + "cejst_cbgs_ca_only": {}, + "ces_tracts_count": {} } }, "source": [ @@ -319,14 +322,6 @@ "\n", "Out of every CalEnviroScreen Disadvantaged Community census tract, {{all_100_sum}} ({{all_100_sum_percent}}) of these census tracts have 100% of the included census block groups within them considered priority communities by the current version of the CEJST score." ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "db3c7d38", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -345,7 +340,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.0" + "version": "3.7.1" } }, "nbformat": 4,