score comparison updated

score/ipython/census_etl.ipynb

@@ -16,6 +16,7 @@
     "\n",
     "DATA_PATH = Path.cwd().parent / \"data\"\n",
     "FIPS_CSV_PATH = DATA_PATH / \"fips_states_2010.csv\"\n",
+    "OUTPUT_PATH = DATA_PATH / \"dataset\" / f\"census_acs_{ACS_YEAR}\"\n",
     "\n",
     "GEOID_FIELD_NAME = \"GEOID10\"\n",
     "UNEMPLOYED_FIELD_NAME = \"Unemployed Civilians (fraction)\"\n",
@@ -64,7 +65,7 @@
     "            line_count += 1\n",
     "        else:\n",
     "            fips = row[0].strip()\n",
-    "            print(f\"Downloading data for state with FIPS code {fips}\")\n",
+    "            print(f\"Downloading data for state/territory with FIPS code {fips}\")\n",
     "\n",
     "            dfs.append(\n",
     "                censusdata.download(\n",
@@ -89,7 +90,7 @@
    "execution_count": null,
    "id": "803cce31",
    "metadata": {
-    "scrolled": true
+    "scrolled": false
    },
    "outputs": [],
    "source": [
@@ -105,41 +106,15 @@
    "execution_count": null,
    "id": "2a269bb1",
    "metadata": {
-    "scrolled": false
+    "scrolled": true
    },
    "outputs": [],
    "source": [
     "# mkdir census\n",
     "columns_to_include = [GEOID_FIELD_NAME, UNEMPLOYED_FIELD_NAME]\n",
     "\n",
-    "df[columns_to_include].to_csv(\n",
-    "    path_or_buf=DATA_PATH / \"tmp\" / \"census\" / \"census.csv\", index=False\n",
-    ")"
+    "df[columns_to_include].to_csv(path_or_buf=OUTPUT_PATH / \"usa.csv\", index=False)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "05b93deb",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "43784bc1",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "1abc16a5",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

score/ipython/score_calc.ipynb

@@ -7,7 +7,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Before running this notebook, you must run the notebook `ejscreen_etl.ipynb`.\n",
+    "# Before running this notebook, you must run the following notebooks:\n",
+    "# 1. `ejscreen_etl.ipynb`\n",
+    "# 2. `census_etl.ipynb`\n",
     "\n",
     "import collections\n",
     "from pathlib import Path\n",
@@ -15,6 +17,7 @@
     "import csv\n",
     "\n",
     "# Define some global parameters\n",
+    "GEOID_FIELD_NAME = \"GEOID10\"\n",
     "BUCKET_SOCIOECONOMIC = \"Socioeconomic Factors\"\n",
     "BUCKET_SENSITIVE = \"Sensitive populations\"\n",
     "BUCKET_ENVIRONMENTAL = \"Environmental effects\"\n",
@@ -51,7 +54,40 @@
    "source": [
     "# EJSCreen csv Load\n",
     "ejscreen_csv = data_path / \"dataset\" / \"ejscreen_2020\" / \"usa.csv\"\n",
-    "df = pd.read_csv(ejscreen_csv, dtype={\"ID\": \"string\"}, low_memory=False)\n",
+    "ejscreen_df = pd.read_csv(ejscreen_csv, dtype={\"ID\": \"string\"}, low_memory=False)\n",
+    "ejscreen_df.rename(columns={\"ID\": GEOID_FIELD_NAME}, inplace=True)\n",
+    "ejscreen_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "daba69fb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load census data\n",
+    "census_csv = data_path / \"dataset\" / \"census_acs_2019\" / \"usa.csv\"\n",
+    "census_df = pd.read_csv(\n",
+    "    census_csv, dtype={GEOID_FIELD_NAME: \"string\"}, low_memory=False\n",
+    ")\n",
+    "census_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bf89efd8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Join the two datasets\n",
+    "df = ejscreen_df.merge(\n",
+    "    census_df,\n",
+    "    how=\"left\",\n",
+    "    on=GEOID_FIELD_NAME,\n",
+    ")\n",
+    "\n",
     "df.head()"
    ]
   },
@@ -70,9 +106,10 @@
     "data_sets = [\n",
     "    # The following data sets have `bucket=None`, because it's not used in the score.\n",
     "    DataSet(\n",
-    "        input_field=\"ID\", \n",
+    "        input_field=GEOID_FIELD_NAME,\n",
     "        # Use the name `GEOID10` to enable geoplatform.gov's workflow.\n",
-    "        renamed_field=\"GEOID10\", bucket=None\n",
+    "        renamed_field=GEOID_FIELD_NAME,\n",
+    "        bucket=None,\n",
     "    ),\n",
     "    DataSet(input_field=\"ACSTOTPOP\", renamed_field=\"Total population\", bucket=None),\n",
     "    # The following data sets have buckets, because they're used in the score\n",
@@ -152,6 +189,12 @@
     "        renamed_field=\"Percent individuals age 25 or over with less than high school degree\",\n",
     "        bucket=BUCKET_SOCIOECONOMIC,\n",
     "    ),\n",
+    "    DataSet(\n",
+    "        input_field=\"Unemployed Civilians (fraction)\",\n",
+    "        # Following EJSCREEN conventions, where fractional data is named as a percent.\n",
+    "        renamed_field=\"Unemployed Civilians (percent)\",\n",
+    "        bucket=BUCKET_SOCIOECONOMIC,\n",
+    "    ),\n",
     "]"
    ]
   },

score/ipython/scoring_comparison.ipynb

@@ -71,10 +71,10 @@
     "# Rename unclear name \"id\" to \"census_block_group_id\", as well as other renamings.\n",
     "cejst_df.rename(\n",
     "    columns={\n",
-    "        \"ID\": CENSUS_BLOCK_GROUP_ID_FIELD,\n",
-    "        \"ACSTOTPOP\": CENSUS_BLOCK_GROUP_POPULATION_FIELD,\n",
-    "        \"score_a\": CEJST_SCORE_FIELD,\n",
-    "        \"score_a_percentile\": CEJST_PERCENTILE_FIELD,\n",
+    "        \"GEOID10\": CENSUS_BLOCK_GROUP_ID_FIELD,\n",
+    "        \"Total population\": CENSUS_BLOCK_GROUP_POPULATION_FIELD,\n",
+    "        \"Score C\": CEJST_SCORE_FIELD,\n",
+    "        \"Score C (percentile)\": CEJST_PERCENTILE_FIELD,\n",
     "    },\n",
     "    inplace=True,\n",
     "    errors=\"raise\",\n",
@@ -110,10 +110,13 @@
     "# Data from https://calenviroscreen-oehha.hub.arcgis.com/#Data, specifically:\n",
     "# https://oehha.ca.gov/media/downloads/calenviroscreen/document/calenviroscreen40resultsdatadictionaryd12021.zip\n",
     "\n",
-    "download = requests.get(\"https://justice40-data.s3.amazonaws.com/CalEnviroScreen/CalEnviroScreen_4.0_2021.zip\", verify=False)\n",
+    "download = requests.get(\n",
+    "    \"https://justice40-data.s3.amazonaws.com/CalEnviroScreen/CalEnviroScreen_4.0_2021.zip\",\n",
+    "    verify=False,\n",
+    ")\n",
     "file_contents = download.content\n",
     "zip_file_path = TEMP_DATA_DIR\n",
-    "zip_file = open(zip_file_path  / \"downloaded.zip\", \"wb\")\n",
+    "zip_file = open(zip_file_path / \"downloaded.zip\", \"wb\")\n",
     "zip_file.write(file_contents)\n",
     "zip_file.close()"
    ]
@@ -298,12 +301,12 @@
    "id": "0c534966",
    "metadata": {
     "variables": {
-     "all_100_sum": "1168",
-     "all_100_sum_percent": "59%",
-     "at_least_one_sum": "1817",
-     "at_least_one_sum_percent": "92%",
-     "cejst_cbgs_ca_only": "6987",
-     "ces_tracts_count": "1983"
+     "all_100_sum": {},
+     "all_100_sum_percent": {},
+     "at_least_one_sum": {},
+     "at_least_one_sum_percent": {},
+     "cejst_cbgs_ca_only": {},
+     "ces_tracts_count": {}
     }
    },
    "source": [
@@ -319,14 +322,6 @@
     "\n",
     "Out of every CalEnviroScreen Disadvantaged Community census tract, {{all_100_sum}} ({{all_100_sum_percent}}) of these census tracts have 100% of the included census block groups within them considered priority communities by the current version of the CEJST score."
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "db3c7d38",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
@@ -345,7 +340,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.0"
+   "version": "3.7.1"
   }
  },
  "nbformat": 4,