Ingest census data directly, add unemployment to the score (#214)

* Ingest two data sources and add to score

Co-authored-by: Jorge Escobar <jorge.e.escobar@omb.eop.gov>

score/ipython/score_calc.ipynb

@@ -7,14 +7,19 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Before running this notebook, you must run the notebook `ejscreen_etl.ipynb`.\n",
+    "# Before running this notebook, you must run the following notebooks (in any order):\n",
+    "# 1. `ejscreen_etl.ipynb`\n",
+    "# 2. `census_etl.ipynb`\n",
+    "# 3. `housing_and_transportation_etl.ipynb`\n",
     "\n",
     "import collections\n",
+    "import functools\n",
     "from pathlib import Path\n",
     "import pandas as pd\n",
     "import csv\n",
     "\n",
     "# Define some global parameters\n",
+    "GEOID_FIELD_NAME = \"GEOID10\"\n",
     "BUCKET_SOCIOECONOMIC = \"Socioeconomic Factors\"\n",
     "BUCKET_SENSITIVE = \"Sensitive populations\"\n",
     "BUCKET_ENVIRONMENTAL = \"Environmental effects\"\n",
@@ -51,7 +56,62 @@
    "source": [
     "# EJSCreen csv Load\n",
     "ejscreen_csv = data_path / \"dataset\" / \"ejscreen_2020\" / \"usa.csv\"\n",
-    "df = pd.read_csv(ejscreen_csv, dtype={\"ID\": \"string\"}, low_memory=False)\n",
+    "ejscreen_df = pd.read_csv(ejscreen_csv, dtype={\"ID\": \"string\"}, low_memory=False)\n",
+    "ejscreen_df.rename(columns={\"ID\": GEOID_FIELD_NAME}, inplace=True)\n",
+    "ejscreen_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "daba69fb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load census data\n",
+    "census_csv = data_path / \"dataset\" / \"census_acs_2019\" / \"usa.csv\"\n",
+    "census_df = pd.read_csv(\n",
+    "    census_csv, dtype={GEOID_FIELD_NAME: \"string\"}, low_memory=False\n",
+    ")\n",
+    "census_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "144bdde2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load housing and transportation data\n",
+    "housing_and_transportation_index_csv = (\n",
+    "    data_path / \"dataset\" / \"housing_and_transportation_index\" / \"usa.csv\"\n",
+    ")\n",
+    "housing_and_transportation_df = pd.read_csv(\n",
+    "    housing_and_transportation_index_csv,\n",
+    "    dtype={GEOID_FIELD_NAME: \"string\"},\n",
+    "    low_memory=False,\n",
+    ")\n",
+    "housing_and_transportation_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bf89efd8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Join all the data sources that use census block groups\n",
+    "dfs = [ejscreen_df, census_df, housing_and_transportation_df]\n",
+    "\n",
+    "df = functools.reduce(\n",
+    "    lambda left, right: pd.merge(\n",
+    "        left=left, right=right, on=GEOID_FIELD_NAME, how=\"outer\"\n",
+    "    ),\n",
+    "    dfs,\n",
+    ")\n",
+    "\n",
     "df.head()"
    ]
   },
@@ -70,9 +130,10 @@
     "data_sets = [\n",
     "    # The following data sets have `bucket=None`, because it's not used in the score.\n",
     "    DataSet(\n",
-    "        input_field=\"ID\", \n",
+    "        input_field=GEOID_FIELD_NAME,\n",
     "        # Use the name `GEOID10` to enable geoplatform.gov's workflow.\n",
-    "        renamed_field=\"GEOID10\", bucket=None\n",
+    "        renamed_field=GEOID_FIELD_NAME,\n",
+    "        bucket=None,\n",
     "    ),\n",
     "    DataSet(input_field=\"ACSTOTPOP\", renamed_field=\"Total population\", bucket=None),\n",
     "    # The following data sets have buckets, because they're used in the score\n",
@@ -152,6 +213,17 @@
     "        renamed_field=\"Percent individuals age 25 or over with less than high school degree\",\n",
     "        bucket=BUCKET_SOCIOECONOMIC,\n",
     "    ),\n",
+    "    DataSet(\n",
+    "        input_field=\"Unemployed Civilians (fraction)\",\n",
+    "        # Following EJSCREEN conventions, where fractional data is named as a percent.\n",
+    "        renamed_field=\"Unemployed Civilians (percent)\",\n",
+    "        bucket=BUCKET_SOCIOECONOMIC,\n",
+    "    ),\n",
+    "    DataSet(\n",
+    "        input_field=\"ht_ami\",\n",
+    "        renamed_field=\"Housing + Transportation Costs % Income for the Regional Typical Household\",\n",
+    "        bucket=BUCKET_SOCIOECONOMIC,\n",
+    "    ),\n",
     "]"
    ]
   },
@@ -305,6 +377,14 @@
     "            # we need to name the file data01.csv for ogr2ogr csv merge to work\n",
     "            df1.to_csv(score_csv_path / f\"data{states_fips}.csv\", index=False)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "167ebba3",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -323,7 +403,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.1"
+   "version": "3.9.0"
   }
  },
  "nbformat": 4,