mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-02-23 10:04:18 -08:00
65ceb7900f
* fixing dependency issue * fixing more dependencies * including fraction of state AMI * wip * nitpick whitespace * etl working now * wip on scoring * fix rename error * reducing metrics * fixing score f * fixing readme * adding dependency * passing tests; * linting/black * removing unnecessary sample * fixing error * adding verify flag on etl/base Co-authored-by: Jorge Escobar <jorge.e.escobar@omb.eop.gov>
131 lines
No EOL
3.8 KiB
Text
131 lines
No EOL
3.8 KiB
Text
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"import censusdata\n",
|
|
"import csv\n",
|
|
"from pathlib import Path\n",
|
|
"import os\n",
|
|
"import sys\n",
|
|
"\n",
|
|
"module_path = os.path.abspath(os.path.join(\"../..\"))\n",
|
|
"if module_path not in sys.path:\n",
|
|
" sys.path.append(module_path)\n",
|
|
"\n",
|
|
"from data_pipeline.etl.sources.census.etl_utils import get_state_fips_codes\n",
|
|
"\n",
|
|
"\n",
|
|
"ACS_YEAR = 2019\n",
|
|
"\n",
|
|
"DATA_PATH = Path.cwd().parent / \"data\"\n",
|
|
"FIPS_CSV_PATH = DATA_PATH / \"fips_states_2010.csv\"\n",
|
|
"\n",
|
|
"GEOID_FIELD_NAME = \"GEOID10\"\n",
|
|
"UNEMPLOYED_FIELD_NAME = \"Unemployed Civilians (fraction)\"\n",
|
|
"\n",
|
|
"# Some display settings to make pandas outputs more readable.\n",
|
|
"pd.set_option(\"display.expand_frame_repr\", False)\n",
|
|
"pd.set_option(\"display.precision\", 2)"
|
|
],
|
|
"outputs": [],
|
|
"metadata": {}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"source": [
|
|
"# Following the tutorial at https://jtleider.github.io/censusdata/example1.html.\n",
|
|
"# Full list of fields is at https://www2.census.gov/programs-surveys/acs/summary_file/2019/documentation/user_tools/ACS2019_Table_Shells.xlsx\n",
|
|
"censusdata.printtable(censusdata.censustable(src=\"acs5\", year=ACS_YEAR, table=\"B19013\"))"
|
|
],
|
|
"outputs": [],
|
|
"metadata": {
|
|
"scrolled": true
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"source": [
|
|
"def fips_from_censusdata_censusgeo(censusgeo: censusdata.censusgeo) -> str:\n",
|
|
" \"\"\"Create a FIPS code from the proprietary censusgeo index.\"\"\"\n",
|
|
" fips = \"\".join([value for (key, value) in censusgeo.params()])\n",
|
|
" return fips\n",
|
|
"\n",
|
|
"\n",
|
|
"dfs = []\n",
|
|
"for fips in get_state_fips_codes(DATA_PATH):\n",
|
|
" print(f\"Fetching data for fips {fips}\")\n",
|
|
" dfs.append(\n",
|
|
" censusdata.download(\n",
|
|
" src=\"acs5\",\n",
|
|
" year=ACS_YEAR,\n",
|
|
" geo=censusdata.censusgeo(\n",
|
|
" [\n",
|
|
" (\"state\", fips) \n",
|
|
" #, (\"county\", \"*\"), (\"block group\", \"*\")\n",
|
|
" ]\n",
|
|
" ),\n",
|
|
" var=[\"B23025_005E\", \"B23025_003E\", \"B19013_001E\"],\n",
|
|
" )\n",
|
|
" )\n",
|
|
"\n",
|
|
"df = pd.concat(dfs)\n",
|
|
"\n",
|
|
"df[GEOID_FIELD_NAME] = df.index.to_series().apply(func=fips_from_censusdata_censusgeo)\n",
|
|
"\n",
|
|
"df.head()"
|
|
],
|
|
"outputs": [],
|
|
"metadata": {
|
|
"scrolled": true
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"source": [
|
|
"columns_to_include = [\"GEOID2\", \"Median household income (State)\"]\n",
|
|
"\n",
|
|
"df.rename(columns={\"GEOID10\": \"GEOID2\", \"B19013_001E\": \"Median household income (State)\"}, inplace=True)\n",
|
|
"\n",
|
|
"# df[columns_to_include].to_csv(path_or_buf= \"/Users/lucas/Documents/usds/repos/justice40-tool/data/data-pipeline/data_pipeline/data/needs_to_be_moved_to_s3/2014_to_2019_state_median_income.csv\", index=False)"
|
|
],
|
|
"outputs": [],
|
|
"metadata": {
|
|
"scrolled": true
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"source": [],
|
|
"outputs": [],
|
|
"metadata": {}
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.6"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
} |