publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-08-20 08:31:39 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Scores D & E (#266)

Browse source 
* running black throughout

* adding housing

* hud housing etl working

* got score d and e working

* updating scoring comparison

* minor fixes

* small changes

* small comments
This commit is contained in:
Lucas Merrill Brown 2021-06-29 08:20:23 -07:00 • committed by GitHub
commit 41e394972c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 625 additions and 74 deletions
Download patch file Download diff file Expand all files Collapse all files

83
score/ipython/census_etl.ipynb
View file

@ -14,7 +14,7 @@
"import os\n",
"import sys\n",
"\n",
"module_path = os.path.abspath(os.path.join('..'))\n",
"module_path = os.path.abspath(os.path.join(\"..\"))\n",
"if module_path not in sys.path:\n",
" sys.path.append(module_path)\n",
"\n",
@ -26,13 +26,36 @@
"OUTPUT_PATH = DATA_PATH / \"dataset\" / f\"census_acs_{ACS_YEAR}\"\n",
"\n",
"GEOID_FIELD_NAME = \"GEOID10\"\n",
"UNEMPLOYED_FIELD_NAME = \"Unemployed Civilians (fraction)\"\n",
"UNEMPLOYED_FIELD_NAME = \"Unemployed civilians (percent)\"\n",
"LINGUISTIC_ISOLATION_FIELD_NAME = \"Linguistic isolation (percent)\"\n",
"LINGUISTIC_ISOLATION_TOTAL_FIELD_NAME = \"Linguistic isolation (total)\"\n",
"\n",
"LINGUISTIC_ISOLATION_FIELDS = [\n",
" \"C16002_001E\",\n",
" \"C16002_004E\",\n",
" \"C16002_007E\",\n",
" \"C16002_010E\",\n",
" \"C16002_013E\",\n",
"]\n",
"\n",
"# Some display settings to make pandas outputs more readable.\n",
"pd.set_option(\"display.expand_frame_repr\", False)\n",
"pd.set_option(\"display.precision\", 2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "64df0b63",
"metadata": {},
"outputs": [],
"source": [
"# For variable discovery, if necessary.\n",
"# censusdata.search(\n",
"# \"acs5\", 2019, \"label\", \"Limited English speaking\"\n",
"# )"
]
},
{
"cell_type": "code",
"execution_count": null,
@ -44,7 +67,8 @@
"source": [
"# Following the tutorial at https://jtleider.github.io/censusdata/example1.html.\n",
"# Full list of fields is at https://www2.census.gov/programs-surveys/acs/summary_file/2019/documentation/user_tools/ACS2019_Table_Shells.xlsx\n",
"censusdata.printtable(censusdata.censustable(src=\"acs5\", year=ACS_YEAR, table=\"B23025\"))"
"censusdata.printtable(censusdata.censustable(src=\"acs5\", year=ACS_YEAR, table=\"B23025\"))\n",
"censusdata.printtable(censusdata.censustable(src=\"acs5\", year=ACS_YEAR, table=\"C16002\"))"
]
},
{
@ -73,10 +97,16 @@
" geo=censusdata.censusgeo(\n",
" [(\"state\", fips), (\"county\", \"*\"), (\"block group\", \"*\")]\n",
" ),\n",
" var=[\"B23025_005E\", \"B23025_003E\"],\n",
" var=[\n",
" # Emploment fields\n",
" \"B23025_005E\",\n",
" \"B23025_003E\",\n",
" ]\n",
" + LINGUISTIC_ISOLATION_FIELDS,\n",
" )\n",
" )\n",
"\n",
"\n",
"df = pd.concat(dfs)\n",
"\n",
"df[GEOID_FIELD_NAME] = df.index.to_series().apply(func=fips_from_censusdata_censusgeo)\n",
@ -97,7 +127,34 @@
"# TODO: remove small-sample data that should be `None` instead of a high-variance fraction.\n",
"df[UNEMPLOYED_FIELD_NAME] = df.B23025_005E / df.B23025_003E\n",
"\n",
"df.head()"
"df[UNEMPLOYED_FIELD_NAME].describe()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e475472c",
"metadata": {
"scrolled": false
},
"outputs": [],
"source": [
"# Calculate linguistic isolation.\n",
"individual_limited_english_fields = [\n",
" \"C16002_004E\",\n",
" \"C16002_007E\",\n",
" \"C16002_010E\",\n",
" \"C16002_013E\",\n",
"]\n",
"\n",
"df[LINGUISTIC_ISOLATION_TOTAL_FIELD_NAME] = df[individual_limited_english_fields].sum(\n",
" axis=1, skipna=True\n",
")\n",
"df[LINGUISTIC_ISOLATION_FIELD_NAME] = (\n",
" df[LINGUISTIC_ISOLATION_TOTAL_FIELD_NAME].astype(float) / df[\"C16002_001E\"]\n",
")\n",
"\n",
"df[LINGUISTIC_ISOLATION_FIELD_NAME].describe()"
]
},
{
@ -112,18 +169,14 @@
"# mkdir census\n",
"OUTPUT_PATH.mkdir(parents=True, exist_ok=True)\n",
"\n",
"columns_to_include = [GEOID_FIELD_NAME, UNEMPLOYED_FIELD_NAME]\n",
"columns_to_include = [\n",
" GEOID_FIELD_NAME,\n",
" UNEMPLOYED_FIELD_NAME,\n",
" LINGUISTIC_ISOLATION_FIELD_NAME,\n",
"]\n",
"\n",
"df[columns_to_include].to_csv(path_or_buf=OUTPUT_PATH / \"usa.csv\", index=False)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "91932af5",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
@ -142,7 +195,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.0"
"version": "3.7.1"
}
},
"nbformat": 4,