publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-08-20 08:31:39 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Modularization + Poetry + Docker (#213)

Browse source 
* reorg

* added configuration management; initial click cmds

* reset dirs completed

* major modularization effort

* prepping mbtiles

* first round of PR review updates

* round 2 of feedback review

* checkpoint

* habemus dockerfile 🎉

* updated dock-er-compose with long running container

* census generation works

* logging working

* updated README

* updated README

* last small update to README

* added instructions for log visualization

* census etl update for reusable fips module

* ejscreem etl updated

* further modularization

* score modularization

* tmp cleanup
This commit is contained in:
Jorge Escobar 2021-06-28 16:16:14 -04:00 • committed by GitHub
commit 67c73dde2a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 2383 additions and 433 deletions
Download patch file Download diff file Expand all files Collapse all files

40
score/ipython/census_etl.ipynb
View file

@ -12,11 +12,17 @@
"import csv\n",
"from pathlib import Path\n",
"import os\n",
"import sys\n",
"\n",
"module_path = os.path.abspath(os.path.join('..'))\n",
"if module_path not in sys.path:\n",
" sys.path.append(module_path)\n",
"\n",
"from etl.sources.census.etl_utils import get_state_fips_codes\n",
"\n",
"ACS_YEAR = 2019\n",
"\n",
"DATA_PATH = Path.cwd().parent / \"data\"\n",
"FIPS_CSV_PATH = DATA_PATH / \"fips_states_2010.csv\"\n",
"OUTPUT_PATH = DATA_PATH / \"dataset\" / f\"census_acs_{ACS_YEAR}\"\n",
"\n",
"GEOID_FIELD_NAME = \"GEOID10\"\n",
@ -57,27 +63,19 @@
"\n",
"\n",
"dfs = []\n",
"with open(FIPS_CSV_PATH) as csv_file:\n",
" csv_reader = csv.reader(csv_file, delimiter=\",\")\n",
" line_count = 0\n",
"for fips in get_state_fips_codes(DATA_PATH):\n",
" print(f\"Downloading data for state/territory with FIPS code {fips}\")\n",
"\n",
" for row in csv_reader:\n",
" if line_count == 0:\n",
" line_count += 1\n",
" else:\n",
" fips = row[0].strip()\n",
" print(f\"Downloading data for state/territory with FIPS code {fips}\")\n",
"\n",
" dfs.append(\n",
" censusdata.download(\n",
" src=\"acs5\",\n",
" year=ACS_YEAR,\n",
" geo=censusdata.censusgeo(\n",
" [(\"state\", fips), (\"county\", \"*\"), (\"block group\", \"*\")]\n",
" ),\n",
" var=[\"B23025_005E\", \"B23025_003E\"],\n",
" )\n",
" )\n",
" dfs.append(\n",
" censusdata.download(\n",
" src=\"acs5\",\n",
" year=ACS_YEAR,\n",
" geo=censusdata.censusgeo(\n",
" [(\"state\", fips), (\"county\", \"*\"), (\"block group\", \"*\")]\n",
" ),\n",
" var=[\"B23025_005E\", \"B23025_003E\"],\n",
" )\n",
" )\n",
"\n",
"df = pd.concat(dfs)\n",
"\n",