mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-02-23 18:14:19 -08:00
* starting etl for score * projection fix * projection flags * proper ejscreen etl csv generation * failing CSV merge -- investigating * checkpoint * some etl changes * completed ticket * small typo
202 lines
5.4 KiB
Text
202 lines
5.4 KiB
Text
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"id": "20aa3891",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from pathlib import Path\n",
|
|
"import requests\n",
|
|
"import zipfile\n",
|
|
"import numpy as np\n",
|
|
"import pandas as pd\n",
|
|
"import csv\n",
|
|
"\n",
|
|
"data_path = Path.cwd().parent / \"data\"\n",
|
|
"fips_csv_path = data_path / \"fips_states_2010.csv\"\n",
|
|
"csv_path = data_path / \"dataset\" / \"ejscreen_2020\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"id": "67a58c24",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"c:\\opt\\justice40-tool\\score\\venv\\lib\\site-packages\\urllib3\\connectionpool.py:1013: InsecureRequestWarning: Unverified HTTPS request is being made to host 'gaftp.epa.gov'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
|
|
" warnings.warn(\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"download = requests.get(\"https://gaftp.epa.gov/EJSCREEN/2020/EJSCREEN_2020_StatePctile.csv.zip\", verify=False)\n",
|
|
"file_contents = download.content\n",
|
|
"zip_file_path = data_path / \"tmp\"\n",
|
|
"zip_file = open(zip_file_path / \"downloaded.zip\", \"wb\")\n",
|
|
"zip_file.write(file_contents)\n",
|
|
"zip_file.close()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"id": "cc3fb9ec",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"with zipfile.ZipFile(zip_file_path / \"downloaded.zip\", \"r\") as zip_ref:\n",
|
|
" zip_ref.extractall(zip_file_path)\n",
|
|
"ejscreen_csv = data_path / \"tmp\" / \"EJSCREEN_2020_StatePctile.csv\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"id": "b25738bb",
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"df = pd.read_csv(ejscreen_csv, dtype={'ID': 'string'}, low_memory=False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"id": "e6994f2d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df = df[[\"ID\", \"ACSTOTPOP\", \"LESSHSPCT\", \"LOWINCPCT\"]]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"id": "9fa2077a",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# write nationwide csv\n",
|
|
"df.to_csv(csv_path / f\"usa.csv\", index = False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"id": "5e5cc12a",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Generating data01 csv\n",
|
|
"Generating data02 csv\n",
|
|
"Generating data04 csv\n",
|
|
"Generating data05 csv\n",
|
|
"Generating data06 csv\n",
|
|
"Generating data08 csv\n",
|
|
"Generating data09 csv\n",
|
|
"Generating data10 csv\n",
|
|
"Generating data11 csv\n",
|
|
"Generating data12 csv\n",
|
|
"Generating data13 csv\n",
|
|
"Generating data15 csv\n",
|
|
"Generating data16 csv\n",
|
|
"Generating data17 csv\n",
|
|
"Generating data18 csv\n",
|
|
"Generating data19 csv\n",
|
|
"Generating data20 csv\n",
|
|
"Generating data21 csv\n",
|
|
"Generating data22 csv\n",
|
|
"Generating data23 csv\n",
|
|
"Generating data24 csv\n",
|
|
"Generating data25 csv\n",
|
|
"Generating data26 csv\n",
|
|
"Generating data27 csv\n",
|
|
"Generating data28 csv\n",
|
|
"Generating data29 csv\n",
|
|
"Generating data30 csv\n",
|
|
"Generating data31 csv\n",
|
|
"Generating data32 csv\n",
|
|
"Generating data33 csv\n",
|
|
"Generating data34 csv\n",
|
|
"Generating data35 csv\n",
|
|
"Generating data36 csv\n",
|
|
"Generating data37 csv\n",
|
|
"Generating data38 csv\n",
|
|
"Generating data39 csv\n",
|
|
"Generating data40 csv\n",
|
|
"Generating data41 csv\n",
|
|
"Generating data42 csv\n",
|
|
"Generating data44 csv\n",
|
|
"Generating data45 csv\n",
|
|
"Generating data46 csv\n",
|
|
"Generating data47 csv\n",
|
|
"Generating data48 csv\n",
|
|
"Generating data49 csv\n",
|
|
"Generating data50 csv\n",
|
|
"Generating data51 csv\n",
|
|
"Generating data53 csv\n",
|
|
"Generating data54 csv\n",
|
|
"Generating data55 csv\n",
|
|
"Generating data56 csv\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# write per state csvs\n",
|
|
"with open(fips_csv_path) as csv_file:\n",
|
|
" csv_reader = csv.reader(csv_file, delimiter=\",\")\n",
|
|
" line_count = 0\n",
|
|
"\n",
|
|
" for row in csv_reader:\n",
|
|
" if line_count == 0:\n",
|
|
" line_count += 1\n",
|
|
" else:\n",
|
|
" fips = row[0].strip()\n",
|
|
" print(f\"Generating data{fips} csv\")\n",
|
|
" df1 = df[df.ID.str[:2] == fips]\n",
|
|
" # we need to name the file data01.csv for ogr2ogr csv merge to work\n",
|
|
" df1.to_csv(csv_path / f\"data{fips}.csv\", index = False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2674fb20",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.0"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|