publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-07-25 20:00:17 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

updating pylint

Browse source
This commit is contained in:
lucasmbrown-usds 2022-09-28 13:35:52 -04:00
commit 5ff988ab29
25 changed files with 154 additions and 101 deletions
Download patch file Download diff file Expand all files Collapse all files

24
data/data-pipeline/data_pipeline/ipython/agricultural_loss_indicator.ipynb
View file

@ -211,7 +211,9 @@
}
],
"source": [
"tmp = sns.FacetGrid(data=score_m, col=\"Urban Heuristic Flag\", col_wrap=2, height=7)\n",
"tmp = sns.FacetGrid(\n",
" data=score_m, col=\"Urban Heuristic Flag\", col_wrap=2, height=7\n",
")\n",
"tmp.map(\n",
" sns.distplot,\n",
" \"Expected agricultural loss rate (Natural Hazards Risk Index) (percentile)\",\n",
@ -250,7 +252,9 @@
")\n",
"\n",
"nri_with_flag[\"total_ag_loss\"] = nri_with_flag.filter(like=\"EALA\").sum(axis=1)\n",
"nri_with_flag[\"total_ag_loss_pctile\"] = nri_with_flag[\"total_ag_loss\"].rank(pct=True)\n",
"nri_with_flag[\"total_ag_loss_pctile\"] = nri_with_flag[\"total_ag_loss\"].rank(\n",
" pct=True\n",
")\n",
"\n",
"nri_with_flag.groupby(\"Urban Heuristic Flag\")[\"total_ag_loss_pctile\"].mean()"
]
@ -779,9 +783,9 @@
" \"Greater than or equal to the 90th percentile for expected agriculture loss rate, is low income, and has a low percent of higher ed students?\"\n",
"].astype(int)\n",
"\n",
"score_m_adjusted_tracts = set(score_m[score_m[\"adjusted\"] > 0][\"GEOID10_TRACT\"]).union(\n",
" all_ag_loss_tracts\n",
")\n",
"score_m_adjusted_tracts = set(\n",
" score_m[score_m[\"adjusted\"] > 0][\"GEOID10_TRACT\"]\n",
").union(all_ag_loss_tracts)\n",
"display(len(set(all_scorem_tracts).difference(score_m_adjusted_tracts)))"
]
},
@ -832,7 +836,11 @@
" left_clip = nri_with_flag[nri_with_flag[\"Urban Heuristic Flag\"] == 0][\n",
" \"AGRIVALUE\"\n",
" ].quantile(threshold)\n",
" print(\"At threshold {:.2f}, minimum value is ${:,.0f}\".format(threshold, left_clip))\n",
" print(\n",
" \"At threshold {:.2f}, minimum value is ${:,.0f}\".format(\n",
" threshold, left_clip\n",
" )\n",
" )\n",
" tmp_value = nri_with_flag[\"AGRIVALUE\"].clip(lower=left_clip)\n",
" nri_with_flag[\"total_ag_loss_pctile_{:.2f}\".format(threshold)] = (\n",
" nri_with_flag[\"total_ag_loss\"] / tmp_value\n",
@ -889,7 +897,9 @@
" .set_index(\"Left clip value\")[[\"Rural\", \"Urban\"]]\n",
" .stack()\n",
" .reset_index()\n",
" .rename(columns={\"level_1\": \"Tract classification\", 0: \"Average percentile\"})\n",
" .rename(\n",
" columns={\"level_1\": \"Tract classification\", 0: \"Average percentile\"}\n",
" )\n",
")"
]
},

13
data/data-pipeline/data_pipeline/ipython/check_tribal_count.ipynb
View file

@ -21,6 +21,7 @@
"source": [
"import os\n",
"import sys\n",
"\n",
"module_path = os.path.abspath(os.path.join(\"../..\"))\n",
"if module_path not in sys.path:\n",
" sys.path.append(module_path)"
@ -94,9 +95,13 @@
"bia_aian_supplemental_geojson = (\n",
" GEOJSON_BASE_PATH / \"bia_national_lar\" / \"BIA_AIAN_Supplemental.json\"\n",
")\n",
"bia_tsa_geojson_geojson = GEOJSON_BASE_PATH / \"bia_national_lar\" / \"BIA_TSA.json\"\n",
"bia_tsa_geojson_geojson = (\n",
" GEOJSON_BASE_PATH / \"bia_national_lar\" / \"BIA_TSA.json\"\n",
")\n",
"alaska_native_villages_geojson = (\n",
" GEOJSON_BASE_PATH / \"alaska_native_villages\" / \"AlaskaNativeVillages.gdb.geojson\"\n",
" GEOJSON_BASE_PATH\n",
" / \"alaska_native_villages\"\n",
" / \"AlaskaNativeVillages.gdb.geojson\"\n",
")"
]
},
@ -131,7 +136,9 @@
"len(\n",
" sorted(\n",
" list(\n",
" bia_national_lar_df.LARName.str.replace(r\"\\(.*\\) \", \"\", regex=True).unique()\n",
" bia_national_lar_df.LARName.str.replace(\n",
" r\"\\(.*\\) \", \"\", regex=True\n",
" ).unique()\n",
" )\n",
" )\n",
")"

15
data/data-pipeline/data_pipeline/ipython/compare_tiles_and_geoJson_files.ipynb
View file

@ -45,6 +45,7 @@
"source": [
"# Read in the score geojson file\n",
"from data_pipeline.etl.score.constants import DATA_SCORE_CSV_TILES_FILE_PATH\n",
"\n",
"nation = gpd.read_file(DATA_SCORE_CSV_TILES_FILE_PATH)"
]
},
@ -93,10 +94,14 @@
" random_tile_features = json.loads(f.read())\n",
"\n",
"# Flatten data around the features key:\n",
"flatten_features = pd.json_normalize(random_tile_features, record_path=[\"features\"])\n",
"flatten_features = pd.json_normalize(\n",
" random_tile_features, record_path=[\"features\"]\n",
")\n",
"\n",
"# index into the feature properties, get keys and turn into a sorted list\n",
"random_tile = sorted(list(flatten_features[\"features\"][0][0][\"properties\"].keys()))"
"random_tile = sorted(\n",
" list(flatten_features[\"features\"][0][0][\"properties\"].keys())\n",
")"
]
},
{
@ -291,8 +296,8 @@
}
],
"source": [
"nation_HRS_GEO = nation[['GEOID10', 'SF', 'CF', 'HRS_ET', 'AML_ET', 'FUDS_ET']]\n",
"nation_HRS_GEO.loc[nation_HRS_GEO['FUDS_ET'] == '0']"
"nation_HRS_GEO = nation[[\"GEOID10\", \"SF\", \"CF\", \"HRS_ET\", \"AML_ET\", \"FUDS_ET\"]]\n",
"nation_HRS_GEO.loc[nation_HRS_GEO[\"FUDS_ET\"] == \"0\"]"
]
},
{
@ -321,7 +326,7 @@
}
],
"source": [
"nation['HRS_ET'].unique()"
"nation[\"HRS_ET\"].unique()"
]
}
],

11
data/data-pipeline/data_pipeline/ipython/explore_adjacency.ipynb
View file

@ -18,7 +18,10 @@
" sys.path.append(module_path)\n",
"\n",
"from data_pipeline.config import settings\n",
"from data_pipeline.etl.sources.geo_utils import add_tracts_for_geometries, get_tract_geojson\n"
"from data_pipeline.etl.sources.geo_utils import (\n",
" add_tracts_for_geometries,\n",
" get_tract_geojson,\n",
")"
]
},
{
@ -655,9 +658,9 @@
}
],
"source": [
"adjacent_tracts.groupby(\"ORIGINAL_TRACT\")[[\"included\"]].mean().reset_index().rename(\n",
" columns={\"ORIGINAL_TRACT\": \"GEOID10_TRACT\"}\n",
")"
"adjacent_tracts.groupby(\"ORIGINAL_TRACT\")[\n",
" [\"included\"]\n",
"].mean().reset_index().rename(columns={\"ORIGINAL_TRACT\": \"GEOID10_TRACT\"})"
]
},
{

3
data/data-pipeline/data_pipeline/ipython/explore_eamlis.ipynb
View file

@ -65,7 +65,8 @@
"tmp_path.mkdir(parents=True, exist_ok=True)\n",
"\n",
"eamlis_path_in_s3 = (\n",
" settings.AWS_JUSTICE40_DATASOURCES_URL + \"/eAMLIS export of all data.tsv.zip\"\n",
" settings.AWS_JUSTICE40_DATASOURCES_URL\n",
" + \"/eAMLIS export of all data.tsv.zip\"\n",
")\n",
"\n",
"unzip_file_from_url(\n",

24
data/data-pipeline/data_pipeline/ipython/generate_fuds_test_data.ipynb
View file

@ -460,7 +460,9 @@
"outputs": [],
"source": [
"object_ids_to_keep = set(\n",
" merged_exaple_data[merged_exaple_data[\"_merge\"] == \"both\"].OBJECTID.astype(\"int\")\n",
" merged_exaple_data[merged_exaple_data[\"_merge\"] == \"both\"].OBJECTID.astype(\n",
" \"int\"\n",
" )\n",
")\n",
"features = []\n",
"for feature in raw_fuds_geojson[\"features\"]:\n",
@ -476,7 +478,11 @@
"outputs": [],
"source": [
"def make_fake_feature(\n",
" state: str, has_projects: bool, is_eligible: bool, latitude: float, longitude: float\n",
" state: str,\n",
" has_projects: bool,\n",
" is_eligible: bool,\n",
" latitude: float,\n",
" longitude: float,\n",
"):\n",
" \"\"\"For tracts where we don't have a FUDS, fake one.\"\"\"\n",
" make_fake_feature._object_id += 1\n",
@ -537,7 +543,9 @@
"# Create FUDS in CA for each tract that doesn't have a FUDS\n",
"for tract_id, point in points.items():\n",
" for bools in [(True, True), (True, False), (False, False)]:\n",
" features.append(make_fake_feature(\"CA\", bools[0], bools[1], point.y, point.x))"
" features.append(\n",
" make_fake_feature(\"CA\", bools[0], bools[1], point.y, point.x)\n",
" )"
]
},
{
@ -596,9 +604,9 @@
}
],
"source": [
"test_frame_with_tracts_full = test_frame_with_tracts = add_tracts_for_geometries(\n",
" test_frame\n",
")"
"test_frame_with_tracts_full = (\n",
" test_frame_with_tracts\n",
") = add_tracts_for_geometries(test_frame)"
]
},
{
@ -680,7 +688,9 @@
}
],
"source": [
"tracts = test_frame_with_tracts_full[[\"GEOID10_TRACT\", \"geometry\"]].drop_duplicates()\n",
"tracts = test_frame_with_tracts_full[\n",
" [\"GEOID10_TRACT\", \"geometry\"]\n",
"].drop_duplicates()\n",
"tracts[\"lat_long\"] = test_frame_with_tracts_full.geometry.apply(\n",
" lambda point: (point.x, point.y)\n",
")\n",

37
data/data-pipeline/data_pipeline/ipython/geojson_compare_tiles.ipynb
View file

@ -13,7 +13,7 @@
"import geopandas as gpd\n",
"\n",
"# Read in the above json file\n",
"nation=gpd.read_file(\"/Users/vims/Downloads/usa-high-1822-637b.json\")"
"nation = gpd.read_file(\"/Users/vims/Downloads/usa-high-1822-637b.json\")"
]
},
{
@ -45,7 +45,7 @@
}
],
"source": [
"nation['FUDS_RAW']"
"nation[\"FUDS_RAW\"]"
]
},
{
@ -248,7 +248,18 @@
}
],
"source": [
"nation_new_ind = nation[['GEOID10', 'SF', 'CF', 'HRS_ET', 'AML_ET', 'AML_RAW','FUDS_ET', 'FUDS_RAW']]\n",
"nation_new_ind = nation[\n",
" [\n",
" \"GEOID10\",\n",
" \"SF\",\n",
" \"CF\",\n",
" \"HRS_ET\",\n",
" \"AML_ET\",\n",
" \"AML_RAW\",\n",
" \"FUDS_ET\",\n",
" \"FUDS_RAW\",\n",
" ]\n",
"]\n",
"nation_new_ind"
]
},
@ -270,7 +281,7 @@
}
],
"source": [
"nation_new_ind['HRS_ET'].unique()"
"nation_new_ind[\"HRS_ET\"].unique()"
]
},
{
@ -293,7 +304,7 @@
}
],
"source": [
"nation_new_ind['HRS_ET'].value_counts()"
"nation_new_ind[\"HRS_ET\"].value_counts()"
]
},
{
@ -314,7 +325,7 @@
}
],
"source": [
"nation_new_ind['AML_ET'].unique()"
"nation_new_ind[\"AML_ET\"].unique()"
]
},
{
@ -337,7 +348,7 @@
}
],
"source": [
"nation_new_ind['AML_ET'].value_counts()"
"nation_new_ind[\"AML_ET\"].value_counts()"
]
},
{
@ -358,7 +369,7 @@
}
],
"source": [
"nation_new_ind['AML_RAW'].unique()"
"nation_new_ind[\"AML_RAW\"].unique()"
]
},
{
@ -380,7 +391,7 @@
}
],
"source": [
"nation_new_ind['AML_RAW'].value_counts()"
"nation_new_ind[\"AML_RAW\"].value_counts()"
]
},
{
@ -401,7 +412,7 @@
}
],
"source": [
"nation_new_ind['FUDS_ET'].unique()"
"nation_new_ind[\"FUDS_ET\"].unique()"
]
},
{
@ -424,7 +435,7 @@
}
],
"source": [
"nation_new_ind['FUDS_ET'].value_counts()"
"nation_new_ind[\"FUDS_ET\"].value_counts()"
]
},
{
@ -445,7 +456,7 @@
}
],
"source": [
"nation_new_ind['FUDS_RAW'].unique()"
"nation_new_ind[\"FUDS_RAW\"].unique()"
]
},
{
@ -468,7 +479,7 @@
}
],
"source": [
"nation_new_ind['FUDS_RAW'].value_counts()"
"nation_new_ind[\"FUDS_RAW\"].value_counts()"
]
}
],

12
data/data-pipeline/data_pipeline/ipython/geopandas_speed_test.ipynb
View file

@ -36,8 +36,8 @@
" engine=\"pyogrio\",\n",
")\n",
"end = time.time()\n",
" \n",
"print(\"Time taken to execute the function using pyogrio is\", end-begin)"
"\n",
"print(\"Time taken to execute the function using pyogrio is\", end - begin)"
]
},
{
@ -59,11 +59,13 @@
"census_tract_gdf = gpd.read_file(\n",
" CensusETL.NATIONAL_TRACT_JSON_PATH,\n",
" engine=\"fiona\",\n",
" include_fields=[\"GEOID10\"]\n",
" include_fields=[\"GEOID10\"],\n",
")\n",
"end2 = time.time()\n",
" \n",
"print(\"Time taken to execute the function using include fields is\", end2-begin2)"
"\n",
"print(\n",
" \"Time taken to execute the function using include fields is\", end2 - begin2\n",
")"
]
},
{

4
data/data-pipeline/data_pipeline/ipython/tribal_and_tracts_overlap.ipynb
View file

@ -1369,7 +1369,9 @@
"\n",
"results = results.reset_index()\n",
"\n",
"results.to_csv(\"~/Downloads/tribal_area_as_a_share_of_tract_area.csv\", index=False)\n",
"results.to_csv(\n",
" \"~/Downloads/tribal_area_as_a_share_of_tract_area.csv\", index=False\n",
")\n",
"\n",
"# Printing results\n",
"print(results)"