mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-07-26 01:51:17 -07:00
Definition L updates (#862)
* Changing FEMA risk measure * Adding "basic stats" feature to comparison tool * Tweaking Definition L
This commit is contained in:
Lucas Merrill Brown
GitHub
parent
0a65060c0c
commit
03e59f2abd
9 changed files with 265 additions and 63 deletions
|
|
@ -318,6 +318,28 @@
|
|||
"# )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4b74b0bf",
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Create a FEMA risk index score\n",
|
||||
"# Note: this can be deleted at a later date.\n",
|
||||
"FEMA_EXPECTED_ANNUAL_LOSS_RATE_FIELD = (\n",
|
||||
" \"FEMA Risk Index Expected Annual Loss Rate\"\n",
|
||||
")\n",
|
||||
"FEMA_COMMUNITIES = \"FEMA Risk Index (top 30th percentile)\"\n",
|
||||
"merged_df[FEMA_COMMUNITIES] = (\n",
|
||||
" merged_df[f\"{FEMA_EXPECTED_ANNUAL_LOSS_RATE_FIELD} (percentile)\"] > 0.70\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"merged_df[FEMA_COMMUNITIES].describe()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
|
|
@ -406,6 +428,11 @@
|
|||
" priority_communities_field=PERSISTENT_POVERTY_CBG_LEVEL_FIELD,\n",
|
||||
" other_census_tract_fields_to_keep=[],\n",
|
||||
" ),\n",
|
||||
" Index(\n",
|
||||
" method_name=FEMA_COMMUNITIES,\n",
|
||||
" priority_communities_field=FEMA_COMMUNITIES,\n",
|
||||
" other_census_tract_fields_to_keep=[],\n",
|
||||
" ),\n",
|
||||
" ]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
|
|
@ -439,11 +466,6 @@
|
|||
"\n",
|
||||
"census_tract_indices = [\n",
|
||||
" Index(\n",
|
||||
" method_name=\"Persistent Poverty\",\n",
|
||||
" priority_communities_field=PERSISTENT_POVERTY_TRACT_LEVEL_FIELD,\n",
|
||||
" other_census_tract_fields_to_keep=[],\n",
|
||||
" ),\n",
|
||||
" Index(\n",
|
||||
" method_name=\"CalEnviroScreen 4.0\",\n",
|
||||
" priority_communities_field=\"calenviroscreen_priority_community\",\n",
|
||||
" other_census_tract_fields_to_keep=[\n",
|
||||
|
|
@ -451,6 +473,27 @@
|
|||
" CALENVIROSCREEN_PERCENTILE_FIELD,\n",
|
||||
" ],\n",
|
||||
" ),\n",
|
||||
" Index(\n",
|
||||
" method_name=\"Persistent Poverty\",\n",
|
||||
" priority_communities_field=PERSISTENT_POVERTY_TRACT_LEVEL_FIELD,\n",
|
||||
" other_census_tract_fields_to_keep=[],\n",
|
||||
" ),\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"# These fields will be used for statistical comparisons.\n",
|
||||
"comparison_fields = [\n",
|
||||
" \"Percent of individuals < 100% Federal Poverty Line\",\n",
|
||||
" \"Percent of individuals < 200% Federal Poverty Line\",\n",
|
||||
" \"Median household income (% of AMI)\",\n",
|
||||
" \"Percent of households in linguistic isolation\",\n",
|
||||
" \"Percent individuals age 25 or over with less than high school degree\",\n",
|
||||
" \"Linguistic isolation (percent)\",\n",
|
||||
" \"Unemployed civilians (percent)\",\n",
|
||||
" \"Median household income in the past 12 months\",\n",
|
||||
" URBAN_HEURISTIC_FIELD,\n",
|
||||
" LIFE_EXPECTANCY_FIELD,\n",
|
||||
" HEALTH_INSURANCE_FIELD,\n",
|
||||
" BAD_HEALTH_FIELD,\n",
|
||||
"]"
|
||||
]
|
||||
},
|
||||
|
|
@ -735,7 +778,120 @@
|
|||
"write_state_distribution_excel(\n",
|
||||
" state_distribution_df=state_distribution_df,\n",
|
||||
" file_path=COMPARISON_OUTPUTS_DIR / f\"{file_prefix}.xlsx\",\n",
|
||||
")"
|
||||
")\n",
|
||||
"\n",
|
||||
"# Note: this is helpful because this file is extremely long-running, so it alerts the user when the first step\n",
|
||||
"# of data analysis is done. Can be removed when converted into scripts. -LMB.\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"os.system(\"say 'state analysis is written.'\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c4d0e783",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"directory = COMPARISON_OUTPUTS_DIR / \"cbg_basic_stats\"\n",
|
||||
"directory.mkdir(parents=True, exist_ok=True)\n",
|
||||
"\n",
|
||||
"# TODO: this Excel-writing function is extremely similar to other Excel-writing functions in this notebook.\n",
|
||||
"# Refactor to use the same Excel-writing function.\n",
|
||||
"def write_basic_stats_excel(\n",
|
||||
" basic_stats_df: pd.DataFrame, file_path: pathlib.PosixPath\n",
|
||||
") -> None:\n",
|
||||
" \"\"\"Write the dataframe to excel with special formatting.\"\"\"\n",
|
||||
" # Create a Pandas Excel writer using XlsxWriter as the engine.\n",
|
||||
" writer = pd.ExcelWriter(file_path, engine=\"xlsxwriter\")\n",
|
||||
"\n",
|
||||
" # Convert the dataframe to an XlsxWriter Excel object. We also turn off the\n",
|
||||
" # index column at the left of the output dataframe.\n",
|
||||
" basic_stats_df.to_excel(writer, sheet_name=\"Sheet1\", index=False)\n",
|
||||
"\n",
|
||||
" # Get the xlsxwriter workbook and worksheet objects.\n",
|
||||
" workbook = writer.book\n",
|
||||
" worksheet = writer.sheets[\"Sheet1\"]\n",
|
||||
" worksheet.autofilter(0, 0, basic_stats_df.shape[0], basic_stats_df.shape[1])\n",
|
||||
"\n",
|
||||
" # Set a width parameter for all columns\n",
|
||||
" # Note: this is parameterized because every call to `set_column` requires setting the width.\n",
|
||||
" column_width = 15\n",
|
||||
"\n",
|
||||
" for column in basic_stats_df.columns:\n",
|
||||
" # Turn the column index into excel ranges (e.g., column #95 is \"CR\" and the range may be \"CR2:CR53\").\n",
|
||||
" column_index = basic_stats_df.columns.get_loc(column)\n",
|
||||
" column_character = get_excel_column_name(column_index)\n",
|
||||
"\n",
|
||||
" # Set all columns to larger width\n",
|
||||
" worksheet.set_column(\n",
|
||||
" f\"{column_character}:{column_character}\", column_width\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Add green to red conditional formatting.\n",
|
||||
" column_ranges = (\n",
|
||||
" f\"{column_character}2:{column_character}{len(basic_stats_df)+1}\"\n",
|
||||
" )\n",
|
||||
" worksheet.conditional_format(\n",
|
||||
" column_ranges,\n",
|
||||
" # Min: green, max: red.\n",
|
||||
" {\n",
|
||||
" \"type\": \"2_color_scale\",\n",
|
||||
" \"min_color\": \"#00FF7F\",\n",
|
||||
" \"max_color\": \"#C82538\",\n",
|
||||
" },\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Special formatting for all percent columns\n",
|
||||
" # Note: we can't just search for `percent`, because that's included in the word `percentile`.\n",
|
||||
" if (\n",
|
||||
" \"percent \" in column\n",
|
||||
" or \"(percent)\" in column\n",
|
||||
" or \"Percent \" in column\n",
|
||||
" ):\n",
|
||||
" # Make these columns percentages.\n",
|
||||
" percentage_format = workbook.add_format({\"num_format\": \"0%\"})\n",
|
||||
" worksheet.set_column(\n",
|
||||
" f\"{column_character}:{column_character}\",\n",
|
||||
" column_width,\n",
|
||||
" percentage_format,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" header_format = workbook.add_format(\n",
|
||||
" {\"bold\": True, \"text_wrap\": True, \"valign\": \"bottom\"}\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Overwrite both the value and the format of each header cell\n",
|
||||
" # This is because xlsxwriter / pandas has a known bug where it can't wrap text for a dataframe.\n",
|
||||
" # See https://stackoverflow.com/questions/42562977/xlsxwriter-text-wrap-not-working.\n",
|
||||
" for col_num, value in enumerate(basic_stats_df.columns.values):\n",
|
||||
" worksheet.write(0, col_num, value, header_format)\n",
|
||||
"\n",
|
||||
" writer.save()\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"for index in census_block_group_indices:\n",
|
||||
" print(f\"Basic stats for {index.method_name}\")\n",
|
||||
" temp_df = merged_df\n",
|
||||
" temp_df[index.priority_communities_field] = (\n",
|
||||
" temp_df[index.priority_communities_field] == True\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # print(sum(temp_df[\"is_a_priority_cbg\"]))\n",
|
||||
" grouped_df = (\n",
|
||||
" temp_df.groupby(index.priority_communities_field).mean().reset_index()\n",
|
||||
" )\n",
|
||||
" result_df = grouped_df[\n",
|
||||
" [index.priority_communities_field] + comparison_fields\n",
|
||||
" ]\n",
|
||||
" result_df.to_csv(\n",
|
||||
" directory / f\"{index.method_name} Basic Stats.csv\", index=False\n",
|
||||
" )\n",
|
||||
" write_basic_stats_excel(\n",
|
||||
" basic_stats_df=result_df,\n",
|
||||
" file_path=directory / f\"{index.method_name} Basic Stats.xlsx\",\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -918,21 +1074,6 @@
|
|||
" )\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"comparison_fields = [\n",
|
||||
" \"Percent of individuals < 100% Federal Poverty Line\",\n",
|
||||
" \"Percent of individuals < 200% Federal Poverty Line\",\n",
|
||||
" \"Median household income (% of AMI)\",\n",
|
||||
" \"Percent of households in linguistic isolation\",\n",
|
||||
" \"Percent individuals age 25 or over with less than high school degree\",\n",
|
||||
" \"Linguistic isolation (percent)\",\n",
|
||||
" \"Unemployed civilians (percent)\",\n",
|
||||
" \"Median household income in the past 12 months\",\n",
|
||||
" URBAN_HEURISTIC_FIELD,\n",
|
||||
" LIFE_EXPECTANCY_FIELD,\n",
|
||||
" HEALTH_INSURANCE_FIELD,\n",
|
||||
" BAD_HEALTH_FIELD,\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"for (index_a, index_b) in itertools.combinations(census_block_group_indices, 2):\n",
|
||||
" print(f\"Comparing {index_a} and {index_b}.\")\n",
|
||||
" compare_cbg_scores(\n",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue