publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-08-03 09:04:18 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Updating comparison tool to be easier for pairwise comparisons (#1400)

Browse source 
Creating pairwise comparison tool to compare two lists of prioritized tracts to each other.
This commit is contained in:
Emma Nechamkin 2022-03-30 14:02:06 -04:00 • committed by GitHub
commit cb963cff5f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 1129 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

24
data/data-pipeline/data_pipeline/comparison_tool/src/donut_hole_dacs.yaml Normal file
View file

@ -0,0 +1,24 @@
ADDITIONAL_DEMO_COLUMNS:
- Urban Heuristic Flag
- Percent of individuals below 200% Federal Poverty Line
- Percent individuals age 25 or over with less than high school degree
- Unemployment (percent)
- Percent of households in linguistic isolation
COMPARATOR_COLUMN: donut_hole_dac_additional_constraints
COMPARATOR_FILE: data/donut_hole_dac/donut_hole_data.csv
DEMOGRAPHIC_COLUMNS:
- Percent Black or African American alone
- Percent American Indian and Alaska Native alone
- Percent Asian alone
- Percent Native Hawaiian and Other Pacific alone
- Percent Two or more races
- Percent Non-Hispanic White
- Percent Hispanic or Latino
DEMOGRAPHIC_FILE: ../../data_pipeline/data/dataset/census_acs_2019/usa.csv
OUTPUT_DATA_PATH: output/donut_hole_dac
SCORE_FILE: ../../data_pipeline/data/score/csv/full/usa.csv
OTHER_COMPARATOR_COLUMNS:
- donut_hole_dac
- P200_PFS
- HSEF
KEEP_MISSING_VALUES_FOR_SEGMENTATION: false

82
data/data-pipeline/data_pipeline/comparison_tool/src/run_tract_comparison.py Normal file
View file

@ -0,0 +1,82 @@
"""
This script is a 'starter' for parameterized work pertaining to generating comparisons.
Here, we only compare DAC lists that operate at the tract level. We can change/expand this as we move forward.
Why papermill? Papermill is an easy way to parameterize notebooks. While doing comparison work,
I often had to do a lot of one-off analysis in a notebook that would then get discarded. With parametrized notebooks,
we can run each type of analysis and then store it for posterity. The first template is just for agencies, but you could
imagine generating interactive outputs for many types of analysis, like demographic work.
To see more: https://buildmedia.readthedocs.org/media/pdf/papermill/latest/papermill.pdf
To run:
` $ python src/run_tract_comparison.py --template_notebook=TEMPLATE.ipynb --parameter_yaml=PARAMETERS.yaml`
"""
import os
import datetime
import argparse
import yaml
import papermill as pm
def _read_param_file(param_file: str) -> dict:
"""Reads params and enforces a few constraints:
1. There's a defined output path
2. Change all relative paths to absolute paths
"""
with open(param_file, "r", encoding="utf8") as param_stream:
param_dict = yaml.safe_load(param_stream)
assert (
"OUTPUT_DATA_PATH" in param_dict
), "Error: you need to specify an output data path"
# convert any paths to absolute paths
updated_param_dict = {}
for variable, value in param_dict.items():
if ("PATH" in variable) or ("FILE" in variable):
updated_param_dict[variable] = os.path.abspath(value)
else:
updated_param_dict[variable] = value
# configure output name
updated_param_dict["OUTPUT_NAME"] = updated_param_dict[
"OUTPUT_DATA_PATH"
].split("/")[-1]
return updated_param_dict
def _configure_output(updated_param_dict: dict, input_template: str) -> str:
"""Configures output directory and creates output path"""
if not os.path.exists(updated_param_dict["OUTPUT_DATA_PATH"]):
os.mkdir(updated_param_dict["OUTPUT_DATA_PATH"])
output_notebook_path = os.path.join(
updated_param_dict["OUTPUT_DATA_PATH"],
updated_param_dict["OUTPUT_NAME"]
+ f"__{input_template.replace('template', datetime.datetime.now().strftime('%Y-%m-%d'))}",
)
return output_notebook_path
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"--template_notebook",
help="Please specify which notebook to run",
required=True,
)
parser.add_argument(
"--parameter_yaml",
help="Please specify which parameter file to use",
required=True,
)
args = parser.parse_args()
updated_param_dict = _read_param_file(args.parameter_yaml)
notebook_name = args.template_notebook.split("/")[-1]
output_notebook_path = _configure_output(updated_param_dict, notebook_name)
pm.execute_notebook(
args.template_notebook,
output_notebook_path,
parameters=updated_param_dict,
)

430
data/data-pipeline/data_pipeline/comparison_tool/src/tract_comparison__template.ipynb Normal file
View file

@ -0,0 +1,430 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "81788485-12d4-41f4-b5db-aa4874a32501",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import os\n",
"import datetime\n",
"import sys\n",
"import seaborn as sns\n",
"import matplotlib.pyplot as plt\n",
"\n",
"from data_pipeline.score import field_names\n",
"from data_pipeline.comparison_tool.src import utils \n",
"\n",
"pd.options.display.float_format = \"{:,.3f}\".format\n",
"%load_ext lab_black"
]
},
{
"cell_type": "markdown",
"id": "2820ead7-981d-40e0-8a07-80f9f1c48b9c",
"metadata": {},
"source": [
"# Comparator definition comparison\n",
"\n",
"This notebook answers a few questions:\n",
"1. How many tracts are flagged and what's the size of overlap by comparator?\n",
"2. What are the demographics of each set of tracts by \"category\" of score (CEJST but not comparator, comparator but not CEJST, CEJST and comparator)?\n",
"3. What are the overall demographics of ALL comparator vs ALL CEJST?\n",
"\n",
"It produces a single Excel file of the stats listed, but is interactive even after run-time. This notebook focuses on 1:1 comparison. It can be pointed in the YAML to either a simple output (tract and boolean for highlight) or to the output from an ETL."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2093584e-312b-44f7-87d7-6099d9fe000f",
"metadata": {
"tags": [
"parameters"
]
},
"outputs": [],
"source": [
"## These are parameters and get overridden by the \"injected parameters\" cell below\n",
"ADDITIONAL_DEMO_COLUMNS = []\n",
"COMPARATOR_COLUMN = None\n",
"COMPARATOR_FILE = None\n",
"DEMOGRAPHIC_COLUMNS = []\n",
"DEMOGRAPHIC_FILE = None\n",
"OUTPUT_DATA_PATH = None\n",
"SCORE_FILE = None\n",
"OTHER_COMPARATOR_COLUMNS = None\n",
"OUTPUT_NAME = None\n",
"KEEP_MISSING_VALUES_FOR_SEGMENTATION = True"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bd38aaaa",
"metadata": {},
"outputs": [],
"source": [
"## These are constants for all runs\n",
"GEOID_COLUMN = field_names.GEOID_TRACT_FIELD\n",
"SCORE_COLUMN = field_names.SCORE_M_COMMUNITIES\n",
"TOTAL_POPULATION_COLUMN = field_names.TOTAL_POP_FIELD"
]
},
{
"cell_type": "markdown",
"id": "8a3940be",
"metadata": {},
"source": [
"__Date and time of last run__"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5f19a1e3-fe05-425d-9160-4d4e193fadf7",
"metadata": {},
"outputs": [],
"source": [
"datetime.datetime.now()"
]
},
{
"cell_type": "markdown",
"id": "269b0f06-edee-4fce-8755-a8e7fe92e340",
"metadata": {},
"source": [
"__Congfigure output (autocreated)__"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "acace944-18fa-4b05-a581-8147e8e09299",
"metadata": {},
"outputs": [],
"source": [
"OUTPUT_EXCEL = os.path.join(\n",
" OUTPUT_DATA_PATH,\n",
" f\"{OUTPUT_NAME}__{datetime.datetime.now().strftime('%Y-%m-%d')}.xlsx\",\n",
")"
]
},
{
"cell_type": "markdown",
"id": "806b8a7a",
"metadata": {},
"source": [
"__Validate new data__"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0d089469",
"metadata": {},
"outputs": [],
"source": [
"utils.validate_new_data(\n",
" file_path=COMPARATOR_FILE, score_col=COMPARATOR_COLUMN\n",
")"
]
},
{
"cell_type": "markdown",
"id": "f5517217-f107-4fee-b4ca-be61f6b2b7c3",
"metadata": {},
"source": [
"__Read in data__"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cdefecf2-100b-4ad2-b0bd-101c0bad5a92",
"metadata": {},
"outputs": [],
"source": [
"comparator_cols = [COMPARATOR_COLUMN] + OTHER_COMPARATOR_COLUMNS if OTHER_COMPARATOR_COLUMNS else [COMPARATOR_COLUMN]\n",
"\n",
"#papermill_description=Loading_data\n",
"joined_df = pd.concat(\n",
" [\n",
" utils.read_file(\n",
" file_path=SCORE_FILE,\n",
" columns=[TOTAL_POPULATION_COLUMN, SCORE_COLUMN] + ADDITIONAL_DEMO_COLUMNS,\n",
" geoid=GEOID_COLUMN,\n",
" ),\n",
" utils.read_file(\n",
" file_path=COMPARATOR_FILE,\n",
" columns=comparator_cols,\n",
" geoid=GEOID_COLUMN\n",
" ),\n",
" utils.read_file(\n",
" file_path=DEMOGRAPHIC_FILE,\n",
" columns=DEMOGRAPHIC_COLUMNS,\n",
" geoid=GEOID_COLUMN,\n",
" ),\n",
" ],\n",
" axis=1,\n",
").reset_index()"
]
},
{
"cell_type": "markdown",
"id": "da71a62c-9c2c-46bd-815f-37cdb9ca18a1",
"metadata": {},
"source": [
"## High-level summary"
]
},
{
"cell_type": "markdown",
"id": "bfaf9912-1ffe-49e7-a38a-2f815b07826d",
"metadata": {},
"source": [
"What *shares* of tracts and population highlighted by the comparator are covered by CEJST?"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "93d2990c-edc0-404b-aeea-b62e3ca7b308",
"metadata": {},
"outputs": [],
"source": [
"#papermill_description=Summary_stats\n",
"population_df = utils.produce_summary_stats(\n",
" joined_df=joined_df,\n",
" comparator_column=COMPARATOR_COLUMN,\n",
" score_column=SCORE_COLUMN,\n",
" population_column=TOTAL_POPULATION_COLUMN,\n",
" geoid_column=GEOID_COLUMN\n",
")\n",
"population_df"
]
},
{
"cell_type": "markdown",
"id": "0c945e85-874d-4f1f-921e-ff2f0de42767",
"metadata": {},
"source": [
"## Tract-level stats\n",
"\n",
"First, this walks through overall stats for disadvantaged communities under the comparator definition and under the CEJST's definition. Next, this walks through stats by group (e.g., CEJST and not comparator). This is at the tract level, so the average across tracts, where tracts are not population-weighted. "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "618d2bac-8800-4fd7-a75d-f4abaae30923",
"metadata": {},
"outputs": [],
"source": [
"#papermill_description=Tract_stats\n",
"tract_level_by_identification_df = pd.concat(\n",
" [\n",
" utils.get_demo_series(\n",
" grouping_column=COMPARATOR_COLUMN,\n",
" joined_df=joined_df,\n",
" demo_columns=ADDITIONAL_DEMO_COLUMNS + DEMOGRAPHIC_COLUMNS\n",
" ),\n",
" utils.get_demo_series(\n",
" grouping_column=SCORE_COLUMN,\n",
" joined_df=joined_df,\n",
" demo_columns=ADDITIONAL_DEMO_COLUMNS + DEMOGRAPHIC_COLUMNS\n",
" ),\n",
" ],\n",
" axis=1,\n",
")\n",
"\n",
"tract_level_by_identification_df"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0f37bfa5-5e9e-46d1-97af-8d0e7d53c80b",
"metadata": {},
"outputs": [],
"source": [
"plt.figure(figsize=(11, 11))\n",
"sns.barplot(\n",
" y=\"Variable\",\n",
" x=\"Avg in tracts\",\n",
" hue=\"Definition\",\n",
" data=tract_level_by_identification_df.sort_values(by=COMPARATOR_COLUMN, ascending=False)\n",
" .stack()\n",
" .reset_index()\n",
" .rename(\n",
" columns={\"level_0\": \"Variable\", \"level_1\": \"Definition\", 0: \"Avg in tracts\"}\n",
" ),\n",
" palette=\"Blues\",\n",
")\n",
"plt.xlim(0, 1)\n",
"plt.title(\"Tract level averages by identification strategy\")\n",
"plt.savefig(os.path.join(OUTPUT_DATA_PATH, \"tract_lvl_avg.jpg\"), bbox_inches='tight')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "713f2a04-1b2a-472c-b036-454cd5c9a495",
"metadata": {},
"outputs": [],
"source": [
"#papermill_description=Tract_stats_grouped\n",
"tract_level_by_grouping_df = utils.get_tract_level_grouping(\n",
" joined_df=joined_df,\n",
" score_column=SCORE_COLUMN,\n",
" comparator_column=COMPARATOR_COLUMN,\n",
" demo_columns=ADDITIONAL_DEMO_COLUMNS + DEMOGRAPHIC_COLUMNS,\n",
" keep_missing_values=KEEP_MISSING_VALUES_FOR_SEGMENTATION\n",
")\n",
"\n",
"tract_level_by_grouping_formatted_df = utils.format_multi_index_for_excel(\n",
" df=tract_level_by_grouping_df\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0d166f53-4242-46f1-aedf-51dac517de94",
"metadata": {},
"outputs": [],
"source": [
"tract_level_by_grouping_formatted_df"
]
},
{
"cell_type": "markdown",
"id": "372ac341-b03b-404a-9462-5087b88579b7",
"metadata": {},
"source": [
"## Population-weighted stats"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d172e3c8-60ab-44f0-8034-dda69f1146da",
"metadata": {},
"outputs": [],
"source": [
"#papermill_description=Population_stats\n",
"population_weighted_stats_df = pd.concat(\n",
" [\n",
" utils.construct_weighted_statistics(\n",
" input_df=joined_df,\n",
" weighting_column=COMPARATOR_COLUMN,\n",
" demographic_columns=DEMOGRAPHIC_COLUMNS + ADDITIONAL_DEMO_COLUMNS,\n",
" population_column=TOTAL_POPULATION_COLUMN,\n",
" ),\n",
" utils.construct_weighted_statistics(\n",
" input_df=joined_df,\n",
" weighting_column=SCORE_COLUMN,\n",
" demographic_columns=DEMOGRAPHIC_COLUMNS + ADDITIONAL_DEMO_COLUMNS,\n",
" population_column=TOTAL_POPULATION_COLUMN,\n",
" ),\n",
" ],\n",
" axis=1,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "33c96e68",
"metadata": {},
"outputs": [],
"source": [
"population_weighted_stats_df"
]
},
{
"cell_type": "markdown",
"id": "751d2d21",
"metadata": {},
"source": [
"## Final information about overlap"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9eef9447",
"metadata": {},
"outputs": [],
"source": [
"comparator_and_cejst_proportion_series, states = utils.get_final_summary_info(\n",
" population=population_df,\n",
" comparator_file=COMPARATOR_FILE,\n",
" geoid_col=GEOID_COLUMN\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2c45e7b6",
"metadata": {},
"outputs": [],
"source": [
"states_text = \"States included in comparator: \" + states\n",
"states_text"
]
},
{
"cell_type": "markdown",
"id": "a9fb4dfa-876d-4f25-9552-9831f51ec05c",
"metadata": {},
"source": [
"## Print to excel"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "67e2c71e-68af-44b4-bab0-33bc54afe7a4",
"metadata": {},
"outputs": [],
"source": [
"#papermill_description=Writing_excel\n",
"utils.write_single_comparison_excel(\n",
" output_excel=OUTPUT_EXCEL,\n",
" population_df=population_df,\n",
" tract_level_by_identification_df=tract_level_by_identification_df,\n",
" population_weighted_stats_df=population_weighted_stats_df,\n",
" tract_level_by_grouping_formatted_df=tract_level_by_grouping_formatted_df,\n",
" comparator_and_cejst_proportion_series=comparator_and_cejst_proportion_series,\n",
" states_text=states_text\n",
")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

391
data/data-pipeline/data_pipeline/comparison_tool/src/utils.py Normal file
View file

@ -0,0 +1,391 @@
import pathlib
import pandas as pd
import xlsxwriter
from data_pipeline.score import field_names
from data_pipeline.etl.sources.census.etl_utils import get_state_information
# Some excel parameters
DEFAULT_COLUMN_WIDTH = 18
# the 31 is a limit from excel on how long the tab name can be
MSFT_TAB_NAME_LIMIT = 31
# FIPS information
DATA_PATH = pathlib.Path(__file__).parents[2] / "data"
FIPS_MAP = (
get_state_information(data_path=DATA_PATH)
.set_index("fips")["state_abbreviation"]
.to_dict()
)
def validate_new_data(
file_path: str, score_col: str, geoid: str = field_names.GEOID_TRACT_FIELD
):
"""Ensures user provided data meets the constraints.
Constraints are:
(1) Boolean series for score column
(2) GEOID column is named the same thing as in the rest of our code
Note this only reads the first 10 rows of the file for speed
"""
checking_df = pd.read_csv(
file_path, usecols=[score_col, geoid], dtype={geoid: str}, nrows=10
)
assert (
geoid in checking_df.columns
), f"Error: change your geoid column in the data to {field_names.GEOID_TRACT_FIELD}"
assert (
checking_df[score_col].nunique() <= 3
), f"Error: there are too many values possible in {score_col}"
assert (True in checking_df[score_col].unique()) & (
False in checking_df[score_col].unique()
), f"Error: {score_col} should be a boolean"
def read_file(
file_path: str, columns: list, geoid: str = field_names.GEOID_TRACT_FIELD
) -> pd.DataFrame:
"""Reads standardized csvs in
Parameters:
file_path: the file to read
columns: the columns to include
geoid: the geoid column name (if we change this in usa.csv, will need to change this slightly)
Returns:
dataframe that has been read in with geographic index
"""
assert (
geoid == field_names.GEOID_TRACT_FIELD
), f"Field name specified for geoid is incorrect. Use {field_names.GEOID_TRACT_FIELD}"
return pd.read_csv(
file_path, usecols=columns + [geoid], dtype={geoid: str}
).set_index(geoid)
def produce_summary_stats(
joined_df: pd.DataFrame,
comparator_column: str,
score_column: str,
population_column: str,
geoid_column: str = field_names.GEOID_TRACT_FIELD,
) -> pd.DataFrame:
"""Produces high-level overview dataframe
Parameters:
joined_df: the big df
comparator_column: the column name for the comparator identification bool
score_column: the column name for the CEJST score bool
population_column: the column that includes population count per tract
geoid_column: the geoid10_tract column
Returns:
population: the high-level overview df
"""
# Because this reports high-level statistics across all census tracts, it
# makes sense for the statistics to force all census tracts to be included.
temp_joined_df = joined_df.fillna({comparator_column: "missing"})
population_df = temp_joined_df.groupby(
[comparator_column, score_column]
).agg({population_column: ["sum"], geoid_column: ["count"]})
population_df["share_of_tracts"] = (
population_df[geoid_column] / population_df[geoid_column].sum()
)
population_df["share_of_population_in_tracts"] = (
population_df[population_column]
/ population_df[population_column].sum()
)
population_df.columns = [
"Population",
"Count of tracts",
"Share of tracts",
"Share of population",
]
return population_df
def get_demo_series(
grouping_column: str,
joined_df: pd.DataFrame,
demo_columns: list,
) -> pd.DataFrame:
"""Helper function to produce demographic information"""
# To preserve np.nan, we drop all nans
full_df = joined_df.dropna(subset=[grouping_column])
return (
full_df[full_df[grouping_column]][demo_columns]
.mean()
.T.rename(grouping_column)
)
def get_tract_level_grouping(
joined_df: pd.DataFrame,
score_column: str,
comparator_column: str,
demo_columns: list,
keep_missing_values: bool = True,
) -> pd.DataFrame:
"""Function to produce segmented statistics (tract level)
Here, we are thinking about the following segments:
1. CEJST and comparator
2. Not CEJST and comparator
3. Not CEJST and not comparator
4. CEJST and not comparator
If "keep_missing_values" flag:
5. Missing from CEJST and comparator (this should never be true - it would be a tract we had not seen in CEJST!)
6. Missing from comparator and not highlighted by CEJST
7. Missing from comparator and highlighted by CEJST
This will make sure that comparisons are "apples to apples".
"""
group_list = [score_column, comparator_column]
use_df = joined_df.copy()
if keep_missing_values:
use_df = use_df.fillna({score_column: "nan", comparator_column: "nan"})
grouping_df = use_df.groupby(group_list)[demo_columns].mean().reset_index()
# this will work whether or not there are "nans" present
grouping_df[score_column] = grouping_df[score_column].map(
{
True: "CEJST",
False: "Not CEJST",
"nan": "No CEJST classification",
}
)
grouping_df[comparator_column] = grouping_df[comparator_column].map(
{
True: "Comparator",
False: "Not Comparator",
"nan": "No Comparator classification",
}
)
return grouping_df.set_index([score_column, comparator_column]).T
def format_multi_index_for_excel(
df: pd.DataFrame, rename_str: str = "Variable"
) -> pd.DataFrame:
"""Helper function for multiindex printing"""
df = df.reset_index()
df.columns = [rename_str] + [
", ".join(col_tuple).strip()
for col_tuple in df.columns[1:].to_flat_index()
]
return df
def get_final_summary_info(
population: pd.DataFrame,
comparator_file: str,
geoid_col: str,
) -> tuple[pd.DataFrame, str]:
"""Creates summary table.
This creates a series that tells us what share (%) of census tracts identified
by the comparator are also in CEJST and what states the comparator covers.
"""
try:
comparator_and_cejst_proportion_series = (
population.loc[(True, True)] / population.loc[(True,)].sum()
)
except KeyError:
# for when we are looking at a disjoint set, like donut holes
comparator_and_cejst_proportion_series = pd.DataFrame()
# we pull all fips codes from the comparator column -- this is a very quick
# read
states_represented = (
pd.read_csv(
comparator_file, usecols=[geoid_col], dtype={geoid_col: str}
)[geoid_col]
.str[:2]
.unique()
)
# We join all states into a single string here so they can be printed in a single
# cell in the excel file.
states = ", ".join(
[
FIPS_MAP[state]
if (state in FIPS_MAP)
else f"Comparator code missing: (fips {state})"
for state in states_represented
]
)
return comparator_and_cejst_proportion_series, states
def construct_weighted_statistics(
input_df: pd.DataFrame,
weighting_column: str,
demographic_columns: list,
population_column: str,
) -> pd.DataFrame:
"""Function to produce population weighted stats
Parameters:
input_df: this gets copied and is the big frame
weighting_column: the column to group by for the comparator weights (e.g., grouped by this column, the sum of the weights is 1)
demographic_columns: the columns to get weighted stats for
population_column: the population column
Returns:
population-weighted comparator statistics
"""
comparator_weighted_joined_df = input_df.copy()
comparator_weighted_joined_df[
"tmp_weight"
] = comparator_weighted_joined_df.groupby(weighting_column)[
population_column
].transform(
lambda x: x / x.sum()
)
comparator_weighted_joined_df[
demographic_columns
] = comparator_weighted_joined_df[demographic_columns].transform(
lambda x: x * comparator_weighted_joined_df["tmp_weight"]
)
return (
comparator_weighted_joined_df.groupby(weighting_column)[
demographic_columns
]
.sum()
.T
).rename(columns={True: weighting_column, False: "not " + weighting_column})
def write_excel_tab(
writer: pd.ExcelWriter,
worksheet_name: str,
df: pd.DataFrame,
text_format: xlsxwriter.format.Format,
use_index: bool = True,
):
"""Helper function to set tab width"""
df.to_excel(writer, sheet_name=worksheet_name, index=use_index)
worksheet = writer.sheets[worksheet_name]
for i, column_name in enumerate(df.columns):
# We set variable names to be extra wide, all other columns can take
# cues from their headers
if not column_name == "Variable":
worksheet.set_column(i, i + 1, len(column_name) + 2, text_format)
else:
worksheet.set_column(i, i + 1, DEFAULT_COLUMN_WIDTH, text_format)
def write_excel_tab_about_comparator_scope(
writer: pd.ExcelWriter,
worksheet_name: str,
comparator_and_cejst_proportion_series: pd.Series,
text_format: xlsxwriter.format.Format,
merge_format: xlsxwriter.format.Format,
states_text: str,
):
"""Writes single tab for the excel file about high level comparator stats"""
comparator_and_cejst_proportion_series.to_excel(
writer, sheet_name=worksheet_name
)
worksheet = writer.sheets[worksheet_name[:MSFT_TAB_NAME_LIMIT]]
worksheet.set_column(0, 1, DEFAULT_COLUMN_WIDTH, text_format)
# merge the cells for states text
row_merge = len(comparator_and_cejst_proportion_series) + 2
# changes the row height based on how long the states text is
worksheet.set_row(row_merge, len(states_text) // 2)
worksheet.merge_range(
first_row=row_merge,
last_row=row_merge,
first_col=0,
last_col=1,
data=states_text,
cell_format=merge_format,
)
def write_single_comparison_excel(
output_excel: str,
population_df: pd.DataFrame,
tract_level_by_identification_df: pd.DataFrame,
population_weighted_stats_df: pd.DataFrame,
tract_level_by_grouping_formatted_df: pd.DataFrame,
comparator_and_cejst_proportion_series: pd.Series,
states_text: str,
):
"""Writes the comparison excel file.
Writing excel from python is always a huge pain. Making the functions truly generalizable is not worth
the pay off and (in my experience) is extremely hard to maintain.
"""
with pd.ExcelWriter(output_excel) as writer:
workbook = writer.book
text_format = workbook.add_format(
{
"bold": False,
"text_wrap": True,
"valign": "middle",
"num_format": "#,##0.00",
}
)
merge_format = workbook.add_format(
{
"border": 1,
"align": "center",
"valign": "vcenter",
"text_wrap": True,
}
)
write_excel_tab(
writer=writer,
worksheet_name="Summary",
df=population_df.reset_index(),
text_format=text_format,
use_index=False,
)
write_excel_tab(
writer=writer,
worksheet_name="Tract level stats",
df=tract_level_by_identification_df.reset_index().rename(
columns={"index": "Description of variable"}
),
text_format=text_format,
use_index=False,
)
write_excel_tab(
writer=writer,
worksheet_name="Population level stats",
df=population_weighted_stats_df.reset_index().rename(
columns={"index": "Description of variable"}
),
text_format=text_format,
use_index=False,
)
write_excel_tab(
writer=writer,
worksheet_name="Segmented tract level stats",
df=tract_level_by_grouping_formatted_df,
text_format=text_format,
use_index=False,
)
if not comparator_and_cejst_proportion_series.empty:
write_excel_tab_about_comparator_scope(
writer=writer,
worksheet_name="Comparator and CEJST overlap",
comparator_and_cejst_proportion_series=comparator_and_cejst_proportion_series.rename(
"Comparator and CEJST overlap"
),
text_format=text_format,
states_text=states_text,
merge_format=merge_format,
)