publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-08-02 20:54:19 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Issue 1044: Add low HS education fields to tiles and download (#1046)

Browse source
This commit is contained in:
Lucas Merrill Brown 2021-12-14 15:41:06 -05:00 • committed by GitHub
commit 0d10534725
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 34 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

6
data/data-pipeline/data_pipeline/etl/score/constants.py
View file

@ -140,11 +140,13 @@ TILES_SCORE_COLUMNS = {
field_names.POVERTY_LOW_HS_EDUCATION_FIELD: "PLHSE", field_names.POVERTY_LOW_HS_EDUCATION_FIELD: "PLHSE",
field_names.LOW_MEDIAN_INCOME_LOW_HS_EDUCATION_FIELD: "LMILHSE", field_names.LOW_MEDIAN_INCOME_LOW_HS_EDUCATION_FIELD: "LMILHSE",
field_names.UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD: "ULHSE", field_names.UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD: "ULHSE",
field_names.LOW_HS_EDUCATION_FIELD: "LHE",
field_names.FPL_200_SERIES: "FPL200S", field_names.FPL_200_SERIES: "FPL200S",
field_names.THRESHOLD_COUNT: "TC", field_names.THRESHOLD_COUNT: "TC",
field_names.ISLAND_AREAS_UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD: "IAULHSE", field_names.ISLAND_AREAS_UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD: "IAULHSE",
field_names.ISLAND_AREAS_POVERTY_LOW_HS_EDUCATION_FIELD: "ISPLHSE", field_names.ISLAND_AREAS_POVERTY_LOW_HS_EDUCATION_FIELD: "ISPLHSE",
field_names.ISLAND_AREAS_LOW_MEDIAN_INCOME_LOW_HS_EDUCATION_FIELD: "IALMILHSE", field_names.ISLAND_AREAS_LOW_MEDIAN_INCOME_LOW_HS_EDUCATION_FIELD: "IALMILHSE",
field_names.ISLAND_AREAS_LOW_HS_EDUCATION_FIELD: "IALHE",
} }
# columns to round floats to 2 decimals # columns to round floats to 2 decimals
@ -178,6 +180,8 @@ TILES_SCORE_FLOAT_COLUMNS = [
field_names.TSDF_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, field_names.TSDF_FIELD + field_names.PERCENTILE_FIELD_SUFFIX,
field_names.TRAFFIC_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, field_names.TRAFFIC_FIELD + field_names.PERCENTILE_FIELD_SUFFIX,
field_names.UNEMPLOYMENT_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, field_names.UNEMPLOYMENT_FIELD + field_names.PERCENTILE_FIELD_SUFFIX,
field_names.LOW_HS_EDUCATION_FIELD,
field_names.ISLAND_AREAS_LOW_HS_EDUCATION_FIELD,
field_names.WASTEWATER_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, field_names.WASTEWATER_FIELD + field_names.PERCENTILE_FIELD_SUFFIX,
field_names.SCORE_L + field_names.PERCENTILE_FIELD_SUFFIX, field_names.SCORE_L + field_names.PERCENTILE_FIELD_SUFFIX,
] ]
@ -264,6 +268,7 @@ DOWNLOADABLE_SCORE_COLUMNS = [
field_names.POVERTY_LOW_HS_EDUCATION_FIELD, field_names.POVERTY_LOW_HS_EDUCATION_FIELD,
field_names.HIGH_SCHOOL_ED_FIELD, field_names.HIGH_SCHOOL_ED_FIELD,
field_names.HIGH_SCHOOL_ED_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, field_names.HIGH_SCHOOL_ED_FIELD + field_names.PERCENTILE_FIELD_SUFFIX,
field_names.LOW_HS_EDUCATION_FIELD,
field_names.THRESHOLD_COUNT, field_names.THRESHOLD_COUNT,
field_names.UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD, field_names.UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD,
field_names.COMBINED_UNEMPLOYMENT_2010, field_names.COMBINED_UNEMPLOYMENT_2010,
@ -272,4 +277,5 @@ DOWNLOADABLE_SCORE_COLUMNS = [
field_names.ISLAND_AREAS_UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD, field_names.ISLAND_AREAS_UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD,
field_names.ISLAND_AREAS_POVERTY_LOW_HS_EDUCATION_FIELD, field_names.ISLAND_AREAS_POVERTY_LOW_HS_EDUCATION_FIELD,
field_names.ISLAND_AREAS_LOW_MEDIAN_INCOME_LOW_HS_EDUCATION_FIELD, field_names.ISLAND_AREAS_LOW_MEDIAN_INCOME_LOW_HS_EDUCATION_FIELD,
field_names.ISLAND_AREAS_LOW_HS_EDUCATION_FIELD,
] ]

6
data/data-pipeline/data_pipeline/etl/score/tests/sample_data/score_data_initial.csv
View file

File diff suppressed because one or more lines are too long

BIN
data/data-pipeline/data_pipeline/etl/score/tests/snapshots/downloadable_data_expected.pkl
View file

Binary file not shown.

BIN
data/data-pipeline/data_pipeline/etl/score/tests/snapshots/score_data_expected.pkl
View file

Binary file not shown.

BIN
data/data-pipeline/data_pipeline/etl/score/tests/snapshots/score_transformed_expected.pkl
View file

Binary file not shown.

BIN
data/data-pipeline/data_pipeline/etl/score/tests/snapshots/tile_data_expected.pkl
View file

Binary file not shown.

10
data/data-pipeline/data_pipeline/ipython/compare_two_score_files_for_differences.ipynb
View file

@ -107,11 +107,13 @@
"# Remove the suffix \"_1\"\n", "# Remove the suffix \"_1\"\n",
"duplicate_columns = [x[:-2] for x in merged_df.columns if \"_1\" in x]\n", "duplicate_columns = [x[:-2] for x in merged_df.columns if \"_1\" in x]\n",
"\n", "\n",
"columns_to_exclude_from_duplicates_check = [\n", "columns_to_exclude_from_duplicates_check = [\"Total threshold criteria exceeded\"]\n",
" \"Total threshold criteria exceeded\"\n",
"]\n",
"\n", "\n",
"columns_to_check = [column for column in duplicate_columns if column not in columns_to_exclude_from_duplicates_check]\n", "columns_to_check = [\n",
" column\n",
" for column in duplicate_columns\n",
" if column not in columns_to_exclude_from_duplicates_check\n",
"]\n",
"\n", "\n",
"any_errors_found = False\n", "any_errors_found = False\n",
"for column_to_check in columns_to_check:\n", "for column_to_check in columns_to_check:\n",

6
data/data-pipeline/data_pipeline/score/field_names.py
View file

@ -296,6 +296,8 @@ LOW_MEDIAN_INCOME_LOW_HS_EDUCATION_FIELD = (
f"percent of area median income and has low HS education" f"percent of area median income and has low HS education"
) )
LOW_HS_EDUCATION_FIELD = "Low high school education"
# Workforce for island areas # Workforce for island areas
ISLAND_AREAS_SUFFIX = " in 2009 (island areas)" ISLAND_AREAS_SUFFIX = " in 2009 (island areas)"
ISLAND_AREAS_UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD = ( ISLAND_AREAS_UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD = (
@ -314,6 +316,10 @@ ISLAND_AREAS_LOW_MEDIAN_INCOME_LOW_HS_EDUCATION_FIELD = (
f" and has low HS education{ISLAND_AREAS_SUFFIX}" f" and has low HS education{ISLAND_AREAS_SUFFIX}"
) )
ISLAND_AREAS_LOW_HS_EDUCATION_FIELD = (
f"Low high school education{ISLAND_AREAS_SUFFIX}"
)
# Not currently used in a factor # Not currently used in a factor
EXTREME_HEAT_MEDIAN_HOUSE_VALUE_LOW_INCOME_FIELD = ( EXTREME_HEAT_MEDIAN_HOUSE_VALUE_LOW_INCOME_FIELD = (
f"At or above the {PERCENTILE}th percentile for summer days above 90F and " f"At or above the {PERCENTILE}th percentile for summer days above 90F and "

25
data/data-pipeline/data_pipeline/score/score_l.py
View file

@ -162,9 +162,9 @@ class ScoreL(Score):
non_workforce_factors non_workforce_factors
].any(axis=1) ].any(axis=1)
self.df[field_names.SCORE_L + field_names.PERCENTILE_FIELD_SUFFIX] = self.df[ self.df[
field_names.SCORE_L_COMMUNITIES field_names.SCORE_L + field_names.PERCENTILE_FIELD_SUFFIX
].astype(int) ] = self.df[field_names.SCORE_L_COMMUNITIES].astype(int)
return self.df return self.df
@ -523,7 +523,7 @@ class ScoreL(Score):
field_names.LOW_MEDIAN_INCOME_LOW_HS_EDUCATION_FIELD, field_names.LOW_MEDIAN_INCOME_LOW_HS_EDUCATION_FIELD,
] ]
high_scool_achievement_rate_threshold = ( self.df[field_names.LOW_HS_EDUCATION_FIELD] = (
self.df[field_names.HIGH_SCHOOL_ED_FIELD] self.df[field_names.HIGH_SCHOOL_ED_FIELD]
>= self.LACK_OF_HIGH_SCHOOL_MINIMUM_THRESHOLD >= self.LACK_OF_HIGH_SCHOOL_MINIMUM_THRESHOLD
) )
@ -562,19 +562,20 @@ class ScoreL(Score):
self.df[field_names.LINGUISTIC_ISOLATION_LOW_HS_EDUCATION_FIELD] = ( self.df[field_names.LINGUISTIC_ISOLATION_LOW_HS_EDUCATION_FIELD] = (
linguistic_isolation_threshold linguistic_isolation_threshold
& high_scool_achievement_rate_threshold & self.df[field_names.LOW_HS_EDUCATION_FIELD]
) )
self.df[field_names.POVERTY_LOW_HS_EDUCATION_FIELD] = ( self.df[field_names.POVERTY_LOW_HS_EDUCATION_FIELD] = (
poverty_threshold & high_scool_achievement_rate_threshold poverty_threshold & self.df[field_names.LOW_HS_EDUCATION_FIELD]
) )
self.df[field_names.LOW_MEDIAN_INCOME_LOW_HS_EDUCATION_FIELD] = ( self.df[field_names.LOW_MEDIAN_INCOME_LOW_HS_EDUCATION_FIELD] = (
low_median_income_threshold & high_scool_achievement_rate_threshold low_median_income_threshold
& self.df[field_names.LOW_HS_EDUCATION_FIELD]
) )
self.df[field_names.UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD] = ( self.df[field_names.UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD] = (
unemployment_threshold & high_scool_achievement_rate_threshold unemployment_threshold & self.df[field_names.LOW_HS_EDUCATION_FIELD]
) )
workforce_combined_criteria_for_states = self.df[ workforce_combined_criteria_for_states = self.df[
@ -632,7 +633,7 @@ class ScoreL(Score):
>= self.ENVIRONMENTAL_BURDEN_THRESHOLD >= self.ENVIRONMENTAL_BURDEN_THRESHOLD
) )
island_areas_high_scool_achievement_rate_threshold = ( self.df[field_names.ISLAND_AREAS_LOW_HS_EDUCATION_FIELD] = (
self.df[field_names.CENSUS_DECENNIAL_HIGH_SCHOOL_ED_FIELD_2009] self.df[field_names.CENSUS_DECENNIAL_HIGH_SCHOOL_ED_FIELD_2009]
>= self.LACK_OF_HIGH_SCHOOL_MINIMUM_THRESHOLD >= self.LACK_OF_HIGH_SCHOOL_MINIMUM_THRESHOLD
) )
@ -641,12 +642,12 @@ class ScoreL(Score):
field_names.ISLAND_AREAS_UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD field_names.ISLAND_AREAS_UNEMPLOYMENT_LOW_HS_EDUCATION_FIELD
] = ( ] = (
self.df[island_areas_unemployment_criteria_field_name] self.df[island_areas_unemployment_criteria_field_name]
& island_areas_high_scool_achievement_rate_threshold & self.df[field_names.ISLAND_AREAS_LOW_HS_EDUCATION_FIELD]
) )
self.df[field_names.ISLAND_AREAS_POVERTY_LOW_HS_EDUCATION_FIELD] = ( self.df[field_names.ISLAND_AREAS_POVERTY_LOW_HS_EDUCATION_FIELD] = (
self.df[island_areas_poverty_criteria_field_name] self.df[island_areas_poverty_criteria_field_name]
& island_areas_high_scool_achievement_rate_threshold & self.df[field_names.ISLAND_AREAS_LOW_HS_EDUCATION_FIELD]
) )
self.df[ self.df[
@ -655,7 +656,7 @@ class ScoreL(Score):
self.df[ self.df[
island_areas_low_median_income_as_a_percent_of_ami_criteria_field_name island_areas_low_median_income_as_a_percent_of_ami_criteria_field_name
] ]
& island_areas_high_scool_achievement_rate_threshold & self.df[field_names.ISLAND_AREAS_LOW_HS_EDUCATION_FIELD]
) )
workforce_combined_criteria_for_island_areas = self.df[ workforce_combined_criteria_for_island_areas = self.df[