From 819f3ff47800eb9bce895c67d45c7e011af01e94 Mon Sep 17 00:00:00 2001 From: Shelby Switzer Date: Mon, 6 Dec 2021 13:17:17 -0500 Subject: [PATCH 1/5] Update etl constants to use score field_names and put strings around tract IDs in downloadable CSV (#985) * Update etl constants to use score field_names Put strings around tract IDs in downloadable CSV No need to modify the xls file creation because the string type is preserved and interpreted correctly in Excel already. One note is that this does cause the ID in the CSV to be have quotes around it, which might be annoying. Maybe we don't want this behavior? * Update based on PR feedback and lint needs * Change field we're using in downloadable This reverts the downloadable csv field list to use MEDIAN_INCOME_AS_PERCENT_OF_STATE_FIELD instead of MEDIAN_INCOME_AS_PERCENT_OF_AMI_FIELD in order to get the test to pass. The point of this PR is a refactor (and a small change to the CSV quotations), not to change the output. That will be a different PR later. Co-authored-by: Shelby Switzer --- .../data_pipeline/etl/score/constants.py | 167 +++++++++--------- .../data_pipeline/etl/score/etl_score_post.py | 9 +- .../data_pipeline/score/field_names.py | 8 +- .../data_pipeline/score/score_l.py | 2 +- 4 files changed, 101 insertions(+), 85 deletions(-) diff --git a/data/data-pipeline/data_pipeline/etl/score/constants.py b/data/data-pipeline/data_pipeline/etl/score/constants.py index 11a29b11..d3338a82 100644 --- a/data/data-pipeline/data_pipeline/etl/score/constants.py +++ b/data/data-pipeline/data_pipeline/etl/score/constants.py @@ -4,6 +4,8 @@ import datetime import pandas as pd from data_pipeline.config import settings +from data_pipeline.score import field_names + # Base Paths DATA_PATH = Path(settings.APP_ROOT) / "data" TMP_PATH = DATA_PATH / "tmp" @@ -59,88 +61,92 @@ SCORE_DOWNLOADABLE_ZIP_FILE_PATH = ( # Column subsets CENSUS_COUNTIES_COLUMNS = ["USPS", "GEOID", "NAME"] TILES_SCORE_COLUMNS = [ - "GEOID10_TRACT", - "State Name", - "County Name", - "Total population", - "Score D (percentile)", - "Score D (top 25th percentile)", - "Score E (percentile)", - "Score E (top 25th percentile)", - "Score G (communities)", - "Score G", - "Definition L (communities)", - "Definition L (percentile)", - "Poverty (Less than 200% of federal poverty line) (percentile)", - "Percent individuals age 25 or over with less than high school degree (percentile)", - "Linguistic isolation (percent) (percentile)", - "Unemployed civilians (percent) (percentile)", - "Housing burden (percent) (percentile)", - "Diagnosed diabetes among adults aged >=18 years (percentile)", - "Current asthma among adults aged >=18 years (percentile)", - "Coronary heart disease among adults aged >=18 years (percentile)", - "Life expectancy (years) (percentile)", - "Traffic proximity and volume (percentile)", - "FEMA Risk Index Expected Annual Loss Score (percentile)", - "Energy burden (percentile)", - "Wastewater discharge (percentile)", - "Percent pre-1960s housing (lead paint indicator) (percentile)", - "Diesel particulate matter (percentile)", - "Particulate matter (PM2.5) (percentile)", - "Median household income (% of AMI) (percentile)", - "Percent of individuals < 200% Federal Poverty Line (percentile)", + field_names.GEOID_TRACT_FIELD, + field_names.STATE_FIELD, + field_names.COUNTY_FIELD, + field_names.TOTAL_POP_FIELD, + field_names.SCORE_D + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.SCORE_D + field_names.TOP_25_PERCENTILE_SUFFIX, + field_names.SCORE_E + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.SCORE_E + field_names.TOP_25_PERCENTILE_SUFFIX, + field_names.SCORE_G_COMMUNITIES, + field_names.SCORE_G, + field_names.SCORE_L_COMMUNITIES, + field_names.SCORE_L + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.POVERTY_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.HIGH_SCHOOL_ED_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.LINGUISTIC_ISO_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.UNEMPLOYMENT_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.HOUSING_BURDEN_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.DIABETES_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.ASTHMA_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.HEART_DISEASE_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.LIFE_EXPECTANCY_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.TRAFFIC_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.FEMA_RISK_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.ENERGY_BURDEN_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.WASTEWATER_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.LEAD_PAINT_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.DIESEL_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.PM25_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.MEDIAN_INCOME_AS_PERCENT_OF_AMI_FIELD + + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.POVERTY_LESS_THAN_200_FPL_FIELD + + field_names.PERCENTILE_FIELD_SUFFIX, ] # columns to round floats to 2 decimals TILES_SCORE_FLOAT_COLUMNS = [ - "Score D (percentile)", - "Score D (top 25th percentile)", - "Score E (percentile)", - "Score E (top 25th percentile)", - "Definition L (percentile)", - "Poverty (Less than 200% of federal poverty line)", - "Percent individuals age 25 or over with less than high school degree", - "Linguistic isolation (percent)", - "Unemployed civilians (percent)", - "Housing burden (percent)", - "Poverty (Less than 200% of federal poverty line) (percentile)", - "Percent individuals age 25 or over with less than high school degree (percentile)", - "Linguistic isolation (percent) (percentile)", - "Unemployed civilians (percent) (percentile)", - "Housing burden (percent) (percentile)", - "Diagnosed diabetes among adults aged >=18 years (percentile)", - "Current asthma among adults aged >=18 years (percentile)", - "Coronary heart disease among adults aged >=18 years (percentile)", - "Life expectancy (years) (percentile)", - "Traffic proximity and volume (percentile)", - "FEMA Risk Index Expected Annual Loss Score (percentile)", - "Energy burden (percentile)", - "Wastewater discharge (percentile)", - "Percent pre-1960s housing (lead paint indicator) (percentile)", - "Diesel particulate matter (percentile)", - "Particulate matter (PM2.5) (percentile)", - "Median household income (% of AMI) (percentile)", - "Percent of individuals < 200% Federal Poverty Line (percentile)", + field_names.SCORE_D + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.SCORE_D + field_names.TOP_25_PERCENTILE_SUFFIX, + field_names.SCORE_E + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.SCORE_E + field_names.TOP_25_PERCENTILE_SUFFIX, + field_names.SCORE_L + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.POVERTY_FIELD, + field_names.HIGH_SCHOOL_ED_FIELD, + field_names.LINGUISTIC_ISO_FIELD, + field_names.UNEMPLOYMENT_FIELD, + field_names.HOUSING_BURDEN_FIELD, + field_names.POVERTY_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.HIGH_SCHOOL_ED_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.LINGUISTIC_ISO_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.UNEMPLOYMENT_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.HOUSING_BURDEN_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.DIABETES_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.ASTHMA_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.HEART_DISEASE_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.LIFE_EXPECTANCY_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.TRAFFIC_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.FEMA_RISK_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.ENERGY_BURDEN_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.WASTEWATER_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.LEAD_PAINT_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.DIESEL_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.PM25_FIELD + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.MEDIAN_INCOME_AS_PERCENT_OF_AMI_FIELD + + field_names.PERCENTILE_FIELD_SUFFIX, + field_names.POVERTY_LESS_THAN_200_FPL_FIELD + + field_names.PERCENTILE_FIELD_SUFFIX, ] TILES_ROUND_NUM_DECIMALS = 2 DOWNLOADABLE_SCORE_INDICATOR_COLUMNS_BASIC = [ - "Area Median Income (State or metropolitan)", - "Percent of individuals < 100% Federal Poverty Line", - "Percent individuals age 25 or over with less than high school degree", - "Diagnosed diabetes among adults aged >=18 years", - "Current asthma among adults aged >=18 years", - "Coronary heart disease among adults aged >=18 years", - "Life expectancy (years)", - "Traffic proximity and volume", - "FEMA Risk Index Expected Annual Loss Score", - "Energy burden", - "Housing burden (percent)", - "Wastewater discharge", - "Percent pre-1960s housing (lead paint indicator)", - "Diesel particulate matter", - "Particulate matter (PM2.5)", - "Total population", + field_names.AMI_FIELD, + field_names.POVERTY_LESS_THAN_100_FPL_FIELD, + field_names.HIGH_SCHOOL_ED_FIELD, + field_names.DIABETES_FIELD, + field_names.ASTHMA_FIELD, + field_names.HEART_DISEASE_FIELD, + field_names.LIFE_EXPECTANCY_FIELD, + field_names.TRAFFIC_FIELD, + field_names.FEMA_RISK_FIELD, + field_names.ENERGY_BURDEN_FIELD, + field_names.HOUSING_BURDEN_FIELD, + field_names.WASTEWATER_FIELD, + field_names.LEAD_PAINT_FIELD, + field_names.DIESEL_FIELD, + field_names.PM25_FIELD, + field_names.TOTAL_POP_FIELD, ] # For every indicator above, we want to include percentile and min-max normalized variants also @@ -155,11 +161,12 @@ DOWNLOADABLE_SCORE_INDICATOR_COLUMNS_FULL = list( # Finally we augment with the GEOID10, county, and state DOWNLOADABLE_SCORE_COLUMNS = [ - "GEOID10_TRACT", - "County Name", - "State Name", - "Score G (communities)", - "Median household income (% of AMI)", - "Median household income (% of state median household income) (percentile)", + field_names.GEOID_TRACT_FIELD, + field_names.COUNTY_FIELD, + field_names.STATE_FIELD, + field_names.SCORE_G_COMMUNITIES, + field_names.MEDIAN_INCOME_AS_PERCENT_OF_AMI_FIELD, + field_names.MEDIAN_INCOME_AS_PERCENT_OF_STATE_FIELD + + field_names.PERCENTILE_FIELD_SUFFIX, *DOWNLOADABLE_SCORE_INDICATOR_COLUMNS_FULL, ] diff --git a/data/data-pipeline/data_pipeline/etl/score/etl_score_post.py b/data/data-pipeline/data_pipeline/etl/score/etl_score_post.py index 585595cf..4723296c 100644 --- a/data/data-pipeline/data_pipeline/etl/score/etl_score_post.py +++ b/data/data-pipeline/data_pipeline/etl/score/etl_score_post.py @@ -276,12 +276,15 @@ class PostScoreETL(ExtractTransformLoad): inplace=False, ) - logger.info("Writing downloadable csv") - downloadable_df_copy.to_csv(csv_path, index=False) - logger.info("Writing downloadable excel") downloadable_df_copy.to_excel(excel_path, index=False) + logger.info("Writing downloadable csv") + downloadable_df_copy[self.GEOID_TRACT_FIELD_NAME] = ( + '"' + downloadable_df_copy[self.GEOID_TRACT_FIELD_NAME] + '"' + ) + downloadable_df_copy.to_csv(csv_path, index=False) + logger.info("Compressing files") files_to_compress = [csv_path, excel_path, pdf_path] zip_files(zip_path, files_to_compress) diff --git a/data/data-pipeline/data_pipeline/score/field_names.py b/data/data-pipeline/data_pipeline/score/field_names.py index 39b6b7f9..1bbcb37b 100644 --- a/data/data-pipeline/data_pipeline/score/field_names.py +++ b/data/data-pipeline/data_pipeline/score/field_names.py @@ -1,6 +1,12 @@ # Suffixes PERCENTILE_FIELD_SUFFIX = " (percentile)" MIN_MAX_FIELD_SUFFIX = " (min-max normalized)" +TOP_25_PERCENTILE_SUFFIX = " (top 25th percentile)" + +# Geographic field names +GEOID_TRACT_FIELD = "GEOID10_TRACT" +STATE_FIELD = "State Name" +COUNTY_FIELD = "County Name" # Score file field names SCORE_A = "Score A" @@ -21,6 +27,7 @@ SCORE_I = "Score I" SCORE_I_COMMUNITIES = "Score I (communities)" SCORE_K = "NMTC (communities)" SCORE_K_COMMUNITIES = "Score K (communities)" +SCORE_L = "Definition L" SCORE_L_COMMUNITIES = "Definition L (communities)" L_CLIMATE = "Climate Factor (Definition L)" L_ENERGY = "Energy Factor (Definition L)" @@ -45,7 +52,6 @@ POVERTY_LESS_THAN_150_FPL_FIELD = ( POVERTY_LESS_THAN_100_FPL_FIELD = ( "Percent of individuals < 100% Federal Poverty Line" ) -MEDIAN_INCOME_PERCENT_AMI_FIELD = "Median household income (% of AMI)" STATE_MEDIAN_INCOME_FIELD = ( "Median household income (State; 2019 inflation-adjusted dollars)" ) diff --git a/data/data-pipeline/data_pipeline/score/score_l.py b/data/data-pipeline/data_pipeline/score/score_l.py index d745f5b9..35c56a65 100644 --- a/data/data-pipeline/data_pipeline/score/score_l.py +++ b/data/data-pipeline/data_pipeline/score/score_l.py @@ -528,7 +528,7 @@ class ScoreL(Score): median_income_threshold = ( self.df[ - field_names.MEDIAN_INCOME_PERCENT_AMI_FIELD + field_names.MEDIAN_INCOME_AS_PERCENT_OF_AMI_FIELD + field_names.PERCENTILE_FIELD_SUFFIX ] # Note: a high median income as a % of AMI is good, so take 1 minus the threshold to invert it. From df564658a5aa9d475b4197694f924fd9e01388d8 Mon Sep 17 00:00:00 2001 From: Vim <86254807+vim-usds@users.noreply.github.com> Date: Mon, 6 Dec 2021 22:58:04 -0500 Subject: [PATCH 2/5] Add 4 additional territory buttons (#956) * Add 4 additional territory buttons * Fix bug where territories fires multiple times - move territory handler from J40Map to component * Update SVGs for all territory buttons --- client/src/components/J40Map.tsx | 27 +------ .../src/components/territoryFocusControl.tsx | 72 ++++++++++++++++--- client/src/data/constants.tsx | 5 ++ client/src/data/copy/explore.tsx | 56 ++++++++++++--- client/src/images/mapbtn-48.svg | 22 +++++- client/src/images/mapbtn-AK.svg | 15 +++- client/src/images/mapbtn-AS.svg | 19 +++++ client/src/images/mapbtn-GU.svg | 18 +++++ client/src/images/mapbtn-HI.svg | 14 +++- client/src/images/mapbtn-MP.svg | 17 +++++ client/src/images/mapbtn-PR.svg | 18 ++++- client/src/images/mapbtn-VI.svg | 14 ++++ client/src/styles/global.scss | 24 +++++++ 13 files changed, 274 insertions(+), 47 deletions(-) create mode 100644 client/src/images/mapbtn-AS.svg create mode 100644 client/src/images/mapbtn-GU.svg create mode 100644 client/src/images/mapbtn-MP.svg create mode 100644 client/src/images/mapbtn-VI.svg diff --git a/client/src/components/J40Map.tsx b/client/src/components/J40Map.tsx index ef0cb021..1d63ee39 100644 --- a/client/src/components/J40Map.tsx +++ b/client/src/components/J40Map.tsx @@ -1,6 +1,6 @@ /* eslint-disable no-unused-vars */ // External Libs: -import React, {MouseEvent, useRef, useState, useMemo} from 'react'; +import React, {useRef, useState, useMemo} from 'react'; import {Map, MapboxGeoJSONFeature, LngLatBoundsLike} from 'maplibre-gl'; import ReactMapGL, { MapEvent, @@ -136,29 +136,6 @@ const J40Map = ({location}: IJ40Interface) => { }); }; - const onClickTerritoryFocusButton = (event: MouseEvent) => { - event.stopPropagation(); - const buttonID = event.target && (event.target as HTMLElement).id; - - switch (buttonID) { - case '48': - goToPlace(constants.LOWER_48_BOUNDS); - break; - case 'AK': - goToPlace(constants.ALASKA_BOUNDS); - break; - case 'HI': - goToPlace(constants.HAWAII_BOUNDS); - break; - case 'PR': - goToPlace(constants.PUERTO_RICO_BOUNDS); - break; - - default: - break; - } - }; - const onTransitionStart = () => { setTransitionInProgress(true); }; @@ -273,7 +250,7 @@ const J40Map = ({location}: IJ40Interface) => { onClick={onClickGeolocate} /> : ''} {geolocationInProgress ?
Geolocation in progress...
: ''} - + {'fs' in flags ? :'' } diff --git a/client/src/components/territoryFocusControl.tsx b/client/src/components/territoryFocusControl.tsx index 393462a3..0ec073b0 100644 --- a/client/src/components/territoryFocusControl.tsx +++ b/client/src/components/territoryFocusControl.tsx @@ -1,21 +1,53 @@ import {useIntl} from 'gatsby-plugin-intl'; -import React, {MouseEventHandler} from 'react'; -import {_useMapControl as useMapControl} from 'react-map-gl'; +import React from 'react'; +import {LngLatBoundsLike} from 'mapbox-gl'; import * as styles from './territoryFocusControl.module.scss'; import * as EXPLORE_COPY from '../data/copy/explore'; +import * as constants from '../data/constants'; interface ITerritoryFocusControl { - onClickTerritoryFocusButton: MouseEventHandler; + goToPlace(bounds: LngLatBoundsLike): void; } -const TerritoryFocusControl = ({onClickTerritoryFocusButton}: ITerritoryFocusControl) => { + +const TerritoryFocusControl = ({goToPlace}: ITerritoryFocusControl) => { const intl = useIntl(); - const {containerRef} = useMapControl({ - // @ts-ignore // Types have not caught up yet, see https://github.com/visgl/react-map-gl/issues/1492 - onClick: onClickTerritoryFocusButton, - }); + const onClickTerritoryFocusButton = (event: React.MouseEvent) => { + event.stopPropagation(); + const buttonID = event.target && (event.target as HTMLElement).id; + + switch (buttonID) { + case '48': + goToPlace(constants.LOWER_48_BOUNDS); + break; + case 'AK': + goToPlace(constants.ALASKA_BOUNDS); + break; + case 'HI': + goToPlace(constants.HAWAII_BOUNDS); + break; + case 'PR': + goToPlace(constants.PUERTO_RICO_BOUNDS); + break; + case 'GU': + goToPlace(constants.GUAM_BOUNDS); + break; + case 'AS': + goToPlace(constants.AMERICAN_SAMOA_BOUNDS); + break; + case 'MP': + goToPlace(constants.MARIANA_ISLAND_BOUNDS); + break; + case 'VI': + goToPlace(constants.US_VIRGIN_ISLANDS_BOUNDS); + break; + + default: + break; + } + }; const territories = [ { @@ -34,6 +66,22 @@ const TerritoryFocusControl = ({onClickTerritoryFocusButton}: ITerritoryFocusCon short: intl.formatMessage(EXPLORE_COPY.MAP.PR_SHORT), long: intl.formatMessage(EXPLORE_COPY.MAP.PR_LONG), }, + { + short: intl.formatMessage(EXPLORE_COPY.MAP.GU_SHORT), + long: intl.formatMessage(EXPLORE_COPY.MAP.GU_LONG), + }, + { + short: intl.formatMessage(EXPLORE_COPY.MAP.AS_SHORT), + long: intl.formatMessage(EXPLORE_COPY.MAP.AS_LONG), + }, + { + short: intl.formatMessage(EXPLORE_COPY.MAP.MP_SHORT), + long: intl.formatMessage(EXPLORE_COPY.MAP.MP_LONG), + }, + { + short: intl.formatMessage(EXPLORE_COPY.MAP.VI_SHORT), + long: intl.formatMessage(EXPLORE_COPY.MAP.VI_LONG), + }, ]; // the offset for this array should map the territories variable const territoriesIconClassName = [ @@ -41,16 +89,20 @@ const TerritoryFocusControl = ({onClickTerritoryFocusButton}: ITerritoryFocusCon 'mapboxgl-ctrl-zoom-to-ak', 'mapboxgl-ctrl-zoom-to-hi', 'mapboxgl-ctrl-zoom-to-pr', + 'mapboxgl-ctrl-zoom-to-gu', + 'mapboxgl-ctrl-zoom-to-as', + 'mapboxgl-ctrl-zoom-to-mp', + 'mapboxgl-ctrl-zoom-to-vi', ]; return ( -
+
{territories.map((territory, index) =>
@@ -156,11 +154,9 @@ exports[`rendering of the AreaDetail checks if various text fields are visible 1 Diabetes
- NaN + N/A - - th - +
@@ -178,11 +174,9 @@ exports[`rendering of the AreaDetail checks if various text fields are visible 1 Diesel particulate matter
- NaN + N/A - - th - +
@@ -200,11 +194,9 @@ exports[`rendering of the AreaDetail checks if various text fields are visible 1 Energy burden
- NaN + N/A - - th - +
@@ -222,11 +214,9 @@ exports[`rendering of the AreaDetail checks if various text fields are visible 1 FEMA Risk Index
- NaN + N/A - - th - +
@@ -244,11 +234,9 @@ exports[`rendering of the AreaDetail checks if various text fields are visible 1 Heart disease
- NaN + N/A - - th - +
@@ -288,11 +276,9 @@ exports[`rendering of the AreaDetail checks if various text fields are visible 1 Lead paint
- NaN + N/A - - th - +
@@ -310,11 +296,9 @@ exports[`rendering of the AreaDetail checks if various text fields are visible 1 Life expectancy
- NaN + N/A - - th - +
@@ -332,11 +316,9 @@ exports[`rendering of the AreaDetail checks if various text fields are visible 1 PM2.5
- NaN + N/A - - th - +
@@ -354,11 +336,9 @@ exports[`rendering of the AreaDetail checks if various text fields are visible 1 Traffic proximity and volume
- NaN + N/A - - th - +
@@ -376,11 +356,9 @@ exports[`rendering of the AreaDetail checks if various text fields are visible 1 Wastewater discharge
- NaN + N/A - - th - +