publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-07-29 03:31:17 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into esfoobar-usds/1062-implement-changes-export-files

Browse source
This commit is contained in:
Jorge Escobar 2022-01-03 15:53:41 -05:00
commit 006493ab24
18 changed files with 577 additions and 172 deletions
Download patch file Download diff file Expand all files Collapse all files

96
client/cypress/integration/LegacyTests/mapZoomLatLong.spec.js
View file

@ -9,53 +9,53 @@ describe('Does the map zoom and adjust to lat/long correctly?', () => {
cy.get('.mapboxgl-ctrl-icon.mapboxgl-ctrl-zoom-in').click({force: true}); cy.get('.mapboxgl-ctrl-icon.mapboxgl-ctrl-zoom-in').click({force: true});
cy.url().should('include', '#4'); cy.url().should('include', '#4');
}); });
// it('should show the correct lat/lng coordinates in the URL', it('should show the correct lat/lng coordinates in the URL',
// { {
// retries: { retries: {
// runMode: 3, runMode: 3,
// openMode: 3, openMode: 3,
// }, },
// defaultCommandTimeout: 4000, defaultCommandTimeout: 4000,
// execTimeout: 10000, execTimeout: 10000,
// taskTimeout: 10000, taskTimeout: 10000,
// pageLoadTimeout: 10000, pageLoadTimeout: 10000,
// requestTimeout: 5000, requestTimeout: 5000,
// responseTimeout: 10000, responseTimeout: 10000,
// }, },
// () => { () => {
// cy.getMap().then((map) => { cy.getMap().then((map) => {
// cy.panTo(map, [-77.9, 35.04]); cy.panTo(map, [-77.9, 35.04]);
// cy.url().should('include', '#4/35.04/-77.9'); cy.url().should('include', '#4/35.04/-77.9');
// }); });
// }); });
// This test hangs intermittently (30% of the time) need to investigate why // This test hangs intermittently (30% of the time) need to investigate why
// it('allows user to specify alternative starting URL', it('allows user to specify alternative starting URL',
// { {
// retries: { retries: {
// runMode: 3, runMode: 3,
// openMode: 3, openMode: 3,
// }, },
// defaultCommandTimeout: 4000, defaultCommandTimeout: 4000,
// execTimeout: 10000, execTimeout: 10000,
// taskTimeout: 10000, taskTimeout: 10000,
// pageLoadTimeout: 10000, pageLoadTimeout: 10000,
// requestTimeout: 5000, requestTimeout: 5000,
// responseTimeout: 10000, responseTimeout: 10000,
// }, },
// () => { () => {
// const [expectedZoom, expectedLat, expectedLng] = [12.05, 41.40965, -75.65978]; const [expectedZoom, expectedLat, expectedLng] = [12.05, 41.40965, -75.65978];
// const expectedURL = `http://localhost:8000/en/cejst/#${expectedZoom}/${expectedLat}/${expectedLng}`; const expectedURL = `http://localhost:8000/en/cejst/#${expectedZoom}/${expectedLat}/${expectedLng}`;
// cy.visit(expectedURL); cy.visit(expectedURL);
// cy.getMap().then((map) => { cy.getMap().then((map) => {
// cy.waitForMapIdle(map); cy.waitForMapIdle(map);
// cy.url().should('equal', expectedURL); cy.url().should('equal', expectedURL);
// const actualZoom = map.getZoom(); const actualZoom = map.getZoom();
// const actualCenter = map.getCenter(); const actualCenter = map.getCenter();
// expect(actualCenter.lat).to.eq(expectedLat); expect(actualCenter.lat).to.eq(expectedLat);
// expect(actualCenter.lng).to.eq(expectedLng); expect(actualCenter.lng).to.eq(expectedLng);
// expect(actualZoom).to.eq(expectedZoom); expect(actualZoom).to.eq(expectedZoom);
// }); });
// }); });
}); });

32
client/cypress/integration/LegacyTests/mapZoomTerritories.spec.js
View file

@ -1,6 +1,19 @@
// / <reference types="Cypress" /> // / <reference types="Cypress" />
describe('Will it zoom into territories correctly?', () => { describe('Will it zoom into territories correctly?',
{
retries: {
runMode: 3,
openMode: 3,
},
defaultCommandTimeout: 4000,
execTimeout: 10000,
taskTimeout: 10000,
pageLoadTimeout: 10000,
requestTimeout: 5000,
responseTimeout: 10000,
},
() => {
beforeEach(() => { beforeEach(() => {
cy.viewport('macbook-13'); cy.viewport('macbook-13');
cy.visit('http://localhost:8000/en/cejst'); cy.visit('http://localhost:8000/en/cejst');
@ -11,22 +24,25 @@ describe('Will it zoom into territories correctly?', () => {
// Removing z as each tests has variance on it's value // Removing z as each tests has variance on it's value
const tests = { const tests = {
'Lower 48': '/38.07/-95.87', 'Lower 48': '3/33.47/-97.5',
// Todo: Understand what causes these two to hang intermittently ticket #579 // Todo: Understand what causes these two to hang intermittently ticket #579
// 'Puerto Rico': '/18.2/-66.583', // 'Puerto Rico': '7.71/18.2/-66.583',
// 'Alaska': '/63.28/-162.39', // 'Alaska': '3/63.28/-162.39',
// 'Hawaii': '5.35/20.574/-161.438', // 'American Samoa': '6.55/-13.804/-171.117',
// 'Commonwealth of Northern Mariana Islands': '5.98/16.901/145.472',
// 'Hawaii': '5.73/20.657/-157.697',
}; };
for (let i=0; i<1; i++) {
for (const [territory, xy] of Object.entries(tests)) { for (const [territory, zxy] of Object.entries(tests)) {
it(`Can focus on ${territory} `, () => { it(`Can focus on ${territory} `, () => {
cy.getMap().then((map) => { cy.getMap().then((map) => {
cy.get(`[aria-label="Focus on ${territory}"]`).click(); cy.get(`[aria-label="Focus on ${territory}"]`).click();
cy.waitForMapIdle(map); cy.waitForMapIdle(map);
cy.log(cy.url()); cy.log(cy.url());
cy.url().should('include', xy); cy.url().should('include', zxy);
}); });
}); });
}; };
}
}); });

10
client/src/components/AreaDetail/index.tsx
View file

@ -28,6 +28,7 @@ export interface indicatorInfo {
const AreaDetail = ({properties}:IAreaDetailProps) => { const AreaDetail = ({properties}:IAreaDetailProps) => {
const intl = useIntl(); const intl = useIntl();
// console.log the properties of the census that is selected:
console.log("Area Detail properies: ", properties); console.log("Area Detail properies: ", properties);
const score = properties[constants.SCORE_PROPERTY_HIGH] ? properties[constants.SCORE_PROPERTY_HIGH] as number : 0; const score = properties[constants.SCORE_PROPERTY_HIGH] ? properties[constants.SCORE_PROPERTY_HIGH] as number : 0;
@ -38,10 +39,6 @@ const AreaDetail = ({properties}:IAreaDetailProps) => {
const isCommunityFocus = score >= constants.SCORE_BOUNDARY_PRIORITIZED; const isCommunityFocus = score >= constants.SCORE_BOUNDARY_PRIORITIZED;
// const sidePanelFeedbackHref = `
// mailto:screeningtool.feedback@usds.gov?subject=Feedback on Census Tract: ${blockGroup}
// `;
// Define each indicator in the side panel with constants from copy file (for intl) // Define each indicator in the side panel with constants from copy file (for intl)
// Indicators are grouped by category // Indicators are grouped by category
const expAgLoss:indicatorInfo = { const expAgLoss:indicatorInfo = {
@ -382,8 +379,9 @@ const AreaDetail = ({properties}:IAreaDetailProps) => {
totalCount: constants.TOTAL_NUMBER_OF_INDICATORS, totalCount: constants.TOTAL_NUMBER_OF_INDICATORS,
}}/> }}/>
</div> </div>
{/* eslint-disable-next-line max-len */} {/* <a className={styles.feedbackLink} href={sidePanelFeedbackHref}>
{/* <a className={styles.feedbackLink} href={sidePanelFeedbackHref}>{EXPLORE_COPY.COMMUNITY.SEND_FEEDBACK}</a> */} {EXPLORE_COPY.COMMUNITY.SEND_FEEDBACK}
</a> */}
</div> </div>

91
client/src/components/J40Map.tsx
View file

@ -1,3 +1,4 @@
/* eslint-disable valid-jsdoc */
/* eslint-disable no-unused-vars */ /* eslint-disable no-unused-vars */
// External Libs: // External Libs:
import React, {useRef, useState, useMemo} from 'react'; import React, {useRef, useState, useMemo} from 'react';
@ -55,12 +56,23 @@ export interface IDetailViewInterface {
const J40Map = ({location}: IJ40Interface) => { const J40Map = ({location}: IJ40Interface) => {
// Hash portion of URL is of the form #zoom/lat/lng /**
* Initializes the zoom, and the map's center point (lat, lng) via the URL hash #{z}/{lat}/{long}
* where:
* z = zoom
* lat = map center's latitude
* long = map center's longitude
*/
const [zoom, lat, lng] = location.hash.slice(1).split('/'); const [zoom, lat, lng] = location.hash.slice(1).split('/');
/**
* If the URL has no #{z}/{lat}/{long} specified in the hash, then set the map's intial viewport state
* to use constants. This is so that we can load URLs with certain zoom/lat/long specified:
*/
const [viewport, setViewport] = useState<ViewportProps>({ const [viewport, setViewport] = useState<ViewportProps>({
latitude: lat && parseFloat(lat) || constants.DEFAULT_CENTER[0], latitude: lat && parseFloat(lat) ? parseFloat(lat) : constants.DEFAULT_CENTER[0],
longitude: lng && parseFloat(lng) || constants.DEFAULT_CENTER[1], longitude: lng && parseFloat(lng) ? parseFloat(lng) : constants.DEFAULT_CENTER[1],
zoom: zoom && parseFloat(zoom) || constants.GLOBAL_MIN_ZOOM, zoom: zoom && parseFloat(zoom) ? parseFloat(zoom) : constants.GLOBAL_MIN_ZOOM,
}); });
const [selectedFeature, setSelectedFeature] = useState<MapboxGeoJSONFeature>(); const [selectedFeature, setSelectedFeature] = useState<MapboxGeoJSONFeature>();
@ -76,8 +88,73 @@ const J40Map = ({location}: IJ40Interface) => {
const selectedFeatureId = (selectedFeature && selectedFeature.id) || ''; const selectedFeatureId = (selectedFeature && selectedFeature.id) || '';
const filter = useMemo(() => ['in', constants.GEOID_PROPERTY, selectedFeatureId], [selectedFeature]); const filter = useMemo(() => ['in', constants.GEOID_PROPERTY, selectedFeatureId], [selectedFeature]);
const onClick = (event: MapEvent) => {
/**
* This function will return the bounding box of the current map. Comment in when needed.
* {
* _ne: {lng:number, lat:number}
* _sw: {lng:number, lat:number}
* }
* @returns {LngLatBounds}
*/
const getCurrentMapBoundingBox = () => {
return mapRef.current ? console.log('mapRef getBounds(): ', mapRef.current.getMap().getBounds()) : null;
};
/**
* This onClick event handler will listen and handle clicks on the map. It will listen for clicks on the
* territory controls and it will listen to clicks on the map.
*
* It will NOT listen to clicks into the search field or the zoom controls. These clickHandlers are
* captured in their own respective components.
*/
const onClick = (event: MapEvent | React.MouseEvent<HTMLButtonElement>) => {
// Stop all propagation / bubbling / capturing
event.preventDefault();
event.stopPropagation();
getCurrentMapBoundingBox();
// Check if the click is for territories. Given the territories component's design, it can be
// guaranteed that each territory control will have an id. We use this ID to determine
// if the click is coming from a territory control
if (event.target && (event.target as HTMLElement).id) {
const buttonID = event.target && (event.target as HTMLElement).id;
switch (buttonID) {
case '48':
goToPlace(constants.LOWER_48_BOUNDS);
break;
case 'AK':
goToPlace(constants.ALASKA_BOUNDS);
break;
case 'HI':
goToPlace(constants.HAWAII_BOUNDS);
break;
case 'PR':
goToPlace(constants.PUERTO_RICO_BOUNDS);
break;
case 'GU':
goToPlace(constants.GUAM_BOUNDS);
break;
case 'AS':
goToPlace(constants.AMERICAN_SAMOA_BOUNDS);
break;
case 'MP':
goToPlace(constants.MARIANA_ISLAND_BOUNDS);
break;
case 'VI':
goToPlace(constants.US_VIRGIN_ISLANDS_BOUNDS);
break;
default:
break;
}
} else {
// This else clause will fire when the ID is null or empty. This is the case where the map is clicked
const feature = event.features && event.features[0]; const feature = event.features && event.features[0];
console.log(feature);
if (feature) { if (feature) {
const [minLng, minLat, maxLng, maxLat] = bbox(feature); const [minLng, minLat, maxLng, maxLat] = bbox(feature);
const newViewPort = new WebMercatorViewport({height: viewport.height!, width: viewport.width!}); const newViewPort = new WebMercatorViewport({height: viewport.height!, width: viewport.width!});
@ -92,7 +169,6 @@ const J40Map = ({location}: IJ40Interface) => {
); );
if (feature.id !== selectedFeatureId) { if (feature.id !== selectedFeatureId) {
setSelectedFeature(feature); setSelectedFeature(feature);
console.log(feature.properties);
} else { } else {
setSelectedFeature(undefined); setSelectedFeature(undefined);
} }
@ -108,6 +184,7 @@ const J40Map = ({location}: IJ40Interface) => {
]); ]);
setDetailViewData(popupInfo); setDetailViewData(popupInfo);
} }
}
}; };
const onLoad = () => { const onLoad = () => {
@ -251,7 +328,7 @@ const J40Map = ({location}: IJ40Interface) => {
onClick={onClickGeolocate} onClick={onClickGeolocate}
/> : ''} /> : ''}
{geolocationInProgress ? <div>Geolocation in progress...</div> : ''} {geolocationInProgress ? <div>Geolocation in progress...</div> : ''}
<TerritoryFocusControl goToPlace={goToPlace}/> <TerritoryFocusControl onClick={onClick}/>
{'fs' in flags ? <FullscreenControl className={styles.fullscreenControl}/> :'' } {'fs' in flags ? <FullscreenControl className={styles.fullscreenControl}/> :'' }
</ReactMapGL> </ReactMapGL>

45
client/src/components/territoryFocusControl.tsx
View file

@ -1,54 +1,18 @@
import {useIntl} from 'gatsby-plugin-intl'; import {useIntl} from 'gatsby-plugin-intl';
import React from 'react'; import React from 'react';
import {LngLatBoundsLike} from 'mapbox-gl'; import {MapEvent} from 'react-map-gl';
import * as styles from './territoryFocusControl.module.scss'; import * as styles from './territoryFocusControl.module.scss';
import * as EXPLORE_COPY from '../data/copy/explore'; import * as EXPLORE_COPY from '../data/copy/explore';
import * as constants from '../data/constants';
interface ITerritoryFocusControl { interface ITerritoryFocusControl {
goToPlace(bounds: LngLatBoundsLike): void; onClick(event: MapEvent | React.MouseEvent<HTMLButtonElement>):void;
} }
const TerritoryFocusControl = ({goToPlace}: ITerritoryFocusControl) => { const TerritoryFocusControl = ({onClick}: ITerritoryFocusControl) => {
const intl = useIntl(); const intl = useIntl();
const onClickTerritoryFocusButton = (event: React.MouseEvent<HTMLButtonElement>) => {
event.stopPropagation();
const buttonID = event.target && (event.target as HTMLElement).id;
switch (buttonID) {
case '48':
goToPlace(constants.LOWER_48_BOUNDS);
break;
case 'AK':
goToPlace(constants.ALASKA_BOUNDS);
break;
case 'HI':
goToPlace(constants.HAWAII_BOUNDS);
break;
case 'PR':
goToPlace(constants.PUERTO_RICO_BOUNDS);
break;
case 'GU':
goToPlace(constants.GUAM_BOUNDS);
break;
case 'AS':
goToPlace(constants.AMERICAN_SAMOA_BOUNDS);
break;
case 'MP':
goToPlace(constants.MARIANA_ISLAND_BOUNDS);
break;
case 'VI':
goToPlace(constants.US_VIRGIN_ISLANDS_BOUNDS);
break;
default:
break;
}
};
const territories = [ const territories = [
{ {
short: intl.formatMessage(EXPLORE_COPY.MAP.LOWER48_SHORT), short: intl.formatMessage(EXPLORE_COPY.MAP.LOWER48_SHORT),
@ -102,7 +66,8 @@ const TerritoryFocusControl = ({goToPlace}: ITerritoryFocusControl) => {
<button <button
id={territory.short} id={territory.short}
key={territory.short} key={territory.short}
onClick={(e) => onClickTerritoryFocusButton(e)} // onClickCapture={(e) => onClickTerritoryFocusButton(e)}
onClickCapture={(e) => onClick(e)}
className={'mapboxgl-ctrl-icon ' + territoriesIconClassName[index]} className={'mapboxgl-ctrl-icon ' + territoriesIconClassName[index]}
aria-label={intl.formatMessage( aria-label={intl.formatMessage(
{ {

16
client/src/data/constants.tsx
View file

@ -128,15 +128,15 @@ export const GLOBAL_MAX_ZOOM_HIGHLIGHT = 22;
export const GLOBAL_MIN_ZOOM_HIGH = 7; export const GLOBAL_MIN_ZOOM_HIGH = 7;
export const GLOBAL_MAX_ZOOM_HIGH = 11; export const GLOBAL_MAX_ZOOM_HIGH = 11;
// Bounds // Bounds - these bounds can be obtained by using the getCurrentMapBoundingBox() function in the map
export const GLOBAL_MAX_BOUNDS: LngLatBoundsLike = [ export const GLOBAL_MAX_BOUNDS: LngLatBoundsLike = [
[-180.118306, 5.499550], [-180.118306, 5.499550],
[-65.0, 83.162102], [-65.0, 83.162102],
]; ];
export const LOWER_48_BOUNDS: LngLatBoundsLike = [ export const LOWER_48_BOUNDS: LngLatBoundsLike = [
[-124.7844079, 24.7433195], [-134.943542, 1.301806],
[-66.9513812, 49.3457868], [-60.060729, 57.050462],
]; ];
export const ALASKA_BOUNDS: LngLatBoundsLike = [ export const ALASKA_BOUNDS: LngLatBoundsLike = [
@ -145,8 +145,8 @@ export const ALASKA_BOUNDS: LngLatBoundsLike = [
]; ];
export const HAWAII_BOUNDS: LngLatBoundsLike = [ export const HAWAII_BOUNDS: LngLatBoundsLike = [
[-168.118306, 18.748115], [-161.174534, 17.652170],
[-154.757881, 22.378413], [-154.218940, 23.603623],
]; ];
export const PUERTO_RICO_BOUNDS: LngLatBoundsLike = [ export const PUERTO_RICO_BOUNDS: LngLatBoundsLike = [
@ -165,8 +165,8 @@ export const MARIANA_ISLAND_BOUNDS: LngLatBoundsLike = [
]; ];
export const AMERICAN_SAMOA_BOUNDS: LngLatBoundsLike = [ export const AMERICAN_SAMOA_BOUNDS: LngLatBoundsLike = [
[-171.089874, -14.548699], [-172.589874, -15.548699],
[-168.1433, -11.046934], [-169.6433, -12.046934],
]; ];
export const US_VIRGIN_ISLANDS_BOUNDS: LngLatBoundsLike = [ export const US_VIRGIN_ISLANDS_BOUNDS: LngLatBoundsLike = [
@ -174,7 +174,7 @@ export const US_VIRGIN_ISLANDS_BOUNDS: LngLatBoundsLike = [
[-64.2704123, 18.7495796], [-64.2704123, 18.7495796],
]; ];
export const DEFAULT_CENTER = [32.4687126, -86.502136]; export const DEFAULT_CENTER = [33.4687126, -97.502136];
// Opacity // Opacity
export const DEFAULT_LAYER_OPACITY = 0.6; export const DEFAULT_LAYER_OPACITY = 0.6;

2
client/src/data/copy/explore.tsx
View file

@ -128,7 +128,7 @@ export const MAP = defineMessages({
}, },
AS_LONG: { AS_LONG: {
id: 'map.territoryFocus.american.samoa.long', id: 'map.territoryFocus.american.samoa.long',
defaultMessage: 'American Somoa', defaultMessage: 'American Samoa',
description: 'The full name indicating the bounds of American Somoa', description: 'The full name indicating the bounds of American Somoa',
}, },
MP_SHORT: { MP_SHORT: {

10
data/data-pipeline/data_pipeline/etl/constants.py
View file

@ -89,11 +89,21 @@ DATASET_LIST = [
"module_dir": "hud_recap", "module_dir": "hud_recap",
"class_name": "HudRecapETL", "class_name": "HudRecapETL",
}, },
{
"name": "energy_definition_alternative_draft",
"module_dir": "energy_definition_alternative_draft",
"class_name": "EnergyDefinitionAlternativeDraft",
},
{ {
"name": "tree_equity_score", "name": "tree_equity_score",
"module_dir": "tree_equity_score", "module_dir": "tree_equity_score",
"class_name": "TreeEquityScoreETL", "class_name": "TreeEquityScoreETL",
}, },
{
"name": "michigan_ejscreen",
"module_dir": "michigan_ejscreen",
"class_name": "MichiganEnviroScreenETL",
},
] ]
CENSUS_INFO = { CENSUS_INFO = {
"name": "census", "name": "census",

0
data/data-pipeline/data_pipeline/etl/sources/energy_definition_alternative_draft/README.md Normal file
View file

0
data/data-pipeline/data_pipeline/etl/sources/energy_definition_alternative_draft/__init__.py Normal file
View file

113
data/data-pipeline/data_pipeline/etl/sources/energy_definition_alternative_draft/etl.py Normal file
View file

@ -0,0 +1,113 @@
from pathlib import Path
import pandas as pd
from data_pipeline.config import settings
from data_pipeline.etl.base import ExtractTransformLoad
from data_pipeline.score import field_names
from data_pipeline.utils import get_module_logger, unzip_file_from_url
logger = get_module_logger(__name__)
class EnergyDefinitionAlternativeDraft(ExtractTransformLoad):
def __init__(self):
self.DEFINITION_ALTERNATIVE_FILE_URL = (
settings.AWS_JUSTICE40_DATASOURCES_URL
+ "/alternative DAC definition.csv.zip"
)
self.OUTPUT_PATH: Path = (
self.DATA_PATH / "dataset" / "energy_definition_alternative_draft"
)
self.TRACT_INPUT_COLUMN_NAME = "GEOID"
self.ALTERNATIVE_DEFINITION_INPUT_COLUMN_NAME = "J40_DAC"
# Constants for output
self.COLUMNS_TO_KEEP = [
self.GEOID_TRACT_FIELD_NAME,
field_names.ENERGY_RELATED_COMMUNITIES_DEFINITION_ALTERNATIVE,
field_names.COAL_EMPLOYMENT,
field_names.OUTAGE_EVENTS,
field_names.HOMELESSNESS,
field_names.DISABLED_POPULATION,
field_names.OUTAGE_DURATION,
field_names.JOB_ACCESS,
field_names.FOSSIL_ENERGY_EMPLOYMENT,
field_names.FOOD_DESERT,
field_names.INCOMPLETE_PLUMBING,
field_names.NON_GRID_CONNECTED_HEATING_FUEL,
field_names.PARKS,
field_names.GREATER_THAN_30_MIN_COMMUTE,
field_names.INTERNET_ACCESS,
field_names.MOBILE_HOME,
field_names.SINGLE_PARENT,
field_names.TRANSPORTATION_COSTS,
]
self.df: pd.DataFrame
def extract(self) -> None:
logger.info("Starting data download.")
unzip_file_from_url(
file_url=self.DEFINITION_ALTERNATIVE_FILE_URL,
download_path=self.TMP_PATH,
unzipped_file_path=self.TMP_PATH
/ "energy_definition_alternative_draft",
)
self.df = pd.read_csv(
filepath_or_buffer=self.TMP_PATH
/ "energy_definition_alternative_draft"
/ "J40 alternative DAC definition.csv",
# The following need to remain as strings for all of their digits, not get converted to numbers.
dtype={
self.TRACT_INPUT_COLUMN_NAME: "string",
},
low_memory=False,
)
def transform(self) -> None:
logger.info("Starting transforms.")
self.df = self.df.rename(
columns={
self.TRACT_INPUT_COLUMN_NAME: self.GEOID_TRACT_FIELD_NAME,
self.ALTERNATIVE_DEFINITION_INPUT_COLUMN_NAME: field_names.ENERGY_RELATED_COMMUNITIES_DEFINITION_ALTERNATIVE,
"Coal_Emp_Ratio": field_names.COAL_EMPLOYMENT,
"COUNT_Outage_Events": field_names.OUTAGE_EVENTS,
"den_hmls_pop": field_names.HOMELESSNESS,
"disability_pct": field_names.DISABLED_POPULATION,
"Duration_in_Minutes": field_names.OUTAGE_DURATION,
"emp_ovrll_ndx": field_names.JOB_ACCESS,
"FE_Emp_Ratio": field_names.FOSSIL_ENERGY_EMPLOYMENT,
"Food_LAhalfand10": field_names.FOOD_DESERT,
"incomplete_plumbing_pct": field_names.INCOMPLETE_PLUMBING,
"nongrid_heat_pct": field_names.NON_GRID_CONNECTED_HEATING_FUEL,
"num_parks": field_names.PARKS,
"Per_MoT_Dur_gte30": field_names.GREATER_THAN_30_MIN_COMMUTE,
"Per_NoInt": field_names.INTERNET_ACCESS,
"population_mobile_home_pct": field_names.MOBILE_HOME,
"single_parent_pct": field_names.SINGLE_PARENT,
"t_ami": field_names.TRANSPORTATION_COSTS,
}
)
# Convert to boolean:
self.df[field_names.ENERGY_RELATED_COMMUNITIES_DEFINITION_ALTERNATIVE] = \
self.df[field_names.ENERGY_RELATED_COMMUNITIES_DEFINITION_ALTERNATIVE
].astype('bool')
def validate(self) -> None:
logger.info("Validating data")
pass
def load(self) -> None:
logger.info("Saving CSV")
self.OUTPUT_PATH.mkdir(parents=True, exist_ok=True)
self.df[self.COLUMNS_TO_KEEP].to_csv(
path_or_buf=self.OUTPUT_PATH / "usa.csv", index=False
)

32
data/data-pipeline/data_pipeline/etl/sources/michigan_ejscreen/README.md Normal file
View file

@ -0,0 +1,32 @@
# Michigan EJSCREEN
The Michigan EJSCREEN description and publication can be found [here](https://deepblue.lib.umich.edu/bitstream/handle/2027.42/149105/AssessingtheStateofEnvironmentalJusticeinMichigan_344.pdf).
#### Some notes about the input source data column fields:
There are two pertinent columns used - `EJ_Score_Cal_Min` and `Pct_CalMin` that are referenced in the source codebase. To our knowledge, these columns reflect the adoption and the comparative quantitative analysis from two different approaches. The "Cal" prefix reflects CalEPA's CalEnviroScreen that omits racial and ethnic data. The "Min" abbreviation reflects Minnesota Pollution Control Agency’s (MPCA) approach to including this data. Please see pages 37 - 39 in the above reference for further details. Briefly, the authors adopted a combination of both the CalEnviroScreen's methodology and the MCPA's methodology. The scores and percentile rankings in the input data source sheet are the same as those reflected in the cited report, included in Appendix I and in the latest version of the mapping [tool](https://www.arcgis.com/apps/webappviewer/index.html?id=dc4f0647dda34959963488d3f519fd24).
#### Additional information on the adoption of the methodology from CalEnviroScreen and MCPA
Both CalEPA's CalEnviroScreen and the Minnesota Pollution Control Agency’s (MPCA) methodology are adopted and used for both comparative purposes and for the identification of areas of concern. The latter, in particular, is used to identify tribal areas. According to the authors, to make permitting decisions, MPCA assesses whether the community, measured at the census tract level, fits at least one of the following criteria:
* Percent of the non-white population is at least 50%
* "More than 40% of the households have a household income of less than 185% of the federal
poverty level (FPL)”
* If the facility is within the boundaries of a “tribal community” (MPCA 2015).
Furthermore, the authors state that the MCPA methodology included data on tribal community boundaries, as defined by the US Census Bureau, and data on poverty, race, and ethnicity. However, the authors also note that the MCPA's methodology does not rank any census tracts.
In addition, although the CalEPA does not analyze data on race and ethnicity in CalEnviroScreen, the researchers incorporated race and ethnicity data in their assessment of environmental justice in Michigan. To justify the incorporation of race and ethnic data, the team compared the tract rankings with and without the data.
A Spearman's rank-order correlation was calculated for the 2,741 census tracts within Michigan with the two variables being environmental justice scores using the CalEPA methodology 1) without racial and ethnic data and 2) with racial and ethnic data. These scores were then ranked and the Spearman rank-order correlation was calculated. These statistics are not included in the output of this ETL process. Please see Chapter 5 and Chapter 6 for further details.
Finally, please see pages 104 -106 for details on the justification and details for the applicability of the upper quartile as a means to identify communities in Michigan with the potential for environmental justice concerns. It should also be noted that, according to the authors, that CalEPA also designates the top 25% scoring tracts as “disadvantaged communities".
Sources:
* Minnesota Pollution Control Agency. (2015, December 15). Environmental Justice Framework Report.
Retrieved from https://www.pca.state.mn.us/sites/default/files/p-gen5-05.pdf.
* Faust, J., L. August, K. Bangia, V. Galaviz, J. Leichty, S. Prasad… and L. Zeise. (2017, January). Update to the California Communities Environmental Health Screening Tool CalEnviroScreen 3.0. Retrieved from OEHHA website: https://oehha.ca.gov/media/downloads/calenviroscreen/report/ces3report.pdf

0
data/data-pipeline/data_pipeline/etl/sources/michigan_ejscreen/__init__.py Normal file
View file

69
data/data-pipeline/data_pipeline/etl/sources/michigan_ejscreen/etl.py Normal file
View file

@ -0,0 +1,69 @@
import pandas as pd
from data_pipeline.etl.base import ExtractTransformLoad
from data_pipeline.utils import get_module_logger
from data_pipeline.score import field_names
from data_pipeline.config import settings
logger = get_module_logger(__name__)
class MichiganEnviroScreenETL(ExtractTransformLoad):
"""Michigan EJ Screen class that ingests dataset represented
here: https://www.arcgis.com/apps/webappviewer/index.html?id=dc4f0647dda34959963488d3f519fd24
This class ingests the data presented in "Assessing the State of Environmental
Justice in Michigan." Please see the README in this module for further details.
"""
def __init__(self):
self.MICHIGAN_EJSCREEN_S3_URL = (
settings.AWS_JUSTICE40_DATASOURCES_URL
+ "/michigan_ejscore_12212021.csv"
)
self.CSV_PATH = self.DATA_PATH / "dataset" / "michigan_ejscreen"
self.MICHIGAN_EJSCREEN_PRIORITY_COMMUNITY_THRESHOLD: float = 0.75
self.COLUMNS_TO_KEEP = [
self.GEOID_TRACT_FIELD_NAME,
field_names.MICHIGAN_EJSCREEN_SCORE_FIELD,
field_names.MICHIGAN_EJSCREEN_PERCENTILE_FIELD,
field_names.MICHIGAN_EJSCREEN_PRIORITY_COMMUNITY_FIELD,
]
self.df: pd.DataFrame
def extract(self) -> None:
logger.info("Downloading Michigan EJSCREEN Data")
self.df = pd.read_csv(
filepath_or_buffer=self.MICHIGAN_EJSCREEN_S3_URL,
dtype={"GEO_ID": "string"},
low_memory=False,
)
def transform(self) -> None:
logger.info("Transforming Michigan EJSCREEN Data")
self.df.rename(
columns={
"GEO_ID": self.GEOID_TRACT_FIELD_NAME,
"EJ_Score_Cal_Min": field_names.MICHIGAN_EJSCREEN_SCORE_FIELD,
"Pct_CalMin": field_names.MICHIGAN_EJSCREEN_PERCENTILE_FIELD,
},
inplace=True,
)
# Calculate the top quartile of prioritized communities
# Please see pg. 104 - 109 from source:
# pg. https://deepblue.lib.umich.edu/bitstream/handle/2027.42/149105/AssessingtheStateofEnvironmentalJusticeinMichigan_344.pdf
self.df[field_names.MICHIGAN_EJSCREEN_PRIORITY_COMMUNITY_FIELD] = (
self.df[field_names.MICHIGAN_EJSCREEN_PERCENTILE_FIELD]
>= self.MICHIGAN_EJSCREEN_PRIORITY_COMMUNITY_THRESHOLD
)
def load(self) -> None:
logger.info("Saving Michigan Environmental Screening Tool to CSV")
# write nationwide csv
self.CSV_PATH.mkdir(parents=True, exist_ok=True)
self.df[self.COLUMNS_TO_KEEP].to_csv(
self.CSV_PATH / "michigan_ejscreen.csv", index=False
)

50
data/data-pipeline/data_pipeline/etl/sources/tree_equity_score/etl.py
View file

@ -7,6 +7,18 @@ logger = get_module_logger(__name__)
class TreeEquityScoreETL(ExtractTransformLoad): class TreeEquityScoreETL(ExtractTransformLoad):
"""Tree equity score methodology: https://www.treeequityscore.org/methodology/
A lower Tree Equity Score indicates a greater priority for closing the tree canopy gap
In order to estimate a general number of trees associated with an increase in tree
canopy, the authors utilize a basic multiplier of 600 sq-ft (55.74 sq-m) of canopy area
per urban tree assuming a medium-size urban tree crown width of 25-30 ft.
Sources:
1. Tree canopy cover. High resolution tree canopy where available.
In the event tree canopy is not defer to National Land Cover Database.
2. Census American Community Survey (ACS) 2018 5-year Block Group population estimates.
3. Census ACS 2018 5-year city and block group Median Income estimates.
"""
def __init__(self): def __init__(self):
self.TES_URL = "https://national-tes-data-share.s3.amazonaws.com/national_tes_share/" self.TES_URL = "https://national-tes-data-share.s3.amazonaws.com/national_tes_share/"
self.TES_CSV = self.TMP_PATH / "tes_2021_data.csv" self.TES_CSV = self.TMP_PATH / "tes_2021_data.csv"
@ -83,8 +95,42 @@ class TreeEquityScoreETL(ExtractTransformLoad):
pd.concat(tes_state_dfs), crs=tes_state_dfs[0].crs pd.concat(tes_state_dfs), crs=tes_state_dfs[0].crs
) )
# rename ID to Tract ID
self.df.rename(
# Block group ID delegated to attribute in superclass
columns={"geoid": ExtractTransformLoad.GEOID_FIELD_NAME},
inplace=True,
)
def load(self) -> None: def load(self) -> None:
logger.info("Saving Tree Equity Score GeoJSON") logger.info("Saving Tree Equity Score CSV")
# write nationwide csv # write nationwide csv
self.CSV_PATH.mkdir(parents=True, exist_ok=True) self.CSV_PATH.mkdir(parents=True, exist_ok=True)
self.df.to_file(self.CSV_PATH / "tes_conus.geojson", driver="GeoJSON") self.df = self.df[
[
ExtractTransformLoad.GEOID_FIELD_NAME,
"total_pop", # Total Population according to ACS Estimates
"state",
"county",
"dep_ratio", # Dependent ratio
"child_perc", # Children (Age 0 -17) (ACS 2014 - 2018)
"seniorperc", # Seniors (Age 65+) (ACS 2014 - 2018)
"treecanopy", # Tree canopy cover
"area", # Source: https://www.fs.fed.us/nrs/pubs/gtr/gtr_nrs200.pdf
"source",
"avg_temp", # Average Temperature from USGS Earth Explorer
"ua_name",
"incorpname", # Incorporated place name
"congressio", # Congressional District
"biome",
"bgpopdense",
"popadjust", # Adjusted population estimate
"tc_gap", # Tree canopy gap
"tc_goal", # Tree canopy goal
"priority", # Priority community according to the index
"tes", # Tree equity score
"tesctyscor", # Tree equity score for the county
"geometry", # Block group geometry coordinates
]
]
self.df.to_csv(self.CSV_PATH / "usa.csv", index=False)

57
data/data-pipeline/data_pipeline/ipython/scoring_comparison.ipynb
View file

@ -276,6 +276,44 @@
"mapping_inequality_df" "mapping_inequality_df"
] ]
}, },
{
"cell_type": "code",
"execution_count": null,
"id": "605af1ff",
"metadata": {},
"outputs": [],
"source": [
"# Load alternative energy-related definition \n",
"energy_definition_alternative_draft_path = (\n",
" DATA_DIR / \"dataset\" / \"energy_definition_alternative_draft\" / \"usa.csv\"\n",
")\n",
"energy_definition_alternative_draft_df = pd.read_csv(\n",
" energy_definition_alternative_draft_path,\n",
" dtype={ExtractTransformLoad.GEOID_TRACT_FIELD_NAME: \"string\"},\n",
")\n",
"\n",
"energy_definition_alternative_draft_df"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fe4a2939",
"metadata": {},
"outputs": [],
"source": [
"# Load Michigan EJSCREEN\n",
"michigan_ejscreen_data_path = (\n",
" DATA_DIR / \"dataset\" / \"michigan_ejscreen\" / \"michigan_ejscreen.csv\"\n",
")\n",
"michigan_ejscreen_df = pd.read_csv(\n",
" michigan_ejscreen_data_path,\n",
" dtype={ExtractTransformLoad.GEOID_TRACT_FIELD_NAME: \"string\"},\n",
")\n",
"\n",
"michigan_ejscreen_df.head()"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
@ -291,6 +329,8 @@
" calenviroscreen_df,\n", " calenviroscreen_df,\n",
" persistent_poverty_df,\n", " persistent_poverty_df,\n",
" mapping_inequality_df,\n", " mapping_inequality_df,\n",
" energy_definition_alternative_draft_df,\n",
" michigan_ejscreen_df\n",
"]\n", "]\n",
"\n", "\n",
"merged_df = functools.reduce(\n", "merged_df = functools.reduce(\n",
@ -431,6 +471,19 @@
" priority_communities_field=PERSISTENT_POVERTY_TRACT_LEVEL_FIELD,\n", " priority_communities_field=PERSISTENT_POVERTY_TRACT_LEVEL_FIELD,\n",
" other_census_tract_fields_to_keep=[],\n", " other_census_tract_fields_to_keep=[],\n",
" ),\n", " ),\n",
" Index(\n",
" method_name=field_names.ENERGY_RELATED_COMMUNITIES_DEFINITION_ALTERNATIVE,\n",
" priority_communities_field=field_names.ENERGY_RELATED_COMMUNITIES_DEFINITION_ALTERNATIVE,\n",
" other_census_tract_fields_to_keep=[],\n",
" ),\n",
" Index(\n",
" method_name=\"Michigan EJSCREEN\",\n",
" priority_communities_field=field_names.MICHIGAN_EJSCREEN_PRIORITY_COMMUNITY_FIELD,\n",
" other_census_tract_fields_to_keep=[\n",
" field_names.MICHIGAN_EJSCREEN_SCORE_FIELD,\n",
" field_names.MICHIGAN_EJSCREEN_PERCENTILE_FIELD,\n",
" ],\n",
" ), \n",
" ]\n", " ]\n",
" # Insert indices for each of the HOLC factors.\n", " # Insert indices for each of the HOLC factors.\n",
" # Note: since these involve no renaming, we write them using list comprehension.\n", " # Note: since these involve no renaming, we write them using list comprehension.\n",
@ -1273,7 +1326,7 @@
], ],
"metadata": { "metadata": {
"kernelspec": { "kernelspec": {
"display_name": "Python 3 (ipykernel)", "display_name": "Python 3",
"language": "python", "language": "python",
"name": "python3" "name": "python3"
}, },
@ -1287,7 +1340,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.9.6" "version": "3.6.2"
} }
}, },
"nbformat": 4, "nbformat": 4,

28
data/data-pipeline/data_pipeline/score/field_names.py
View file

@ -210,13 +210,18 @@ EJSCREEN_AREAS_OF_CONCERN_STATE_90TH_PERCENTILE_COMMUNITIES_FIELD = (
EJSCREEN_AREAS_OF_CONCERN_STATE_95TH_PERCENTILE_COMMUNITIES_FIELD = ( EJSCREEN_AREAS_OF_CONCERN_STATE_95TH_PERCENTILE_COMMUNITIES_FIELD = (
"EJSCREEN Areas of Concern, State, 95th percentile (communities)" "EJSCREEN Areas of Concern, State, 95th percentile (communities)"
) )
# Mapping inequality data. # Mapping inequality data.
HOLC_GRADE_D_TRACT_PERCENT_FIELD: str = "Percent of tract that is HOLC Grade D" HOLC_GRADE_D_TRACT_PERCENT_FIELD: str = "Percent of tract that is HOLC Grade D"
HOLC_GRADE_D_TRACT_20_PERCENT_FIELD: str = "Tract is >20% HOLC Grade D" HOLC_GRADE_D_TRACT_20_PERCENT_FIELD: str = "Tract is >20% HOLC Grade D"
HOLC_GRADE_D_TRACT_50_PERCENT_FIELD: str = "Tract is >50% HOLC Grade D" HOLC_GRADE_D_TRACT_50_PERCENT_FIELD: str = "Tract is >50% HOLC Grade D"
HOLC_GRADE_D_TRACT_75_PERCENT_FIELD: str = "Tract is >75% HOLC Grade D" HOLC_GRADE_D_TRACT_75_PERCENT_FIELD: str = "Tract is >75% HOLC Grade D"
# Michigan Environmental Screening Tool ETL Constants
MICHIGAN_EJSCREEN_SCORE_FIELD: str = "Michigan EJSCREEN Score Field"
MICHIGAN_EJSCREEN_PERCENTILE_FIELD: str = "Michigan EJSCREEN Percentile Field"
MICHIGAN_EJSCREEN_PRIORITY_COMMUNITY_FIELD: str = (
"Michigan EJSCREEN Priority Community"
)
# Child Opportunity Index data # Child Opportunity Index data
# Summer days with maximum temperature above 90F. # Summer days with maximum temperature above 90F.
@ -234,6 +239,27 @@ IMPENETRABLE_SURFACES_FIELD = "Percent impenetrable surface areas"
READING_FIELD = "Third grade reading proficiency" READING_FIELD = "Third grade reading proficiency"
LOW_READING_FIELD = "Low third grade reading proficiency" LOW_READING_FIELD = "Low third grade reading proficiency"
# Alternative energy-related definition of DACs
ENERGY_RELATED_COMMUNITIES_DEFINITION_ALTERNATIVE = (
"Energy-related alternative definition of communities"
)
COAL_EMPLOYMENT = "Coal employment"
OUTAGE_EVENTS = "Outage Events"
HOMELESSNESS = "Homelessness"
DISABLED_POPULATION = "Disabled population"
OUTAGE_DURATION = "Outage Duration"
JOB_ACCESS = "Job Access"
FOSSIL_ENERGY_EMPLOYMENT = "Fossil energy employment"
FOOD_DESERT = "Food Desert"
INCOMPLETE_PLUMBING = "Incomplete Plumbing"
NON_GRID_CONNECTED_HEATING_FUEL = "Non-grid-connected heating fuel"
PARKS = "Parks"
GREATER_THAN_30_MIN_COMMUTE = "Greater than 30 min commute"
INTERNET_ACCESS = "Internet Access"
MOBILE_HOME = "Mobile Home"
SINGLE_PARENT = "Single Parent"
TRANSPORTATION_COSTS = "Transportation Costs"
##### #####
# Names for individual factors being exceeded # Names for individual factors being exceeded
# Climate Change # Climate Change

6
docs/decisions/0002-mapping-visualization-library.md
View file

@ -58,8 +58,8 @@ We provide more detail on these factors below.
![GIS Feature set](./0002-files/GIS_Features.png) ![GIS Feature set](./0002-files/GIS_Features.png)
- **Popularity** : OpenLayers is second only to Leaflet in the number of Github [stars](https://github.com/openlayers/openlayers/stargazers) it has received, close to 8000 - **Popularity** : OpenLayers is second only to Leaflet in the number of Github [stars](https://github.com/openlayers/openlayers/stargazers) it has received, close to 8000
- **Performance** : - **Performance** :
- The below chart comes from an September 2020 [study](https://doi.org/10.3390/ijgi9100563). The purpose of this study was to compare OpenLayers to Mapbox-GL-JS and Leaflet (both raster and vector tile variants) as the potential basis for a Life Quality Index for 55,000+ census radius jurisdictions in Argentina. - The below chart comes from an September 2020 study (www dot mdpi dot com/2220-9964/9/10/563). The purpose of this study was to compare OpenLayers to Mapbox-GL-JS and Leaflet (both raster and vector tile variants) as the potential basis for a Life Quality Index for 55,000+ census radius jurisdictions in Argentina.
![Execution Time](./0002-files/ExecutionTime.png) ([Source](https://doi.org/10.3390/ijgi9100563)) ![Execution Time](./0002-files/ExecutionTime.png) (Source (www dot mdpi dot com/2220-9964/9/10/563))
In this chart, the two letters following the library name are for basemap layer and feature layer. Further, "R" is "raster" and "V" is vector, and lower numbers indicate faster load times. "OpenLayersRR" and "OpenLayersRV", (signifying a raster base layer and vector feature layer), performed quite well across all device types compared to other libraries. In this chart, the two letters following the library name are for basemap layer and feature layer. Further, "R" is "raster" and "V" is vector, and lower numbers indicate faster load times. "OpenLayersRR" and "OpenLayersRV", (signifying a raster base layer and vector feature layer), performed quite well across all device types compared to other libraries.
- We also performed local testing using puppeteer and web performance APIs, tested against a choropleth map of the cenus block groups, which represents a likely usecase for us. The results were as follows: - We also performed local testing using puppeteer and web performance APIs, tested against a choropleth map of the cenus block groups, which represents a likely usecase for us. The results were as follows:
![Choropleth Map Performance](./0002-files/ChoroplethMapPerformance.png) ([Source](./0002-files/Maryland.csv)) ![Choropleth Map Performance](./0002-files/ChoroplethMapPerformance.png) ([Source](./0002-files/Maryland.csv))
@ -70,7 +70,7 @@ We provide more detail on these factors below.
- OL+MB did not file the `tiledidload` event and thus there was not a separate measure for style loaded - OL+MB did not file the `tiledidload` event and thus there was not a separate measure for style loaded
- Apparent performance was different from measured/reported overall, particularly when it comes to zoom performance. This is an area to dig into further and measured at a later time to understand better. - Apparent performance was different from measured/reported overall, particularly when it comes to zoom performance. This is an area to dig into further and measured at a later time to understand better.
- **Data Usage** : The same study above also analyzed the amount of data usage for each of the libraries under investigation, and the result was the below chart (Lower values are better). OpenLayers overall performed quite well - **Data Usage** : The same study above also analyzed the amount of data usage for each of the libraries under investigation, and the result was the below chart (Lower values are better). OpenLayers overall performed quite well
![Data Usage](./0002-files/NetworkTraffic.png) ([Source](https://doi.org/10.3390/ijgi9100563)) ![Data Usage](./0002-files/NetworkTraffic.png) (source (www dot mdpi dot com/2220-9964/9/10/563))
### Negative Consequences ### Negative Consequences