From 426328e378126760a61388552463183801dbc2f6 Mon Sep 17 00:00:00 2001 From: Emma Nechamkin <97977170+emma-nechamkin@users.noreply.github.com> Date: Wed, 7 Sep 2022 17:13:31 -0400 Subject: [PATCH] =?UTF-8?q?Updating=20traffic=20barriers=C2=A0to=20include?= =?UTF-8?q?=20low=20pop=20threshold=20(#1889)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changing the traffic barriers to only be included for places with recorded population --- .../data_pipeline/etl/score/etl_score.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/data/data-pipeline/data_pipeline/etl/score/etl_score.py b/data/data-pipeline/data_pipeline/etl/score/etl_score.py index 62a5006d..56682d49 100644 --- a/data/data-pipeline/data_pipeline/etl/score/etl_score.py +++ b/data/data-pipeline/data_pipeline/etl/score/etl_score.py @@ -380,7 +380,8 @@ class ScoreETL(ExtractTransformLoad): ), "Join against national tract list ADDED rows" logger.info( "Dropped %s tracts not in the 2010 tract data", - pre_join_len - census_tract_df[field_names.GEOID_TRACT_FIELD].nunique() + pre_join_len + - census_tract_df[field_names.GEOID_TRACT_FIELD].nunique(), ) # Now sanity-check the merged df. @@ -551,6 +552,9 @@ class ScoreETL(ExtractTransformLoad): # For *Non-Natural Space*, we may only want to include tracts that have at least 35 acreas, I think. This will # get rid of tracts that we think are aberrations statistically. Right now, we have left this out # pending ground-truthing. + # + # For *Traffic Barriers*, we want to exclude low population tracts, which may have high burden because they are + # low population alone. We set this low population constant in the if statement. for numeric_column in numeric_columns: drop_tracts = [] @@ -575,6 +579,17 @@ class ScoreETL(ExtractTransformLoad): f"Dropping {len(drop_tracts)} tracts from Linguistic Isolation" ) + elif numeric_column == field_names.DOT_TRAVEL_BURDEN_FIELD: + # Not having any people appears to be correlated with transit burden, but also doesn't represent + # on the ground need. For now, we remove these tracts from the percentile calculation. (To be QAed live) + low_population = 20 + drop_tracts = df_copy[ + df_copy[field_names.TOTAL_POP_FIELD] <= low_population + ][field_names.GEOID_TRACT_FIELD].to_list() + logger.info( + f"Dropping {len(drop_tracts)} tracts from DOT traffic burden" + ) + df_copy = self._add_percentiles_to_df( df=df_copy, input_column_name=numeric_column,