From 65ceb7900f87200b4bf4e533abf8f5db0e58d982 Mon Sep 17 00:00:00 2001 From: Lucas Merrill Brown Date: Tue, 24 Aug 2021 15:40:54 -0500 Subject: [PATCH] Score F, testing methodology (#510) * fixing dependency issue * fixing more dependencies * including fraction of state AMI * wip * nitpick whitespace * etl working now * wip on scoring * fix rename error * reducing metrics * fixing score f * fixing readme * adding dependency * passing tests; * linting/black * removing unnecessary sample * fixing error * adding verify flag on etl/base Co-authored-by: Jorge Escobar --- data/data-pipeline/.vscode/settings.json | 8 +- data/data-pipeline/.vscode/tasks.json | 14 +- data/data-pipeline/README.md | 11 - data/data-pipeline/data_pipeline/etl/base.py | 12 +- .../data_pipeline/etl/constants.py | 5 + .../data_pipeline/etl/score/etl_score.py | 121 +++++++++-- .../etl/sources/calenviroscreen/etl.py | 11 +- .../etl/sources/cdc_places/README.md | 0 .../etl/sources/cdc_places/__init__.py | 0 .../etl/sources/cdc_places/etl.py | 66 ++++++ .../data_pipeline/etl/sources/census/etl.py | 16 +- .../etl/sources/census_acs/etl.py | 4 +- .../data_pipeline/etl/sources/ejscreen/etl.py | 5 +- .../sources/housing_and_transportation/etl.py | 10 +- .../etl/sources/hud_housing/etl.py | 16 +- .../etl/sources/hud_recap/etl.py | 4 +- .../etl/sources/tree_equity_score/etl.py | 12 +- .../ipython/census_explore.ipynb | 49 ++--- .../ipython/scoring_comparison.ipynb | 61 +++--- data/data-pipeline/data_pipeline/utils.py | 63 ++++-- data/data-pipeline/poetry.lock | 201 ++++++++++++++++-- data/data-pipeline/pyproject.toml | 6 + data/data-pipeline/requirements.txt | 15 +- 23 files changed, 557 insertions(+), 153 deletions(-) create mode 100644 data/data-pipeline/data_pipeline/etl/sources/cdc_places/README.md create mode 100644 data/data-pipeline/data_pipeline/etl/sources/cdc_places/__init__.py create mode 100644 data/data-pipeline/data_pipeline/etl/sources/cdc_places/etl.py diff --git a/data/data-pipeline/.vscode/settings.json b/data/data-pipeline/.vscode/settings.json index 4c0377b4..e5a4989f 100644 --- a/data/data-pipeline/.vscode/settings.json +++ b/data/data-pipeline/.vscode/settings.json @@ -1,10 +1,16 @@ { "python.formatting.provider": "black", + "python.formatting.blackArgs": [ + "--line-length=80" + ], "python.linting.enabled": true, "python.linting.flake8Enabled": true, "python.linting.pylintEnabled": true, "python.testing.pytestEnabled": true, - "python.testing.pytestArgs": ["-s", "."], + "python.testing.pytestArgs": [ + "-s", + "." + ], "python.testing.unittestEnabled": false, "python.testing.nosetestsEnabled": false } diff --git a/data/data-pipeline/.vscode/tasks.json b/data/data-pipeline/.vscode/tasks.json index e85d3b5d..79e165ea 100644 --- a/data/data-pipeline/.vscode/tasks.json +++ b/data/data-pipeline/.vscode/tasks.json @@ -4,7 +4,7 @@ "version": "2.0.0", "tasks": [ { - "label": "test with tox", + "label": "Test with tox", "type": "shell", "command": "tox", "group": { @@ -16,19 +16,25 @@ "label": "Run Black Formatter", "type": "shell", "command": "black", - "args": ["data_pipeline"] + "args": [ + "data_pipeline" + ] }, { "label": "Run Flake8 Style Enforcer", "type": "shell", "command": "black", - "args": ["data_pipeline"] + "args": [ + "data_pipeline" + ] }, { "label": "Run Pylint", "type": "shell", "command": "pylint", - "args": ["data_pipeline"] + "args": [ + "data_pipeline" + ] } ] } diff --git a/data/data-pipeline/README.md b/data/data-pipeline/README.md index c6dcfdc5..ce0c78f7 100644 --- a/data/data-pipeline/README.md +++ b/data/data-pipeline/README.md @@ -120,17 +120,6 @@ To run this comparison tool: 1. Make sure you've gone through the above steps to run the data ETL and score generation. 1. From the package directory (`data/data-pipeline/data_pipeline/`), navigate to the `ipython` directory: `cd ipython`. 1. Ensure you have `pandoc` installed on your computer. If you're on a Mac, run `brew install pandoc`; for other OSes, see pandoc's [installation guide](https://pandoc.org/installing.html). -1. Install the extra dependencies: - -```python - pip install pypandoc - pip install requests - pip install us - pip install tqdm - pip install dynaconf - pip instal xlsxwriter -``` - 1. Start the notebooks: `jupyter notebook` 1. In your browser, navigate to one of the URLs returned by the above command. 1. Select `scoring_comparison.ipynb` from the options in your browser. diff --git a/data/data-pipeline/data_pipeline/etl/base.py b/data/data-pipeline/data_pipeline/etl/base.py index 25470099..a0128119 100644 --- a/data/data-pipeline/data_pipeline/etl/base.py +++ b/data/data-pipeline/data_pipeline/etl/base.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Optional from data_pipeline.config import settings from data_pipeline.utils import unzip_file_from_url, remove_all_from_dir @@ -33,14 +34,21 @@ class ExtractTransformLoad: pass - def extract(self, source_url: str = None, extract_path: Path = None) -> None: + def extract( + self, + source_url: str = None, + extract_path: Path = None, + verify: Optional[bool] = True, + ) -> None: """Extract the data from a remote source. By default it provides code to get the file from a source url, unzips it and stores it on an extract_path.""" # this can be accessed via super().extract() if source_url and extract_path: - unzip_file_from_url(source_url, self.TMP_PATH, extract_path) + unzip_file_from_url( + source_url, self.TMP_PATH, extract_path, verify=verify + ) def transform(self) -> None: """Transform the data extracted into a format that can be consumed by the diff --git a/data/data-pipeline/data_pipeline/etl/constants.py b/data/data-pipeline/data_pipeline/etl/constants.py index cef7c80c..c0b16128 100644 --- a/data/data-pipeline/data_pipeline/etl/constants.py +++ b/data/data-pipeline/data_pipeline/etl/constants.py @@ -34,6 +34,11 @@ DATASET_LIST = [ "module_dir": "hud_recap", "class_name": "HudRecapETL", }, + { + "name": "cdc_places", + "module_dir": "cdc_places", + "class_name": "CDCPlacesETL", + }, ] CENSUS_INFO = { "name": "census", diff --git a/data/data-pipeline/data_pipeline/etl/score/etl_score.py b/data/data-pipeline/data_pipeline/etl/score/etl_score.py index 4010eaa3..450ce946 100644 --- a/data/data-pipeline/data_pipeline/etl/score/etl_score.py +++ b/data/data-pipeline/data_pipeline/etl/score/etl_score.py @@ -50,6 +50,7 @@ class ScoreETL(ExtractTransformLoad): self.census_df: pd.DataFrame self.housing_and_transportation_df: pd.DataFrame self.hud_housing_df: pd.DataFrame + self.cdc_places_df: pd.DataFrame def data_sets(self) -> list: # Define a named tuple that will be used for each data set input. @@ -81,6 +82,36 @@ class ScoreETL(ExtractTransformLoad): renamed_field=self.MEDIAN_INCOME_AS_PERCENT_OF_STATE_FIELD_NAME, bucket=None, ), + DataSet( + input_field="Current asthma among adults aged >=18 years", + renamed_field="Current asthma among adults aged >=18 years", + bucket=None, + ), + DataSet( + input_field="Coronary heart disease among adults aged >=18 years", + renamed_field="Coronary heart disease among adults aged >=18 years", + bucket=None, + ), + DataSet( + input_field="Cancer (excluding skin cancer) among adults aged >=18 years", + renamed_field="Cancer (excluding skin cancer) among adults aged >=18 years", + bucket=None, + ), + DataSet( + input_field="Current lack of health insurance among adults aged 18-64 years", + renamed_field="Current lack of health insurance among adults aged 18-64 years", + bucket=None, + ), + DataSet( + input_field="Diagnosed diabetes among adults aged >=18 years", + renamed_field="Diagnosed diabetes among adults aged >=18 years", + bucket=None, + ), + DataSet( + input_field="Physical health not good for >=14 days among adults aged >=18 years", + renamed_field="Physical health not good for >=14 days among adults aged >=18 years", + bucket=None, + ), # The following data sets have buckets, because they're used in Score C DataSet( input_field="CANCER", @@ -218,6 +249,14 @@ class ScoreETL(ExtractTransformLoad): low_memory=False, ) + # Load CDC Places data + cdc_places_csv = self.DATA_PATH / "dataset" / "cdc_places" / "usa.csv" + self.cdc_places_df = pd.read_csv( + cdc_places_csv, + dtype={self.GEOID_TRACT_FIELD_NAME: "string"}, + low_memory=False, + ) + def transform(self) -> None: ## IMPORTANT: THIS METHOD IS CLOSE TO THE LIMIT OF STATEMENTS @@ -247,8 +286,28 @@ class ScoreETL(ExtractTransformLoad): ) # Join all the data sources that use census tracts - # TODO: when there's more than one data source using census tract, reduce/merge them here. - census_tract_df = self.hud_housing_df + census_tract_dfs = [ + self.hud_housing_df, + self.cdc_places_df, + ] + census_tract_df = functools.reduce( + lambda left, right: pd.merge( + left=left, + right=right, + on=self.GEOID_TRACT_FIELD_NAME, + how="outer", + ), + census_tract_dfs, + ) + + # Sanity check the join. + if ( + len(census_tract_df[self.GEOID_TRACT_FIELD_NAME].str.len().unique()) + != 1 + ): + raise ValueError( + f"One of the input CSVs uses {self.GEOID_TRACT_FIELD_NAME} with a different length." + ) # Calculate the tract for the CBG data. census_block_group_df[ @@ -437,12 +496,56 @@ class ScoreETL(ExtractTransformLoad): ) self.df[meets_burden_field_name] = ( - self.df["Particulate matter (PM2.5)"] > 10 - ) | (self.df["Respiratory hazard " "index"] > 0.75) + (self.df["Particulate matter (PM2.5) (percentile)"] > 0.9) + | (self.df["Respiratory hazard index (percentile)"] > 0.9) + | (self.df["Traffic proximity and volume (percentile)"] > 0.9) + | ( + self.df[ + "Percent pre-1960s housing (lead paint indicator) (percentile)" + ] + > 0.9 + ) + | (self.df["Proximity to RMP sites (percentile)"] > 0.9) + | ( + self.df[ + "Current asthma among adults aged >=18 years (percentile)" + ] + > 0.9 + ) + | ( + self.df[ + "Coronary heart disease among adults aged >=18 years (percentile)" + ] + > 0.9 + ) + | ( + self.df[ + "Cancer (excluding skin cancer) among adults aged >=18 years (percentile)" + ] + > 0.9 + ) + # | ( + # self.df[ + # "Current lack of health insurance among adults aged 18-64 years (percentile)" + # ] + # > 0.9 + # ) + | ( + self.df[ + "Diagnosed diabetes among adults aged >=18 years (percentile)" + ] + > 0.9 + ) + # | ( + # self.df[ + # "Physical health not good for >=14 days among adults aged >=18 years (percentile)" + # ] + # > 0.9 + # ) + ) self.df["Score F (communities)"] = ( - self.df[ami_and_high_school_field_name] - & self.df[meets_burden_field_name] + self.df[meets_socio_field_name] & self.df[meets_burden_field_name] ) def load(self) -> None: @@ -450,10 +553,4 @@ class ScoreETL(ExtractTransformLoad): # write nationwide csv self.SCORE_CSV_PATH.mkdir(parents=True, exist_ok=True) - - # TODO: drop - self.df[0:10000].to_csv( - self.SCORE_CSV_PATH / "usa-10000.csv", index=False - ) - self.df.to_csv(self.SCORE_CSV_PATH / "usa.csv", index=False) diff --git a/data/data-pipeline/data_pipeline/etl/sources/calenviroscreen/etl.py b/data/data-pipeline/data_pipeline/etl/sources/calenviroscreen/etl.py index ec5e53d6..8312de24 100644 --- a/data/data-pipeline/data_pipeline/etl/sources/calenviroscreen/etl.py +++ b/data/data-pipeline/data_pipeline/etl/sources/calenviroscreen/etl.py @@ -10,14 +10,19 @@ logger = get_module_logger(__name__) class CalEnviroScreenETL(ExtractTransformLoad): def __init__(self): self.CALENVIROSCREEN_FTP_URL = ( - settings.AWS_JUSTICE40_DATASOURCES_URL + "/CalEnviroScreen_4.0_2021.zip" + settings.AWS_JUSTICE40_DATASOURCES_URL + + "/CalEnviroScreen_4.0_2021.zip" + ) + self.CALENVIROSCREEN_CSV = ( + self.TMP_PATH / "CalEnviroScreen_4.0_2021.csv" ) - self.CALENVIROSCREEN_CSV = self.TMP_PATH / "CalEnviroScreen_4.0_2021.csv" self.CSV_PATH = self.DATA_PATH / "dataset" / "calenviroscreen4" # Definining some variable names self.CALENVIROSCREEN_SCORE_FIELD_NAME = "calenviroscreen_score" - self.CALENVIROSCREEN_PERCENTILE_FIELD_NAME = "calenviroscreen_percentile" + self.CALENVIROSCREEN_PERCENTILE_FIELD_NAME = ( + "calenviroscreen_percentile" + ) self.CALENVIROSCREEN_PRIORITY_COMMUNITY_FIELD_NAME = ( "calenviroscreen_priority_community" ) diff --git a/data/data-pipeline/data_pipeline/etl/sources/cdc_places/README.md b/data/data-pipeline/data_pipeline/etl/sources/cdc_places/README.md new file mode 100644 index 00000000..e69de29b diff --git a/data/data-pipeline/data_pipeline/etl/sources/cdc_places/__init__.py b/data/data-pipeline/data_pipeline/etl/sources/cdc_places/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/data/data-pipeline/data_pipeline/etl/sources/cdc_places/etl.py b/data/data-pipeline/data_pipeline/etl/sources/cdc_places/etl.py new file mode 100644 index 00000000..337c60ef --- /dev/null +++ b/data/data-pipeline/data_pipeline/etl/sources/cdc_places/etl.py @@ -0,0 +1,66 @@ +import pandas as pd + +from data_pipeline.etl.base import ExtractTransformLoad +from data_pipeline.utils import get_module_logger, download_file_from_url + +logger = get_module_logger(__name__) + + +class CDCPlacesETL(ExtractTransformLoad): + def __init__(self): + self.OUTPUT_PATH = self.DATA_PATH / "dataset" / "cdc_places" + + self.CDC_PLACES_URL = "https://chronicdata.cdc.gov/api/views/cwsq-ngmh/rows.csv?accessType=DOWNLOAD" + self.CDC_GEOID_FIELD_NAME = "LocationID" + self.CDC_VALUE_FIELD_NAME = "Data_Value" + self.CDC_MEASURE_FIELD_NAME = "Measure" + + self.df: pd.DataFrame + + def extract(self) -> None: + logger.info("Starting to download 520MB CDC Places file.") + file_path = download_file_from_url( + file_url=self.CDC_PLACES_URL, + download_file_name=self.TMP_PATH + / "cdc_places" + / "census_tract.csv", + ) + + self.df = pd.read_csv( + filepath_or_buffer=file_path, + dtype={self.CDC_GEOID_FIELD_NAME: "string"}, + low_memory=False, + ) + + def transform(self) -> None: + logger.info("Starting CDC Places transform") + + # Rename GEOID field + self.df.rename( + columns={self.CDC_GEOID_FIELD_NAME: self.GEOID_TRACT_FIELD_NAME}, + inplace=True, + errors="raise", + ) + + # Note: Puerto Rico not included. + self.df = self.df.pivot( + index=self.GEOID_TRACT_FIELD_NAME, + columns=self.CDC_MEASURE_FIELD_NAME, + values=self.CDC_VALUE_FIELD_NAME, + ) + + # Make the index (the census tract ID) a column, not the index. + self.df.reset_index(inplace=True) + + def load(self) -> None: + logger.info("Saving CDC Places Data") + + # mkdir census + self.OUTPUT_PATH.mkdir(parents=True, exist_ok=True) + + self.df.to_csv(path_or_buf=self.OUTPUT_PATH / "usa.csv", index=False) + + def validate(self) -> None: + logger.info("Validating Census ACS Data") + + pass diff --git a/data/data-pipeline/data_pipeline/etl/sources/census/etl.py b/data/data-pipeline/data_pipeline/etl/sources/census/etl.py index 929fc76d..803ef5f9 100644 --- a/data/data-pipeline/data_pipeline/etl/sources/census/etl.py +++ b/data/data-pipeline/data_pipeline/etl/sources/census/etl.py @@ -33,7 +33,9 @@ class CensusETL(ExtractTransformLoad): self.NATIONAL_CBG_CSV_PATH = self.CSV_BASE_PATH / "us.csv" self.NATIONAL_CBG_JSON_PATH = self.GEOJSON_BASE_PATH / "us.json" - def _path_for_fips_file(self, fips_code: str, file_type: GeoFileType) -> Path: + def _path_for_fips_file( + self, fips_code: str, file_type: GeoFileType + ) -> Path: """Get paths for associated geospatial files for the provided FIPS code Args: @@ -93,7 +95,9 @@ class CensusETL(ExtractTransformLoad): None """ shp_file_path = self._path_for_fips_file(fips_code, GeoFileType.SHP) - geojson_file_path = self._path_for_fips_file(fips_code, GeoFileType.GEOJSON) + geojson_file_path = self._path_for_fips_file( + fips_code, GeoFileType.GEOJSON + ) logger.info(f"Checking if {fips_code} geoJSON file exists ") if not geojson_file_path.is_file(): logger.info( @@ -176,7 +180,9 @@ class CensusETL(ExtractTransformLoad): if not self.NATIONAL_CBG_CSV_PATH.is_file(): logger.info(f"Creating {self.NATIONAL_CBG_CSV_PATH}") - with open(self.NATIONAL_CBG_CSV_PATH, mode="w", newline="") as cbg_csv_file: + with open( + self.NATIONAL_CBG_CSV_PATH, mode="w", newline="" + ) as cbg_csv_file: cbg_csv_file_writer = csv.writer( cbg_csv_file, delimiter=",", @@ -205,7 +211,9 @@ class CensusETL(ExtractTransformLoad): state_gdf = gpd.read_file(file_name) usa_df = usa_df.append(state_gdf) - usa_df = usa_df.to_crs("+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs") + usa_df = usa_df.to_crs( + "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs" + ) logger.info("Writing national geojson file") usa_df.to_file(self.NATIONAL_CBG_JSON_PATH, driver="GeoJSON") diff --git a/data/data-pipeline/data_pipeline/etl/sources/census_acs/etl.py b/data/data-pipeline/data_pipeline/etl/sources/census_acs/etl.py index a948bd12..c8dfc93c 100644 --- a/data/data-pipeline/data_pipeline/etl/sources/census_acs/etl.py +++ b/data/data-pipeline/data_pipeline/etl/sources/census_acs/etl.py @@ -41,10 +41,10 @@ class CensusACSETL(ExtractTransformLoad): self.STATE_MEDIAN_INCOME_FTP_URL = ( settings.AWS_JUSTICE40_DATASOURCES_URL - + "/2014_to_2019_state_median_income.zip" + + "/2015_to_2019_state_median_income.zip" ) self.STATE_MEDIAN_INCOME_FILE_PATH = ( - self.TMP_PATH / "2014_to_2019_state_median_income.csv" + self.TMP_PATH / "2015_to_2019_state_median_income.csv" ) def _fips_from_censusdata_censusgeo( diff --git a/data/data-pipeline/data_pipeline/etl/sources/ejscreen/etl.py b/data/data-pipeline/data_pipeline/etl/sources/ejscreen/etl.py index fd69f3ec..cef48080 100644 --- a/data/data-pipeline/data_pipeline/etl/sources/ejscreen/etl.py +++ b/data/data-pipeline/data_pipeline/etl/sources/ejscreen/etl.py @@ -8,9 +8,7 @@ logger = get_module_logger(__name__) class EJScreenETL(ExtractTransformLoad): def __init__(self): - self.EJSCREEN_FTP_URL = ( - "https://gaftp.epa.gov/EJSCREEN/2019/EJSCREEN_2019_StatePctile.csv.zip" - ) + self.EJSCREEN_FTP_URL = "https://gaftp.epa.gov/EJSCREEN/2019/EJSCREEN_2019_StatePctile.csv.zip" self.EJSCREEN_CSV = self.TMP_PATH / "EJSCREEN_2019_StatePctiles.csv" self.CSV_PATH = self.DATA_PATH / "dataset" / "ejscreen_2019" self.df: pd.DataFrame @@ -20,6 +18,7 @@ class EJScreenETL(ExtractTransformLoad): super().extract( self.EJSCREEN_FTP_URL, self.TMP_PATH, + verify=False, # EPA EJScreen end point has certificate issues often ) def transform(self) -> None: diff --git a/data/data-pipeline/data_pipeline/etl/sources/housing_and_transportation/etl.py b/data/data-pipeline/data_pipeline/etl/sources/housing_and_transportation/etl.py index 911a2048..6e054df6 100644 --- a/data/data-pipeline/data_pipeline/etl/sources/housing_and_transportation/etl.py +++ b/data/data-pipeline/data_pipeline/etl/sources/housing_and_transportation/etl.py @@ -35,7 +35,9 @@ class HousingTransportationETL(ExtractTransformLoad): ) # New file name: - tmp_csv_file_path = zip_file_dir / f"htaindex_data_blkgrps_{fips}.csv" + tmp_csv_file_path = ( + zip_file_dir / f"htaindex_data_blkgrps_{fips}.csv" + ) tmp_df = pd.read_csv(filepath_or_buffer=tmp_csv_file_path) dfs.append(tmp_df) @@ -47,9 +49,9 @@ class HousingTransportationETL(ExtractTransformLoad): # Rename and reformat block group ID self.df.rename(columns={"blkgrp": self.GEOID_FIELD_NAME}, inplace=True) - self.df[self.GEOID_FIELD_NAME] = self.df[self.GEOID_FIELD_NAME].str.replace( - '"', "" - ) + self.df[self.GEOID_FIELD_NAME] = self.df[ + self.GEOID_FIELD_NAME + ].str.replace('"', "") def load(self) -> None: logger.info("Saving Housing and Transportation Data") diff --git a/data/data-pipeline/data_pipeline/etl/sources/hud_housing/etl.py b/data/data-pipeline/data_pipeline/etl/sources/hud_housing/etl.py index 3cc2f2af..0dbfaee0 100644 --- a/data/data-pipeline/data_pipeline/etl/sources/hud_housing/etl.py +++ b/data/data-pipeline/data_pipeline/etl/sources/hud_housing/etl.py @@ -9,16 +9,16 @@ class HudHousingETL(ExtractTransformLoad): def __init__(self): self.OUTPUT_PATH = self.DATA_PATH / "dataset" / "hud_housing" self.GEOID_TRACT_FIELD_NAME = "GEOID10_TRACT" - self.HOUSING_FTP_URL = ( - "https://www.huduser.gov/portal/datasets/cp/2012thru2016-140-csv.zip" - ) + self.HOUSING_FTP_URL = "https://www.huduser.gov/portal/datasets/cp/2012thru2016-140-csv.zip" self.HOUSING_ZIP_FILE_DIR = self.TMP_PATH / "hud_housing" # We measure households earning less than 80% of HUD Area Median Family Income by county # and paying greater than 30% of their income to housing costs. self.HOUSING_BURDEN_FIELD_NAME = "Housing burden (percent)" self.HOUSING_BURDEN_NUMERATOR_FIELD_NAME = "HOUSING_BURDEN_NUMERATOR" - self.HOUSING_BURDEN_DENOMINATOR_FIELD_NAME = "HOUSING_BURDEN_DENOMINATOR" + self.HOUSING_BURDEN_DENOMINATOR_FIELD_NAME = ( + "HOUSING_BURDEN_DENOMINATOR" + ) # Note: some variable definitions. # HUD-adjusted median family income (HAMFI). @@ -55,7 +55,9 @@ class HudHousingETL(ExtractTransformLoad): ) # Rename and reformat block group ID - self.df.rename(columns={"geoid": self.GEOID_TRACT_FIELD_NAME}, inplace=True) + self.df.rename( + columns={"geoid": self.GEOID_TRACT_FIELD_NAME}, inplace=True + ) # The CHAS data has census tract ids such as `14000US01001020100` # Whereas the rest of our data uses, for the same tract, `01001020100`. @@ -273,7 +275,9 @@ class HudHousingETL(ExtractTransformLoad): # TODO: add small sample size checks self.df[self.HOUSING_BURDEN_FIELD_NAME] = self.df[ self.HOUSING_BURDEN_NUMERATOR_FIELD_NAME - ].astype(float) / self.df[self.HOUSING_BURDEN_DENOMINATOR_FIELD_NAME].astype( + ].astype(float) / self.df[ + self.HOUSING_BURDEN_DENOMINATOR_FIELD_NAME + ].astype( float ) diff --git a/data/data-pipeline/data_pipeline/etl/sources/hud_recap/etl.py b/data/data-pipeline/data_pipeline/etl/sources/hud_recap/etl.py index e40abbfd..04fdbf70 100644 --- a/data/data-pipeline/data_pipeline/etl/sources/hud_recap/etl.py +++ b/data/data-pipeline/data_pipeline/etl/sources/hud_recap/etl.py @@ -18,7 +18,9 @@ class HudRecapETL(ExtractTransformLoad): self.CSV_PATH = self.DATA_PATH / "dataset" / "hud_recap" # Definining some variable names - self.HUD_RECAP_PRIORITY_COMMUNITY_FIELD_NAME = "hud_recap_priority_community" + self.HUD_RECAP_PRIORITY_COMMUNITY_FIELD_NAME = ( + "hud_recap_priority_community" + ) self.df: pd.DataFrame diff --git a/data/data-pipeline/data_pipeline/etl/sources/tree_equity_score/etl.py b/data/data-pipeline/data_pipeline/etl/sources/tree_equity_score/etl.py index a4351bf8..4c93d911 100644 --- a/data/data-pipeline/data_pipeline/etl/sources/tree_equity_score/etl.py +++ b/data/data-pipeline/data_pipeline/etl/sources/tree_equity_score/etl.py @@ -8,9 +8,7 @@ logger = get_module_logger(__name__) class TreeEquityScoreETL(ExtractTransformLoad): def __init__(self): - self.TES_URL = ( - "https://national-tes-data-share.s3.amazonaws.com/national_tes_share/" - ) + self.TES_URL = "https://national-tes-data-share.s3.amazonaws.com/national_tes_share/" self.TES_CSV = self.TMP_PATH / "tes_2021_data.csv" self.CSV_PATH = self.DATA_PATH / "dataset" / "tree_equity_score" self.df: gpd.GeoDataFrame @@ -78,8 +76,12 @@ class TreeEquityScoreETL(ExtractTransformLoad): logger.info("Transforming Tree Equity Score Data") tes_state_dfs = [] for state in self.states: - tes_state_dfs.append(gpd.read_file(f"{self.TMP_PATH}/{state}/{state}.shp")) - self.df = gpd.GeoDataFrame(pd.concat(tes_state_dfs), crs=tes_state_dfs[0].crs) + tes_state_dfs.append( + gpd.read_file(f"{self.TMP_PATH}/{state}/{state}.shp") + ) + self.df = gpd.GeoDataFrame( + pd.concat(tes_state_dfs), crs=tes_state_dfs[0].crs + ) def load(self) -> None: logger.info("Saving Tree Equity Score GeoJSON") diff --git a/data/data-pipeline/data_pipeline/ipython/census_explore.ipynb b/data/data-pipeline/data_pipeline/ipython/census_explore.ipynb index 76d22fd7..abb5dea7 100644 --- a/data/data-pipeline/data_pipeline/ipython/census_explore.ipynb +++ b/data/data-pipeline/data_pipeline/ipython/census_explore.ipynb @@ -3,9 +3,6 @@ { "cell_type": "code", "execution_count": null, - "id": "0491828b", - "metadata": {}, - "outputs": [], "source": [ "import pandas as pd\n", "import censusdata\n", @@ -32,30 +29,26 @@ "# Some display settings to make pandas outputs more readable.\n", "pd.set_option(\"display.expand_frame_repr\", False)\n", "pd.set_option(\"display.precision\", 2)" - ] + ], + "outputs": [], + "metadata": {} }, { "cell_type": "code", "execution_count": null, - "id": "654f25a1", - "metadata": { - "scrolled": true - }, - "outputs": [], "source": [ "# Following the tutorial at https://jtleider.github.io/censusdata/example1.html.\n", "# Full list of fields is at https://www2.census.gov/programs-surveys/acs/summary_file/2019/documentation/user_tools/ACS2019_Table_Shells.xlsx\n", "censusdata.printtable(censusdata.censustable(src=\"acs5\", year=ACS_YEAR, table=\"B19013\"))" - ] + ], + "outputs": [], + "metadata": { + "scrolled": true + } }, { "cell_type": "code", "execution_count": null, - "id": "8999cea4", - "metadata": { - "scrolled": true - }, - "outputs": [], "source": [ "def fips_from_censusdata_censusgeo(censusgeo: censusdata.censusgeo) -> str:\n", " \"\"\"Create a FIPS code from the proprietary censusgeo index.\"\"\"\n", @@ -85,31 +78,33 @@ "df[GEOID_FIELD_NAME] = df.index.to_series().apply(func=fips_from_censusdata_censusgeo)\n", "\n", "df.head()" - ] + ], + "outputs": [], + "metadata": { + "scrolled": true + } }, { "cell_type": "code", "execution_count": null, - "id": "2a269bb1", - "metadata": { - "scrolled": true - }, - "outputs": [], "source": [ "columns_to_include = [\"GEOID2\", \"Median household income (State)\"]\n", "\n", "df.rename(columns={\"GEOID10\": \"GEOID2\", \"B19013_001E\": \"Median household income (State)\"}, inplace=True)\n", "\n", - "df[columns_to_include].to_csv(path_or_buf= \"/Users/lucas/Documents/usds/repos/justice40-tool/data/data-pipeline/data_pipeline/data/needs_to_be_moved_to_s3/2014_to_2019_state_median_income.csv\", index=False)" - ] + "# df[columns_to_include].to_csv(path_or_buf= \"/Users/lucas/Documents/usds/repos/justice40-tool/data/data-pipeline/data_pipeline/data/needs_to_be_moved_to_s3/2014_to_2019_state_median_income.csv\", index=False)" + ], + "outputs": [], + "metadata": { + "scrolled": true + } }, { "cell_type": "code", "execution_count": null, - "id": "91932af5", - "metadata": {}, + "source": [], "outputs": [], - "source": [] + "metadata": {} } ], "metadata": { @@ -133,4 +128,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/data/data-pipeline/data_pipeline/ipython/scoring_comparison.ipynb b/data/data-pipeline/data_pipeline/ipython/scoring_comparison.ipynb index a3db0c87..22d7b865 100644 --- a/data/data-pipeline/data_pipeline/ipython/scoring_comparison.ipynb +++ b/data/data-pipeline/data_pipeline/ipython/scoring_comparison.ipynb @@ -28,7 +28,7 @@ "from datetime import datetime\n", "from tqdm.notebook import tqdm_notebook\n", "\n", - "module_path = os.path.abspath(os.path.join(\"..\"))\n", + "module_path = os.path.abspath(os.path.join(\"../..\"))\n", "if module_path not in sys.path:\n", " sys.path.append(module_path)\n", "\n", @@ -215,7 +215,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8b795fb4", + "id": "274f6bc6", "metadata": {}, "outputs": [], "source": [ @@ -234,6 +234,21 @@ "# (`census_tract_indices`).\n", "census_block_group_indices = [\n", " Index(\n", + " method_name=\"Score F\",\n", + " priority_communities_field=\"Score F (communities)\",\n", + " other_census_tract_fields_to_keep=[],\n", + " ),\n", + " Index(\n", + " method_name=\"Score F (socioeconomic only)\",\n", + " priority_communities_field=\"Meets socioeconomic criteria\",\n", + " other_census_tract_fields_to_keep=[],\n", + " ),\n", + " Index(\n", + " method_name=\"Score F (burden only)\",\n", + " priority_communities_field=\"Meets burden criteria\",\n", + " other_census_tract_fields_to_keep=[],\n", + " ),\n", + " Index(\n", " method_name=\"Score A\",\n", " priority_communities_field=\"Score A (top 25th percentile)\",\n", " other_census_tract_fields_to_keep=[],\n", @@ -253,21 +268,21 @@ " priority_communities_field=\"Score D (top 25th percentile)\",\n", " other_census_tract_fields_to_keep=[],\n", " ),\n", - " Index(\n", - " method_name=\"Score D (30th percentile)\",\n", - " priority_communities_field=\"Score D (top 30th percentile)\",\n", - " other_census_tract_fields_to_keep=[],\n", - " ),\n", - " Index(\n", - " method_name=\"Score D (35th percentile)\",\n", - " priority_communities_field=\"Score D (top 35th percentile)\",\n", - " other_census_tract_fields_to_keep=[],\n", - " ),\n", - " Index(\n", - " method_name=\"Score D (40th percentile)\",\n", - " priority_communities_field=\"Score D (top 40th percentile)\",\n", - " other_census_tract_fields_to_keep=[],\n", - " ),\n", + "# Index(\n", + "# method_name=\"Score D (30th percentile)\",\n", + "# priority_communities_field=\"Score D (top 30th percentile)\",\n", + "# other_census_tract_fields_to_keep=[],\n", + "# ),\n", + "# Index(\n", + "# method_name=\"Score D (35th percentile)\",\n", + "# priority_communities_field=\"Score D (top 35th percentile)\",\n", + "# other_census_tract_fields_to_keep=[],\n", + "# ),\n", + "# Index(\n", + "# method_name=\"Score D (40th percentile)\",\n", + "# priority_communities_field=\"Score D (top 40th percentile)\",\n", + "# other_census_tract_fields_to_keep=[],\n", + "# ),\n", " Index(\n", " method_name=\"Poverty\",\n", " priority_communities_field=\"Poverty (Less than 200% of federal poverty line) (top 25th percentile)\",\n", @@ -534,7 +549,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d7acf80d", + "id": "eeb9699d", "metadata": {}, "outputs": [], "source": [ @@ -682,7 +697,7 @@ { "cell_type": "code", "execution_count": null, - "id": "777a4623", + "id": "4f44426c", "metadata": {}, "outputs": [], "source": [ @@ -1140,14 +1155,6 @@ "\n", "print(file_paths)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e679502a", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/data/data-pipeline/data_pipeline/utils.py b/data/data-pipeline/data_pipeline/utils.py index 4263a1a5..d0dd81a6 100644 --- a/data/data-pipeline/data_pipeline/utils.py +++ b/data/data-pipeline/data_pipeline/utils.py @@ -98,11 +98,50 @@ def remove_all_dirs_from_dir(dir_path: Path) -> None: logging.info(f"Removing directory {file_path}") +def download_file_from_url( + file_url: str, + download_file_name: Path, + verify: bool = True, +) -> str: + """Downloads a file from a remote URL location and returns the file location. + + Args: + file_url (str): URL where the zip file is located + download_file_name (pathlib.Path): file path where the file will be downloaded (called downloaded.zip by default) + verify (bool): A flag to check if the certificate is valid. If truthy, an invalid certificate will throw an error (optional, default to False) + + Returns: + None + + """ + # disable https warning + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + if not os.path.isdir(download_file_name.parent): + os.mkdir(download_file_name.parent) + + logger.info(f"Downloading {file_url}") + response = requests.get(file_url, verify=verify) + if response.status_code == 200: + file_contents = response.content + else: + sys.exit( + f"HTTP response {response.status_code} from url {file_url}. Info: {response.content}" + ) + + # Write the contents to disk. + file = open(download_file_name, "wb") + file.write(file_contents) + file.close() + + return download_file_name + + def unzip_file_from_url( file_url: str, download_path: Path, unzipped_file_path: Path, - verify: bool = False, + verify: bool = True, ) -> None: """Downloads a zip file from a remote URL location and unzips it in a specific directory, removing the temporary file after @@ -116,23 +155,11 @@ def unzip_file_from_url( None """ - - # disable https warning - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - - logger.info(f"Downloading {file_url}") - response = requests.get(file_url, verify=verify) - if response.status_code == 200: - file_contents = response.content - else: - sys.exit( - f"HTTP response {response.status_code} from url {file_url}. Info: {response.content}" - ) - - zip_file_path = download_path / "downloaded.zip" - zip_file = open(zip_file_path, "wb") - zip_file.write(file_contents) - zip_file.close() + zip_file_path = download_file_from_url( + file_url=file_url, + download_file_name=download_path / "downloaded.zip", + verify=verify, + ) logger.info(f"Extracting {zip_file_path}") with zipfile.ZipFile(zip_file_path, "r") as zip_ref: diff --git a/data/data-pipeline/poetry.lock b/data/data-pipeline/poetry.lock index 1b3a6456..cb4adbe5 100644 --- a/data/data-pipeline/poetry.lock +++ b/data/data-pipeline/poetry.lock @@ -14,6 +14,20 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "argcomplete" +version = "1.12.3" +description = "Bash tab completion for argparse" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +importlib-metadata = {version = ">=0.23,<5", markers = "python_version == \"3.7\""} + +[package.extras] +test = ["coverage", "flake8", "pexpect", "wheel"] + [[package]] name = "argon2-cffi" version = "20.1.0" @@ -237,6 +251,17 @@ python-versions = ">=3.6" docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "pytest-enabler", "pytest-black (>=0.3.7)", "pytest-mypy"] +[[package]] +name = "cycler" +version = "0.10.0" +description = "Composable style cycles" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = "*" + [[package]] name = "debugpy" version = "1.4.1" @@ -387,7 +412,7 @@ python-versions = ">=3.5" [[package]] name = "importlib-metadata" -version = "3.10.1" +version = "4.6.3" description = "Read metadata from Python packages" category = "main" optional = false @@ -399,7 +424,8 @@ zipp = ">=0.5" [package.extras] docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] +perf = ["ipython"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -411,7 +437,7 @@ python-versions = "*" [[package]] name = "ipykernel" -version = "6.0.3" +version = "6.1.0" description = "IPython Kernel for Jupyter" category = "main" optional = false @@ -419,8 +445,9 @@ python-versions = ">=3.7" [package.dependencies] appnope = {version = "*", markers = "platform_system == \"Darwin\""} +argcomplete = {version = ">=1.12.3", markers = "python_version < \"3.8.0\""} debugpy = ">=1.0.0,<2.0" -importlib-metadata = {version = "<4", markers = "python_version < \"3.8.0\""} +importlib-metadata = {version = "<5", markers = "python_version < \"3.8.0\""} ipython = ">=7.23.1,<8.0" jupyter-client = "<7.0" matplotlib-inline = ">=0.1.0,<0.2.0" @@ -727,6 +754,14 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "kiwisolver" +version = "1.3.1" +description = "A fast implementation of the Cassowary constraint solver" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "lazy-object-proxy" version = "1.6.0" @@ -770,6 +805,22 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "matplotlib" +version = "3.4.3" +description = "Python plotting package" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +cycler = ">=0.10" +kiwisolver = ">=1.0.1" +numpy = ">=1.16" +pillow = ">=6.2.0" +pyparsing = ">=2.2.1" +python-dateutil = ">=2.7" + [[package]] name = "matplotlib-inline" version = "0.1.2" @@ -916,7 +967,7 @@ python-versions = ">=3.5" [[package]] name = "notebook" -version = "6.4.2" +version = "6.4.3" description = "A web-based notebook environment for interactive computing" category = "main" optional = false @@ -1036,6 +1087,14 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "pillow" +version = "8.3.1" +description = "Python Imaging Library (Fork)" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "platformdirs" version = "2.2.0" @@ -1373,7 +1432,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "terminado" -version = "0.10.1" +version = "0.11.0" description = "Tornado websocket backend for the Xterm.js Javascript terminal emulator library." category = "main" optional = false @@ -1525,7 +1584,7 @@ jellyfish = "0.6.1" [[package]] name = "virtualenv" -version = "20.7.1" +version = "20.7.2" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -1601,7 +1660,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "14f5225f41212af2785865f984c8ce402712df0398b59c460cb849a0792b3f17" +content-hash = "7fd9dc6d4902d1010da038192ba1a074d9c50264cb4b5d5fabfc45869f9f718a" [metadata.files] appdirs = [ @@ -1612,6 +1671,10 @@ appnope = [ {file = "appnope-0.1.2-py2.py3-none-any.whl", hash = "sha256:93aa393e9d6c54c5cd570ccadd8edad61ea0c4b9ea7a01409020c9aa019eb442"}, {file = "appnope-0.1.2.tar.gz", hash = "sha256:dd83cd4b5b460958838f6eb3000c660b1f9caf2a5b1de4264e941512f603258a"}, ] +argcomplete = [ + {file = "argcomplete-1.12.3-py2.py3-none-any.whl", hash = "sha256:291f0beca7fd49ce285d2f10e4c1c77e9460cf823eef2de54df0c0fec88b0d81"}, + {file = "argcomplete-1.12.3.tar.gz", hash = "sha256:2c7dbffd8c045ea534921e63b0be6fe65e88599990d8dc408ac8c542b72a5445"}, +] argon2-cffi = [ {file = "argon2-cffi-20.1.0.tar.gz", hash = "sha256:d8029b2d3e4b4cea770e9e5a0104dd8fa185c1724a0f01528ae4826a6d25f97d"}, {file = "argon2_cffi-20.1.0-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:6ea92c980586931a816d61e4faf6c192b4abce89aa767ff6581e6ddc985ed003"}, @@ -1741,6 +1804,10 @@ configparser = [ {file = "configparser-5.0.2-py3-none-any.whl", hash = "sha256:af59f2cdd7efbdd5d111c1976ecd0b82db9066653362f0962d7bf1d3ab89a1fa"}, {file = "configparser-5.0.2.tar.gz", hash = "sha256:85d5de102cfe6d14a5172676f09d19c465ce63d6019cf0a4ef13385fc535e828"}, ] +cycler = [ + {file = "cycler-0.10.0-py2.py3-none-any.whl", hash = "sha256:1d8a5ae1ff6c5cf9b93e8811e581232ad8920aeec647c37316ceac982b08cb2d"}, + {file = "cycler-0.10.0.tar.gz", hash = "sha256:cd7b2d1018258d7247a71425e9f26463dfb444d411c39569972f4ce586b0c9d8"}, +] debugpy = [ {file = "debugpy-1.4.1-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:a2c5a1c49239707ed5bc8e97d8f9252fb392d9e13c79c7b477593d7dde4ae24a"}, {file = "debugpy-1.4.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:ebc241351791595796864a960892e1cd58627064feda939d0377edd0730bbff2"}, @@ -1855,16 +1922,16 @@ idna = [ {file = "idna-3.2.tar.gz", hash = "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"}, ] importlib-metadata = [ - {file = "importlib_metadata-3.10.1-py3-none-any.whl", hash = "sha256:2ec0faae539743ae6aaa84b49a169670a465f7f5d64e6add98388cc29fd1f2f6"}, - {file = "importlib_metadata-3.10.1.tar.gz", hash = "sha256:c9356b657de65c53744046fa8f7358afe0714a1af7d570c00c3835c2d724a7c1"}, + {file = "importlib_metadata-4.6.3-py3-none-any.whl", hash = "sha256:51c6635429c77cf1ae634c997ff9e53ca3438b495f10a55ba28594dd69764a8b"}, + {file = "importlib_metadata-4.6.3.tar.gz", hash = "sha256:0645585859e9a6689c523927a5032f2ba5919f1f7d0e84bd4533312320de1ff9"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] ipykernel = [ - {file = "ipykernel-6.0.3-py3-none-any.whl", hash = "sha256:9f9f41a14caf2fde2b7802446adf83885afcbf50585a46d6c687292599a3c3af"}, - {file = "ipykernel-6.0.3.tar.gz", hash = "sha256:0df34a78c7e1422800d6078cde65ccdcdb859597046c338c759db4dbc535c58f"}, + {file = "ipykernel-6.1.0-py3-none-any.whl", hash = "sha256:804202fb4a621dce163bf88ce2687c98450d7ed728ef1d17d6f5ed20744c6e02"}, + {file = "ipykernel-6.1.0.tar.gz", hash = "sha256:e21a718c696ded7d4d5e25b13d2bdd88e099e782fd3be66f9d2e66397543d283"}, ] ipython = [ {file = "ipython-7.26.0-py3-none-any.whl", hash = "sha256:892743b65c21ed72b806a3a602cca408520b3200b89d1924f4b3d2cdb3692362"}, @@ -1940,6 +2007,40 @@ jupyterlab-widgets = [ {file = "jupyterlab_widgets-1.0.0-py3-none-any.whl", hash = "sha256:caeaf3e6103180e654e7d8d2b81b7d645e59e432487c1d35a41d6d3ee56b3fef"}, {file = "jupyterlab_widgets-1.0.0.tar.gz", hash = "sha256:5c1a29a84d3069208cb506b10609175b249b6486d6b1cbae8fcde2a11584fb78"}, ] +kiwisolver = [ + {file = "kiwisolver-1.3.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:fd34fbbfbc40628200730bc1febe30631347103fc8d3d4fa012c21ab9c11eca9"}, + {file = "kiwisolver-1.3.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:d3155d828dec1d43283bd24d3d3e0d9c7c350cdfcc0bd06c0ad1209c1bbc36d0"}, + {file = "kiwisolver-1.3.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:5a7a7dbff17e66fac9142ae2ecafb719393aaee6a3768c9de2fd425c63b53e21"}, + {file = "kiwisolver-1.3.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:f8d6f8db88049a699817fd9178782867bf22283e3813064302ac59f61d95be05"}, + {file = "kiwisolver-1.3.1-cp36-cp36m-manylinux2014_ppc64le.whl", hash = "sha256:5f6ccd3dd0b9739edcf407514016108e2280769c73a85b9e59aa390046dbf08b"}, + {file = "kiwisolver-1.3.1-cp36-cp36m-win32.whl", hash = "sha256:225e2e18f271e0ed8157d7f4518ffbf99b9450fca398d561eb5c4a87d0986dd9"}, + {file = "kiwisolver-1.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:cf8b574c7b9aa060c62116d4181f3a1a4e821b2ec5cbfe3775809474113748d4"}, + {file = "kiwisolver-1.3.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:232c9e11fd7ac3a470d65cd67e4359eee155ec57e822e5220322d7b2ac84fbf0"}, + {file = "kiwisolver-1.3.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:b38694dcdac990a743aa654037ff1188c7a9801ac3ccc548d3341014bc5ca278"}, + {file = "kiwisolver-1.3.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ca3820eb7f7faf7f0aa88de0e54681bddcb46e485beb844fcecbcd1c8bd01689"}, + {file = "kiwisolver-1.3.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:c8fd0f1ae9d92b42854b2979024d7597685ce4ada367172ed7c09edf2cef9cb8"}, + {file = "kiwisolver-1.3.1-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:1e1bc12fb773a7b2ffdeb8380609f4f8064777877b2225dec3da711b421fda31"}, + {file = "kiwisolver-1.3.1-cp37-cp37m-win32.whl", hash = "sha256:72c99e39d005b793fb7d3d4e660aed6b6281b502e8c1eaf8ee8346023c8e03bc"}, + {file = "kiwisolver-1.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:8be8d84b7d4f2ba4ffff3665bcd0211318aa632395a1a41553250484a871d454"}, + {file = "kiwisolver-1.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:31dfd2ac56edc0ff9ac295193eeaea1c0c923c0355bf948fbd99ed6018010b72"}, + {file = "kiwisolver-1.3.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:563c649cfdef27d081c84e72a03b48ea9408c16657500c312575ae9d9f7bc1c3"}, + {file = "kiwisolver-1.3.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:78751b33595f7f9511952e7e60ce858c6d64db2e062afb325985ddbd34b5c131"}, + {file = "kiwisolver-1.3.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:a357fd4f15ee49b4a98b44ec23a34a95f1e00292a139d6015c11f55774ef10de"}, + {file = "kiwisolver-1.3.1-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:5989db3b3b34b76c09253deeaf7fbc2707616f130e166996606c284395da3f18"}, + {file = "kiwisolver-1.3.1-cp38-cp38-win32.whl", hash = "sha256:c08e95114951dc2090c4a630c2385bef681cacf12636fb0241accdc6b303fd81"}, + {file = "kiwisolver-1.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:44a62e24d9b01ba94ae7a4a6c3fb215dc4af1dde817e7498d901e229aaf50e4e"}, + {file = "kiwisolver-1.3.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:50af681a36b2a1dee1d3c169ade9fdc59207d3c31e522519181e12f1b3ba7000"}, + {file = "kiwisolver-1.3.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:a53d27d0c2a0ebd07e395e56a1fbdf75ffedc4a05943daf472af163413ce9598"}, + {file = "kiwisolver-1.3.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:834ee27348c4aefc20b479335fd422a2c69db55f7d9ab61721ac8cd83eb78882"}, + {file = "kiwisolver-1.3.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:5c3e6455341008a054cccee8c5d24481bcfe1acdbc9add30aa95798e95c65621"}, + {file = "kiwisolver-1.3.1-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:acef3d59d47dd85ecf909c359d0fd2c81ed33bdff70216d3956b463e12c38a54"}, + {file = "kiwisolver-1.3.1-cp39-cp39-win32.whl", hash = "sha256:c5518d51a0735b1e6cee1fdce66359f8d2b59c3ca85dc2b0813a8aa86818a030"}, + {file = "kiwisolver-1.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:b9edd0110a77fc321ab090aaa1cfcaba1d8499850a12848b81be2222eab648f6"}, + {file = "kiwisolver-1.3.1-pp36-pypy36_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0cd53f403202159b44528498de18f9285b04482bab2a6fc3f5dd8dbb9352e30d"}, + {file = "kiwisolver-1.3.1-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:33449715e0101e4d34f64990352bce4095c8bf13bed1b390773fc0a7295967b3"}, + {file = "kiwisolver-1.3.1-pp36-pypy36_pp73-win32.whl", hash = "sha256:401a2e9afa8588589775fe34fc22d918ae839aaaf0c0e96441c0fdbce6d8ebe6"}, + {file = "kiwisolver-1.3.1.tar.gz", hash = "sha256:950a199911a8d94683a6b10321f9345d5a3a8433ec58b217ace979e18f16e248"}, +] lazy-object-proxy = [ {file = "lazy-object-proxy-1.6.0.tar.gz", hash = "sha256:489000d368377571c6f982fba6497f2aa13c6d1facc40660963da62f5c379726"}, {file = "lazy_object_proxy-1.6.0-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:c6938967f8528b3668622a9ed3b31d145fab161a32f5891ea7b84f6b790be05b"}, @@ -2052,6 +2153,29 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"}, {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"}, ] +matplotlib = [ + {file = "matplotlib-3.4.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c988bb43414c7c2b0a31bd5187b4d27fd625c080371b463a6d422047df78913"}, + {file = "matplotlib-3.4.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f1c5efc278d996af8a251b2ce0b07bbeccb821f25c8c9846bdcb00ffc7f158aa"}, + {file = "matplotlib-3.4.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:eeb1859efe7754b1460e1d4991bbd4a60a56f366bc422ef3a9c5ae05f0bc70b5"}, + {file = "matplotlib-3.4.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:844a7b0233e4ff7fba57e90b8799edaa40b9e31e300b8d5efc350937fa8b1bea"}, + {file = "matplotlib-3.4.3-cp37-cp37m-win32.whl", hash = "sha256:85f0c9cf724715e75243a7b3087cf4a3de056b55e05d4d76cc58d610d62894f3"}, + {file = "matplotlib-3.4.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c70b6311dda3e27672f1bf48851a0de816d1ca6aaf3d49365fbdd8e959b33d2b"}, + {file = "matplotlib-3.4.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b884715a59fec9ad3b6048ecf3860f3b2ce965e676ef52593d6fa29abcf7d330"}, + {file = "matplotlib-3.4.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:a78a3b51f29448c7f4d4575e561f6b0dbb8d01c13c2046ab6c5220eb25c06506"}, + {file = "matplotlib-3.4.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:6a724e3a48a54b8b6e7c4ae38cd3d07084508fa47c410c8757e9db9791421838"}, + {file = "matplotlib-3.4.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:48e1e0859b54d5f2e29bb78ca179fd59b971c6ceb29977fb52735bfd280eb0f5"}, + {file = "matplotlib-3.4.3-cp38-cp38-win32.whl", hash = "sha256:01c9de93a2ca0d128c9064f23709362e7fefb34910c7c9e0b8ab0de8258d5eda"}, + {file = "matplotlib-3.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:ebfb01a65c3f5d53a8c2a8133fec2b5221281c053d944ae81ff5822a68266617"}, + {file = "matplotlib-3.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b8b53f336a4688cfce615887505d7e41fd79b3594bf21dd300531a4f5b4f746a"}, + {file = "matplotlib-3.4.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:fcd6f1954943c0c192bfbebbac263f839d7055409f1173f80d8b11a224d236da"}, + {file = "matplotlib-3.4.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:6be8df61b1626e1a142c57e065405e869e9429b4a6dab4a324757d0dc4d42235"}, + {file = "matplotlib-3.4.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:41b6e307458988891fcdea2d8ecf84a8c92d53f84190aa32da65f9505546e684"}, + {file = "matplotlib-3.4.3-cp39-cp39-win32.whl", hash = "sha256:f72657f1596199dc1e4e7a10f52a4784ead8a711f4e5b59bea95bdb97cf0e4fd"}, + {file = "matplotlib-3.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:f15edcb0629a0801738925fe27070480f446fcaa15de65946ff946ad99a59a40"}, + {file = "matplotlib-3.4.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:556965514b259204637c360d213de28d43a1f4aed1eca15596ce83f768c5a56f"}, + {file = "matplotlib-3.4.3-pp37-pypy37_pp73-manylinux2010_x86_64.whl", hash = "sha256:54a026055d5f8614f184e588f6e29064019a0aa8448450214c0b60926d62d919"}, + {file = "matplotlib-3.4.3.tar.gz", hash = "sha256:fc4f526dfdb31c9bd6b8ca06bf9fab663ca12f3ec9cdf4496fb44bc680140318"}, +] matplotlib-inline = [ {file = "matplotlib-inline-0.1.2.tar.gz", hash = "sha256:f41d5ff73c9f5385775d5c0bc13b424535c8402fe70ea8210f93e11f3683993e"}, {file = "matplotlib_inline-0.1.2-py3-none-any.whl", hash = "sha256:5cf1176f554abb4fa98cb362aa2b55c500147e4bdbb07e3fda359143e1da0811"}, @@ -2114,8 +2238,8 @@ nest-asyncio = [ {file = "nest_asyncio-1.5.1.tar.gz", hash = "sha256:afc5a1c515210a23c461932765691ad39e8eba6551c055ac8d5546e69250d0aa"}, ] notebook = [ - {file = "notebook-6.4.2-py3-none-any.whl", hash = "sha256:5ae23d7f831a5788e8bd51a0ba65c486db3bfd43e9db97a62330b6273e3175e3"}, - {file = "notebook-6.4.2.tar.gz", hash = "sha256:ba9db5e5a9bd2d272b67e3de9143cca2be5125578f1c4f2902d7178ce2f0b4ff"}, + {file = "notebook-6.4.3-py3-none-any.whl", hash = "sha256:b50eafa8208d5db966efd1caa4076b4dfc51815e02a805b32ecd717e9e6cc071"}, + {file = "notebook-6.4.3.tar.gz", hash = "sha256:e6b6dfed36b00cf950f63c0d42e947c101d4258aec21624de62b9e0c11ed5c0d"}, ] numpy = [ {file = "numpy-1.21.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38e8648f9449a549a7dfe8d8755a5979b45b3538520d1e735637ef28e8c2dc50"}, @@ -2195,6 +2319,47 @@ pickleshare = [ {file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"}, {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, ] +pillow = [ + {file = "Pillow-8.3.1-1-cp36-cp36m-win_amd64.whl", hash = "sha256:fd7eef578f5b2200d066db1b50c4aa66410786201669fb76d5238b007918fb24"}, + {file = "Pillow-8.3.1-1-cp37-cp37m-win_amd64.whl", hash = "sha256:75e09042a3b39e0ea61ce37e941221313d51a9c26b8e54e12b3ececccb71718a"}, + {file = "Pillow-8.3.1-1-cp38-cp38-win_amd64.whl", hash = "sha256:c0e0550a404c69aab1e04ae89cca3e2a042b56ab043f7f729d984bf73ed2a093"}, + {file = "Pillow-8.3.1-1-cp39-cp39-win_amd64.whl", hash = "sha256:479ab11cbd69612acefa8286481f65c5dece2002ffaa4f9db62682379ca3bb77"}, + {file = "Pillow-8.3.1-1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:f156d6ecfc747ee111c167f8faf5f4953761b5e66e91a4e6767e548d0f80129c"}, + {file = "Pillow-8.3.1-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:196560dba4da7a72c5e7085fccc5938ab4075fd37fe8b5468869724109812edd"}, + {file = "Pillow-8.3.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c9569049d04aaacd690573a0398dbd8e0bf0255684fee512b413c2142ab723"}, + {file = "Pillow-8.3.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c088a000dfdd88c184cc7271bfac8c5b82d9efa8637cd2b68183771e3cf56f04"}, + {file = "Pillow-8.3.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:fc214a6b75d2e0ea7745488da7da3c381f41790812988c7a92345978414fad37"}, + {file = "Pillow-8.3.1-cp36-cp36m-win32.whl", hash = "sha256:a17ca41f45cf78c2216ebfab03add7cc350c305c38ff34ef4eef66b7d76c5229"}, + {file = "Pillow-8.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:67b3666b544b953a2777cb3f5a922e991be73ab32635666ee72e05876b8a92de"}, + {file = "Pillow-8.3.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:ff04c373477723430dce2e9d024c708a047d44cf17166bf16e604b379bf0ca14"}, + {file = "Pillow-8.3.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9364c81b252d8348e9cc0cb63e856b8f7c1b340caba6ee7a7a65c968312f7dab"}, + {file = "Pillow-8.3.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a2f381932dca2cf775811a008aa3027671ace723b7a38838045b1aee8669fdcf"}, + {file = "Pillow-8.3.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:d0da39795049a9afcaadec532e7b669b5ebbb2a9134576ebcc15dd5bdae33cc0"}, + {file = "Pillow-8.3.1-cp37-cp37m-win32.whl", hash = "sha256:2b6dfa068a8b6137da34a4936f5a816aba0ecc967af2feeb32c4393ddd671cba"}, + {file = "Pillow-8.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a4eef1ff2d62676deabf076f963eda4da34b51bc0517c70239fafed1d5b51500"}, + {file = "Pillow-8.3.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:660a87085925c61a0dcc80efb967512ac34dbb256ff7dd2b9b4ee8dbdab58cf4"}, + {file = "Pillow-8.3.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:15a2808e269a1cf2131930183dcc0419bc77bb73eb54285dde2706ac9939fa8e"}, + {file = "Pillow-8.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:969cc558cca859cadf24f890fc009e1bce7d7d0386ba7c0478641a60199adf79"}, + {file = "Pillow-8.3.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2ee77c14a0299d0541d26f3d8500bb57e081233e3fa915fa35abd02c51fa7fae"}, + {file = "Pillow-8.3.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c11003197f908878164f0e6da15fce22373ac3fc320cda8c9d16e6bba105b844"}, + {file = "Pillow-8.3.1-cp38-cp38-win32.whl", hash = "sha256:3f08bd8d785204149b5b33e3b5f0ebbfe2190ea58d1a051c578e29e39bfd2367"}, + {file = "Pillow-8.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:70af7d222df0ff81a2da601fab42decb009dc721545ed78549cb96e3a1c5f0c8"}, + {file = "Pillow-8.3.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:37730f6e68bdc6a3f02d2079c34c532330d206429f3cee651aab6b66839a9f0e"}, + {file = "Pillow-8.3.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4bc3c7ef940eeb200ca65bd83005eb3aae8083d47e8fcbf5f0943baa50726856"}, + {file = "Pillow-8.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c35d09db702f4185ba22bb33ef1751ad49c266534339a5cebeb5159d364f6f82"}, + {file = "Pillow-8.3.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0b2efa07f69dc395d95bb9ef3299f4ca29bcb2157dc615bae0b42c3c20668ffc"}, + {file = "Pillow-8.3.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:cc866706d56bd3a7dbf8bac8660c6f6462f2f2b8a49add2ba617bc0c54473d83"}, + {file = "Pillow-8.3.1-cp39-cp39-win32.whl", hash = "sha256:9a211b663cf2314edbdb4cf897beeb5c9ee3810d1d53f0e423f06d6ebbf9cd5d"}, + {file = "Pillow-8.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:c2a5ff58751670292b406b9f06e07ed1446a4b13ffced6b6cab75b857485cbc8"}, + {file = "Pillow-8.3.1-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:c379425c2707078dfb6bfad2430728831d399dc95a7deeb92015eb4c92345eaf"}, + {file = "Pillow-8.3.1-pp36-pypy36_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:114f816e4f73f9ec06997b2fde81a92cbf0777c9e8f462005550eed6bae57e63"}, + {file = "Pillow-8.3.1-pp36-pypy36_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8960a8a9f4598974e4c2aeb1bff9bdd5db03ee65fd1fce8adf3223721aa2a636"}, + {file = "Pillow-8.3.1-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:147bd9e71fb9dcf08357b4d530b5167941e222a6fd21f869c7911bac40b9994d"}, + {file = "Pillow-8.3.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1fd5066cd343b5db88c048d971994e56b296868766e461b82fa4e22498f34d77"}, + {file = "Pillow-8.3.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f4ebde71785f8bceb39dcd1e7f06bcc5d5c3cf48b9f69ab52636309387b097c8"}, + {file = "Pillow-8.3.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:1c03e24be975e2afe70dfc5da6f187eea0b49a68bb2b69db0f30a61b7031cee4"}, + {file = "Pillow-8.3.1.tar.gz", hash = "sha256:2cac53839bfc5cece8fdbe7f084d5e3ee61e1303cccc86511d351adcb9e2c792"}, +] platformdirs = [ {file = "platformdirs-2.2.0-py3-none-any.whl", hash = "sha256:4666d822218db6a262bdfdc9c39d21f23b4cfdb08af331a81e92751daf6c866c"}, {file = "platformdirs-2.2.0.tar.gz", hash = "sha256:632daad3ab546bd8e6af0537d09805cec458dce201bccfe23012df73332e181e"}, @@ -2481,8 +2646,8 @@ six = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] terminado = [ - {file = "terminado-0.10.1-py3-none-any.whl", hash = "sha256:c89ace5bffd0e7268bdcf22526830eb787fd146ff9d78691a0528386f92b9ae3"}, - {file = "terminado-0.10.1.tar.gz", hash = "sha256:89d5dac2f4e2b39758a0ff9a3b643707c95a020a6df36e70583b88297cd59cbe"}, + {file = "terminado-0.11.0-py3-none-any.whl", hash = "sha256:221eef83e6a504894842f7dccfa971ca2e98ec22a8a9118577e5257527674b42"}, + {file = "terminado-0.11.0.tar.gz", hash = "sha256:1e01183885f64c1bba3cf89a5a995ad4acfed4e5f00aebcce1bf7f089b0825a1"}, ] testpath = [ {file = "testpath-0.5.0-py3-none-any.whl", hash = "sha256:8044f9a0bab6567fc644a3593164e872543bb44225b0e24846e2c89237937589"}, @@ -2600,8 +2765,8 @@ us = [ {file = "us-2.0.2.tar.gz", hash = "sha256:cb11ad0d43deff3a1c3690c74f0c731cff5b862c73339df2edd91133e1496fbc"}, ] virtualenv = [ - {file = "virtualenv-20.7.1-py2.py3-none-any.whl", hash = "sha256:73863dc3be1efe6ee638e77495c0c195a6384ae7b15c561f3ceb2698ae7267c1"}, - {file = "virtualenv-20.7.1.tar.gz", hash = "sha256:57bcb59c5898818bd555b1e0cfcf668bd6204bc2b53ad0e70a52413bd790f9e4"}, + {file = "virtualenv-20.7.2-py2.py3-none-any.whl", hash = "sha256:e4670891b3a03eb071748c569a87cceaefbf643c5bac46d996c5a45c34aa0f06"}, + {file = "virtualenv-20.7.2.tar.gz", hash = "sha256:9ef4e8ee4710826e98ff3075c9a4739e2cb1040de6a2a8d35db0055840dc96a0"}, ] wcwidth = [ {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, diff --git a/data/data-pipeline/pyproject.toml b/data/data-pipeline/pyproject.toml index b87c97ca..55ace4b1 100644 --- a/data/data-pipeline/pyproject.toml +++ b/data/data-pipeline/pyproject.toml @@ -12,6 +12,7 @@ geopandas = "^0.9.0" ipython = "^7.24.1" jupyter = "^1.0.0" jupyter-contrib-nbextensions = "^0.5.1" +matplotlib = "^3.4.2" numpy = "^1.21.0" pandas = "^1.2.5" python = "^3.7.1" @@ -61,6 +62,7 @@ disable = [ "W0613", # Disables unused argument "C0116", # Disables missing function or method docstring "C0115", # Disables missing class docstring + "R0915", # Disables too many statements (score generation transform) ] [tool.pylint.FORMAT] @@ -73,6 +75,9 @@ ignore-docstrings = "yes" ignore-imports = "yes" min-similarity-lines = 4 +[tool.black] +line-length = 80 + [tool.liccheck] # Authorized and unauthorized licenses in LOWER CASE authorized_licenses = [ @@ -102,6 +107,7 @@ authorized_licenses = [ "python software foundation", "zpl 2.1", "gpl v3", + "historical permission notice and disclaimer (hpnd)", ] [tool.poetry.scripts] diff --git a/data/data-pipeline/requirements.txt b/data/data-pipeline/requirements.txt index 253b738a..f47cdde1 100644 --- a/data/data-pipeline/requirements.txt +++ b/data/data-pipeline/requirements.txt @@ -1,4 +1,5 @@ appnope==0.1.2; sys_platform == "darwin" and python_version >= "3.7" and platform_system == "Darwin" +argcomplete==1.12.3; python_version < "3.8.0" and python_version >= "3.7" argon2-cffi==20.1.0; python_version >= "3.6" async-generator==1.10; python_full_version >= "3.6.1" and python_version >= "3.7" attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" @@ -12,6 +13,7 @@ click-plugins==1.1.1; python_version >= "3.6" click==8.0.1; python_version >= "3.6" cligj==0.7.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version < "4" and python_version >= "3.6" colorama==0.4.4; python_version >= "3.7" and python_full_version < "3.0.0" and platform_system == "Windows" and sys_platform == "win32" or platform_system == "Windows" and python_version >= "3.7" and python_full_version >= "3.5.0" and sys_platform == "win32" +cycler==0.10.0; python_version >= "3.7" debugpy==1.4.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7" decorator==5.0.9; python_version >= "3.7" defusedxml==0.7.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7" @@ -20,8 +22,8 @@ entrypoints==0.3; python_version >= "3.7" fiona==1.8.20; python_version >= "3.6" geopandas==0.9.0; python_version >= "3.6" idna==3.2; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5" -importlib-metadata==3.10.1; python_version < "3.8" and python_version >= "3.7" -ipykernel==6.0.3; python_version >= "3.7" +importlib-metadata==4.6.3; python_version == "3.7" +ipykernel==6.1.0; python_version >= "3.7" ipython-genutils==0.2.0; python_version >= "3.7" ipython==7.26.0; python_version >= "3.7" ipywidgets==7.6.3 @@ -40,16 +42,18 @@ jupyter-nbextensions-configurator==0.4.1 jupyter==1.0.0 jupyterlab-pygments==0.1.2; python_version >= "3.7" jupyterlab-widgets==1.0.0; python_version >= "3.6" +kiwisolver==1.3.1; python_version >= "3.7" lxml==4.6.3; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" markupsafe==2.0.1; python_version >= "3.7" matplotlib-inline==0.1.2; python_version >= "3.7" +matplotlib==3.4.3; python_version >= "3.7" mistune==0.8.4; python_version >= "3.7" munch==2.5.0; python_version >= "3.6" nbclient==0.5.3; python_full_version >= "3.6.1" and python_version >= "3.7" nbconvert==6.1.0; python_version >= "3.7" nbformat==5.1.3; python_full_version >= "3.6.1" and python_version >= "3.7" nest-asyncio==1.5.1; python_full_version >= "3.6.1" and python_version >= "3.7" -notebook==6.4.2; python_version >= "3.6" +notebook==6.4.3; python_version >= "3.6" numpy==1.21.1; python_version >= "3.7" packaging==21.0; python_version >= "3.7" pandas==1.3.1; python_full_version >= "3.7.1" @@ -57,6 +61,7 @@ pandocfilters==1.4.3; python_version >= "3.7" and python_full_version < "3.0.0" parso==0.8.2; python_version >= "3.7" pexpect==4.8.0; sys_platform != "win32" and python_version >= "3.7" pickleshare==0.7.5; python_version >= "3.7" +pillow==8.3.1; python_version >= "3.7" prometheus-client==0.11.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" prompt-toolkit==3.0.19; python_full_version >= "3.6.1" and python_version >= "3.7" ptyprocess==0.7.0; sys_platform != "win32" and python_version >= "3.7" and os_name != "nt" @@ -78,8 +83,8 @@ qtpy==1.9.0; python_version >= "3.6" requests==2.26.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") send2trash==1.8.0; python_version >= "3.6" shapely==1.7.1; python_version >= "3.6" -six==1.16.0; python_full_version >= "3.7.1" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7") and (python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.5") -terminado==0.10.1; python_version >= "3.6" +six==1.16.0; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7" +terminado==0.11.0; python_version >= "3.6" testpath==0.5.0; python_version >= "3.7" tornado==6.1; python_full_version >= "3.6.1" and python_version >= "3.7" tqdm==4.62.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")