2021-06-28 16:16:14 -04:00
|
|
|
[tool.poetry]
|
2022-03-21 18:55:15 -04:00
|
|
|
name = "justice40-data-pipeline"
|
2021-06-28 16:16:14 -04:00
|
|
|
version = "0.1.0"
|
2022-03-21 18:55:15 -04:00
|
|
|
description = "ETL, Score and Map Generation of Justice 40 Tool"
|
|
|
|
|
authors = ["Justice40 Engineering <j40-engineering@lists.usds.gov>"]
|
|
|
|
|
keywords = ["justice40", "environmental_justice", "python", "etl"]
|
|
|
|
|
readme = "README.md"
|
|
|
|
|
license = "MIT"
|
|
|
|
|
homepage = "https://github.com/usds/justice40-tool/tree/main/data/data-pipeline"
|
|
|
|
|
repository = "https://github.com/usds/justice40-tool"
|
|
|
|
|
include = [
|
|
|
|
|
"LICENSE",
|
|
|
|
|
]
|
|
|
|
|
packages = [
|
|
|
|
|
{include = "data_pipeline"}
|
|
|
|
|
]
|
2021-06-28 16:16:14 -04:00
|
|
|
|
|
|
|
|
[tool.poetry.dependencies]
|
2021-07-20 14:55:39 -04:00
|
|
|
CensusData = "^1.13"
|
2022-09-26 12:05:40 -04:00
|
|
|
click = "8.0.4" # pinning for now per https://github.com/psf/black/issues/2964
|
2021-07-20 14:55:39 -04:00
|
|
|
dynaconf = "^3.1.4"
|
Add FUDS ETL (#1817)
* Add spatial join method (#1871)
Since we'll need to figure out the tracts for a large number of points
in future tickets, add a utility to handle grabbing the tract geometries
and adding tract data to a point dataset.
* Add FUDS, also jupyter lab (#1871)
* Add YAML configs for FUDS (#1871)
* Allow input geoid to be optional (#1871)
* Add FUDS ETL, tests, test-datae noteobook (#1871)
This adds the ETL class for Formerly Used Defense Sites (FUDS). This is
different from most other ETLs since these FUDS are not provided by
tract, but instead by geographic point, so we need to assign FUDS to
tracts and then do calculations from there.
* Floats -> Ints, as I intended (#1871)
* Floats -> Ints, as I intended (#1871)
* Formatting fixes (#1871)
* Add test false positive GEOIDs (#1871)
* Add gdal binaries (#1871)
* Refactor pandas code to be more idiomatic (#1871)
Per Emma, the more pandas-y way of doing my counts is using np.where to
add the values i need, then groupby and size. It is definitely more
compact, and also I think more correct!
* Update configs per Emma suggestions (#1871)
* Type fixed! (#1871)
* Remove spurious import from vscode (#1871)
* Snapshot update after changing col name (#1871)
* Move up GDAL (#1871)
* Adjust geojson strategy (#1871)
* Try running census separately first (#1871)
* Fix import order (#1871)
* Cleanup cache strategy (#1871)
* Download census data from S3 instead of re-calculating (#1871)
* Clarify pandas code per Emma (#1871)
2022-08-16 13:28:39 -04:00
|
|
|
geopandas = "^0.11.0"
|
2022-03-17 23:19:23 -04:00
|
|
|
ipdb = "^0.13.9"
|
2022-01-27 17:36:14 -05:00
|
|
|
ipython = "^7.31.1"
|
2021-06-28 16:16:14 -04:00
|
|
|
jupyter = "^1.0.0"
|
|
|
|
|
jupyter-contrib-nbextensions = "^0.5.1"
|
2022-03-31 13:56:10 -04:00
|
|
|
marshmallow-dataclass = "^8.5.3"
|
|
|
|
|
marshmallow-enum = "^1.5.1"
|
2021-08-24 15:40:54 -05:00
|
|
|
matplotlib = "^3.4.2"
|
2022-02-03 17:05:51 -05:00
|
|
|
numpy = "^1.22.1"
|
2021-06-28 16:16:14 -04:00
|
|
|
pandas = "^1.2.5"
|
2022-03-17 23:19:23 -04:00
|
|
|
pylint = "^2.11.1"
|
2022-03-02 16:50:04 -05:00
|
|
|
pillow = "9.0.1"
|
2022-02-03 17:05:51 -05:00
|
|
|
python = "^3.8"
|
2021-08-09 22:24:14 -05:00
|
|
|
pypandoc = "^1.6.3"
|
2022-03-17 23:19:23 -04:00
|
|
|
PyYAML = "^6.0"
|
2021-06-28 16:16:14 -04:00
|
|
|
requests = "^2.25.1"
|
2021-08-09 22:24:14 -05:00
|
|
|
tqdm = "4.62.0"
|
2021-06-28 16:16:14 -04:00
|
|
|
types-requests = "^2.25.0"
|
2021-08-09 22:24:14 -05:00
|
|
|
us = "^2.0.2"
|
|
|
|
|
xlsxwriter = "^2.0.0"
|
2022-04-27 15:59:10 -04:00
|
|
|
pydantic = "^1.9.0"
|
Add FUDS ETL (#1817)
* Add spatial join method (#1871)
Since we'll need to figure out the tracts for a large number of points
in future tickets, add a utility to handle grabbing the tract geometries
and adding tract data to a point dataset.
* Add FUDS, also jupyter lab (#1871)
* Add YAML configs for FUDS (#1871)
* Allow input geoid to be optional (#1871)
* Add FUDS ETL, tests, test-datae noteobook (#1871)
This adds the ETL class for Formerly Used Defense Sites (FUDS). This is
different from most other ETLs since these FUDS are not provided by
tract, but instead by geographic point, so we need to assign FUDS to
tracts and then do calculations from there.
* Floats -> Ints, as I intended (#1871)
* Floats -> Ints, as I intended (#1871)
* Formatting fixes (#1871)
* Add test false positive GEOIDs (#1871)
* Add gdal binaries (#1871)
* Refactor pandas code to be more idiomatic (#1871)
Per Emma, the more pandas-y way of doing my counts is using np.where to
add the values i need, then groupby and size. It is definitely more
compact, and also I think more correct!
* Update configs per Emma suggestions (#1871)
* Type fixed! (#1871)
* Remove spurious import from vscode (#1871)
* Snapshot update after changing col name (#1871)
* Move up GDAL (#1871)
* Adjust geojson strategy (#1871)
* Try running census separately first (#1871)
* Fix import order (#1871)
* Cleanup cache strategy (#1871)
* Download census data from S3 instead of re-calculating (#1871)
* Clarify pandas code per Emma (#1871)
2022-08-16 13:28:39 -04:00
|
|
|
Rtree = "^1.0.0"
|
2021-06-28 16:16:14 -04:00
|
|
|
|
|
|
|
|
[tool.poetry.dev-dependencies]
|
|
|
|
|
black = {version = "^21.6b0", allow-prereleases = true}
|
2021-08-02 12:16:38 -04:00
|
|
|
flake8 = "^3.9.2"
|
|
|
|
|
liccheck = "^0.6.2"
|
2021-08-05 15:35:54 -04:00
|
|
|
mypy = "^0.910"
|
2021-08-09 10:39:59 -04:00
|
|
|
openpyxl = "^3.0.7"
|
2021-08-05 15:35:54 -04:00
|
|
|
pylint = "^2.9.6"
|
|
|
|
|
pytest = "^6.2.4"
|
2021-08-02 12:16:38 -04:00
|
|
|
safety = "^1.10.3"
|
2021-08-05 15:35:54 -04:00
|
|
|
tox = "^3.24.0"
|
2021-09-10 14:17:34 -04:00
|
|
|
pytest-mock = "^3.6.1"
|
2022-01-10 16:43:56 -05:00
|
|
|
tox-poetry = "^0.4.1"
|
2022-01-13 13:17:30 -05:00
|
|
|
pandas-vet = "^0.2.2"
|
2022-03-11 21:34:07 -05:00
|
|
|
pytest-snapshot = "^0.8.1"
|
2022-03-02 16:50:04 -05:00
|
|
|
nb-black = "^1.0.7"
|
2022-03-30 14:02:06 -04:00
|
|
|
seaborn = "^0.11.2"
|
|
|
|
|
papermill = "^2.3.4"
|
Add FUDS ETL (#1817)
* Add spatial join method (#1871)
Since we'll need to figure out the tracts for a large number of points
in future tickets, add a utility to handle grabbing the tract geometries
and adding tract data to a point dataset.
* Add FUDS, also jupyter lab (#1871)
* Add YAML configs for FUDS (#1871)
* Allow input geoid to be optional (#1871)
* Add FUDS ETL, tests, test-datae noteobook (#1871)
This adds the ETL class for Formerly Used Defense Sites (FUDS). This is
different from most other ETLs since these FUDS are not provided by
tract, but instead by geographic point, so we need to assign FUDS to
tracts and then do calculations from there.
* Floats -> Ints, as I intended (#1871)
* Floats -> Ints, as I intended (#1871)
* Formatting fixes (#1871)
* Add test false positive GEOIDs (#1871)
* Add gdal binaries (#1871)
* Refactor pandas code to be more idiomatic (#1871)
Per Emma, the more pandas-y way of doing my counts is using np.where to
add the values i need, then groupby and size. It is definitely more
compact, and also I think more correct!
* Update configs per Emma suggestions (#1871)
* Type fixed! (#1871)
* Remove spurious import from vscode (#1871)
* Snapshot update after changing col name (#1871)
* Move up GDAL (#1871)
* Adjust geojson strategy (#1871)
* Try running census separately first (#1871)
* Fix import order (#1871)
* Cleanup cache strategy (#1871)
* Download census data from S3 instead of re-calculating (#1871)
* Clarify pandas code per Emma (#1871)
2022-08-16 13:28:39 -04:00
|
|
|
jupyterlab = "^3.4.4"
|
2021-06-28 16:16:14 -04:00
|
|
|
|
|
|
|
|
[build-system]
|
|
|
|
|
build-backend = "poetry.core.masonry.api"
|
2021-07-21 16:10:32 -04:00
|
|
|
requires = ["poetry-core>=1.0.0"]
|
2021-08-02 12:16:38 -04:00
|
|
|
|
|
|
|
|
[tool.pylint]
|
|
|
|
|
|
|
|
|
|
[tool.pylint."MESSAGE CONTROL"]
|
|
|
|
|
disable = [
|
2021-08-05 15:35:54 -04:00
|
|
|
"C0114", # Disables module docstrings
|
|
|
|
|
"R0903", # Disables too few public methods
|
|
|
|
|
"C0103", # Disables name case styling
|
|
|
|
|
"W0511", # Disables FIXME warning
|
|
|
|
|
"W1203", # Disables f-string interpolation for logging warning # Errors temporarily ignored for further discussion
|
|
|
|
|
"W0107", # Disables unnecessary pass
|
|
|
|
|
"W0221", # Disables arguments differ
|
|
|
|
|
"R0902", # Disables too many instance attributes
|
|
|
|
|
"R0914", # Disables too many local variables
|
|
|
|
|
"W0621", # Disables redefined outer name
|
|
|
|
|
"C0302", # Disables too many lines in module
|
|
|
|
|
"R1732", # Disables consider using "with"
|
|
|
|
|
"R1720", # Disables unnecessary "else" after "raise"
|
|
|
|
|
"C0206", # Disables consider iteratig with ".items()"
|
|
|
|
|
"C0200", # Disables consider using "enumerate" instead of "range" + "len"
|
|
|
|
|
"W0612", # Disables unused variable
|
|
|
|
|
"W0613", # Disables unused argument
|
|
|
|
|
"C0116", # Disables missing function or method docstring
|
|
|
|
|
"C0115", # Disables missing class docstring
|
2021-08-24 15:40:54 -05:00
|
|
|
"R0915", # Disables too many statements (score generation transform)
|
2021-10-13 15:54:15 -04:00
|
|
|
"W0231", # Disables super init not called
|
2021-12-03 15:46:10 -05:00
|
|
|
"R0801", # Disables duplicate code. There are a couple places we have similar code and
|
|
|
|
|
# unfortunately you can't disable this rule for individual lines or files, it's a
|
|
|
|
|
# known bug. https://github.com/PyCQA/pylint/issues/214#
|
2021-08-02 12:16:38 -04:00
|
|
|
]
|
|
|
|
|
|
|
|
|
|
[tool.pylint.FORMAT]
|
2021-08-05 15:35:54 -04:00
|
|
|
max-line-length = 150
|
2021-08-02 12:16:38 -04:00
|
|
|
|
2021-09-07 20:51:34 -04:00
|
|
|
[tool.pylint.typecheck]
|
2021-10-13 15:54:15 -04:00
|
|
|
generated-members = "pandas.*" # fixes E1101 for ETL.df
|
2021-09-07 20:51:34 -04:00
|
|
|
|
2021-08-02 12:16:38 -04:00
|
|
|
[tool.pylint.SIMILARITIES]
|
|
|
|
|
# Configures how pylint detects repetitive code
|
|
|
|
|
ignore-comments = "yes"
|
|
|
|
|
ignore-docstrings = "yes"
|
|
|
|
|
ignore-imports = "yes"
|
2021-08-05 15:35:54 -04:00
|
|
|
min-similarity-lines = 4
|
2021-08-02 12:16:38 -04:00
|
|
|
|
2021-08-24 15:40:54 -05:00
|
|
|
[tool.black]
|
|
|
|
|
line-length = 80
|
|
|
|
|
|
2021-08-02 12:16:38 -04:00
|
|
|
[tool.liccheck]
|
|
|
|
|
# Authorized and unauthorized licenses in LOWER CASE
|
|
|
|
|
authorized_licenses = [
|
2021-08-05 15:35:54 -04:00
|
|
|
"bsd",
|
|
|
|
|
"new bsd",
|
|
|
|
|
"bsd license",
|
|
|
|
|
"bsd 3-clause",
|
|
|
|
|
"new bsd license",
|
|
|
|
|
"simplified bsd",
|
|
|
|
|
"apache",
|
|
|
|
|
"apache 2.0",
|
|
|
|
|
"apache license 2.0",
|
|
|
|
|
"apache software license",
|
|
|
|
|
"apache software",
|
|
|
|
|
"gnu lgpl",
|
|
|
|
|
"gnu lesser general public license v2 (lgplv2)",
|
|
|
|
|
"gnu general public license v2 (gplv2)",
|
|
|
|
|
"gnu library or lesser general public license (lgpl)",
|
|
|
|
|
"lgpl with exceptions or zpl",
|
|
|
|
|
"isc license",
|
|
|
|
|
"isc license (iscl)",
|
|
|
|
|
"mit",
|
|
|
|
|
"mit license",
|
|
|
|
|
"mozilla public license 2.0 (mpl 2.0)",
|
|
|
|
|
"public domain",
|
|
|
|
|
"python software foundation license",
|
|
|
|
|
"python software foundation",
|
|
|
|
|
"zpl 2.1",
|
|
|
|
|
"gpl v3",
|
2021-08-24 15:40:54 -05:00
|
|
|
"historical permission notice and disclaimer (hpnd)",
|
2022-09-26 12:03:18 -04:00
|
|
|
]
|
