mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-02-23 10:04:18 -08:00
d1273b63c5
* Adds dev dependencies to requirements.txt and re-runs black on codebase * Adds test and code for national risk index etl, still in progress * Removes test_data from .gitignore * Adds test data to nation_risk_index tests * Creates tests and ETL class for NRI data * Adds tests for load() and transform() methods of NationalRiskIndexETL * Updates README.md with info about the NRI dataset * Adds to dos * Moves tests and test data into a tests/ dir in national_risk_index * Moves tmp_dir for tests into data/tmp/tests/ * Promotes fixtures to conftest and relocates national_risk_index tests: The relocation of national_risk_index tests is necessary because tests can only use fixtures specified in conftests within the same package * Fixes issue with df.equals() in test_transform() * Files reformatted by black * Commit changes to other files after re-running black * Fixes unused import that caused lint checks to fail * Moves tests/ directory to app root for data_pipeline * Adds new methods to ExtractTransformLoad base class: - __init__() Initializes class attributes - _get_census_fips_codes() Loads a dataframe with the fips codes for census block group and tract - validate_init() Checks that the class was initialized correctly - validate_output() Checks that the output was loaded correctly * Adds test for ExtractTransformLoad.__init__() and base.py * Fixes failing flake8 test * Changes geo_col to geoid_col and changes is_dataset to is_census in yaml * Adds test for validate_output() * Adds remaining tests * Removes is_dataset from init method * Makes CENSUS_CSV a class attribute instead of a class global: This ensures that CENSUS_CSV is only set when the ETL class is for a non-census dataset and removes the need to overwrite the value in mock_etl fixture * Re-formats files with black and fixes broken tox tests
126 lines
3.6 KiB
TOML
126 lines
3.6 KiB
TOML
[tool.poetry]
|
|
authors = ["Your Name <you@example.com>"]
|
|
description = "ETL and Generation of Justice 40 Score"
|
|
name = "data-pipeline"
|
|
version = "0.1.0"
|
|
|
|
[tool.poetry.dependencies]
|
|
CensusData = "^1.13"
|
|
click = "^8.0.1"
|
|
dynaconf = "^3.1.4"
|
|
geopandas = "^0.9.0"
|
|
ipython = "^7.24.1"
|
|
jupyter = "^1.0.0"
|
|
jupyter-contrib-nbextensions = "^0.5.1"
|
|
matplotlib = "^3.4.2"
|
|
numpy = "^1.21.0"
|
|
pandas = "^1.2.5"
|
|
python = "^3.7.1"
|
|
pypandoc = "^1.6.3"
|
|
requests = "^2.25.1"
|
|
tqdm = "4.62.0"
|
|
types-requests = "^2.25.0"
|
|
us = "^2.0.2"
|
|
xlsxwriter = "^2.0.0"
|
|
|
|
[tool.poetry.dev-dependencies]
|
|
black = {version = "^21.6b0", allow-prereleases = true}
|
|
flake8 = "^3.9.2"
|
|
liccheck = "^0.6.2"
|
|
mypy = "^0.910"
|
|
openpyxl = "^3.0.7"
|
|
pylint = "^2.9.6"
|
|
pytest = "^6.2.4"
|
|
safety = "^1.10.3"
|
|
tox = "^3.24.0"
|
|
pytest-mock = "^3.6.1"
|
|
|
|
[build-system]
|
|
build-backend = "poetry.core.masonry.api"
|
|
requires = ["poetry-core>=1.0.0"]
|
|
|
|
[tool.pylint]
|
|
|
|
[tool.pylint."MESSAGE CONTROL"]
|
|
disable = [
|
|
"C0114", # Disables module docstrings
|
|
"R0201", # Disables method could have been a function
|
|
"R0903", # Disables too few public methods
|
|
"C0103", # Disables name case styling
|
|
"W0511", # Disables FIXME warning
|
|
"W1203", # Disables f-string interpolation for logging warning # Errors temporarily ignored for further discussion
|
|
"W0107", # Disables unnecessary pass
|
|
"W0221", # Disables arguments differ
|
|
"R0902", # Disables too many instance attributes
|
|
"R0914", # Disables too many local variables
|
|
"W0621", # Disables redefined outer name
|
|
"C0302", # Disables too many lines in module
|
|
"R1732", # Disables consider using "with"
|
|
"R1720", # Disables unnecessary "else" after "raise"
|
|
"C0206", # Disables consider iteratig with ".items()"
|
|
"C0200", # Disables consider using "enumerate" instead of "range" + "len"
|
|
"W0612", # Disables unused variable
|
|
"W0613", # Disables unused argument
|
|
"C0116", # Disables missing function or method docstring
|
|
"C0115", # Disables missing class docstring
|
|
"R0915", # Disables too many statements (score generation transform)
|
|
"W0231", # Disables super init not called
|
|
]
|
|
|
|
[tool.pylint.FORMAT]
|
|
max-line-length = 150
|
|
|
|
[tool.pylint.typecheck]
|
|
generated-members = "pandas.*" # fixes E1101 for ETL.df
|
|
|
|
[tool.pylint.SIMILARITIES]
|
|
# Configures how pylint detects repetitive code
|
|
ignore-comments = "yes"
|
|
ignore-docstrings = "yes"
|
|
ignore-imports = "yes"
|
|
min-similarity-lines = 4
|
|
|
|
[tool.black]
|
|
line-length = 80
|
|
|
|
[tool.liccheck]
|
|
# Authorized and unauthorized licenses in LOWER CASE
|
|
authorized_licenses = [
|
|
"bsd",
|
|
"new bsd",
|
|
"bsd license",
|
|
"bsd 3-clause",
|
|
"new bsd license",
|
|
"simplified bsd",
|
|
"apache",
|
|
"apache 2.0",
|
|
"apache license 2.0",
|
|
"apache software license",
|
|
"apache software",
|
|
"gnu lgpl",
|
|
"gnu lesser general public license v2 (lgplv2)",
|
|
"gnu general public license v2 (gplv2)",
|
|
"gnu library or lesser general public license (lgpl)",
|
|
"lgpl with exceptions or zpl",
|
|
"isc license",
|
|
"isc license (iscl)",
|
|
"mit",
|
|
"mit license",
|
|
"mozilla public license 2.0 (mpl 2.0)",
|
|
"public domain",
|
|
"python software foundation license",
|
|
"python software foundation",
|
|
"zpl 2.1",
|
|
"gpl v3",
|
|
"historical permission notice and disclaimer (hpnd)",
|
|
]
|
|
|
|
[tool.poetry.scripts]
|
|
cleanup_census = 'data_pipeline.application:census_cleanup'
|
|
cleanup_data = 'data_pipeline.application:data_cleanup'
|
|
download_census = 'data_pipeline.application:census_data_download'
|
|
etl = 'data_pipeline.application:etl_run'
|
|
generate_tiles = 'data_pipeline.application:generate_map_tiles'
|
|
score = 'data_pipeline.application:score_run'
|
|
score_geo = 'data_pipeline.application:geo_score'
|
|
etl_and_score = 'data_pipeline.application:score_full_run'
|