mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-02-23 18:14:19 -08:00
5504528fdf
* Adds flake8, pylint, liccheck, flake8 to dependencies for data-pipeline * Sets up and runs black autoformatting * Adds flake8 to tox linting * Fixes flake8 error F541 f string missing placeholders * Fixes flake8 E501 line too long * Fixes flake8 F401 imported but not used * Adds pylint to tox and disables the following pylint errors: - C0114: module docstrings - R0201: method could have been a function - R0903: too few public methods - C0103: name case styling - W0511: fix me - W1203: f-string interpolation in logging * Adds utils.py to tox.ini linting, runs black on utils.py * Fixes import related pylint errors: C0411 and C0412 * Fixes or ignores remaining pylint errors (for discussion later) * Adds safety and liccheck to tox.ini
100 lines
2.1 KiB
Python
100 lines
2.1 KiB
Python
import click
|
|
|
|
from config import settings
|
|
from utils import (
|
|
get_module_logger,
|
|
data_folder_cleanup,
|
|
score_folder_cleanup,
|
|
temp_folder_cleanup,
|
|
)
|
|
from etl.sources.census.etl import download_census_csvs
|
|
from etl.sources.census.etl_utils import reset_data_directories as census_reset
|
|
from etl.runner import etl_runner, score_generate, score_geo
|
|
|
|
logger = get_module_logger(__name__)
|
|
|
|
|
|
@click.group()
|
|
def cli():
|
|
"""Defines a click group for the commands below"""
|
|
|
|
pass
|
|
|
|
|
|
@cli.command(
|
|
help="Clean up all census data folders",
|
|
)
|
|
def census_cleanup():
|
|
"""CLI command to clean up the census data folder"""
|
|
|
|
data_path = settings.APP_ROOT / "data"
|
|
|
|
# census directories
|
|
logger.info("Initializing all census data")
|
|
census_reset(data_path)
|
|
|
|
logger.info("Cleaned up all census data files")
|
|
|
|
|
|
@cli.command(
|
|
help="Clean up all data folders",
|
|
)
|
|
def data_cleanup():
|
|
"""CLI command to clean up the all the data folders"""
|
|
|
|
data_folder_cleanup()
|
|
score_folder_cleanup()
|
|
temp_folder_cleanup()
|
|
|
|
logger.info("Cleaned up all data folders")
|
|
|
|
|
|
@cli.command(
|
|
help="Census data download",
|
|
)
|
|
def census_data_download():
|
|
"""CLI command to download all census shape files from the Census FTP and extract the geojson
|
|
to generate national and by state Census Block Group CSVs"""
|
|
|
|
logger.info("Downloading census data")
|
|
data_path = settings.APP_ROOT / "data"
|
|
download_census_csvs(data_path)
|
|
|
|
logger.info("Completed downloading census data")
|
|
|
|
|
|
@cli.command(
|
|
help="Run all ETL processes or a specific one",
|
|
)
|
|
@click.option("-d", "--dataset", required=False, type=str)
|
|
def etl_run(dataset: str):
|
|
"""Run a specific or all ETL processes
|
|
|
|
Args:
|
|
dataset (str): Name of the ETL module to be run (optional)
|
|
|
|
Returns:
|
|
None
|
|
"""
|
|
|
|
etl_runner(dataset)
|
|
|
|
|
|
@cli.command(
|
|
help="Generate Score",
|
|
)
|
|
def score_run():
|
|
"""CLI command to generate the score"""
|
|
score_generate()
|
|
|
|
|
|
@cli.command(
|
|
help="Generate Geojson files with scores baked in",
|
|
)
|
|
def geo_score():
|
|
"""CLI command to generate the score"""
|
|
score_geo()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cli()
|