mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-02-23 10:04:18 -08:00
67c73dde2a
* reorg
* added configuration management; initial click cmds
* reset dirs completed
* major modularization effort
* prepping mbtiles
* first round of PR review updates
* round 2 of feedback review
* checkpoint
* habemus dockerfile 🎉
* updated dock-er-compose with long running container
* census generation works
* logging working
* updated README
* updated README
* last small update to README
* added instructions for log visualization
* census etl update for reusable fips module
* ejscreem etl updated
* further modularization
* score modularization
* tmp cleanup
60 lines
1.4 KiB
Python
60 lines
1.4 KiB
Python
from config import settings
|
|
import click
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
from etl.sources.census.etl_utils import reset_data_directories as census_reset
|
|
from utils import remove_files_from_dir, remove_all_from_dir, get_module_logger
|
|
from etl.sources.census.etl import download_census_csvs
|
|
|
|
|
|
settings.APP_ROOT = Path.cwd()
|
|
logger = get_module_logger(__name__)
|
|
|
|
|
|
@click.group()
|
|
def cli():
|
|
pass
|
|
|
|
|
|
@cli.command(
|
|
help="Clean up all data folders",
|
|
)
|
|
def data_cleanup():
|
|
|
|
data_path = settings.APP_ROOT / "data"
|
|
|
|
# census directories
|
|
logger.info(f"Initializing all census data")
|
|
census_reset(data_path)
|
|
|
|
# dataset directory
|
|
logger.info(f"Initializing all dataset directoriees")
|
|
remove_all_from_dir(data_path / "dataset")
|
|
|
|
# score directory
|
|
logger.info(f"Initializing all score data")
|
|
remove_files_from_dir(data_path / "score" / "csv", ".csv")
|
|
remove_files_from_dir(data_path / "score" / "geojson", ".json")
|
|
|
|
# cleanup tmp dir
|
|
logger.info(f"Initializing all temp directoriees")
|
|
remove_all_from_dir(data_path / "tmp")
|
|
|
|
logger.info("Cleaned up all data files")
|
|
|
|
|
|
@cli.command(
|
|
help="Census data download",
|
|
)
|
|
def census_data_download():
|
|
logger.info("Downloading census data")
|
|
data_path = settings.APP_ROOT / "data"
|
|
download_census_csvs(data_path)
|
|
|
|
logger.info("Completed downloading census data")
|
|
exit()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cli()
|