mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-08-11 11:34:18 -07:00
Modularization + Poetry + Docker (#213)
* reorg
* added configuration management; initial click cmds
* reset dirs completed
* major modularization effort
* prepping mbtiles
* first round of PR review updates
* round 2 of feedback review
* checkpoint
* habemus dockerfile 🎉
* updated dock-er-compose with long running container
* census generation works
* logging working
* updated README
* updated README
* last small update to README
* added instructions for log visualization
* census etl update for reusable fips module
* ejscreem etl updated
* further modularization
* score modularization
* tmp cleanup
This commit is contained in:
parent
6f4087d247
commit
67c73dde2a
29 changed files with 2383 additions and 433 deletions
60
score/application.py
Normal file
60
score/application.py
Normal file
|
@ -0,0 +1,60 @@
|
|||
from config import settings
|
||||
import click
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
from etl.sources.census.etl_utils import reset_data_directories as census_reset
|
||||
from utils import remove_files_from_dir, remove_all_from_dir, get_module_logger
|
||||
from etl.sources.census.etl import download_census_csvs
|
||||
|
||||
|
||||
settings.APP_ROOT = Path.cwd()
|
||||
logger = get_module_logger(__name__)
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
pass
|
||||
|
||||
|
||||
@cli.command(
|
||||
help="Clean up all data folders",
|
||||
)
|
||||
def data_cleanup():
|
||||
|
||||
data_path = settings.APP_ROOT / "data"
|
||||
|
||||
# census directories
|
||||
logger.info(f"Initializing all census data")
|
||||
census_reset(data_path)
|
||||
|
||||
# dataset directory
|
||||
logger.info(f"Initializing all dataset directoriees")
|
||||
remove_all_from_dir(data_path / "dataset")
|
||||
|
||||
# score directory
|
||||
logger.info(f"Initializing all score data")
|
||||
remove_files_from_dir(data_path / "score" / "csv", ".csv")
|
||||
remove_files_from_dir(data_path / "score" / "geojson", ".json")
|
||||
|
||||
# cleanup tmp dir
|
||||
logger.info(f"Initializing all temp directoriees")
|
||||
remove_all_from_dir(data_path / "tmp")
|
||||
|
||||
logger.info("Cleaned up all data files")
|
||||
|
||||
|
||||
@cli.command(
|
||||
help="Census data download",
|
||||
)
|
||||
def census_data_download():
|
||||
logger.info("Downloading census data")
|
||||
data_path = settings.APP_ROOT / "data"
|
||||
download_census_csvs(data_path)
|
||||
|
||||
logger.info("Completed downloading census data")
|
||||
exit()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
Loading…
Add table
Add a link
Reference in a new issue