publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-07-27 15:51:16 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add pre-cleaning to a couple of zip files during geo-score (#2151)

Browse source 
Two zip files were not getting cleared prior to running geo score. This was resulting in them growing between runs until the application ground to a halt. This fix clears those two zip files before geo score and before the full run.
This commit is contained in:
Travis Newby 2023-02-06 12:43:12 -06:00 • committed by GitHub
commit 9ba4e790a7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 21 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

18
data/data-pipeline/data_pipeline/utils.py
View file

@ -17,6 +17,7 @@ from data_pipeline.config import settings
from data_pipeline.content.schemas.download_schemas import CodebookConfig
from data_pipeline.content.schemas.download_schemas import CSVConfig
from data_pipeline.content.schemas.download_schemas import ExcelConfig
from data_pipeline.etl.score.constants import SCORE_VERSIONING_SHAPEFILE_CODEBOOK_FILE_PATH
from marshmallow import ValidationError
from marshmallow_dataclass import class_schema
@ -218,9 +219,26 @@ def score_folder_cleanup() -> None:
remove_all_from_dir(data_path / "score" / "csv")
remove_all_from_dir(data_path / "score" / "geojson")
remove_all_from_dir(data_path / "score" / "tiles")
remove_all_from_dir(data_path / "score" / "shapefile")
downloadable_cleanup()
def geo_score_folder_cleanup() -> None:
"""Removes the necessary files to run geo-score. This works out to be
zip files, since if we don't remove them python's zip utils continuously
add to them instead of overwriting the contents."""
data_path = settings.APP_ROOT / "data"
logger.info("Removing zip files")
remove_files_from_dir(data_path / "score" / "shapefile", ".zip")
shapefile_and_codebook_zipped = SCORE_VERSIONING_SHAPEFILE_CODEBOOK_FILE_PATH
if os.path.isfile(shapefile_and_codebook_zipped):
os.remove(shapefile_and_codebook_zipped)
def downloadable_cleanup() -> None:
"""Remove all files from downloadable directory in the local data/score path"""