mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-08-11 00:34:18 -07:00
* added tribalId for Supplemental dataset (#1804) * Setting zoom levels for tribal map (#1810) * NRI dataset and initial score YAML configuration (#1534) * update be staging gha * NRI dataset and initial score YAML configuration * checkpoint * adding data checks for release branch * passing tests * adding INPUT_EXTRACTED_FILE_NAME to base class * lint * columns to keep and tests * update be staging gha * checkpoint * update be staging gha * NRI dataset and initial score YAML configuration * checkpoint * adding data checks for release branch * passing tests * adding INPUT_EXTRACTED_FILE_NAME to base class * lint * columns to keep and tests * checkpoint * PR Review * renoving source url * tests * stop execution of ETL if there's a YAML schema issue * update be staging gha * adding source url as class var again * clean up * force cache bust * gha cache bust * dynamically set score vars from YAML * docsctrings * removing last updated year - optional reverse percentile * passing tests * sort order * column ordening * PR review * class level vars * Updating DatasetsConfig * fix pylint errors * moving metadata hint back to code Co-authored-by: lucasmbrown-usds <lucas.m.brown@omb.eop.gov> * Correct copy typo (#1809) * Add basic test suite for COI (#1518) * Update COI to use new yaml (#1518) * Add tests for DOE energy budren (1518 * Add dataset config for energy budren (1518) * Refactor ETL to use datasets.yml (#1518) * Add fake GEOIDs to COI tests (#1518) * Refactor _setup_etl_instance_and_run_extract to base (#1518) For the three classes we've done so far, a generic _setup_etl_instance_and_run_extract will work fine, for the moment we can reuse the same setup method until we decide future classes need more flexibility --- but they can also always subclass so... * Add output-path tests (#1518) * Update YAML to match constant (#1518) * Don't blindly set float format (#1518) * Add defaults for extract (#1518) * Run YAML load on all subclasses (#1518) * Update description fields (#1518) * Update YAML per final format (#1518) * Update fixture tract IDs (#1518) * Update base class refactor (#1518) Now that NRI is final I needed to make a small number of updates to my refactored code. * Remove old comment (#1518) * Fix type signature and return (#1518) * Update per code review (#1518) Co-authored-by: Jorge Escobar <83969469+esfoobar-usds@users.noreply.github.com> Co-authored-by: lucasmbrown-usds <lucas.m.brown@omb.eop.gov> Co-authored-by: Vim <86254807+vim-usds@users.noreply.github.com>
108 lines
3.9 KiB
Python
108 lines
3.9 KiB
Python
import os
|
|
from pathlib import Path
|
|
from subprocess import call
|
|
|
|
from data_pipeline.utils import get_module_logger, remove_all_from_dir
|
|
|
|
logger = get_module_logger(__name__)
|
|
|
|
|
|
def generate_tiles(data_path: Path, generate_tribal_layer: bool) -> None:
|
|
"""Generates map tiles from geojson files
|
|
|
|
Args:
|
|
data_path (Path): Path to data folder
|
|
generate_tribal_layer (bool): If true, generate the tribal layer of the map
|
|
|
|
Returns:
|
|
None
|
|
"""
|
|
|
|
def _generate_score_tiles() -> None:
|
|
"""Generates score map tiles"""
|
|
score_tiles_path = data_path / "score" / "tiles"
|
|
high_tile_path = score_tiles_path / "high"
|
|
low_tile_path = score_tiles_path / "low"
|
|
score_geojson_dir = data_path / "score" / "geojson"
|
|
|
|
USA_HIGH_MIN_ZOOM = 5
|
|
USA_HIGH_MAX_ZOOM = 11
|
|
USA_LOW_MIN_ZOOM = 0
|
|
USA_LOW_MAX_ZOOM = 7
|
|
|
|
# remove existing mbtiles file
|
|
remove_all_from_dir(score_tiles_path)
|
|
|
|
# create dirs
|
|
os.mkdir(high_tile_path)
|
|
os.mkdir(low_tile_path)
|
|
|
|
# generate high mbtiles file
|
|
logger.info("Generating USA High mbtiles file")
|
|
cmd = "tippecanoe "
|
|
cmd += f"--minimum-zoom={USA_HIGH_MIN_ZOOM} --maximum-zoom={USA_HIGH_MAX_ZOOM} --layer=blocks "
|
|
cmd += f"--output={high_tile_path}/usa_high.mbtiles "
|
|
cmd += str(score_geojson_dir / "usa-high.json")
|
|
call(cmd, shell=True)
|
|
|
|
# generate high mvts
|
|
logger.info("Generating USA High mvt folders and files")
|
|
cmd = "tippecanoe "
|
|
cmd += f"--minimum-zoom={USA_HIGH_MIN_ZOOM} --maximum-zoom={USA_HIGH_MAX_ZOOM} --no-tile-compression "
|
|
cmd += "--drop-densest-as-needed "
|
|
cmd += f"--output-to-directory={high_tile_path} --layer=blocks "
|
|
cmd += str(score_geojson_dir / "usa-high.json")
|
|
call(cmd, shell=True)
|
|
|
|
# generate low mbtiles file
|
|
logger.info("Generating USA Low mbtiles file")
|
|
cmd = "tippecanoe "
|
|
cmd += f"--minimum-zoom={USA_LOW_MIN_ZOOM} --maximum-zoom={USA_LOW_MAX_ZOOM} --layer=blocks "
|
|
cmd += f"--output={low_tile_path}/usa_low.mbtiles "
|
|
cmd += str(score_geojson_dir / "usa-low.json")
|
|
call(cmd, shell=True)
|
|
|
|
# generate low mvts
|
|
logger.info("Generating USA Low mvt folders and files")
|
|
cmd = "tippecanoe "
|
|
cmd += f"--minimum-zoom={USA_LOW_MIN_ZOOM} --maximum-zoom={USA_LOW_MAX_ZOOM} --no-tile-compression "
|
|
cmd += "--drop-densest-as-needed "
|
|
cmd += f"--output-to-directory={low_tile_path} --layer=blocks "
|
|
cmd += str(score_geojson_dir / "usa-low.json")
|
|
call(cmd, shell=True)
|
|
|
|
def _generate_tribal_tiles() -> None:
|
|
"""Generates tribal layer tiles"""
|
|
|
|
USA_TRIBAL_MIN_ZOOM = 0
|
|
USA_TRIBAL_MAX_ZOOM = 11
|
|
|
|
tribal_tiles_path = data_path / "tribal" / "tiles"
|
|
tribal_geojson_dir = data_path / "tribal" / "geojson"
|
|
|
|
# remove existing mbtiles file
|
|
remove_all_from_dir(tribal_tiles_path)
|
|
|
|
# generate mbtiles file
|
|
logger.info("Generating Tribal mbtiles file")
|
|
cmd = "tippecanoe "
|
|
cmd += "--layer=blocks "
|
|
cmd += f"--minimum-zoom={USA_TRIBAL_MIN_ZOOM} --maximum-zoom={USA_TRIBAL_MAX_ZOOM} "
|
|
cmd += f"--output={tribal_tiles_path}/usa.mbtiles "
|
|
cmd += str(tribal_geojson_dir / "usa.json")
|
|
call(cmd, shell=True)
|
|
|
|
# generate mvts
|
|
logger.info("Generating Tribal mvt folders and files")
|
|
cmd = "tippecanoe "
|
|
cmd += "--no-tile-compression "
|
|
cmd += "--drop-densest-as-needed "
|
|
cmd += f"--minimum-zoom={USA_TRIBAL_MIN_ZOOM} --maximum-zoom={USA_TRIBAL_MAX_ZOOM} "
|
|
cmd += f"--output-to-directory={tribal_tiles_path} --layer=blocks "
|
|
cmd += str(tribal_geojson_dir / "usa.json")
|
|
call(cmd, shell=True)
|
|
|
|
if generate_tribal_layer:
|
|
_generate_tribal_tiles()
|
|
else:
|
|
_generate_score_tiles()
|