Starting Tribal Boundaries Work (#1736)

* starting tribal pr

* further pipeline work

* bia merge working

* alaska villages and tribal geo generate

* tribal folders

* adding data full run

* tile generation

* tribal tile deploy

data/data-pipeline/data_pipeline/application.py

@@ -13,6 +13,9 @@ from data_pipeline.etl.sources.census.etl_utils import (
     reset_data_directories as census_reset,
     zip_census_data,
 )
+from data_pipeline.etl.sources.tribal.etl_utils import (
+    reset_data_directories as tribal_reset,
+)
 from data_pipeline.tile.generate import generate_tiles
 from data_pipeline.utils import (
     data_folder_cleanup,
@@ -57,6 +60,7 @@ def data_cleanup():
 
     census_reset(data_path)
     data_folder_cleanup()
+    tribal_reset(data_path)
     score_folder_cleanup()
     temp_folder_cleanup()
 
@@ -168,13 +172,21 @@ def geo_score(data_source: str):
 
 
 @cli.command(
-    help="Generate map tiles",
+    help="Generate map tiles. Pass -t to generate tribal layer as well.",
 )
-def generate_map_tiles():
+@click.option(
+    "-t",
+    "--generate-tribal-layer",
+    default=False,
+    required=False,
+    is_flag=True,
+    type=bool,
+)
+def generate_map_tiles(generate_tribal_layer):
     """CLI command to generate the map tiles"""
 
     data_path = settings.APP_ROOT / "data"
-    generate_tiles(data_path)
+    generate_tiles(data_path, generate_tribal_layer)
     sys.exit()
 
 
@@ -271,7 +283,7 @@ def data_full_run(check: bool, data_source: str):
     score_geo(data_source)
 
     logger.info("*** Generating Map Tiles")
-    generate_tiles(data_path)
+    generate_tiles(data_path, True)
 
     file = "first_run.txt"
     cmd = f"touch {data_path}/{file}"

data/data-pipeline/data_pipeline/etl/constants.py

@@ -125,8 +125,15 @@ DATASET_LIST = [
         "class_name": "MarylandEJScreenETL",
     },
 ]
+
 CENSUS_INFO = {
     "name": "census",
     "module_dir": "census",
     "class_name": "CensusETL",
 }
+
+TRIBAL_INFO = {
+    "name": "tribal",
+    "module_dir": "tribal",
+    "class_name": "TribalETL",
+}

data/data-pipeline/data_pipeline/etl/runner.py

@@ -22,7 +22,9 @@ def _get_datasets_to_run(dataset_to_run: str) -> typing.List[dict]:
         None
     """
     dataset_list = constants.DATASET_LIST
-    etls_to_search = dataset_list + [constants.CENSUS_INFO]
+    etls_to_search = (
+        dataset_list + [constants.CENSUS_INFO] + [constants.TRIBAL_INFO]
+    )
 
     if dataset_to_run:
         dataset_element = next(

data/data-pipeline/data_pipeline/etl/sources/census/etl.py

@@ -3,8 +3,8 @@ import json
 import subprocess
 from enum import Enum
 from pathlib import Path
-
 import geopandas as gpd
+
 from data_pipeline.etl.base import ExtractTransformLoad
 from data_pipeline.utils import get_module_logger, unzip_file_from_url
 

data/data-pipeline/data_pipeline/etl/sources/tribal/etl.py

@@ -0,0 +1,195 @@
+from pathlib import Path
+import geopandas as gpd
+import pandas as pd
+
+from data_pipeline.etl.base import ExtractTransformLoad
+from data_pipeline.utils import get_module_logger, unzip_file_from_url
+
+logger = get_module_logger(__name__)
+
+
+class TribalETL(ExtractTransformLoad):
+    def __init__(self):
+        self.GEOJSON_BASE_PATH = self.DATA_PATH / "tribal" / "geojson"
+        self.CSV_BASE_PATH = self.DATA_PATH / "tribal" / "csv"
+        self.NATIONAL_TRIBAL_GEOJSON_PATH = self.GEOJSON_BASE_PATH / "usa.json"
+        self.USA_TRIBAL_DF_LIST = []
+
+    def extract(self) -> None:
+        """Extract the tribal geojson zip files from Justice40 S3 data folder
+
+        Returns:
+            None
+        """
+        logger.info("Downloading Tribal Data")
+
+        bia_geojson_url = "https://justice40-data.s3.amazonaws.com/data-sources/BIA_National_LAR_json.zip"
+        alaska_geojson_url = "https://justice40-data.s3.amazonaws.com/data-sources/Alaska_Native_Villages_json.zip"
+
+        unzip_file_from_url(
+            bia_geojson_url,
+            self.TMP_PATH,
+            self.DATA_PATH / "tribal" / "geojson" / "bia_national_lar",
+        )
+
+        unzip_file_from_url(
+            alaska_geojson_url,
+            self.TMP_PATH,
+            self.DATA_PATH / "tribal" / "geojson" / "alaska_native_villages",
+        )
+        pass
+
+    def _transform_bia_national_lar(self, tribal_geojson_path: Path) -> None:
+        """Transform the Tribal BIA National Lar Geodataframe and appends it to the
+        national Tribal Dataframe List
+
+        Args:
+            tribal_geojson_path (Path): the Path to the Tribal Geojson
+
+        Returns:
+            None
+        """
+
+        bia_national_lar_df = gpd.read_file(tribal_geojson_path)
+
+        bia_national_lar_df.drop(
+            ["OBJECTID", "GISAcres", "Shape_Length", "Shape_Area"],
+            axis=1,
+            inplace=True,
+        )
+
+        bia_national_lar_df.rename(
+            columns={"TSAID": "tribalId", "LARName": "landAreaName"},
+            inplace=True,
+        )
+
+        self.USA_TRIBAL_DF_LIST.append(bia_national_lar_df)
+
+    def _transform_bia_aian_supplemental(
+        self, tribal_geojson_path: Path
+    ) -> None:
+        """Transform the Tribal BIA Supplemental Geodataframe and appends it to the
+        national Tribal Dataframe List
+
+        Args:
+            tribal_geojson_path (Path): the Path to the Tribal Geojson
+
+        Returns:
+            None
+        """
+
+        bia_aian_supplemental_df = gpd.read_file(tribal_geojson_path)
+
+        bia_aian_supplemental_df.drop(
+            ["OBJECTID", "GISAcres", "Source", "Shape_Length", "Shape_Area"],
+            axis=1,
+            inplace=True,
+        )
+
+        bia_aian_supplemental_df.rename(
+            columns={"Land_Area_": "landAreaName"},
+            inplace=True,
+        )
+
+        self.USA_TRIBAL_DF_LIST.append(bia_aian_supplemental_df)
+
+    def _transform_bia_tsa(self, tribal_geojson_path: Path) -> None:
+        """Transform the Tribal BIA TSA Geodataframe and appends it to the
+        national Tribal Dataframe List
+
+        Args:
+            tribal_geojson_path (Path): the Path to the Tribal Geojson
+
+        Returns:
+            None
+        """
+
+        bia_tsa_df = gpd.read_file(tribal_geojson_path)
+
+        bia_tsa_df.drop(
+            ["OBJECTID", "GISAcres", "Shape_Length", "Shape_Area"],
+            axis=1,
+            inplace=True,
+        )
+
+        bia_tsa_df.rename(
+            columns={"TSAID": "tribalId", "LARName": "landAreaName"},
+            inplace=True,
+        )
+
+        self.USA_TRIBAL_DF_LIST.append(bia_tsa_df)
+
+    def _transform_alaska_native_villages(
+        self, tribal_geojson_path: Path
+    ) -> None:
+        """Transform the Alaska Native Villages Geodataframe and appends it to the
+        national Tribal Dataframe List
+
+        Args:
+            tribal_geojson_path (Path): the Path to the Tribal Geojson
+
+        Returns:
+            None
+        """
+
+        alaska_native_villages_df = gpd.read_file(tribal_geojson_path)
+
+        alaska_native_villages_df.rename(
+            columns={
+                "GlobalID": "tribalId",
+                "TRIBALOFFICENAME": "landAreaName",
+            },
+            inplace=True,
+        )
+
+        self.USA_TRIBAL_DF_LIST.append(alaska_native_villages_df)
+
+    def transform(self) -> None:
+        """Transform the tribal geojson files to generate national CSVs and GeoJSONs
+
+        Returns:
+            None
+        """
+        logger.info("Transforming Tribal Data")
+
+        # load the geojsons
+        bia_national_lar_geojson = (
+            self.GEOJSON_BASE_PATH / "bia_national_lar" / "BIA_TSA.json"
+        )
+        bia_aian_supplemental_geojson = (
+            self.GEOJSON_BASE_PATH
+            / "bia_national_lar"
+            / "BIA_AIAN_Supplemental.json"
+        )
+        bia_tsa_geojson_geojson = (
+            self.GEOJSON_BASE_PATH / "bia_national_lar" / "BIA_TSA.json"
+        )
+        alaska_native_villages_geojson = (
+            self.GEOJSON_BASE_PATH
+            / "alaska_native_villages"
+            / "AlaskaNativeVillages.gdb.geojson"
+        )
+
+        self._transform_bia_national_lar(bia_national_lar_geojson)
+        self._transform_bia_aian_supplemental(bia_aian_supplemental_geojson)
+        self._transform_bia_tsa(bia_tsa_geojson_geojson)
+        self._transform_alaska_native_villages(alaska_native_villages_geojson)
+
+    def load(self) -> None:
+        """Create tribal national CSV and GeoJSON
+
+        Returns:
+            None
+        """
+        logger.info("Saving Tribal GeoJson and CSV")
+
+        usa_tribal_df = gpd.GeoDataFrame(
+            pd.concat(self.USA_TRIBAL_DF_LIST, ignore_index=True)
+        )
+        usa_tribal_df = usa_tribal_df.to_crs(
+            "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
+        )
+        logger.info("Writing national geojson file")
+        usa_tribal_df.to_file(
+            self.NATIONAL_TRIBAL_GEOJSON_PATH, driver="GeoJSON"
+        )

data/data-pipeline/data_pipeline/etl/sources/tribal/etl_utils.py

@@ -0,0 +1,28 @@
+from pathlib import Path
+
+from data_pipeline.utils import (
+    get_module_logger,
+    remove_all_from_dir,
+    remove_files_from_dir,
+)
+
+
+logger = get_module_logger(__name__)
+
+
+def reset_data_directories(
+    data_path: Path,
+) -> None:
+    """Empties all tribal files"""
+    tribal_data_path = data_path / "tribal"
+
+    # csv
+    csv_path = tribal_data_path / "csv"
+    remove_files_from_dir(
+        csv_path,
+        ".csv",
+    )
+
+    # geojson
+    geojson_path = tribal_data_path / "geojson"
+    remove_all_from_dir(geojson_path)

data/data-pipeline/data_pipeline/tile/generate.py

@@ -7,55 +7,96 @@ from data_pipeline.utils import get_module_logger, remove_all_from_dir
 logger = get_module_logger(__name__)
 
 
-def generate_tiles(data_path: Path) -> None:
+def generate_tiles(data_path: Path, generate_tribal_layer: bool) -> None:
+    """Generates map tiles from geojson files
 
-    score_tiles_path = data_path / "score" / "tiles"
-    high_tile_path = score_tiles_path / "high"
-    low_tile_path = score_tiles_path / "low"
-    score_geojson_dir = data_path / "score" / "geojson"
+    Args:
+        data_path (Path):  Path to data folder
+        generate_tribal_layer (bool): If true, generate the tribal layer of the map
 
-    USA_HIGH_MIN_ZOOM = 5
-    USA_HIGH_MAX_ZOOM = 11
-    USA_LOW_MIN_ZOOM = 0
-    USA_LOW_MAX_ZOOM = 7
+    Returns:
+        None
+    """
 
-    # remove existing mbtiles file
-    remove_all_from_dir(score_tiles_path)
+    def _generate_score_tiles() -> None:
+        """Generates score map tiles"""
+        score_tiles_path = data_path / "score" / "tiles"
+        high_tile_path = score_tiles_path / "high"
+        low_tile_path = score_tiles_path / "low"
+        score_geojson_dir = data_path / "score" / "geojson"
 
-    # create dirs
-    os.mkdir(high_tile_path)
-    os.mkdir(low_tile_path)
+        USA_HIGH_MIN_ZOOM = 5
+        USA_HIGH_MAX_ZOOM = 11
+        USA_LOW_MIN_ZOOM = 0
+        USA_LOW_MAX_ZOOM = 7
 
-    # generate high mbtiles file
-    logger.info("Generating USA High mbtiles file")
-    cmd = "tippecanoe "
-    cmd += f"--minimum-zoom={USA_HIGH_MIN_ZOOM} --maximum-zoom={USA_HIGH_MAX_ZOOM} --layer=blocks "
-    cmd += f"--output={high_tile_path}/usa_high.mbtiles "
-    cmd += str(score_geojson_dir / "usa-high.json")
-    call(cmd, shell=True)
+        # remove existing mbtiles file
+        remove_all_from_dir(score_tiles_path)
 
-    # generate high mvts
-    logger.info("Generating USA High mvt folders and files")
-    cmd = "tippecanoe "
-    cmd += f"--minimum-zoom={USA_HIGH_MIN_ZOOM} --maximum-zoom={USA_HIGH_MAX_ZOOM} --no-tile-compression "
-    cmd += "--drop-densest-as-needed "
-    cmd += f"--output-to-directory={high_tile_path} --layer=blocks "
-    cmd += str(score_geojson_dir / "usa-high.json")
-    call(cmd, shell=True)
+        # create dirs
+        os.mkdir(high_tile_path)
+        os.mkdir(low_tile_path)
 
-    # generate low mbtiles file
-    logger.info("Generating USA Low mbtiles file")
-    cmd = "tippecanoe "
-    cmd += f"--minimum-zoom={USA_LOW_MIN_ZOOM} --maximum-zoom={USA_LOW_MAX_ZOOM} --layer=blocks "
-    cmd += f"--output={low_tile_path}/usa_low.mbtiles "
-    cmd += str(score_geojson_dir / "usa-low.json")
-    call(cmd, shell=True)
+        # generate high mbtiles file
+        logger.info("Generating USA High mbtiles file")
+        cmd = "tippecanoe "
+        cmd += f"--minimum-zoom={USA_HIGH_MIN_ZOOM} --maximum-zoom={USA_HIGH_MAX_ZOOM} --layer=blocks "
+        cmd += f"--output={high_tile_path}/usa_high.mbtiles "
+        cmd += str(score_geojson_dir / "usa-high.json")
+        call(cmd, shell=True)
 
-    # generate low mvts
-    logger.info("Generating USA Low mvt folders and files")
-    cmd = "tippecanoe "
-    cmd += f"--minimum-zoom={USA_LOW_MIN_ZOOM} --maximum-zoom={USA_LOW_MAX_ZOOM} --no-tile-compression "
-    cmd += "--drop-densest-as-needed "
-    cmd += f"--output-to-directory={low_tile_path} --layer=blocks "
-    cmd += str(score_geojson_dir / "usa-low.json")
-    call(cmd, shell=True)
+        # generate high mvts
+        logger.info("Generating USA High mvt folders and files")
+        cmd = "tippecanoe "
+        cmd += f"--minimum-zoom={USA_HIGH_MIN_ZOOM} --maximum-zoom={USA_HIGH_MAX_ZOOM} --no-tile-compression "
+        cmd += "--drop-densest-as-needed "
+        cmd += f"--output-to-directory={high_tile_path} --layer=blocks "
+        cmd += str(score_geojson_dir / "usa-high.json")
+        call(cmd, shell=True)
+
+        # generate low mbtiles file
+        logger.info("Generating USA Low mbtiles file")
+        cmd = "tippecanoe "
+        cmd += f"--minimum-zoom={USA_LOW_MIN_ZOOM} --maximum-zoom={USA_LOW_MAX_ZOOM} --layer=blocks "
+        cmd += f"--output={low_tile_path}/usa_low.mbtiles "
+        cmd += str(score_geojson_dir / "usa-low.json")
+        call(cmd, shell=True)
+
+        # generate low mvts
+        logger.info("Generating USA Low mvt folders and files")
+        cmd = "tippecanoe "
+        cmd += f"--minimum-zoom={USA_LOW_MIN_ZOOM} --maximum-zoom={USA_LOW_MAX_ZOOM} --no-tile-compression "
+        cmd += "--drop-densest-as-needed "
+        cmd += f"--output-to-directory={low_tile_path} --layer=blocks "
+        cmd += str(score_geojson_dir / "usa-low.json")
+        call(cmd, shell=True)
+
+    def _generate_tribal_tiles() -> None:
+        """Generates tribal layer tiles"""
+        tribal_tiles_path = data_path / "tribal" / "tiles"
+        tribal_geojson_dir = data_path / "tribal" / "geojson"
+
+        # remove existing mbtiles file
+        remove_all_from_dir(tribal_tiles_path)
+
+        # generate mbtiles file
+        logger.info("Generating Tribal mbtiles file")
+        cmd = "tippecanoe "
+        cmd += "--layer=blocks "
+        cmd += f"--output={tribal_tiles_path}/usa.mbtiles "
+        cmd += str(tribal_geojson_dir / "usa.json")
+        call(cmd, shell=True)
+
+        # generate mvts
+        logger.info("Generating Tribal mvt folders and files")
+        cmd = "tippecanoe "
+        cmd += "--no-tile-compression "
+        cmd += "--drop-densest-as-needed "
+        cmd += f"--output-to-directory={tribal_tiles_path} --layer=blocks "
+        cmd += str(tribal_geojson_dir / "usa.json")
+        call(cmd, shell=True)
+
+    if generate_tribal_layer:
+        _generate_tribal_tiles()
+    else:
+        _generate_score_tiles()