publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-07-28 08:11:16 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

ArcGIS zipping (#1391)

Browse source 
* ArcGIS zipping

* lint

* shapefile zip

* removing space in GMT

* adding shapefile to be staging gha
This commit is contained in:
Jorge Escobar 2022-03-09 18:00:20 -05:00 • committed by GitHub
commit 7f91e2b06b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 17 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

2
data/data-pipeline/data_pipeline/etl/score/constants.py
View file

@ -45,7 +45,7 @@ DATA_SCORE_TILES_DIR = DATA_SCORE_DIR / "tiles"
# Downloadable paths
current_dt = datetime.datetime.now()
timestamp_str = current_dt.strftime("%Y-%m-%d-%H%M")
timestamp_str = current_dt.strftime("%Y-%m-%d-%H%MGMT")
SCORE_DOWNLOADABLE_DIR = DATA_SCORE_DIR / "downloadable"
SCORE_DOWNLOADABLE_PDF_FILE_NAME = "Draft_Communities_List.pdf"
SCORE_DOWNLOADABLE_PDF_FILE_PATH = FILES_PATH / SCORE_DOWNLOADABLE_PDF_FILE_NAME

13
data/data-pipeline/data_pipeline/etl/score/etl_score_geo.py
View file

@ -1,5 +1,6 @@
import concurrent.futures
import math
import os
import numpy as np
import pandas as pd
import geopandas as gpd
@ -11,7 +12,7 @@ from data_pipeline.etl.sources.census.etl_utils import (
)
from data_pipeline.etl.score.etl_utils import check_score_data_source
from data_pipeline.score import field_names
from data_pipeline.utils import get_module_logger
from data_pipeline.utils import get_module_logger, zip_files
logger = get_module_logger(__name__)
@ -298,11 +299,21 @@ class GeoScoreETL(ExtractTransformLoad):
# kept as strings because no downstream impacts
columns={0: "column", "index": "meaning"}
).to_csv(self.SCORE_SHP_CODE_CSV, index=False)
self.geojson_score_usa_high.rename(columns=renaming_map).to_file(
self.SCORE_SHP_FILE
)
logger.info("Completed writing shapefile")
arcgis_zip_file_path = self.SCORE_SHP_PATH / "usa.zip"
arcgis_files = []
for file in os.listdir(self.SCORE_SHP_PATH):
# don't remove __init__ files as they conserve dir structure
if file != "__init__.py":
arcgis_files.append(self.SCORE_SHP_PATH / file)
zip_files(arcgis_zip_file_path, arcgis_files)
logger.info("Completed zipping shapefiles")
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = {
executor.submit(task)