mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-10-19 10:23:51 -07:00
Update score version workflow to also upload tiles (#2042)
* Actually upload the score * Add back tile generation (#2035) * Move score GEO up (#2035) * Copy beta README to 1.0 (#2035) * Remove dates from file names (#2035) * Remove unused score ETLs from constants (#2035) * Put back child opp index (#2035) * Remove unused import (#2035)
This commit is contained in:
Matt Bowen
GitHub
parent
d7a0be7ee7
commit
0d0cf2a5f9
4 changed files with 146 additions and 63 deletions
75
.github/workflows/create-score-version.yml
vendored
75
.github/workflows/create-score-version.yml
vendored
|
|
@ -10,15 +10,10 @@ on:
|
|||
options:
|
||||
- beta
|
||||
- 1.0
|
||||
score_date:
|
||||
description: "What is today's date, in YYYY-MM-DD format?"
|
||||
required: true
|
||||
type: string
|
||||
|
||||
env:
|
||||
CENSUS_API_KEY: ${{ secrets.CENSUS_API_KEY }}
|
||||
J40_VERSION_LABEL_STRING: ${{ inputs.score_version }}
|
||||
J40_VERSION_DATE_STRING: ${{ inputs.score_date }}
|
||||
|
||||
jobs:
|
||||
generate-score-tiles:
|
||||
|
|
@ -57,49 +52,49 @@ jobs:
|
|||
aws-access-key-id: ${{ secrets.DATA_DEV_AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.DATA_DEV_AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: us-east-1
|
||||
- name: Install GDAL/ogr2ogr
|
||||
run: |
|
||||
sudo add-apt-repository ppa:ubuntugis/ppa
|
||||
sudo apt-get update
|
||||
sudo apt-get -y install gdal-bin
|
||||
ogrinfo --version
|
||||
- name: Generate Score
|
||||
run: |
|
||||
poetry run python3 data_pipeline/application.py score-full-run
|
||||
- name: Generate Score Post
|
||||
run: |
|
||||
poetry run python3 data_pipeline/application.py generate-score-post -s aws
|
||||
- name: Generate Score Geo
|
||||
run: |
|
||||
poetry run python3 data_pipeline/application.py geo-score
|
||||
- name: Deploy Score to Geoplatform AWS
|
||||
run: |
|
||||
poetry run s4cmd put ./data_pipeline/data/score/csv/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/csv --recursive --force --API-ACL=public-read
|
||||
poetry run s4cmd put ./data_pipeline/files/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read
|
||||
poetry run s4cmd put ./data_pipeline/data/score/downloadable/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read
|
||||
|
||||
# - name: Install GDAL/ogr2ogr
|
||||
# run: |
|
||||
# sudo add-apt-repository ppa:ubuntugis/ppa
|
||||
# sudo apt-get update
|
||||
# sudo apt-get -y install gdal-bin
|
||||
# ogrinfo --version
|
||||
# - name: Set timezone for tippecanoe
|
||||
# uses: szenius/set-timezone@v1.0
|
||||
# with:
|
||||
# timezoneLinux: "America/Los_Angeles"
|
||||
# - name: Get tippecanoe
|
||||
# run: |
|
||||
# sudo apt-get install -y software-properties-common libsqlite3-dev zlib1g-dev
|
||||
# sudo apt-add-repository -y ppa:git-core/ppa
|
||||
# sudo mkdir -p /tmp/tippecanoe-src
|
||||
# sudo git clone https://github.com/mapbox/tippecanoe.git /tmp/tippecanoe-src
|
||||
# - name: Make tippecanoe
|
||||
# working-directory: /tmp/tippecanoe-src
|
||||
# run: |
|
||||
# sudo /usr/bin/bash -c make
|
||||
# mkdir -p /usr/local/bin
|
||||
# cp tippecanoe /usr/local/bin/tippecanoe
|
||||
# tippecanoe -v
|
||||
# - name: Generate Score Geo
|
||||
# run: |
|
||||
# poetry run python3 data_pipeline/application.py geo-score
|
||||
# - name: Generate Tiles
|
||||
# run: |
|
||||
# poetry run python3 data_pipeline/application.py generate-map-tiles
|
||||
# - name: Deploy Map to Geoplatform AWS
|
||||
# run: |
|
||||
# poetry run s4cmd put ./data_pipeline/data/score/geojson/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/data/score/geojson --recursive --force --API-ACL=public-read --num-threads=250
|
||||
# poetry run s4cmd put ./data_pipeline/data/score/shapefile/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/${{env.SHA_NUMBER}}/data/score/shapefile --recursive --force --API-ACL=public-read
|
||||
# poetry run s4cmd put ./data_pipeline/data/score/tiles/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/data/score/tiles --recursive --force --API-ACL=public-read --num-threads=250
|
||||
# poetry run s4cmd put ./data_pipeline/data/score/downloadable/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read
|
||||
- name: Set timezone for tippecanoe
|
||||
uses: szenius/set-timezone@v1.0
|
||||
with:
|
||||
timezoneLinux: "America/Los_Angeles"
|
||||
- name: Get tippecanoe
|
||||
run: |
|
||||
sudo apt-get install -y software-properties-common libsqlite3-dev zlib1g-dev
|
||||
sudo apt-add-repository -y ppa:git-core/ppa
|
||||
sudo mkdir -p /tmp/tippecanoe-src
|
||||
sudo git clone https://github.com/mapbox/tippecanoe.git /tmp/tippecanoe-src
|
||||
- name: Make tippecanoe
|
||||
working-directory: /tmp/tippecanoe-src
|
||||
run: |
|
||||
sudo /usr/bin/bash -c make
|
||||
mkdir -p /usr/local/bin
|
||||
cp tippecanoe /usr/local/bin/tippecanoe
|
||||
tippecanoe -v
|
||||
- name: Generate Tiles
|
||||
run: |
|
||||
poetry run python3 data_pipeline/application.py generate-map-tiles
|
||||
- name: Deploy Map to Geoplatform AWS
|
||||
run: |
|
||||
poetry run s4cmd put ./data_pipeline/data/score/geojson/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/geojson --recursive --force --API-ACL=public-read --num-threads=250
|
||||
poetry run s4cmd put ./data_pipeline/data/score/shapefile/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/${{env.SHA_NUMBER}}/data/score/shapefile --recursive --force --API-ACL=public-read
|
||||
poetry run s4cmd put ./data_pipeline/data/score/tiles/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/tiles --recursive --force --API-ACL=public-read --num-threads=250
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue