mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-02-22 09:41:26 -08:00
Staging Data Pipeline
This commit is contained in:
ericiwamoto
Carlos Felix
parent
cce91fb47b
commit
999c1f6dee
1 changed files with 126 additions and 0 deletions
126
.github/workflows/deploy_backend_main.yml
vendored
Normal file
126
.github/workflows/deploy_backend_main.yml
vendored
Normal file
|
|
@ -0,0 +1,126 @@
|
||||||
|
name: Deploy Backend Main
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
score_version:
|
||||||
|
description: "Which version of the score are you generating?"
|
||||||
|
required: true
|
||||||
|
default: '2.0'
|
||||||
|
type: choice
|
||||||
|
options:
|
||||||
|
- beta
|
||||||
|
- 1.0
|
||||||
|
- 2.0
|
||||||
|
- test
|
||||||
|
|
||||||
|
env:
|
||||||
|
CENSUS_API_KEY: ${{ secrets.CENSUS_API_KEY }}
|
||||||
|
J40_VERSION_LABEL_STRING: ${{ inputs.score_version }}
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
generate-score-tiles:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
working-directory: data/data-pipeline
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
python-version: [3.10]
|
||||||
|
steps:
|
||||||
|
- name: Checkout source
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
- name: Print variables to help debug
|
||||||
|
uses: hmarr/debug-action@v2
|
||||||
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
|
uses: actions/setup-python@v2
|
||||||
|
with:
|
||||||
|
python-version: ${{ matrix.python-version }}
|
||||||
|
- name: Load cached Poetry installation
|
||||||
|
id: cached-poetry-dependencies
|
||||||
|
uses: actions/cache@v4
|
||||||
|
with:
|
||||||
|
path: ~/.cache/pypoetry/virtualenvs
|
||||||
|
key: env-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/deploy_backend_main.yml') }}
|
||||||
|
- name: Install poetry
|
||||||
|
uses: snok/install-poetry@v1
|
||||||
|
- name: Print Poetry settings
|
||||||
|
run: poetry show -v
|
||||||
|
- name: Install dependencies
|
||||||
|
run: poetry add s4cmd && poetry install
|
||||||
|
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
|
||||||
|
- name: Configure AWS Credentials
|
||||||
|
uses: aws-actions/configure-aws-credentials@v1
|
||||||
|
with:
|
||||||
|
aws-access-key-id: ${{ secrets.DATA_DEV_AWS_ACCESS_KEY_ID }}
|
||||||
|
aws-secret-access-key: ${{ secrets.DATA_DEV_AWS_SECRET_ACCESS_KEY }}
|
||||||
|
aws-region: us-east-1
|
||||||
|
- name: Install GDAL/ogr2ogr
|
||||||
|
run: |
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get -y install gdal-bin
|
||||||
|
ogrinfo --version
|
||||||
|
- name: Generate Score
|
||||||
|
run: |
|
||||||
|
poetry run python3 data_pipeline/application.py score-full-run
|
||||||
|
- name: Generate Score Post
|
||||||
|
run: |
|
||||||
|
poetry run python3 data_pipeline/application.py generate-score-post -s aws
|
||||||
|
- name: Confirm we generated the version of the score we think we did
|
||||||
|
if: ${{ env.J40_VERSION_LABEL_STRING == '1.0' || env.J40_VERSION_LABEL_STRING == 'test' }}
|
||||||
|
run: |
|
||||||
|
grep "Identified as disadvantaged due to tribal overlap" data_pipeline/data/score/downloadable/* > /dev/null
|
||||||
|
- name: Confirm we generated the version of the score we think we did
|
||||||
|
if: ${{ env.J40_VERSION_LABEL_STRING == '2.0' || env.J40_VERSION_LABEL_STRING == 'beta' }}
|
||||||
|
run: |
|
||||||
|
grep -v "Identified as disadvantaged due to tribal overlap" data_pipeline/data/score/downloadable/* > /dev/null
|
||||||
|
- name: Generate Score Geo
|
||||||
|
run: |
|
||||||
|
poetry run python3 data_pipeline/application.py geo-score
|
||||||
|
- name: Run smoketest for 1.0
|
||||||
|
if: ${{ env.J40_VERSION_LABEL_STRING == '1.0' || env.J40_VERSION_LABEL_STRING == 'test' }}
|
||||||
|
run: |
|
||||||
|
poetry run pytest data_pipeline/ -m smoketest
|
||||||
|
- name: Deploy Score to Geoplatform AWS
|
||||||
|
run: |
|
||||||
|
poetry run s4cmd put ./data_pipeline/data/score/csv/ s3://${{secrets.S3_DATA_BUCKET}}/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/csv --recursive --force
|
||||||
|
poetry run s4cmd put ./data_pipeline/files/ s3://${{secrets.S3_DATA_BUCKET}}/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force
|
||||||
|
poetry run s4cmd put ./data_pipeline/data/score/downloadable/ s3://${{secrets.S3_DATA_BUCKET}}/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force
|
||||||
|
- name: Confirm we generated the version of the score we think we did
|
||||||
|
if: ${{ env.J40_VERSION_LABEL_STRING == '1.0' || env.J40_VERSION_LABEL_STRING == 'test' }}
|
||||||
|
run: |
|
||||||
|
curl "${{secrets.DATA_URL}}/data-versions/1.0/data/score/downloadable/1.0-shapefile-codebook.zip" -s -f -I -o /dev/null && \
|
||||||
|
curl "${{secrets.DATA_URL}}/data-versions/1.0/data/score/downloadable/1.0-communities.xlsx" -s -f -I -o /dev/null && \
|
||||||
|
curl "${{secrets.DATA_URL}}/data-versions/1.0/data/score/downloadable/1.0-communities.csv" -s -f -I -o /dev/null && \
|
||||||
|
curl "${{secrets.DATA_URL}}/data-versions/1.0/data/score/downloadable/1.0-shapefile-codebook.zip" -s -f -I -o /dev/null && \
|
||||||
|
curl "${{secrets.DATA_URL}}/data-versions/1.0/data/score/downloadable/cejst-technical-support-document.pdf" -s -f -I -o /dev/null && \
|
||||||
|
curl "${{secrets.DATA_URL}}/data-versions/1.0/data/score/downloadable/draft-communities-list.pdf" -s -f -I -o /dev/null
|
||||||
|
- name: Confirm we generated the version of the score we think we did
|
||||||
|
if: ${{ env.J40_VERSION_LABEL_STRING == '2.0' || env.J40_VERSION_LABEL_STRING == 'beta' }}
|
||||||
|
run: |
|
||||||
|
curl "${{secrets.DATA_URL}}/data-versions/2.0/data/score/downloadable/2.0-data-documentation.zip" -s -f -I -o /dev/null && \
|
||||||
|
curl "${{secrets.DATA_URL}}/data-versions/2.0/data/score/downloadable/2.0-shapefile-codebook.zip" -s -f -I -o /dev/null
|
||||||
|
- name: Set timezone for tippecanoe
|
||||||
|
uses: szenius/set-timezone@v1.0
|
||||||
|
with:
|
||||||
|
timezoneLinux: "America/Los_Angeles"
|
||||||
|
- name: Get tippecanoe
|
||||||
|
run: |
|
||||||
|
sudo apt-get install -y software-properties-common libsqlite3-dev zlib1g-dev
|
||||||
|
sudo apt-add-repository -y ppa:git-core/ppa
|
||||||
|
sudo mkdir -p /tmp/tippecanoe-src
|
||||||
|
sudo git clone https://github.com/mapbox/tippecanoe.git /tmp/tippecanoe-src
|
||||||
|
- name: Make tippecanoe
|
||||||
|
working-directory: /tmp/tippecanoe-src
|
||||||
|
run: |
|
||||||
|
sudo /usr/bin/bash -c make
|
||||||
|
mkdir -p /usr/local/bin
|
||||||
|
cp tippecanoe /usr/local/bin/tippecanoe
|
||||||
|
tippecanoe -v
|
||||||
|
- name: Generate Tiles
|
||||||
|
run: |
|
||||||
|
poetry run python3 data_pipeline/application.py generate-map-tiles
|
||||||
|
- name: Deploy Map to Geoplatform AWS
|
||||||
|
run: |
|
||||||
|
poetry run s4cmd put ./data_pipeline/data/score/geojson/ s3://${{secrets.S3_DATA_BUCKET}}/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/geojson --recursive --force --num-threads=250
|
||||||
|
poetry run s4cmd put ./data_pipeline/data/score/shapefile/ s3://${{secrets.S3_DATA_BUCKET}}/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/shapefile --recursive --force
|
||||||
|
poetry run s4cmd put ./data_pipeline/data/score/tiles/ s3://${{secrets.S3_DATA_BUCKET}}/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/tiles --recursive --force --num-threads=250
|
||||||
Loading…
Add table
Reference in a new issue