mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-02-23 10:04:18 -08:00
7b05ee9c76
* installation step * trigger action * installing to home dir * dry-run * pyenv * py 2.8 * trying s4cmd * removing pyenv * poetry s4cmd * num-threads * public read * poetry cache * s4cmd all around * poetry cache * poetry cache * install poetry packages * poetry echo * let's do this * s4cmd install on run * s4cmd * ad aws back * add aws back * testing census api key and poetry caching * census api key * census api * census api key #3 * 250 * poetry update * poetry change * check census api key * force flag * update score gen and tilefy; remove cached fips * small gdal update * invalidation * missing cache ids
59 lines
2.4 KiB
YAML
59 lines
2.4 KiB
YAML
name: Generate Score
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
confirm-action:
|
|
description: This will rebuild the data sources and regenerate the score, are you sure you want to proceed? (Y/n)
|
|
default: n
|
|
required: true
|
|
|
|
jobs:
|
|
deploy_data:
|
|
runs-on: ubuntu-latest
|
|
defaults:
|
|
run:
|
|
working-directory: data/data-pipeline
|
|
strategy:
|
|
matrix:
|
|
python-version: [3.9]
|
|
steps:
|
|
- name: Checkout source
|
|
uses: actions/checkout@v2
|
|
- name: Print variables to help debug
|
|
uses: hmarr/debug-action@v2
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@v2
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
- name: Load cached Poetry installation
|
|
id: cached-poetry-dependencies
|
|
uses: actions/cache@v2
|
|
with:
|
|
path: ~/.cache/pypoetry/virtualenvs
|
|
key: env-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/generate-score.yml') }}
|
|
- name: Install poetry
|
|
uses: snok/install-poetry@v1
|
|
- name: Print Poetry settings
|
|
run: poetry show -v
|
|
- name: Install dependencies
|
|
run: poetry install
|
|
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
|
|
- name: Configure AWS Credentials
|
|
uses: aws-actions/configure-aws-credentials@v1
|
|
with:
|
|
aws-access-key-id: ${{ secrets.DATA_DEV_AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.DATA_DEV_AWS_SECRET_ACCESS_KEY }}
|
|
aws-region: us-east-1
|
|
- name: Generate Score
|
|
run: |
|
|
poetry run python3 data_pipeline/application.py score-full-run
|
|
- name: Upload Score to AWS
|
|
run: |
|
|
aws s3 sync ./data_pipeline/data/score/csv/ s3://justice40-data/data-pipeline/data/score/csv --acl public-read --delete
|
|
- name: Generate Score Post
|
|
run: |
|
|
poetry run python3 data_pipeline/application.py generate-score-post -s aws
|
|
- name: Upload Score Post to AWS
|
|
run: |
|
|
aws s3 sync ./data_pipeline/data/score/csv/ s3://justice40-data/data-pipeline/data/score/csv --acl public-read --delete
|
|
aws s3 sync ./data_pipeline/data/score/downloadable/ s3://justice40-data/data-pipeline/data/score/downloadable --acl public-read --delete
|