mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-08-11 19:54:18 -07:00
Backend change for Zipfile pt. 2 (#469)
* Fixes #303 : adding downloadable zip archive logic * linter recommendations * Pushes data directory to AWS. We'll want to move to use AWS for this ASAP, but this works for now * updating pattern
This commit is contained in:
parent
6eb7382d48
commit
9a9d5fdf7f
8 changed files with 223 additions and 10 deletions
60
.github/workflows/deploy_data.yml
vendored
Normal file
60
.github/workflows/deploy_data.yml
vendored
Normal file
|
@ -0,0 +1,60 @@
|
|||
name: Deploy Data
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- 'data/data-pipeline/*'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'data/data-pipeline/*'
|
||||
|
||||
jobs:
|
||||
deploy_data:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: data/data-pipeline
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: [3.9]
|
||||
steps:
|
||||
- name: Checkout source
|
||||
uses: actions/checkout@v2
|
||||
- name: Print variables to help debug
|
||||
uses: hmarr/debug-action@v2
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Setup Poetry
|
||||
uses: Gr1N/setup-poetry@v7
|
||||
- name: Print poetry version
|
||||
run: poetry --version
|
||||
- name: Install dependencies
|
||||
run: poetry install
|
||||
- name: Install GDAL/ogr2ogr
|
||||
run: |
|
||||
sudo apt-add-repository ppa:ubuntugis/ubuntugis-unstable
|
||||
sudo apt-get update
|
||||
sudo apt-get install gdal-bin libgdal-dev
|
||||
pip install GDAL==3.2.3
|
||||
- name: Run Scripts
|
||||
run: |
|
||||
poetry run download_census
|
||||
poetry run etl_and_score
|
||||
- name: Configure AWS Credentials
|
||||
uses: aws-actions/configure-aws-credentials@v1
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.DATA_DEV_AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.DATA_DEV_AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: us-east-1
|
||||
- name: Deploy to Geoplatform AWS
|
||||
run: |
|
||||
aws s3 sync ./data_pipeline/data/ s3://justice40-data/data-pipeline/data --delete
|
||||
- name: Update PR with Comment about deployment
|
||||
uses: mshick/add-pr-comment@v1
|
||||
with:
|
||||
message: |
|
||||
Data Synced! Find it here: s3://justice40-data/data-pipeline/data/
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
repo-token-user-login: 'github-actions[bot]' # The user.login for temporary GitHub tokens
|
||||
allow-repeats: false # This is the default
|
Loading…
Add table
Add a link
Reference in a new issue