name: Create Score Version on: workflow_dispatch: inputs: score_version: description: "Which version of the score are you generating?" required: true default: 'beta' type: choice options: - beta - 1.0 env: CENSUS_API_KEY: ${{ secrets.CENSUS_API_KEY }} J40_VERSION_LABEL_STRING: ${{ inputs.score_version }} jobs: generate-score-tiles: runs-on: ubuntu-latest defaults: run: working-directory: data/data-pipeline strategy: matrix: python-version: [3.9] steps: - name: Checkout source uses: actions/checkout@v2 - name: Print variables to help debug uses: hmarr/debug-action@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Load cached Poetry installation id: cached-poetry-dependencies uses: actions/cache@v2 with: path: ~/.cache/pypoetry/virtualenvs key: env-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/create-score-version.yml') }} - name: Install poetry uses: snok/install-poetry@v1 - name: Print Poetry settings run: poetry show -v - name: Install dependencies run: poetry add s4cmd && poetry install if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: aws-access-key-id: ${{ secrets.DATA_DEV_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.DATA_DEV_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: Install GDAL/ogr2ogr run: | sudo add-apt-repository ppa:ubuntugis/ppa sudo apt-get update sudo apt-get -y install gdal-bin ogrinfo --version - name: Generate Score run: | poetry run python3 data_pipeline/application.py score-full-run - name: Generate Score Post run: | poetry run python3 data_pipeline/application.py generate-score-post -s aws - name: Generate Score Geo run: | poetry run python3 data_pipeline/application.py geo-score - name: Deploy Score to Geoplatform AWS run: | poetry run s4cmd put ./data_pipeline/data/score/csv/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/csv --recursive --force --API-ACL=public-read poetry run s4cmd put ./data_pipeline/files/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read poetry run s4cmd put ./data_pipeline/data/score/downloadable/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read - name: Set timezone for tippecanoe uses: szenius/set-timezone@v1.0 with: timezoneLinux: "America/Los_Angeles" - name: Get tippecanoe run: | sudo apt-get install -y software-properties-common libsqlite3-dev zlib1g-dev sudo apt-add-repository -y ppa:git-core/ppa sudo mkdir -p /tmp/tippecanoe-src sudo git clone https://github.com/mapbox/tippecanoe.git /tmp/tippecanoe-src - name: Make tippecanoe working-directory: /tmp/tippecanoe-src run: | sudo /usr/bin/bash -c make mkdir -p /usr/local/bin cp tippecanoe /usr/local/bin/tippecanoe tippecanoe -v - name: Generate Tiles run: | poetry run python3 data_pipeline/application.py generate-map-tiles - name: Deploy Map to Geoplatform AWS run: | poetry run s4cmd put ./data_pipeline/data/score/geojson/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/geojson --recursive --force --API-ACL=public-read --num-threads=250 poetry run s4cmd put ./data_pipeline/data/score/shapefile/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/${{env.SHA_NUMBER}}/data/score/shapefile --recursive --force --API-ACL=public-read poetry run s4cmd put ./data_pipeline/data/score/tiles/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/tiles --recursive --force --API-ACL=public-read --num-threads=250