j40-cejst-2/.github/workflows/create-score-version.yml

name: Create Score Version
on:
  workflow_dispatch:
    inputs:
      score_version:
        description: "Which version of the score are you generating?"
        required: true
        default: '1.0'
        type: choice
        options:
        - beta
        - 1.0
        - test

env:
  CENSUS_API_KEY: ${{ secrets.CENSUS_API_KEY }}
  J40_VERSION_LABEL_STRING: ${{ inputs.score_version }}

jobs:
  generate-score-tiles:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: data/data-pipeline
    strategy:
      matrix:
        python-version: [3.9]
    steps:
      # Pull in source and setup the build environment
      - name: Checkout source
        uses: actions/checkout@v2
      - name: Print variables to help debug
        uses: hmarr/debug-action@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Load cached Poetry installation
        id: cached-poetry-dependencies
        uses: actions/cache@v2
        with:
          path: ~/.cache/pypoetry/virtualenvs
          key: env-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/create-score-version.yml') }}
      - name: Install poetry
        uses: snok/install-poetry@v1
      - name: Print Poetry settings
        run: poetry show -v
      - name: Install dependencies
        run: poetry add s4cmd && poetry install
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.DATA_DEV_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.DATA_DEV_AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-1
      - name: Install GDAL/ogr2ogr
        run: |
          sudo apt-get update
          sudo apt-get -y install gdal-bin
          ogrinfo --version
      - name: Set timezone for tippecanoe
        uses: szenius/set-timezone@v1.0
        with:
          timezoneLinux: "America/Los_Angeles"
      - name: Get tippecanoe
        run: |
          sudo apt-get install -y software-properties-common libsqlite3-dev zlib1g-dev
          sudo apt-add-repository -y ppa:git-core/ppa
          sudo mkdir -p /tmp/tippecanoe-src
          sudo git clone https://github.com/mapbox/tippecanoe.git /tmp/tippecanoe-src
      - name: Make tippecanoe
        working-directory: /tmp/tippecanoe-src
        run: |
          sudo /usr/bin/bash -c make
          mkdir -p /usr/local/bin
          cp tippecanoe /usr/local/bin/tippecanoe
          tippecanoe -v
          
      # Generate score and map tiles
      - name: Generate Score
        run: |
          poetry run python3 data_pipeline/application.py score-full-run
      - name: Generate Score Post
        run: |
          poetry run python3 data_pipeline/application.py generate-score-post -s aws
      - name: Generate Score Geo
        run: |
          poetry run python3 data_pipeline/application.py geo-score
      - name: Generate Tiles
        run: |
          poetry run python3 data_pipeline/application.py generate-map-tiles
          
      # Run basic smoke tests on score and map tiles
      - name: Confirm we generated the version of the score we think we did
        if: ${{ env.J40_VERSION_LABEL_STRING != 'beta' }}
        run: |
          grep "Identified as disadvantaged due to tribal overlap" data_pipeline/data/score/downloadable/* > /dev/null
      - name: Confirm we generated the version of the score we think we did
        if: ${{ env.J40_VERSION_LABEL_STRING == 'beta' }}
        run: |
          grep -v "Identified as disadvantaged due to tribal overlap" data_pipeline/data/score/downloadable/* > /dev/null
      - name: Run smoketest
        if: ${{ env.J40_VERSION_LABEL_STRING != 'beta' }}
        run: |
          poetry run pytest data_pipeline/ -m smoketest
      
      # Deploy to AWS
      - name: Deploy Score to Geoplatform AWS
        run: |
          poetry run s4cmd put ./data_pipeline/data/score/csv/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/csv --recursive --force --API-ACL=public-read
          poetry run s4cmd put ./data_pipeline/files/static/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read
          poetry run s4cmd put ./data_pipeline/data/score/downloadable/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read
      - name: Deploy Map to Geoplatform AWS
        run: |
          poetry run s4cmd put ./data_pipeline/data/score/geojson/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/geojson --recursive --force  --API-ACL=public-read --num-threads=250
          poetry run s4cmd put ./data_pipeline/data/score/shapefile/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/${{env.SHA_NUMBER}}/data/score/shapefile --recursive --force  --API-ACL=public-read
          poetry run s4cmd put ./data_pipeline/data/score/tiles/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/tiles --recursive --force  --API-ACL=public-read --num-threads=250
      - name: Deploy Test Files to Geoplatform AWS
        if: ${{ env.J40_VERSION_LABEL_STRING == 'test' }}
        run: |
          poetry run s4cmd put ./data_pipeline/files/test/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read
          
      # Run more smoke tests to ensure we deployed everything correctly
      - name: Confirm we deployed downloadables to AWS
        if: ${{ env.J40_VERSION_LABEL_STRING != 'beta' && env.J40_VERSION_LABEL_STRING != 'test' }}
        run: |
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/${{env.J40_VERSION_LABEL_STRING}}-shapefile-codebook.zip" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/${{env.J40_VERSION_LABEL_STRING}}-communities.xlsx" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/${{env.J40_VERSION_LABEL_STRING}}-communities.csv" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/${{env.J40_VERSION_LABEL_STRING}}-communities-xls.zip" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/${{env.J40_VERSION_LABEL_STRING}}-communities-csv.zip" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/${{env.J40_VERSION_LABEL_STRING}}-codebook.csv" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/${{env.J40_VERSION_LABEL_STRING}}-data-documentation.zip" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/cejst-technical-support-document.pdf" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/CEQ-CEJST-Instructions.pdf" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/climate-and-environmental-burden-tool-comparison.pdf" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/communities-list.pdf" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/readme.md" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/technical-training-slides.pptx" -s -f -I -o /dev/null
      - name: Confirm we deployed downloadables to AWS (beta)
        if: ${{ env.J40_VERSION_LABEL_STRING == 'beta' }}
        run: |
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/beta/data/score/downloadable/beta-data-documentation.zip" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/beta/data/score/downloadable/beta-shapefile-codebook.zip" -s -f -I -o /dev/null    
      - name: Confirm we deployed additional downloadables – from the files directory – to AWS (test)
        if: ${{ env.J40_VERSION_LABEL_STRING == 'test' }}
        run: |
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/test.md" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/cejst-technical-support-document.pdf" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/CEQ-CEJST-Instructions.pdf" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/climate-and-environmental-burden-tool-comparison.pdf" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/communities-list.pdf" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/readme.md" -s -f -I -o /dev/null && \
          curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/technical-training-slides.pptx" -s -f -I -o /dev/null