j40-cejst-2/.github/workflows/pr_backend.yml

name: Pull Request Backend 
on:
  pull_request:
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
  cancel-in-progress: true
env:
  python-version: '3.10'
  J40_VERSION_LABEL_STRING: ${{ vars.SCORE_VERSION }}
jobs:
  # JOB to run change detection
  detect-be-changes:
    name: Detect backend changes
    runs-on: ubuntu-latest
    # Required permissions
    permissions:
      pull-requests: read
    # Set job outputs to values from filter step
    outputs:
      backend: ${{ steps.filter.outputs.backend }}
    steps:
    # For pull requests it's not necessary to checkout the code
    - uses: dorny/paths-filter@v3
      id: filter
      with:
        filters: |
          backend:
            - 'data/**'
            - '.github/workflows/pr_backend.yml'
  code-quality-checks:
    name: Code quality checks and tests
    needs: detect-be-changes
    if: ${{ needs.detect-be-changes.outputs.backend == 'true' }}
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: data/data-pipeline
    environment: PR
    steps:
      - name: Checkout source
        uses: actions/checkout@v4
      - name: Print variables to help debug
        uses: hmarr/debug-action@v3
      - name: Set up Python ${{ env.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.python-version }}
      - name: Load cached Poetry installation
        id: cached-poetry-dependencies
        uses: actions/cache@v4
        with:
          path: ~/.cache/pypoetry/virtualenvs
          key: env-${{ runner.os }}-${{ env.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/deploy_backend_main.yml') }}
      - name: Install poetry
        uses: snok/install-poetry@v1
      - name: Install dependencies
        run: poetry install
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
      - name: Check code is formatted
        run: poetry run black --check data_pipeline/
      - name: Check code style consistency
        run: poetry run flake8 -v data_pipeline/
      - name: Run static code analysis
        run: poetry run pylint data_pipeline/
      - name: Check library safety
        run: poetry run safety check --ignore 51457 --ignore 44715 --ignore 70612
      - name: Run unit tests
        run: |
          poetry run pytest data_pipeline/
  generate-score-tiles:
    name: Score and tile generation
    needs: detect-be-changes
    if: ${{ needs.detect-be-changes.outputs.backend == 'true' }}
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: data/data-pipeline
    environment: PR
    steps:
      - name: Checkout source
        uses: actions/checkout@v4
      - name: Print variables to help debug
        uses: hmarr/debug-action@v3
      - name: Set up Python ${{ env.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.python-version }}
      - name: Load cached Poetry installation
        id: cached-poetry-dependencies
        uses: actions/cache@v4
        with:
          path: ~/.cache/pypoetry/virtualenvs
          key: env-${{ runner.os }}-${{ env.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/deploy_backend_main.yml') }}
      - name: Install poetry
        uses: snok/install-poetry@v1
      - name: Print Poetry settings
        run: poetry show -v
      - name: Install dependencies
        run: poetry add s4cmd && poetry install
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
      - name: Install GDAL/ogr2ogr
        run: |
          sudo apt-get update
          sudo apt-get -y install gdal-bin
          ogrinfo --version
      - name: Cleanup Data
        run: |
          poetry run python3 -m data_pipeline.application data-cleanup
      - name: Get Census Data
        run: |
          poetry run python3 -m data_pipeline.application census-data-download
      - name: Run ETL
        run: |
          poetry run python3 -m data_pipeline.application etl-run
          poetry run python3 -m data_pipeline.application etl-run --dataset tribal
      - name: Generate Score
        run: |
          poetry run python3 -m data_pipeline.application score-run
      - name: Score Compare
        run: |
          poetry run python3 -m data_pipeline.comparator compare-score
      - name: Generate Score Post
        run: |
          poetry run python3 -m data_pipeline.application generate-score-post
      - name: Confirm we generated the version of the score we think we did
        if: ${{ env.J40_VERSION_LABEL_STRING == '2.0' || env.J40_VERSION_LABEL_STRING == 'beta' }}
        run: |
          grep -v "Identified as disadvantaged due to tribal overlap" data_pipeline/data/score/downloadable/* > /dev/null
      - name: Generate Score Geo
        run: |
          poetry run python3 -m data_pipeline.application geo-score
      - name: Run smoketest for 1.0
        if: ${{ env.J40_VERSION_LABEL_STRING == '1.0' }}
        run: |
          poetry run pytest data_pipeline/ -m smoketest
      - name: Set timezone for tippecanoe
        uses: szenius/set-timezone@v2.0
        with:
          timezoneLinux: "America/Los_Angeles"
      - name: Get tippecanoe
        run: |
          sudo apt-get install -y software-properties-common libsqlite3-dev zlib1g-dev
          sudo apt-add-repository -y ppa:git-core/ppa
          sudo mkdir -p /tmp/tippecanoe-src
          sudo git clone https://github.com/mapbox/tippecanoe.git /tmp/tippecanoe-src
      - name: Make tippecanoe
        working-directory: /tmp/tippecanoe-src
        run: |
          sudo /usr/bin/bash -c make
          mkdir -p /usr/local/bin
          cp tippecanoe /usr/local/bin/tippecanoe
          tippecanoe -v
      - name: Generate Tiles
        run: |
          poetry run python3 -m data_pipeline.application generate-map-tiles
          poetry run python3 -m data_pipeline.application generate-map-tiles --generate-tribal-layer