From 5f588672a7c468603fdcc0ec92d5cde419f28e85 Mon Sep 17 00:00:00 2001
From: Matt Bowen <83967628+mattbowen-usds@users.noreply.github.com>
Date: Wed, 26 Oct 2022 15:01:19 -0400
Subject: [PATCH] Add action for creating new score versions (#2039)

* Add action for creating new score versions (2035)

Add a new action that allows us to manually create and upload score
versions to s3.

* Remove PR-related chat message

* Temportarily remove tile generation (#2035)

We want to see how this works, so we cut the time down for actually
running it by removing tile generation.

* Remove another PR-comment block
---
 .github/workflows/create-score-version.yml | 105 +++++++++++++++++++++
 1 file changed, 105 insertions(+)
 create mode 100644 .github/workflows/create-score-version.yml

diff --git a/.github/workflows/create-score-version.yml b/.github/workflows/create-score-version.yml
new file mode 100644
index 00000000..479876ad
--- /dev/null
+++ b/.github/workflows/create-score-version.yml
@@ -0,0 +1,105 @@
+name: Create Score Version
+on:
+  workflow_dispatch:
+    inputs:
+      score_version:
+        description: "Which version of the score are you generating?"
+        required: true
+        default: 'beta'
+        type: choice
+        options:
+        - beta
+        - 1.0
+      score_date:
+        description: "What is today's date, in YYYY-MM-DD format?"
+        required: true
+        type: string
+
+env:
+  CENSUS_API_KEY: ${{ secrets.CENSUS_API_KEY }}
+  J40_VERSION_LABEL_STRING: ${{ inputs.score_version }}
+  J40_VERSION_DATE_STRING: ${{ inputs.score_date }}
+
+jobs:
+  generate-score-tiles:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: data/data-pipeline
+    strategy:
+      matrix:
+        python-version: [3.9]
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2
+      - name: Print variables to help debug
+        uses: hmarr/debug-action@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Load cached Poetry installation
+        id: cached-poetry-dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: env-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/create-score-version.yml') }}
+      - name: Install poetry
+        uses: snok/install-poetry@v1
+      - name: Print Poetry settings
+        run: poetry show -v
+      - name: Install dependencies
+        run: poetry add s4cmd && poetry install
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.DATA_DEV_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.DATA_DEV_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+      - name: Generate Score
+        run: |
+          poetry run python3 data_pipeline/application.py score-full-run
+      - name: Generate Score Post
+        run: |
+          poetry run python3 data_pipeline/application.py generate-score-post -s aws
+      - name: Deploy Score to Geoplatform AWS
+        run: |
+          poetry run s4cmd put ./data_pipeline/data/score/csv/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/data/score/csv --recursive --force --API-ACL=public-read
+          poetry run s4cmd put ./data_pipeline/files/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read
+
+      # - name: Install GDAL/ogr2ogr
+      #   run: |
+      #     sudo add-apt-repository ppa:ubuntugis/ppa
+      #     sudo apt-get update
+      #     sudo apt-get -y install gdal-bin
+      #     ogrinfo --version
+      # - name: Set timezone for tippecanoe
+      #   uses: szenius/set-timezone@v1.0
+      #   with:
+      #     timezoneLinux: "America/Los_Angeles"
+      # - name: Get tippecanoe
+      #   run: |
+      #     sudo apt-get install -y software-properties-common libsqlite3-dev zlib1g-dev
+      #     sudo apt-add-repository -y ppa:git-core/ppa
+      #     sudo mkdir -p /tmp/tippecanoe-src
+      #     sudo git clone https://github.com/mapbox/tippecanoe.git /tmp/tippecanoe-src
+      # - name: Make tippecanoe
+      #   working-directory: /tmp/tippecanoe-src
+      #   run: |
+      #     sudo /usr/bin/bash -c make
+      #     mkdir -p /usr/local/bin
+      #     cp tippecanoe /usr/local/bin/tippecanoe
+      #     tippecanoe -v
+      # - name: Generate Score Geo
+      #   run: |
+      #     poetry run python3 data_pipeline/application.py geo-score
+      # - name: Generate Tiles
+      #   run: |
+      #     poetry run python3 data_pipeline/application.py generate-map-tiles
+      # - name: Deploy Map to Geoplatform AWS
+      #   run: |
+      #     poetry run s4cmd put ./data_pipeline/data/score/geojson/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/data/score/geojson --recursive --force  --API-ACL=public-read --num-threads=250
+      #     poetry run s4cmd put ./data_pipeline/data/score/shapefile/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/${{env.SHA_NUMBER}}/data/score/shapefile --recursive --force  --API-ACL=public-read
+      #     poetry run s4cmd put ./data_pipeline/data/score/tiles/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/data/score/tiles --recursive --force  --API-ACL=public-read --num-threads=250
+      #     poetry run s4cmd put ./data_pipeline/data/score/downloadable/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read