diff --git a/.github/workflows/create-score-version.yml b/.github/workflows/create-score-version.yml new file mode 100644 index 00000000..479876ad --- /dev/null +++ b/.github/workflows/create-score-version.yml @@ -0,0 +1,105 @@ +name: Create Score Version +on: + workflow_dispatch: + inputs: + score_version: + description: "Which version of the score are you generating?" + required: true + default: 'beta' + type: choice + options: + - beta + - 1.0 + score_date: + description: "What is today's date, in YYYY-MM-DD format?" + required: true + type: string + +env: + CENSUS_API_KEY: ${{ secrets.CENSUS_API_KEY }} + J40_VERSION_LABEL_STRING: ${{ inputs.score_version }} + J40_VERSION_DATE_STRING: ${{ inputs.score_date }} + +jobs: + generate-score-tiles: + runs-on: ubuntu-latest + defaults: + run: + working-directory: data/data-pipeline + strategy: + matrix: + python-version: [3.9] + steps: + - name: Checkout source + uses: actions/checkout@v2 + - name: Print variables to help debug + uses: hmarr/debug-action@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Load cached Poetry installation + id: cached-poetry-dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pypoetry/virtualenvs + key: env-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/create-score-version.yml') }} + - name: Install poetry + uses: snok/install-poetry@v1 + - name: Print Poetry settings + run: poetry show -v + - name: Install dependencies + run: poetry add s4cmd && poetry install + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.DATA_DEV_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.DATA_DEV_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Generate Score + run: | + poetry run python3 data_pipeline/application.py score-full-run + - name: Generate Score Post + run: | + poetry run python3 data_pipeline/application.py generate-score-post -s aws + - name: Deploy Score to Geoplatform AWS + run: | + poetry run s4cmd put ./data_pipeline/data/score/csv/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/data/score/csv --recursive --force --API-ACL=public-read + poetry run s4cmd put ./data_pipeline/files/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read + + # - name: Install GDAL/ogr2ogr + # run: | + # sudo add-apt-repository ppa:ubuntugis/ppa + # sudo apt-get update + # sudo apt-get -y install gdal-bin + # ogrinfo --version + # - name: Set timezone for tippecanoe + # uses: szenius/set-timezone@v1.0 + # with: + # timezoneLinux: "America/Los_Angeles" + # - name: Get tippecanoe + # run: | + # sudo apt-get install -y software-properties-common libsqlite3-dev zlib1g-dev + # sudo apt-add-repository -y ppa:git-core/ppa + # sudo mkdir -p /tmp/tippecanoe-src + # sudo git clone https://github.com/mapbox/tippecanoe.git /tmp/tippecanoe-src + # - name: Make tippecanoe + # working-directory: /tmp/tippecanoe-src + # run: | + # sudo /usr/bin/bash -c make + # mkdir -p /usr/local/bin + # cp tippecanoe /usr/local/bin/tippecanoe + # tippecanoe -v + # - name: Generate Score Geo + # run: | + # poetry run python3 data_pipeline/application.py geo-score + # - name: Generate Tiles + # run: | + # poetry run python3 data_pipeline/application.py generate-map-tiles + # - name: Deploy Map to Geoplatform AWS + # run: | + # poetry run s4cmd put ./data_pipeline/data/score/geojson/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/data/score/geojson --recursive --force --API-ACL=public-read --num-threads=250 + # poetry run s4cmd put ./data_pipeline/data/score/shapefile/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/${{env.SHA_NUMBER}}/data/score/shapefile --recursive --force --API-ACL=public-read + # poetry run s4cmd put ./data_pipeline/data/score/tiles/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/data/score/tiles --recursive --force --API-ACL=public-read --num-threads=250 + # poetry run s4cmd put ./data_pipeline/data/score/downloadable/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read