diff --git a/.github/workflows/create-score-version.yml b/.github/workflows/create-score-version.yml index 103da843..adf368f9 100644 --- a/.github/workflows/create-score-version.yml +++ b/.github/workflows/create-score-version.yml @@ -26,6 +26,7 @@ jobs: matrix: python-version: [3.9] steps: + # Pull in source and setup the build environment - name: Checkout source uses: actions/checkout@v2 - name: Print variables to help debug @@ -58,46 +59,6 @@ jobs: sudo apt-get update sudo apt-get -y install gdal-bin ogrinfo --version - - name: Generate Score - run: | - poetry run python3 data_pipeline/application.py score-full-run - - name: Generate Score Post - run: | - poetry run python3 data_pipeline/application.py generate-score-post -s aws - - name: Confirm we generated the version of the score we think we did - if: ${{ env.J40_VERSION_LABEL_STRING == '1.0' || env.J40_VERSION_LABEL_STRING == 'test' }} - run: | - grep "Identified as disadvantaged due to tribal overlap" data_pipeline/data/score/downloadable/* > /dev/null - - name: Confirm we generated the version of the score we think we did - if: ${{ env.J40_VERSION_LABEL_STRING == 'beta' }} - run: | - grep -v "Identified as disadvantaged due to tribal overlap" data_pipeline/data/score/downloadable/* > /dev/null - - name: Generate Score Geo - run: | - poetry run python3 data_pipeline/application.py geo-score - - name: Run smoketest for 1.0 - if: ${{ env.J40_VERSION_LABEL_STRING == '1.0' || env.J40_VERSION_LABEL_STRING == 'test' }} - run: | - poetry run pytest data_pipeline/ -m smoketest - - name: Deploy Score to Geoplatform AWS - run: | - poetry run s4cmd put ./data_pipeline/data/score/csv/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/csv --recursive --force --API-ACL=public-read - poetry run s4cmd put ./data_pipeline/files/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read - poetry run s4cmd put ./data_pipeline/data/score/downloadable/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read - - name: Confirm we generated the version of the score we think we did - if: ${{ env.J40_VERSION_LABEL_STRING == '1.0' || env.J40_VERSION_LABEL_STRING == 'test' }} - run: | - curl "https://static-data-screeningtool.geoplatform.gov/data-versions/1.0/data/score/downloadable/1.0-shapefile-codebook.zip" -s -f -I -o /dev/null && \ - curl "https://static-data-screeningtool.geoplatform.gov/data-versions/1.0/data/score/downloadable/1.0-communities.xlsx" -s -f -I -o /dev/null && \ - curl "https://static-data-screeningtool.geoplatform.gov/data-versions/1.0/data/score/downloadable/1.0-communities.csv" -s -f -I -o /dev/null && \ - curl "https://static-data-screeningtool.geoplatform.gov/data-versions/1.0/data/score/downloadable/1.0-shapefile-codebook.zip" -s -f -I -o /dev/null && \ - curl "https://static-data-screeningtool.geoplatform.gov/data-versions/1.0/data/score/downloadable/cejst-technical-support-document.pdf" -s -f -I -o /dev/null && \ - curl "https://static-data-screeningtool.geoplatform.gov/data-versions/1.0/data/score/downloadable/draft-communities-list.pdf" -s -f -I -o /dev/null - - name: Confirm we generated the version of the score we think we did - if: ${{ env.J40_VERSION_LABEL_STRING == 'beta' }} - run: | - curl "https://static-data-screeningtool.geoplatform.gov/data-versions/beta/data/score/downloadable/beta-data-documentation.zip" -s -f -I -o /dev/null && \ - curl "https://static-data-screeningtool.geoplatform.gov/data-versions/beta/data/score/downloadable/beta-shapefile-codebook.zip" -s -f -I -o /dev/null - name: Set timezone for tippecanoe uses: szenius/set-timezone@v1.0 with: @@ -115,11 +76,64 @@ jobs: mkdir -p /usr/local/bin cp tippecanoe /usr/local/bin/tippecanoe tippecanoe -v + + # Generate score and map tiles + - name: Generate Score + run: | + poetry run python3 data_pipeline/application.py score-full-run + - name: Generate Score Post + run: | + poetry run python3 data_pipeline/application.py generate-score-post -s aws + - name: Generate Score Geo + run: | + poetry run python3 data_pipeline/application.py geo-score - name: Generate Tiles run: | poetry run python3 data_pipeline/application.py generate-map-tiles + + # Run basic smoke tests on score and map tiles + - name: Confirm we generated the version of the score we think we did + if: ${{ env.J40_VERSION_LABEL_STRING != 'beta' }} + run: | + grep "Identified as disadvantaged due to tribal overlap" data_pipeline/data/score/downloadable/* > /dev/null + - name: Confirm we generated the version of the score we think we did + if: ${{ env.J40_VERSION_LABEL_STRING == 'beta' }} + run: | + grep -v "Identified as disadvantaged due to tribal overlap" data_pipeline/data/score/downloadable/* > /dev/null + - name: Run smoketest + if: ${{ env.J40_VERSION_LABEL_STRING != 'beta' }} + run: | + poetry run pytest data_pipeline/ -m smoketest + + # Deploy to AWS + - name: Deploy Score to Geoplatform AWS + run: | + poetry run s4cmd put ./data_pipeline/data/score/csv/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/csv --recursive --force --API-ACL=public-read + poetry run s4cmd put ./data_pipeline/files/${{env.J40_VERSION_LABEL_STRING}}/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read + poetry run s4cmd put ./data_pipeline/data/score/downloadable/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read - name: Deploy Map to Geoplatform AWS run: | poetry run s4cmd put ./data_pipeline/data/score/geojson/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/geojson --recursive --force --API-ACL=public-read --num-threads=250 poetry run s4cmd put ./data_pipeline/data/score/shapefile/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/${{env.SHA_NUMBER}}/data/score/shapefile --recursive --force --API-ACL=public-read poetry run s4cmd put ./data_pipeline/data/score/tiles/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/tiles --recursive --force --API-ACL=public-read --num-threads=250 + + # Run more smoke tests to ensure we deployed everything correctly + - name: Confirm we deployed downloadables to AWS + if: ${{ env.J40_VERSION_LABEL_STRING != 'beta' }} + run: | + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/1.0-shapefile-codebook.zip" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/1.0-communities.xlsx" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/1.0-communities.csv" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/1.0-shapefile-codebook.zip" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/cejst-technical-support-document.pdf" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/draft-communities-list.pdf" -s -f -I -o /dev/null + - name: Confirm we deployed downloadables to AWS (beta) + if: ${{ env.J40_VERSION_LABEL_STRING == 'beta' }} + run: | + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/beta/data/score/downloadable/beta-data-documentation.zip" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/beta/data/score/downloadable/beta-shapefile-codebook.zip" -s -f -I -o /dev/null + - name: Confirm we deployed additional downloadables – from the files directory – to AWS (test) + if: ${{ env.J40_VERSION_LABEL_STRING == 'test' }} + run: | + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/beta/data/score/downloadable/test.md" -s -f -I -o /dev/null + \ No newline at end of file diff --git a/data/data-pipeline/data_pipeline/files/cejst-technical-support-document-es.pdf b/data/data-pipeline/data_pipeline/files/beta/cejst-technical-support-document-es.pdf similarity index 100% rename from data/data-pipeline/data_pipeline/files/cejst-technical-support-document-es.pdf rename to data/data-pipeline/data_pipeline/files/beta/cejst-technical-support-document-es.pdf diff --git a/data/data-pipeline/data_pipeline/files/cejst-technical-support-document.pdf b/data/data-pipeline/data_pipeline/files/beta/cejst-technical-support-document.pdf similarity index 100% rename from data/data-pipeline/data_pipeline/files/cejst-technical-support-document.pdf rename to data/data-pipeline/data_pipeline/files/beta/cejst-technical-support-document.pdf diff --git a/data/data-pipeline/data_pipeline/files/draft-communities-list.pdf b/data/data-pipeline/data_pipeline/files/beta/draft-communities-list.pdf similarity index 100% rename from data/data-pipeline/data_pipeline/files/draft-communities-list.pdf rename to data/data-pipeline/data_pipeline/files/beta/draft-communities-list.pdf diff --git a/data/data-pipeline/data_pipeline/files/readme-version-1.0.md b/data/data-pipeline/data_pipeline/files/beta/readme-version-1.0.md similarity index 100% rename from data/data-pipeline/data_pipeline/files/readme-version-1.0.md rename to data/data-pipeline/data_pipeline/files/beta/readme-version-1.0.md diff --git a/data/data-pipeline/data_pipeline/files/readme-version-beta.md b/data/data-pipeline/data_pipeline/files/beta/readme-version-beta.md similarity index 100% rename from data/data-pipeline/data_pipeline/files/readme-version-beta.md rename to data/data-pipeline/data_pipeline/files/beta/readme-version-beta.md diff --git a/data/data-pipeline/data_pipeline/files/technical-training-slides.pptx b/data/data-pipeline/data_pipeline/files/beta/technical-training-slides.pptx similarity index 100% rename from data/data-pipeline/data_pipeline/files/technical-training-slides.pptx rename to data/data-pipeline/data_pipeline/files/beta/technical-training-slides.pptx diff --git a/data/data-pipeline/data_pipeline/files/test/test.md b/data/data-pipeline/data_pipeline/files/test/test.md new file mode 100644 index 00000000..03a5ef48 --- /dev/null +++ b/data/data-pipeline/data_pipeline/files/test/test.md @@ -0,0 +1,3 @@ +# Test + +This file exists as a test to ensure that the file(s) in this folder are deployed to AWS during the deployment process. \ No newline at end of file