diff --git a/.github/workflows/create-score-version.yml b/.github/workflows/create-score-version.yml index adf368f9..464ee58a 100644 --- a/.github/workflows/create-score-version.yml +++ b/.github/workflows/create-score-version.yml @@ -109,24 +109,35 @@ jobs: - name: Deploy Score to Geoplatform AWS run: | poetry run s4cmd put ./data_pipeline/data/score/csv/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/csv --recursive --force --API-ACL=public-read - poetry run s4cmd put ./data_pipeline/files/${{env.J40_VERSION_LABEL_STRING}}/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read + poetry run s4cmd put ./data_pipeline/files/static/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read poetry run s4cmd put ./data_pipeline/data/score/downloadable/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read - name: Deploy Map to Geoplatform AWS run: | poetry run s4cmd put ./data_pipeline/data/score/geojson/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/geojson --recursive --force --API-ACL=public-read --num-threads=250 poetry run s4cmd put ./data_pipeline/data/score/shapefile/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/${{env.SHA_NUMBER}}/data/score/shapefile --recursive --force --API-ACL=public-read poetry run s4cmd put ./data_pipeline/data/score/tiles/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/tiles --recursive --force --API-ACL=public-read --num-threads=250 - + - name: Deploy Test Files to Geoplatform AWS + if: ${{ env.J40_VERSION_LABEL_STRING == 'test' }} + run: | + poetry run s4cmd put ./data_pipeline/files/test/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read + # Run more smoke tests to ensure we deployed everything correctly - name: Confirm we deployed downloadables to AWS - if: ${{ env.J40_VERSION_LABEL_STRING != 'beta' }} + if: ${{ env.J40_VERSION_LABEL_STRING != 'beta' && env.J40_VERSION_LABEL_STRING != 'test' }} run: | - curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/1.0-shapefile-codebook.zip" -s -f -I -o /dev/null && \ - curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/1.0-communities.xlsx" -s -f -I -o /dev/null && \ - curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/1.0-communities.csv" -s -f -I -o /dev/null && \ - curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/1.0-shapefile-codebook.zip" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/${{env.J40_VERSION_LABEL_STRING}}-shapefile-codebook.zip" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/${{env.J40_VERSION_LABEL_STRING}}-communities.xlsx" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/${{env.J40_VERSION_LABEL_STRING}}-communities.csv" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/${{env.J40_VERSION_LABEL_STRING}}-communities-xls.zip" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/${{env.J40_VERSION_LABEL_STRING}}-communities-csv.zip" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/${{env.J40_VERSION_LABEL_STRING}}-codebook.csv" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/${{env.J40_VERSION_LABEL_STRING}}-data-documentation.zip" -s -f -I -o /dev/null && \ curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/cejst-technical-support-document.pdf" -s -f -I -o /dev/null && \ - curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/draft-communities-list.pdf" -s -f -I -o /dev/null + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/CEQ-CEJST-Instructions.pdf" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/climate-and-environmental-burden-tool-comparison.pdf" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/communities-list.pdf" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/readme.md" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/technical-training-slides.pptx" -s -f -I -o /dev/null - name: Confirm we deployed downloadables to AWS (beta) if: ${{ env.J40_VERSION_LABEL_STRING == 'beta' }} run: | @@ -135,5 +146,11 @@ jobs: - name: Confirm we deployed additional downloadables – from the files directory – to AWS (test) if: ${{ env.J40_VERSION_LABEL_STRING == 'test' }} run: | - curl "https://static-data-screeningtool.geoplatform.gov/data-versions/beta/data/score/downloadable/test.md" -s -f -I -o /dev/null + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/test.md" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/cejst-technical-support-document.pdf" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/CEQ-CEJST-Instructions.pdf" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/climate-and-environmental-burden-tool-comparison.pdf" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/communities-list.pdf" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/readme.md" -s -f -I -o /dev/null && \ + curl "https://static-data-screeningtool.geoplatform.gov/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable/technical-training-slides.pptx" -s -f -I -o /dev/null \ No newline at end of file diff --git a/data/data-pipeline/data_pipeline/etl/score/constants.py b/data/data-pipeline/data_pipeline/etl/score/constants.py index ea63f854..035d71a2 100644 --- a/data/data-pipeline/data_pipeline/etl/score/constants.py +++ b/data/data-pipeline/data_pipeline/etl/score/constants.py @@ -10,7 +10,7 @@ from data_pipeline.score import field_names # Base Paths DATA_PATH = Path(settings.APP_ROOT) / "data" TMP_PATH = DATA_PATH / "tmp" -FILES_PATH = Path(settings.APP_ROOT) / "files" +FILES_PATH = Path(settings.APP_ROOT) / "files" / "static" # Remote Paths CENSUS_COUNTIES_ZIP_URL = "https://www2.census.gov/geo/docs/maps-data/data/gazetteer/Gaz_counties_national.zip" @@ -52,7 +52,7 @@ else: version_str = os.environ.get("J40_VERSION_LABEL_STRING") SCORE_DOWNLOADABLE_DIR = DATA_SCORE_DIR / "downloadable" -SCORE_DOWNLOADABLE_PDF_FILE_NAME = "draft-communities-list.pdf" +SCORE_DOWNLOADABLE_PDF_FILE_NAME = "communities-list.pdf" SCORE_DOWNLOADABLE_PDF_FILE_PATH = FILES_PATH / SCORE_DOWNLOADABLE_PDF_FILE_NAME SCORE_DOWNLOADABLE_TSD_FILE_NAME = "cejst-technical-support-document.pdf" SCORE_DOWNLOADABLE_TSD_FILE_PATH = FILES_PATH / SCORE_DOWNLOADABLE_TSD_FILE_NAME @@ -77,7 +77,7 @@ SCORE_VERSIONING_DATA_DOCUMENTATION_ZIP_FILE_PATH = ( SCORE_VERSIONING_SHAPEFILE_CODEBOOK_FILE_PATH = ( SCORE_DOWNLOADABLE_DIR / f"{version_str}-shapefile-codebook.zip" ) -SCORE_VERSIONING_README_FILE_NAME = f"readme-version-{version_str}.md" +SCORE_VERSIONING_README_FILE_NAME = "readme.md" SCORE_VERSIONING_README_FILE_PATH = ( FILES_PATH / SCORE_VERSIONING_README_FILE_NAME ) diff --git a/data/data-pipeline/data_pipeline/etl/score/tests/test_score_post.py b/data/data-pipeline/data_pipeline/etl/score/tests/test_score_post.py index 7d8291f4..8823e10a 100644 --- a/data/data-pipeline/data_pipeline/etl/score/tests/test_score_post.py +++ b/data/data-pipeline/data_pipeline/etl/score/tests/test_score_post.py @@ -125,7 +125,7 @@ def test_load_tile_csv(etl, tile_data_expected): def test_load_downloadable_zip(etl, monkeypatch, score_data_expected): reload(constants) static_files_path = ( - Path.cwd() / "data_pipeline" / "files" / "beta" + Path.cwd() / "data_pipeline" / "files" / "static" ) # need to monkeypatch to real dir monkeypatch.setattr(constants, "FILES_PATH", static_files_path) monkeypatch.setattr( diff --git a/data/data-pipeline/data_pipeline/files/beta/cejst-technical-support-document-es.pdf b/data/data-pipeline/data_pipeline/files/beta/cejst-technical-support-document-es.pdf deleted file mode 100644 index 92ff73fb..00000000 Binary files a/data/data-pipeline/data_pipeline/files/beta/cejst-technical-support-document-es.pdf and /dev/null differ diff --git a/data/data-pipeline/data_pipeline/files/beta/cejst-technical-support-document.pdf b/data/data-pipeline/data_pipeline/files/beta/cejst-technical-support-document.pdf deleted file mode 100644 index 71294948..00000000 Binary files a/data/data-pipeline/data_pipeline/files/beta/cejst-technical-support-document.pdf and /dev/null differ diff --git a/data/data-pipeline/data_pipeline/files/beta/draft-communities-list.pdf b/data/data-pipeline/data_pipeline/files/beta/draft-communities-list.pdf deleted file mode 100644 index 8989fc81..00000000 Binary files a/data/data-pipeline/data_pipeline/files/beta/draft-communities-list.pdf and /dev/null differ diff --git a/data/data-pipeline/data_pipeline/files/beta/readme-version-1.0.md b/data/data-pipeline/data_pipeline/files/beta/readme-version-1.0.md deleted file mode 100644 index e69de29b..00000000 diff --git a/data/data-pipeline/data_pipeline/files/beta/readme-version-beta.md b/data/data-pipeline/data_pipeline/files/beta/readme-version-beta.md deleted file mode 100644 index d0dbec53..00000000 --- a/data/data-pipeline/data_pipeline/files/beta/readme-version-beta.md +++ /dev/null @@ -1,3 +0,0 @@ -# README - -This is the beta version of the CEJST tool \ No newline at end of file diff --git a/data/data-pipeline/data_pipeline/files/static/CEQ-CEJST-Instructions.pdf b/data/data-pipeline/data_pipeline/files/static/CEQ-CEJST-Instructions.pdf new file mode 100644 index 00000000..d3dd3e89 Binary files /dev/null and b/data/data-pipeline/data_pipeline/files/static/CEQ-CEJST-Instructions.pdf differ diff --git a/data/data-pipeline/data_pipeline/files/static/cejst-technical-support-document.pdf b/data/data-pipeline/data_pipeline/files/static/cejst-technical-support-document.pdf new file mode 100644 index 00000000..a5eb54f2 Binary files /dev/null and b/data/data-pipeline/data_pipeline/files/static/cejst-technical-support-document.pdf differ diff --git a/data/data-pipeline/data_pipeline/files/static/climate-and-environmental-burden-tool-comparison.pdf b/data/data-pipeline/data_pipeline/files/static/climate-and-environmental-burden-tool-comparison.pdf new file mode 100644 index 00000000..08c1f353 Binary files /dev/null and b/data/data-pipeline/data_pipeline/files/static/climate-and-environmental-burden-tool-comparison.pdf differ diff --git a/data/data-pipeline/data_pipeline/files/static/communities-list.pdf b/data/data-pipeline/data_pipeline/files/static/communities-list.pdf new file mode 100644 index 00000000..4fa20ad1 Binary files /dev/null and b/data/data-pipeline/data_pipeline/files/static/communities-list.pdf differ diff --git a/data/data-pipeline/data_pipeline/files/static/readme.md b/data/data-pipeline/data_pipeline/files/static/readme.md new file mode 100644 index 00000000..ea94e38e --- /dev/null +++ b/data/data-pipeline/data_pipeline/files/static/readme.md @@ -0,0 +1,3 @@ +# README + +This is the 1.0 version of the CEJST tool \ No newline at end of file diff --git a/data/data-pipeline/data_pipeline/files/beta/technical-training-slides.pptx b/data/data-pipeline/data_pipeline/files/static/technical-training-slides.pptx similarity index 100% rename from data/data-pipeline/data_pipeline/files/beta/technical-training-slides.pptx rename to data/data-pipeline/data_pipeline/files/static/technical-training-slides.pptx