Score versioning (#1986)

* Score versioning

* adding new zip assets

* tests passing

* debugging

* debugging 2

* get codebook from downloadable path

* upload version file for codebook and shapefile

.github/workflows/deploy_be_staging.yml

@@ -56,7 +56,6 @@ jobs:
       - name: Deploy Score to Geoplatform AWS
         run: |
           poetry run s4cmd put ./data_pipeline/data/score/csv/ s3://justice40-data/data-pipeline-staging/${{env.PR_NUMBER}}/${{env.SHA_NUMBER}}/data/score/csv --recursive --force --API-ACL=public-read
-          poetry run s4cmd put ./data_pipeline/data/score/downloadable/ s3://justice40-data/data-pipeline-staging/${{env.PR_NUMBER}}/${{env.SHA_NUMBER}}/data/score/downloadable --recursive --force --API-ACL=public-read
           poetry run s4cmd put ./data_pipeline/files/ s3://justice40-data/data-pipeline-staging/${{env.PR_NUMBER}}/${{env.SHA_NUMBER}}/data/score/downloadable --recursive --force --API-ACL=public-read
 
       - name: Update PR with deployed Score URLs
@@ -105,6 +104,7 @@ jobs:
           poetry run s4cmd put ./data_pipeline/data/score/geojson/ s3://justice40-data/data-pipeline-staging/${{env.PR_NUMBER}}/${{env.SHA_NUMBER}}/data/score/geojson --recursive --force  --API-ACL=public-read --num-threads=250
           poetry run s4cmd put ./data_pipeline/data/score/shapefile/ s3://justice40-data/data-pipeline-staging/${{env.PR_NUMBER}}/${{env.SHA_NUMBER}}/data/score/shapefile --recursive --force  --API-ACL=public-read
           poetry run s4cmd put ./data_pipeline/data/score/tiles/ s3://justice40-data/data-pipeline-staging/${{env.PR_NUMBER}}/${{env.SHA_NUMBER}}/data/score/tiles --recursive --force  --API-ACL=public-read --num-threads=250
+          poetry run s4cmd put ./data_pipeline/data/score/downloadable/ s3://justice40-data/data-pipeline-staging/${{env.PR_NUMBER}}/${{env.SHA_NUMBER}}/data/score/downloadable --recursive --force --API-ACL=public-read
       - name: Update PR with deployed Map URL
         uses: mshick/add-pr-comment@v1
         with:

client/.env.development

@@ -15,7 +15,7 @@ GATSBY_FILE_DL_PATH_TSD_PDF=downloadable/cejst_technical_support_document.pdf
 GATSBY_FILE_DL_PATH_TSD_ES_PDF=downloadable/cejst-technical-support-document-es.pdf
 GATSBY_FILE_DL_PATH_COMMUNITIES_LIST_XLS=downloadable/communities-2022-05-12-1914GMT.xlsx
 GATSBY_FILE_DL_PATH_COMMUNITIES_LIST_CSV=downloadable/communities-2022-05-12-1914GMT.csv
-GATSBY_FILE_DL_PATH_HOW_TO_COMMUNITIES_PDF=downloadable/draft_communities_list.pdf
+GATSBY_FILE_DL_PATH_HOW_TO_COMMUNITIES_PDF=downloadable/draft-communities-list.pdf
 
 GATSBY_MAP_TILES_PATH=tiles
 

client/.env.production

@@ -13,6 +13,6 @@ GATSBY_FILE_DL_PATH_TSD_PDF=downloadable/cejst_technical_support_document.pdf
 GATSBY_FILE_DL_PATH_TSD_ES_PDF=downloadable/cejst-technical-support-document-es.pdf
 GATSBY_FILE_DL_PATH_COMMUNITIES_LIST_XLS=downloadable/communities-2022-05-31-1915GMT.xlsx
 GATSBY_FILE_DL_PATH_COMMUNITIES_LIST_CSV=downloadable/communities-2022-05-31-1915GMT.csv
-GATSBY_FILE_DL_PATH_HOW_TO_COMMUNITIES_PDF=downloadable/Draft_Communities_List.pdf
+GATSBY_FILE_DL_PATH_HOW_TO_COMMUNITIES_PDF=downloadable/draft-communities-list.pdf
 
 GATSBY_MAP_TILES_PATH=tiles

data/data-pipeline/data_pipeline/etl/score/constants.py

@@ -57,8 +57,10 @@ else:
     version_str = os.environ.get("J40_VERSION_LABEL_STRING")
 
 SCORE_DOWNLOADABLE_DIR = DATA_SCORE_DIR / "downloadable"
-SCORE_DOWNLOADABLE_PDF_FILE_NAME = "draft_communities_list.pdf"
+SCORE_DOWNLOADABLE_PDF_FILE_NAME = "draft-communities-list.pdf"
 SCORE_DOWNLOADABLE_PDF_FILE_PATH = FILES_PATH / SCORE_DOWNLOADABLE_PDF_FILE_NAME
+SCORE_DOWNLOADABLE_TSD_FILE_NAME = "cejst-technical-support-document.pdf"
+SCORE_DOWNLOADABLE_TSD_FILE_PATH = FILES_PATH / SCORE_DOWNLOADABLE_TSD_FILE_NAME
 SCORE_DOWNLOADABLE_CSV_FILE_PATH = (
     SCORE_DOWNLOADABLE_DIR / f"{version_str}-communities-{timestamp_str}.csv"
 )
@@ -76,6 +78,18 @@ SCORE_DOWNLOADABLE_XLS_ZIP_FILE_PATH = (
     SCORE_DOWNLOADABLE_DIR
     / f"{version_str}-communities-xls-{timestamp_str}.zip"
 )
+SCORE_VERSIONING_DATA_DOCUMENTATION_ZIP_FILE_PATH = (
+    SCORE_DOWNLOADABLE_DIR
+    / f"{version_str}-data-documentation-{timestamp_str}.zip"
+)
+SCORE_VERSIONING_SHAPEFILE_CODEBOOK_FILE_PATH = (
+    SCORE_DOWNLOADABLE_DIR
+    / f"{version_str}-shapefile-codebook-{timestamp_str}.zip"
+)
+SCORE_VERSIONING_README_FILE_NAME = f"README-version-{version_str}.md"
+SCORE_VERSIONING_README_FILE_PATH = (
+    FILES_PATH / SCORE_VERSIONING_README_FILE_NAME
+)
 
 # For the codebook
 CEJST_SCORE_COLUMN_NAME = "score_name"

data/data-pipeline/data_pipeline/etl/score/etl_score_geo.py

@@ -368,6 +368,37 @@ class GeoScoreETL(ExtractTransformLoad):
             zip_files(arcgis_zip_file_path, arcgis_files)
             logger.info("Completed zipping shapefiles")
 
+            # Per #1557:
+            # Zip file that contains the shapefiles, codebook and checksum file.
+            # Normally we get the codebook file path using this constant:
+            # - codebook_path = constants.SCORE_DOWNLOADABLE_CODEBOOK_FILE_PATH
+            # However since we generate it on a separate script (etl_score_post)
+            # the time stamp can be generated again, and thus the file is not found.
+            # So we grab it from the downloadable dir and if we don't find it, it
+            # means we haven't run etl_score_post, and continue
+
+            logger.info("Getting codebook from downloadable dir")
+            codebook_path = None
+            for file in os.listdir(constants.SCORE_DOWNLOADABLE_DIR):
+                if "codebook" in file:
+                    codebook_path = constants.SCORE_DOWNLOADABLE_DIR / file
+
+            if codebook_path:
+                version_shapefile_codebook_zip_path = (
+                    constants.SCORE_VERSIONING_SHAPEFILE_CODEBOOK_FILE_PATH
+                )
+                readme_path = constants.SCORE_VERSIONING_README_FILE_PATH
+
+                logger.info("Compressing shapefile and codebook files")
+                files_to_compress = [
+                    arcgis_zip_file_path,
+                    codebook_path,
+                    readme_path,
+                ]
+                zip_files(
+                    version_shapefile_codebook_zip_path, files_to_compress
+                )
+
         with concurrent.futures.ThreadPoolExecutor() as executor:
             futures = {
                 executor.submit(task)

data/data-pipeline/data_pipeline/etl/score/etl_score_post.py

@@ -527,8 +527,18 @@ class PostScoreETL(ExtractTransformLoad):
         csv_path = constants.SCORE_DOWNLOADABLE_CSV_FILE_PATH
         excel_path = constants.SCORE_DOWNLOADABLE_EXCEL_FILE_PATH
         codebook_path = constants.SCORE_DOWNLOADABLE_CODEBOOK_FILE_PATH
+        readme_path = constants.SCORE_VERSIONING_README_FILE_PATH
         csv_zip_path = constants.SCORE_DOWNLOADABLE_CSV_ZIP_FILE_PATH
         xls_zip_path = constants.SCORE_DOWNLOADABLE_XLS_ZIP_FILE_PATH
+        score_downloadable_pdf_file_path = (
+            constants.SCORE_DOWNLOADABLE_PDF_FILE_PATH
+        )
+        score_downloadable_tsd_file_path = (
+            constants.SCORE_DOWNLOADABLE_TSD_FILE_PATH
+        )
+        version_data_documentation_zip_path = (
+            constants.SCORE_VERSIONING_DATA_DOCUMENTATION_ZIP_FILE_PATH
+        )
 
         logger.info("Writing downloadable excel")
         excel_config = self._load_excel_from_df(
@@ -577,19 +587,26 @@ class PostScoreETL(ExtractTransformLoad):
         # load codebook to disk
         codebook_df.to_csv(codebook_path, index=False)
 
+        # zip assets
         logger.info("Compressing csv files")
-        files_to_compress = [
-            csv_path,
-            codebook_path,
-        ]
+        files_to_compress = [csv_path, codebook_path, readme_path]
         zip_files(csv_zip_path, files_to_compress)
 
         logger.info("Compressing xls files")
+        files_to_compress = [excel_path, codebook_path, readme_path]
+        zip_files(xls_zip_path, files_to_compress)
+
+        # Per #1557
+        # zip file that contains the .xls, .csv, .pdf, tech support document, checksum file
+        logger.info("Compressing data and documentation files")
         files_to_compress = [
             excel_path,
-            codebook_path,
+            csv_path,
+            score_downloadable_pdf_file_path,
+            score_downloadable_tsd_file_path,
+            readme_path,
         ]
-        zip_files(xls_zip_path, files_to_compress)
+        zip_files(version_data_documentation_zip_path, files_to_compress)
 
     def load(self) -> None:
         self._load_score_csv_full(

data/data-pipeline/data_pipeline/etl/score/tests/conftest.py

@@ -73,7 +73,7 @@ def score_data_initial(sample_data_dir):
 
 @pytest.fixture()
 def score_pdf_initial(sample_data_dir):
-    return sample_data_dir / "draft_communities_list.pdf"
+    return sample_data_dir / "draft-communities-list.pdf"
 
 
 @pytest.fixture()

data/data-pipeline/data_pipeline/etl/score/tests/test_score_post.py

@@ -138,6 +138,16 @@ def test_load_downloadable_zip(etl, monkeypatch, score_data_expected):
         "SCORE_DOWNLOADABLE_PDF_FILE_PATH",
         static_files_path / constants.SCORE_DOWNLOADABLE_PDF_FILE_NAME,
     )
+    monkeypatch.setattr(
+        constants,
+        "SCORE_VERSIONING_README_FILE_PATH",
+        static_files_path / constants.SCORE_VERSIONING_README_FILE_NAME,
+    )
+    monkeypatch.setattr(
+        constants,
+        "SCORE_DOWNLOADABLE_TSD_FILE_PATH",
+        static_files_path / constants.SCORE_DOWNLOADABLE_TSD_FILE_NAME,
+    )
     etl.output_score_county_state_merged_df = score_data_expected
     etl._load_downloadable_zip(constants.SCORE_DOWNLOADABLE_DIR)
     assert constants.SCORE_DOWNLOADABLE_DIR.is_dir()

data/data-pipeline/data_pipeline/files/README-version-beta.md

@@ -0,0 +1,104 @@
+Draft, Deliberative and Pre-Decisional
+
+**Release Notes for the Climate and Economic Justice Screening Tool (CEJST) version 1.0**
+
+**Changes between version 1.0 and the beta version**
+
+**Release update - DATE**
+
+- Added Federally Recognized Tribal Lands and Alaska Native Villages data from the Bureau of Indian Affairs at the Department of the Interior to the tool and map
+- Added new data for indicators of burden
+  - Climate change
+    - Projected flood risk
+    - Projected wildfire risk
+  - Housing
+    - Lack of plumbing
+    - Lack of green space
+    - Historic underinvestment (due to redlining)
+  - Legacy pollution
+    - Abandoned mine lands
+    - Formerly used defense sites
+  - Transportation
+    - Transportation barriers
+  - Water
+    - Leaking underground storage tanks
+- Added an adjacency indicator to methodology to include low income communities that had been completely surrounded by other disadvantaged communities, but which had just missed the thresholds in the beta version.
+- Made technical changes to enhance accuracy of the tool
+  - Removed income data for students enrolled in higher education in the low income indicator
+  - Imputed income for census tracts missing that data that have a population greater than zero
+  - Used transportation barriers and population loss data indicators only for census tracts with populations of 20 or more
+
+- Improved the user interface
+  - Added demographics (race & age) for census tracts to map side panel
+  - Improved the design of the map side panel
+  - Updated the site copy of the website and associated Spanish translations
+
+- Added new data for all the U.S. Territories, and also made a small methodology change for Puerto Rico
+  - USVI
+    - Included data from 2010 decennial census for US Virgin Islands
+    - New data from EJScreen 2.1
+      - Sustainable Housing:
+        - Lead paint
+      - Legacy pollution:
+        - Proximity to Superfund (National Priorities List (NPL)) sites
+        - Proximity to Risk Management Plan (RMP) facilities
+        - Proximity to hazardous waste facilities
+        - Leaking underground storage tanks
+  - Guam
+    - Included data from 2010 decennial census for Guam
+    - New data from EJScreen 2.1
+      - Sustainable Housing:
+        - Lead paint
+      - Legacy pollution:
+        - Proximity to Superfund (National Priorities List (NPL)) sites
+        - Proximity to Risk Management Plan (RMP) facilities
+        - Proximity to hazardous waste facilities
+        - Leaking underground storage tanks
+  - Samoa
+    - New data from EJScreen 2.1
+      - Sustainable Housing:
+        - Lead paint
+      - Legacy pollution:
+        - Proximity to Risk Management Plan (RMP) facilities
+        - Leaking underground storage tanks
+  - Marianna Islands
+    - New data from EJScreen 2.1
+      - Sustainable Housing:
+        - Lead paint
+      - Legacy pollution:
+        - Leaking underground storage tanks
+  - Puerto Rico
+    - Removed linguistic isolation as an indicator in the methodology for Puerto Rico
+    - Imported additional available data for Puerto Rico
+      - Energy cost
+      - Housing cost
+      - Abandoned mine lands
+      - Proximity to hazardous waste sites
+      - Proximity to Superfund (National Priorities List (NPL)) sites
+      - Proximity to Risk Management Plan (RMP) sites
+- Updated data from EJScreen 2.1 across the entire tool:
+  - Sustainable Housing:
+    - Lead paint - 2016-2020
+  - Legacy pollution:
+    - Proximity to Superfund (National Priorities List (NPL)) sites - 2022
+    - Proximity to Risk Management Plans (RMP) facilities - 2022
+    - Proximity to hazardous waste facilities - 2022
+  - Water and wastewater
+    - Leaking underground storage tanks - 2022
+
+- Enhanced the technical files:
+  - Added all new data indicators and demographics to .xls, .csv, and shapefiles
+  - Added versions page to house beta version of download files
+  - Updated codebook
+  - Updated Technical Support Document
+
+- Improved the way that users provide feedback on the tool:
+  - Data survey
+  - Site experience survey
+  - Census tract feedback
+  - General contact form
+- Added a link to sign up for email mailing list managed by CEQ
+- Fixes:
+
+  - Bug fix: loaded missing life expectancy data for Maine and Wisconsin
+  - Bug fix: Census tracts that have 0 land should not be included on the map