Update score version workflow to also upload tiles (#2042)

* Actually upload the score * Add back tile generation (#2035) * Move score GEO up (#2035) * Copy beta README to 1.0 (#2035) * Remove dates from file names (#2035) * Remove unused score ETLs from constants (#2035) * Put back child opp index (#2035) * Remove unused import (#2035)
2025-02-22 01:31:25 -08:00 · 2022-10-28 10:53:19 -04:00 · 2022-10-28 10:53:19 -04:00 · 0d0cf2a5f9
commit 0d0cf2a5f9
parent d7a0be7ee7
4 changed files with 146 additions and 63 deletions
--- a/.github/workflows/create-score-version.yml
+++ b/.github/workflows/create-score-version.yml
@ -10,15 +10,10 @@ on:
        options:
        - beta
        - 1.0
-      score_date:
-        description: "What is today's date, in YYYY-MM-DD format?"
-        required: true
-        type: string

 env:
  CENSUS_API_KEY: ${{ secrets.CENSUS_API_KEY }}
  J40_VERSION_LABEL_STRING: ${{ inputs.score_version }}
-  J40_VERSION_DATE_STRING: ${{ inputs.score_date }}

 jobs:
  generate-score-tiles:
@ -57,49 +52,49 @@ jobs:
          aws-access-key-id: ${{ secrets.DATA_DEV_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.DATA_DEV_AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-1
+      - name: Install GDAL/ogr2ogr
+        run: |
+          sudo add-apt-repository ppa:ubuntugis/ppa
+          sudo apt-get update
+          sudo apt-get -y install gdal-bin
+          ogrinfo --version
      - name: Generate Score
        run: |
          poetry run python3 data_pipeline/application.py score-full-run
      - name: Generate Score Post
        run: |
          poetry run python3 data_pipeline/application.py generate-score-post -s aws
+      - name: Generate Score Geo
+        run: |
+          poetry run python3 data_pipeline/application.py geo-score
      - name: Deploy Score to Geoplatform AWS
        run: |
          poetry run s4cmd put ./data_pipeline/data/score/csv/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/csv --recursive --force --API-ACL=public-read
          poetry run s4cmd put ./data_pipeline/files/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read
+          poetry run s4cmd put ./data_pipeline/data/score/downloadable/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read

-      # - name: Install GDAL/ogr2ogr
-      #   run: |
-      #     sudo add-apt-repository ppa:ubuntugis/ppa
-      #     sudo apt-get update
-      #     sudo apt-get -y install gdal-bin
-      #     ogrinfo --version
-      # - name: Set timezone for tippecanoe
-      #   uses: szenius/set-timezone@v1.0
-      #   with:
-      #     timezoneLinux: "America/Los_Angeles"
-      # - name: Get tippecanoe
-      #   run: |
-      #     sudo apt-get install -y software-properties-common libsqlite3-dev zlib1g-dev
-      #     sudo apt-add-repository -y ppa:git-core/ppa
-      #     sudo mkdir -p /tmp/tippecanoe-src
-      #     sudo git clone https://github.com/mapbox/tippecanoe.git /tmp/tippecanoe-src
-      # - name: Make tippecanoe
-      #   working-directory: /tmp/tippecanoe-src
-      #   run: |
-      #     sudo /usr/bin/bash -c make
-      #     mkdir -p /usr/local/bin
-      #     cp tippecanoe /usr/local/bin/tippecanoe
-      #     tippecanoe -v
-      # - name: Generate Score Geo
-      #   run: |
-      #     poetry run python3 data_pipeline/application.py geo-score
-      # - name: Generate Tiles
-      #   run: |
-      #     poetry run python3 data_pipeline/application.py generate-map-tiles
-      # - name: Deploy Map to Geoplatform AWS
-      #   run: |
-      #     poetry run s4cmd put ./data_pipeline/data/score/geojson/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/data/score/geojson --recursive --force  --API-ACL=public-read --num-threads=250
-      #     poetry run s4cmd put ./data_pipeline/data/score/shapefile/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/${{env.SHA_NUMBER}}/data/score/shapefile --recursive --force  --API-ACL=public-read
-      #     poetry run s4cmd put ./data_pipeline/data/score/tiles/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/data/score/tiles --recursive --force  --API-ACL=public-read --num-threads=250
-      #     poetry run s4cmd put ./data_pipeline/data/score/downloadable/ s3://justice40-data/data-versions/${{J40_VERSION_LABEL_STRING}}/data/score/downloadable --recursive --force --API-ACL=public-read
+      - name: Set timezone for tippecanoe
+        uses: szenius/set-timezone@v1.0
+        with:
+          timezoneLinux: "America/Los_Angeles"
+      - name: Get tippecanoe
+        run: |
+          sudo apt-get install -y software-properties-common libsqlite3-dev zlib1g-dev
+          sudo apt-add-repository -y ppa:git-core/ppa
+          sudo mkdir -p /tmp/tippecanoe-src
+          sudo git clone https://github.com/mapbox/tippecanoe.git /tmp/tippecanoe-src
+      - name: Make tippecanoe
+        working-directory: /tmp/tippecanoe-src
+        run: |
+          sudo /usr/bin/bash -c make
+          mkdir -p /usr/local/bin
+          cp tippecanoe /usr/local/bin/tippecanoe
+          tippecanoe -v
+      - name: Generate Tiles
+        run: |
+          poetry run python3 data_pipeline/application.py generate-map-tiles
+      - name: Deploy Map to Geoplatform AWS
+        run: |
+          poetry run s4cmd put ./data_pipeline/data/score/geojson/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/geojson --recursive --force  --API-ACL=public-read --num-threads=250
+          poetry run s4cmd put ./data_pipeline/data/score/shapefile/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/${{env.SHA_NUMBER}}/data/score/shapefile --recursive --force  --API-ACL=public-read
+          poetry run s4cmd put ./data_pipeline/data/score/tiles/ s3://justice40-data/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/tiles --recursive --force  --API-ACL=public-read --num-threads=250
--- a/data/data-pipeline/data_pipeline/etl/constants.py
+++ b/data/data-pipeline/data_pipeline/etl/constants.py
@ -29,11 +29,6 @@ DATASET_LIST = [
        "module_dir": "census_decennial",
        "class_name": "CensusDecennialETL",
    },
-    {
-        "name": "housing_and_transportation",
-        "module_dir": "housing_and_transportation",
-        "class_name": "HousingTransportationETL",
-    },
    {
        "name": "mapping_for_ej",
        "module_dir": "mapping_for_ej",
--- a/data/data-pipeline/data_pipeline/etl/score/constants.py
+++ b/data/data-pipeline/data_pipeline/etl/score/constants.py
@ -1,6 +1,5 @@
 import os
 from pathlib import Path
-import datetime

 from data_pipeline.config import settings

@ -45,12 +44,6 @@ DATA_SCORE_JSON_INDEX_FILE_PATH = (
 DATA_SCORE_TILES_DIR = DATA_SCORE_DIR / "tiles"

 # Downloadable paths
-if not os.environ.get("J40_VERSION_DATE_STRING"):
-    current_dt = datetime.datetime.now()
-    timestamp_str = current_dt.strftime("%Y-%m-%d-%H%MGMT")
-else:
-    timestamp_str = os.environ.get("J40_VERSION_DATE_STRING")
-
 if not os.environ.get("J40_VERSION_LABEL_STRING"):
    version_str = "beta"
 else:
@ -62,29 +55,25 @@ SCORE_DOWNLOADABLE_PDF_FILE_PATH = FILES_PATH / SCORE_DOWNLOADABLE_PDF_FILE_NAME
 SCORE_DOWNLOADABLE_TSD_FILE_NAME = "cejst-technical-support-document.pdf"
 SCORE_DOWNLOADABLE_TSD_FILE_PATH = FILES_PATH / SCORE_DOWNLOADABLE_TSD_FILE_NAME
 SCORE_DOWNLOADABLE_CSV_FILE_PATH = (
-    SCORE_DOWNLOADABLE_DIR / f"{version_str}-communities-{timestamp_str}.csv"
+    SCORE_DOWNLOADABLE_DIR / f"{version_str}-communities.csv"
 )
 SCORE_DOWNLOADABLE_EXCEL_FILE_PATH = (
-    SCORE_DOWNLOADABLE_DIR / f"{version_str}-communities-{timestamp_str}.xlsx"
+    SCORE_DOWNLOADABLE_DIR / f"{version_str}-communities.xlsx"
 )
 SCORE_DOWNLOADABLE_CODEBOOK_FILE_PATH = (
-    SCORE_DOWNLOADABLE_DIR / f"{version_str}-codebook-{timestamp_str}.csv"
+    SCORE_DOWNLOADABLE_DIR / f"{version_str}-codebook.csv"
 )
 SCORE_DOWNLOADABLE_CSV_ZIP_FILE_PATH = (
-    SCORE_DOWNLOADABLE_DIR
-    / f"{version_str}-communities-csv-{timestamp_str}.zip"
+    SCORE_DOWNLOADABLE_DIR / f"{version_str}-communities-csv.zip"
 )
 SCORE_DOWNLOADABLE_XLS_ZIP_FILE_PATH = (
-    SCORE_DOWNLOADABLE_DIR
-    / f"{version_str}-communities-xls-{timestamp_str}.zip"
+    SCORE_DOWNLOADABLE_DIR / f"{version_str}-communities-xls.zip"
 )
 SCORE_VERSIONING_DATA_DOCUMENTATION_ZIP_FILE_PATH = (
-    SCORE_DOWNLOADABLE_DIR
-    / f"{version_str}-data-documentation-{timestamp_str}.zip"
+    SCORE_DOWNLOADABLE_DIR / f"{version_str}-data-documentation.zip"
 )
 SCORE_VERSIONING_SHAPEFILE_CODEBOOK_FILE_PATH = (
-    SCORE_DOWNLOADABLE_DIR
-    / f"{version_str}-shapefile-codebook-{timestamp_str}.zip"
+    SCORE_DOWNLOADABLE_DIR / f"{version_str}-shapefile-codebook.zip"
 )
 SCORE_VERSIONING_README_FILE_NAME = f"README-version-{version_str}.md"
 SCORE_VERSIONING_README_FILE_PATH = (
--- a/data/data-pipeline/data_pipeline/files/README-version-1.0.md
+++ b/data/data-pipeline/data_pipeline/files/README-version-1.0.md
@ -0,0 +1,104 @@
+Draft, Deliberative and Pre-Decisional
+
+**Release Notes for the Climate and Economic Justice Screening Tool (CEJST) version 1.0**
+
+**Changes between version 1.0 and the beta version**
+
+**Release update - DATE**
+
+- Added Federally Recognized Tribal Lands and Alaska Native Villages data from the Bureau of Indian Affairs at the Department of the Interior to the tool and map
+- Added new data for indicators of burden
+  - Climate change
+    - Projected flood risk
+    - Projected wildfire risk
+  - Housing
+    - Lack of plumbing
+    - Lack of green space
+    - Historic underinvestment (due to redlining)
+  - Legacy pollution
+    - Abandoned mine lands
+    - Formerly used defense sites
+  - Transportation
+    - Transportation barriers
+  - Water
+    - Leaking underground storage tanks
+- Added an adjacency indicator to methodology to include low income communities that had been completely surrounded by other disadvantaged communities, but which had just missed the thresholds in the beta version.
+- Made technical changes to enhance accuracy of the tool
+  - Removed income data for students enrolled in higher education in the low income indicator
+  - Imputed income for census tracts missing that data that have a population greater than zero
+  - Used transportation barriers and population loss data indicators only for census tracts with populations of 20 or more
+
+- Improved the user interface
+  - Added demographics (race & age) for census tracts to map side panel
+  - Improved the design of the map side panel
+  - Updated the site copy of the website and associated Spanish translations
+
+- Added new data for all the U.S. Territories, and also made a small methodology change for Puerto Rico
+  - USVI
+    - Included data from 2010 decennial census for US Virgin Islands
+    - New data from EJScreen 2.1
+      - Sustainable Housing:
+        - Lead paint
+      - Legacy pollution:
+        - Proximity to Superfund (National Priorities List (NPL)) sites
+        - Proximity to Risk Management Plan (RMP) facilities
+        - Proximity to hazardous waste facilities
+        - Leaking underground storage tanks
+  - Guam
+    - Included data from 2010 decennial census for Guam
+    - New data from EJScreen 2.1
+      - Sustainable Housing:
+        - Lead paint
+      - Legacy pollution:
+        - Proximity to Superfund (National Priorities List (NPL)) sites
+        - Proximity to Risk Management Plan (RMP) facilities
+        - Proximity to hazardous waste facilities
+        - Leaking underground storage tanks
+  - Samoa
+    - New data from EJScreen 2.1
+      - Sustainable Housing:
+        - Lead paint
+      - Legacy pollution:
+        - Proximity to Risk Management Plan (RMP) facilities
+        - Leaking underground storage tanks
+  - Marianna Islands
+    - New data from EJScreen 2.1
+      - Sustainable Housing:
+        - Lead paint
+      - Legacy pollution:
+        - Leaking underground storage tanks
+  - Puerto Rico
+    - Removed linguistic isolation as an indicator in the methodology for Puerto Rico
+    - Imported additional available data for Puerto Rico
+      - Energy cost
+      - Housing cost
+      - Abandoned mine lands
+      - Proximity to hazardous waste sites
+      - Proximity to Superfund (National Priorities List (NPL)) sites
+      - Proximity to Risk Management Plan (RMP) sites
+- Updated data from EJScreen 2.1 across the entire tool:
+  - Sustainable Housing:
+    - Lead paint - 2016-2020
+  - Legacy pollution:
+    - Proximity to Superfund (National Priorities List (NPL)) sites - 2022
+    - Proximity to Risk Management Plans (RMP) facilities - 2022
+    - Proximity to hazardous waste facilities - 2022
+  - Water and wastewater
+    - Leaking underground storage tanks - 2022
+
+- Enhanced the technical files:
+  - Added all new data indicators and demographics to .xls, .csv, and shapefiles
+  - Added versions page to house beta version of download files
+  - Updated codebook
+  - Updated Technical Support Document
+
+- Improved the way that users provide feedback on the tool:
+  - Data survey
+  - Site experience survey
+  - Census tract feedback
+  - General contact form
+- Added a link to sign up for email mailing list managed by CEQ
+- Fixes:
+
+  - Bug fix: loaded missing life expectancy data for Maine and Wisconsin
+  - Bug fix: Census tracts that have 0 land should not be included on the map