From a38caf7b75b949643629924dc8c317d9b9e4c36c Mon Sep 17 00:00:00 2001
From: ericiwamoto <100735505+ericiwamoto@users.noreply.github.com>
Date: Thu, 5 Dec 2024 08:54:40 -0800
Subject: [PATCH 01/17] Add PR build checks, disable deploy concurrency

---
 .github/workflows/deploy_backend_main.yml  |  25 ++++-
 .github/workflows/deploy_frontend_main.yml |   3 +
 .github/workflows/pr_backend.yml           | 103 +++++++++++++++++++++
 .github/workflows/pr_frontend.yml          |  44 +++++++++
 4 files changed, 170 insertions(+), 5 deletions(-)
 create mode 100644 .github/workflows/pr_backend.yml
 create mode 100644 .github/workflows/pr_frontend.yml

diff --git a/.github/workflows/deploy_backend_main.yml b/.github/workflows/deploy_backend_main.yml
index 1a420687..a3be4576 100644
--- a/.github/workflows/deploy_backend_main.yml
+++ b/.github/workflows/deploy_backend_main.yml
@@ -4,6 +4,9 @@ on:
     branches: [main]
     paths:
       - "data/**"
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 env:
   CENSUS_API_KEY: ${{ secrets.CENSUS_API_KEY }}
   J40_VERSION_LABEL_STRING: ${{ vars.SCORE_VERSION }}
@@ -51,15 +54,27 @@ jobs:
           sudo apt-get update
           sudo apt-get -y install gdal-bin
           ogrinfo --version
+      - name: Cleanup Data
+        run: |
+          poetry run python3 -m data_pipeline.application data-cleanup
       - name: Get Census Data
         run: |
-          poetry run python3 data_pipeline/application.py pull-census-data -s aws
+          poetry run python3 -m data_pipeline.application census-data-download
+          - name: Extract Data Sources
+          run: |
+            poetry run python3 -m data_pipeline.application extract-data-sources
+      - name: Run ETL
+        run: |
+            poetry run python3 -m data_pipeline.application etl-run
       - name: Generate Score
         run: |
-          poetry run python3 data_pipeline/application.py score-full-run
+          poetry run python3 -m data_pipeline.application score-run
+      - name: Score Compare
+        run: |
+          poetry run python3 -m data_pipeline.comparator compare-score
       - name: Generate Score Post
         run: |
-          poetry run python3 data_pipeline/application.py generate-score-post -s aws
+          poetry run python3 -m data_pipeline.application generate-score-post
       - name: Confirm we generated the version of the score we think we did
         if: ${{ env.J40_VERSION_LABEL_STRING == '1.0' || env.J40_VERSION_LABEL_STRING == 'test' }}
         run: |
@@ -70,7 +85,7 @@ jobs:
           grep -v "Identified as disadvantaged due to tribal overlap" data_pipeline/data/score/downloadable/* > /dev/null
       - name: Generate Score Geo
         run: |
-          poetry run python3 data_pipeline/application.py geo-score
+          poetry run python3 -m data_pipeline.application geo-score
       - name: Run smoketest for 1.0
         if: ${{ env.J40_VERSION_LABEL_STRING == '1.0' }}
         run: |
@@ -116,7 +131,7 @@ jobs:
           tippecanoe -v
       - name: Generate Tiles
         run: |
-          poetry run python3 data_pipeline/application.py generate-map-tiles
+          poetry run python3 -m data_pipeline.application generate-map-tiles
       - name: Deploy Map to Geoplatform AWS
         run: |
           poetry run s4cmd put ./data_pipeline/data/score/geojson/ s3://${{secrets.S3_DATA_BUCKET}}/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/geojson --sync-check --recursive --force --delete-removed --num-threads=250
diff --git a/.github/workflows/deploy_frontend_main.yml b/.github/workflows/deploy_frontend_main.yml
index 662edeaa..c94a3c15 100644
--- a/.github/workflows/deploy_frontend_main.yml
+++ b/.github/workflows/deploy_frontend_main.yml
@@ -4,6 +4,9 @@ on:
     branches: [main]
     paths:
       - "client/**/*"
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 jobs:
   build:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/pr_backend.yml b/.github/workflows/pr_backend.yml
new file mode 100644
index 00000000..f7a13009
--- /dev/null
+++ b/.github/workflows/pr_backend.yml
@@ -0,0 +1,103 @@
+name: Pull Request Backend 
+on:
+  pull_request:
+    paths:
+      - "data/**"
+env:
+  CENSUS_API_KEY: ${{ secrets.CENSUS_API_KEY }}
+  J40_VERSION_LABEL_STRING: ${{ vars.SCORE_VERSION }}
+
+jobs:
+  generate-score-tiles:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: data/data-pipeline
+    strategy:
+      matrix:
+        python-version: ['3.10']
+    environment: Staging
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+      - name: Print variables to help debug
+        uses: hmarr/debug-action@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Load cached Poetry installation
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: env-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/deploy_backend_main.yml') }}
+      - name: Install poetry
+        uses: snok/install-poetry@v1
+      - name: Print Poetry settings
+        run: poetry show -v
+      - name: Install dependencies
+        run: poetry add s4cmd && poetry install
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+      - name: Install GDAL/ogr2ogr
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install gdal-bin
+          ogrinfo --version
+      - name: Cleanup Data
+        run: |
+          poetry run python3 -m data_pipeline.application data-cleanup
+      - name: Get Census Data
+        run: |
+          poetry run python3 -m data_pipeline.application census-data-download
+      - name: Extract Data Sources
+        run: |
+          poetry run python3 -m data_pipeline.application extract-data-sources
+      - name: Run ETL
+        run: |
+          poetry run python3 -m data_pipeline.application etl-run
+      - name: Generate Score
+        run: |
+          poetry run python3 -m data_pipeline.application score-run
+      - name: Score Compare
+        run: |
+          poetry run python3 -m data_pipeline.comparator compare-score
+      - name: Generate Score Post
+        run: |
+          poetry run python3 -m data_pipeline.application generate-score-post
+      - name: Confirm we generated the version of the score we think we did
+        if: ${{ env.J40_VERSION_LABEL_STRING == '1.0' || env.J40_VERSION_LABEL_STRING == 'test' }}
+        run: |
+          grep "Identified as disadvantaged due to tribal overlap" data_pipeline/data/score/downloadable/* > /dev/null
+      - name: Confirm we generated the version of the score we think we did
+        if: ${{ env.J40_VERSION_LABEL_STRING == '2.0' || env.J40_VERSION_LABEL_STRING == 'beta' }}
+        run: |
+          grep -v "Identified as disadvantaged due to tribal overlap" data_pipeline/data/score/downloadable/* > /dev/null
+      - name: Generate Score Geo
+        run: |
+          poetry run python3 -m data_pipeline.application geo-score
+      - name: Run smoketest for 1.0
+        if: ${{ env.J40_VERSION_LABEL_STRING == '1.0' }}
+        run: |
+          poetry run pytest data_pipeline/ -m smoketest
+      - name: Set timezone for tippecanoe
+        uses: szenius/set-timezone@v2.0
+        with:
+          timezoneLinux: "America/Los_Angeles"
+      - name: Get tippecanoe
+        run: |
+          sudo apt-get install -y software-properties-common libsqlite3-dev zlib1g-dev
+          sudo apt-add-repository -y ppa:git-core/ppa
+          sudo mkdir -p /tmp/tippecanoe-src
+          sudo git clone https://github.com/mapbox/tippecanoe.git /tmp/tippecanoe-src
+      - name: Make tippecanoe
+        working-directory: /tmp/tippecanoe-src
+        run: |
+          sudo /usr/bin/bash -c make
+          mkdir -p /usr/local/bin
+          cp tippecanoe /usr/local/bin/tippecanoe
+          tippecanoe -v
+      - name: Generate Tiles
+        run: |
+          poetry run python3 -m data_pipeline.application generate-map-tiles
+
diff --git a/.github/workflows/pr_frontend.yml b/.github/workflows/pr_frontend.yml
new file mode 100644
index 00000000..4b7fb3a0
--- /dev/null
+++ b/.github/workflows/pr_frontend.yml
@@ -0,0 +1,44 @@
+name: Pull Request Frontend
+on:
+  push:
+    branches: [main]
+    paths:
+      - "client/**/*"
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    environment: Staging
+    defaults:
+      run:
+        working-directory: client
+    strategy:
+      matrix:
+        node-version: [14.x]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Install
+        run: npm ci
+      - name: Build
+        run: npm run build --if-present
+        env:
+          # See the client readme for more info on environment variables:
+          # https://github.com/usds/justice40-tool/blob/main/client/README.md
+          DATA_SOURCE: cdn
+          # TODO: Update main URL when either is back up
+          SITE_URL: "${{ secrets.SITE_URL }}"
+          MAPBOX_STYLES_READ_TOKEN: "${{ secrets.MAPBOX_STYLES_READ_TOKEN }}"
+      - name: Get directory contents
+        run: ls -la public
+      - name: Lint
+        run: npm run lint
+        # Disabling for now due to jsonlint - TODO: put this back
+      # - name: License Check
+      #   run: npm run licenses
+      - name: Test
+        run: npm test
+      # - name: Check for security vulnerabilities
+      #   run: npm audit --production
\ No newline at end of file

From c864fdb792ac7e36f4e1dceb4c2938badc48dea7 Mon Sep 17 00:00:00 2001
From: ericiwamoto <100735505+ericiwamoto@users.noreply.github.com>
Date: Thu, 5 Dec 2024 09:30:24 -0800
Subject: [PATCH 02/17] update env for PR pipelines

---
 .github/workflows/pr_backend.yml  | 2 +-
 .github/workflows/pr_frontend.yml | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr_backend.yml b/.github/workflows/pr_backend.yml
index f7a13009..e41bc87c 100644
--- a/.github/workflows/pr_backend.yml
+++ b/.github/workflows/pr_backend.yml
@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         python-version: ['3.10']
-    environment: Staging
+    environment: PR
     steps:
       - name: Checkout source
         uses: actions/checkout@v4
diff --git a/.github/workflows/pr_frontend.yml b/.github/workflows/pr_frontend.yml
index 4b7fb3a0..9deb6495 100644
--- a/.github/workflows/pr_frontend.yml
+++ b/.github/workflows/pr_frontend.yml
@@ -1,13 +1,12 @@
 name: Pull Request Frontend
 on:
   push:
-    branches: [main]
     paths:
       - "client/**/*"
 jobs:
   build:
     runs-on: ubuntu-latest
-    environment: Staging
+    environment: PR
     defaults:
       run:
         working-directory: client

From 4efd9895e4eb9bbfbb50acdc71885dc4403122fa Mon Sep 17 00:00:00 2001
From: ericiwamoto <100735505+ericiwamoto@users.noreply.github.com>
Date: Thu, 5 Dec 2024 09:43:36 -0800
Subject: [PATCH 03/17] null change to test PR build test

---
 client/README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/client/README.md b/client/README.md
index da2079b3..48b8d7e3 100644
--- a/client/README.md
+++ b/client/README.md
@@ -214,7 +214,6 @@ When developing, to use a flag:
 6. Set breakpoints in VS code!
 
 
-
 ## Package Versions
 
 The following attemps to explain why certain packages versions have been chosen and what their current limitations are

From 5dbb7eea401e61cec932c3577f0613fbca19eccb Mon Sep 17 00:00:00 2001
From: ericiwamoto <100735505+ericiwamoto@users.noreply.github.com>
Date: Thu, 5 Dec 2024 10:09:03 -0800
Subject: [PATCH 04/17] Update pr build test to not trigger on main branch

---
 .github/workflows/pr_backend.yml  | 3 +++
 .github/workflows/pr_frontend.yml | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/.github/workflows/pr_backend.yml b/.github/workflows/pr_backend.yml
index e41bc87c..c6871c4b 100644
--- a/.github/workflows/pr_backend.yml
+++ b/.github/workflows/pr_backend.yml
@@ -1,6 +1,9 @@
 name: Pull Request Backend 
 on:
   pull_request:
+    branches:
+      - "**"
+      - "!main"
     paths:
       - "data/**"
 env:
diff --git a/.github/workflows/pr_frontend.yml b/.github/workflows/pr_frontend.yml
index 9deb6495..8587ea9b 100644
--- a/.github/workflows/pr_frontend.yml
+++ b/.github/workflows/pr_frontend.yml
@@ -1,6 +1,9 @@
 name: Pull Request Frontend
 on:
   push:
+    branches:
+      - "**"
+      - "!main"
     paths:
       - "client/**/*"
 jobs:

From 850c50ba6bb48735a5a75a88434ba06f8c691204 Mon Sep 17 00:00:00 2001
From: ericiwamoto <100735505+ericiwamoto@users.noreply.github.com>
Date: Thu, 5 Dec 2024 10:42:41 -0800
Subject: [PATCH 05/17] null change to test PR build test #2

---
 client/README.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/client/README.md b/client/README.md
index 48b8d7e3..6d0c4637 100644
--- a/client/README.md
+++ b/client/README.md
@@ -213,7 +213,6 @@ When developing, to use a flag:
 5. Install the [CORS chrome extension](https://chrome.google.com/webstore/detail/allow-cors-access-control/lhobafahddgcelffkeicbaginigeejlf?hl=en) in the browser that is launched by the debugger.
 6. Set breakpoints in VS code!
 
-
 ## Package Versions
 
 The following attemps to explain why certain packages versions have been chosen and what their current limitations are
@@ -224,4 +223,4 @@ The following attemps to explain why certain packages versions have been chosen
 | gatsby-cli | 3.14.2          | 4.15.2         | No            | when attempting to update - breaks all unit tests. Compatibility warning come up with all plugins but this doesn't seems to effect functionality. This is the latest version we can release without investigating unit tests.|  
 | sass       | 1.32.12         | 1.52.3         | No            | This version is needed to surpress the dart warnings on / as division for each component. See [here](https://github.com/twbs/bootstrap/issues/34051#issuecomment-845884423) for more information |
 | uswds      | 2.13.3          | 3.0.2          | No            | Needs to stay at 2.13.3 for peer dependency on trussworks|   
-| trussworks | 3.1.0           | 3.1.0          | No            | latest! |
\ No newline at end of file
+| trussworks | 3.1.0           | 3.1.0          | No            | latest! |

From 6213ed1ac4335c0cce333db41b0be774917ff063 Mon Sep 17 00:00:00 2001
From: ericiwamoto <100735505+ericiwamoto@users.noreply.github.com>
Date: Thu, 5 Dec 2024 11:45:48 -0800
Subject: [PATCH 06/17] fix pr workflow triggers

---
 .github/workflows/pr_backend.yml  | 3 ---
 .github/workflows/pr_frontend.yml | 5 +----
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/.github/workflows/pr_backend.yml b/.github/workflows/pr_backend.yml
index c6871c4b..e41bc87c 100644
--- a/.github/workflows/pr_backend.yml
+++ b/.github/workflows/pr_backend.yml
@@ -1,9 +1,6 @@
 name: Pull Request Backend 
 on:
   pull_request:
-    branches:
-      - "**"
-      - "!main"
     paths:
       - "data/**"
 env:
diff --git a/.github/workflows/pr_frontend.yml b/.github/workflows/pr_frontend.yml
index 8587ea9b..0503629a 100644
--- a/.github/workflows/pr_frontend.yml
+++ b/.github/workflows/pr_frontend.yml
@@ -1,9 +1,6 @@
 name: Pull Request Frontend
 on:
-  push:
-    branches:
-      - "**"
-      - "!main"
+  pull_request:
     paths:
       - "client/**/*"
 jobs:

From 95246c9df02212393e636c986165c4090f2aceab Mon Sep 17 00:00:00 2001
From: Carlos Felix <63804190+carlosfelix2@users.noreply.github.com>
Date: Thu, 5 Dec 2024 15:34:10 -0500
Subject: [PATCH 07/17] Pipeline PR workflow now runs code quality checks

---
 .github/workflows/data-checks.yml             | 44 -------------------
 .github/workflows/pr_backend.yml              | 40 +++++++++++++++++
 .github/workflows/pr_frontend.yml             |  2 +-
 .../data_pipeline/etl/score/tests/conftest.py |  4 +-
 .../etl/score/tests/test_score_post.py        | 10 +++--
 data/data-pipeline/tox.ini                    | 27 ------------
 6 files changed, 51 insertions(+), 76 deletions(-)
 delete mode 100644 .github/workflows/data-checks.yml
 delete mode 100644 data/data-pipeline/tox.ini

diff --git a/.github/workflows/data-checks.yml b/.github/workflows/data-checks.yml
deleted file mode 100644
index ac111d2a..00000000
--- a/.github/workflows/data-checks.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-# This runs tox in the two directories under data
-name: Data Checks
-on:
-  pull_request:
-    paths:
-      - "data/**"
-jobs:
-  data-pipeline:
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: data/data-pipeline
-    strategy:
-      matrix:
-        # checks all of the versions allowed in pyproject.toml
-        python-version: [3.10.15]
-    steps:
-      # installs Python
-      # one execution of the tests per version listed above
-      - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Upgrade wheel
-        run: pip install -U wheel
-      - name: Print variables to help debug
-        uses: hmarr/debug-action@v2
-      - name: Load cached Poetry installation
-        id: cached-poetry-dependencies
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: env-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/data-checks.yml') }}
-      - name: Install poetry
-        uses: snok/install-poetry@v1
-      - name: Print Poetry settings
-        run: poetry show -v
-      - name: Install dependencies
-        run: poetry install
-        # TODO: investigate why caching layer started failing.
-        # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-      - name: Run tox
-        run: poetry run tox
diff --git a/.github/workflows/pr_backend.yml b/.github/workflows/pr_backend.yml
index e41bc87c..b579deba 100644
--- a/.github/workflows/pr_backend.yml
+++ b/.github/workflows/pr_backend.yml
@@ -8,6 +8,43 @@ env:
   J40_VERSION_LABEL_STRING: ${{ vars.SCORE_VERSION }}
 
 jobs:
+  code-quality-checks:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: data/data-pipeline
+    strategy:
+      matrix:
+        python-version: ['3.10']
+    environment: PR
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+      - name: Print variables to help debug
+        uses: hmarr/debug-action@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Load cached Poetry installation
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: env-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/deploy_backend_main.yml') }}
+      - name: Install poetry
+        uses: snok/install-poetry@v1
+      - name: Install dependencies
+        run: poetry install
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+      - name: Check code is formatted
+        run: poetry run black --check data_pipeline/
+      - name: Check code style consistency
+        run: poetry run flake8 -v data_pipeline/
+      - name: Run static code analysis
+        run: poetry run pylint data_pipeline/ --ignore-paths data_pipeline/comparator.py
+      - name: Check library safety
+        run: poetry run safety check --ignore 51457 --ignore 44715 --ignore 70612
   generate-score-tiles:
     runs-on: ubuntu-latest
     defaults:
@@ -44,6 +81,9 @@ jobs:
           sudo apt-get update
           sudo apt-get -y install gdal-bin
           ogrinfo --version
+      - name: Run unit tests
+        run: |
+          poetry run pytest data_pipeline/
       - name: Cleanup Data
         run: |
           poetry run python3 -m data_pipeline.application data-cleanup
diff --git a/.github/workflows/pr_frontend.yml b/.github/workflows/pr_frontend.yml
index 0503629a..f3ba113d 100644
--- a/.github/workflows/pr_frontend.yml
+++ b/.github/workflows/pr_frontend.yml
@@ -2,7 +2,7 @@ name: Pull Request Frontend
 on:
   pull_request:
     paths:
-      - "client/**/*"
+      - "client/**"
 jobs:
   build:
     runs-on: ubuntu-latest
diff --git a/data/data-pipeline/data_pipeline/etl/score/tests/conftest.py b/data/data-pipeline/data_pipeline/etl/score/tests/conftest.py
index 81a029a2..4450672f 100644
--- a/data/data-pipeline/data_pipeline/etl/score/tests/conftest.py
+++ b/data/data-pipeline/data_pipeline/etl/score/tests/conftest.py
@@ -132,7 +132,9 @@ def tile_data_expected():
 
 @pytest.fixture()
 def create_tile_score_data_input():
-    return pd.read_pickle(pytest.SNAPSHOT_DIR / "create_tile_score_data_input.pkl")
+    return pd.read_pickle(
+        pytest.SNAPSHOT_DIR / "create_tile_score_data_input.pkl"
+    )
 
 
 @pytest.fixture()
diff --git a/data/data-pipeline/data_pipeline/etl/score/tests/test_score_post.py b/data/data-pipeline/data_pipeline/etl/score/tests/test_score_post.py
index d3c762c6..23114378 100644
--- a/data/data-pipeline/data_pipeline/etl/score/tests/test_score_post.py
+++ b/data/data-pipeline/data_pipeline/etl/score/tests/test_score_post.py
@@ -83,7 +83,9 @@ def test_create_score_data(
     )
 
 
-def test_create_tile_data(etl, create_tile_score_data_input, create_tile_data_expected):
+def test_create_tile_data(
+    etl, create_tile_score_data_input, create_tile_data_expected
+):
     output_tiles_df_actual = etl._create_tile_data(create_tile_score_data_input)
     pdt.assert_frame_equal(
         output_tiles_df_actual,
@@ -158,8 +160,10 @@ def test_load_downloadable_zip(etl, monkeypatch, score_data_expected):
 def test_create_tract_search_data(census_geojson_sample_data: gpd.GeoDataFrame):
     # Sanity check
     assert len(census_geojson_sample_data) > 0
-    
-    result = PostScoreETL()._create_tract_search_data(census_geojson_sample_data)
+
+    result = PostScoreETL()._create_tract_search_data(
+        census_geojson_sample_data
+    )
     assert isinstance(result, pd.DataFrame)
     assert not result.columns.empty
     columns = ["GEOID10", "INTPTLAT10", "INTPTLON10"]
diff --git a/data/data-pipeline/tox.ini b/data/data-pipeline/tox.ini
deleted file mode 100644
index ebf462eb..00000000
--- a/data/data-pipeline/tox.ini
+++ /dev/null
@@ -1,27 +0,0 @@
-[tox]
-# required because we use pyproject.toml
-isolated_build = true
-envlist = py310, lint, checkdeps, pytest
-# only checks python versions installed locally
-skip_missing_interpreters = true
-
-[testenv:lint]
-deps = pytest
-# lints python code in src and tests
-commands = black data_pipeline
-           flake8 data_pipeline
-           pylint data_pipeline
-
-[testenv:checkdeps]
-# checks the dependencies for security vulnerabilities and open source licenses
-allowlist_externals = bash
-commands = pip install -U wheel
-           # known issue: https://github.com/pyupio/safety/issues/364
-           # jinja2 false positive for our use: https://data.safetycli.com/v/70612/f17
-           safety check --ignore 51457 --ignore 44715 --ignore 70612
-           bash scripts/run-liccheck.sh
-
-[testenv:pytest]
-# Run tests
-deps = pytest
-commands = pytest --full-trace

From 3a41b2a2f8484165ef33b690d5f69423fdfb2ef2 Mon Sep 17 00:00:00 2001
From: Carlos Felix <carlos.felix@agile6.com>
Date: Thu, 5 Dec 2024 15:35:18 -0500
Subject: [PATCH 08/17] Remove temp ignore

---
 .github/workflows/pr_backend.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr_backend.yml b/.github/workflows/pr_backend.yml
index b579deba..bf1db49a 100644
--- a/.github/workflows/pr_backend.yml
+++ b/.github/workflows/pr_backend.yml
@@ -42,7 +42,7 @@ jobs:
       - name: Check code style consistency
         run: poetry run flake8 -v data_pipeline/
       - name: Run static code analysis
-        run: poetry run pylint data_pipeline/ --ignore-paths data_pipeline/comparator.py
+        run: poetry run pylint data_pipeline/
       - name: Check library safety
         run: poetry run safety check --ignore 51457 --ignore 44715 --ignore 70612
   generate-score-tiles:

From d22c34850404dd7410045a4976822046d3dc609d Mon Sep 17 00:00:00 2001
From: ericiwamoto <100735505+ericiwamoto@users.noreply.github.com>
Date: Thu, 5 Dec 2024 13:00:24 -0800
Subject: [PATCH 09/17] Remove concurrency for build tests in a PR

---
 .github/workflows/deploy_backend_main.yml | 8 ++++----
 .github/workflows/pr_backend.yml          | 3 +++
 .github/workflows/pr_frontend.yml         | 3 +++
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/deploy_backend_main.yml b/.github/workflows/deploy_backend_main.yml
index a3be4576..3d8cb281 100644
--- a/.github/workflows/deploy_backend_main.yml
+++ b/.github/workflows/deploy_backend_main.yml
@@ -60,12 +60,12 @@ jobs:
       - name: Get Census Data
         run: |
           poetry run python3 -m data_pipeline.application census-data-download
-          - name: Extract Data Sources
-          run: |
-            poetry run python3 -m data_pipeline.application extract-data-sources
+      - name: Extract Data Sources
+        run: |
+          poetry run python3 -m data_pipeline.application extract-data-sources
       - name: Run ETL
         run: |
-            poetry run python3 -m data_pipeline.application etl-run
+          poetry run python3 -m data_pipeline.application etl-run
       - name: Generate Score
         run: |
           poetry run python3 -m data_pipeline.application score-run
diff --git a/.github/workflows/pr_backend.yml b/.github/workflows/pr_backend.yml
index bf1db49a..2b9c7ae4 100644
--- a/.github/workflows/pr_backend.yml
+++ b/.github/workflows/pr_backend.yml
@@ -3,6 +3,9 @@ on:
   pull_request:
     paths:
       - "data/**"
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
 env:
   CENSUS_API_KEY: ${{ secrets.CENSUS_API_KEY }}
   J40_VERSION_LABEL_STRING: ${{ vars.SCORE_VERSION }}
diff --git a/.github/workflows/pr_frontend.yml b/.github/workflows/pr_frontend.yml
index f3ba113d..033dea58 100644
--- a/.github/workflows/pr_frontend.yml
+++ b/.github/workflows/pr_frontend.yml
@@ -3,6 +3,9 @@ on:
   pull_request:
     paths:
       - "client/**"
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
 jobs:
   build:
     runs-on: ubuntu-latest

From a58edbc72489150867852075576575097979ac12 Mon Sep 17 00:00:00 2001
From: Carlos Felix <63804190+carlosfelix2@users.noreply.github.com>
Date: Thu, 5 Dec 2024 16:31:53 -0500
Subject: [PATCH 10/17] Added tract grandfathering language to UI

---
 .../src/components/AreaDetail/AreaDetail.tsx  |   9 +++-
 .../PrioritizationCopy/PrioritizationCopy.tsx |   6 ++-
 client/src/data/constants.tsx                 |   1 +
 client/src/data/copy/explore.tsx              |   8 ++++
 client/src/intl/en.json                       |   4 ++
 client/src/intl/es.json                       |   1 +
 .../data_pipeline/etl/score/constants.py      |   1 +
 .../etl/score/tests/snapshots/README.md       |  42 +++++++++---------
 .../snapshots/create_tile_data_expected.pkl   | Bin 6657 -> 7231 bytes
 .../create_tile_score_data_input.pkl          | Bin 28696 -> 30157 bytes
 10 files changed, 49 insertions(+), 23 deletions(-)

diff --git a/client/src/components/AreaDetail/AreaDetail.tsx b/client/src/components/AreaDetail/AreaDetail.tsx
index 2f3656b8..3d6d2667 100644
--- a/client/src/components/AreaDetail/AreaDetail.tsx
+++ b/client/src/components/AreaDetail/AreaDetail.tsx
@@ -1163,6 +1163,9 @@ const AreaDetail = ({properties}: IAreaDetailProps) => {
                 null
             }
             percentTractTribal={percentTractTribal}
+            isGrandfathered={
+              properties[constants.IS_GRANDFATHERED]
+            }
           />
           <PrioritizationCopy2
             totalCategoriesPrioritized={
@@ -1189,9 +1192,11 @@ const AreaDetail = ({properties}: IAreaDetailProps) => {
         </div>
       </div>
 
-      {/* Only show the DonutCopy if Adjacency index is true and the total number of disadv ind == 0 */}
+      {/* Only show the DonutCopy if Adjacency index is true, the total number of disadv ind == 0,
+          and not grandfathered. */}
       {properties[constants.ADJACENCY_EXCEEDS_THRESH] &&
-        properties[constants.TOTAL_NUMBER_OF_DISADVANTAGE_INDICATORS] === 0 && (
+        properties[constants.TOTAL_NUMBER_OF_DISADVANTAGE_INDICATORS] === 0 &&
+        !properties[constants.IS_GRANDFATHERED] && (
         <DonutCopy
           isAdjacent={properties[constants.ADJACENCY_EXCEEDS_THRESH]}
           povertyBelow200Percentile={
diff --git a/client/src/components/PrioritizationCopy/PrioritizationCopy.tsx b/client/src/components/PrioritizationCopy/PrioritizationCopy.tsx
index 6769ebc3..19fffda7 100644
--- a/client/src/components/PrioritizationCopy/PrioritizationCopy.tsx
+++ b/client/src/components/PrioritizationCopy/PrioritizationCopy.tsx
@@ -10,6 +10,7 @@ interface IPrioritizationCopy {
   isAdjacencyThreshMet: boolean,
   isAdjacencyLowIncome: boolean,
   isIslandLowIncome: boolean,
+  isGrandfathered: boolean,
   tribalCountAK: number | null,
   tribalCountUS: null, // when this signal is supported add number type
   percentTractTribal: number | null
@@ -38,6 +39,7 @@ const PrioritizationCopy =
      totalBurdensPrioritized,
      isAdjacencyThreshMet,
      isAdjacencyLowIncome,
+     isGrandfathered,
      isIslandLowIncome,
      tribalCountAK,
      tribalCountUS,
@@ -57,8 +59,10 @@ const PrioritizationCopy =
        } else if (isAdjacencyThreshMet && !isAdjacencyLowIncome) {
          // if 1-2-1
          if ( tribalCountAK === null && tribalCountUS === null) {
+           if (isGrandfathered) {
+             prioCopyRendered = EXPLORE_COPY.PRIORITIZATION_COPY.PRIO_GRANDFATHERED_LI;
            // if 1-2-1-1
-           if (percentTractTribal === null) {
+           } else if (percentTractTribal === null) {
              prioCopyRendered = EXPLORE_COPY.PRIORITIZATION_COPY.NOT_PRIO_SURR_LI;
              // if 1-2-1-2
            } else if (percentTractTribal === 0) {
diff --git a/client/src/data/constants.tsx b/client/src/data/constants.tsx
index 2f88b502..4b76025d 100644
--- a/client/src/data/constants.tsx
+++ b/client/src/data/constants.tsx
@@ -71,6 +71,7 @@ export const TOTAL_NUMBER_OF_INDICATORS = "THRHLD";
 export const COUNT_OF_CATEGORIES_DISADV = "CC";
 export const SCORE_N_COMMUNITIES = "SN_C";
 export const SCORE_N_TRIBAL = "SN_T";
+export const IS_GRANDFATHERED = "SN_GRAND";
 
 export const SIDE_PANEL_STATE = "UI_EXP";
 export const SIDE_PANEL_STATE_VALUES = {
diff --git a/client/src/data/copy/explore.tsx b/client/src/data/copy/explore.tsx
index 2429f59d..4815b14c 100644
--- a/client/src/data/copy/explore.tsx
+++ b/client/src/data/copy/explore.tsx
@@ -660,6 +660,14 @@ export const PRIORITIZATION_COPY = {
       bold: boldFn,
     }}
   />,
+  PRIO_GRANDFATHERED_LI: <FormattedMessage
+    id={'explore.map.page.side.panel.prio.copy.prio.grandfathered'}
+    defaultMessage={'This tract is considered disadvantaged because it was identified as disadvantaged in version 1.0 of the tool.'}
+    description={`Navigate to the explore the map page. Click on tract, The side panel will show This tract is considered disadvantaged. This tract is considered disadvantaged because it was identified as disadvantaged in version 1.0 of the tool.`}
+    values={{
+      bold: boldFn,
+    }}
+  />,
 };
 
 export const getPrioNBurdenCopy = (burdens:string) => {
diff --git a/client/src/intl/en.json b/client/src/intl/en.json
index faec8fff..7a0b7a38 100644
--- a/client/src/intl/en.json
+++ b/client/src/intl/en.json
@@ -1427,6 +1427,10 @@
     "defaultMessage": "The {numPoints} that are Federally Recognized Tribes in this tract are are {also} considered disadvantaged.",
     "description": "Navigate to the explore the map page. Click on tract, The {numPoints} that are Federally Recognized Tribes in this tract ares are {also} considered disadvantaged."
   },
+  "explore.map.page.side.panel.prio.copy.prio.grandfathered": {
+    "defaultMessage": "This tract is considered disadvantaged because it was identified as disadvantaged in version 1.0 of the tool.",
+    "description": "Navigate to the explore the map page. Click on tract, The side panel will show This tract is considered disadvantaged. This tract is considered disadvantaged because it was identified as disadvantaged in version 1.0 of the tool."
+  },
   "explore.map.page.side.panel.prio.copy.prio.island.li": {
     "defaultMessage": "This tract is considered disadvantaged because it meets the low income threshold <bold>AND</bold> is located in a U.S. Territory.",
     "description": "Navigate to the explore the map page. Click on tract, The side panel will show This tract is considered disadvantaged. It is an island territory that meets an adjusted low income threshold."
diff --git a/client/src/intl/es.json b/client/src/intl/es.json
index 7bd2b14d..06ccbe5b 100644
--- a/client/src/intl/es.json
+++ b/client/src/intl/es.json
@@ -356,6 +356,7 @@
   "explore.map.page.side.panel.prio.copy.prio.akus": "Los {numAKpoints} pueblos nativos de Alaska y las {numUSpoints} tribus de esta zona que están reconocidas a nivel federal también se consideran desfavorecidos.",
   "explore.map.page.side.panel.prio.copy.prio.anv": "Los {numAKpoints} pueblos nativos de Alaska y las tribus de esta zona que están reconocidas a nivel federal {also} se consideran desfavorecidos.",
   "explore.map.page.side.panel.prio.copy.prio.donut": "Este distrito censal se considera desfavorecido. Está rodeado de distritos censales desfavorecidos <bold>Y</bold> cumple con el umbral ajustado de bajos ingresos. El ajuste no corresponde a ninguna de las categorías.",
+  "explore.map.page.side.panel.prio.copy.prio.grandfathered": "Este distrito censal se considera desfavorecido porque fue identificado como desfavorecido en la versión 1.0 de esta herramienta.",
   "explore.map.page.side.panel.prio.copy.prio.frt": "Las tierras de las tribus reconocidas a nivel federal que cubren {amount} de esta extensión se consideran {also} desfavorecidas.",
   "explore.map.page.side.panel.prio.copy.prio.frt.n.points": "Los {numPoints} que son tribus reconocidas a nivel federal en este distrito censal se consideran {also} desfavorecidos.",
   "explore.map.page.side.panel.prio.copy.prio.n.burden": "Este distrito censal se considera desfavorecido porque cumple con el umbral de carga <bold>Y</bold> con el umbral socioeconómico asociado.",
diff --git a/data/data-pipeline/data_pipeline/etl/score/constants.py b/data/data-pipeline/data_pipeline/etl/score/constants.py
index 964a76f3..acbee81c 100644
--- a/data/data-pipeline/data_pipeline/etl/score/constants.py
+++ b/data/data-pipeline/data_pipeline/etl/score/constants.py
@@ -282,6 +282,7 @@ TILES_SCORE_COLUMNS = {
     # The NEW final score value INCLUDES the adjacency index.
     field_names.FINAL_SCORE_N_BOOLEAN: "SN_C",
     field_names.FINAL_SCORE_N_BOOLEAN_V1_0: "SN_C_V10",
+    field_names.GRANDFATHERED_N_COMMUNITIES_V1_0: "SN_GRAND",
     field_names.IS_TRIBAL_DAC: "SN_T",
     field_names.DIABETES_LOW_INCOME_FIELD: "DLI",
     field_names.ASTHMA_LOW_INCOME_FIELD: "ALI",
diff --git a/data/data-pipeline/data_pipeline/etl/score/tests/snapshots/README.md b/data/data-pipeline/data_pipeline/etl/score/tests/snapshots/README.md
index 01ae8488..4c2693ee 100644
--- a/data/data-pipeline/data_pipeline/etl/score/tests/snapshots/README.md
+++ b/data/data-pipeline/data_pipeline/etl/score/tests/snapshots/README.md
@@ -1,23 +1,25 @@
-These files are used as inputs to unit tests. Some notes in their creation is below.
+# How to generate the sample data in this folder
+
+The sample data in this folder can be easily generated by debugging the `data_pipeline/etl/score/etl_score_post.py` file
+and exporting data using the debugger console. Examples of this exporting are below.
+
+## Why in pickle format?
+
+Exporting as a Pickle file keeps all the metadata about the columns including the data types. If we were to export as CSV then we will need
+to code the data types in the test fixtures for all the columns for the comparison to be correct.
+
+## Exporting the test data
+
+First, verify the code works as expected before exporting the data. You will not be able to inspect the data exports as they are in binary.
+You will be using the debugger to export the data. Note that it is best to export a small subset of the data for faster test execution.
+
+### create_tile_data test
+1. Place a breakpoint in `data_pipeline/etl/score/etl_score_post.py` in the `transform` method right after the call to
+`_create_tile_data` and start the debugger running the Generate Post Score command (`generate-score-post`).
+1. Partially export the `output_score_county_state_merged_df` and `self.output_score_tiles_df` data to a pickle file once the debugger pauses 
+at the breakpoint. Use these sample commands in the debugger console. Note that we are using head and tail to have territories in the sample data.
 
-### create_tile_data_expected.pkl
-1. Set a breakpoint in the `test_create_tile_data` method in `data_pipeline/etl/score/tests/test_score_post.py` 
-after the call to `_create_tile_data` and debug the test.
-2. Extract a subset of the `output_tiles_df_actual` dataframe. Do not extract the whole score as the file 
-will be too big and the test will run slow. Also, you need to extract the same tracts that are in
-the `create_tile_score_data_input.pkl` input data. For example, use the following command once the breakpoint is reached 
-to extract a few rows at the top and bottom of the score. This will some capture states and territories.
 ```python
-import pandas as pd
-pd.concat([output_tiles_df_actual.head(3), output_tiles_df_actual.tail(3)], ignore_index=True).to_pickle('data_pipeline/etl/score/tests/snapshots/create_tile_data_expected.pkl')
+pd.concat([output_score_county_state_merged_df.head(3), output_score_county_state_merged_df.tail(4)], ignore_index=True).to_pickle('data_pipeline/etl/score/tests/snapshots/create_tile_score_data_input.pkl')
+pd.concat([self.output_score_tiles_df.head(3), self.output_score_tiles_df.tail(4)], ignore_index=True).to_pickle('data_pipeline/etl/score/tests/snapshots/create_tile_data_expected.pkl')
 ```
-
-### create_tile_score_data_input.pkl
-1. Set a breakpoint in the transform method in `data_pipeline/etl/score/etl_score_post.py` before the call to
-`_create_tile_data` and run the post scoring.
-2. Extract a subset of the `output_score_county_state_merged_df` dataframe. Do not extract the whole score as the file 
-will be too big and the test will run slow. For example, use the following command once the breakpoint is reached 
-to extract a few rows at the top and bottom of the score. This will some capture states and territories.
-```python
-pd.concat([output_score_county_state_merged_df.head(3), output_score_county_state_merged_df.tail(3)], ignore_index=True).to_pickle('data_pipeline/etl/score/tests/snapshots/create_tile_score_data_input.pkl')
-```
\ No newline at end of file
diff --git a/data/data-pipeline/data_pipeline/etl/score/tests/snapshots/create_tile_data_expected.pkl b/data/data-pipeline/data_pipeline/etl/score/tests/snapshots/create_tile_data_expected.pkl
index 3257e33c17b61cf489876893d14e893da23c9f03..387230d7efdbca2cc8272a4ccff2f94b1521cb7e 100644
GIT binary patch
delta 1975
zcmZuyU2GIp6rQ`ocJ`<2|FT=eZIOuW64~X)b|xqjZ3DIkcBNbJ0h6}Y(CJSovio9N
zEdHQAG$HnkCK&&H;e`@HVu;Z+QKK)Kn)u)&!I%&&{=As*V1nn)+*><4-sH@gx#ynm
zeD|Dl@4U2sMGu}2zO{ebwctEs8otaX*&e*fo~QR?fpost>RpVTm^eE=ak`_wK0SK+
zxw>ujS$#MgXjczehw!QJeYhORF$2E|80`F#+Jhxk$6=C-Br7C;lC*Equ}ZQ)a+TyJ
zNj#{dNisw70m&-Kz2M%G_L{xOdhLciXj=xTkOuZTy+eRMt*BM)d|Ol8!5C9;xMkN4
z^0A&49%;z<J<0*R($Y;_61di~8y{9f*rpYUYY2O_GVzJw3t9<3)H(uhT*1$@exe$~
zaH!~*4AOk*Sg1@CBMUW%z{ZdnDELDj6(pr1vW(ujaOG<G<xYyHv<U|$eaLKEqYzDa
zf)}6*N74^a#omk`q=JNTH6AP*&;><5sgp90h#{dFei6>&R3y<O4erAK<HtE{_E0E7
zkDyS9?35Jp2K0dLr|=>slUZy;QVSG9;0spnhS#TEDss|GOnOQHVcoEZ(@c42>M*d8
zj^;L#j{ig+VgX{6!+p_|K2)hx2JX>t9-uVrXMuTe1^_nVNEp7YrDz^Mj%FrnO)daO
z{4QCsZN6c-4`+&har&w!>=CH?F)ri%t?f9Yb~+8ZLj676S`K~x9Upg0e7AM$Sgi@C
z3WI)}nB=m~M3M-2NDRV|-&yA*hW9z+N7F_6LFl`@%rx#U*j95^hTym#Z2jDbln6o$
zSHrmq0sYqxj)gw?LZNshaY{V2kHd&R(q+88ncr6-EX2xmUnTHFtdk$`oy{p5fib^(
zmRe@ZZa1*MT4Qj^@1o=F*dzS*DRPp{={N=BewP?OE?JK6EyOrX_>cVo114b74<av2
zH$6^od0ToCrUcra#QF(7XuuRq`_Z0MU7Q<*3v0)2{9(3pRruZPdgkJne|P?3zH^C>
zUvVj6Vn;HB#nuGolBxJyX8G`~S=)5=t5{HTG{2SB#N?Wgi}AUMwPZKmB4bHy*{&#V
zsAcmG1-h_yO+b8YMt=I_%wv&nIaHgXc&)bSdcBW7C;Oc;?@E;&eKPfzqj~A2P}$Qp
zh<mzb8s<b+UP(WJpQg9boi+#a{*37OTe|e1NQ)xRwQYUWbPdeIIqc3nO25l`ri-1s
z{$6I5g+sN<@ZrJ7D}C3qTMU&Qz-rsx{8!Y}mBnDaF?zOPgR;1(-k3RKgNB#dtbYNH
C{uW#S

delta 1719
zcmZvcOKcNY6o!30@z{xtZD!($qX;JoQe+PShDT!~0TD@26%GpIL0IufL|~kRs3qwp
zq%y5U5hw`5KdriJ7FCz65JG|#k=P&vQg&T*k?gBVSpcG40QZh3*y9^Z_vN|g|G#tY
z+<8{`{$S`v=%;tB4?~NLsrZ3S;eU3?RDX2r-0gEtb}4*pW_)7&>bPU~*?mj#vEsEW
z#o3O5xta0VOLLChk2}G3<+yzukApc@z*_JS3!qiW<6d0FdBsFovG78%9yC}e@b+~B
zpQ`wy!NfXWmK7D5kcrbF3oD@{ehGE((c;fkeBl3Es^u!pswS3Ia}NhwV*Sp%Gwj#}
zQb;Fui_a-2dOK<lW`9-#l^<ga|Earnid@FL0G*oUfOUqi!acaj+w-uAw_%Ips5lVW
zi>XK&cOv~R&fgJ%sX}iwv8Jn_M)OO@9;PO$b!euH>~Ppas*>>;b5|U4BW`ptcg2y#
z1JOJ7#X98-W$`iD$xmrt)6w3^d1r!GAWJ#fZqW1LN)?a#o}1o?8hF|i!<+c_l<?Z-
z%OXcTp4Z-J*ACCChw`%54DLsDl%pDNu~H=gFSlwkBBxCzKGstC7nQs<A4{GFWLv97
zPD~r!HIBqI*jf@-Vj3GkBAn{1J(~eKv4s*I#&(yga^gst58wom9Hvzx>QWQcRD1$N
zgR(%mDi8=@F|K1-$zmfE$Ge73oK_;G_M86<J<0srTiN*DV|-r{gnolT8j=lSn+BV;
z=bVBq=wlnA5joVYF6QHHoI@?GZb{&;(Fl#oGJT&3+)0I_=J0)m^Pajx%{>SNx<#-6
zkNUtz>4NN7#z-QM2MG%=6INC()CHQ5gEamMr@Do)d~_2uDF;}XO@3jD?e(w9Bwd!b
z$HWzJFPXpem&ukdv`s}>Ve^w#si&aySHDP8p3<xBT^aAXPSLciv>HUKmO-y}6yL>@
zc&%ypvu56|_psmaO`-+g>Kg9p7S8Hv{<X8v(rn>W^9E02m=eB&n5G5K{MS?;Hd5Qt
z1zMo%vbTjDMhm))G)@{#`f^<k*J%+&<1owPcO%CZAFUb-?9rKYUSWrDw{1UL!AhIK
KKYo9;*`EN8F4M69

diff --git a/data/data-pipeline/data_pipeline/etl/score/tests/snapshots/create_tile_score_data_input.pkl b/data/data-pipeline/data_pipeline/etl/score/tests/snapshots/create_tile_score_data_input.pkl
index 8d21a5b94e2d4a3a99ded3171fda0c663aa82c12..3458d992faa3717f44084f03a79e189a5806d269 100644
GIT binary patch
delta 2890
zcmZuz2~ZPf6lO!nZcwriz=VKAFoYz41gsdQ>Vo!A)+F97j7p&%5s`r4*y#baPNjCD
zOnoc$sM^|EbiC_$blQs6vFeN#k9Im8kJO{BRt3h|Iv!K{{@vvW$<BWJ{`cPhj{p5{
z_uqBDQRnVb5${H<{eH>*h_@+<hWDrnDhK>j4UQ)(2E(x!HB>34Bh{+r)(PdFd^n@X
zSJ<OL89oXM%nTHS+rV##g346A;)C5#pPB&4N;{OsGT@SFy-fGW)SaY)TS}WaM-w5<
z9V62!nQoTpj}gzozM%{#RCbZ(sxrYFuZB`pwrIavl_#dL%^D56tqfdI=|%n_&L`=a
zXd84TtHDi6^Xq9T-X57=BQ!w`EF<a1Fp|DXrkjzrVZ1h3|DP-$rk3K)QA=?IwN0GY
zs+Rn;qny+rIa11hN@S+U*GJ0nWqM7fngNo0%z*J^9=l*v&C!p3xWsKsf4JeFSI<L=
z*-Edxl-2q9po?(ToWo{aFaG-eq8nV%>~r5d^4}!-P%@@&Q^Hqy^N(}z(m>HMtRCnD
z&dhZ0Rr^3W&nStyWLhuNJyFBNI$nv&71zgVbo52T?hA(9cOXrZ!@g2NEsWSTowM#J
z`{MMBT1W}kL5oJq9$8&jm49y?r@N#6Zr<n>qWrMQ0-r=h!%rG5oph=O?{DFEO%5Gh
zd3;di@n(o;bJ!0jpYVUFKgs<vU~Xhf+bPj*4y&baSn@~Luigqv*a>1CkFrv}?QADB
z87*)vTH-Y^QvJsMb?D@J?VsGen^&haHva*W&9P7!ZjLG9=@o*H8Yk2VQv^>wuY_eW
zCN;`D)|Ima3)E;6vDx1s4G#|OG(s&><MS?lDo-_{8LuEu&j3%6rI`+sFNz9-1KNzC
z#MKnp`BZ*du!ThnRBRp?b!u?Ll4fy06QUK-fgnwj9%l!4oC6vVdl7yF8!zczj<@3r
z;D9ZNHbfU9dyu4?KgbSEIBrFBAdET(xDmu>qs|U{aqLI1dIxX_FM{Y=^w{J$?m}cI
zpbXJODC$ln*r5Z*Mg#gnG$4rlut6*|bNxcb>rV&Y=b+1AXV#@|S+d-EgM;iuJ2N7E
z*^x=L$dn@Ucb5M_mLHiWWL9i|f|=XzaL}44H6@)SIXkoJ%IQ^CZgb#9K~oa01kr&o
z4wkmpJy?p~i2Pn0`w?t1t_I;nh^tA))!?`bk!^GwC^be?q(_+9*W>YS=fFF}&a9ny
zMwoc!76)690qgoT*7X-TXd8ldD0tlL#TzrsPO&p38xIz5Joqav1Q~H$d>0qeqah1P
z84q14lSq8VaJl35+WlA&cj^e~HA^&;-s#&J)A*L)&cNlSrc(;|OrwK;Q?(6lm&;Yu
zg_ro?!|?mdQvq&T`0c@N`m`iAWtiZZ*UxG@za!YHTJGbsjy1DN#$Dt}Zo};;O{i}Q
ztL)5(EWdi6JNsbkl06@{lXok}#VZ0b+rjS)juEHmep|vQu$xT4nc`(cW{_NauaKN9
zq#(u=$H>zu$VrQXWD^Y|(@b#8l!%ULXf|oVWm3UDO$_o{lpAHLpG0;0mOMzDEbP{W
zniAF>zONk$<peY&iRb(2hzZ$%i2=pg$v+A%PBlDDGpeNTHR#Q`O!15E<wqIpqaw3m
zX8-y=;vWaQ=-K%&G1uUof6PA#wyK6!v$5~t!o1~Jnzy47dt8%lXy_mL3I8nEhog7r
z<Wb0C_0?giNwFkCQe^^a^vtQGNE|lsd+UH2O+1`P*W+m)_nIrPv(dyEAzB3*_0**|
zSipOPMov`-I|rngbTG>jjbMOFi!K*DZ}tnRqMURH)9#ScB7<K-G;%T_>>O~-lGV`3
zT|Tr3@4MV|@%ot)JpI?oP)>CS2T9#KA+5EyY2^(eYEo@w-)f^LXH>5<sJE(=c(_T$
z-eeVv&Gt(G&RYxdFHvjDg;6%WD&MK0oGfH1Qo^yeP;X08vADk#EPQH<$756t2W>hk
zAI{l`6`L_McBDh_DTe0><am8YXcT`g3o<MddJbbBU07LGUbd)AsK_O*$it_Zv0(B1
z1yw^P)K->N&8ZbCh7%Qe_!!sRr9+Cnix-M6@mF<MM&U^MefenjDGDbF730Wk^6)8U
zOb{wwAS`+K6t3ccOpKXhDffZ$%<ys4WSHm8)qC;d<?=<;)|D-;6ZkNnvbN4!E#Ol3
Hc|HFF%~gj-

delta 2260
zcmZveZA?>F7{_~Bdv5{DEiZ)%77C5F1&YT`v2H3bw^=w|0t1akH$Y^o_%amPOs0`-
zQ8BW__}ndK`$DoTVEnKx#JGiQaoLBNkOh)$nTeXQELk+Vgv}XW!sh<>wqW6M(%(6^
zr{{m3=RD6j_w<K5?EECN_t`&ic7I_%%9up&vMzeecH^nl>ZKFbEmG%UZC5->e_PjD
z+bk5et)qjsGU~_)hOI(6sSU7=+F|WwEglt%ge2H0JPWH{$1~z}!4J75QK%;1rOtpm
ztO};<;!R1FBxJLy(V0|Tw{gl@#7Thnz$mx}%I%ywz{lW6`zlf-PMf3(dK>MCR7Hm?
zJ(MPUC?s=gm&=IIjscs4({rE)Tm-*^!Zc3J;0*XKt%_JWr?u&2v={9s;ClKh%Fn1E
zkpUmTD7Xj87supJ1OI$<f7=B8S`?!773gy&=y1kp#HnJ%Ipojev;z<r12(0S)+!O|
zQ9MKn`V{WSzn%8=YwJ%|Q@1;hZY$ZuU71wpDy1G59CdNWFBzTbL*w5Y1F|fZXTPe6
zym)h@`nYBkD#)s%y;<-#Yl=p*;a_%)L{34DDmKlDwk#UxFYdG1g+$-xY#>K&3mwR9
z2_&*C8bq@>8B#KZ7RG2gx7?qay=Kv37MxY6-d#ogZcbOgL%{Pm#laxB23+}^VxSLv
z38p~M!>J9N2NS@tgi{ST0zL<~fUkhlHgFbPE#T)0IYmJ?7zUHTvy@XK=m%H8L%_W-
z4+g<C;3|Uoq8JULy#`#xT)N+=;6qJHDNT6O=$+zCG*KL*pf5&8eDDC=0=|+UHI_u_
zYzaIl;pa->0q6$9U=n!z@BsAt`N$Q&gUSEh82|Ui_@Blu;mhF4vM3EN3(9Zq3gGjH
z(eGN$<zr))kB(iwYkcop9-|2m46N4A&2kl~NsWAq7&1p5D+~l^JW#s70j2&4ictv*
zPVu~hHzbD56QtytSu9$-cb;fiTri8;s5Mw@Z2fP71p!qUFNWCzenDI`ORh3n7g|C*
zgq=Q%WMj-iPlE0>Xi<{&%=XiJjAwJ57{X{W?4`+25$2SzS$_*J#W|Aj&5afqtwQgY
zIo3Qizzj2qI%zUtXO0ue@zfW}r^3+U4Jkg%j6;_mu+4MQbWuDoufC0H%ZsVQ>7l72
z4-J+V?3f?42jY=gHk%wH5a#4O5|7QIHp;6g*56Gv&0{Y;2w%Eeoe0*$<1?)mpKtxK
zm^M#@tsly(Pg!?DZ^SAkhPqphIU$%<7qNw=D?v&kVGq@>Sll}AR)9jaFCdzYfML)S
z0cccd0U4(fr0r(7wDt7d{fyD7yntZFPGVhVJ2F!p3&=RNR)!02Rc|rM&((Tl=iGMm
zS*2UWt>{U{4*N|HXvz*s{GeOJb^7$0RMVL1T+pH(#j+Jf6m#cUgF?-u_Lb{!oe!;u
z<ULcxqpLOjRewx7rQhmmCQ}uf;(iP$<V$5Qoayp1I%v(M9Xwx(stQwG$r&zjKi%X`
z7Nt8p_=4_R==3XEa6qoz*S@QLf4kOIt;_XlAam=UL$B{S;H&G|*M4AkkJeSA_c|In
z-|}sL<IrBi2mGnnxw1AQotT-(TDw{6YSib@s{!lQI<2e8cxu?5>NWke@_Dv%BtK%?
U$X=zw-UvHN4ZV3-LuYUNe;yH|p8x;=


From 2f97674413696dd6e119672049b9d50abaa85897 Mon Sep 17 00:00:00 2001
From: Carlos Felix <63804190+carlosfelix2@users.noreply.github.com>
Date: Fri, 6 Dec 2024 09:57:31 -0500
Subject: [PATCH 11/17] Updates to comparator and libraries

---
 .../data-pipeline/data_pipeline/comparator.py | 395 ++++++++++--------
 data/data-pipeline/poetry.lock                |   2 +-
 data/data-pipeline/pyproject.toml             |   5 +
 3 files changed, 229 insertions(+), 173 deletions(-)

diff --git a/data/data-pipeline/data_pipeline/comparator.py b/data/data-pipeline/data_pipeline/comparator.py
index 860be7bb..97e512c3 100644
--- a/data/data-pipeline/data_pipeline/comparator.py
+++ b/data/data-pipeline/data_pipeline/comparator.py
@@ -17,6 +17,7 @@ pd.set_option("display.width", 10000)
 pd.set_option("display.colheader_justify", "left")
 
 result_text = []
+WORKING_PATH = constants.TMP_PATH / "Comparator" / "Score"
 
 
 def _add_text(text: str):
@@ -38,7 +39,12 @@ def _get_result_doc() -> str:
 
 
 def _read_from_file(file_path: Path):
-    """Read a CSV file into a Dataframe."""
+    """
+    Read a CSV file into a Dataframe.
+
+    Args:
+        file_path (Path): the path of the file to read
+    """
     if not file_path.is_file():
         logger.error(
             f"- No score file exists at {file_path}. "
@@ -53,6 +59,219 @@ def _read_from_file(file_path: Path):
     ).sort_index()
 
 
+def _add_tract_list(tract_list: list[str]):
+    """
+    Adds a list of tracts to the output grouped by Census state.
+
+    Args:
+        tract_list (list[str]): a list of tracts
+    """
+    if len(tract_list) > 0:
+        _add_text("Those tracts are:\n")
+        # First extract the Census states/territories
+        states_by_tract = []
+        for tract in tract_list:
+            states_by_tract.append(tract[0:2])
+        states = set(states_by_tract)
+        # Now output the grouped tracts
+        for state in sorted(states):
+            tracts_for_state = [
+                item for item in tract_list if item.startswith(state)
+            ]
+            _add_text(
+                f"\t{state} = {len(tracts_for_state)} = {', '.join(tracts_for_state)}\n"
+            )
+
+
+def _compare_score_columns(prod_df: pd.DataFrame, local_df: pd.DataFrame):
+    """
+    Compare the columns between scores.
+
+    Args:
+        prod_df (pd.DataFrame): the production score
+        local_df (pd.DataFrame): the local score
+    """
+    log_info("Comparing columns (production vs local)")
+    _add_text("## Columns\n")
+    local_score_df_columns = sorted(local_df.columns.array.tolist())
+    production_score_df_columns = sorted(prod_df.columns.array.tolist())
+    extra_cols_in_local = set(local_score_df_columns) - set(
+        production_score_df_columns
+    )
+    extra_cols_in_prod = set(production_score_df_columns) - set(
+        local_score_df_columns
+    )
+    if len(extra_cols_in_local) == 0 and len(extra_cols_in_prod) == 0:
+        _add_text("* There are no differences in the column names.\n")
+    else:
+        _add_text(
+            f"* There are {len(extra_cols_in_local)} columns that were added as compared to the production score."
+        )
+        if len(extra_cols_in_local) > 0:
+            _add_text(f" Those colums are:\n{extra_cols_in_local}")
+        _add_text(
+            f"\n* There are {len(extra_cols_in_prod)} columns that were removed as compared to the production score."
+        )
+        if len(extra_cols_in_prod) > 0:
+            _add_text(f" Those colums are:\n{extra_cols_in_prod}")
+
+
+def _compare_score_results(prod_df: pd.DataFrame, local_df: pd.DataFrame):
+    """
+    Compare the scores.
+
+    Args:
+        prod_df (pd.DataFrame): the production score
+        local_df (pd.DataFrame): the local score
+    """
+    log_info("Comparing dataframe contents (production vs local)")
+    _add_text("\n\n## Scores\n")
+
+    production_row_count = len(prod_df.index)
+    local_row_count = len(local_df.index)
+
+    # Tract comparison
+    _add_text(
+        f"* The production score has {production_row_count:,} census tracts, and the freshly calculated score has {local_row_count:,}."
+    )
+    if production_row_count == local_row_count:
+        _add_text(" They match!\n")
+    else:
+        _add_text(" They don't match. The differences are:\n")
+        _add_text(
+            "  * New tracts added to the local score are:\n"
+            f"{local_df.index.difference(prod_df.index).to_list()}"
+            "\n  * Tracts removed from the local score are:\n"
+            f"{prod_df.index.difference(local_df.index).to_list()}"
+            "\n"
+        )
+
+    # Population comparison
+    production_total_population = prod_df[field_names.TOTAL_POP_FIELD].sum()
+    local_total_population = local_df[field_names.TOTAL_POP_FIELD].sum()
+
+    _add_text(
+        f"* The total population in all census tracts in the production score is {production_total_population:,}. "
+        f"The total population in all census tracts locally is {local_total_population:,}. "
+    )
+    _add_text(
+        "They match!\n"
+        if production_total_population == local_total_population
+        else f"The difference is {abs(production_total_population - local_total_population):,}.\n"
+    )
+
+    dacs_query = f"`{field_names.FINAL_SCORE_N_BOOLEAN}` == True"
+    production_disadvantaged_tracts_df = prod_df.query(dacs_query)
+    local_disadvantaged_tracts_df = local_df.query(dacs_query)
+
+    production_disadvantaged_tracts_set = set(
+        production_disadvantaged_tracts_df.index.array
+    )
+    local_disadvantaged_tracts_set = set(
+        local_disadvantaged_tracts_df.index.array
+    )
+
+    production_pct_of_population_represented = (
+        production_disadvantaged_tracts_df[field_names.TOTAL_POP_FIELD].sum()
+        / production_total_population
+    )
+    local_pct_of_population_represented = (
+        local_disadvantaged_tracts_df[field_names.TOTAL_POP_FIELD].sum()
+        / local_total_population
+    )
+
+    # DACS comparison
+    _add_text(
+        f"* There are {len(production_disadvantaged_tracts_set):,} disadvantaged tracts in the production score representing"
+        f" {production_pct_of_population_represented:.1%} of the total population, and {len(local_disadvantaged_tracts_set):,}"
+    )
+    _add_text(
+        f" in the locally generated score representing {local_pct_of_population_represented:.1%} of the total population."
+    )
+    _add_text(
+        " The number of tracts match!\n "
+        if len(production_disadvantaged_tracts_set)
+        == len(local_disadvantaged_tracts_set)
+        else f" The difference is {abs(len(production_disadvantaged_tracts_set) - len(local_disadvantaged_tracts_set))} tract(s).\n "
+    )
+
+    removed_tracts = production_disadvantaged_tracts_set.difference(
+        local_disadvantaged_tracts_set
+    )
+    added_tracts = local_disadvantaged_tracts_set.difference(
+        production_disadvantaged_tracts_set
+    )
+    _add_text(
+        f"* There are {len(removed_tracts):,} tract(s) marked as disadvantaged in the production score that are not disadvantaged in the locally"
+        f" generated score (i.e. disadvantaged tracts that were removed by the new score). "
+    )
+    _add_tract_list(removed_tracts)
+
+    _add_text(
+        f"\n* There are {len(added_tracts):,} tract(s) marked as disadvantaged in the locally generated score that are not disadvantaged in the"
+        f" production score (i.e. disadvantaged tracts that were added by the new score). "
+    )
+    _add_tract_list(added_tracts)
+
+    # Grandfathered tracts from v1.0
+    grandfathered_tracts = local_df.loc[
+        local_df[field_names.GRANDFATHERED_N_COMMUNITIES_V1_0]
+    ].index
+    if len(grandfathered_tracts) > 0:
+        _add_text(
+            f"* This includes {len(grandfathered_tracts)} grandfathered tract(s) from v1.0 scoring."
+        )
+        _add_tract_list(grandfathered_tracts)
+    else:
+        _add_text("* There are NO grandfathered tracts from v1.0 scoring.\n")
+
+
+def _generate_delta(prod_df: pd.DataFrame, local_df: pd.DataFrame):
+    """
+    Generate a delta of scores
+
+    Args:
+        prod_df (pd.DataFrame): the production score
+        local_df (pd.DataFrame): the local score
+    """
+    _add_text("\n## Delta\n")
+    # First we make the columns on two dataframes to be the same to be able to compare
+    local_score_df_columns = local_df.columns.array.tolist()
+    production_score_df_columns = prod_df.columns.array.tolist()
+    extra_cols_in_local = set(local_score_df_columns) - set(
+        production_score_df_columns
+    )
+    extra_cols_in_prod = set(production_score_df_columns) - set(
+        local_score_df_columns
+    )
+    trimmed_prod_df = prod_df.drop(extra_cols_in_prod, axis=1)
+    trimmed_local_df = local_df.drop(extra_cols_in_local, axis=1)
+    try:
+
+        comparison_results_df = trimmed_prod_df.compare(
+            trimmed_local_df, align_axis=1, keep_shape=False, keep_equal=False
+        ).rename({"self": "Production", "other": "Local"}, axis=1, level=1)
+
+        _add_text(
+            "* I compared all values across all census tracts. Note this ignores any columns that have been added or removed."
+            f" There are {len(comparison_results_df.index):,} tracts with at least one difference.\n"
+        )
+
+        comparison_path = WORKING_PATH / "deltas.csv"
+        comparison_results_df.to_csv(path_or_buf=comparison_path)
+
+        _add_text(f"* Wrote comparison results to {comparison_path}")
+
+    except ValueError as e:
+        _add_text(
+            "* I could not run a full comparison. This is likely because there are column or index (census tract) differences."
+            " Please examine the logs or run the score comparison locally to find out more.\n"
+        )
+        _add_text(
+            f"Encountered an exception while performing the comparison: {repr(e)}\n"
+        )
+
+
 @click.group()
 def cli():
     """
@@ -101,7 +320,6 @@ def compare_score(
     """
 
     FLOAT_ROUNDING_PLACES = 2
-    WORKING_PATH = constants.TMP_PATH / "Comparator" / "Score"
 
     log_title("Compare Score", "Compare production score to local score")
 
@@ -132,188 +350,21 @@ def compare_score(
     production_score_df = production_score_df.round(FLOAT_ROUNDING_PLACES)
     local_score_df = local_score_df.round(FLOAT_ROUNDING_PLACES)
 
-    local_score_df_columns = sorted(local_score_df.columns.array.tolist())
-    production_score_df_columns = sorted(
-        production_score_df.columns.array.tolist()
-    )
-    extra_cols_in_local = set(local_score_df_columns) - set(
-        production_score_df_columns
-    )
-    extra_cols_in_prod = set(production_score_df_columns) - set(
-        local_score_df_columns
-    )
-
     _add_text("# Score Comparison Summary\n")
     _add_text(
         f"Hi! I'm the Score Comparator. I compared the score in production (version {compare_to_version}) to the"
         " locally calculated score. Here are the results:\n\n"
     )
 
-    #####################
-    # Compare the columns
-    #####################
-    log_info("Comparing columns (production vs local)")
-    _add_text("## Columns\n")
-    if len(extra_cols_in_local) == 0 and len(extra_cols_in_prod) == 0:
-        _add_text("* There are no differences in the column names.\n")
-    else:
-        _add_text(
-            f"* There are {len(extra_cols_in_local)} columns that were added as compared to the production score."
-        )
-        if len(extra_cols_in_local) > 0:
-            _add_text(f" Those colums are:\n{extra_cols_in_local}")
-        _add_text(
-            f"\n* There are {len(extra_cols_in_prod)} columns that were removed as compared to the production score."
-        )
-        if len(extra_cols_in_prod) > 0:
-            _add_text(f" Those colums are:\n{extra_cols_in_prod}")
-
-    ####################
-    # Compare the scores
-    ####################
-    log_info("Comparing dataframe contents (production vs local)")
-    _add_text("\n\n## Scores\n")
-
-    production_row_count = len(production_score_df.index)
-    local_row_count = len(local_score_df.index)
-
-    # Tract comparison
-    _add_text(
-        f"* The production score has {production_row_count:,} census tracts, and the freshly calculated score has {local_row_count:,}."
-    )
-    if production_row_count == local_row_count:
-        _add_text(" They match!\n")
-    else:
-        _add_text(" They don't match. The differences are:\n")
-        _add_text(
-            "  * New tracts added to the local score are:\n"
-            f"{local_score_df.index.difference(production_score_df.index).to_list()}"
-            "\n  * Tracts removed from the local score are:\n"
-            f"{production_score_df.index.difference(local_score_df.index).to_list()}"
-            "\n"
-        )
-
-    # Population comparison
-    production_total_population = production_score_df[
-        field_names.TOTAL_POP_FIELD
-    ].sum()
-    local_total_population = local_score_df[field_names.TOTAL_POP_FIELD].sum()
-
-    _add_text(
-        f"* The total population in all census tracts in the production score is {production_total_population:,}. "
-        f"The total population in all census tracts locally is {local_total_population:,}. "
-    )
-    _add_text(
-        "They match!\n"
-        if production_total_population == local_total_population
-        else f"The difference is {abs(production_total_population - local_total_population):,}.\n"
-    )
-
-    dacs_query = f"`{field_names.FINAL_SCORE_N_BOOLEAN}` == True"
-    production_disadvantaged_tracts_df = production_score_df.query(dacs_query)
-    local_disadvantaged_tracts_df = local_score_df.query(dacs_query)
-
-    production_disadvantaged_tracts_set = set(
-        production_disadvantaged_tracts_df.index.array
-    )
-    local_disadvantaged_tracts_set = set(
-        local_disadvantaged_tracts_df.index.array
-    )
-
-    production_pct_of_population_represented = (
-        production_disadvantaged_tracts_df[field_names.TOTAL_POP_FIELD].sum()
-        / production_total_population
-    )
-    local_pct_of_population_represented = (
-        local_disadvantaged_tracts_df[field_names.TOTAL_POP_FIELD].sum()
-        / local_total_population
-    )
-
-    # DACS comparison
-    _add_text(
-        f"* There are {len(production_disadvantaged_tracts_set):,} disadvantaged tracts in the production score representing"
-        f" {production_pct_of_population_represented:.1%} of the total population, and {len(local_disadvantaged_tracts_set):,}"
-    )
-    _add_text(
-        f" in the locally generated score representing {local_pct_of_population_represented:.1%} of the total population."
-    )
-    _add_text(
-        " The number of tracts match!\n "
-        if len(production_disadvantaged_tracts_set)
-        == len(local_disadvantaged_tracts_set)
-        else f" The difference is {abs(len(production_disadvantaged_tracts_set) - len(local_disadvantaged_tracts_set))} tract(s).\n "
-    )
-
-    removed_tracts = production_disadvantaged_tracts_set.difference(
-        local_disadvantaged_tracts_set
-    )
-    added_tracts = local_disadvantaged_tracts_set.difference(
-        production_disadvantaged_tracts_set
-    )
-    _add_text(
-        f"* There are {len(removed_tracts):,} tract(s) marked as disadvantaged in the production score that are not disadvantaged in the locally"
-        f" generated score (i.e. disadvantaged tracts that were removed by the new score). "
-    )
-    if len(removed_tracts) > 0:
-        _add_text(f"Those tracts are:\n{removed_tracts}")
-
-    _add_text(
-        f"\n* There are {len(added_tracts):,} tract(s) marked as disadvantaged in the locally generated score that are not disadvantaged in the"
-        f" production score (i.e. disadvantaged tracts that were added by the new score). "
-    )
-    if len(added_tracts) > 0:
-        _add_text(f"Those tracts are:\n{added_tracts}\n")
-
-    # Grandfathered tracts from v1.0
-    grandfathered_tracts = local_score_df.loc[
-        local_score_df[field_names.GRANDFATHERED_N_COMMUNITIES_V1_0]
-    ].index
-    if len(grandfathered_tracts) > 0:
-        _add_text(
-            f"* This includes {len(grandfathered_tracts)} grandfathered tract(s) from v1.0 scoring. They are:\n"
-            f"{grandfathered_tracts.to_list()}\n"
-        )
-    else:
-        _add_text("* There are NO grandfathered tracts from v1.0 scoring.\n")
-
-    ################
-    # Create a delta
-    ################
-    _add_text("\n## Delta\n")
-    # First we make the columns on two dataframes to be the same to be able to compare
-    trimmed_prod_df = production_score_df.drop(extra_cols_in_prod, axis=1)
-    trimmed_local_df = local_score_df.drop(extra_cols_in_local, axis=1)
-    try:
-
-        comparison_results_df = trimmed_prod_df.compare(
-            trimmed_local_df, align_axis=1, keep_shape=False, keep_equal=False
-        ).rename({"self": "Production", "other": "Local"}, axis=1, level=1)
-
-        _add_text(
-            "* I compared all values across all census tracts. Note this ignores any columns that have been added or removed."
-            f" There are {len(comparison_results_df.index):,} tracts with at least one difference.\n"
-        )
-
-        comparison_path = WORKING_PATH / "deltas.csv"
-        comparison_results_df.to_csv(path_or_buf=comparison_path)
-
-        _add_text(f"* Wrote comparison results to {comparison_path}")
-
-    except ValueError as e:
-        _add_text(
-            "* I could not run a full comparison. This is likely because there are column or index (census tract) differences."
-            " Please examine the logs or run the score comparison locally to find out more.\n"
-        )
-        _add_text(
-            f"Encountered an exception while performing the comparison: {repr(e)}\n"
-        )
+    _compare_score_columns(production_score_df, local_score_df)
+    _compare_score_results(production_score_df, local_score_df)
+    _generate_delta(production_score_df, local_score_df)
 
     result_doc = _get_result_doc()
     print(result_doc)
 
     # Write the report
     summary_path = WORKING_PATH / "comparison-summary.md"
-
     with open(summary_path, "w", encoding="utf-8") as f:
         f.write(result_doc)
         log_info(f"Wrote comparison summary to {summary_path}")
diff --git a/data/data-pipeline/poetry.lock b/data/data-pipeline/poetry.lock
index 51b54284..6141851e 100644
--- a/data/data-pipeline/poetry.lock
+++ b/data/data-pipeline/poetry.lock
@@ -5053,4 +5053,4 @@ test = ["mypy", "pre-commit", "pytest", "pytest-asyncio", "websockets (>=10.0)"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "bdce0f2249243262fbfd1e73df3f2525c8ca624df6da458480636a19db26c4fe"
+content-hash = "04639d2eaf33218ba4fef190f76620b00fb2285d86d58458511d85dafd304658"
diff --git a/data/data-pipeline/pyproject.toml b/data/data-pipeline/pyproject.toml
index e6fff8e8..f7122078 100644
--- a/data/data-pipeline/pyproject.toml
+++ b/data/data-pipeline/pyproject.toml
@@ -60,6 +60,11 @@ seaborn = "^0.11.2"
 papermill = "^2.3.4"
 jupyterlab = "^3.6.7"
 
+
+[tool.poetry.group.test.dependencies]
+openpyxl = "^3.1.5"
+pytest-snapshot = "^0.9.0"
+
 [build-system]
 build-backend = "poetry.core.masonry.api"
 requires = ["poetry-core>=1.0.0"]

From c7dfd8fd29b7265928c4b79b3214f3354caf4c8a Mon Sep 17 00:00:00 2001
From: ericiwamoto <100735505+ericiwamoto@users.noreply.github.com>
Date: Fri, 6 Dec 2024 08:14:49 -0800
Subject: [PATCH 12/17] Fix frontend variables, add tract search to prod
 backend deploy

---
 .github/workflows/deploy_backend_main.yml | 1 +
 client/.env.development                   | 3 +++
 client/.env.production                    | 2 ++
 client/src/data/constants.tsx             | 6 ++++--
 4 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/deploy_backend_main.yml b/.github/workflows/deploy_backend_main.yml
index 3d8cb281..a68beb69 100644
--- a/.github/workflows/deploy_backend_main.yml
+++ b/.github/workflows/deploy_backend_main.yml
@@ -95,6 +95,7 @@ jobs:
           poetry run s4cmd put ./data_pipeline/data/score/csv/ s3://${{secrets.S3_DATA_BUCKET}}/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/csv --sync-check --recursive --force
           poetry run s4cmd put ./data_pipeline/files/ s3://${{secrets.S3_DATA_BUCKET}}/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --sync-check --recursive --force
           poetry run s4cmd put ./data_pipeline/data/score/downloadable/ s3://${{secrets.S3_DATA_BUCKET}}/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/downloadable --sync-check --recursive --force
+          poetry run s4cmd put ./data_pipeline/data/score/search/ s3://${{secrets.S3_DATA_BUCKET}}/data-versions/${{env.J40_VERSION_LABEL_STRING}}/data/score/search --sync-check --recursive --force
       - name: Deploy 1.0 score post
         if: ${{ env.J40_VERSION_LABEL_STRING == '1.0' }}
         run: |
diff --git a/client/.env.development b/client/.env.development
index a9594395..b599cfb5 100644
--- a/client/.env.development
+++ b/client/.env.development
@@ -12,6 +12,9 @@ GATSBY_DATA_PIPELINE_TRIBAL_PATH=data-pipeline/data/tribal
 GATSBY_BETA_SCORE_PATH = data-versions/beta/data/score
 GATSBY_2_0_SCORE_PATH = data-versions/2.0/data/score
 
+GATSBY_DATA_PIPELINE_SEARCH_PATH_LOCAL = data_pipeline/data/score/search/tracts.json
+GATSBY_2_0_MAP_TRACT_SEARCH_PATH = data-versions/2.0/data/score/search/tracts.json
+
 GATSBY_FILE_DL_PATH_BETA_COMMUNITIES_LIST_XLS=downloadable/beta-communities.xlsx
 GATSBY_FILE_DL_PATH_BETA_COMMUNITIES_LIST_CSV=downloadable/beta-communities.csv
 GATSBY_FILE_DL_PATH_BETA_SHAPE_FILE_ZIP=downloadable/beta-shapefile-codebook.zip
diff --git a/client/.env.production b/client/.env.production
index a925c4a2..0e91cc36 100644
--- a/client/.env.production
+++ b/client/.env.production
@@ -10,6 +10,8 @@ GATSBY_DATA_PIPELINE_TRIBAL_PATH=data-pipeline/data/tribal
 GATSBY_BETA_SCORE_PATH = data-versions/beta/data/score
 GATSBY_2_0_SCORE_PATH = data-versions/2.0/data/score
 
+GATSBY_2_0_MAP_TRACT_SEARCH_PATH = data-versions/2.0/data/score/search/tracts.json
+
 GATSBY_FILE_DL_PATH_BETA_COMMUNITIES_LIST_XLS=downloadable/beta-communities.xlsx
 GATSBY_FILE_DL_PATH_BETA_COMMUNITIES_LIST_CSV=downloadable/beta-communities.csv
 GATSBY_FILE_DL_PATH_BETA_SHAPE_FILE_ZIP=downloadable/beta-shapefile-codebook.zip
diff --git a/client/src/data/constants.tsx b/client/src/data/constants.tsx
index 4b76025d..73631584 100644
--- a/client/src/data/constants.tsx
+++ b/client/src/data/constants.tsx
@@ -388,6 +388,8 @@ process.env.GATSBY_CDN_TILES_BASE_URL;
 
 export const TILE_PATH = process.env.DATA_SOURCE === "local" ?
 process.env.GATSBY_DATA_PIPELINE_SCORE_PATH_LOCAL :
-process.env.GATSBY_1_0_SCORE_PATH;
+process.env.GATSBY_2_0_SCORE_PATH;
 
-export const MAP_TRACT_SEARCH_PATH = "data_pipeline/data/score/search/tracts.json";
+export const MAP_TRACT_SEARCH_PATH = process.env.DATA_SOURCE === "local" ?
+process.env.GATSBY_DATA_PIPELINE_SEARCH_PATH_LOCAL :
+process.env.GATSBY_2_0_MAP_TRACT_SEARCH_PATH;

From c6c9a1a1f750b1ba9d23b49557749ffbd8ef39c0 Mon Sep 17 00:00:00 2001
From: Carlos Felix <63804190+carlosfelix2@users.noreply.github.com>
Date: Fri, 6 Dec 2024 15:08:07 -0500
Subject: [PATCH 13/17] Update job name for the PR frontend workflow

---
 .github/workflows/pr_frontend.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr_frontend.yml b/.github/workflows/pr_frontend.yml
index 033dea58..c0682062 100644
--- a/.github/workflows/pr_frontend.yml
+++ b/.github/workflows/pr_frontend.yml
@@ -7,7 +7,7 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 jobs:
-  build:
+  frontend-build:
     runs-on: ubuntu-latest
     environment: PR
     defaults:

From dd43b64579b4d9171cff79eba17ac0203c4adcaf Mon Sep 17 00:00:00 2001
From: Ryon Coleman <ryon55@gmail.com>
Date: Fri, 6 Dec 2024 15:36:45 -0500
Subject: [PATCH 14/17] Show island low income percentiles in sidebar

---
 .../src/components/AreaDetail/AreaDetail.tsx  |  14 +++--
 .../IslandCopy/IslandCopy.module.scss         |  44 +++++++++++++++
 .../IslandCopy/IslandCopy.module.scss.d.ts    |  19 +++++++
 .../src/components/IslandCopy/IslandCopy.tsx  |  51 ++++++++++++++++++
 .../PrioritizationCopy2.tsx                   |   9 ++--
 .../PrioritizationCopy2.test.tsx.snap         |   8 ++-
 client/src/data/constants.tsx                 |   2 +
 client/src/data/copy/explore.tsx              |   7 +++
 client/src/intl/en.json                       |   4 ++
 client/src/intl/es.json                       |   1 +
 client/tsconfig.json                          |   2 +-
 .../data_pipeline/etl/score/constants.py      |   2 +
 .../snapshots/create_tile_data_expected.pkl   | Bin 7231 -> 7391 bytes
 .../data_pipeline/score/field_names.py        |   5 ++
 .../data_pipeline/score/score_narwhal.py      |   2 +
 15 files changed, 158 insertions(+), 12 deletions(-)
 create mode 100644 client/src/components/IslandCopy/IslandCopy.module.scss
 create mode 100644 client/src/components/IslandCopy/IslandCopy.module.scss.d.ts
 create mode 100644 client/src/components/IslandCopy/IslandCopy.tsx

diff --git a/client/src/components/AreaDetail/AreaDetail.tsx b/client/src/components/AreaDetail/AreaDetail.tsx
index 3d6d2667..d51c62db 100644
--- a/client/src/components/AreaDetail/AreaDetail.tsx
+++ b/client/src/components/AreaDetail/AreaDetail.tsx
@@ -1,8 +1,8 @@
 /* eslint-disable quotes */
 // External Libs:
-import React from "react";
-import {MessageDescriptor, useIntl} from "gatsby-plugin-intl";
 import {Accordion, Button} from "@trussworks/react-uswds";
+import {MessageDescriptor, useIntl} from "gatsby-plugin-intl";
+import React from "react";
 
 // Components:
 import Category from "../Category";
@@ -15,11 +15,12 @@ import TractInfo from "../TractInfo";
 import TractPrioritization from "../TractPrioritization";
 
 // Styles and constants
-import * as styles from "./areaDetail.module.scss";
 import * as constants from "../../data/constants";
 import * as EXPLORE_COPY from "../../data/copy/explore";
+import * as styles from "./areaDetail.module.scss";
 
 // @ts-ignore
+import IslandCopy from "../IslandCopy/IslandCopy";
 import launchIcon from "/node_modules/uswds/dist/img/usa-icons/launch.svg";
 
 interface IAreaDetailProps {
@@ -1192,6 +1193,13 @@ const AreaDetail = ({properties}: IAreaDetailProps) => {
         </div>
       </div>
 
+      {/* Show IslandCopy if the GeoID matches an island prefix */}
+      {constants.TILES_ISLAND_AREA_FIPS_CODES.some((code) => {
+        return properties[constants.GEOID_PROPERTY].startsWith(code);
+      }) && (
+        <IslandCopy povertyPercentile={ properties[constants.CENSUS_DECENNIAL_POVERTY_LESS_THAN_200_FPL_PERCENTILE]} />
+      )}
+
       {/* Only show the DonutCopy if Adjacency index is true, the total number of disadv ind == 0,
           and not grandfathered. */}
       {properties[constants.ADJACENCY_EXCEEDS_THRESH] &&
diff --git a/client/src/components/IslandCopy/IslandCopy.module.scss b/client/src/components/IslandCopy/IslandCopy.module.scss
new file mode 100644
index 00000000..d357652b
--- /dev/null
+++ b/client/src/components/IslandCopy/IslandCopy.module.scss
@@ -0,0 +1,44 @@
+@use '../../styles/design-system.scss' as *;
+@import "../utils.scss";
+
+.islandCopyContainer{
+    @include u-display('flex');
+    flex-direction: column;
+    @include u-padding-left(2);
+    @include u-padding-right(2.5);
+    @include u-padding-top(2);
+
+    .islandRow {
+        @include u-display('flex');
+        justify-content: space-between;
+
+        .islandRowLabel {
+            font-weight: bold;
+        }
+
+        .valueSubTextContainer {
+            @include u-display('flex');
+            flex-direction: column;
+            .subTextContainer{
+                @include indicatorValueSubTextContainer();
+            }
+        }
+    }
+
+    .islandRow:first-child {
+        @include u-padding-bottom("05");
+    }
+
+    .valueContainer {
+        @include u-text('bold');
+    }
+
+    .invert {
+        align-self: flex-end;
+        @include invert();
+    }
+
+    .noInvert {
+        align-self: flex-end;
+    }
+}
diff --git a/client/src/components/IslandCopy/IslandCopy.module.scss.d.ts b/client/src/components/IslandCopy/IslandCopy.module.scss.d.ts
new file mode 100644
index 00000000..e95628dd
--- /dev/null
+++ b/client/src/components/IslandCopy/IslandCopy.module.scss.d.ts
@@ -0,0 +1,19 @@
+declare namespace IslandCopyNamespace {
+    export interface IIslandCopyScss {
+      islandCopyContainer: string;
+      islandRow: string;
+      islandRowLabel: string;
+      invert: string;
+      noInvert: string;
+      valueSubTextContainer: string;
+      valueContainer: string;
+      subTextContainer: string;
+    }
+  }
+
+declare const IslandCopyScssModule: IslandCopyNamespace.IIslandCopyScss & {
+    /** WARNING: Only available when "css-loader" is used without "style-loader" or "mini-css-extract-plugin" */
+    locals: IslandCopyNamespace.IIslandCopyScss;
+  };
+
+  export = IslandCopyScssModule;
diff --git a/client/src/components/IslandCopy/IslandCopy.tsx b/client/src/components/IslandCopy/IslandCopy.tsx
new file mode 100644
index 00000000..e7878142
--- /dev/null
+++ b/client/src/components/IslandCopy/IslandCopy.tsx
@@ -0,0 +1,51 @@
+import {useIntl} from 'gatsby-plugin-intl';
+import React from 'react';
+
+import {IndicatorValue, IndicatorValueSubText} from '../Indicator/Indicator';
+
+import * as styles from './IslandCopy.module.scss';
+
+import * as EXPLORE_COPY from '../../data/copy/explore';
+
+export interface IIslandCopyProps {
+  povertyPercentile: number | null
+}
+
+const IslandCopy = ({povertyPercentile}: IIslandCopyProps) => {
+  const intl = useIntl();
+  const percentileWhole = povertyPercentile ?
+    parseFloat((povertyPercentile*100).toFixed()) : null;
+  const threshold = 65;
+
+  return (
+    <div className={styles.islandCopyContainer}>
+      <div className={styles.islandRow}>
+        <div className={styles.islandRowLabel}>
+          {intl.formatMessage(EXPLORE_COPY.ISLAND_COPY.LOW_INC)}
+        </div>
+        <div className={styles.valueSubTextContainer}>
+          <div className={`${styles.valueContainer}
+            ${ percentileWhole && percentileWhole >= threshold ?
+              styles.invert :
+              styles.noInvert }
+          `}>
+            <IndicatorValue
+              type={'percentile'}
+              displayStat={percentileWhole}
+            />
+          </div>
+          <div className={styles.subTextContainer}>
+            <IndicatorValueSubText
+              value={percentileWhole}
+              isAboveThresh={percentileWhole && percentileWhole >= threshold ? true : false}
+              threshold={threshold}
+              type={'percentile'}
+            />
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default IslandCopy;
diff --git a/client/src/components/PrioritizationCopy2/PrioritizationCopy2.tsx b/client/src/components/PrioritizationCopy2/PrioritizationCopy2.tsx
index de021bf1..63c92926 100644
--- a/client/src/components/PrioritizationCopy2/PrioritizationCopy2.tsx
+++ b/client/src/components/PrioritizationCopy2/PrioritizationCopy2.tsx
@@ -40,8 +40,8 @@ const PrioritizationCopy2 =
      tribalCountUS,
      percentTractTribal,
    }:IPrioritizationCopy2) => {
-     let noStyles = false;
-     let prioCopy2Rendered;
+     let prioCopy2Rendered = <></>;
+
 
      // if 1
      if (
@@ -165,13 +165,10 @@ const PrioritizationCopy2 =
       (tribalCountAK !== null && tribalCountAK >= 1)
      ) {
        prioCopy2Rendered = EXPLORE_COPY.getPrioANVCopy(tribalCountAK, false);
-     } else {
-       prioCopy2Rendered = <></>;
-       noStyles = true;
      };
 
      return (
-       <div className={noStyles ? '' : styles.prioritizationCopy2Container}>
+       <div className={prioCopy2Rendered !== <></> ? '' : styles.prioritizationCopy2Container}>
          {prioCopy2Rendered}
        </div>
      );
diff --git a/client/src/components/PrioritizationCopy2/__snapshots__/PrioritizationCopy2.test.tsx.snap b/client/src/components/PrioritizationCopy2/__snapshots__/PrioritizationCopy2.test.tsx.snap
index 348e5bbb..393064e7 100644
--- a/client/src/components/PrioritizationCopy2/__snapshots__/PrioritizationCopy2.test.tsx.snap
+++ b/client/src/components/PrioritizationCopy2/__snapshots__/PrioritizationCopy2.test.tsx.snap
@@ -2,7 +2,9 @@
 
 exports[`rendering of PrioritizationCopy2 Component checks if component renders The lands of Federally Recognized Tribes that cover 2% of this tract are also considered disadvantaged. when totCats = 0, totBurds = 0, isAdj = true, isAdjLI = true, tribal % = 2, 1`] = `
 <DocumentFragment>
-  <div>
+  <div
+    class=""
+  >
     The lands of Federally Recognized Tribes that cover 2% of this tract are also considered disadvantaged.
   </div>
 </DocumentFragment>
@@ -10,7 +12,9 @@ exports[`rendering of PrioritizationCopy2 Component checks if component renders
 
 exports[`rendering of PrioritizationCopy2 Component checks if component renders The lands of Federally Recognized Tribes that cover 4% of this tract are also considered disadvantaged. when totCats = 0, totBurds = 1, isAdj = true, isAdjLI = true, tribal % = 4, 1`] = `
 <DocumentFragment>
-  <div>
+  <div
+    class=""
+  >
     The lands of Federally Recognized Tribes that cover 4% of this tract are also considered disadvantaged.
   </div>
 </DocumentFragment>
diff --git a/client/src/data/constants.tsx b/client/src/data/constants.tsx
index 73631584..585d7eca 100644
--- a/client/src/data/constants.tsx
+++ b/client/src/data/constants.tsx
@@ -110,6 +110,8 @@ export const IS_EXCEED_BOTH_SOCIO_INDICATORS = "N_EBSI";
 
 export const POVERTY_BELOW_200_PERCENTILE = "P200_I_PFS";
 export const IS_FEDERAL_POVERTY_LEVEL_200 = "FPL200S";
+// Percentile FPL 200 for islands only
+export const CENSUS_DECENNIAL_POVERTY_LESS_THAN_200_FPL_PERCENTILE = "FPL200P";
 
 export const HIGHER_ED_PERCENTILE = "CA";
 export const IS_HIGHER_ED_PERCENTILE = "CA_LT20";
diff --git a/client/src/data/copy/explore.tsx b/client/src/data/copy/explore.tsx
index 4815b14c..5ec6c40b 100644
--- a/client/src/data/copy/explore.tsx
+++ b/client/src/data/copy/explore.tsx
@@ -772,6 +772,13 @@ export const DONUT_COPY = defineMessages({
     description: `Navigate to the explore the map page. Click on side panel, this copy may show up`,
   },
 });
+export const ISLAND_COPY = defineMessages({
+  LOW_INC: {
+    id: 'explore.map.page.side.panel.island.copy.low.income',
+    defaultMessage: 'Low income',
+    description: `Navigate to the explore the map page. Click on side panel, this copy may show up`,
+  },
+});
 export const COMMUNITY = {
   OF_FOCUS: <FormattedMessage
     id={'explore.map.page.side.panel.community.of.focus'}
diff --git a/client/src/intl/en.json b/client/src/intl/en.json
index 7a0b7a38..25fd22f1 100644
--- a/client/src/intl/en.json
+++ b/client/src/intl/en.json
@@ -1375,6 +1375,10 @@
     "defaultMessage": "Identified as disadvantaged?",
     "description": "Navigate to the explore the map page. When the map is in view, click on the map. The side panel will show asking IF the communities is focused on"
   },
+  "explore.map.page.side.panel.island.copy.low.income": {
+    "defaultMessage": "Low income",
+    "description": "Navigate to the explore the map page. Click on side panel, this copy may show up"
+  },
   "explore.map.page.side.panel.not.community.of.focus": {
     "defaultMessage": "NO",
     "description": "Navigate to the explore the map page. When the map is in view, click on the map. The side panel will show the communities the score currently is not focused on"
diff --git a/client/src/intl/es.json b/client/src/intl/es.json
index 06ccbe5b..e6665a9c 100644
--- a/client/src/intl/es.json
+++ b/client/src/intl/es.json
@@ -213,6 +213,7 @@
   "explore.map.page.side.panel.demo.two.or.more": "Dos o más razas",
   "explore.map.page.side.panel.demo.white": "Personas de raza blanca",
   "explore.map.page.side.panel.donut.copy.adj.low.income": "Bajos ingresos ajustados",
+  "explore.map.page.side.panel.island.copy.low.income": "Bajos ingresos",
   "explore.map.page.side.panel.donut.copy.complete.surround": "completamente rodeada",
   "explore.map.page.side.panel.exceed.burden.answer.no": "No",
   "explore.map.page.side.panel.exceed.burden.answer.yes": "Sí",
diff --git a/client/tsconfig.json b/client/tsconfig.json
index 09b97a00..7064fdae 100644
--- a/client/tsconfig.json
+++ b/client/tsconfig.json
@@ -2,7 +2,7 @@
   "compilerOptions": {
     "module": "commonjs",
     "target": "es6",
-    "jsx": "preserve",
+    "jsx": "react",
     "lib": ["dom", "es2015", "es2017", "es2019"],
     "strict": true,
     "noEmit": true,
diff --git a/data/data-pipeline/data_pipeline/etl/score/constants.py b/data/data-pipeline/data_pipeline/etl/score/constants.py
index acbee81c..ac53d1f9 100644
--- a/data/data-pipeline/data_pipeline/etl/score/constants.py
+++ b/data/data-pipeline/data_pipeline/etl/score/constants.py
@@ -347,6 +347,8 @@ TILES_SCORE_COLUMNS = {
     # These are the booleans for socioeconomic indicators
     ## this measures low income boolean
     field_names.FPL_200_SERIES_IMPUTED_AND_ADJUSTED: "FPL200S",
+    # Percentile FPL 200 for islands only
+    field_names.CENSUS_DECENNIAL_POVERTY_LESS_THAN_200_FPL_PERCENTILE: "FPL200P",
     ## Low high school for t&wd
     field_names.WORKFORCE_SOCIO_INDICATORS_EXCEEDED: "N_WKFC_EBSI",
     field_names.DOT_BURDEN_PCTILE_THRESHOLD: "TD_ET",
diff --git a/data/data-pipeline/data_pipeline/etl/score/tests/snapshots/create_tile_data_expected.pkl b/data/data-pipeline/data_pipeline/etl/score/tests/snapshots/create_tile_data_expected.pkl
index 387230d7efdbca2cc8272a4ccff2f94b1521cb7e..e01164dfcfcd221bb3c91f58d943a1baa5746a46 100644
GIT binary patch
delta 238
zcmdmQao>`qfpzMYjVzb=80{xt=W`RCl9A}m>CN6YB_q6KO3;)HOYczcxXJT)6eid5
z3GpBPsu9zF=&gMk7`&T2k58F7&O3gx0-r1sBje<U0@BR!-iebP_^&W0dZ$bl7jR@u
znS4M%Wby_cj>)A0Ud-v<89>F-lkW=1GiP{bP5v)X$eiV!J^419)Z|`4$;nJYLX-If
zWhP${)ML)^&fUx^RKm=h>s<p>s6BbPXasYOcP-F0Qj=xGyqIgf>n3N4*)V!eUM8l=
mShx9tSQ|43yIX*dk%2+L<`5|hMn><+z0yXES0*2jt_J|U^-r$=

delta 170
zcmca_x!;1Nfpx0MMwUx_jJ}hv^SMpl%r7+gFDD;!sCUBTYy4N36TFiruN81)PWDcl
z{81o<In6tLvI3v<<VYUg$qj-+%<0~llkJ7HCm#@WW6t!>ntYp0YO=DBBy*N`Hc&j0
zIomsD^K79KX6794YOq3Eu?XgB?;4<CQj<4{c`?^`*G~Q~X2WPX*+yKGv37HscpLNP
VZ&GHAjMkHNWsDdPPfn1j2LMbTIAH()

diff --git a/data/data-pipeline/data_pipeline/score/field_names.py b/data/data-pipeline/data_pipeline/score/field_names.py
index 9b548f47..57fd387f 100644
--- a/data/data-pipeline/data_pipeline/score/field_names.py
+++ b/data/data-pipeline/data_pipeline/score/field_names.py
@@ -200,6 +200,11 @@ CENSUS_DECENNIAL_MEDIAN_INCOME_2019 = (
 CENSUS_DECENNIAL_POVERTY_LESS_THAN_100_FPL_FIELD_2019 = f"Percentage households below 100% of federal poverty line in {DEC_DATA_YEAR}"
 CENSUS_DECENNIAL_POVERTY_LESS_THAN_200_FPL_FIELD_2019 = f"Percentage households below 200% of federal poverty line in {DEC_DATA_YEAR}"
 CENSUS_DECENNIAL_ADJUSTED_POVERTY_LESS_THAN_200_FPL_FIELD_2019 = f"{CENSUS_DECENNIAL_POVERTY_LESS_THAN_200_FPL_FIELD_2019}, adjusted and imputed"
+CENSUS_DECENNIAL_POVERTY_LESS_THAN_200_FPL_PERCENTILE = (
+    CENSUS_DECENNIAL_ADJUSTED_POVERTY_LESS_THAN_200_FPL_FIELD_2019
+    + ISLAND_AREAS_PERCENTILE_ADJUSTMENT_FIELD
+    + PERCENTILE_FIELD_SUFFIX
+)
 CENSUS_DECENNIAL_HIGH_SCHOOL_ED_FIELD_2019 = f"Percent individuals age 25 or over with less than high school degree in {DEC_DATA_YEAR}"
 CENSUS_DECENNIAL_UNEMPLOYMENT_FIELD_2019 = (
     f"Unemployment (percent) in {DEC_DATA_YEAR}"
diff --git a/data/data-pipeline/data_pipeline/score/score_narwhal.py b/data/data-pipeline/data_pipeline/score/score_narwhal.py
index 3a42879e..63f1a163 100644
--- a/data/data-pipeline/data_pipeline/score/score_narwhal.py
+++ b/data/data-pipeline/data_pipeline/score/score_narwhal.py
@@ -1053,6 +1053,8 @@ class ScoreNarwhal(Score):
 
         # Now we set the low income flag only for territories, but we need to rank them
         # with all other tracts.
+        # Note: This specific method call will generate the
+        # CENSUS_DECENNIAL_POVERTY_LESS_THAN_200_FPL_PERCENTILE column in the score.
         (
             self.df,
             island_areas_poverty_200_criteria_field_name,

From 666eac2df4593accb6ac9932ca38b476cdb6f437 Mon Sep 17 00:00:00 2001
From: Carlos Felix <63804190+carlosfelix2@users.noreply.github.com>
Date: Mon, 9 Dec 2024 09:26:38 -0500
Subject: [PATCH 15/17] Skip PR workflows if no changes to files

---
 .github/workflows/pr_backend.yml  | 32 +++++++++++++++++++++++--------
 .github/workflows/pr_frontend.yml | 21 ++++++++++++++++++--
 2 files changed, 43 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/pr_backend.yml b/.github/workflows/pr_backend.yml
index 2b9c7ae4..ca36481a 100644
--- a/.github/workflows/pr_backend.yml
+++ b/.github/workflows/pr_backend.yml
@@ -1,17 +1,31 @@
 name: Pull Request Backend 
 on:
   pull_request:
-    paths:
-      - "data/**"
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
-env:
-  CENSUS_API_KEY: ${{ secrets.CENSUS_API_KEY }}
-  J40_VERSION_LABEL_STRING: ${{ vars.SCORE_VERSION }}
 
 jobs:
+  # JOB to run change detection
+  detect-be-changes:
+    runs-on: ubuntu-latest
+    # Required permissions
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step
+    outputs:
+      backend: ${{ steps.filter.outputs.backend }}
+    steps:
+    # For pull requests it's not necessary to checkout the code
+    - uses: dorny/paths-filter@v3
+      id: filter
+      with:
+        filters: |
+          backend:
+            - 'data/**'
   code-quality-checks:
+    needs: detect-be-changes
+    if: ${{ needs.detect-be-changes.outputs.backend == 'true' }}
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -48,7 +62,12 @@ jobs:
         run: poetry run pylint data_pipeline/
       - name: Check library safety
         run: poetry run safety check --ignore 51457 --ignore 44715 --ignore 70612
+      - name: Run unit tests
+        run: |
+          poetry run pytest data_pipeline/
   generate-score-tiles:
+    needs: detect-be-changes
+    if: ${{ needs.detect-be-changes.outputs.backend == 'true' }}
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -84,9 +103,6 @@ jobs:
           sudo apt-get update
           sudo apt-get -y install gdal-bin
           ogrinfo --version
-      - name: Run unit tests
-        run: |
-          poetry run pytest data_pipeline/
       - name: Cleanup Data
         run: |
           poetry run python3 -m data_pipeline.application data-cleanup
diff --git a/.github/workflows/pr_frontend.yml b/.github/workflows/pr_frontend.yml
index c0682062..70503d13 100644
--- a/.github/workflows/pr_frontend.yml
+++ b/.github/workflows/pr_frontend.yml
@@ -1,13 +1,30 @@
 name: Pull Request Frontend
 on:
   pull_request:
-    paths:
-      - "client/**"
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 jobs:
+  # JOB to run change detection
+  detect-fe-changes:
+    runs-on: ubuntu-latest
+    # Required permissions
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step
+    outputs:
+      backend: ${{ steps.filter.outputs.frontend }}
+    steps:
+    # For pull requests it's not necessary to checkout the code
+    - uses: dorny/paths-filter@v3
+      id: filter
+      with:
+        filters: |
+          frontend:
+            - 'client/**'
   frontend-build:
+    needs: detect-fe-changes
+    if: ${{ needs.detect-fe-changes.outputs.frontend == 'true' }}
     runs-on: ubuntu-latest
     environment: PR
     defaults:

From dbcbf6b7e14556952b5f744a2cc3e37a243740ea Mon Sep 17 00:00:00 2001
From: Carlos Felix <63804190+carlosfelix2@users.noreply.github.com>
Date: Mon, 9 Dec 2024 11:46:39 -0500
Subject: [PATCH 16/17] Frontend workflow not detecting changes correctly

---
 .github/workflows/pr_frontend.yml | 2 +-
 client/README.md                  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr_frontend.yml b/.github/workflows/pr_frontend.yml
index 70503d13..b1c84293 100644
--- a/.github/workflows/pr_frontend.yml
+++ b/.github/workflows/pr_frontend.yml
@@ -13,7 +13,7 @@ jobs:
       pull-requests: read
     # Set job outputs to values from filter step
     outputs:
-      backend: ${{ steps.filter.outputs.frontend }}
+      frontend: ${{ steps.filter.outputs.frontend }}
     steps:
     # For pull requests it's not necessary to checkout the code
     - uses: dorny/paths-filter@v3
diff --git a/client/README.md b/client/README.md
index 6d0c4637..8984887f 100644
--- a/client/README.md
+++ b/client/README.md
@@ -1,7 +1,7 @@
 [![Staging](https://github.com/usds/justice40-tool/actions/workflows/deploy_fe_staging.yml/badge.svg)](https://github.com/usds/justice40-tool/actions/workflows/deploy_fe_staging.yml)
 [![Production](https://github.com/usds/justice40-tool/actions/workflows/deploy_fe_main.yml/badge.svg)](https://github.com/usds/justice40-tool/actions/workflows/deploy_fe_main.yml)
 
-# Justice40 Client
+# Justice40 Clientss
 
 This README contains the following content:
 

From e7be2b923687f641e822cf15cb8d2f8de517081d Mon Sep 17 00:00:00 2001
From: Carlos Felix <63804190+carlosfelix2@users.noreply.github.com>
Date: Mon, 9 Dec 2024 12:44:05 -0500
Subject: [PATCH 17/17] Force use of matrix in GitHub Action job names

---
 .github/workflows/pr_backend.yml  | 4 ++++
 .github/workflows/pr_frontend.yml | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/.github/workflows/pr_backend.yml b/.github/workflows/pr_backend.yml
index ca36481a..cba7be0f 100644
--- a/.github/workflows/pr_backend.yml
+++ b/.github/workflows/pr_backend.yml
@@ -8,6 +8,7 @@ concurrency:
 jobs:
   # JOB to run change detection
   detect-be-changes:
+    name: Detect backend changes
     runs-on: ubuntu-latest
     # Required permissions
     permissions:
@@ -23,7 +24,9 @@ jobs:
         filters: |
           backend:
             - 'data/**'
+            - '.github/workflows/pr_backend.yml'
   code-quality-checks:
+    name: Code quality checks and tests - ${{ matrix.python-version }}
     needs: detect-be-changes
     if: ${{ needs.detect-be-changes.outputs.backend == 'true' }}
     runs-on: ubuntu-latest
@@ -66,6 +69,7 @@ jobs:
         run: |
           poetry run pytest data_pipeline/
   generate-score-tiles:
+    name: Score and tile generation - ${{ matrix.python-version }}
     needs: detect-be-changes
     if: ${{ needs.detect-be-changes.outputs.backend == 'true' }}
     runs-on: ubuntu-latest
diff --git a/.github/workflows/pr_frontend.yml b/.github/workflows/pr_frontend.yml
index b1c84293..de4232a2 100644
--- a/.github/workflows/pr_frontend.yml
+++ b/.github/workflows/pr_frontend.yml
@@ -7,6 +7,7 @@ concurrency:
 jobs:
   # JOB to run change detection
   detect-fe-changes:
+    name: Detect frontend changes
     runs-on: ubuntu-latest
     # Required permissions
     permissions:
@@ -22,7 +23,9 @@ jobs:
         filters: |
           frontend:
             - 'client/**'
+            - '.github/workflows/pr_frontend.yml'
   frontend-build:
+    name: Frontend build - ${{ matrix.node-version }}
     needs: detect-fe-changes
     if: ${{ needs.detect-fe-changes.outputs.frontend == 'true' }}
     runs-on: ubuntu-latest