diff --git a/.github/workflows/deploy_backend_main.yml b/.github/workflows/deploy_backend_main.yml index c55b9b43..93aae5f4 100644 --- a/.github/workflows/deploy_backend_main.yml +++ b/.github/workflows/deploy_backend_main.yml @@ -60,9 +60,6 @@ jobs: - name: Get Census Data run: | poetry run python3 -m data_pipeline.application census-data-download - - name: Extract Data Sources - run: | - poetry run python3 -m data_pipeline.application extract-data-sources - name: Run ETL run: | poetry run python3 -m data_pipeline.application etl-run diff --git a/.github/workflows/pr_backend.yml b/.github/workflows/pr_backend.yml index cd953570..ee77c63a 100644 --- a/.github/workflows/pr_backend.yml +++ b/.github/workflows/pr_backend.yml @@ -4,7 +4,9 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true - +env: + python-version: '3.10' + J40_VERSION_LABEL_STRING: ${{ vars.SCORE_VERSION }} jobs: # JOB to run change detection detect-be-changes: @@ -26,32 +28,29 @@ jobs: - 'data/**' - '.github/workflows/pr_backend.yml' code-quality-checks: - name: Code quality checks and tests - ${{ matrix.python-version }} + name: Code quality checks and tests needs: detect-be-changes if: ${{ needs.detect-be-changes.outputs.backend == 'true' }} runs-on: ubuntu-latest defaults: run: working-directory: data/data-pipeline - strategy: - matrix: - python-version: ['3.10'] environment: PR steps: - name: Checkout source uses: actions/checkout@v4 - name: Print variables to help debug uses: hmarr/debug-action@v3 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python ${{ env.python-version }} uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ env.python-version }} - name: Load cached Poetry installation id: cached-poetry-dependencies uses: actions/cache@v4 with: path: ~/.cache/pypoetry/virtualenvs - key: env-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/deploy_backend_main.yml') }} + key: env-${{ runner.os }}-${{ env.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/deploy_backend_main.yml') }} - name: Install poetry uses: snok/install-poetry@v1 - name: Install dependencies @@ -69,32 +68,29 @@ jobs: run: | poetry run pytest data_pipeline/ generate-score-tiles: - name: Score and tile generation - ${{ matrix.python-version }} + name: Score and tile generation needs: detect-be-changes if: ${{ needs.detect-be-changes.outputs.backend == 'true' }} runs-on: ubuntu-latest defaults: run: working-directory: data/data-pipeline - strategy: - matrix: - python-version: ['3.10'] environment: PR steps: - name: Checkout source uses: actions/checkout@v4 - name: Print variables to help debug uses: hmarr/debug-action@v3 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python ${{ env.python-version }} uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ env.python-version }} - name: Load cached Poetry installation id: cached-poetry-dependencies uses: actions/cache@v4 with: path: ~/.cache/pypoetry/virtualenvs - key: env-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/deploy_backend_main.yml') }} + key: env-${{ runner.os }}-${{ env.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/deploy_backend_main.yml') }} - name: Install poetry uses: snok/install-poetry@v1 - name: Print Poetry settings @@ -113,9 +109,6 @@ jobs: - name: Get Census Data run: | poetry run python3 -m data_pipeline.application census-data-download - - name: Extract Data Sources - run: | - poetry run python3 -m data_pipeline.application extract-data-sources - name: Run ETL run: | poetry run python3 -m data_pipeline.application etl-run @@ -129,10 +122,6 @@ jobs: - name: Generate Score Post run: | poetry run python3 -m data_pipeline.application generate-score-post - - name: Confirm we generated the version of the score we think we did - if: ${{ env.J40_VERSION_LABEL_STRING == '1.0' || env.J40_VERSION_LABEL_STRING == 'test' }} - run: | - grep "Identified as disadvantaged due to tribal overlap" data_pipeline/data/score/downloadable/* > /dev/null - name: Confirm we generated the version of the score we think we did if: ${{ env.J40_VERSION_LABEL_STRING == '2.0' || env.J40_VERSION_LABEL_STRING == 'beta' }} run: | diff --git a/.github/workflows/pr_frontend.yml b/.github/workflows/pr_frontend.yml index de4232a2..361aa090 100644 --- a/.github/workflows/pr_frontend.yml +++ b/.github/workflows/pr_frontend.yml @@ -4,6 +4,8 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true +env: + node-version: 14.x jobs: # JOB to run change detection detect-fe-changes: @@ -25,7 +27,7 @@ jobs: - 'client/**' - '.github/workflows/pr_frontend.yml' frontend-build: - name: Frontend build - ${{ matrix.node-version }} + name: Frontend build needs: detect-fe-changes if: ${{ needs.detect-fe-changes.outputs.frontend == 'true' }} runs-on: ubuntu-latest @@ -33,15 +35,12 @@ jobs: defaults: run: working-directory: client - strategy: - matrix: - node-version: [14.x] steps: - uses: actions/checkout@v4 - - name: Use Node.js ${{ matrix.node-version }} + - name: Use Node.js ${{ env.node-version }} uses: actions/setup-node@v2 with: - node-version: ${{ matrix.node-version }} + node-version: ${{ env.node-version }} - name: Install run: npm ci - name: Build diff --git a/client/README.md b/client/README.md index 8984887f..6d0c4637 100644 --- a/client/README.md +++ b/client/README.md @@ -1,7 +1,7 @@ [![Staging](https://github.com/usds/justice40-tool/actions/workflows/deploy_fe_staging.yml/badge.svg)](https://github.com/usds/justice40-tool/actions/workflows/deploy_fe_staging.yml) [![Production](https://github.com/usds/justice40-tool/actions/workflows/deploy_fe_main.yml/badge.svg)](https://github.com/usds/justice40-tool/actions/workflows/deploy_fe_main.yml) -# Justice40 Clientss +# Justice40 Client This README contains the following content: diff --git a/data/data-pipeline/data_pipeline/application.py b/data/data-pipeline/data_pipeline/application.py index 66323542..e405d9ec 100644 --- a/data/data-pipeline/data_pipeline/application.py +++ b/data/data-pipeline/data_pipeline/application.py @@ -439,7 +439,6 @@ def full_run(ctx, use_cache): if not use_cache: ctx.invoke(data_cleanup) ctx.invoke(census_data_download, zip_compress=False, use_cache=use_cache) - ctx.invoke(extract_data_sources, dataset=None, use_cache=use_cache) ctx.invoke(etl_run, dataset=None, use_cache=use_cache) ctx.invoke(full_post_etl) diff --git a/data/data-pipeline/data_pipeline/etl/constants.py b/data/data-pipeline/data_pipeline/etl/constants.py index 00f3e0eb..5796025f 100644 --- a/data/data-pipeline/data_pipeline/etl/constants.py +++ b/data/data-pipeline/data_pipeline/etl/constants.py @@ -160,13 +160,15 @@ DATASET_LIST = [ "name": "census_acs", "module_dir": "census_acs", "class_name": "CensusACSETL", - "is_memory_intensive": False, + # Not memory intensive, but run at the end + "is_memory_intensive": True, }, { "name": "census_acs_2010", "module_dir": "census_acs_2010", "class_name": "CensusACS2010ETL", - "is_memory_intensive": False, + # Not memory intensive, but run at the end + "is_memory_intensive": True, }, { "name": "us_army_fuds",