diff --git a/.github/workflows/deploy_be_staging.yml b/.github/workflows/deploy_be_staging.yml index b4e7f2ef..a1edd959 100644 --- a/.github/workflows/deploy_be_staging.yml +++ b/.github/workflows/deploy_be_staging.yml @@ -1,9 +1,9 @@ -name: Deploy Frontend staging +name: Deploy Backend staging on: pull_request: branches: [main] paths: - - "data/*" + - "data/**" env: PR_NUMBER: ${{github.event.pull_request.number}} SHA_NUMBER: ${{github.event.pull_request.head.sha}} diff --git a/.github/workflows/score-checks.yml b/.github/workflows/score-checks.yml deleted file mode 100644 index 5cb84e1d..00000000 --- a/.github/workflows/score-checks.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Score Checks -on: - pull_request: - branches: [main] # runs on any PR against main - paths: - - "data/**" -jobs: - generate-score: - runs-on: ubuntu-latest - defaults: - run: - working-directory: data/data-pipeline - strategy: - matrix: - python-version: [3.9] - steps: - - name: Checkout source - uses: actions/checkout@v2 - - name: Print variables to help debug - uses: hmarr/debug-action@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Setup Poetry - uses: Gr1N/setup-poetry@v7 - - name: Print poetry version - run: poetry --version - - name: Install dependencies - run: poetry install - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.DATA_DEV_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.DATA_DEV_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - name: Generate Score - run: | - poetry run python3 data_pipeline/application.py score-full-run - - name: Generate Score Post - run: | - poetry run python3 data_pipeline/application.py generate-score-post -s aws - - name: Generate Score Geo - run: | - poetry run python3 data_pipeline/application.py geo-score diff --git a/data/data-pipeline/data_pipeline/etl/base.py b/data/data-pipeline/data_pipeline/etl/base.py index 67a61f0a..be1dd86a 100644 --- a/data/data-pipeline/data_pipeline/etl/base.py +++ b/data/data-pipeline/data_pipeline/etl/base.py @@ -104,7 +104,7 @@ class ExtractTransformLoad: # of the instance which is often a child class. tmp_path = self.DATA_PATH / "tmp" / str(self.__class__.__name__) - # Create directory if it doesn't exist already + # Create directory if it doesn't exist tmp_path.mkdir(parents=True, exist_ok=True) return tmp_path