mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-07-28 02:11:16 -07:00
Adds National Risk Index data to ETL pipeline (#549)
* Adds dev dependencies to requirements.txt and re-runs black on codebase * Adds test and code for national risk index etl, still in progress * Removes test_data from .gitignore * Adds test data to nation_risk_index tests * Creates tests and ETL class for NRI data * Adds tests for load() and transform() methods of NationalRiskIndexETL * Updates README.md with info about the NRI dataset * Adds to dos * Moves tests and test data into a tests/ dir in national_risk_index * Moves tmp_dir for tests into data/tmp/tests/ * Promotes fixtures to conftest and relocates national_risk_index tests: The relocation of national_risk_index tests is necessary because tests can only use fixtures specified in conftests within the same package * Fixes issue with df.equals() in test_transform() * Files reformatted by black * Commit changes to other files after re-running black * Fixes unused import that caused lint checks to fail * Moves tests/ directory to app root for data_pipeline
This commit is contained in:
parent
94298635c2
commit
f0900f7b69
14 changed files with 307 additions and 7 deletions
33
data/data-pipeline/data_pipeline/tests/conftest.py
Normal file
33
data/data-pipeline/data_pipeline/tests/conftest.py
Normal file
|
@ -0,0 +1,33 @@
|
|||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from data_pipeline.config import settings
|
||||
from data_pipeline.etl.base import ExtractTransformLoad
|
||||
|
||||
TMP_DIR = settings.APP_ROOT / "data" / "tmp" / "tests"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def mock_paths(tmp_path_factory) -> tuple:
|
||||
"""Creates new DATA_PATH and TMP_PATH that point to a temporary local
|
||||
file structure that can be used to mock data folder during testing
|
||||
"""
|
||||
# sets location of the temp directory inside the national_risk_index folder
|
||||
os.environ["PYTEST_DEBUG_TEMPROOT"] = str(TMP_DIR)
|
||||
TMP_DIR.mkdir(parents=True, exist_ok=True)
|
||||
# creates DATA_PATH and TMP_PATH directories in temp directory
|
||||
data_path = tmp_path_factory.mktemp("data", numbered=False)
|
||||
tmp_path = data_path / "tmp"
|
||||
tmp_path.mkdir()
|
||||
return data_path, tmp_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_etl(monkeypatch, mock_paths) -> None:
|
||||
"""Creates a mock version of the base ExtractTransformLoad class and resets
|
||||
global the variables for DATA_PATH and TMP_PATH to the local mock_paths
|
||||
"""
|
||||
data_path, tmp_path = mock_paths
|
||||
monkeypatch.setattr(ExtractTransformLoad, "DATA_PATH", data_path)
|
||||
monkeypatch.setattr(ExtractTransformLoad, "TMP_PATH", tmp_path)
|
Loading…
Add table
Add a link
Reference in a new issue