publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-07-30 06:31:16 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Issue 1075: Add refactored ETL tests to NRI (#1088)

Browse source 
* Adds a substantially refactored ETL test to the National Risk Index, to be used as a model for other tests
This commit is contained in:
Lucas Merrill Brown 2022-02-08 19:05:32 -05:00 • committed by GitHub
commit 43e005cc10
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 1155 additions and 619 deletions
Download patch file Download diff file Expand all files Collapse all files

18
data/data-pipeline/data_pipeline/tests/conftest.py
View file

@ -22,8 +22,9 @@ def copy_data_files(src: Path, dst: Path) -> None:
"""
if not dst.exists():
dst.parent.mkdir(parents=True, exist_ok=True)
copyfile(src, dst)
assert dst.exists()
copyfile(src, dst)
assert dst.exists()
@pytest.fixture(scope="session")
@ -34,24 +35,17 @@ def mock_paths(tmp_path_factory) -> tuple:
# sets location of the temp directory inside the national_risk_index folder
os.environ["PYTEST_DEBUG_TEMPROOT"] = str(TMP_DIR)
TMP_DIR.mkdir(parents=True, exist_ok=True)
# creates DATA_PATH and TMP_PATH directories in temp directory
data_path = tmp_path_factory.mktemp("data", numbered=False)
tmp_path = data_path / "tmp"
tmp_path.mkdir()
return data_path, tmp_path
@pytest.fixture(scope="session")
def mock_census(mock_paths) -> Path:
data_path, tmp_path = mock_paths
census_src = settings.APP_ROOT / "tests" / "base" / "data" / "census.csv"
census_dst = data_path / "census" / "csv" / "us.csv"
copy_data_files(census_src, census_dst)
return census_dst
@pytest.fixture
def mock_etl(monkeypatch, mock_paths, mock_census) -> None:
def mock_etl(monkeypatch, mock_paths) -> None:
"""Creates a mock version of the base ExtractTransformLoad class and resets
global the variables for DATA_PATH and TMP_PATH to the local mock_paths
"""