publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-02-23 01:54:18 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

Cleanup source tests (#1912)

Browse source 
* Move test to base for broader coverage (#1848)

* Remove duplicate line (#1848)

* FUDS needed an extra mock (#1848)
This commit is contained in:
Matt Bowen 2022-09-23 13:20:23 -04:00 • committed by GitHub
parent f70f30d610
commit d8dd4cf047
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 24 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
data/data-pipeline/data_pipeline/etl/base.py
View file

@ -164,9 +164,6 @@ class ExtractTransformLoad:
for field in dataset_config["load_fields"]: for field in dataset_config["load_fields"]:
cls.COLUMNS_TO_KEEP.append(field["long_name"]) cls.COLUMNS_TO_KEEP.append(field["long_name"])
setattr(cls, field["df_field_name"], field["long_name"]) setattr(cls, field["df_field_name"], field["long_name"])
# set the constants for the class
setattr(cls, field["df_field_name"], field["long_name"])
return dataset_config return dataset_config
# This is a classmethod so it can be used by `get_data_frame` without # This is a classmethod so it can be used by `get_data_frame` without

10
data/data-pipeline/data_pipeline/tests/sources/doe_energy_burden/test_etl.py
View file

@ -59,13 +59,3 @@ class TestDOEEnergyBurdenETL(TestETL):
data_path / "dataset" / "doe_energy_burden" / "usa.csv" data_path / "dataset" / "doe_energy_burden" / "usa.csv"
) )
assert output_file_path == expected_output_file_path assert output_file_path == expected_output_file_path
def test_tract_id_lengths(self, mock_etl, mock_paths):
etl = self._setup_etl_instance_and_run_extract(
mock_etl=mock_etl, mock_paths=mock_paths
)
etl.transform()
etl.validate()
etl.load()
df = etl.get_data_frame()
assert (df[etl.GEOID_TRACT_FIELD_NAME].str.len() == 11).all()

7
data/data-pipeline/data_pipeline/tests/sources/eamlis/test_etl.py
View file

@ -150,3 +150,10 @@ class TestAbandondedLandMineETL(TestETL):
assert len(df[etl.GEOID_TRACT_FIELD_NAME]) == len( assert len(df[etl.GEOID_TRACT_FIELD_NAME]) == len(
self._FIXTURES_SHARED_TRACT_IDS self._FIXTURES_SHARED_TRACT_IDS
) )
def test_tract_id_lengths(self, mock_etl, mock_paths):
with mock.patch(
"data_pipeline.etl.sources.eamlis.etl.add_tracts_for_geometries",
new=_fake_add_tracts_for_geometries,
):
super().test_tract_id_lengths(mock_etl, mock_paths)

10
data/data-pipeline/data_pipeline/tests/sources/example/test_etl.py
View file

@ -209,6 +209,16 @@ class TestETL:
assert actual_file_path == expected_file_path assert actual_file_path == expected_file_path
def test_tract_id_lengths(self, mock_etl, mock_paths):
etl = self._setup_etl_instance_and_run_extract(
mock_etl=mock_etl, mock_paths=mock_paths
)
etl.transform()
etl.validate()
etl.load()
df = etl.get_data_frame()
assert (df[etl.GEOID_TRACT_FIELD_NAME].str.len() == 11).all()
def test_fixtures_contain_shared_tract_ids_base(self, mock_etl, mock_paths): def test_fixtures_contain_shared_tract_ids_base(self, mock_etl, mock_paths):
"""Check presence of necessary shared tract IDs. """Check presence of necessary shared tract IDs.
Note: We used shared census tract IDs so that later our tests can join all the Note: We used shared census tract IDs so that later our tests can join all the

7
data/data-pipeline/data_pipeline/tests/sources/us_army_fuds/test_etl.py
View file

@ -185,3 +185,10 @@ class TestUSArmyFUDSETL(TestETL):
assert len(df[etl.GEOID_TRACT_FIELD_NAME]) == len( assert len(df[etl.GEOID_TRACT_FIELD_NAME]) == len(
self._FIXTURES_SHARED_TRACT_IDS self._FIXTURES_SHARED_TRACT_IDS
) )
def test_tract_id_lengths(self, mock_etl, mock_paths):
with mock.patch(
"data_pipeline.etl.sources.us_army_fuds.etl.add_tracts_for_geometries",
new=_fake_add_tracts_for_geometries,
):
return super().test_tract_id_lengths(mock_etl, mock_paths)