Add pytest to tox run in CI/CD (#713)

* Add pytest to tox run in CI/CD

* Try fixing tox dependencies for pytest

* update poetry to get ci/cd passing

* Run poetry export with --dev flag to include dev dependencies such as pytest

* WIP updating test fixtures to include PDF

* Remove dev dependencies from reqs and add pytest to envlist to make build faster

* passing score_post tests

* Add pytest tox (#729)

* Fix failing pytest

* Fixes failing tox tests and updates requirements.txt to include dev deps

* pickle protocol 4

Co-authored-by: Shelby Switzer <shelby.switzer@cms.hhs.gov>
Co-authored-by: Jorge Escobar <jorge.e.escobar@omb.eop.gov>
Co-authored-by: Billy Daly <williamdaly422@gmail.com>
Co-authored-by: Jorge Escobar <83969469+esfoobar-usds@users.noreply.github.com>

data/data-pipeline/data_pipeline/tests/sources/national_risk_index/data/input.csv

@@ -1,6 +1,6 @@
-TRACT,TRACTFIPS,RISK_SCORE,RISK_RATNG,RISK_NPCTL
-40300,05007040300,10.492015,Very Low,15.3494
-20100,05001020100,14.705854,Relatively Low,36.725828
-40500,15007040500,10.234981,Very Low,13.997993
-21010,15001021010,21.537231,Relatively Moderate,59.488033
-21101,15001021101,19.434585,Relatively Low,53.392265
+TRACT,TRACTFIPS,RISK_SCORE,RISK_RATNG,RISK_NPCTL,EAL_SCORE
+40300,05007040300,10.492015,Very Low,15.3494,11.5
+20100,05001020100,14.705854,Relatively Low,36.725828,12.5
+40500,15007040500,10.234981,Very Low,13.997993,13.5
+21010,15001021010,21.537231,Relatively Moderate,59.488033,14.5
+21101,15001021101,19.434585,Relatively Low,53.392265,15.5

data/data-pipeline/data_pipeline/tests/sources/national_risk_index/data/output.csv

@@ -1,11 +1,11 @@
-GEOID10,GEOID10_TRACT,TRACT,RISK_SCORE,RISK_RATNG,RISK_NPCTL
-050070403001,05007040300,40300,10.492015,Very Low,15.3494
-050070403002,05007040300,40300,10.492015,Very Low,15.3494
-050010201001,05001020100,20100,14.705854,Relatively Low,36.725828
-050010201002,05001020100,20100,14.705854,Relatively Low,36.725828
-150070405001,15007040500,40500,10.234981,Very Low,13.997993
-150070405002,15007040500,40500,10.234981,Very Low,13.997993
-150010210101,15001021010,21010,21.537231,Relatively Moderate,59.488033
-150010210102,15001021010,21010,21.537231,Relatively Moderate,59.488033
-150010211011,15001021101,21101,19.434585,Relatively Low,53.392265
-150010211012,15001021101,21101,19.434585,Relatively Low,53.392265
+GEOID10,FEMA Risk Index Expected Annual Loss Score
+050070403001,11.5
+050070403002,11.5
+050010201001,12.5
+050010201002,12.5
+150070405001,13.5
+150070405002,13.5
+150010210101,14.5
+150010210102,14.5
+150010211011,15.5
+150010211012,15.5

data/data-pipeline/data_pipeline/tests/sources/national_risk_index/data/transform.csv

@@ -0,0 +1,11 @@
+GEOID10,GEOID10_TRACT,FEMA Risk Index Expected Annual Loss Score
+050070403001,05007040300,11.5
+050070403002,05007040300,11.5
+050010201001,05001020100,12.5
+050010201002,05001020100,12.5
+150070405001,15007040500,13.5
+150070405002,15007040500,13.5
+150010210101,15001021010,14.5
+150010210102,15001021010,14.5
+150010211011,15001021101,15.5
+150010211012,15001021101,15.5

data/data-pipeline/data_pipeline/tests/sources/national_risk_index/test_etl.py

@@ -73,13 +73,13 @@ class TestNationalRiskIndexETL:
         TRACT_COL = etl.GEOID_TRACT_FIELD_NAME
         BLOCK_COL = etl.GEOID_FIELD_NAME
         expected = pd.read_csv(
-            DATA_DIR / "output.csv",
+            DATA_DIR / "transform.csv",
             dtype={BLOCK_COL: "string", TRACT_COL: "string"},
         )
         # execution
         etl.transform()
         # validation
-        assert etl.df.shape == (10, 6)
+        assert etl.df.shape == (10, 3)
         assert etl.df.equals(expected)
 
     def test_load(self, mock_etl):
@@ -90,21 +90,23 @@ class TestNationalRiskIndexETL:
           self.OUTPUT_DIR
         - The content of the file that's written matches the data in self.df
         """
-        # setup
+        # setup - input variables
         etl = NationalRiskIndexETL()
-        output_path = etl.OUTPUT_DIR / "usa.csv"
         TRACT_COL = etl.GEOID_TRACT_FIELD_NAME
         BLOCK_COL = etl.GEOID_FIELD_NAME
-        expected = pd.read_csv(
-            DATA_DIR / "output.csv",
-            dtype={BLOCK_COL: str, TRACT_COL: str},
+        output_path = etl.OUTPUT_DIR / "usa.csv"
+        # setup - mock transform step
+        df_transform = pd.read_csv(
+            DATA_DIR / "transform.csv",
+            dtype={BLOCK_COL: "string", TRACT_COL: "string"},
         )
-        etl.df = expected
+        etl.df = df_transform
+        # setup - load expected output
+        expected = pd.read_csv(DATA_DIR / "output.csv", dtype={BLOCK_COL: str})
         # execution
         etl.load()
-        output = pd.read_csv(
-            output_path, dtype={BLOCK_COL: str, TRACT_COL: str}
-        )
+        output = pd.read_csv(output_path, dtype={BLOCK_COL: str})
         # validation
         assert output_path.exists()
+        assert output.shape == (10, 2)
         assert output.equals(expected)