Merge branch 'main' into emma-nechamkin/release/score-narwhal

data/data-pipeline/data_pipeline/etl/score/constants.py

@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 import datetime
 
@@ -47,22 +48,36 @@ DATA_SCORE_JSON_INDEX_FILE_PATH = (
 DATA_SCORE_TILES_DIR = DATA_SCORE_DIR / "tiles"
 
 # Downloadable paths
-current_dt = datetime.datetime.now()
-timestamp_str = current_dt.strftime("%Y-%m-%d-%H%MGMT")
+if not os.environ.get("J40_VERSION_DATE_STRING"):
+    current_dt = datetime.datetime.now()
+    timestamp_str = current_dt.strftime("%Y-%m-%d-%H%MGMT")
+else:
+    timestamp_str = os.environ.get("J40_VERSION_DATE_STRING")
+
+if not os.environ.get("J40_VERSION_LABEL_STRING"):
+    version_str = "beta"
+else:
+    version_str = os.environ.get("J40_VERSION_LABEL_STRING")
+
 SCORE_DOWNLOADABLE_DIR = DATA_SCORE_DIR / "downloadable"
-SCORE_DOWNLOADABLE_PDF_FILE_NAME = "Draft_Communities_List.pdf"
+SCORE_DOWNLOADABLE_PDF_FILE_NAME = "draft_communities_list.pdf"
 SCORE_DOWNLOADABLE_PDF_FILE_PATH = FILES_PATH / SCORE_DOWNLOADABLE_PDF_FILE_NAME
 SCORE_DOWNLOADABLE_CSV_FILE_PATH = (
-    SCORE_DOWNLOADABLE_DIR / f"communities-{timestamp_str}.csv"
+    SCORE_DOWNLOADABLE_DIR / f"{version_str}-communities-{timestamp_str}.csv"
 )
 SCORE_DOWNLOADABLE_EXCEL_FILE_PATH = (
-    SCORE_DOWNLOADABLE_DIR / f"communities-{timestamp_str}.xlsx"
+    SCORE_DOWNLOADABLE_DIR / f"{version_str}-communities-{timestamp_str}.xlsx"
 )
 SCORE_DOWNLOADABLE_CODEBOOK_FILE_PATH = (
-    SCORE_DOWNLOADABLE_DIR / f"codebook-{timestamp_str}.csv"
+    SCORE_DOWNLOADABLE_DIR / f"{version_str}-codebook-{timestamp_str}.csv"
 )
-SCORE_DOWNLOADABLE_ZIP_FILE_PATH = (
-    SCORE_DOWNLOADABLE_DIR / "Screening_Tool_Data.zip"
+SCORE_DOWNLOADABLE_CSV_ZIP_FILE_PATH = (
+    SCORE_DOWNLOADABLE_DIR
+    / f"{version_str}-communities-csv-{timestamp_str}.zip"
+)
+SCORE_DOWNLOADABLE_XLS_ZIP_FILE_PATH = (
+    SCORE_DOWNLOADABLE_DIR
+    / f"{version_str}-communities-xls-{timestamp_str}.zip"
 )
 
 # For the codebook

data/data-pipeline/data_pipeline/etl/score/etl_score_post.py

@@ -492,8 +492,8 @@ class PostScoreETL(ExtractTransformLoad):
         csv_path = constants.SCORE_DOWNLOADABLE_CSV_FILE_PATH
         excel_path = constants.SCORE_DOWNLOADABLE_EXCEL_FILE_PATH
         codebook_path = constants.SCORE_DOWNLOADABLE_CODEBOOK_FILE_PATH
-        zip_path = constants.SCORE_DOWNLOADABLE_ZIP_FILE_PATH
-        pdf_path = constants.SCORE_DOWNLOADABLE_PDF_FILE_PATH
+        csv_zip_path = constants.SCORE_DOWNLOADABLE_CSV_ZIP_FILE_PATH
+        xls_zip_path = constants.SCORE_DOWNLOADABLE_XLS_ZIP_FILE_PATH
 
         logger.info("Writing downloadable excel")
         excel_config = self._load_excel_from_df(
@@ -542,14 +542,19 @@ class PostScoreETL(ExtractTransformLoad):
         # load codebook to disk
         codebook_df.to_csv(codebook_path, index=False)
 
-        logger.info("Compressing files")
+        logger.info("Compressing csv files")
         files_to_compress = [
             csv_path,
+            codebook_path,
+        ]
+        zip_files(csv_zip_path, files_to_compress)
+
+        logger.info("Compressing xls files")
+        files_to_compress = [
             excel_path,
             codebook_path,
-            pdf_path,
         ]
-        zip_files(zip_path, files_to_compress)
+        zip_files(xls_zip_path, files_to_compress)
 
     def load(self) -> None:
         self._load_score_csv_full(

data/data-pipeline/data_pipeline/etl/score/tests/conftest.py

@@ -73,7 +73,7 @@ def score_data_initial(sample_data_dir):
 
 @pytest.fixture()
 def score_pdf_initial(sample_data_dir):
-    return sample_data_dir / "Draft_Communities_List.pdf"
+    return sample_data_dir / "draft_communities_list.pdf"
 
 
 @pytest.fixture()

data/data-pipeline/data_pipeline/etl/score/tests/test_score_post.py

@@ -141,4 +141,5 @@ def test_load_downloadable_zip(etl, monkeypatch, score_data_expected):
     assert constants.SCORE_DOWNLOADABLE_DIR.is_dir()
     assert constants.SCORE_DOWNLOADABLE_CSV_FILE_PATH.is_file()
     assert constants.SCORE_DOWNLOADABLE_EXCEL_FILE_PATH.is_file()
-    assert constants.SCORE_DOWNLOADABLE_ZIP_FILE_PATH.is_file()
+    assert constants.SCORE_DOWNLOADABLE_CSV_ZIP_FILE_PATH.is_file()
+    assert constants.SCORE_DOWNLOADABLE_XLS_ZIP_FILE_PATH.is_file()

data/data-pipeline/data_pipeline/tests/sources/example/test_etl.py

@@ -172,7 +172,6 @@ class TestETL:
         """
         # Setup
         etl = self._get_instance_of_etl_class()
-        etl.__init__()
         data_path, tmp_path = mock_paths
 
         assert etl.DATA_PATH == data_path
@@ -200,7 +199,6 @@ class TestETL:
         etl = self._get_instance_of_etl_class()
         data_path, tmp_path = mock_paths
 
-        etl.__init__()
         actual_file_path = etl._get_output_file_path()
 
         expected_file_path = data_path / "dataset" / etl.NAME / "usa.csv"
@@ -319,7 +317,6 @@ class TestETL:
         etl = self._setup_etl_instance_and_run_extract(
             mock_etl=mock_etl, mock_paths=mock_paths
         )
-        etl.__init__()
         etl.transform()
 
         assert etl.output_df is not None
@@ -337,7 +334,6 @@ class TestETL:
         """
         # setup - input variables
         etl = self._get_instance_of_etl_class()
-        etl.__init__()
 
         # setup - mock transform step
         df_transform = pd.read_csv(