Merge branch 'main' into emma-nechamkin/release/score-narwhal

client/.env.development

@@ -15,7 +15,7 @@ GATSBY_FILE_DL_PATH_TSD_PDF=downloadable/cejst_technical_support_document.pdf
 GATSBY_FILE_DL_PATH_TSD_ES_PDF=downloadable/cejst-technical-support-document-es.pdf
 GATSBY_FILE_DL_PATH_COMMUNITIES_LIST_XLS=downloadable/communities-2022-05-12-1914GMT.xlsx
 GATSBY_FILE_DL_PATH_COMMUNITIES_LIST_CSV=downloadable/communities-2022-05-12-1914GMT.csv
-GATSBY_FILE_DL_PATH_HOW_TO_COMMUNITIES_PDF=downloadable/Draft_Communities_List.pdf
+GATSBY_FILE_DL_PATH_HOW_TO_COMMUNITIES_PDF=downloadable/draft_communities_list.pdf
 
 GATSBY_MAP_TILES_PATH=tiles
 

client/src/components/J40Map.tsx

@@ -324,7 +324,7 @@ const J40Map = ({location}: IJ40Interface) => {
     setGeolocationInProgress(true);
   };
 
-  const mapBoxBaseLayer = 'tl' in flags ? `mapbox://styles/justice40/cl2qimpi2000014qeb1egpox8` : `mapbox://styles/mapbox/streets-v11`;
+  const mapBoxBaseLayer = 'tl' in flags ? `mapbox://styles/justice40/cl2qimpi2000014qeb1egpox8` : `mapbox://styles/justice40/cl4gb253h000014s6r4xwjm10`;
 
   return (
     <>

data/data-pipeline/data_pipeline/etl/score/constants.py

@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 import datetime
 
@@ -47,22 +48,36 @@ DATA_SCORE_JSON_INDEX_FILE_PATH = (
 DATA_SCORE_TILES_DIR = DATA_SCORE_DIR / "tiles"
 
 # Downloadable paths
-current_dt = datetime.datetime.now()
+if not os.environ.get("J40_VERSION_DATE_STRING"):
-timestamp_str = current_dt.strftime("%Y-%m-%d-%H%MGMT")
+    current_dt = datetime.datetime.now()
+    timestamp_str = current_dt.strftime("%Y-%m-%d-%H%MGMT")
+else:
+    timestamp_str = os.environ.get("J40_VERSION_DATE_STRING")
+
+if not os.environ.get("J40_VERSION_LABEL_STRING"):
+    version_str = "beta"
+else:
+    version_str = os.environ.get("J40_VERSION_LABEL_STRING")
+
 SCORE_DOWNLOADABLE_DIR = DATA_SCORE_DIR / "downloadable"
-SCORE_DOWNLOADABLE_PDF_FILE_NAME = "Draft_Communities_List.pdf"
+SCORE_DOWNLOADABLE_PDF_FILE_NAME = "draft_communities_list.pdf"
 SCORE_DOWNLOADABLE_PDF_FILE_PATH = FILES_PATH / SCORE_DOWNLOADABLE_PDF_FILE_NAME
 SCORE_DOWNLOADABLE_CSV_FILE_PATH = (
-    SCORE_DOWNLOADABLE_DIR / f"communities-{timestamp_str}.csv"
+    SCORE_DOWNLOADABLE_DIR / f"{version_str}-communities-{timestamp_str}.csv"
 )
 SCORE_DOWNLOADABLE_EXCEL_FILE_PATH = (
-    SCORE_DOWNLOADABLE_DIR / f"communities-{timestamp_str}.xlsx"
+    SCORE_DOWNLOADABLE_DIR / f"{version_str}-communities-{timestamp_str}.xlsx"
 )
 SCORE_DOWNLOADABLE_CODEBOOK_FILE_PATH = (
-    SCORE_DOWNLOADABLE_DIR / f"codebook-{timestamp_str}.csv"
+    SCORE_DOWNLOADABLE_DIR / f"{version_str}-codebook-{timestamp_str}.csv"
 )
-SCORE_DOWNLOADABLE_ZIP_FILE_PATH = (
+SCORE_DOWNLOADABLE_CSV_ZIP_FILE_PATH = (
-    SCORE_DOWNLOADABLE_DIR / "Screening_Tool_Data.zip"
+    SCORE_DOWNLOADABLE_DIR
+    / f"{version_str}-communities-csv-{timestamp_str}.zip"
+)
+SCORE_DOWNLOADABLE_XLS_ZIP_FILE_PATH = (
+    SCORE_DOWNLOADABLE_DIR
+    / f"{version_str}-communities-xls-{timestamp_str}.zip"
 )
 
 # For the codebook

data/data-pipeline/data_pipeline/etl/score/etl_score_post.py

@@ -492,8 +492,8 @@ class PostScoreETL(ExtractTransformLoad):
         csv_path = constants.SCORE_DOWNLOADABLE_CSV_FILE_PATH
         excel_path = constants.SCORE_DOWNLOADABLE_EXCEL_FILE_PATH
         codebook_path = constants.SCORE_DOWNLOADABLE_CODEBOOK_FILE_PATH
-        zip_path = constants.SCORE_DOWNLOADABLE_ZIP_FILE_PATH
+        csv_zip_path = constants.SCORE_DOWNLOADABLE_CSV_ZIP_FILE_PATH
-        pdf_path = constants.SCORE_DOWNLOADABLE_PDF_FILE_PATH
+        xls_zip_path = constants.SCORE_DOWNLOADABLE_XLS_ZIP_FILE_PATH
 
         logger.info("Writing downloadable excel")
         excel_config = self._load_excel_from_df(
@@ -542,14 +542,19 @@ class PostScoreETL(ExtractTransformLoad):
         # load codebook to disk
         codebook_df.to_csv(codebook_path, index=False)
 
-        logger.info("Compressing files")
+        logger.info("Compressing csv files")
         files_to_compress = [
             csv_path,
+            codebook_path,
+        ]
+        zip_files(csv_zip_path, files_to_compress)
+
+        logger.info("Compressing xls files")
+        files_to_compress = [
             excel_path,
             codebook_path,
-            pdf_path,
         ]
-        zip_files(zip_path, files_to_compress)
+        zip_files(xls_zip_path, files_to_compress)
 
     def load(self) -> None:
         self._load_score_csv_full(

data/data-pipeline/data_pipeline/etl/score/tests/conftest.py

@@ -73,7 +73,7 @@ def score_data_initial(sample_data_dir):
 
 @pytest.fixture()
 def score_pdf_initial(sample_data_dir):
-    return sample_data_dir / "Draft_Communities_List.pdf"
+    return sample_data_dir / "draft_communities_list.pdf"
 
 
 @pytest.fixture()

data/data-pipeline/data_pipeline/etl/score/tests/test_score_post.py

@@ -141,4 +141,5 @@ def test_load_downloadable_zip(etl, monkeypatch, score_data_expected):
     assert constants.SCORE_DOWNLOADABLE_DIR.is_dir()
     assert constants.SCORE_DOWNLOADABLE_CSV_FILE_PATH.is_file()
     assert constants.SCORE_DOWNLOADABLE_EXCEL_FILE_PATH.is_file()
-    assert constants.SCORE_DOWNLOADABLE_ZIP_FILE_PATH.is_file()
+    assert constants.SCORE_DOWNLOADABLE_CSV_ZIP_FILE_PATH.is_file()
+    assert constants.SCORE_DOWNLOADABLE_XLS_ZIP_FILE_PATH.is_file()

data/data-pipeline/data_pipeline/tests/sources/example/test_etl.py

@@ -172,7 +172,6 @@ class TestETL:
         """
         # Setup
         etl = self._get_instance_of_etl_class()
-        etl.__init__()
         data_path, tmp_path = mock_paths
 
         assert etl.DATA_PATH == data_path
@@ -200,7 +199,6 @@ class TestETL:
         etl = self._get_instance_of_etl_class()
         data_path, tmp_path = mock_paths
 
-        etl.__init__()
         actual_file_path = etl._get_output_file_path()
 
         expected_file_path = data_path / "dataset" / etl.NAME / "usa.csv"
@@ -319,7 +317,6 @@ class TestETL:
         etl = self._setup_etl_instance_and_run_extract(
             mock_etl=mock_etl, mock_paths=mock_paths
         )
-        etl.__init__()
         etl.transform()
 
         assert etl.output_df is not None
@@ -337,7 +334,6 @@ class TestETL:
         """
         # setup - input variables
         etl = self._get_instance_of_etl_class()
-        etl.__init__()
 
         # setup - mock transform step
         df_transform = pd.read_csv(

data/data-pipeline/poetry.lock

@@ -1537,6 +1537,21 @@ category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 
+[[package]]
+name = "pydantic"
+version = "1.10.2"
+description = "Data validation and settings management using python type hints"
+category = "main"
+optional = false
+python-versions = ">=3.6.1"
+
+[package.dependencies]
+typing-extensions = ">=3.7.4.3"
+
+[package.extras]
+dotenv = ["python-dotenv (>=0.10.4)"]
+email = ["email-validator (>=1.0.3)"]
+
 [[package]]
 name = "pyflakes"
 version = "2.3.1"
@@ -2192,7 +2207,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8"
-content-hash = "5d55ed06ec57aa8b2fe957ff8a5cfd557f310f8fe65377ffd2d5ec3545f2ec82"
+content-hash = "f3d61a8c4ca54c580ba8d459aa76a9d956e046c2b8339602497ce53393ba9983"
 
 [metadata.files]
 ansiwrap = []
@@ -2547,6 +2562,7 @@ pycparser = [
     {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"},
     {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"},
 ]
+pydantic = []
 pyflakes = []
 pygments = []
 pylint = []

data/data-pipeline/pyproject.toml

@@ -39,6 +39,7 @@ tqdm = "4.62.0"
 types-requests = "^2.25.0"
 us = "^2.0.2"
 xlsxwriter = "^2.0.0"
+pydantic = "^1.9.0"
 Rtree = "^1.0.0"
 
 [tool.poetry.dev-dependencies]