publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-02-23 01:54:18 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into emma-nechamkin/release/score-narwhal

Browse source
This commit is contained in:
Jorge Escobar 2022-10-04 16:03:46 -04:00
commit baa34ec038
10 changed files with 56 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
client/.env.development
View file

@ -15,7 +15,7 @@ GATSBY_FILE_DL_PATH_TSD_PDF=downloadable/cejst_technical_support_document.pdf
GATSBY_FILE_DL_PATH_TSD_ES_PDF=downloadable/cejst-technical-support-document-es.pdf
GATSBY_FILE_DL_PATH_COMMUNITIES_LIST_XLS=downloadable/communities-2022-05-12-1914GMT.xlsx
GATSBY_FILE_DL_PATH_COMMUNITIES_LIST_CSV=downloadable/communities-2022-05-12-1914GMT.csv
GATSBY_FILE_DL_PATH_HOW_TO_COMMUNITIES_PDF=downloadable/Draft_Communities_List.pdf
GATSBY_FILE_DL_PATH_HOW_TO_COMMUNITIES_PDF=downloadable/draft_communities_list.pdf
GATSBY_MAP_TILES_PATH=tiles

2
client/src/components/J40Map.tsx
View file

@ -324,7 +324,7 @@ const J40Map = ({location}: IJ40Interface) => {
setGeolocationInProgress(true);
};
const mapBoxBaseLayer = 'tl' in flags ? `mapbox://styles/justice40/cl2qimpi2000014qeb1egpox8` : `mapbox://styles/mapbox/streets-v11`;
const mapBoxBaseLayer = 'tl' in flags ? `mapbox://styles/justice40/cl2qimpi2000014qeb1egpox8` : `mapbox://styles/justice40/cl4gb253h000014s6r4xwjm10`;
return (
<>

31
data/data-pipeline/data_pipeline/etl/score/constants.py
View file

@ -1,3 +1,4 @@
import os
from pathlib import Path
import datetime
@ -47,22 +48,36 @@ DATA_SCORE_JSON_INDEX_FILE_PATH = (
DATA_SCORE_TILES_DIR = DATA_SCORE_DIR / "tiles"
# Downloadable paths
current_dt = datetime.datetime.now()
timestamp_str = current_dt.strftime("%Y-%m-%d-%H%MGMT")
if not os.environ.get("J40_VERSION_DATE_STRING"):
current_dt = datetime.datetime.now()
timestamp_str = current_dt.strftime("%Y-%m-%d-%H%MGMT")
else:
timestamp_str = os.environ.get("J40_VERSION_DATE_STRING")
if not os.environ.get("J40_VERSION_LABEL_STRING"):
version_str = "beta"
else:
version_str = os.environ.get("J40_VERSION_LABEL_STRING")
SCORE_DOWNLOADABLE_DIR = DATA_SCORE_DIR / "downloadable"
SCORE_DOWNLOADABLE_PDF_FILE_NAME = "Draft_Communities_List.pdf"
SCORE_DOWNLOADABLE_PDF_FILE_NAME = "draft_communities_list.pdf"
SCORE_DOWNLOADABLE_PDF_FILE_PATH = FILES_PATH / SCORE_DOWNLOADABLE_PDF_FILE_NAME
SCORE_DOWNLOADABLE_CSV_FILE_PATH = (
SCORE_DOWNLOADABLE_DIR / f"communities-{timestamp_str}.csv"
SCORE_DOWNLOADABLE_DIR / f"{version_str}-communities-{timestamp_str}.csv"
)
SCORE_DOWNLOADABLE_EXCEL_FILE_PATH = (
SCORE_DOWNLOADABLE_DIR / f"communities-{timestamp_str}.xlsx"
SCORE_DOWNLOADABLE_DIR / f"{version_str}-communities-{timestamp_str}.xlsx"
)
SCORE_DOWNLOADABLE_CODEBOOK_FILE_PATH = (
SCORE_DOWNLOADABLE_DIR / f"codebook-{timestamp_str}.csv"
SCORE_DOWNLOADABLE_DIR / f"{version_str}-codebook-{timestamp_str}.csv"
)
SCORE_DOWNLOADABLE_ZIP_FILE_PATH = (
SCORE_DOWNLOADABLE_DIR / "Screening_Tool_Data.zip"
SCORE_DOWNLOADABLE_CSV_ZIP_FILE_PATH = (
SCORE_DOWNLOADABLE_DIR
/ f"{version_str}-communities-csv-{timestamp_str}.zip"
)
SCORE_DOWNLOADABLE_XLS_ZIP_FILE_PATH = (
SCORE_DOWNLOADABLE_DIR
/ f"{version_str}-communities-xls-{timestamp_str}.zip"
)
# For the codebook

15
data/data-pipeline/data_pipeline/etl/score/etl_score_post.py
View file

@ -492,8 +492,8 @@ class PostScoreETL(ExtractTransformLoad):
csv_path = constants.SCORE_DOWNLOADABLE_CSV_FILE_PATH
excel_path = constants.SCORE_DOWNLOADABLE_EXCEL_FILE_PATH
codebook_path = constants.SCORE_DOWNLOADABLE_CODEBOOK_FILE_PATH
zip_path = constants.SCORE_DOWNLOADABLE_ZIP_FILE_PATH
pdf_path = constants.SCORE_DOWNLOADABLE_PDF_FILE_PATH
csv_zip_path = constants.SCORE_DOWNLOADABLE_CSV_ZIP_FILE_PATH
xls_zip_path = constants.SCORE_DOWNLOADABLE_XLS_ZIP_FILE_PATH
logger.info("Writing downloadable excel")
excel_config = self._load_excel_from_df(
@ -542,14 +542,19 @@ class PostScoreETL(ExtractTransformLoad):
# load codebook to disk
codebook_df.to_csv(codebook_path, index=False)
logger.info("Compressing files")
logger.info("Compressing csv files")
files_to_compress = [
csv_path,
codebook_path,
]
zip_files(csv_zip_path, files_to_compress)
logger.info("Compressing xls files")
files_to_compress = [
excel_path,
codebook_path,
pdf_path,
]
zip_files(zip_path, files_to_compress)
zip_files(xls_zip_path, files_to_compress)
def load(self) -> None:
self._load_score_csv_full(

2
data/data-pipeline/data_pipeline/etl/score/tests/conftest.py
View file

@ -73,7 +73,7 @@ def score_data_initial(sample_data_dir):
@pytest.fixture()
def score_pdf_initial(sample_data_dir):
return sample_data_dir / "Draft_Communities_List.pdf"
return sample_data_dir / "draft_communities_list.pdf"
@pytest.fixture()

3
data/data-pipeline/data_pipeline/etl/score/tests/test_score_post.py
View file

@ -141,4 +141,5 @@ def test_load_downloadable_zip(etl, monkeypatch, score_data_expected):
assert constants.SCORE_DOWNLOADABLE_DIR.is_dir()
assert constants.SCORE_DOWNLOADABLE_CSV_FILE_PATH.is_file()
assert constants.SCORE_DOWNLOADABLE_EXCEL_FILE_PATH.is_file()
assert constants.SCORE_DOWNLOADABLE_ZIP_FILE_PATH.is_file()
assert constants.SCORE_DOWNLOADABLE_CSV_ZIP_FILE_PATH.is_file()
assert constants.SCORE_DOWNLOADABLE_XLS_ZIP_FILE_PATH.is_file()

0
data/data-pipeline/data_pipeline/files/Draft_Communities_List.pdf → data/data-pipeline/data_pipeline/files/draft_communities_list.pdf
View file

4
data/data-pipeline/data_pipeline/tests/sources/example/test_etl.py
View file

@ -172,7 +172,6 @@ class TestETL:
"""
# Setup
etl = self._get_instance_of_etl_class()
etl.__init__()
data_path, tmp_path = mock_paths
assert etl.DATA_PATH == data_path
@ -200,7 +199,6 @@ class TestETL:
etl = self._get_instance_of_etl_class()
data_path, tmp_path = mock_paths
etl.__init__()
actual_file_path = etl._get_output_file_path()
expected_file_path = data_path / "dataset" / etl.NAME / "usa.csv"
@ -319,7 +317,6 @@ class TestETL:
etl = self._setup_etl_instance_and_run_extract(
mock_etl=mock_etl, mock_paths=mock_paths
)
etl.__init__()
etl.transform()
assert etl.output_df is not None
@ -337,7 +334,6 @@ class TestETL:
"""
# setup - input variables
etl = self._get_instance_of_etl_class()
etl.__init__()
# setup - mock transform step
df_transform = pd.read_csv(

18
data/data-pipeline/poetry.lock generated
View file

@ -1537,6 +1537,21 @@ category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[[package]]
name = "pydantic"
version = "1.10.2"
description = "Data validation and settings management using python type hints"
category = "main"
optional = false
python-versions = ">=3.6.1"
[package.dependencies]
typing-extensions = ">=3.7.4.3"
[package.extras]
dotenv = ["python-dotenv (>=0.10.4)"]
email = ["email-validator (>=1.0.3)"]
[[package]]
name = "pyflakes"
version = "2.3.1"
@ -2192,7 +2207,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-
[metadata]
lock-version = "1.1"
python-versions = "^3.8"
content-hash = "5d55ed06ec57aa8b2fe957ff8a5cfd557f310f8fe65377ffd2d5ec3545f2ec82"
content-hash = "f3d61a8c4ca54c580ba8d459aa76a9d956e046c2b8339602497ce53393ba9983"
[metadata.files]
ansiwrap = []
@ -2547,6 +2562,7 @@ pycparser = [
{file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"},
{file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"},
]
pydantic = []
pyflakes = []
pygments = []
pylint = []

1
data/data-pipeline/pyproject.toml
View file

@ -39,6 +39,7 @@ tqdm = "4.62.0"
types-requests = "^2.25.0"
us = "^2.0.2"
xlsxwriter = "^2.0.0"
pydantic = "^1.9.0"
Rtree = "^1.0.0"
[tool.poetry.dev-dependencies]