publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-02-23 10:04:18 -08:00
Code Issues Projects Releases Packages Wiki Activity Actions

Moving from relative imports to absolute to enable poetry run python data-pipeline/application.py [command]

Browse source
This commit is contained in:
Nat Hillard 2021-08-06 11:12:19 -04:00
parent 45a8b1c026
commit 32a1278903
3 changed files with 14 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
data/data-pipeline/data_pipeline/application.py
View file

@ -1,10 +1,13 @@
import click import click
from .config import settings from data_pipeline.config import settings
from .etl.runner import etl_runner, score_generate, score_geo from data_pipeline.etl.runner import etl_runner, score_generate, score_geo
from .etl.sources.census.etl_utils import reset_data_directories as census_reset from data_pipeline.etl.sources.census.etl import download_census_csvs
from .tile.generate import generate_tiles from data_pipeline.etl.sources.census.etl_utils import (
from .utils import ( reset_data_directories as census_reset,
)
from data_pipeline.tile.generate import generate_tiles
from data_pipeline.utils import (
data_folder_cleanup, data_folder_cleanup,
get_module_logger, get_module_logger,
score_folder_cleanup, score_folder_cleanup,

2
data/data-pipeline/data_pipeline/etl/sources/census/etl.py
View file

@ -8,7 +8,7 @@ import geopandas as gpd
from data_pipeline.etl.base import ExtractTransformLoad from data_pipeline.etl.base import ExtractTransformLoad
from data_pipeline.utils import get_module_logger, unzip_file_from_url from data_pipeline.utils import get_module_logger, unzip_file_from_url
from .etl_utils import get_state_fips_codes from data_pipeline.etl.sources.census.etl_utils import get_state_fips_codes
logger = get_module_logger(__name__) logger = get_module_logger(__name__)

7
data/data-pipeline/data_pipeline/utils.py
View file

@ -7,7 +7,7 @@ from pathlib import Path
import requests import requests
import urllib3 import urllib3
from .config import settings from data_pipeline.config import settings
def get_module_logger(module_name: str) -> logging.Logger: def get_module_logger(module_name: str) -> logging.Logger:
@ -97,7 +97,10 @@ def remove_all_dirs_from_dir(dir_path: Path) -> None:
def unzip_file_from_url( def unzip_file_from_url(
file_url: str, download_path: Path, unzipped_file_path: Path, verify: bool = False, file_url: str,
download_path: Path,
unzipped_file_path: Path,
verify: bool = False,
) -> None: ) -> None:
"""Downloads a zip file from a remote URL location and unzips it in a specific directory, removing the temporary file after """Downloads a zip file from a remote URL location and unzips it in a specific directory, removing the temporary file after