publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-08-03 03:14:18 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Changes to allow local runs

Browse source
This commit is contained in:
Carlos Felix 2024-11-05 11:31:54 -05:00 • committed by Carlos Felix
commit ff9e7b9aa2
11 changed files with 3231 additions and 1867 deletions
Download patch file Download diff file Expand all files Collapse all files

5
data/data-pipeline/data_pipeline/etl/downloader.py
View file

@ -6,7 +6,9 @@ import shutil
from pathlib import Path
from data_pipeline.config import settings
from data_pipeline.utils import get_module_logger
logger = get_module_logger(__name__)
class Downloader:
"""A simple class to encapsulate the download capabilities of the application"""
@ -34,12 +36,13 @@ class Downloader:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
download_file_name.parent.mkdir(parents=True, exist_ok=True)
logger.debug(f"Downloading {file_url}")
response = requests.get(
file_url, verify=verify, timeout=settings.REQUESTS_DEFAULT_TIMOUT
)
if response.status_code == 200:
file_contents = response.content
logger.debug("Downloaded.")
else:
raise Exception(
f"HTTP response {response.status_code} from url {file_url}. Info: {response.content}"