publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-07-28 05:51:17 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add retry logic to downloader

Browse source
This commit is contained in:
ericiwamoto 2024-12-26 08:35:22 -08:00 • committed by Carlos Felix
commit 7af92f575b
4 changed files with 19 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

9
data/data-pipeline/data_pipeline/etl/downloader.py
View file

@ -7,6 +7,7 @@ import shutil
from pathlib import Path
from data_pipeline.config import settings
from data_pipeline.utils import get_module_logger
from tenacity import retry, stop_after_attempt, wait_exponential
logger = get_module_logger(__name__)
@ -15,6 +16,10 @@ class Downloader:
"""A simple class to encapsulate the download capabilities of the application"""
@classmethod
@retry(
stop=stop_after_attempt(3),
wait=wait_exponential(multiplier=1, min=4, max=10),
)
def download_file_from_url(
cls,
file_url: str,
@ -58,6 +63,10 @@ class Downloader:
return download_file_name
@classmethod
@retry(
stop=stop_after_attempt(3),
wait=wait_exponential(multiplier=1, min=4, max=10),
)
def download_zip_file_from_url(
cls,
file_url: str,