publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-08-03 00:34:19 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Changes to allow local runs

Browse source
This commit is contained in:
Carlos Felix 2024-11-05 11:31:54 -05:00 • committed by Carlos Felix
commit ff9e7b9aa2
11 changed files with 3231 additions and 1867 deletions
Download patch file Download diff file Expand all files Collapse all files

5
data/data-pipeline/data_pipeline/etl/datasource.py
View file

@ -22,6 +22,7 @@ from pathlib import Path
from typing import List
from dataclasses import dataclass
from abc import ABC, abstractmethod
from data_pipeline.constants import NO_SSL_VERIFY
from data_pipeline.etl.downloader import Downloader
from data_pipeline.etl.sources.census_acs.etl_utils import (
@ -65,7 +66,7 @@ class FileDataSource(DataSource):
Downloader.download_file_from_url(
file_url=self.source,
download_file_name=self.destination,
verify=True,
verify=not NO_SSL_VERIFY,
)
def __str__(self):
@ -85,7 +86,7 @@ class ZIPDataSource(DataSource):
Downloader.download_zip_file_from_url(
file_url=self.source,
unzipped_file_path=self.destination,
verify=True,
verify=not NO_SSL_VERIFY,
)
def __str__(self):