Add ability to cache ETL data sources (#2169)

* Add a rough prototype allowing a developer to pre-download data sources for all ETLs

* Update code to be more production-ish

* Move fetch to Extract part of ETL
* Create a downloader to house all downloading operations
* Remove unnecessary "name" in data source

* Format source files with black

* Fix issues from pylint and get the tests working with the new folder structure

* Clean up files with black

* Fix unzip test

* Add caching notes to README

* Fix tests (linting and case sensitivity bug)

* Address PR comments and add API keys for census where missing

* Merging comparator changes from main into this branch for the sake of the PR

* Add note on using cache (-u) during pipeline

data/data-pipeline/data_pipeline/etl/sources/persistent_poverty/etl.py

@@ -3,9 +3,10 @@ import functools
 import pandas as pd
 from data_pipeline.config import settings
 from data_pipeline.etl.base import ExtractTransformLoad
+from data_pipeline.etl.datasource import DataSource
+from data_pipeline.etl.datasource import ZIPDataSource
 from data_pipeline.etl.base import ValidGeoLevel
 from data_pipeline.utils import get_module_logger
-from data_pipeline.utils import unzip_file_from_url
 
 logger = get_module_logger(__name__)
 
@@ -23,6 +24,26 @@ class PersistentPovertyETL(ExtractTransformLoad):
     PUERTO_RICO_EXPECTED_IN_DATA = False
 
     def __init__(self):
+
+        # fetch
+        self.poverty_url = (
+            settings.AWS_JUSTICE40_DATASOURCES_URL + "/LTDB_Std_All_Sample.zip"
+        )
+
+        # source
+        self.poverty_sources = [
+            self.get_sources_path()
+            / "ltdb_std_all_sample"
+            / "ltdb_std_1990_sample.csv",
+            self.get_sources_path()
+            / "ltdb_std_all_sample"
+            / "ltdb_std_2000_sample.csv",
+            self.get_sources_path()
+            / "ltdb_std_all_sample"
+            / "ltdb_std_2010_sample.csv",
+        ]
+
+        # output
         self.OUTPUT_PATH = self.DATA_PATH / "dataset" / "persistent_poverty"
 
         # Need to change hyperlink to S3
@@ -44,6 +65,13 @@ class PersistentPovertyETL(ExtractTransformLoad):
 
         self.df: pd.DataFrame
 
+    def get_data_sources(self) -> [DataSource]:
+        return [
+            ZIPDataSource(
+                source=self.poverty_url, destination=self.get_sources_path()
+            )
+        ]
+
     def _join_input_dfs(self, dfs: list) -> pd.DataFrame:
         df = functools.reduce(
             lambda df_a, df_b: pd.merge(
@@ -75,28 +103,17 @@ class PersistentPovertyETL(ExtractTransformLoad):
 
         return df
 
-    def extract(self) -> None:
-        unzipped_file_path = self.get_tmp_path()
+    def extract(self, use_cached_data_sources: bool = False) -> None:
 
-        unzip_file_from_url(
-            file_url=settings.AWS_JUSTICE40_DATASOURCES_URL
-            + "/LTDB_Std_All_Sample.zip",
-            download_path=self.get_tmp_path(),
-            unzipped_file_path=unzipped_file_path,
-        )
-
-        file_names = [
-            "ltdb_std_1990_sample.csv",
-            "ltdb_std_2000_sample.csv",
-            "ltdb_std_2010_sample.csv",
-        ]
+        super().extract(
+            use_cached_data_sources
+        )  # download and extract data sources
 
         temporary_input_dfs = []
 
-        for file_name in file_names:
+        for file_name in self.poverty_sources:
             temporary_input_df = pd.read_csv(
-                filepath_or_buffer=unzipped_file_path
-                / f"ltdb_std_all_sample/{file_name}",
+                filepath_or_buffer=file_name,
                 dtype={
                     self.GEOID_TRACT_INPUT_FIELD_NAME_1: "string",
                     self.GEOID_TRACT_INPUT_FIELD_NAME_2: "string",