Issue 675 & 676: Adding life expectancy and DOE energy burden data (#683)

* Adding two new data sources.

data/data-pipeline/data_pipeline/etl/sources/cdc_life_expectancy/etl.py

@@ -0,0 +1,69 @@
+from pathlib import Path
+import pandas as pd
+
+from data_pipeline.etl.base import ExtractTransformLoad
+from data_pipeline.utils import get_module_logger, download_file_from_url
+
+logger = get_module_logger(__name__)
+
+
+class CDCLifeExpectancy(ExtractTransformLoad):
+    def __init__(self):
+        self.FILE_URL: str = "https://ftp.cdc.gov/pub/Health_Statistics/NCHS/Datasets/NVSS/USALEEP/CSV/US_A.CSV"
+        self.OUTPUT_PATH: Path = (
+            self.DATA_PATH / "dataset" / "cdc_life_expectancy"
+        )
+
+        self.TRACT_INPUT_COLUMN_NAME = "Tract ID"
+        self.LIFE_EXPECTANCY_FIELD_NAME = "Life expectancy (years)"
+
+        # Constants for output
+        self.COLUMNS_TO_KEEP = [
+            self.GEOID_TRACT_FIELD_NAME,
+            self.LIFE_EXPECTANCY_FIELD_NAME,
+        ]
+
+        self.raw_df: pd.DataFrame
+        self.output_df: pd.DataFrame
+
+    def extract(self) -> None:
+        logger.info("Starting data download.")
+
+        download_file_name = self.TMP_PATH / "cdc_life_expectancy" / "usa.csv"
+        download_file_from_url(
+            file_url=self.FILE_URL,
+            download_file_name=download_file_name,
+            verify=True,
+        )
+
+        self.raw_df = pd.read_csv(
+            filepath_or_buffer=download_file_name,
+            dtype={
+            # The following need to remain as strings for all of their digits, not get converted to numbers.
+                self.TRACT_INPUT_COLUMN_NAME: "string",
+            },
+            low_memory=False,
+        )
+
+    def transform(self) -> None:
+        logger.info("Starting DOE energy burden transform.")
+
+        self.output_df = self.raw_df.rename(
+            columns={
+                "e(0)": self.LIFE_EXPECTANCY_FIELD_NAME,
+                self.TRACT_INPUT_COLUMN_NAME: self.GEOID_TRACT_FIELD_NAME,
+            }
+        )
+
+    def validate(self) -> None:
+        logger.info("Validating CDC Life Expectancy Data")
+
+        pass
+
+    def load(self) -> None:
+        logger.info("Saving CDC Life Expectancy CSV")
+
+        self.OUTPUT_PATH.mkdir(parents=True, exist_ok=True)
+        self.output_df[self.COLUMNS_TO_KEEP].to_csv(
+            path_or_buf=self.OUTPUT_PATH / "usa.csv", index=False
+        )

data/data-pipeline/data_pipeline/etl/sources/census_acs/etl.py

@@ -100,7 +100,9 @@ class CensusACSETL(ExtractTransformLoad):
         ]
 
         # Handle null values for CBG median income, which are `-666666666`.
-        missing_value_count = sum(self.df[self.MEDIAN_INCOME_FIELD_NAME]==-666666666)
+        missing_value_count = sum(
+            self.df[self.MEDIAN_INCOME_FIELD_NAME] == -666666666
+        )
         logger.info(
             f"There are {missing_value_count} ({int(100*missing_value_count/self.df[self.MEDIAN_INCOME_FIELD_NAME].count())}%) values of "
             + f"`{self.MEDIAN_INCOME_FIELD_NAME}` being marked as null values."

data/data-pipeline/data_pipeline/etl/sources/doe_energy_burden/etl.py

@@ -0,0 +1,86 @@
+from pathlib import Path
+import pandas as pd
+
+from data_pipeline.config import settings
+from data_pipeline.etl.base import ExtractTransformLoad
+from data_pipeline.utils import get_module_logger, unzip_file_from_url
+
+logger = get_module_logger(__name__)
+
+
+class DOEEnergyBurden(ExtractTransformLoad):
+    def __init__(self):
+        self.DOE_FILE_URL = (
+            settings.AWS_JUSTICE40_DATASOURCES_URL
+            + "/DOE_LEAD_with_EJSCREEN.csv.zip"
+        )
+
+        self.OUTPUT_PATH: Path = (
+            self.DATA_PATH / "dataset" / "doe_energy_burden"
+        )
+
+        self.TRACT_INPUT_COLUMN_NAME = "GEOID"
+        self.ENERGY_BURDEN_FIELD_NAME = "Energy burden"
+
+        # Constants for output
+        self.COLUMNS_TO_KEEP = [
+            self.GEOID_TRACT_FIELD_NAME,
+            self.ENERGY_BURDEN_FIELD_NAME,
+        ]
+
+        self.raw_df: pd.DataFrame
+        self.output_df: pd.DataFrame
+
+    def extract(self) -> None:
+        logger.info("Starting data download.")
+
+        unzip_file_from_url(
+            file_url=self.DOE_FILE_URL,
+            download_path=self.TMP_PATH,
+            unzipped_file_path=self.TMP_PATH / "doe_energy_burden",
+        )
+
+        self.raw_df = pd.read_csv(
+            filepath_or_buffer=self.TMP_PATH
+            / "doe_energy_burden"
+            / "DOE_LEAD_with_EJSCREEN.csv",
+            # The following need to remain as strings for all of their digits, not get converted to numbers.
+            dtype={
+                self.TRACT_INPUT_COLUMN_NAME: "string",
+            },
+            low_memory=False,
+        )
+
+    def transform(self) -> None:
+        logger.info("Starting transforms.")
+
+        output_df = self.raw_df.rename(
+            columns={
+                "AvgEnergyBurden": self.ENERGY_BURDEN_FIELD_NAME,
+                self.TRACT_INPUT_COLUMN_NAME: self.GEOID_TRACT_FIELD_NAME,
+            }
+        )
+
+        # Convert energy burden to a fraction, since we represent all other percentages as fractions.
+        output_df[self.ENERGY_BURDEN_FIELD_NAME] = output_df[self.ENERGY_BURDEN_FIELD_NAME] / 100
+
+        # Left-pad the tracts with 0s
+        expected_length_of_census_tract_field = 11
+        output_df[self.GEOID_TRACT_FIELD_NAME] = output_df[self.GEOID_TRACT_FIELD_NAME].astype(str).apply(
+            lambda x: x.zfill(expected_length_of_census_tract_field)
+        )
+
+        self.output_df = output_df
+
+    def validate(self) -> None:
+        logger.info("Validating DOE Energy Burden Data")
+
+        pass
+
+    def load(self) -> None:
+        logger.info("Saving DOE Energy Burden CSV")
+
+        self.OUTPUT_PATH.mkdir(parents=True, exist_ok=True)
+        self.output_df[self.COLUMNS_TO_KEEP].to_csv(
+            path_or_buf=self.OUTPUT_PATH / "usa.csv", index=False
+        )