Data directory should adopt standard Poetry-suggested python package structure (#457)

* Fixes #456 - Our data directory should adopt standard python package structure
* a few missed references
* updating readme
* updating requirements
* Running Black
* Fixes for flake8
* updating pylint

data/data-pipeline/data_pipeline/etl/sources/census/etl.py

@@ -0,0 +1,124 @@
+import csv
+import json
+import os
+from pathlib import Path
+
+import geopandas as gpd
+from data_pipeline.utils import get_module_logger, unzip_file_from_url
+
+from .etl_utils import get_state_fips_codes
+
+logger = get_module_logger(__name__)
+
+
+def download_census_csvs(data_path: Path) -> None:
+    """Download all census shape files from the Census FTP and extract the geojson
+    to generate national and by state Census Block Group CSVs and GeoJSONs
+
+    Args:
+        data_path (pathlib.Path): Name of the directory where the files and directories will
+        be created
+
+    Returns:
+        None
+    """
+
+    # the fips_states_2010.csv is generated from data here
+    # https://www.census.gov/geographies/reference-files/time-series/geo/tallies.html
+    state_fips_codes = get_state_fips_codes(data_path)
+    geojson_dir_path = data_path / "census" / "geojson"
+
+    for fips in state_fips_codes:
+        # check if file exists
+        shp_file_path = data_path / "census" / "shp" / fips / f"tl_2010_{fips}_bg10.shp"
+
+        logger.info(f"Checking if {fips} file exists")
+        if not os.path.isfile(shp_file_path):
+            logger.info(f"Downloading and extracting {fips} shape file")
+            # 2020 tiger data is here: https://www2.census.gov/geo/tiger/TIGER2020/BG/
+            # But using 2010 for now
+            cbg_state_url = f"https://www2.census.gov/geo/tiger/TIGER2010/BG/2010/tl_2010_{fips}_bg10.zip"
+            unzip_file_from_url(
+                cbg_state_url,
+                data_path / "tmp",
+                data_path / "census" / "shp" / fips,
+            )
+
+            cmd = (
+                "ogr2ogr -f GeoJSON data/census/geojson/"
+                + fips
+                + ".json data/census/shp/"
+                + fips
+                + "/tl_2010_"
+                + fips
+                + "_bg10.shp"
+            )
+            os.system(cmd)
+
+    # generate CBG CSV table for pandas
+    ## load in memory
+    cbg_national = []  # in-memory global list
+    cbg_per_state: dict = {}  # in-memory dict per state
+    for file in os.listdir(geojson_dir_path):
+        if file.endswith(".json"):
+            logger.info(f"Ingesting geoid10 for file {file}")
+            with open(geojson_dir_path / file) as f:
+                geojson = json.load(f)
+                for feature in geojson["features"]:
+                    geoid10 = feature["properties"]["GEOID10"]
+                    cbg_national.append(str(geoid10))
+                    geoid10_state_id = geoid10[:2]
+                    if not cbg_per_state.get(geoid10_state_id):
+                        cbg_per_state[geoid10_state_id] = []
+                    cbg_per_state[geoid10_state_id].append(geoid10)
+
+    csv_dir_path = data_path / "census" / "csv"
+    ## write to individual state csv
+    for state_id in cbg_per_state:
+        geoid10_list = cbg_per_state[state_id]
+        with open(
+            csv_dir_path / f"{state_id}.csv", mode="w", newline=""
+        ) as cbg_csv_file:
+            cbg_csv_file_writer = csv.writer(
+                cbg_csv_file,
+                delimiter=",",
+                quotechar='"',
+                quoting=csv.QUOTE_MINIMAL,
+            )
+
+            for geoid10 in geoid10_list:
+                cbg_csv_file_writer.writerow(
+                    [
+                        geoid10,
+                    ]
+                )
+
+    ## write US csv
+    with open(csv_dir_path / "us.csv", mode="w", newline="") as cbg_csv_file:
+        cbg_csv_file_writer = csv.writer(
+            cbg_csv_file,
+            delimiter=",",
+            quotechar='"',
+            quoting=csv.QUOTE_MINIMAL,
+        )
+        for geoid10 in cbg_national:
+            cbg_csv_file_writer.writerow(
+                [
+                    geoid10,
+                ]
+            )
+
+    ## create national geojson
+    logger.info("Generating national geojson file")
+    usa_df = gpd.GeoDataFrame()
+
+    for file_name in geojson_dir_path.rglob("*.json"):
+        logger.info(f"Ingesting {file_name}")
+        state_gdf = gpd.read_file(file_name)
+        usa_df = usa_df.append(state_gdf)
+
+    usa_df = usa_df.to_crs("+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs")
+    logger.info("Writing national geojson file")
+    usa_df.to_file(geojson_dir_path / "us.json", driver="GeoJSON")
+
+    logger.info("Census block groups downloading complete")

data/data-pipeline/data_pipeline/etl/sources/census/etl_utils.py

@@ -0,0 +1,71 @@
+import csv
+import os
+from pathlib import Path
+
+import pandas as pd
+from data_pipeline.config import settings
+from data_pipeline.utils import (
+    get_module_logger,
+    remove_all_dirs_from_dir,
+    remove_files_from_dir,
+    unzip_file_from_url,
+)
+
+logger = get_module_logger(__name__)
+
+
+def reset_data_directories(data_path: Path) -> None:
+    census_data_path = data_path / "census"
+
+    # csv
+    csv_path = census_data_path / "csv"
+    remove_files_from_dir(csv_path, ".csv")
+
+    # geojson
+    geojson_path = census_data_path / "geojson"
+    remove_files_from_dir(geojson_path, ".json")
+
+    # shp
+    shp_path = census_data_path / "shp"
+    remove_all_dirs_from_dir(shp_path)
+
+
+def get_state_fips_codes(data_path: Path) -> list:
+    fips_csv_path = data_path / "census" / "csv" / "fips_states_2010.csv"
+
+    # check if file exists
+    if not os.path.isfile(fips_csv_path):
+        logger.info("Downloading fips from S3 repository")
+        unzip_file_from_url(
+            settings.AWS_JUSTICE40_DATASOURCES_URL + "/fips_states_2010.zip",
+            data_path / "tmp",
+            data_path / "census" / "csv",
+        )
+
+    fips_state_list = []
+    with open(fips_csv_path) as csv_file:
+        csv_reader = csv.reader(csv_file, delimiter=",")
+        line_count = 0
+
+        for row in csv_reader:
+            if line_count == 0:
+                line_count += 1
+            else:
+                fips = row[0].strip()
+                fips_state_list.append(fips)
+    return fips_state_list
+
+
+def get_state_information(data_path: Path) -> pd.DataFrame:
+    """Load the full state file as a dataframe.
+
+    Useful because of the state regional information.
+    """
+    fips_csv_path = data_path / "census" / "csv" / "fips_states_2010.csv"
+
+    df = pd.read_csv(fips_csv_path)
+
+    # Left pad the FIPS codes with 0s
+    df["fips"] = df["fips"].astype(str).apply(lambda x: x.zfill(2))
+
+    return df