j40-cejst-2/docs/architecture/geodata-pipeline-arch.mmd

graph TD
    Dataset1["Dataset 1"]-->ETL1
    Dataset2["Dataset 2"]-->ETL2
    subgraph "ETL and Score Generation"
        ETL1["ETL for Dataset 1"]-->ncsv1("Normalized CSV (S3)")
        ETL2["ETL for Dataset 2"]-->ncsv2("Normalized CSV (S3)")
        ncsv1-->Score
        ncsv2-->Score
        Score-->DL("Downloadable zip")
        Score["Generate Score (score-run)"]-->CSV
    end
    DL-->Client

    Census["Census TIGER Data Shapefiles (hosted by Census)"]-->CGTiger
    subgraph "Census Data ETL"
        CGTiger["Create GeoJSON from Shapefile with ogr2ogr"]-->TS3
        TS3("TIGER GeoJSON State Files(S3)")-->CombineCensus["Combine Census State Files with Geopandas"]
        CombineCensus-->NCS3("National Census GeoJSON (S3)")
    end
    
    CSV("Full CSV (S3)")-->CGJ
    NCS3-->CGJ
    CGJ["Combine with ogr2ogr + Create GeoJSON (score-geo)"]-->GeoJSON
    GeoJSON("GeoJSON files (high and low zoom) (S3)")-->Tip
    Tip["Create and Send Tiles using Tippecanoe"]-->Uncompressed
    Tip-->Compressed
    subgraph production
        Uncompressed("Uncompressed MVT high and low directories (S3)")
    end
    subgraph development
        Local("Locally stored tiles")--"Option 1"-->TS
        Compressed("Compressed high and low .mbtile files (S3)")--"Option 2"-->TS[/Tileserver-GL/]
    end
    
    TS--"XYZ URL"-->Client
    Uncompressed--"XYZ URL"-->Client["Gatsby+MapLibre"]