publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-08-16 13:51:39 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Data Pipeline performance improvements for Census GeoJson and Score file

Browse source
This commit is contained in:
Carlos Felix 2025-01-13 09:28:14 -05:00 • committed by Carlos Felix
commit c32bd1f363
37 changed files with 1305 additions and 1413 deletions
Download patch file Download diff file Expand all files Collapse all files

157
data/data-pipeline/.vscode/launch.json vendored
View file

@ -4,27 +4,9 @@
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Score Run",
"type": "python",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"score-run"
]
},
{
"name": "Generate Score Post",
"type": "python",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"generate-score-post"
]
},
{
"name": "Data Cleanup",
"type": "python",
"type": "debugpy",
"request": "launch",
"module": "data_pipeline.application",
"args": [
@ -33,7 +15,7 @@
},
{
"name": "Census Cleanup",
"type": "python",
"type": "debugpy",
"request": "launch",
"module": "data_pipeline.application",
"args": [
@ -42,73 +24,25 @@
},
{
"name": "Download Census",
"type": "python",
"type": "debugpy",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"census-data-download"
]
},
{
"name": "Score Full Run",
"type": "python",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"score-full-run"
]
},
{
"name": "Combine Score and GeoJSON",
"type": "python",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"geo-score"
]
},
{
"name": "Generate Score Tiles",
"type": "python",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"generate-map-tiles"
]
},
{
"name": "Generate Tribal Tiles",
"type": "python",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"generate-map-tiles",
"-t"
"census-data-download", "-u"
]
},
{
"name": "ETL Run",
"type": "python",
"type": "debugpy",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"etl-run"
]
},
{
"name": "ETL Run NRI",
"type": "python",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"etl-run",
"--dataset",
"national_risk_index"
]
},
{
"name": "ETL Run Tribal",
"type": "python",
"type": "debugpy",
"request": "launch",
"module": "data_pipeline.application",
"args": [
@ -117,18 +51,91 @@
"tribal"
]
},
{
"name": "Score Run",
"type": "debugpy",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"score-run"
]
},
{
"name": "Combine Score and GeoJSON",
"type": "debugpy",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"geo-score"
]
},
{
"name": "Generate Score Post",
"type": "debugpy",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"generate-score-post"
]
},
{
"name": "Generate Score Tiles",
"type": "debugpy",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"generate-map-tiles"
]
},
{
"name": "Generate Tribal Tiles",
"type": "debugpy",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"generate-map-tiles",
"-t"
]
},
{
"name": "Score Full Run",
"type": "debugpy",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"score-full-run"
]
},
{
"name": "Data Full Run",
"type": "python",
"type": "debugpy",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"data-full-run",
]
},
{
"name": "Comparator",
"type": "debugpy",
"request": "launch",
"module": "data_pipeline.comparator",
"args": [
"compare-score",
]
},
{
"name": "Convert score to CSV",
"type": "debugpy",
"request": "launch",
"module": "data_pipeline.application",
"args": [
"convert-score",
]
},
{
"name": "poetry install",
"type": "python",
"type": "debugpy",
"request": "launch",
"module": "poetry",
"args": [
@ -137,7 +144,7 @@
},
{
"name": "poetry update",
"type": "python",
"type": "debugpy",
"request": "launch",
"module": "poetry",
"args": [