mirror of
https://github.com/DOI-DO/j40-cejst-2.git
synced 2025-08-04 02:04:18 -07:00
Improve stability by limiting multithreading when running ETL
This commit is contained in:
parent
d4898b8f55
commit
6f3432d48a
2 changed files with 18 additions and 5 deletions
|
@ -139,9 +139,14 @@ def pull_census_data(data_source: str):
|
|||
@cli.command(
|
||||
help="Run all ETL processes or a specific one",
|
||||
)
|
||||
@click.option(
|
||||
"--no-concurrency",
|
||||
is_flag=True,
|
||||
help="Run ETLs sequentially instead of concurrently.",
|
||||
)
|
||||
@dataset_option
|
||||
@use_cache_option
|
||||
def etl_run(dataset: str, use_cache: bool):
|
||||
def etl_run(dataset: str, use_cache: bool, no_concurrency: bool):
|
||||
"""Run a specific or all ETL processes
|
||||
|
||||
Args:
|
||||
|
@ -153,7 +158,7 @@ def etl_run(dataset: str, use_cache: bool):
|
|||
log_title("Run ETL")
|
||||
|
||||
log_info("Running dataset(s)")
|
||||
etl_runner(dataset, use_cache)
|
||||
etl_runner(dataset, use_cache, no_concurrency)
|
||||
|
||||
log_goodbye()
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue