publicdata/j40-cejst-2
1
0
Fork 0
mirror of https://github.com/DOI-DO/j40-cejst-2.git synced 2025-07-29 01:41:16 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Adding eamlis and fuds data to legacy pollution in score (#1832)

Browse source 
Update to add EAMLIS and FUDS data to score
This commit is contained in:
Emma Nechamkin 2022-08-18 13:32:29 -04:00 • committed by GitHub
commit cb4866b93f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 93 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

28
data/data-pipeline/data_pipeline/etl/runner.py
View file

@ -93,21 +93,23 @@ def etl_runner(dataset_to_run: str = None) -> None:
dataset for dataset in dataset_list if dataset["is_memory_intensive"]
]
logger.info("Running concurrent jobs")
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = {
executor.submit(_run_one_dataset, dataset=dataset)
for dataset in concurrent_datasets
}
if concurrent_datasets:
logger.info("Running concurrent jobs")
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = {
executor.submit(_run_one_dataset, dataset=dataset)
for dataset in concurrent_datasets
}
for fut in concurrent.futures.as_completed(futures):
# Calling result will raise an exception if one occurred.
# Otherwise, the exceptions are silently ignored.
fut.result()
for fut in concurrent.futures.as_completed(futures):
# Calling result will raise an exception if one occurred.
# Otherwise, the exceptions are silently ignored.
fut.result()
logger.info("Running high-memory jobs")
for dataset in high_memory_datasets:
_run_one_dataset(dataset=dataset)
if high_memory_datasets:
logger.info("Running high-memory jobs")
for dataset in high_memory_datasets:
_run_one_dataset(dataset=dataset)
def score_generate() -> None: