From 473af2295e7c53e36556947c968e53ab903f4718 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Al=C3=A1n=20Mu=C3=B1oz?= <amuoz@ed.ac.uk>
Date: Thu, 3 Feb 2022 10:41:17 +0000
Subject: [PATCH] bugfix positions where real ntps < ntps in meta
---
aliby/pipeline.py | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/aliby/pipeline.py b/aliby/pipeline.py
index d6057dea..3231b8a2 100644
--- a/aliby/pipeline.py
+++ b/aliby/pipeline.py
@@ -300,7 +300,8 @@ class Pipeline(ProcessABC):
ext = Extractor.from_tiler(exparams, store=filename, tiler=tiler)
# RUN
- tps = general_config["tps"]
+ # Adjust tps based on how many tps are available on the server
+ tps = min(general_config["tps"], image.data.shape[0])
frac_clogged_traps = 0
for i in tqdm(
range(process_from, tps), desc=image.name, initial=process_from
@@ -359,6 +360,17 @@ class Pipeline(ProcessABC):
overwrite=swriter.datatypes.keys(),
)
meta.add_fields({"last_processed": i})
+
+ import pickle as pkl
+
+ with open(
+ Path(bwriter.file).parent / f"{i}_live_state.pkl", "wb"
+ ) as f:
+ pkl.dump(runner.crawler.tracker_states, f)
+ with open(
+ Path(bwriter.file).parent / f"{i}_read_state.pkl", "wb"
+ ) as f:
+ pkl.dump(StateReader(bwriter.file).get_formatted_states(), f)
# Run post processing
meta.add_fields({"end_status": "Success"})
--
GitLab