From 5370f42124665d2a6ee2ee87c21d71122d64635d Mon Sep 17 00:00:00 2001
From: Arin Wongprommoon <arin.wongprommoon@ed.ac.uk>
Date: Fri, 30 Sep 2022 15:03:15 +0100
Subject: [PATCH] fix(postproc): groups signal not present in all HDFs in dir
WHY IS THIS CHANGE NEEDED?:
- if a signal is not present in all HDFs files in a directory, defining
a grouper object and then invoking
grouper.concat_signal('<name_of_signal>') throws an error
HOW DOES THE CHANGE SOLVE THE PROBLEM?:
- modify pool_function() so that a dictionary of signals is an argument.
this dictionary of signals indicates which files have the signal of interest.
- pass the dictionary of signals into concat_signal(), so that it knows
which ones to process (i.e. concatenate)
---
src/postprocessor/grouper.py | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/src/postprocessor/grouper.py b/src/postprocessor/grouper.py
index 7a18a244..1531c052 100644
--- a/src/postprocessor/grouper.py
+++ b/src/postprocessor/grouper.py
@@ -116,7 +116,12 @@ class Grouper(ABC):
)
signals = self.pool_function(
- path=path, f=concat_signal_ind, mode=mode, pool=pool, **kwargs
+ path=path,
+ f=concat_signal_ind,
+ mode=mode,
+ pool=pool,
+ signals=sitems,
+ **kwargs,
)
errors = [k for s, k in zip(signals, self.signals.keys()) if s is None]
@@ -196,15 +201,17 @@ class Grouper(ABC):
path: str,
f: t.Callable,
pool: t.Optional[int] = None,
+ signals: t.Dict[str, Signal] = None,
**kwargs,
):
"""
Wrapper to add support for threading to process independent signals.
Particularly useful when aggregating multiple elements.
"""
- if pool or pool is None:
- if pool is None:
- pool = 8
+ pool = pool or 8
+ signals = signals or self.signals
+
+ if pool:
with Pool(pool) as p:
kymographs = p.map(
@@ -214,7 +221,7 @@ class Grouper(ABC):
group=self.positions_groups[x[0]],
**kwargs,
),
- self.signals.items(),
+ signals.items(),
)
else:
kymographs = [
--
GitLab