From 7bd0b38243466f7a385e8743691756e535f36f30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Al=C3=A1n=20Mu=C3=B1oz?= <amuoz@ed.ac.uk>
Date: Wed, 16 Jun 2021 18:34:23 +0100
Subject: [PATCH] restructure
Former-commit-id: fa061950b491d4934f26f7c87d5a882dad81cf32
---
core/functions/test_hdf.py | 6 +--
core/io/writer.py | 9 +++--
core/processes/base.py | 6 +++
core/processes/picker.py | 2 +-
core/processor.py | 78 ++++++++++++++++----------------------
5 files changed, 48 insertions(+), 53 deletions(-)
diff --git a/core/functions/test_hdf.py b/core/functions/test_hdf.py
index 6d2f8d02..683c005b 100644
--- a/core/functions/test_hdf.py
+++ b/core/functions/test_hdf.py
@@ -10,8 +10,6 @@ f = h5py.File(
)
tracks = f["/extraction/general/None/area"][()]
cell = Cells.from_source("/home/alan/Documents/sync_docs/PhD/tmp/DO6MS2_003store.h5")
-from postprocessor.core.picker import Picker, PickerParameters
+from postprocessor.core.processes.picker import Picker, PickerParameters
-picker = Picker(
- signals=pd.DataFrame(tracks), cells=cell, parameters=PickerParameters.default()
-)
+picker = Picker(cells=cell, parameters=PickerParameters.default())
diff --git a/core/io/writer.py b/core/io/writer.py
index de1660e0..480c7e03 100644
--- a/core/io/writer.py
+++ b/core/io/writer.py
@@ -1,9 +1,12 @@
-import numpy as np
+import h5py
import pandas as pd
-def Writer(filename):
+from postprocessor.core.io.base import BridgeH5
+
+
+def Writer(BridgeH5):
def __init__(self, filename):
- self._file = h5py.File(filename)
+ self._hdf = h5py.File(filename, "a")
def write(self, address, data):
self._file.add_group(address)
diff --git a/core/processes/base.py b/core/processes/base.py
index a53cee6d..8412fdb5 100644
--- a/core/processes/base.py
+++ b/core/processes/base.py
@@ -35,9 +35,15 @@ class ProcessABC(ABC):
"Base class for processes"
def __init__(self, parameters):
+ self._parameters = parameters
+
for k, v in parameters.to_dict().items(): # access parameters directly
setattr(self, k, v)
+ @property
+ def parameters(self):
+ return self._parameters
+
@abstractmethod
def run(self):
pass
diff --git a/core/processes/picker.py b/core/processes/picker.py
index 26a0320b..9342ff17 100644
--- a/core/processes/picker.py
+++ b/core/processes/picker.py
@@ -6,7 +6,7 @@ import pandas as pd
from core.cells import CellsHDF
-from postprocessor.core.base import ParametersABC, ProcessABC
+from postprocessor.core.processes.base import ParametersABC, ProcessABC
from postprocessor.core.functions.tracks import max_ntps, max_nonstop_ntps
diff --git a/core/processor.py b/core/processor.py
index f1e006cf..9e433076 100644
--- a/core/processor.py
+++ b/core/processor.py
@@ -1,29 +1,54 @@
+from typing import List, Dict, Union
import pandas as pd
-from postprocessor.core.base import ParametersABC
-from core import Cells
+
+from postprocessor.core.processes.base import ParametersABC
+from postprocessor.core.processes.merger import MergerParameters, Merger
+from postprocessor.core.processes.picker import PickerParameters, Picker
+from postprocessor.core.io.writer import Writer
+from postprocessor.core.io.signal import Signal
+
+from core.cells import Cells
class PostProParameters(ParametersABC):
"""
Anthology of parameters used for postprocessing
+ :merger:
+ :picker: parameters for picker
+ :processes: List of processes that can be found in ./processes
+ :datasets: Dictionary
"""
- def __init__(self, merger=None, picker=None, processes=None, datasets=None):
- self.merger = merger
- self.picker = picker
- self.processes = processes
+ def __init__(self, merger=None, picker=None, processes=[], datasets=None):
+ self.merger: MergerParameters = merger
+ self.picker: PickerParameters = picker
+ self.processes: List = processes
- self.datasets = datasets
+ self.datasets: Dict = datasets
def __getitem__(self, item):
return getattr(self, item)
+ @classmethod
+ def default(cls, kind=None):
+ if kind == "defaults" or kind == None:
+ return cls(
+ merger=MergerParameters.default(),
+ picker=PickerParameters.default(),
+ datasets={
+ "merger": "/extraction/general/None/area",
+ "picker": "/extraction/general/None/area",
+ "processes": [],
+ },
+ )
+
class PostProcessor:
def __init__(self, fname, parameters, signals):
self.parameters = parameters
-
self._signals = Signals(fname)
+ self._writer = Writer(fname)
+
self.datasets = parameters["datasets"]
self.merger = Merger(parameters["merger"])
self.picker = Picker(
@@ -39,40 +64,3 @@ class PostProcessor:
for process, dataset in zip(self.processes, self.datasets["processes"]):
process_result = process.run(signals.get_dataset(dataset))
self.writer.write(process_result, dataset)
-
-
-class Signals:
- """
- Class that fetches data from the hdf5 storage for post-processing
- """
-
- def __init__(self, file):
- self._hdf = h5py.File(file, "r")
-
- def __getitem__(self, dataset):
- dset = self._hdf[dataset][()]
- attrs = self._hdf[dataset].attrs
- first_dataset = "/" + dataset.split("/")[0] + "/"
- timepoints = self._hdf[first_dataset].attrs["processed_timepoints"]
-
- if "cell_label" in self._hdf[dataset].attrs:
- ids = pd.MultiIndex.from_tuples(
- zip(attrs["trap"], attrs["cell_label"]), names=["trap", "cell_label"]
- )
- else:
- ids = pd.Index(attrs["trap"], names=["trap"])
-
- return pd.DataFrame(dset, index=ids, columns=timepoints)
-
- @staticmethod
- def _if_ext_or_post(name):
- if name.startswith("extraction") or name.startswith("postprocessing"):
- if len(name.split("/")) > 3:
- return name
-
- @property
- def datasets(self):
- return signals._hdf.visit(self._if_ext_or_post)
-
- def close(self):
- self._hdf.close()
--
GitLab