Merge branch 'dev' into 'master'

Dev

See merge request swain-lab/python-pipeline/post-processing!1

Former-commit-id: 68bdd63a8b485c0818aaaa8435a0e504454fb80b

core/processes/base.py

-from pathlib import Path, PosixPath
-from typing import Union
+from agora.base import ParametersABC, ProcessABC
\ No newline at end of file
-from abc import ABC, abstractmethod
-
-from yaml import safe_load, dump
-
-
-class ParametersABC(ABC):
-    """
-    Base class to add yaml functionality to parameters
-
-    """
-
-    def to_dict(self):
-        return self.__dict__
-
-    @classmethod
-    def from_dict(cls, d: dict):
-        return cls(**d)
-
-    def to_yaml(self, path: Union[PosixPath, str] = None):
-        if path:
-            with open(Path(path), "w") as f:
-                dump(self.to_dict(), f)
-        return dump(self.to_dict())
-
-    @classmethod
-    def from_yaml(cls, path: Union[PosixPath, str]):
-        with open(Path(file)) as f:
-            params = safe_load(f)
-        return cls(**params)
-
-    @classmethod
-    @abstractmethod
-    def default(cls):
-        pass
-
-
-class ProcessABC(ABC):
-    "Base class for processes"
-
-    def __init__(self, parameters):
-        self._parameters = parameters
-
-        for k, v in parameters.to_dict().items():  # access parameters directly
-            setattr(self, k, v)
-
-    @property
-    def parameters(self):
-        return self._parameters
-
-    @abstractmethod
-    def run(self):
-        pass

core/processes/dsignal.py

 import pandas as pd
 
 from postprocessor.core.processes.base import ParametersABC, ProcessABC
-from postprocessor.core.functions.tracks import clean_tracks, merge_tracks, join_tracks
 
 
 class dsignalParameters(ParametersABC):

core/processes/gpsignal.py

+"""Gaussian process fit of a Signal."""
+import logging
+
+from postprocessor.core.processes.base import ParametersABC, ProcessABC
+import numpy as np
+import pandas as pd
+
+import gaussian_processes.gaussianprocess as gp
+
+def estimate_gr(volume, dt, noruns, bounds, verbose):
+    """
+    Parameters
+    ----------
+
+    volume : pd.Series
+        The volume series of a given cell
+    dt : float
+        The time interval in hours
+    noruns : int
+        The number of runs for optimisation
+    bounds : dict
+        The hyperparameter bounds used for optimisation
+    verbose : bool
+        If True, prints results
+
+    Returns
+    -------
+    """
+    volume = volume.values
+    n = len(volume)
+    idx = np.arange(n)
+    t = idx * dt
+    y = volume[volume > 0]
+    x = t[volume > 0]
+    idx = idx[volume > 0]
+    # Fit the GP
+    mg = gp.maternGP(bounds, x, y)
+    mg.findhyperparameters(noruns=noruns)
+    if verbose:
+        mg.results()  # Prints out the hyperparameters
+    mg.predict(x, derivs=2)  # Saves the predictions to object
+    # Save the predictions to a csv file so they can be reused
+    results = dict(time=mg.x, volume=mg.y, fit_time=mg.xnew, fit_volume=mg.f,
+                   growth_rate=mg.df, d_growth_rate=mg.ddf,
+                   volume_var=mg.fvar, growth_rate_var=mg.dfvar,
+                   d_growth_rate_var=mg.ddfvar)
+    for name, value in results.items():
+        results[name] = np.full((n, ), np.nan)
+        results[name][idx] = value
+    return results
+
+# Give that to a writer: NOTE the writer needs to learn how to write the
+# output of a process that returns multiple signals like this one does.
+
+class gpParameters(ParametersABC):
+    default_dict = dict(dt=5,
+             noruns=5,
+             bounds={0: (-2, 3),
+                     1: (-2, 1),
+                     2: (-4, -1)},
+             verbose=False)
+    def __init__(self, dt, noruns, bounds, verbose):
+        """
+        Parameters
+        ----------
+            dt : float
+                The time step between time points, in minutes
+            noruns : int
+                The number of times the optimisation is tried
+            bounds : dict
+                Hyperparameter bounds for the Matern Kernel
+            verbose : bool
+                Determines whether to print hyperparameter results
+        """
+        self.dt = dt
+        self.noruns = noruns
+        self.bounds = bounds
+        self.verbose = verbose
+
+    @classmethod
+    def default(cls):
+        return cls.from_dict(cls.default_dict)
+
+
+class GPSignal(ProcessABC):
+    """Gaussian process fit of a Signal.
+    """
+    def __init__(self, parameters: gpParameters):
+        super().__init__(parameters)
+
+    def run(self, signal: pd.DataFrame):
+        results = signal.apply(lambda x: estimate_gr(x,
+                                                     **self.parameters.to_dict()),
+                               result_type='expand').T
+        multi_signal = {name: pd.DataFrame(np.vstack(results[name]))
+                        for name in results.columns}
+        return multi_signal

core/processes/merger.py

-from postprocessor.core.processes.base import ParametersABC, ProcessABC
+from agora.base import ParametersABC, ProcessABC
 from postprocessor.core.functions.tracks import get_joinable
 
 

core/processes/picker.py

@@ -6,7 +6,7 @@ import pandas as pd
 
 from core.cells import CellsHDF
 
-from postprocessor.core.processes.base import ParametersABC, ProcessABC
+from agora.base import ParametersABC, ProcessABC
 from postprocessor.core.functions.tracks import max_ntps, max_nonstop_ntps
 
 
@@ -106,7 +106,7 @@ class picker(ProcessABC):
     ):
         threshold_asint = _as_int(threshold, signals.shape[1])
         case_mgr = {
-            "present": signals.apply(max_ntps, axis=1) > threshold_asint,
+            "present": signals.notna().sum(axis=1) > threshold_asint,
             "nonstoply_present": signals.apply(max_nonstop_ntps, axis=1)
             > threshold_asint,
             "quantile": [np.quantile(signals.values[signals.notna()], threshold)],

core/processor.py

@@ -5,13 +5,13 @@ from pydoc import locate
 import numpy as np
 import pandas as pd
 
-from postprocessor.core.processes.base import ParametersABC
-from postprocessor.core.processes.merger import mergerParameters, merger
-from postprocessor.core.processes.picker import pickerParameters, picker
+from agora.base import ParametersABC
 from core.io.writer import Writer
 from core.io.signal import Signal
 
 from core.cells import Cells
+from postprocessor.core.processes.merger import mergerParameters, merger
+from postprocessor.core.processes.picker import pickerParameters, picker
 
 
 class PostProcessorParameters(ParametersABC):
@@ -165,9 +165,17 @@ class PostProcessor:
                 else:
                     raise ("Outpath not defined", type(dataset))
 
-                self.write_result(
-                    "/postprocessing/" + process + "/" + outpath, result, metadata={}
-                )
+                if isinstance(result, dict): # Multiple Signals as output
+                    for k, v in result:
+                        self.write_result(
+                            "/postprocessing/" + process + "/" + outpath +
+                            f'/{k}',
+                            v, metadata={}
+                        )
+                else:
+                    self.write_result(
+                        "/postprocessing/" + process + "/" + outpath, result, metadata={}
+                    )
 
     def write_result(
         self, path: str, result: Union[List, pd.DataFrame, np.ndarray], metadata: Dict