From 4728053309f9ae40c199f694b7eb6795f8d87531 Mon Sep 17 00:00:00 2001
From: Diane Adjavon <diane.adjavon@ed.ac.uk>
Date: Thu, 12 Jan 2023 11:51:14 +0000
Subject: [PATCH] refactor: Remove unused code and obsolete tests
Relates to #52
---
examples/extraction/pos_example.py | 1 -
src/agora/io/bridge.py | 6 -
src/agora/io/signal.py | 18 --
src/agora/io/utils.py | 23 --
src/agora/io/writer.py | 229 ------------------
src/agora/utils/example.py | 53 ----
src/agora/utils/lineage.py | 57 -----
src/aliby/baby_client.py | 69 ------
src/aliby/haystack.py | 31 ---
src/aliby/io/omero.py | 5 +-
src/aliby/io/utils.py | 50 ----
src/aliby/pipeline.py | 32 ---
src/aliby/tile/tiler.py | 2 -
src/aliby/tile/traps.py | 100 +-------
src/aliby/utils/argo.py | 55 +----
src/aliby/utils/cache.py | 141 -----------
src/extraction/core/extractor.py | 1 +
src/extraction/core/functions/io.py | 12 -
src/extraction/core/functions/utils.py | 19 --
src/extraction/core/functions/versioning.py | 1 +
src/extraction/core/omero.py | 35 ---
src/logfile_parser/swainlab_parser.py | 17 +-
.../benchmarks/post_processing.py | 1 +
src/postprocessor/core/abc.py | 1 +
src/postprocessor/core/group.py | 137 -----------
src/postprocessor/core/lineageprocess.py | 9 +-
src/postprocessor/core/processor.py | 41 +---
.../core/reshapers/bud_metric.py | 1 -
src/postprocessor/core/reshapers/buddings.py | 3 +-
src/postprocessor/core/reshapers/merger.py | 7 +-
src/postprocessor/core/reshapers/picker.py | 26 +-
src/postprocessor/grouper.py | 5 -
tests/agora/example_test.py | 25 --
tests/aliby/network/test_baby_client.py | 91 -------
tests/aliby/network/test_post_processing.py | 8 -
35 files changed, 23 insertions(+), 1289 deletions(-)
delete mode 100644 src/agora/utils/example.py
delete mode 100644 src/aliby/io/utils.py
delete mode 100644 src/aliby/utils/cache.py
delete mode 100644 src/extraction/core/functions/io.py
delete mode 100644 src/extraction/core/functions/utils.py
delete mode 100644 src/extraction/core/omero.py
delete mode 100644 src/postprocessor/core/group.py
delete mode 100644 tests/agora/example_test.py
delete mode 100644 tests/aliby/network/test_baby_client.py
diff --git a/examples/extraction/pos_example.py b/examples/extraction/pos_example.py
index 9372fa1d..da44f464 100644
--- a/examples/extraction/pos_example.py
+++ b/examples/extraction/pos_example.py
@@ -14,5 +14,4 @@ params = Parameters(
ext = Extractor(params, omero_id=19310)
-# ext.extract_exp(tile_size=117)
d = ext.extract_tp(tp=1, tile_size=117)
diff --git a/src/agora/io/bridge.py b/src/agora/io/bridge.py
index 478408f3..9eb0c640 100644
--- a/src/agora/io/bridge.py
+++ b/src/agora/io/bridge.py
@@ -151,12 +151,6 @@ def attrs_from_h5(fpath: str):
return dict(f.attrs)
-def parameters_from_h5(fpath: str):
- """Return parameters from an h5 file."""
- attrs = attrs_from_h5(fpath)
- return yaml.safe_load(attrs["parameters"])
-
-
def image_creds_from_h5(fpath: str):
"""Return image id and server credentials from an h5."""
attrs = attrs_from_h5(fpath)
diff --git a/src/agora/io/signal.py b/src/agora/io/signal.py
index 4c6bf0f7..61568824 100644
--- a/src/agora/io/signal.py
+++ b/src/agora/io/signal.py
@@ -351,24 +351,6 @@ class Signal(BridgeH5):
def stem(self):
return self.filename.stem
- # def dataset_to_df(self, f: h5py.File, path: str):
-
- # all_indices = self.index_names
-
- # valid_indices = {
- # k: f[path][k][()] for k in all_indices if k in f[path].keys()
- # }
-
- # new_index = pd.MultiIndex.from_arrays(
- # list(valid_indices.values()), names=valid_indices.keys()
- # )
-
- # return pd.DataFrame(
- # f[path + "/values"][()],
- # index=new_index,
- # columns=f[path + "/timepoint"][()],
- # )
-
def store_signal_url(
self, fullname: str, node: t.Union[h5py.Dataset, h5py.Group]
):
diff --git a/src/agora/io/utils.py b/src/agora/io/utils.py
index 0acca82c..b32b9314 100644
--- a/src/agora/io/utils.py
+++ b/src/agora/io/utils.py
@@ -35,29 +35,6 @@ def imread(path):
return cv2.imread(str(path), -1)
-class ImageCache:
- """HDF5-based image cache for faster loading of the images once they've
- been read.
- """
-
- def __init__(self, file, name, shape, remote_fn):
- self.store = h5py.File(file, "a")
- # Create a dataset
- self.dataset = self.store.create_dataset(
- name, shape, dtype=np.float, fill_value=np.nan
- )
- self.remote_fn = remote_fn
-
- def __getitem__(self, item):
- cached = self.dataset[item]
- if np.any(np.isnan(cached)):
- full = self.remote_fn(item)
- self.dataset[item] = full
- return full
- else:
- return cached
-
-
class Cache:
"""
Fixed-length mapping to use as a cache.
diff --git a/src/agora/io/writer.py b/src/agora/io/writer.py
index f11827fa..bc8769a2 100644
--- a/src/agora/io/writer.py
+++ b/src/agora/io/writer.py
@@ -135,10 +135,6 @@ class DynamicWriter:
# write all data, signified by the empty tuple
hgroup[key][()] = data
- # def _check_key(self, key):
- # if key not in self.datatypes:
- # raise KeyError(f"No defined data type for key {key}")
-
def write(self, data: dict, overwrite: list, meta: dict = {}):
"""
Write data and metadata to h5 file.
@@ -222,220 +218,6 @@ class TilerWriter(DynamicWriter):
super().write(data=data, overwrite=overwrite, meta=meta)
-# Alan: we use complex numbers because...
-@timed()
-def save_complex(array, dataset):
- # append array, an 1D array of complex numbers, onto dataset, a 2D array of real numbers
- n = len(array)
- if n > 0:
- dataset.resize(dataset.shape[0] + n, axis=0)
- dataset[-n:, 0] = array.real
- dataset[-n:, 1] = array.imag
-
-
-@timed()
-def load_complex(dataset):
- # convert 2D dataset into a 1D array of complex numbers
- array = dataset[:, 0] + 1j * dataset[:, 1]
- return array
-
-
-class BabyWriter(DynamicWriter):
- """
- Write data stored in a Baby instance to h5 files.
-
- Assumes the edgemasks are of form ((max_ncells, max_tps, tile_size, tile_size), bool).
- """
-
- compression = "gzip"
- max_ncells = 2e5 # Alan: Could just make this None
- max_tps = 1e3 # Could just make this None
- # the number of cells in a chunk for edge masks
- chunk_cells = 25
- default_tile_size = 117
- datatypes = {
- "centres": ((None, 2), np.uint16),
- "position": ((None,), np.uint16),
- "angles": ((None,), h5py.vlen_dtype(np.float32)),
- "radii": ((None,), h5py.vlen_dtype(np.float32)),
- "edgemasks": (
- (max_ncells, max_tps, default_tile_size, default_tile_size),
- bool,
- ),
- "ellipse_dims": ((None, 2), np.float32),
- "cell_label": ((None,), np.uint16),
- "trap": ((None,), np.uint16),
- "timepoint": ((None,), np.uint16),
- # "mother_assign": ((None,), h5py.vlen_dtype(np.uint16)),
- "mother_assign_dynamic": ((None,), np.uint16),
- "volumes": ((None,), np.float32),
- }
- group = "cell_info"
-
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self._traps_initialised = False
-
- def __init_trap_info(self):
- # requires traps to have been initialised
- trap_metadata = load_attributes(self.file, "trap_info")
- tile_size = trap_metadata.get("tile_size", self.default_tile_size)
- max_tps = self.metadata["time_settings/ntimepoints"][0]
- self.datatypes["edgemasks"] = (
- (self.max_ncells, max_tps, tile_size, tile_size),
- bool,
- )
- self._traps_initialised = True
-
- def __init_edgemasks(self, hgroup, edgemasks, current_indices, n_cells):
- # create the values dataset in the h5 file
- # holds the edge masks and has shape (n_tps, n_cells, tile_size, tile_size)
- key = "edgemasks"
- max_shape, dtype = self.datatypes[key]
- shape = (n_cells, 1) + max_shape[2:]
- chunks = (self.chunk_cells, 1) + max_shape[2:]
- val_dset = hgroup.create_dataset(
- "values",
- shape=shape,
- maxshape=max_shape,
- dtype=dtype,
- chunks=chunks,
- compression=self.compression,
- )
- val_dset[:, 0] = edgemasks
- # create index dataset in the h5 file:
- # the (trap, cell_id) description used to index into the values and has shape (n_cells, 2)
- ix_max_shape = (max_shape[0], 2)
- ix_shape = (0, 2)
- ix_dtype = np.uint16
- ix_dset = hgroup.create_dataset(
- "indices",
- shape=ix_shape,
- maxshape=ix_max_shape,
- dtype=ix_dtype,
- compression=self.compression,
- )
- save_complex(current_indices, ix_dset)
-
- def __append_edgemasks(self, hgroup, edgemasks, current_indices):
- val_dset = hgroup["values"]
- ix_dset = hgroup["indices"]
- existing_indices = load_complex(ix_dset)
- # check if there are any new labels
- available = np.in1d(current_indices, existing_indices)
- missing = current_indices[~available]
- all_indices = np.concatenate([existing_indices, missing])
- # resizing
- debug_t = perf_counter() # for timing code for debugging
- n_tps = val_dset.shape[1] + 1
- n_add_cells = len(missing)
- # resize dataset for Time and Cells
- new_shape = (val_dset.shape[0] + n_add_cells, n_tps) + val_dset.shape[
- 2:
- ]
- val_dset.resize(new_shape)
- logging.debug(f"Timing:resizing:{perf_counter() - debug_t}")
- # write data
- cell_indices = np.where(np.in1d(all_indices, current_indices))[0]
- for ix, mask in zip(cell_indices, edgemasks):
- try:
- val_dset[ix, n_tps - 1] = mask
- except Exception as e:
- logging.debug(
- "Exception: {}:{}, {}, {}".format(
- e, ix, n_tps, val_dset.shape
- )
- )
- # save the index values
- save_complex(missing, ix_dset)
-
- def write_edgemasks(self, data, keys, hgroup):
- """
- Write edgemasks to h5 file.
-
- Parameters
- ----------
- data: list of arrays
- Data to be written, in the form (trap_ids, cell_labels, edgemasks)
- keys: list of str
- Names corresponding to the elements of data.
- For example: ["trap", "cell_label", "edgemasks"]
- hgroup: group object
- Group to write to in h5 file.
- """
- if not self._traps_initialised:
- self.__init_trap_info()
- key = "edgemasks"
- val_key = "values"
- traps, cell_labels, edgemasks = data
- n_cells = len(cell_labels)
- hgroup = hgroup.require_group(key)
- # create complex indices with traps as real part and cell_labels as imaginary part
- current_indices = np.array(traps) + 1j * np.array(cell_labels)
- if val_key not in hgroup:
- self.__init_edgemasks(hgroup, edgemasks, current_indices, n_cells)
- else:
- self.__append_edgemasks(hgroup, edgemasks, current_indices)
-
- def write(
- self, data: dict, overwrite: list, tp: int = None, meta: dict = {}
- ):
- """
- Write data from a Baby instance, including edgemasks.
-
- Parameters
- ----------
- data: dict
- A dict of datasets and data
- overwrite: list of str
- A list of datasets to overwrite
- tp: int
- The time point of interest
- meta: dict, optional
- Metadata to be written as attributes of the h5 file
- """
- with h5py.File(self.file, "a") as store:
- hgroup = store.require_group(self.group)
- # write data
- for key, value in data.items():
- if key not in self.datatypes:
- raise KeyError(
- f"BabyWriter: No defined data type for key {key}"
- )
- else:
- try:
- if key.startswith("attrs/"):
- # metadata
- key = key.split("/")[1]
- hgroup.attrs[key] = value
- elif key in overwrite:
- # delete and replace existing dataset
- self._overwrite(value, key, hgroup)
- elif key == "edgemasks":
- keys = ["trap", "cell_label", "edgemasks"]
- value = [data[x] for x in keys]
- edgemask_dset = hgroup.get(key + "/values", None)
- if (
- edgemask_dset
- and tp < edgemask_dset[()].shape[1]
- ):
- # data already exists
- print(
- f"BabyWriter: Skipping edgemasks in tp {tp}"
- )
- else:
- self.write_edgemasks(value, keys, hgroup)
- else:
- # append or create new dataset
- self._append(value, key, hgroup)
- except Exception as e:
- print(key, value)
- raise (e)
- # write metadata
- for key, value in meta.items():
- hgroup.attrs[key] = value
-
-
class LinearBabyWriter(DynamicWriter):
"""
Write data stored in a Baby instance to h5 files.
@@ -456,7 +238,6 @@ class LinearBabyWriter(DynamicWriter):
"cell_label": ((None,), np.uint16),
"trap": ((None,), np.uint16),
"timepoint": ((None,), np.uint16),
- # "mother_assign": ((None,), h5py.vlen_dtype(np.uint16)),
"mother_assign_dynamic": ((None,), np.uint16),
"volumes": ((None,), np.float32),
}
@@ -667,14 +448,6 @@ class Writer(BridgeH5):
if overwrite == "overwrite": # TODO refactor overwriting
if path in f:
del f[path]
- # elif overwrite == "accumulate": # Add a number if needed
- # if path in f:
- # parent, name = path.rsplit("/", maxsplit=1)
- # n = sum([x.startswith(name) for x in f[path]])
- # path = path + str(n).zfill(3)
- # elif overwrite == "skip":
- # if path in f:
- # logging.debug("Skipping dataset {}".format(path))
logging.debug(
"{} {} to {} and {} metadata fields".format(
overwrite, type(data), path, len(meta)
@@ -931,8 +704,6 @@ class Writer(BridgeH5):
return existing_cells, new_cells
-
-
def locate_indices(existing, new):
if new.any():
if new.shape[1] > 1:
diff --git a/src/agora/utils/example.py b/src/agora/utils/example.py
deleted file mode 100644
index e3ff571a..00000000
--- a/src/agora/utils/example.py
+++ /dev/null
@@ -1,53 +0,0 @@
-"""This is an example module to show the structure."""
-from typing import Union
-
-
-class ExampleClass:
- """This is an example class to show the structure."""
-
- def __init__(self, parameter: int):
- """This class takes one parameter and is used to add one to that
- parameter.
-
- :param parameter: The parameter for this class
- """
- self.parameter = parameter
-
- def add_one(self):
- """Takes the parameter and adds one.
-
- >>> x = ExampleClass(1)
- >>> x.add_one()
- 2
-
- :return: the parameter + 1
- """
- return self.parameter + 1
-
- def add_n(self, n: int):
- """Adds n to the class instance's parameter.
-
- For instance
- >>> x = ExampleClass(1)
- >>> x.add_n(10)
- 11
-
- :param n: The number to add
- :return: the parameter + n
- """
- return self.parameter + n
-
-
-def example_function(parameter: Union[int, str]):
- """This is a factory function for an ExampleClass.
-
- :param parameter: the parameter to give to the example class
- :return: An example class
- """
- try:
- return ExampleClass(int(parameter))
- except ValueError as e:
- raise ValueError(
- f"The parameter {parameter} could not be turned "
- f"into an integer."
- ) from e
diff --git a/src/agora/utils/lineage.py b/src/agora/utils/lineage.py
index 5b668686..52fb552b 100644
--- a/src/agora/utils/lineage.py
+++ b/src/agora/utils/lineage.py
@@ -20,60 +20,3 @@ def mb_array_to_dict(mb_array: np.ndarray):
for mo, daughters in groupsort(mo_da).items()
}
-
-def mb_array_to_indices(mb_array: np.ndarray):
- """
- Convert a lineage ndarray (trap, mother_id, daughter_id)
- into a dictionary of lists ( mother_id ->[daughters_ids] )
- """
- return pd.MultiIndex.from_arrays(mb_array[:, :2].T).union(
- pd.MultiIndex.from_arrays(mb_array[:, [0, 2]].T)
- )
-
-
-def group_matrix(
- matrix: np.ndarray,
- n_keys: int = 2,
-) -> t.Dict[t.Tuple[int], t.List[int]]:
- """Group a matrix of integers by grouping the first two columns
- and setting the third one in a list.
-
-
- Parameters
- ----------
- matrix : np.ndarray
- id_matrix, generally its columns are three integers indicating trap,
- mother and daughter.
- n_keys : int
- number of keys to use to determine groups.
-
- Returns
- -------
- t.Dict[t.Tuple[int], t.Collection[int, ...]]
- The column(s) not used for generaeting keys are grouped as values.
-
- Examples
- --------
- FIXME: Add docs.
-
- """
- lineage_dict = {}
- if len(matrix):
-
- daughter = matrix[:, n_keys]
- mother_global_id = matrix[:, :n_keys]
-
- iterator = groupby(
- zip(mother_global_id, daughter), lambda x: str(x[0])
- )
- lineage_dict = {key: [x[1] for x in group] for key, group in iterator}
-
- def str_to_tuple(k: str) -> t.Tuple[int, ...]:
- return tuple([int(x) for x in re.findall("[0-9]+", k)])
-
- # Convert keys from str to tuple
- lineage_dict = {
- str_to_tuple(k): sorted(v) for k, v in lineage_dict.items()
- }
-
- return lineage_dict
diff --git a/src/aliby/baby_client.py b/src/aliby/baby_client.py
index f13c5b02..6004af88 100644
--- a/src/aliby/baby_client.py
+++ b/src/aliby/baby_client.py
@@ -174,75 +174,6 @@ class BabyRunner:
return format_segmentation(segmentation, tp)
-class BabyClient:
- """A dummy BabyClient object for Dask Demo.
-
-
- Does segmentation one time point at a time.
- Should work better with the parallelisation.
- """
-
- bf_channel = 0
- model_name = "prime95b_brightfield_60x_5z"
- url = "http://localhost:5101"
- max_tries = 50
- sleep_time = 0.1
-
- def __init__(self, tiler):
- self.tiler = tiler
- self._session = None
-
- @property
- def session(self):
- if self._session is None:
- r_session = requests.get(self.url + f"/session/{self.model_name}")
- r_session.raise_for_status()
- self._session = r_session.json()["sessionid"]
- return self._session
-
- def get_data(self, tp):
- return self.tiler.get_tp_data(tp, self.bf_channel).swapaxes(1, 3)
-
- # def queue_image(self, img, **kwargs):
- # bit_depth = img.dtype.itemsize * 8 # bit depth = byte_size * 8
- # data = create_request(img.shape, bit_depth, img, **kwargs)
- # status = requests.post(
- # self.url + f"/segment?sessionid={self.session}",
- # data=data,
- # headers={"Content-Type": data.content_type},
- # )
- # status.raise_for_status()
- # return status
-
- def get_segmentation(self):
- try:
- seg_response = requests.get(
- self.url + f"/segment?sessionid={self.session}", timeout=120
- )
- seg_response.raise_for_status()
- result = seg_response.json()
- except Timeout as e:
- raise e
- except HTTPError as e:
- raise e
- return result
-
- def run_tp(self, tp, **kwargs):
- # Get data
- img = self.get_data(tp)
- # Queue image
- _ = self.queue_image(img, **kwargs)
- # Get segmentation
- for _ in range(self.max_tries):
- try:
- seg = self.get_segmentation()
- break
- except (Timeout, HTTPError):
- time.sleep(self.sleep_time)
- continue
- return format_segmentation(seg, tp)
-
-
def choose_model_from_params(
modelset_filter=None,
camera="prime95b",
diff --git a/src/aliby/haystack.py b/src/aliby/haystack.py
index 340584c5..d1368ffd 100644
--- a/src/aliby/haystack.py
+++ b/src/aliby/haystack.py
@@ -37,8 +37,6 @@ def timer(func, *args, **kwargs):
################## CUSTOM OBJECTS ##################################
-
-
class ModelPredictor:
"""Generic object that takes a NN and returns the prediction.
@@ -77,32 +75,3 @@ class ModelPredictorWriter(DynamicWriter):
"timepoint": ((None,), np.uint16),
}
self.group = f"{self.name}_info"
-
-
-class Saver:
- channel_names = {0: "BrightField", 1: "GFP"}
-
- def __init__(self, tiler, save_directory, pos_name):
- """This class straight up saves the trap data for use with neural networks in the future."""
- self.tiler = tiler
- self.name = pos_name
- self.save_dir = Path(save_directory)
-
- def channel_dir(self, index):
- ch_dir = self.save_dir / self.channel_names[index]
- if not ch_dir.exists():
- ch_dir.mkdir()
- return ch_dir
-
- def get_data(self, tp, ch):
- return self.tiler.get_tp_data(tp, ch).swapaxes(1, 3).swapaxes(1, 2)
-
- def cache(self, tp):
- # Get a given time point
- # split into channels
- for ch in self.channel_names:
- ch_dir = self.channel_dir(ch)
- data = self.get_data(tp, ch)
- for tid, trap in enumerate(data):
- np.save(ch_dir / f"{self.name}_{tid}_{tp}.npy", trap)
- return
diff --git a/src/aliby/io/omero.py b/src/aliby/io/omero.py
index 97b4f33d..5377c48a 100644
--- a/src/aliby/io/omero.py
+++ b/src/aliby/io/omero.py
@@ -127,7 +127,7 @@ class BridgeOmero:
@abstractmethod
def init_interface(self):
- ...
+ pass
@property
def file_annotations(self):
@@ -302,9 +302,6 @@ class Image(BridgeOmero):
# metadata = load_attributes(filepath)
bridge = BridgeH5(filepath)
image_id = bridge.meta_h5["image_id"]
- # server_info = safe_load(bridge.meta_h5["parameters"])["general"][
- # "server_info"
- # ]
return cls(image_id, **cls.server_info_from_h5(filepath))
@property
diff --git a/src/aliby/io/utils.py b/src/aliby/io/utils.py
deleted file mode 100644
index d59a2f65..00000000
--- a/src/aliby/io/utils.py
+++ /dev/null
@@ -1,50 +0,0 @@
-import re
-import struct
-
-
-def clean_ascii(text):
- return re.sub(r"[^\x20-\x7F]", ".", text)
-
-
-def xxd(x, start=0, stop=None):
- if stop is None:
- stop = len(x)
- for i in range(start, stop, 8):
- # Row number
- print("%04d" % i, end=" ")
- # Hexadecimal bytes
- for r in range(i, i + 8):
- print("%02x" % x[r], end="")
- if (r + 1) % 4 == 0:
- print(" ", end="")
- # ASCII
- print(
- " ",
- clean_ascii(x[i : i + 8].decode("utf-8", errors="ignore")),
- " ",
- end="",
- )
- # Int32
- print(
- "{:>10} {:>10}".format(*struct.unpack("II", x[i : i + 8])),
- end=" ",
- )
- print("") # Newline
- return
-
-
-# Buffer reading functions
-def read_int(buffer, n=1):
- res = struct.unpack("I" * n, buffer.read(4 * n))
- if n == 1:
- res = res[0]
- return res
-
-
-def read_string(buffer):
- return "".join([x.decode() for x in iter(lambda: buffer.read(1), b"\x00")])
-
-
-def read_delim(buffer, n):
- delim = read_int(buffer, n)
- assert all([x == 0 for x in delim]), "Unknown nonzero value in delimiter"
diff --git a/src/aliby/pipeline.py b/src/aliby/pipeline.py
index ea566746..04484584 100644
--- a/src/aliby/pipeline.py
+++ b/src/aliby/pipeline.py
@@ -522,15 +522,6 @@ class Pipeline(ProcessABC):
finally:
_close_session(session)
- # try:
- # compiler = ExperimentCompiler(None, filepath)
- # tmp = compiler.run()
- # po = PageOrganiser(tmp, grid_spec=(3, 2))
- # po.plot()
- # po.save(fullpath / f"{directory}report.pdf")
- # except Exception as e:
- # print("Report failed: {}".format(e))
-
@staticmethod
def check_earlystop(filename: str, es_parameters: dict, tile_size: int):
s = Signal(filename)
@@ -685,29 +676,6 @@ class Pipeline(ProcessABC):
except Exception:
pass
- # Delete datasets to overwrite and update pipeline data
- # Use existing parameters
- # with h5py.File(filename, "a") as f:
- # pparams = PipelineParameters.from_yaml(
- # f.attrs["parameters"]
- # ).to_dict()
-
- # for k, v in ow.items():
- # if v:
- # for gname in self.writer_groups[k]:
- # if gname in f:
- # del f[gname]
-
- # pparams[k] = config[k]
- # meta.add_fields(
- # {
- # "parameters": PipelineParameters.from_dict(
- # pparams
- # ).to_yaml()
- # },
- # overwrite=True,
- # )
-
meta.run()
meta.add_fields( # Add non-logfile metadata
{
diff --git a/src/aliby/tile/tiler.py b/src/aliby/tile/tiler.py
index 16f9b2f8..e0f4ff7d 100644
--- a/src/aliby/tile/tiler.py
+++ b/src/aliby/tile/tiler.py
@@ -281,8 +281,6 @@ class Tiler(ProcessABC):
trap_locs = TrapLocations.read_hdf5(filepath)
metadata = BridgeH5(filepath).meta_h5
metadata["channels"] = image.metadata["channels"]
- # metadata["zsectioning/nsections"] = image.metadata["zsectioning/nsections"]
- # metadata["channels/zsect"] = image.metadata["channels/zsect"]
if parameters is None:
parameters = TilerParameters.default()
tiler = cls(
diff --git a/src/aliby/tile/traps.py b/src/aliby/tile/traps.py
index 8d1d776c..4eddeb7e 100644
--- a/src/aliby/tile/traps.py
+++ b/src/aliby/tile/traps.py
@@ -140,9 +140,6 @@ def segment_traps(
return traps_retry
-###
-
-
def identify_trap_locations(
image, trap_template, optimize_scale=True, downscale=0.35, trap_size=None
):
@@ -240,103 +237,10 @@ def identify_trap_locations(
def stretch_image(image):
+ # FIXME Used in aliby.utils.imageViewer
image = ((image - image.min()) / (image.max() - image.min())) * 255
minval = np.percentile(image, 2)
maxval = np.percentile(image, 98)
image = np.clip(image, minval, maxval)
image = (image - minval) / (maxval - minval)
- return image
-
-
-def get_tile_shapes(x, tile_size):
- half_size = tile_size // 2
- xmin = int(x[0] - half_size)
- ymin = max(0, int(x[1] - half_size))
-
- return xmin, xmin + tile_size, ymin, ymin + tile_size
-
-
-def in_image(img, xmin, xmax, ymin, ymax, xidx=2, yidx=3):
- if xmin >= 0 and ymin >= 0:
- if xmax < img.shape[xidx] and ymax < img.shape[yidx]:
- return True
- else:
- return False
-
-
-def get_xy_tile(img, xmin, xmax, ymin, ymax, xidx=2, yidx=3, pad_val=None):
- if pad_val is None:
- pad_val = np.median(img)
- # Get the tile from the image
- idx = [slice(None)] * len(img.shape)
- idx[xidx] = slice(max(0, xmin), min(xmax, img.shape[xidx]))
- idx[yidx] = slice(max(0, ymin), min(ymax, img.shape[yidx]))
- tile = img[tuple(idx)]
- # Check if the tile is in the image
- if in_image(img, xmin, xmax, ymin, ymax, xidx, yidx):
- return tile
- else:
- # Add padding
- pad_shape = [(0, 0)] * len(img.shape)
- pad_shape[xidx] = (max(-xmin, 0), max(xmax - img.shape[xidx], 0))
- pad_shape[yidx] = (max(-ymin, 0), max(ymax - img.shape[yidx], 0))
- tile = np.pad(tile, pad_shape, constant_values=pad_val)
- return tile
-
-
-def tile_where(centre, x, y, MAX_X, MAX_Y):
- # Find the position of the tile
- xmin = int(centre[1] - x // 2)
- ymin = int(centre[0] - y // 2)
- xmax = xmin + x
- ymax = ymin + y
- # What do we actually have available?
- r_xmin = max(0, xmin)
- r_xmax = min(MAX_X, xmax)
- r_ymin = max(0, ymin)
- r_ymax = min(MAX_Y, ymax)
- return xmin, ymin, xmax, ymax, r_xmin, r_ymin, r_xmax, r_ymax
-
-
-def get_tile(shape, center, raw_expt, ch, t, z):
- """Returns a tile from the raw experiment with a given shape.
-
- :param shape: The shape of the tile in (C, T, Z, Y, X) order.
- :param center: The x,y position of the centre of the tile
- :param
- """
- _, _, x, y, _ = shape
- _, _, MAX_X, MAX_Y, _ = raw_expt.shape
- tile = np.full(shape, np.nan)
-
- # Find the position of the tile
- xmin = int(center[1] - x // 2)
- ymin = int(center[0] - y // 2)
- xmax = xmin + x
- ymax = ymin + y
- # What do we actually have available?
- r_xmin = max(0, xmin)
- r_xmax = min(MAX_X, xmax)
- r_ymin = max(0, ymin)
- r_ymax = min(MAX_Y, ymax)
-
- # Fill values
- tile[
- :,
- :,
- (r_xmin - xmin) : (r_xmax - xmin),
- (r_ymin - ymin) : (r_ymax - ymin),
- :,
- ] = raw_expt[ch, t, r_xmin:r_xmax, r_ymin:r_ymax, z]
- # fill_val = np.nanmedian(tile)
- # np.nan_to_num(tile, nan=fill_val, copy=False)
- return tile
-
-
-def centre(img, percentage=0.3):
- y, x = img.shape
- cropx = int(np.ceil(x * percentage))
- cropy = int(np.ceil(y * percentage))
- startx = int(x // 2 - (cropx // 2))
- starty = int(y // 2 - (cropy // 2))
- return img[starty : starty + cropy, startx : startx + cropx]
+ return image
\ No newline at end of file
diff --git a/src/aliby/utils/argo.py b/src/aliby/utils/argo.py
index 24255065..b4d80c36 100644
--- a/src/aliby/utils/argo.py
+++ b/src/aliby/utils/argo.py
@@ -282,10 +282,6 @@ class OmeroExplorer:
for k, v in self.cache.items()
}
- # @staticfunction
- # def number_of_X(logfile: str):
- # return re.findall("X", logfile)
-
def dset_count(
self,
dset: t.Union[int, _DatasetWrapper],
@@ -351,17 +347,6 @@ class Argo(OmeroExplorer):
super().__init__(*args, **kwargs)
-def get_creds():
- return (
- "upload",
- "***REMOVED***", # OMERO Password
- )
-
-
-def list_files(dset):
- return {x for x in dset.listAnnotations() if hasattr(x, "getFileName")}
-
-
def annot_from_dset(dset, kind):
v = [x for x in dset.listAnnotations() if hasattr(x, "getFileName")]
infname = kind if kind == "log" else kind.title()
@@ -374,7 +359,6 @@ def annot_from_dset(dset, kind):
except Exception as e:
print(f"Conversion from acquisition file failed: {e}")
return {}
-
return acq
@@ -393,6 +377,7 @@ def check_channels(acq, channels, _all=True):
def get_chs(exptype):
+ # TODO Documentation
exptypes = {
"dual_ph": ("GFP", "pHluorin405", "mCherry"),
"ph": ("GFP", "pHluorin405"),
@@ -403,6 +388,7 @@ def get_chs(exptype):
def load_annot_from_cache(exp_id, cache_dir="cache/"):
+ # TODO Documentation
if type(cache_dir) is not PosixPath:
cache_dir = Path(cache_dir)
@@ -428,16 +414,6 @@ def parse_annot(str_io, fmt):
return parser.parse(io.StringIO(str_io))
-def get_log_date(annot_sets):
- log = get_annot(annot_sets, "log")
- return log.get("date", None)
-
-
-def get_log_microscope(annot_sets):
- log = get_annot(annot_sets, "log")
- return log.get("microscope", None)
-
-
def get_annotsets(dset):
annot_files = [
annot.getFile()
@@ -457,12 +433,8 @@ def get_annotsets(dset):
return annot_sets
-# def has_tags(d, tags):
-# if set(tags).intersection(annot_from_dset(d, "log").get("omero_tags", [])):
-# return True
-
-
def load_acq(dset):
+ # TODO Documentation
try:
acq = annot_from_dset(dset, kind="acq")
return acq
@@ -472,6 +444,7 @@ def load_acq(dset):
def has_channels(dset, exptype):
+ # TODO Documentation
acq = load_acq(dset)
if acq:
return check_channels(acq, get_chs(exptype))
@@ -479,26 +452,8 @@ def has_channels(dset, exptype):
return
-# Custom functions
-def compare_dsets_voltages_exp(dsets):
- a = {}
- for d in dsets:
- try:
- acq = annot_from_dset(d, kind="acq")["channels"]
- a[d.getId()] = {
- k: (v, e)
- for k, v, e in zip(
- acq["channel"], acq["voltage"], acq["exposure"]
- )
- }
-
- except Exception as e:
- print(d, f"Data set voltage comparison did not work:{e}")
-
- return a
-
-
def get_logfile(dset):
+ # TODO Documentation
annot_file = [
annot.getFile()
for annot in dset.listAnnotations()
diff --git a/src/aliby/utils/cache.py b/src/aliby/utils/cache.py
deleted file mode 100644
index a256c635..00000000
--- a/src/aliby/utils/cache.py
+++ /dev/null
@@ -1,141 +0,0 @@
-"""
-Utility functions and classes
-"""
-import itertools
-import logging
-import operator
-from functools import partial, wraps
-from pathlib import Path
-from time import perf_counter
-from typing import Callable
-
-import cv2
-import h5py
-import imageio
-import numpy as np
-
-
-def repr_obj(obj, indent=0):
- """
- Helper function to display info about OMERO objects.
- Not all objects will have a "name" or owner field.
- """
- string = """%s%s:%s Name:"%s" (owner=%s)""" % (
- " " * indent,
- obj.OMERO_CLASS,
- obj.getId(),
- obj.getName(),
- obj.getAnnotation(),
- )
-
- return string
-
-
-def imread(path):
- return cv2.imread(str(path), -1)
-
-
-class ImageCache:
- """HDF5-based image cache for faster loading of the images once they've
- been read.
- """
-
- def __init__(self, file, name, shape, remote_fn):
- self.store = h5py.File(file, "a")
- # Create a dataset
- self.dataset = self.store.create_dataset(
- name, shape, dtype=np.float, fill_value=np.nan
- )
- self.remote_fn = remote_fn
-
- def __getitem__(self, item):
- cached = self.dataset[item]
- if np.any(np.isnan(cached)):
- full = self.remote_fn(item)
- self.dataset[item] = full
- return full
- else:
- return cached
-
-
-class Cache:
- """
- Fixed-length mapping to use as a cache.
- Deletes items in FIFO manner when maximum allowed length is reached.
- """
-
- def __init__(self, max_len=5000, load_fn: Callable = imread):
- """
- :param max_len: Maximum number of items in the cache.
- :param load_fn: The function used to load new items if they are not
- available in the Cache
- """
- self._dict = dict()
- self._queue = []
- self.load_fn = load_fn
- self.max_len = max_len
-
- def __getitem__(self, item):
- if item not in self._dict:
- self.load_item(item)
- return self._dict[item]
-
- def load_item(self, item):
- self._dict[item] = self.load_fn(item)
- # Clean up the queue
- self._queue.append(item)
- if len(self._queue) > self.max_len:
- del self._dict[self._queue.pop(0)]
-
- def clear(self):
- self._dict.clear()
- self._queue.clear()
-
-
-def accumulate(lst: list):
- lst = sorted(lst)
- it = itertools.groupby(lst, operator.itemgetter(0))
- for key, sub_iter in it:
- yield key, [x[1] for x in sub_iter]
-
-
-def get_store_path(save_dir, store, name):
- """Create a path to a position-specific store.
-
- This combines the name and the store's base name into a file path within save_dir.
- For example.
- >>> get_store_path('data', 'baby_seg.h5', 'pos001')
- Path(data/pos001baby_seg.h5')
-
- :param save_dir: The root directory in which to save the file, absolute
- path.
- :param store: The base name of the store
- :param name: The name of the position
- :return: Path(save_dir) / name+store
- """
- store = Path(save_dir) / store
- store = store.with_name(name + store.name)
- return store
-
-
-def parametrized(dec):
- def layer(*args, **kwargs):
- def repl(f):
- return dec(f, *args, **kwargs)
-
- return repl
-
- return layer
-
-
-@parametrized
-def timed(f, name=None):
- @wraps(f)
- def decorated(*args, **kwargs):
- t = perf_counter()
- res = f(*args, **kwargs)
- to_print = name or f.__name__
- logging.debug(f"Timing:{to_print}:{perf_counter() - t}s")
- return res
-
- return decorated
diff --git a/src/extraction/core/extractor.py b/src/extraction/core/extractor.py
index 6a5af6bc..a44d6ea6 100644
--- a/src/extraction/core/extractor.py
+++ b/src/extraction/core/extractor.py
@@ -451,6 +451,7 @@ class Extractor(ProcessABC):
An example is d["GFP"]["np_max"]["mean"][0], which gives a tuple of the calculated mean GFP fluorescence for all cells.
"""
+ # TODO Can we split the different extraction types into sub-methods to make this easier to read?
if tree is None:
# use default
tree: extraction_tree = self.params.tree
diff --git a/src/extraction/core/functions/io.py b/src/extraction/core/functions/io.py
deleted file mode 100644
index 3b377bdf..00000000
--- a/src/extraction/core/functions/io.py
+++ /dev/null
@@ -1,12 +0,0 @@
-from yaml import dump, load
-
-
-def dict_to_yaml(d, f):
- with open(f, "w") as f:
- dump(d, f)
-
-
-def add_attrs(hdfile, path, files):
- group = hdfile.create_group(path)
- for k, v in files:
- group.attrs[k] = v
diff --git a/src/extraction/core/functions/utils.py b/src/extraction/core/functions/utils.py
deleted file mode 100644
index dcae68c9..00000000
--- a/src/extraction/core/functions/utils.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from collections import deque
-
-
-def depth(d):
- """
- Copied from https://stackoverflow.com/a/23499088
-
- Used to determine the depth of our config trees and fill them
- """
- queue = deque([(id(d), d, 1)])
- memo = set()
- while queue:
- id_, o, level = queue.popleft()
- if id_ in memo:
- continue
- memo.add(id_)
- if isinstance(o, dict):
- queue += ((id(v), v, level + 1) for v in o.values())
- return level
diff --git a/src/extraction/core/functions/versioning.py b/src/extraction/core/functions/versioning.py
index 40c1e8f9..4c77a9f8 100644
--- a/src/extraction/core/functions/versioning.py
+++ b/src/extraction/core/functions/versioning.py
@@ -2,6 +2,7 @@ import git
def get_sha():
+ # FIXME Unused, but *should* be used...
repo = git.Repo(search_parent_directories=True)
sha = repo.head.object.hexsha
return sha
diff --git a/src/extraction/core/omero.py b/src/extraction/core/omero.py
deleted file mode 100644
index e69aa747..00000000
--- a/src/extraction/core/omero.py
+++ /dev/null
@@ -1,35 +0,0 @@
-from omero.gateway import BlitzGateway
-from tqdm import tqdm
-
-
-# Helper funs
-def connect_omero():
- conn = BlitzGateway(*get_creds(), host="islay.bio.ed.ac.uk", port=4064)
- conn.connect()
- return conn
-
-
-def get_creds():
- return (
- "upload",
- "***REMOVED***", # OMERO Password
- )
-
-
-def download_file(f):
- """
- Download file in chunks using FileWrapper object
- """
- desc = (
- "Downloading "
- + f.getFileName()
- + " ("
- + str(round(f.getFileSize() / 1000**2, 2))
- + "Mb)"
- )
-
- down_file = bytearray()
- for c in tqdm(f.getFileInChunks(), desc=desc):
- down_file += c
-
- return down_file
diff --git a/src/logfile_parser/swainlab_parser.py b/src/logfile_parser/swainlab_parser.py
index a83cb87c..c5f200da 100644
--- a/src/logfile_parser/swainlab_parser.py
+++ b/src/logfile_parser/swainlab_parser.py
@@ -1,5 +1,5 @@
#!/usr/bin/env jupyter
-
+# TODO should this be merged to the regular logfile_parser structure?
"""
Description of new logfile:
@@ -298,18 +298,3 @@ def parse_x(string: str, type: str, **kwargs):
def parse_from_swainlab_grammar(filepath: t.Union[str, PosixPath]):
return parse_from_grammar(filepath, grammar)
-
-
-# test_file = "/home/alan/Documents/dev/skeletons/scripts/dev/C1_60x.log"
-# test_file = "/home/alan/Documents/dev/skeletons/scripts/dev/bak"
-# test_file = "/home/alan/Documents/dev/skeletons/scripts/dev/two_tables.log"
-# test_file = "/home/alan/Downloads/pH_med_to_low 1.log"
-# test_file = "/home/alan/Documents/dev/skeletons/scripts/data/577_2022_12_20_pHCalibrate6_7_00/pHCalibrate6_7.log"
-
-
-# d = parse_from_grammar(test_file, grammar)
-# print(d)
-
-# from logfile_parser.legacy import get_legacy_log_example_interface
-
-# original = get_legacy_log_example_interface()
diff --git a/src/postprocessor/benchmarks/post_processing.py b/src/postprocessor/benchmarks/post_processing.py
index d17651a8..d36e2afb 100644
--- a/src/postprocessor/benchmarks/post_processing.py
+++ b/src/postprocessor/benchmarks/post_processing.py
@@ -1,3 +1,4 @@
+# TODO remove/to snippets?
"""
Post-processing utilities
diff --git a/src/postprocessor/core/abc.py b/src/postprocessor/core/abc.py
index a299e19f..ffb677b4 100644
--- a/src/postprocessor/core/abc.py
+++ b/src/postprocessor/core/abc.py
@@ -16,6 +16,7 @@ class PostProcessABC(ProcessABC):
@classmethod
def as_function(cls, data, *extra_data, **kwargs):
+ # FIXME can this be a __call__ method instead?
# Find the parameter's default
parameters = cls.default_parameters(**kwargs)
return cls(parameters=parameters).run(data, *extra_data)
diff --git a/src/postprocessor/core/group.py b/src/postprocessor/core/group.py
deleted file mode 100644
index 5613e5de..00000000
--- a/src/postprocessor/core/group.py
+++ /dev/null
@@ -1,137 +0,0 @@
-"""
-Class to group multiple positions into one using one different available criteria.
-"""
-
-import re
-from pathlib import Path
-
-import h5py
-import pandas as pd
-from agora.io.bridge import groupsort
-from agora.io.signal import Signal
-
-from postprocessor.core.abc import ParametersABC, ProcessABC
-
-
-class GroupParameters(ParametersABC):
- def __init__(self, by="name", processes=[], signals=[]):
- self.by = by
- self.signals = signals
- self.processes = processes
-
- @classmethod
- def default(cls):
- return cls.from_dict({"by": "name", "signals": [], "processes": []})
-
-
-class Group(ProcessABC):
- def __init__(self, parameters):
- super().__init__(parameters)
-
- def get_position_filenames(self, exp_root, poses):
- """
- Get filenames as a dictionary where the key is the position and value the filename.
- """
- central_store = Path(exp_root) / "store.h5"
- if central_store.exists():
- hdf = h5py.File(central_store, "r")
- self.filenames = [
- pos.attrs["filename"] for pos in hdf["/positions/"]
- ]
- hdf.close()
- else: # If no central store just list position files in expt root folder
- fullfiles = [x for x in Path(exp_root).glob("*store.h5")]
- files = [x.name for x in Path(exp_root).glob("*store.h5")]
- filenames = [False for _ in poses]
- for i, pos in enumerate(poses):
- matches = [
- True if re.match(pos + ".*.h5", fname) else False
- for fname in files
- ]
- if any(matches):
- assert sum(matches) == 1, "More than one match"
- filenames[i] = (pos, fullfiles[matches.index(True)])
-
- self.filenames = {
- fname[0]: fname[1] for fname in filenames if fname
- }
-
- self.positions = list(self.filenames.keys())
- return self.filenames
-
- def get_signals(self):
- # hdf = h5py.File(central_store, "r")
- # keys_d = groupsort(keys)
- self.signals = {pos: {} for pos in self.filenames.keys()}
- for pos, fname in self.filenames.items():
- for signal in self.parameters.signals:
- self.signals[pos][signal] = pd.read_hdf(fname, signal)
-
- return self.signals
-
- def gen_groups(self):
- if self.by == "group": # Use group names in metadata
- pass
- elif self.by == "name": # Infer groups from signal concatenation
- # Remove last four characters and find commonalities larger than 4
- # characters between posnames and group them that way.
- groupnames = list(set([x[:-3] for x in self.positions]))
- self.group_signal_tree = {group: [] for group in groupnames}
- self.poses_grouped = {group: [] for group in groupnames}
- for pos in self.positions:
- group = groupnames[groupnames.index(pos[:-3])]
- self.group_signal_tree[group].append(self.signals[pos])
- self.poses_grouped[group].append(pos)
-
- elif (
- type(self.by) == tuple
- ): # Manually give groups as tuple or list of positions
- pass
-
- def concat_signals(self):
- self.concated_signals = {group: {} for group in self.group_signal_tree}
- for k, group in self.group_signal_tree.items():
- for signal in self.parameters.signals:
- self.concated_signals[k][signal] = pd.concat(
- [g[signal] for g in group], keys=self.poses_grouped[k]
- )
-
- return self.concated_signals
-
- def process_signals(self, grouped_signals):
- pass
-
- def run(self, central_store, poses):
-
- self.get_position_filenames(central_store, poses)
- self.get_signals()
- self.gen_groups()
- self.concat_signals()
- # processed_signals = self.process_signals(grouped_signals)
-
- return self.concated_signals
- # return processed_signals
-
-
-poses = [
- x.name.split("store")[0]
- for x in Path(
- "/shared_libs/pipeline-core/scripts/data/ph_calibration_dual_phl_ura8_5_04_5_83_7_69_7_13_6_59__01"
- ).rglob("*")
- if x.name != "images.h5"
-]
-gr = Group(
- GroupParameters(
- signals=[
- "/extraction/general/None/area",
- "/extraction/mCherry/np_max/median",
- ]
- )
-)
-gr.run(
- central_store="/shared_libs/pipeline-core/scripts/data/ph_calibration_dual_phl_ura8_5_04_5_83_7_69_7_13_6_59__01",
- poses=poses,
-)
-signal = Signal(
- "/shared_libs/pipeline-core/scripts/data/ph_calibration_dual_phl_ura8_5_04_5_83_7_69_7_13_6_59__01/ph_5_04_001store.h5"
-)
diff --git a/src/postprocessor/core/lineageprocess.py b/src/postprocessor/core/lineageprocess.py
index 34ccf234..f10d5b3e 100644
--- a/src/postprocessor/core/lineageprocess.py
+++ b/src/postprocessor/core/lineageprocess.py
@@ -1,3 +1,4 @@
+# TODO Module docstring
import typing as t
from abc import abstractmethod
@@ -7,8 +8,6 @@ import pandas as pd
from agora.abc import ParametersABC
from postprocessor.core.abc import PostProcessABC
-# from agora.utils.lineage import group_matrix
-
class LineageProcessParameters(ParametersABC):
"""
@@ -47,18 +46,14 @@ class LineageProcess(PostProcessABC):
Overrides PostProcess.as_function classmethod.
Lineage functions require lineage information to be passed if run as function.
"""
- # if isinstance(lineage, np.ndarray):
- # lineage = group_matrix(lineage, n_keys=2)
-
parameters = cls.default_parameters(**kwargs)
return cls(parameters=parameters).run(
data, lineage=lineage, *extra_data
)
- # super().as_function(data, *extra_data, lineage=lineage, **kwargs)
def load_lineage(self, lineage):
"""
Reshape the lineage information if needed
"""
-
+ # TODO does this need to be a function?
self.lineage = lineage
diff --git a/src/postprocessor/core/processor.py b/src/postprocessor/core/processor.py
index 37efd0d4..4b9161c0 100644
--- a/src/postprocessor/core/processor.py
+++ b/src/postprocessor/core/processor.py
@@ -56,26 +56,16 @@ class PostProcessorParameters(ParametersABC):
"/extraction/general/None/volume",
],
],
- # [
- # "savgol",
- # [
- # "/extraction/general/None/volume",
- # ],
- # ],
[
"dsignal",
[
"/extraction/general/None/volume",
- # "/postprocessing/savgol/extraction_general_None_volume",
],
],
[
"bud_metric",
[
"/extraction/general/None/volume",
- # "/postprocessing/dsignal/postprocessing_savgol_extraction_general_None_volume",
- # "/postprocessing/savgol/extraction_general_None_volume",
- # "/postprocessing/dsignal/extraction_general_None_volume",
],
],
[
@@ -84,15 +74,6 @@ class PostProcessorParameters(ParametersABC):
"/postprocessing/bud_metric/extraction_general_None_volume",
],
],
- # [
- # "aggregate",
- # [
- # [
- # "/extraction/general/None/volume",
- # "postprocessing/dsignal/extraction_general_None_volume",
- # ],
- # ],
- # ],
],
}
param_sets = {
@@ -105,22 +86,12 @@ class PostProcessorParameters(ParametersABC):
outpaths["aggregate"] = "/postprocessing/experiment_wide/aggregated/"
if "ph_batman" in kind:
- # targets["processes"]["bud_metric"].append(
- # [
- # [
- # "/extraction/em_ratio/np_max/mean",
- # "/extraction/em_ratio/np_max/median",
- # ],
- # ]
- # )
targets["processes"]["dsignal"].append(
[
"/extraction/em_ratio/np_max/mean",
"/extraction/em_ratio/np_max/median",
"/extraction/em_ratio_bgsub/np_max/mean",
"/extraction/em_ratio_bgsub/np_max/median",
- # "/postprocessing/bud_metric/extraction_em_ratio_np_max_mean",
- # "/postprocessing/bud_metric/extraction_em_ratio_np_max_median",
]
)
targets["processes"]["aggregate"].append(
@@ -132,10 +103,6 @@ class PostProcessorParameters(ParametersABC):
"/extraction/em_ratio_bgsub/np_max/median",
"/extraction/gsum/np_max/median",
"/extraction/gsum/np_max/mean",
- # "postprocessing/bud_metric/extraction_em_ratio_np_max_mean",
- # "postprocessing/bud_metric/extraction_em_ratio_np_max_median",
- # "postprocessing/dsignal/postprocessing_bud_metric_extraction_em_ratio_np_max_median",
- # "postprocessing/dsignal/postprocessing_bud_metric_extraction_em_ratio_np_max_mean",
]
],
)
@@ -178,6 +145,7 @@ class PostProcessor(ProcessABC):
self.targets = parameters["targets"]
def run_prepost(self):
+ # TODO Split function
"""Important processes run before normal post-processing ones"""
merge_events = self.merger.run(
@@ -301,12 +269,7 @@ class PostProcessor(ProcessABC):
return x
def run(self):
- # import cProfile
- # import pstats
-
- # profile = cProfile.Profile()
- # profile.enable()
-
+ # TODO Documentation :) + Split
self.run_prepost()
for process, datasets in tqdm(self.targets["processes"]):
diff --git a/src/postprocessor/core/reshapers/bud_metric.py b/src/postprocessor/core/reshapers/bud_metric.py
index 0893db66..c527134d 100644
--- a/src/postprocessor/core/reshapers/bud_metric.py
+++ b/src/postprocessor/core/reshapers/bud_metric.py
@@ -33,7 +33,6 @@ class bud_metric(LineageProcess):
mother_bud_ids: Dict[pd.Index, Tuple[pd.Index]] = None,
):
if mother_bud_ids is None:
- # filtered_lineage = self.filter_signal_cells(signal)
mother_bud_ids = mb_array_to_dict(self.lineage)
return self.get_bud_metric(signal, mother_bud_ids)
diff --git a/src/postprocessor/core/reshapers/buddings.py b/src/postprocessor/core/reshapers/buddings.py
index 0b01dad7..4b3fbba9 100644
--- a/src/postprocessor/core/reshapers/buddings.py
+++ b/src/postprocessor/core/reshapers/buddings.py
@@ -25,16 +25,17 @@ class buddingsParameters(LineageProcessParameters):
FIXME: Add docs.
"""
-
_defaults = {"lineage_location": "postprocessing/lineage_merged"}
+# TODO Why not capitalized?
class buddings(LineageProcess):
"""
Calculate buddings in a trap assuming one mother per trap
returns a pandas series with the buddings
"""
+ # TODO might want to define "buddings" more scientifically
def __init__(self, parameters: buddingsParameters):
super().__init__(parameters)
diff --git a/src/postprocessor/core/reshapers/merger.py b/src/postprocessor/core/reshapers/merger.py
index 16d2b598..1fbf4155 100644
--- a/src/postprocessor/core/reshapers/merger.py
+++ b/src/postprocessor/core/reshapers/merger.py
@@ -3,7 +3,7 @@ from agora.abc import ParametersABC
from postprocessor.core.abc import PostProcessABC
from postprocessor.core.functions.tracks import get_joinable
-
+# TODO Why not capitalized?
class mergerParameters(ParametersABC):
"""
:param tol: float or int threshold of average (prediction error/std) necessary
@@ -22,6 +22,7 @@ class mergerParameters(ParametersABC):
}
+# TODO Why not capitalized?
class merger(PostProcessABC):
"""
Combines rows of tracklet that are likely to be the same.
@@ -34,8 +35,4 @@ class merger(PostProcessABC):
joinable = []
if signal.shape[1] > 4:
joinable = get_joinable(signal, tol=self.parameters.tolerance)
- # merged, _ = merge_tracks(signal) # , min_len=self.window + 1)
- # indices = (*zip(*merged.index.tolist()),)
- # names = merged.index.names
- # return {name: ids for name, ids in zip(names, indices)}
return joinable
diff --git a/src/postprocessor/core/reshapers/picker.py b/src/postprocessor/core/reshapers/picker.py
index bfabc76a..b198b7fd 100644
--- a/src/postprocessor/core/reshapers/picker.py
+++ b/src/postprocessor/core/reshapers/picker.py
@@ -1,34 +1,20 @@
-# from abc import ABC, abstractmethod
-
-# from copy import copy
-# from itertools import groupby
-# from typing import List, Tuple, Union
import typing as t
-from typing import Union
-# import igraph as ig
import numpy as np
import pandas as pd
from agora.abc import ParametersABC
from agora.io.cells import Cells
-# from postprocessor.core.functions.tracks import max_nonstop_ntps, max_ntps
from agora.utils.association import validate_association
from postprocessor.core.lineageprocess import LineageProcess
-# from utils_find_1st import cmp_equal, find_1st
-
class pickerParameters(ParametersABC):
_defaults = {
"sequence": [
["lineage", "intersection", "families"],
- # ["condition", "intersection", "any_present", 0.7],
- # ["condition", "intersection", "growing", 80],
["condition", "intersection", "present", 7],
- # ["condition", "intersection", "mb_guess", 3, 0.7],
- # ("lineage", "intersection", "full_families"),
],
}
@@ -80,16 +66,8 @@ class picker(LineageProcess):
idx = idx[valid_indices]
mothers_daughters = mothers_daughters[valid_lineage]
-
- # return mothers_daughters, idx
return idx
- def loc_lineage(self, kymo: pd.DataFrame, how: str, lineage=None):
- _, valid_indices = self.pick_by_lineage(
- kymo, how, mothers_daughters=lineage
- )
- return kymo.loc[[tuple(x) for x in valid_indices]]
-
def pick_by_condition(self, signals, condition, thresh):
idx = self.switch_case(signals, condition, thresh)
return idx
@@ -131,7 +109,7 @@ class picker(LineageProcess):
self,
signals: pd.DataFrame,
condition: str,
- threshold: Union[float, int, list],
+ threshold: t.Union[float, int, list],
):
if len(threshold) == 1:
threshold = [_as_int(*threshold, signals.shape[1])]
@@ -145,7 +123,7 @@ class picker(LineageProcess):
return set(signals.index[case_mgr[condition](signals, *threshold)])
-def _as_int(threshold: Union[float, int], ntps: int):
+def _as_int(threshold: t.Union[float, int], ntps: int):
if type(threshold) is float:
threshold = ntps * threshold
return threshold
diff --git a/src/postprocessor/grouper.py b/src/postprocessor/grouper.py
index 167c0b5e..ec04a6f5 100644
--- a/src/postprocessor/grouper.py
+++ b/src/postprocessor/grouper.py
@@ -15,11 +15,6 @@ import pandas as pd
import seaborn as sns
from pathos.multiprocessing import Pool
-from agora.utils.kymograph import (
- drop_level,
- get_mother_ilocs_from_daughters,
- intersection_matrix,
-)
from postprocessor.chainer import Chainer
diff --git a/tests/agora/example_test.py b/tests/agora/example_test.py
deleted file mode 100644
index 539b410a..00000000
--- a/tests/agora/example_test.py
+++ /dev/null
@@ -1,25 +0,0 @@
-"""This is an example test file to show the structure."""
-import pytest
-
-from agora.utils.example import ExampleClass, example_function
-
-
-class TestExampleClass:
- x = ExampleClass(1)
-
- def test_add_one(self):
- assert self.x.add_one() == 2
-
- def test_add_n(self):
- assert self.x.add_n(10) == 11
-
-
-def test_example_function():
- x = example_function(1)
- assert isinstance(x, ExampleClass)
- assert x.parameter == 1
-
-
-def test_example_function_fail():
- with pytest.raises(ValueError):
- example_function("hello")
diff --git a/tests/aliby/network/test_baby_client.py b/tests/aliby/network/test_baby_client.py
deleted file mode 100644
index a8d131a0..00000000
--- a/tests/aliby/network/test_baby_client.py
+++ /dev/null
@@ -1,91 +0,0 @@
-import pytest
-
-pytest.mark.skip
-
-import json
-import time
-
-import numpy as np
-
-# from aliby.experiment import ExperimentLocal
-from aliby.baby_client import BabyClient
-from aliby.tile.tiler import Tiler
-
-
-@pytest.mark.skip(
- reason="No longer usable, requires local files. Kept until replaced."
-)
-def test_client():
- root_dir = (
- "/Users/s1893247/PhD/pipeline-core/data/glclvl_0"
- ".1_mig1_msn2_maf1_sfp1_dot6_03"
- )
-
- expt = ExperimentLocal(root_dir, finished=True)
- seg_expt = Tiler(expt, finished=True)
-
- print(seg_expt.positions)
- seg_expt.current_position = "pos007"
-
- config = {
- "camera": "evolve",
- "channel": "brightfield",
- "zoom": "60x",
- "n_stacks": "5z",
- }
-
- baby_client = BabyClient(expt, **config)
-
- print("The session is {}".format(baby_client.sessions["default"]))
-
- # Channel 0, 0, X,Y,Z all
- num_timepoints = 5
-
- traps_tps = [
- seg_expt.get_tiles_timepoint(
- tp, tile_size=81, channels=[0], z=[0, 1, 2, 3, 4]
- ).squeeze()
- for tp in range(num_timepoints)
- ]
-
- segmentations = []
- try:
- for i, timpoint in enumerate(traps_tps):
- print("Sending timepoint {};".format(i))
- status = baby_client.queue_image(
- timpoint,
- baby_client.sessions["default"],
- assign_mothers=True,
- return_baprobs=True,
- with_edgemasks=True,
- )
- while True:
- try:
- print("Loading.", end="")
- result = baby_client.get_segmentation(
- baby_client.sessions["default"]
- )
- except:
- print(".", end="")
- time.sleep(1)
- continue
- break
- print("Received timepoint {}".format(i))
- segmentations.append(result)
- except Exception as e:
- print(segmentations)
- raise e
-
- with open("segmentations.json", "w") as fd:
- json.dump(segmentations, fd)
-
- print("Done.")
- # print(len(segmentations[0]))
- # for i in range(5):
- # print("trap {}".format(i))
- # for k, v in segmentations[0][i].items():
- # print(k, v)
- #
- # import matplotlib.pyplot as plt
- # plt.imshow(np.squeeze(batches[0][0, ..., 0]))
- # plt.savefig('test_baby.pdf')
diff --git a/tests/aliby/network/test_post_processing.py b/tests/aliby/network/test_post_processing.py
index 3ac8ba40..dbf9b8a9 100644
--- a/tests/aliby/network/test_post_processing.py
+++ b/tests/aliby/network/test_post_processing.py
@@ -7,14 +7,6 @@ import skimage.morphology as morph
from scipy import ndimage
from skimage import draw
-# from aliby.post_processing import (
-# circle_outline,
-# conical,
-# ellipse_perimeter,
-# union_of_spheres,
-# volume_of_sphere,
-# )
-
@pytest.mark.skip(
reason="No longer usable, post_processing unused inside aliby. Kept temporarily"
--
GitLab