diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f16af3cbf09a0942f2ee74360bf875e25d285a14..f45d1486a3483a230c0b8f0f4eab8e7cb513f3ce 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -52,7 +52,7 @@ Bump_release:
script:
- git config --global user.email ${GITLAB_USER_EMAIL}
- git config --global user.name ${GITLAB_USER_NAME}
- - git pull origin HEAD:MASTER && poetry version ${BUMP_RULE} && git add pypoetry add pyproject.toml poetry.lock && git commit -m "Bump version" && git push -o ci.skip origin HEAD:master && poetry publish --build --username ${PYPI_USER} --password ${PYPI_PASSWORD}
+ - git pull origin HEAD:MASTER && poetry version ${BUMP_RULE} && git add poetry.lock add pyproject.toml poetry.lock && git commit -m "Bump version" && git push -o ci.skip origin HEAD:master && poetry publish --build --username ${PYPI_USER} --password ${PYPI_PASSWORD}
- echo "TRIGGER_PYPI_NAME=$(cat pyproject.toml | grep '^name =' | head -n 1 | cut -f3 -d' ' | tr -d \")" >> build.env
- echo "Exporting TRIGGER_PYPI_NAME as ${TRIGGER_PYPI_NAME}"
diff --git a/aliby/pipeline.py b/aliby/pipeline.py
index bfe18173f390ecfd53e41cda54de629350ae7a1d..3006081358a1816bf1a456d0b1efea2b196b1225 100644
--- a/aliby/pipeline.py
+++ b/aliby/pipeline.py
@@ -9,6 +9,7 @@ from pathlib import Path
import traceback
from itertools import groupby
+import h5py
import yaml
from tqdm import tqdm
from time import perf_counter
@@ -86,7 +87,7 @@ class PipelineParameters(ParametersABC):
directory=str(directory),
strain="",
earlystop=dict(
- min_tp=180,
+ min_tp=100,
thresh_pos_clogged=0.3,
thresh_trap_clogged=7,
ntps_to_eval=5,
@@ -109,21 +110,17 @@ class PipelineParameters(ParametersABC):
class Pipeline(ProcessABC):
"""
A chained set of Pipeline elements connected through pipes.
- """
-
- # Tiling, Segmentation,Extraction and Postprocessing should use their own default parameters
+ Tiling, Segmentation,Extraction and Postprocessing should use their own default parameters.
+ These can be overriden passing the key:value of parameters to override to a PipelineParameters class
- # Early stop for clogging
- earlystop = {
- "min_tp": 180,
- "thresh_pos_clogged": 0.3,
- "thresh_trap_clogged": 7,
- "ntps_to_eval": 5,
- }
+ """
- def __init__(self, parameters: PipelineParameters):
+ def __init__(self, parameters: PipelineParameters, store=None):
super().__init__(parameters)
- self.store = self.parameters.general["directory"]
+
+ if store is None:
+ store = self.parameters.general["directory"]
+ self.store = store
@classmethod
def from_yaml(cls, fpath):
@@ -192,31 +189,54 @@ class Pipeline(ProcessABC):
earlystop = general_config["earlystop"]
try:
directory = general_config["directory"]
+
with Image(image_id, **self.general["server_info"]) as image:
filename = f"{directory}/{image.name}.h5"
- try:
- os.remove(filename)
- except:
- pass
-
- # Run metadata first
- process_from = 0
- # if True: # not Path(filename).exists():
meta = MetaData(directory, filename)
- meta.run()
- meta.add_fields(
- {"omero_id,": config["general"]["id"], "image_id": image_id}
- )
- tiler = Tiler.from_image(
- image, TilerParameters.from_dict(config["tiler"])
- )
- # else: TODO add support to continue local experiments?
- # tiler = Tiler.from_hdf5(image.data, filename)
- # s = Signal(filename)
- # process_from = s["/general/None/extraction/volume"].columns[-1]
- # if process_from > 2:
- # process_from = process_from - 3
- # tiler.n_processed = process_from
+ from_start = True
+ if (
+ not general_config.get("overwrite", False)
+ and Path(filename).exists()
+ ):
+ try:
+ print(f"Existing file {filename} will be used.")
+ with h5py.File(filename, "r") as f:
+ tiler = Tiler.from_hdf5(image.data, filename)
+ s = Signal(filename)
+ process_from = (
+ f.attrs["last_processed"]
+ or s.get_raw("/general/None/extraction/volume").columns[
+ -1
+ ]
+ or 0
+ )
+ # get state array
+ state_array = f.get("state_array", 0)
+ if process_from > 2:
+ processFalsefrom = process_from - 3
+ tiler.n_processed = process_from
+ from_start = False
+ except:
+ pass
+
+ if from_start: # New experiment or overwriting
+ try:
+ os.remove(filename)
+ except:
+ pass
+
+ process_from = 0
+ try:
+ meta.run()
+ meta.add_fields(
+ {"omero_id,": config["general"]["id"], "image_id": image_id}
+ )
+ tiler = Tiler.from_image(
+ image, TilerParameters.from_dict(config["tiler"])
+ )
+ except:
+ # Remove and try to run again?
+ pass
writer = TilerWriter(filename)
session = initialise_tf(2)
@@ -305,13 +325,14 @@ class Pipeline(ProcessABC):
logging.debug(f"Quality:Clogged_traps:{frac_clogged_traps}")
print("Frac clogged traps: ", frac_clogged_traps)
+ meta.add_fields({"last_processed": i})
# Run post processing
post_proc_params = PostProcessorParameters.from_dict(
self.parameters.postprocessing
).to_dict()
PostProcessor(filename, post_proc_params).run()
- return True
+ return 1
except Exception as e: # bug in the trap getting
logging.exception(
f"Caught exception in worker thread (x = {name}):", exc_info=True
diff --git a/poetry.lock b/poetry.lock
index 5ace4714138993274b557a0d305a939cbd8fbd8a..339baa67a8c853d7ee8017565c73aff2f7582dd6 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -78,7 +78,7 @@ python-versions = "*"
[[package]]
name = "aliby-agora"
-version = "0.2.10"
+version = "0.2.11"
description = "A gathering of shared utilities for the Swain Lab image processing pipeline."
category = "main"
optional = false
@@ -114,14 +114,14 @@ xgboost = "1.4.2"
[[package]]
name = "aliby-post"
-version = "0.1.7"
+version = "0.1.9"
description = "Post-processing tools for aliby pipeline."
category = "main"
optional = false
python-versions = ">=3.7.1,<3.11"
[package.dependencies]
-aliby-agora = ">=0.2.10,<0.3.0"
+aliby-agora = ">=0.2.11,<0.3.0"
catch22 = ">=0.2.0,<0.3.0"
leidenalg = ">=0.8.8,<0.9.0"
more-itertools = ">=8.12.0,<9.0.0"
@@ -1920,7 +1920,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-
[metadata]
lock-version = "1.1"
python-versions = ">=3.7.1,<3.11"
-content-hash = "08ee313e962de1a7c6574a5caef7421dc34acf70cee4885afdc77340b519933c"
+content-hash = "d85b736014517401ba6cbd663015246c8fce0395297fab7dcd7a51487add7da0"
[metadata.files]
absl-py = [
@@ -2018,16 +2018,16 @@ alabaster = [
{file = "alabaster-0.7.12.tar.gz", hash = "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"},
]
aliby-agora = [
- {file = "aliby-agora-0.2.10.tar.gz", hash = "sha256:6cd3850a4b657a6033794cb7e5caa15ed963bf9a95cd8aca220fe1d704f6d94f"},
- {file = "aliby_agora-0.2.10-py3-none-any.whl", hash = "sha256:b1fe92461f79d7a54fc3fac6668ff18ec205bfedd5732dadf0fcaa9dc9f5ad4c"},
+ {file = "aliby-agora-0.2.11.tar.gz", hash = "sha256:9979349a937adcf0b1288b22e2d6fe62d2d3eadd3f9a07855debf8307642a9d0"},
+ {file = "aliby_agora-0.2.11-py3-none-any.whl", hash = "sha256:3675cd754340623ecf3acc7ae0b9efa35afd9d69ccfbcedd0d116cd0d2c53d36"},
]
aliby-baby = [
{file = "aliby-baby-0.1.2.tar.gz", hash = "sha256:564a12b44fc7e4889fb70d5a82e39e7fb01f37eeb60669c684bd18677160de93"},
{file = "aliby_baby-0.1.2-py3-none-any.whl", hash = "sha256:f8f11b82db89a74ad8cc48059370af889f246a83d4cbc45ce75d9475a15605f7"},
]
aliby-post = [
- {file = "aliby-post-0.1.7.tar.gz", hash = "sha256:82b3f6a20642fa2cee98b8647d71a08b95ba23159b97d04407474f98e720ce90"},
- {file = "aliby_post-0.1.7-py3-none-any.whl", hash = "sha256:8b187d886ae8ba6b3ac406a1ba873db79f5784861f73b14187baadaedfe132f1"},
+ {file = "aliby-post-0.1.9.tar.gz", hash = "sha256:900fb4c51e99a1ae722bb510f80230258d9226c6d302bbcbde84122d48096930"},
+ {file = "aliby_post-0.1.9-py3-none-any.whl", hash = "sha256:0ca6b4bb0597684c13da3782e9255fb341cbef6925cbb6ba78a83dbbeb20c057"},
]
appdirs = [
{file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"},
diff --git a/pyproject.toml b/pyproject.toml
index 840962bfab6a0519b8a8227d2ce431d8afa9bbed..b14a1c12bfb0a82d39b78e417f4f8e50d54696e8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "aliby"
-version = "0.1.11"
+version = "0.1.14"
description = ""
authors = ["Alan Munoz <alan.munoz@ed.ac.uk>"]
packages = [
@@ -28,10 +28,10 @@ imageio = "2.8.0"
omero-py = ">=5.6.2"
zeroc-ice = "3.6.5"
tensorflow = ">=1.15,<=2.3"
-aliby-agora = "^0.2.4"
+aliby-agora = "^0.2.11"
aliby-baby = "^0.1.2"
omni-gaussian = "^0.1.1"
-aliby-post = "^0.1.7"
+aliby-post = "^0.1.9"
[tool.poetry.dev-dependencies]