Index of experiments with at least one position failing segmentation
-
19813 (Ice) -
19333 -
19203 -
19995 (pp)
Another odd case, other positions seem to do just fine. Could just be a network failure or something.
phluorin_mig1_024: 79%|██████████████████████████████████ | 214/270 [8:02:45<2:06:19, 135.35s/it, 0% Clogged]
Caught exception in worker thread (x = phluorin_mig1_024):
Traceback (most recent call last):
File "/home/alan/Documents/dev/libs/aliby/aliby/pipeline.py", line 369, in create_pipeline
tmp = ext.run(tps=[i], masks=masks, labels=labels)
File "/home/alan/Documents/dev/libs/aliby/extraction/core/extractor.py", line 444, in run
self.extract_tp(tp=tp, tree=tree, **kwargs),
File "/home/alan/Documents/dev/libs/aliby/extraction/core/extractor.py", line 351, in extract_tp
traps = self.get_traps(tp, tile_size=tile_size, channels=tree_chs)
File "/home/alan/Documents/dev/libs/aliby/extraction/core/extractor.py", line 203, in get_traps
if channel_ids
File "/home/alan/Documents/dev/libs/aliby/aliby/tile/tiler.py", line 323, in get_traps_timepoint
val = self.get_tp_data(tp, c)[:, z] # Only return requested z
File "/home/alan/Documents/dev/libs/aliby/aliby/tile/tiler.py", line 248, in get_tp_data
full = self.get_tc(tp, c)
File "/home/alan/Documents/dev/libs/aliby/aliby/tile/tiler.py", line 189, in get_tc
full = self.image[t, c].compute() # FORCE THE CACHE
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/dask/base.py", line 288, in compute
(result,) = compute(self, traverse=False, **kwargs)
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/dask/base.py", line 571, in compute
results = schedule(dsk, keys, **kwargs)
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/dask/threaded.py", line 87, in get
**kwargs,
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/dask/local.py", line 507, in get_async
raise_exception(exc, tb)
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/dask/local.py", line 315, in reraise
raise exc
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/dask/local.py", line 220, in execute_task
result = _execute_task(task, data)
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/dask/core.py", line 119, in _execute_task
return func(*(_execute_task(a, cache) for a in args))
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/dask/core.py", line 119, in <genexpr>
return func(*(_execute_task(a, cache) for a in args))
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/dask/core.py", line 119, in _execute_task
return func(*(_execute_task(a, cache) for a in args))
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/dask/core.py", line 119, in <genexpr>
return func(*(_execute_task(a, cache) for a in args))
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/dask/core.py", line 119, in _execute_task
return func(*(_execute_task(a, cache) for a in args))
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/dask/core.py", line 119, in <genexpr>
return func(*(_execute_task(a, cache) for a in args))
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/dask/core.py", line 119, in _execute_task
return func(*(_execute_task(a, cache) for a in args))
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/dask/core.py", line 119, in <genexpr>
return func(*(_execute_task(a, cache) for a in args))
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/dask/core.py", line 119, in _execute_task
return func(*(_execute_task(a, cache) for a in args))
File "/home/alan/Documents/dev/libs/aliby/aliby/experiment.py", line 83, in <lambda>
get_plane = delayed(lambda idx: pixels.getPlane(*idx))
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/omero/gateway/__init__.py", line 7492, in getPlane
planeList = list(self.getPlanes([(theZ, theC, theT)]))
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/omero/gateway/__init__.py", line 7559, in getTiles
raise exc
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/omero/gateway/__init__.py", line 7522, in getTiles
rawPixelsStore = self._prepareRawPixelsStore()
File "/home/alan/Documents/dev/libs/aliby/aliby/experiment.py", line 63, in _prepareRawPixelsStore
ps = self._conn.c.sf.createRawPixelsStore()
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/omero_API_ice.py", line 884, in createRawPixelsStore
return _M_omero.api.ServiceFactory._op_createRawPixelsStore.invoke(self, ((), _ctx))
Ice.UnknownException: exception ::Ice::UnknownException
{
unknown = ome.conditions.SessionTimeoutException: Session (started=2022-02-22 07:36:22.102, hits=84752, last access=2022-02-22 15:28:58.335) exceeded timeToIdle (600000) by 18372 ms
at ome.services.sessions.state.SessionCache.getDataNullOrThrowOnTimeout(SessionCache.java:470)
at ome.services.sessions.state.SessionCache.getSessionContext(SessionCache.java:368)
at ome.services.sessions.state.SessionCache.getSessionContext(SessionCache.java:353)
at ome.security.basic.CurrentDetails.login(CurrentDetails.java:164)
at ome.services.util.Executor$Impl.execute(Executor.java:439)
at ome.services.util.Executor$Impl.execute(Executor.java:392)
at ome.services.blitz.impl.ServiceFactoryI.isGuest(ServiceFactoryI.java:571)
at ome.services.blitz.impl.ServiceFactoryI.disallowGuest(ServiceFactoryI.java:585)
at ome.services.blitz.impl.ServiceFactoryI.createByName(ServiceFactoryI.java:483)
at ome.services.blitz.impl.ServiceFactoryI.createByName(ServiceFactoryI.java:476)
at ome.services.blitz.impl.ServiceFactoryI.createRawPixelsStore(ServiceFactoryI.java:395)
at omero.api._ServiceFactoryTie.createRawPixelsStore(_ServiceFactoryTie.java:107)
at omero.api._ServiceFactoryDisp.___createRawPixelsStore(_ServiceFactoryDisp.java:892)
at omero.api._ServiceFactoryDisp.__dispatch(_ServiceFactoryDisp.java:1247)
at IceInternal.Incoming.invoke(Incoming.java:221)
at Ice.ConnectionI.invokeAll(ConnectionI.java:2536)
at Ice.ConnectionI.dispatch(ConnectionI.java:1145)
at Ice.ConnectionI.message(ConnectionI.java:1056)
at IceInternal.ThreadPool.run(ThreadPool.java:395)
at IceInternal.ThreadPool.access$300(ThreadPool.java:12)
at IceInternal.ThreadPool$EventHandlerThread.run(ThreadPool.java:832)
at java.lang.Thread.run(Thread.java:748)
}
Segmented more experiments and found more exceptions, the first one looks like postprocessing and the second one is very cryptic. 19333
ura7hr_021: 100%|██████████████████████████████████████████████████████████████████████████| 240/240 [00:00<?, ?it/s]
Caught exception in worker thread (x = ura7hr_021):
Traceback (most recent call last):
File "/home/alan/Documents/dev/libs/aliby/aliby/pipeline.py", line 415, in create_pipeline
PostProcessor(filename, post_proc_params).run()
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/postprocessor/core/processor.py", line 311, in run
self.run_prepost()
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/postprocessor/core/processor.py", line 264, in run_prepost
self.pick_mother(mothers[mother_ids][0][1], target[1]),
IndexError: index 0 is out of bounds for axis 0 with size 0
ura7ha_010: 2%|▉ | 4/240 [01:33<1:39:55, 25.40s/it, 0% Clogged]Traceback (most recent call last):
File "h5py/_objects.pyx", line 193, in h5py._objects.ObjectID.__dealloc__
RuntimeError: Can't decrement id ref count (wrong B-tree signature)
Exception ignored in: 'h5py._objects.ObjectID.__dealloc__'
Traceback (most recent call last):
File "h5py/_objects.pyx", line 193, in h5py._objects.ObjectID.__dealloc__
RuntimeError: Can't decrement id ref count (wrong B-tree signature)
Traceback (most recent call last):
File "h5py/_objects.pyx", line 193, in h5py._objects.ObjectID.__dealloc__
RuntimeError: Can't decrement id ref count (wrong B-tree signature)
Exception ignored in: 'h5py._objects.ObjectID.__dealloc__'
Traceback (most recent call last):
File "h5py/_objects.pyx", line 193, in h5py._objects.ObjectID.__dealloc__
RuntimeError: Can't decrement id ref count (wrong B-tree signature)
All were run with these parameters
params = PipelineParameters.default(
general={
"expt_id": i,
# "tps": 20,
"distributed": 5,
"earlystop": dict(
min_tp=200,
thresh_pos_clogged=0.4,
thresh_trap_ncells=8,
ntps_to_eval=5,
thresh_trap_area=0.9,
),
"override_meta": True,
"server_info": {
"host": "islay.bio.ed.ac.uk",
"username": "upload",
"password": "gothamc1ty",
},
#
},
tiler={"tile_size": 117},
)
Found another one, in experiment 19203
Existing file ../data/19203_2020_09_30_downUpshift_twice_2_0_2_glu_ura8_ura8h360a_ura8h360r_00/ura8h360a_018.h5 will be used.
Caught exception in worker thread (x = ura8h36r_019):
Traceback (most recent call last):
File "/home/alan/Documents/dev/libs/aliby/aliby/pipeline.py", line 430, in create_pipeline
PostProcessor(filename, post_proc_params).run()
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/postprocessor/core/processor.py", line 311, in run
self.run_prepost()
File "/home/alan/.pyenv/versions/aliby/lib/python3.7/site-packages/postprocessor/core/processor.py", line 264, in run_prepost
self.pick_mother(mothers[mother_ids][0][1], target[1]),
IndexError: index 0 is out of bounds for axis 0 with size 0