CC: @henrypinkard
There appears to be a logic error in bridge.py:
Line 407:
def send_and_receive(self, message, timeout=None, give_up_condition=None):
"""
Send a message over the main socket
"""
with self._communication_lock:
if give_up_condition:
# keep trying until give up condition is true
if timeout is None:
timeout = 0.1
self._send_queue.put(message)
while not give_up_condition():
try:
response = self._response_queue.get(timeout=timeout)
except Empty:
pass
if isinstance(response, Exception):
raise response
return response
# rest of the function omitted for brevity
In the snippet above, there appear to be two issues:
- The while not give_up_condition loop only ever iterates once, because of the return response call which happens regardless of the value of response
- It is possible for response to never be assigned (in the case where queue.get raises the Empty exception). The isinstance(response, Exception) will then raise an exception and cause a crash: UnboundLocalError: cannot access local variable 'response' where it is not associated with a value
Some context:
I've encountered this bug rarely and sporadically in an application I am writing which, as part of some broader automation involving pumps/fluidics/etc., executes a Pycromanager Acquisition in a python Thread. Over the duration of an experiment, which lasts 24h, around 30 separate (non-overlapping) calls to imaging are made. Generally this works without a hitch, but rarely, and in as far as I can tell, non-deterministically, the call to with Acquisition triggers a crash.
Below is the verbose full traceback:
2024-12-09-19:08:51 - INFO: Starting imaging for well b1...
Traceback (most recent call last):
File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pyjavaz\bridge.py", line 724, in __del__
self._close()
File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pyjavaz\bridge.py", line 703, in _close
reply_json = self._send_and_receive(message, give_up_condition=
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pyjavaz\bridge.py", line 717, in _send_and_receive
return self._bridge.send_and_receive(message, give_up_condition=give_up_condition)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pyjavaz\bridge.py", line 422, in send_and_receive
if isinstance(response, Exception):
^^^^^^^^
UnboundLocalError: cannot access local variable 'response' where it is not associated with a value
Exception in thread Event sending:
Traceback (most recent call last):
File "C:\Users\InSituScope\scoop\apps\python\current\Lib\threading.py", line 1073, in _bootstrap_inner
self.run()
File "C:\Users\InSituScope\scoop\apps\python\current\Lib\threading.py", line 1010, in run
Exception in thread Thread-130 (wrapper):
Traceback (most recent call last):
File "C:\Users\InSituScope\scoop\apps\python\current\Lib\threading.py", line 1073, in _bootstrap_inner
self._target(*self._args, **self._kwargs)
File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pycromanager\acquisition\java_backend_acquisitions.py", line 33, in _run_acq_event_source
event_socket = PushSocket(event_port, debug=debug)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pyjavaz\wrappers.py", line 34, in __init__
self.run()
File "C:\Users\InSituScope\scoop\apps\python\current\Lib\threading.py", line 1010, in run
_DataSocket.__init__(self,
File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pyjavaz\bridge.py", line 44, in __init__
self._socket.bind("tcp://{}:{}".format(ip_address, port))
File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\zmq\sugar\socket.py", line 311, in bind
self._target(*self._args, **self._kwargs)
File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\src\opsautomation\scripts\interleaved.py", line 69, in wrapper
super().bind(addr)
File "_zmq.py", line 917, in zmq.backend.cython._zmq.Socket.bind
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\src\opsautomation\imaging\utils.py", line 230, in acquire
with Acquisition(
^^^^^^^^^^^^
File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pycromanager\acquisition\acq_constructor.py", line 45, in __new__
File "_zmq.py", line 179, in zmq.backend.cython._zmq._check_rc
return JavaBackendAcquisition(**named_args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pycromanager\acquisition\java_backend_acquisitions.py", line 314, in __init__
storage_java_class = data_sink.get_storage()
^^^^^^^^^^^^^^^^^^^^^
AttributeError: 'int' object has no attribute 'get_storage'
zmq.error.ZMQError: Invalid argument (addr='tcp://127.0.0.1:None')
CC: @henrypinkard
There appears to be a logic error in bridge.py:
Line 407:
def send_and_receive(self, message, timeout=None, give_up_condition=None): """ Send a message over the main socket """ with self._communication_lock: if give_up_condition: # keep trying until give up condition is true if timeout is None: timeout = 0.1 self._send_queue.put(message) while not give_up_condition(): try: response = self._response_queue.get(timeout=timeout) except Empty: pass if isinstance(response, Exception): raise response return response # rest of the function omitted for brevityIn the snippet above, there appear to be two issues:
Some context:
I've encountered this bug rarely and sporadically in an application I am writing which, as part of some broader automation involving pumps/fluidics/etc., executes a Pycromanager Acquisition in a python Thread. Over the duration of an experiment, which lasts 24h, around 30 separate (non-overlapping) calls to imaging are made. Generally this works without a hitch, but rarely, and in as far as I can tell, non-deterministically, the call to with Acquisition triggers a crash.
Below is the verbose full traceback:
2024-12-09-19:08:51 - INFO: Starting imaging for well b1... Traceback (most recent call last): File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pyjavaz\bridge.py", line 724, in __del__ self._close() File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pyjavaz\bridge.py", line 703, in _close reply_json = self._send_and_receive(message, give_up_condition= ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pyjavaz\bridge.py", line 717, in _send_and_receive return self._bridge.send_and_receive(message, give_up_condition=give_up_condition) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pyjavaz\bridge.py", line 422, in send_and_receive if isinstance(response, Exception): ^^^^^^^^ UnboundLocalError: cannot access local variable 'response' where it is not associated with a value Exception in thread Event sending: Traceback (most recent call last): File "C:\Users\InSituScope\scoop\apps\python\current\Lib\threading.py", line 1073, in _bootstrap_inner self.run() File "C:\Users\InSituScope\scoop\apps\python\current\Lib\threading.py", line 1010, in run Exception in thread Thread-130 (wrapper): Traceback (most recent call last): File "C:\Users\InSituScope\scoop\apps\python\current\Lib\threading.py", line 1073, in _bootstrap_inner self._target(*self._args, **self._kwargs) File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pycromanager\acquisition\java_backend_acquisitions.py", line 33, in _run_acq_event_source event_socket = PushSocket(event_port, debug=debug) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pyjavaz\wrappers.py", line 34, in __init__ self.run() File "C:\Users\InSituScope\scoop\apps\python\current\Lib\threading.py", line 1010, in run _DataSocket.__init__(self, File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pyjavaz\bridge.py", line 44, in __init__ self._socket.bind("tcp://{}:{}".format(ip_address, port)) File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\zmq\sugar\socket.py", line 311, in bind self._target(*self._args, **self._kwargs) File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\src\opsautomation\scripts\interleaved.py", line 69, in wrapper super().bind(addr) File "_zmq.py", line 917, in zmq.backend.cython._zmq.Socket.bind return func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\src\opsautomation\imaging\utils.py", line 230, in acquire with Acquisition( ^^^^^^^^^^^^ File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pycromanager\acquisition\acq_constructor.py", line 45, in __new__ File "_zmq.py", line 179, in zmq.backend.cython._zmq._check_rc return JavaBackendAcquisition(**named_args) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "C:\Users\InSituScope\Documents\OPSAutomation\OPSAutomation\ops_venv\Lib\site-packages\pycromanager\acquisition\java_backend_acquisitions.py", line 314, in __init__ storage_java_class = data_sink.get_storage() ^^^^^^^^^^^^^^^^^^^^^ AttributeError: 'int' object has no attribute 'get_storage' zmq.error.ZMQError: Invalid argument (addr='tcp://127.0.0.1:None')