Merge pull request #395 from vshekar/low-mag-cam-recovery

Retry and recovery code for loop detection

mxbluesky/plans/loop_detection.py

@@ -6,8 +6,9 @@
 import bluesky.plans as bp
 from bluesky.utils import FailedStatus
 from bluesky.preprocessors import finalize_decorator
+from ophyd.utils.errors import WaitTimeoutError
 import daq_utils
-from start_bs import db, two_click_low, loop_detector, gonio
+from start_bs import db, two_click_low, loop_detector, gonio, low_mag_cam_reset_signal
 from mxbluesky.plans.utils import mvr_with_retry, mv_with_retry
 
 logger = getLogger()
@@ -18,16 +19,35 @@ def cleanup_two_click_low():
         yield from bps.abs_set(two_click_low.jpeg.file_write_mode, 2, wait=True)
         yield from bps.abs_set(two_click_low.jpeg.file_template, "%s%s_%d.jpg", wait=True)
 
+def reset_low_mag_cam():
+    yield from bps.abs_set(two_click_low.cam.acquire, 0, wait=True)
+    yield from bps.sleep(3)
+    yield from bps.abs_set(low_mag_cam_reset_signal, 1, wait=True)
+    yield from bps.sleep(5)
+    yield from bps.abs_set(two_click_low.cam.acquire, 1, wait=True)
+
+def trigger_two_click():
+    tries = 0
+    max_tries = 3
+
+    while tries < max_tries:
+        try:
+            yield from bp.count([two_click_low], 1)
+            break  # If trigger succeeds, exit the loop
+        except (FailedStatus, WaitTimeoutError) as e:
+            tries += 1
+            logger.exception(f"Exception while triggering two click, retry #{tries} : {e}")
+            if tries == 3:
+                logger.exception("Resetting low mag camera")
+                yield from reset_low_mag_cam()
+                yield from bp.count([two_click_low], 1)
+
 @finalize_decorator(cleanup_two_click_low)
 def detect_loop(sample_detection: "Dict[str, float|int]"):
     # face on attempt, most features, should work
     yield from bps.abs_set(two_click_low.cam_mode, "two_click", wait=True)
     logger.info("Starting loop centering")
-    #two_click_low.cam_mode.set("two_click")
-    try:
-        yield from bp.count([two_click_low], 1)
-    except FailedStatus:
-        yield from bp.count([two_click_low], 1)
+    yield from trigger_two_click()
 
     loop_detector.filename.set(two_click_low.jpeg.full_file_name.get())
 
@@ -69,6 +89,7 @@ def detect_loop(sample_detection: "Dict[str, float|int]"):
     # orthogonal face, use loop model only if predicted width matches face on
     # otherwise, threshold
     yield from bps.mv(gonio.o, sample_detection["face_on_omega"]+90)
+    yield from trigger_two_click()
 
     try:
         yield from bp.count([two_click_low], 1)

mxbluesky/plans/top_view.py

@@ -24,8 +24,14 @@
 logger = getLogger()
 
 def cleanup_topcam():
-    yield from bps.abs_set(top_aligner_slow.topcam.cam.acquire, 1, wait=True)
-    yield from bps.abs_set(top_aligner_fast.zebra.pos_capt.direction, 0, wait=True)
+    try:
+        yield from bps.abs_set(top_aligner_slow.topcam.cam.acquire, 1, wait=True, timeout=4)
+        yield from bps.abs_set(top_aligner_fast.zebra.pos_capt.direction, 0, wait=True, timeout=4)
+    except WaitTimeoutError as error:
+        logger.exception(f"Exception in cleanup_topcam, trying again: {error}")
+        yield from bps.abs_set(top_aligner_slow.topcam.cam.acquire, 1, wait=True, timeout=4)
+        yield from bps.abs_set(top_aligner_fast.zebra.pos_capt.direction, 0, wait=True, timeout=4)
+
 
 def inner_pseudo_fly_scan(*args, **kwargs):
     scan_uid = yield from bp.count(*args, **kwargs)
@@ -74,6 +80,12 @@ def inner_pseudo_fly_scan(*args, **kwargs):
 
     return delta_y, delta_z, omega_min
 
+def setup_transition_signals(target_state, zebra_dir, cam_mode):
+    yield from bps.abs_set(top_aligner_fast.target_gov_state, target_state, wait=True)
+    yield from bps.abs_set(top_aligner_fast.zebra.pos_capt.direction, zebra_dir, wait=True, timeout=4)
+    yield from bps.abs_set(top_aligner_fast.topcam.cam_mode, cam_mode, wait=True, timeout=4)
+    yield from bps.sleep(0.1)
+
 
 @finalize_decorator(cleanup_topcam)
 def topview_optimized():
@@ -94,10 +106,12 @@ def topview_optimized():
     logger.info("Updated TA work pos, starting transition to TA")
 
     # SE -> TA
-    yield from bps.abs_set(top_aligner_fast.target_gov_state, "TA", wait=True)
-    yield from bps.abs_set(top_aligner_fast.zebra.pos_capt.direction, 0, wait=True)
-    yield from bps.abs_set(top_aligner_fast.topcam.cam_mode, CamMode.COARSE_ALIGN.value)
-    yield from bps.sleep(0.1)
+    try:
+        yield from setup_transition_signals("TA", 0, CamMode.COARSE_ALIGN.value)
+    except WaitTimeoutError as error:
+        logger.exception(f"Exception while setting SE to TA signals, trying again: {error}")
+        yield from setup_transition_signals("TA", 0, CamMode.COARSE_ALIGN.value)
+
     logger.info("Starting 1st inner fly scan")
 
     try:
@@ -124,10 +138,11 @@ def topview_optimized():
     yield from set_SA_work_pos(delta_y, delta_z)
     logger.info("Starting transition to SA")
     # TA -> SA
-    yield from bps.abs_set(top_aligner_fast.target_gov_state, "SA", wait=True)
-    yield from bps.abs_set(top_aligner_fast.zebra.pos_capt.direction, 1, wait=True)
-    yield from bps.abs_set(top_aligner_fast.topcam.cam_mode, CamMode.FINE_FACE.value)
-    yield from bps.sleep(0.1)
+    try:
+        yield from setup_transition_signals("SA", 1, CamMode.FINE_FACE.value)
+    except WaitTimeoutError as error:
+        logger.exception(f"Exception while setting TA to SA signals, trying again: {error}")
+        yield from setup_transition_signals("SA", 1, CamMode.FINE_FACE.value)
     logger.info("Starting 2nd inner fly scan")
     try:
         delta_y, delta_z, omega_min = yield from inner_pseudo_fly_scan(

start_bs.py

@@ -138,6 +138,7 @@ class SampleXYZ(Device):
     work_pos = WorkPositions("XF:17IDB-ES:AMX", name="work_pos")
     mount_pos = MountPositions("XF:17IDB-ES:AMX", name="mount_pos")
     two_click_low = TwoClickLowMag("XF:17IDB-ES:AMX{Cam:6}", name="two_click_low")
+    low_mag_cam_reset_signal = EpicsSignal('XF:17IDB-CT:AMX{IOC:CAM06}:SysReset')
     gonio = GoniometerStack("XF:17IDB-ES:AMX{Gon:1", name="gonio")
     loop_detector = LoopDetector(name="loop_detector")
     top_aligner_fast = TopAlignerFast(name="top_aligner_fast", gov_robot=gov_robot)
@@ -170,6 +171,7 @@ class SampleXYZ(Device):
     work_pos = WorkPositions("XF:17IDC-ES:FMX", name="work_pos")
     mount_pos = MountPositions("XF:17IDC-ES:FMX", name="mount_pos")
     two_click_low = TwoClickLowMag("XF:17IDC-ES:FMX{Cam:7}", name="two_click_low")
+    low_mag_cam_reset_signal = EpicsSignal('XF:17IDC-CT:FMX{IOC:CAM07}:SysReset')
     gonio = GoniometerStack("XF:17IDC-ES:FMX{Gon:1", name="gonio")
     loop_detector = LoopDetector(name="loop_detector")
     top_aligner_fast = TopAlignerFast(name="top_aligner_fast", gov_robot=gov_robot)