From af3590c668fccadc2f1a4bfb88d4047fcfe4a583 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Tue, 31 Jan 2023 10:00:22 -0500 Subject: [PATCH 01/39] change call for insertRasterResult to allow remote call --- ispybLib.py | 3 ++- lsdcGui.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ispybLib.py b/ispybLib.py index bc0fd2ed..b66ec189 100644 --- a/ispybLib.py +++ b/ispybLib.py @@ -387,13 +387,14 @@ def createDataCollection(directory, filePrefix, jpegImageFilename, params, reque # if request_type == 'screening': # params['overlap'] = 89.0 -def insertRasterResult(request,visitName): +def insertRasterResult(request_id,visitName): return try: sessionid = core.retrieve_visit_id(visitName) except ISPyBNoResultException as e: logger.error("insertRasterResult - caught ISPyBNoResultException: '%s'. make sure visit name is in the format mx999999-1234. NOT HAVING MX IN FRONT IS A SIGN OF PROBLEMS - try newVisit() in that case." % e) return + request = db_lib.getRequestByID(request_id) sample = request['sample'] # this needs to be created and linked to a DC group #result_obj = result['result_obj'] this doesn't appear to be used -DK request_obj = request['request_obj'] diff --git a/lsdcGui.py b/lsdcGui.py index 52167feb..68f8f3ea 100644 --- a/lsdcGui.py +++ b/lsdcGui.py @@ -4090,7 +4090,7 @@ def takeRasterSnapshot(self,rasterReq): self.saveVidSnapshotCB("Raster Result from sample " + str(rasterReq["request_obj"]["file_prefix"]),useOlog=False,reqID=rasterReq["uid"],rasterHeatJpeg=jpegImageFilename) self.saveVidSnapshotCB("Raster Result from sample " + str(rasterReq["request_obj"]["file_prefix"]),useOlog=False,reqID=rasterReq["uid"],rasterHeatJpeg=jpegImageFilename) try: - ispybLib.insertRasterResult(rasterReq,visitName) + ispybLib.insertRasterResult(rasterReq["uid"],visitName) except Exception as e: logger.error(f'Exception while writing raster result: {e}') From 54517851fb4f746e28d3febf254ea85adaf02002 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Tue, 31 Jan 2023 10:01:38 -0500 Subject: [PATCH 02/39] send insertRasterResult() call to server --- lsdcGui.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lsdcGui.py b/lsdcGui.py index 68f8f3ea..bb99ad55 100644 --- a/lsdcGui.py +++ b/lsdcGui.py @@ -4089,10 +4089,7 @@ def takeRasterSnapshot(self,rasterReq): logger.info("saving raster snapshot") self.saveVidSnapshotCB("Raster Result from sample " + str(rasterReq["request_obj"]["file_prefix"]),useOlog=False,reqID=rasterReq["uid"],rasterHeatJpeg=jpegImageFilename) self.saveVidSnapshotCB("Raster Result from sample " + str(rasterReq["request_obj"]["file_prefix"]),useOlog=False,reqID=rasterReq["uid"],rasterHeatJpeg=jpegImageFilename) - try: - ispybLib.insertRasterResult(rasterReq["uid"],visitName) - except Exception as e: - logger.error(f'Exception while writing raster result: {e}') + self.send_to_server(f"ispybLib.insertRasterResult({rasterReq['uid']},{visitName})") From 698d8b3ed6b87ee9377777a53987683a64dc0ee1 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Tue, 31 Jan 2023 10:02:17 -0500 Subject: [PATCH 03/39] remove ispybLib requirement from GUI --- lsdcGui.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lsdcGui.py b/lsdcGui.py index bb99ad55..f9263b53 100644 --- a/lsdcGui.py +++ b/lsdcGui.py @@ -70,10 +70,6 @@ def filter(self, record): #handler2.setFormatter(myformat) logger.addHandler(handler1) #logger.addHandler(handler2) -try: - import ispybLib -except Exception as e: - logger.error("lsdcGui: ISPYB import error, %s" % e) import raddoseLib global sampleNameDict From 0b7f3868e9d2810a99ad778a40684aa9cc38cffd Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Wed, 1 Feb 2023 09:30:44 -0500 Subject: [PATCH 04/39] fix string output so literals are surrounded with single quotes --- lsdcGui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lsdcGui.py b/lsdcGui.py index f9263b53..6ad53117 100644 --- a/lsdcGui.py +++ b/lsdcGui.py @@ -4085,7 +4085,7 @@ def takeRasterSnapshot(self,rasterReq): logger.info("saving raster snapshot") self.saveVidSnapshotCB("Raster Result from sample " + str(rasterReq["request_obj"]["file_prefix"]),useOlog=False,reqID=rasterReq["uid"],rasterHeatJpeg=jpegImageFilename) self.saveVidSnapshotCB("Raster Result from sample " + str(rasterReq["request_obj"]["file_prefix"]),useOlog=False,reqID=rasterReq["uid"],rasterHeatJpeg=jpegImageFilename) - self.send_to_server(f"ispybLib.insertRasterResult({rasterReq['uid']},{visitName})") + self.send_to_server(f"ispybLib.insertRasterResult('{rasterReq['uid']}','{visitName}')") From 978b8a267163a04bcef59ea5912ee0afa5945863 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Tue, 16 May 2023 16:22:47 -0400 Subject: [PATCH 05/39] set visitDirectory config variable when starting server * use this as the source of truth for both GUI and server --- daq_main2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/daq_main2.py b/daq_main2.py index 07d457c3..afa1cc5a 100755 --- a/daq_main2.py +++ b/daq_main2.py @@ -35,6 +35,7 @@ logger.addHandler(handler1) logger.addHandler(handler2) +setBlConfig("visitDirectory", os.getcwd()) sitefilename = "" global command_list,immediate_command_list,z command_list = [] From b8f659975b261c6377c118d482932e0f84350a57 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Tue, 16 May 2023 16:23:49 -0400 Subject: [PATCH 06/39] remove os.getcwd() and use visitDirectory config var instead --- daq_macros.py | 2 +- daq_utils.py | 4 ++-- embl_robot.py | 2 +- gui/control_main.py | 10 +++------- gui/data_loc_info.py | 2 +- top_view.py | 4 ++-- 6 files changed, 10 insertions(+), 14 deletions(-) diff --git a/daq_macros.py b/daq_macros.py index 42887266..58be0967 100644 --- a/daq_macros.py +++ b/daq_macros.py @@ -3207,7 +3207,7 @@ def loop_center_xrec(): pic_prefix = "findloop" output_file = 'xrec_result.txt' clean_up_files(pic_prefix, output_file) - zebraCamDaq(0,360,40,.4,pic_prefix,os.getcwd(),0) + zebraCamDaq(0,360,40,.4,pic_prefix,getBlConfig("visitDirectory"),0) comm_s = f'xrec {os.environ["CONFIGDIR"]}/xrec_360_40Fast.txt {output_file}' logger.info(comm_s) try: diff --git a/daq_utils.py b/daq_utils.py index 0f4a10d0..6dca8948 100644 --- a/daq_utils.py +++ b/daq_utils.py @@ -209,7 +209,7 @@ def createDefaultRequest(sample_id,createVisit=True): setProposalID(propNum,createVisit) screenDist, screenEnergy, screenExptime, screenPhiend, screenPhist, screenReso, screenTransmissionPercent, screenWidth, screenbeamHeight, screenbeamWidth = getScreenDefaultParams() sampleName = str(db_lib.getSampleNamebyID(sample_id)) - basePath = os.getcwd() + basePath = getBlConfig("visitDirectory") runNum = db_lib.getSampleRequestCount(sample_id) (puckPosition,samplePositionInContainer,containerID) = db_lib.getCoordsfromSampleID(beamline,sample_id) request = {"sample": sample_id} @@ -284,7 +284,7 @@ def create_filename(prefix,number): else: tmp_filename = "%s_%05d.cbf" % (prefix,int(number)) if (prefix[0] != "/"): - cwd = os.getcwd() + cwd = getBlConfig("visitDirectory") filename = "%s/%s" % (cwd,tmp_filename) else: filename = tmp_filename diff --git a/embl_robot.py b/embl_robot.py index ee719cc4..653f0daf 100644 --- a/embl_robot.py +++ b/embl_robot.py @@ -207,7 +207,7 @@ def preMount(self, gov_robot, puckPos, pinPos, sampID, **kwargs): prefix90 = sampName + "_" + str(puckPos) + "_" + str(pinPos) + "_" + str(reqCount) + "_PA_90" kwargs['prefix1'] = prefix1 kwargs['prefix90'] = prefix90 - top_view.topViewSnap(prefix1,os.getcwd()+"/pinAlign",1,acquire=0) + top_view.topViewSnap(prefix1,getBlConfig("visitDirectory")+"/pinAlign",1,acquire=0) except Exception as e: e_s = str(e) message = "TopView check ERROR, will continue: " + e_s diff --git a/gui/control_main.py b/gui/control_main.py index a856a02c..751bffe7 100644 --- a/gui/control_main.py +++ b/gui/control_main.py @@ -1685,20 +1685,16 @@ def saveVidSnapshotCB( if reqID != None: filePrefix = db_lib.getRequestByID(reqID)["request_obj"]["file_prefix"] imagePath = ( - os.getcwd() + "/snapshots/" + filePrefix + str(int(now)) + ".jpg" + f"{getBlConfig('visitDirectory')}/snapshots/{filePrefix}{int(now)}.jpg" ) else: if self.dataPathGB.prefix_ledit.text() != "": imagePath = ( - os.getcwd() - + "/snapshots/" - + str(self.dataPathGB.prefix_ledit.text()) - + str(int(now)) - + ".jpg" + f"{getBlConfig('visitDirectory')}/snapshots/{self.dataPathGB.prefix_ledit.text()}{int(now)}.jpg" ) else: imagePath = ( - os.getcwd() + "/snapshots/capture" + str(int(now)) + ".jpg" + f"{getBlConfig('visitDirectory')}/snapshots/capture{int(now)}.jpg" ) else: imagePath = rasterHeatJpeg diff --git a/gui/data_loc_info.py b/gui/data_loc_info.py index e868be56..181130f3 100644 --- a/gui/data_loc_info.py +++ b/gui/data_loc_info.py @@ -23,7 +23,7 @@ def __init__(self, parent: "ControlMain"): self.hBoxDPathParams1 = QtWidgets.QHBoxLayout() self.basePathLabel = QtWidgets.QLabel("Base Path:") self.base_path_ledit = QtWidgets.QLabel() # leave editable for now - self.base_path_ledit.setText(os.getcwd()) + self.base_path_ledit.setText(getBlConfig("visitDirectory")) self.base_path_ledit.setTextInteractionFlags(QtCore.Qt.TextSelectableByMouse) # self.base_path_ledit.textChanged[str].connect(self.basePathTextChanged) self.browseBasePathButton = QtWidgets.QPushButton("Browse...") diff --git a/top_view.py b/top_view.py index d76337f1..03b73698 100644 --- a/top_view.py +++ b/top_view.py @@ -31,10 +31,10 @@ def wait90TopviewThread(gov_robot, prefix1,prefix90): time.sleep(0.10) try: topViewWrite() - topViewSnap(prefix90,os.getcwd()+"/pinAlign",1) + topViewSnap(prefix90,getBlConfig("visitDirectory")+"/pinAlign",1) snapshot1Name = prefix1+"_001.jpg" snapshot2Name = prefix90+"_001.jpg" - if (not filecmp.cmp(os.getcwd()+"/pinAlign/"+snapshot1Name,os.getcwd()+"/pinAlign/"+snapshot2Name)): #this would mean something is wrong if true because the pictures are identical + if (not filecmp.cmp(getBlConfig("visitDirectory")+"/pinAlign/"+snapshot1Name,getBlConfig("visitDirectory")+"/pinAlign/"+snapshot2Name)): #this would mean something is wrong if true because the pictures are identical comm_s = os.environ["LSDCHOME"] + "/runPinAlign.py " + snapshot1Name + " " + snapshot2Name logger.info(comm_s) lines = os.popen(comm_s).readlines() From 01ad2931a3dd12e52c4765c5ef9382182c9acfe4 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Tue, 16 May 2023 16:24:59 -0400 Subject: [PATCH 07/39] check that GUI is started in server visit directory * on startup. if not started there, show an error and exit --- lsdcGui.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lsdcGui.py b/lsdcGui.py index 77793b05..1cfc75c5 100644 --- a/lsdcGui.py +++ b/lsdcGui.py @@ -2,6 +2,7 @@ The GUI for the LSDC system """ import sys +import os from qtpy import QtWidgets import daq_utils from utils.healthcheck import perform_checks @@ -32,6 +33,9 @@ def filter(self, record): handler1.setFormatter(myformat) logger.addHandler(handler1) +if daq_utils.getBlConfig("visitDirectory") != os.getcwd(): + logger.error("The GUI has not been started in the visit directory. Aborting!") + sys.exit(1) def main(): logger.info("Starting LSDC...") From ab0662f879bc76774bc8bd1b84817ff370532b80 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Tue, 16 May 2023 16:30:01 -0400 Subject: [PATCH 08/39] monitor server visit directory from GUI * if it has changed, kill the GUI, and do this check in a separate thread --- gui/control_main.py | 5 ++++- threads.py | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/gui/control_main.py b/gui/control_main.py index 751bffe7..ba6001d7 100644 --- a/gui/control_main.py +++ b/gui/control_main.py @@ -49,7 +49,7 @@ ) from gui.raster import RasterCell, RasterGroup from QPeriodicTable import QPeriodicTable -from threads import RaddoseThread, VideoThread +from threads import RaddoseThread, VideoThread, ServerCheckThread logger = logging.getLogger() try: @@ -1392,6 +1392,9 @@ def createSampleTab(self): lambda frame: self.updateCam(self.pixmap_item_HutchTop, frame) ) hutchTopCamThread.start() + serverCheckThread = ServerCheckThread( + parent=self) + serverCheckThread.start() def updateCam(self, pixmapItem: "QGraphicsPixmapItem", frame): pixmapItem.setPixmap(frame) diff --git a/threads.py b/threads.py index 5f0c257b..a946f76a 100644 --- a/threads.py +++ b/threads.py @@ -1,9 +1,12 @@ from qtpy.QtCore import QThread, QTimer, QEventLoop, Signal, QPoint, Qt, QObject from qtpy import QtGui from PIL import Image, ImageQt +import os +import sys import urllib from io import BytesIO import logging +from daq_lib import getBlConfig import raddoseLib logger = logging.getLogger() @@ -88,3 +91,8 @@ def run(self): self.lifetime.emit(lifetime_value) +class ServerCheckThread(QThread): + def run(self): + if getBlConfig("visitDirectory") != os.getcwd(): + logger.error("The server visit directory has changed, stopping!") + sys.exit(1) From 57d0eac3b0acfa40d87178d78c52aae6a0aa74c4 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Wed, 17 May 2023 13:27:02 -0400 Subject: [PATCH 09/39] create a visit directory checker that runs regularly * use QThread to allow ability to repeat --- threads.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/threads.py b/threads.py index a946f76a..a7622bd5 100644 --- a/threads.py +++ b/threads.py @@ -92,7 +92,13 @@ def run(self): class ServerCheckThread(QThread): + def __init__(self, *args, delay=60, **kwargs): + self.delay = delay + QThread.__init__(self, *args, **kwargs) + def run(self): - if getBlConfig("visitDirectory") != os.getcwd(): - logger.error("The server visit directory has changed, stopping!") - sys.exit(1) + while True: + if getBlConfig("visitDirectory") != os.getcwd(): + logger.error("The server visit directory has changed, stopping!") + sys.exit(1) + self.msleep(self.delay) From f629116165bc042c34f6435651e61dc7cef8d237 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Wed, 17 May 2023 13:55:55 -0400 Subject: [PATCH 10/39] move GUI current and visit directory check to healthcheck --- lsdcGui.py | 5 ----- utils/healthcheck.py | 7 ++++++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lsdcGui.py b/lsdcGui.py index 1cfc75c5..81984f0f 100644 --- a/lsdcGui.py +++ b/lsdcGui.py @@ -2,7 +2,6 @@ The GUI for the LSDC system """ import sys -import os from qtpy import QtWidgets import daq_utils from utils.healthcheck import perform_checks @@ -33,10 +32,6 @@ def filter(self, record): handler1.setFormatter(myformat) logger.addHandler(handler1) -if daq_utils.getBlConfig("visitDirectory") != os.getcwd(): - logger.error("The GUI has not been started in the visit directory. Aborting!") - sys.exit(1) - def main(): logger.info("Starting LSDC...") perform_checks() diff --git a/utils/healthcheck.py b/utils/healthcheck.py index ca109644..f57bf24b 100644 --- a/utils/healthcheck.py +++ b/utils/healthcheck.py @@ -61,6 +61,7 @@ def check_daq_utils(): ) def check_working_directory(): import daq_utils + import os working_dir = Path.cwd() home_dir = Path.home() if home_dir in working_dir.parents or home_dir == working_dir: @@ -72,6 +73,11 @@ def check_working_directory(): return False else: return True + if daq_utils.getBlConfig("visitDirectory") != os.getcwd(): + check_working_directory.remediation = ("Working directory mismatch. Please start LSDC GUI in the same folder as the server is running.") + return False + return True + @healthcheck( name='database lib', @@ -121,4 +127,3 @@ def perform_checks(): handle_fail(check) print("End checks") print(u'\u2500' * 20) - \ No newline at end of file From 61b1a0dab23d17154101def10f239bdb0a613702 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Wed, 17 May 2023 14:00:08 -0400 Subject: [PATCH 11/39] set default server (visit directory) check interval to 60 sec * make this a configurable value --- config_params.py | 1 + gui/control_main.py | 3 ++- threads.py | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config_params.py b/config_params.py index 26038eae..5c3b7f26 100644 --- a/config_params.py +++ b/config_params.py @@ -49,6 +49,7 @@ class RasterStatus(Enum): HUTCH_TIMER_DELAY = 500 SAMPLE_TIMER_DELAY = 0 +SERVER_CHECK_DELAY = 60000 ROBOT_MIN_DISTANCE = 200.0 ROBOT_DISTANCE_TOLERANCE = 0.050 diff --git a/gui/control_main.py b/gui/control_main.py index ba6001d7..1c313a40 100644 --- a/gui/control_main.py +++ b/gui/control_main.py @@ -26,6 +26,7 @@ from config_params import ( CRYOSTREAM_ONLINE, HUTCH_TIMER_DELAY, + SERVER_CHECK_DELAY, RASTER_GUI_XREC_FILL_DELAY, SAMPLE_TIMER_DELAY, VALID_DET_DIST, @@ -1393,7 +1394,7 @@ def createSampleTab(self): ) hutchTopCamThread.start() serverCheckThread = ServerCheckThread( - parent=self) + parent=self, delay=SERVER_CHECK_DELAY) serverCheckThread.start() def updateCam(self, pixmapItem: "QGraphicsPixmapItem", frame): diff --git a/threads.py b/threads.py index a7622bd5..62da4d24 100644 --- a/threads.py +++ b/threads.py @@ -7,6 +7,7 @@ from io import BytesIO import logging from daq_lib import getBlConfig +from config_params import SERVER_CHECK_DELAY import raddoseLib logger = logging.getLogger() @@ -92,7 +93,7 @@ def run(self): class ServerCheckThread(QThread): - def __init__(self, *args, delay=60, **kwargs): + def __init__(self, *args, delay=SERVER_CHECK_DELAY, **kwargs): self.delay = delay QThread.__init__(self, *args, **kwargs) From e78a459040e67f39c4104b3153acb2ae0edb4ff7 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Tue, 6 Jun 2023 10:27:47 -0400 Subject: [PATCH 12/39] shorten the visit directory change checking cycle * the GUI should terminate quickly after the visit directory is changed to prevent the user collecting data in an undesired place --- config_params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config_params.py b/config_params.py index 5c3b7f26..55090376 100644 --- a/config_params.py +++ b/config_params.py @@ -49,7 +49,7 @@ class RasterStatus(Enum): HUTCH_TIMER_DELAY = 500 SAMPLE_TIMER_DELAY = 0 -SERVER_CHECK_DELAY = 60000 +SERVER_CHECK_DELAY = 2000 ROBOT_MIN_DISTANCE = 200.0 ROBOT_DISTANCE_TOLERANCE = 0.050 From 0b993f08c7b906b3b92025d026f1cb1d4fef55e9 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Tue, 6 Jun 2023 10:28:53 -0400 Subject: [PATCH 13/39] print out a visit changed message to the terminal * user is not likely to have the GUI log open, so also ensure the message is printed out to the terminal --- threads.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/threads.py b/threads.py index 62da4d24..e1c386e6 100644 --- a/threads.py +++ b/threads.py @@ -100,6 +100,8 @@ def __init__(self, *args, delay=SERVER_CHECK_DELAY, **kwargs): def run(self): while True: if getBlConfig("visitDirectory") != os.getcwd(): - logger.error("The server visit directory has changed, stopping!") + message = "The server visit directory has changed, stopping!" + logger.error(message) + print(message) sys.exit(1) self.msleep(self.delay) From 56162dff3f5bcf8bf388dfd44595bd1ee4ef1b0f Mon Sep 17 00:00:00 2001 From: Shekar V Date: Tue, 6 Jun 2023 14:55:02 -0400 Subject: [PATCH 14/39] Update healthcheck.py Added checks for server: - Checks to if env file exists - Checks if CURRENT_VISIT_DIR has a value and is valid --- utils/healthcheck.py | 61 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/utils/healthcheck.py b/utils/healthcheck.py index f57bf24b..8107a38b 100644 --- a/utils/healthcheck.py +++ b/utils/healthcheck.py @@ -41,6 +41,7 @@ def dec(f): def start(): return True +# GUI checks @healthcheck( name='import daq_utils', remediation='', # Dynamic remediation defined in function @@ -100,30 +101,70 @@ def handle_fail(check): print("End checks") print(u'\u2500' * 20) sys.exit('Fatal error, exiting...') - + +# Server checks +@healthcheck(name="server working directory", remediation="", fatal=True) +def check_curr_visit_dir() -> bool: + if "CURRENT_VISIT_DIR" not in os.environ: + check_curr_visit_dir.remediation = ( + "CURRENT_VISIT_DIR environment variable not found" + ) + return False + if os.environ["CURRENT_VISIT_DIR"] != "": + check_curr_visit_dir.remediation = "CURRENT_VISIT_DIR is empty" + return False + current_visit_dir = Path(os.environ["CURRENT_VISIT_DIR"]) + if not current_visit_dir.exists(): + check_curr_visit_dir.remediation = ( + f"CURRENT_VISIT_DIR = {current_visit_dir} does not exist" + ) + return False + return True + + +@healthcheck(name="existence of environment file", remediation="", fatal=True) +def check_env_file() -> bool: + import daq_utils + + env_path = Path(f"/nsls2/software/mx/current_visit_lsdc_{daq_utils.beamline}") + if not env_path.exists(): + check_env_file.remidiation = f"Environment file not found at {env_path}" + return False + return True + def perform_checks(): - """Call this function to contruct a DAG where each node is evaluated using - breadth first search. DAGs allow certain tests to be run only after passing + """Call this function to contruct a DAG where each node is evaluated using + breadth first search. DAGs allow certain tests to be run only after passing specific tests. For example """ + check_functions = [check_daq_utils, check_working_directory, check_db] + run_checks(check_functions) + + +def perform_server_checks(): + check_functions = [check_env_file, check_curr_visit_dir] + run_checks(check_functions) + + +def run_checks(check_functions): checks = DiGraph() - for c in [check_daq_utils, check_working_directory, check_db]: + for c in check_functions: for d in c.depends: checks.add_edge(d, c) - print(u'\u2500' * 20) + print("\u2500" * 20) print("Begin checks") for check in bfs_tree(checks, start): try: if all([parent.passed for parent in checks.predecessors(check)]): - print(f'Checking {check.name}...', end='\t') + print(f"Checking {check.name}...", end="\t") if check(): - print(f'{bcolors.OKGREEN}Success{bcolors.ENDC}') + print(f"{bcolors.OKGREEN}Success{bcolors.ENDC}") else: handle_fail(check) except Exception as e: - print(f'Exception: {e}') - logger.error(f'Exception during checks {e}') + print(f"Exception: {e}") + logger.error(f"Exception during checks {e}") handle_fail(check) print("End checks") - print(u'\u2500' * 20) + print("\u2500" * 20) From c2724c8eb7fb0759b49a4662cf766b3a1ead9b2f Mon Sep 17 00:00:00 2001 From: Shekar V Date: Tue, 6 Jun 2023 14:56:38 -0400 Subject: [PATCH 15/39] Added server checks to daq_main2.py --- daq_main2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/daq_main2.py b/daq_main2.py index afa1cc5a..a987aa65 100755 --- a/daq_main2.py +++ b/daq_main2.py @@ -5,7 +5,7 @@ import sys import os from daq_main_common import pybass_init, run_server - +from utils.healthcheck import perform_server_checks #TODO understand why imports are required here - GUI requires imports in daq_main_common from daq_macros import * from daq_lib import * @@ -35,6 +35,7 @@ logger.addHandler(handler1) logger.addHandler(handler2) +perform_server_checks() setBlConfig("visitDirectory", os.getcwd()) sitefilename = "" global command_list,immediate_command_list,z From 713237f4da0ec16ebc0cfd737d736bed028fc4e9 Mon Sep 17 00:00:00 2001 From: Shekar V Date: Tue, 6 Jun 2023 15:00:36 -0400 Subject: [PATCH 16/39] Fixed check_curr_visit_dir in healthcheck.py --- utils/healthcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/healthcheck.py b/utils/healthcheck.py index 8107a38b..ec47e5e8 100644 --- a/utils/healthcheck.py +++ b/utils/healthcheck.py @@ -110,7 +110,7 @@ def check_curr_visit_dir() -> bool: "CURRENT_VISIT_DIR environment variable not found" ) return False - if os.environ["CURRENT_VISIT_DIR"] != "": + if os.environ["CURRENT_VISIT_DIR"] == "": check_curr_visit_dir.remediation = "CURRENT_VISIT_DIR is empty" return False current_visit_dir = Path(os.environ["CURRENT_VISIT_DIR"]) From 9fc8f3191f0dcfd6bc6ecb2713230f048bf978ec Mon Sep 17 00:00:00 2001 From: Shekar V Date: Wed, 7 Jun 2023 14:02:42 -0400 Subject: [PATCH 17/39] Checks if current visit dir exists in BASE_DATA_DIRS --- utils/healthcheck.py | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/utils/healthcheck.py b/utils/healthcheck.py index ec47e5e8..25bbc50e 100644 --- a/utils/healthcheck.py +++ b/utils/healthcheck.py @@ -2,7 +2,7 @@ import logging from pathlib import Path from networkx import DiGraph, bfs_tree - +import os logger = logging.getLogger() class bcolors: @@ -105,20 +105,43 @@ def handle_fail(check): # Server checks @healthcheck(name="server working directory", remediation="", fatal=True) def check_curr_visit_dir() -> bool: - if "CURRENT_VISIT_DIR" not in os.environ: + # Check if current visit dir is valid + visit_dir_env_var = "CURRENT_VISIT_DIR" + if visit_dir_env_var not in os.environ: check_curr_visit_dir.remediation = ( - "CURRENT_VISIT_DIR environment variable not found" + f"{visit_dir_env_var} environment variable not found" ) return False - if os.environ["CURRENT_VISIT_DIR"] == "": - check_curr_visit_dir.remediation = "CURRENT_VISIT_DIR is empty" + if os.environ[visit_dir_env_var] == "": + check_curr_visit_dir.remediation = f"{visit_dir_env_var} is empty" return False - current_visit_dir = Path(os.environ["CURRENT_VISIT_DIR"]) + current_visit_dir = Path(os.environ[visit_dir_env_var]) if not current_visit_dir.exists(): check_curr_visit_dir.remediation = ( - f"CURRENT_VISIT_DIR = {current_visit_dir} does not exist" + f"{visit_dir_env_var} = {current_visit_dir} does not exist" ) return False + + # Check if current visit dir is one of the dirs defined in BASE_DATA_DIRS + base_dir_env_var = "BASE_DATA_DIRS" + if base_dir_env_var not in os.environ: + check_curr_visit_dir.remediation = f"{base_dir_env_var} evironment variable not found" + return False + # Splitting on : if there are multiple base dirs + base_dirs = [Path(p) for p in os.environ[base_dir_env_var].split(":")] + pass_dir_found = False + for i, part in enumerate(reversed(current_visit_dir.parts)): + if 'pass-' in part: + pass_dir_found = True + if current_visit_dir.parents[i] not in base_dirs: + check_curr_visit_dir.remediation = f"{current_visit_dir.parents[i]} not found in {base_dirs}" + return False + else: + break + if not pass_dir_found: + check_curr_visit_dir.remediation = f"Pass folder not found in {current_visit_dir}" + return False + return True From 884a802c6613c46a039edf603d22672d6fac785a Mon Sep 17 00:00:00 2001 From: Shekar V Date: Thu, 8 Jun 2023 10:01:11 -0400 Subject: [PATCH 18/39] Added beamline tla to environment variable --- utils/healthcheck.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/healthcheck.py b/utils/healthcheck.py index 25bbc50e..416c2a70 100644 --- a/utils/healthcheck.py +++ b/utils/healthcheck.py @@ -105,6 +105,7 @@ def handle_fail(check): # Server checks @healthcheck(name="server working directory", remediation="", fatal=True) def check_curr_visit_dir() -> bool: + import daq_utils # Check if current visit dir is valid visit_dir_env_var = "CURRENT_VISIT_DIR" if visit_dir_env_var not in os.environ: @@ -123,7 +124,7 @@ def check_curr_visit_dir() -> bool: return False # Check if current visit dir is one of the dirs defined in BASE_DATA_DIRS - base_dir_env_var = "BASE_DATA_DIRS" + base_dir_env_var = f"BASE_DATA_DIRS_{daq_utils.beamline.upper()}" if base_dir_env_var not in os.environ: check_curr_visit_dir.remediation = f"{base_dir_env_var} evironment variable not found" return False From 216df675cd456f81342e7c0a370296538ebd5efe Mon Sep 17 00:00:00 2001 From: Shekar V Date: Thu, 8 Jun 2023 15:26:40 -0400 Subject: [PATCH 19/39] Fixes to get LSDC working --- gui/data_loc_info.py | 2 +- threads.py | 5 +++-- top_view.py | 2 +- utils/healthcheck.py | 16 ++++++++++++---- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/gui/data_loc_info.py b/gui/data_loc_info.py index 181130f3..acd52efc 100644 --- a/gui/data_loc_info.py +++ b/gui/data_loc_info.py @@ -23,7 +23,7 @@ def __init__(self, parent: "ControlMain"): self.hBoxDPathParams1 = QtWidgets.QHBoxLayout() self.basePathLabel = QtWidgets.QLabel("Base Path:") self.base_path_ledit = QtWidgets.QLabel() # leave editable for now - self.base_path_ledit.setText(getBlConfig("visitDirectory")) + self.base_path_ledit.setText(daq_utils.getBlConfig("visitDirectory")) self.base_path_ledit.setTextInteractionFlags(QtCore.Qt.TextSelectableByMouse) # self.base_path_ledit.textChanged[str].connect(self.basePathTextChanged) self.browseBasePathButton = QtWidgets.QPushButton("Browse...") diff --git a/threads.py b/threads.py index e1c386e6..91924160 100644 --- a/threads.py +++ b/threads.py @@ -6,7 +6,6 @@ import urllib from io import BytesIO import logging -from daq_lib import getBlConfig from config_params import SERVER_CHECK_DELAY import raddoseLib @@ -98,8 +97,10 @@ def __init__(self, *args, delay=SERVER_CHECK_DELAY, **kwargs): QThread.__init__(self, *args, **kwargs) def run(self): + import db_lib + beamline = os.environ["BEAMLINE_ID"] while True: - if getBlConfig("visitDirectory") != os.getcwd(): + if db_lib.getBeamlineConfigParam(beamline, "visitDirectory") != os.getcwd(): message = "The server visit directory has changed, stopping!" logger.error(message) print(message) diff --git a/top_view.py b/top_view.py index 03b73698..71e05a65 100644 --- a/top_view.py +++ b/top_view.py @@ -1,4 +1,4 @@ -from daq_utils import setBlConfig +from daq_utils import setBlConfig, getBlConfig import daq_lib import beamline_lib import time diff --git a/utils/healthcheck.py b/utils/healthcheck.py index 416c2a70..04824734 100644 --- a/utils/healthcheck.py +++ b/utils/healthcheck.py @@ -3,6 +3,7 @@ from pathlib import Path from networkx import DiGraph, bfs_tree import os +import getpass logger = logging.getLogger() class bcolors: @@ -72,8 +73,6 @@ def check_working_directory(): # Hacky way to check if amx or fmx is in path. Unless server can tell GUI where its running? check_working_directory.remediation = f'Please start LSDC in {daq_utils.beamline} data directory. Current directory: {working_dir}' return False - else: - return True if daq_utils.getBlConfig("visitDirectory") != os.getcwd(): check_working_directory.remediation = ("Working directory mismatch. Please start LSDC GUI in the same folder as the server is running.") return False @@ -100,9 +99,18 @@ def handle_fail(check): if check.fatal: print("End checks") print(u'\u2500' * 20) - sys.exit('Fatal error, exiting...') + print('Fatal error, exiting...') + os._exit(1) # Server checks +@healthcheck(name="check service user", remediation="LSDC server not being started by a LSDC service user account, aborting!") +def check_service_user() -> bool: + if not getpass.getuser().startswith("lsdc-"): + return False + print(f"continuing as we are using a service user: {getpass.getuser()}") + return True + + @healthcheck(name="server working directory", remediation="", fatal=True) def check_curr_visit_dir() -> bool: import daq_utils @@ -167,7 +175,7 @@ def perform_checks(): def perform_server_checks(): - check_functions = [check_env_file, check_curr_visit_dir] + check_functions = [check_service_user, check_env_file, check_curr_visit_dir] run_checks(check_functions) From 9c5f555e275204ee5ef856da0648bf7530306f05 Mon Sep 17 00:00:00 2001 From: Shekar V Date: Thu, 8 Jun 2023 19:51:23 -0400 Subject: [PATCH 20/39] Cleaner way to quit LSDC from QThread --- gui/control_main.py | 1 + threads.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/gui/control_main.py b/gui/control_main.py index 1c313a40..ee000c57 100644 --- a/gui/control_main.py +++ b/gui/control_main.py @@ -1395,6 +1395,7 @@ def createSampleTab(self): hutchTopCamThread.start() serverCheckThread = ServerCheckThread( parent=self, delay=SERVER_CHECK_DELAY) + serverCheckThread.visit_dir_changed.connect(self.close) serverCheckThread.start() def updateCam(self, pixmapItem: "QGraphicsPixmapItem", frame): diff --git a/threads.py b/threads.py index 91924160..7d2210ed 100644 --- a/threads.py +++ b/threads.py @@ -92,6 +92,7 @@ def run(self): class ServerCheckThread(QThread): + visit_dir_changed = Signal() def __init__(self, *args, delay=SERVER_CHECK_DELAY, **kwargs): self.delay = delay QThread.__init__(self, *args, **kwargs) @@ -104,5 +105,6 @@ def run(self): message = "The server visit directory has changed, stopping!" logger.error(message) print(message) - sys.exit(1) + self.visit_dir_changed.emit() + break self.msleep(self.delay) From 88b4ede2c957339cd59809ddb79b6a45199e2665 Mon Sep 17 00:00:00 2001 From: Shekar V Date: Thu, 8 Jun 2023 20:31:15 -0400 Subject: [PATCH 21/39] Added write permission check to server --- utils/healthcheck.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/utils/healthcheck.py b/utils/healthcheck.py index 04824734..7a123b66 100644 --- a/utils/healthcheck.py +++ b/utils/healthcheck.py @@ -4,6 +4,7 @@ from networkx import DiGraph, bfs_tree import os import getpass +import tempfile logger = logging.getLogger() class bcolors: @@ -110,6 +111,15 @@ def check_service_user() -> bool: print(f"continuing as we are using a service user: {getpass.getuser()}") return True +def has_write_permission(directory): + try: + testfile = tempfile.TemporaryFile(dir = directory) + testfile.close() + except OSError as e: + if e.errno == 13: # Permission denied + return False + raise + return True @healthcheck(name="server working directory", remediation="", fatal=True) def check_curr_visit_dir() -> bool: @@ -150,6 +160,11 @@ def check_curr_visit_dir() -> bool: if not pass_dir_found: check_curr_visit_dir.remediation = f"Pass folder not found in {current_visit_dir}" return False + + # Check if current visit dir is writable + if not has_write_permission(current_visit_dir): + check_curr_visit_dir.remediation = f"Server does not have write permission to {current_visit_dir}" + return False return True From 498a505667923da2379be839337a9e1082b46c4d Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Thu, 15 Jun 2023 17:39:27 -0400 Subject: [PATCH 22/39] Update top_view.py Co-authored-by: Thomas A Caswell --- top_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/top_view.py b/top_view.py index 71e05a65..79305747 100644 --- a/top_view.py +++ b/top_view.py @@ -37,7 +37,7 @@ def wait90TopviewThread(gov_robot, prefix1,prefix90): if (not filecmp.cmp(getBlConfig("visitDirectory")+"/pinAlign/"+snapshot1Name,getBlConfig("visitDirectory")+"/pinAlign/"+snapshot2Name)): #this would mean something is wrong if true because the pictures are identical comm_s = os.environ["LSDCHOME"] + "/runPinAlign.py " + snapshot1Name + " " + snapshot2Name logger.info(comm_s) - lines = os.popen(comm_s).readlines() + lines = os.popen(comm_s, cwd=.getBlConfig("visitDirectory"))readlines() logger.info("printing lines right after popen ") logger.info(lines) logger.info(" done") From e1268f6152f3ecbecc8d9a9c2342f82c444c36e1 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Thu, 15 Jun 2023 17:40:38 -0400 Subject: [PATCH 23/39] fix typo in popen call --- top_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/top_view.py b/top_view.py index 79305747..4c3c732a 100644 --- a/top_view.py +++ b/top_view.py @@ -37,7 +37,7 @@ def wait90TopviewThread(gov_robot, prefix1,prefix90): if (not filecmp.cmp(getBlConfig("visitDirectory")+"/pinAlign/"+snapshot1Name,getBlConfig("visitDirectory")+"/pinAlign/"+snapshot2Name)): #this would mean something is wrong if true because the pictures are identical comm_s = os.environ["LSDCHOME"] + "/runPinAlign.py " + snapshot1Name + " " + snapshot2Name logger.info(comm_s) - lines = os.popen(comm_s, cwd=.getBlConfig("visitDirectory"))readlines() + lines = os.popen(comm_s, cwd=getBlConfig("visitDirectory")).readlines() logger.info("printing lines right after popen ") logger.info(lines) logger.info(" done") From d785be05470ca0d74e3bc46c144c7d40952727b7 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Thu, 15 Jun 2023 17:58:58 -0400 Subject: [PATCH 24/39] remove misleading comment * the object in question is a QLabel(), which is not editable --- gui/data_loc_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/data_loc_info.py b/gui/data_loc_info.py index acd52efc..999c1839 100644 --- a/gui/data_loc_info.py +++ b/gui/data_loc_info.py @@ -22,7 +22,7 @@ def __init__(self, parent: "ControlMain"): self.vBoxDPathParams1 = QtWidgets.QVBoxLayout() self.hBoxDPathParams1 = QtWidgets.QHBoxLayout() self.basePathLabel = QtWidgets.QLabel("Base Path:") - self.base_path_ledit = QtWidgets.QLabel() # leave editable for now + self.base_path_ledit = QtWidgets.QLabel() self.base_path_ledit.setText(daq_utils.getBlConfig("visitDirectory")) self.base_path_ledit.setTextInteractionFlags(QtCore.Qt.TextSelectableByMouse) # self.base_path_ledit.textChanged[str].connect(self.basePathTextChanged) From a437f7c5631d3bca05bea2ef81248847d4eefefe Mon Sep 17 00:00:00 2001 From: Shekar V Date: Thu, 15 Jun 2023 18:17:29 -0400 Subject: [PATCH 25/39] Used QApplication.instace().quit to exit Inspired by: ``` if the goal is murder the app...just murder the app -Tom Caswell ``` --- gui/control_main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gui/control_main.py b/gui/control_main.py index ee000c57..0b501da1 100644 --- a/gui/control_main.py +++ b/gui/control_main.py @@ -17,7 +17,7 @@ from qtpy import QtCore, QtGui, QtWidgets from qtpy.QtCore import QModelIndex, QRectF, Qt, QTimer from qtpy.QtGui import QIntValidator -from qtpy.QtWidgets import QCheckBox, QFrame, QGraphicsPixmapItem +from qtpy.QtWidgets import QCheckBox, QFrame, QGraphicsPixmapItem, QApplication import albulaUtils import daq_utils @@ -1395,7 +1395,7 @@ def createSampleTab(self): hutchTopCamThread.start() serverCheckThread = ServerCheckThread( parent=self, delay=SERVER_CHECK_DELAY) - serverCheckThread.visit_dir_changed.connect(self.close) + serverCheckThread.visit_dir_changed.connect(QApplication.instance().quit) serverCheckThread.start() def updateCam(self, pixmapItem: "QGraphicsPixmapItem", frame): From 85da26a07ba3ad3f5d975fc50bc3d46a4b60eb53 Mon Sep 17 00:00:00 2001 From: Shekar V Date: Tue, 20 Jun 2023 13:45:13 -0400 Subject: [PATCH 26/39] Stopping cam threads when quitting App - Using already existing closeAll() method - Threads are stopped cleanly and quit --- gui/control_main.py | 18 +++++++++++------- threads.py | 6 +++++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/gui/control_main.py b/gui/control_main.py index 0b501da1..a91bf9f5 100644 --- a/gui/control_main.py +++ b/gui/control_main.py @@ -1378,21 +1378,21 @@ def createSampleTab(self): self.vidActionRasterDefRadio.setDisabled(True) self.vidActionDefineCenterRadio.setDisabled(True) - hutchCornerCamThread = VideoThread( + self.hutchCornerCamThread = VideoThread( parent=self, delay=HUTCH_TIMER_DELAY, url=getBlConfig("hutchCornerCamURL") ) - hutchCornerCamThread.frame_ready.connect( + self.hutchCornerCamThread.frame_ready.connect( lambda frame: self.updateCam(self.pixmap_item_HutchCorner, frame) ) - hutchCornerCamThread.start() + self.hutchCornerCamThread.start() - hutchTopCamThread = VideoThread( + self.hutchTopCamThread = VideoThread( parent=self, delay=HUTCH_TIMER_DELAY, url=getBlConfig("hutchTopCamURL") ) - hutchTopCamThread.frame_ready.connect( + self.hutchTopCamThread.frame_ready.connect( lambda frame: self.updateCam(self.pixmap_item_HutchTop, frame) ) - hutchTopCamThread.start() + self.hutchTopCamThread.start() serverCheckThread = ServerCheckThread( parent=self, delay=SERVER_CHECK_DELAY) serverCheckThread.visit_dir_changed.connect(QApplication.instance().quit) @@ -4958,7 +4958,11 @@ def popStaffDialogCB(self): self.popupServerMessage("You don't have control") def closeAll(self): - QtWidgets.QApplication.closeAllWindows() + self.hutchCornerCamThread.stop() + self.hutchTopCamThread.stop() + self.hutchCornerCamThread.wait() + self.hutchTopCamThread.wait() + QtWidgets.QApplication.instance().quit() def initCallbacks(self): self.beamSizeSignal.connect(self.processBeamSize) diff --git a/threads.py b/threads.py index 7d2210ed..45118e26 100644 --- a/threads.py +++ b/threads.py @@ -56,15 +56,19 @@ def __init__(self, *args, delay=1000, url='', camera_object=None, width=None, he self.url = url self.camera_object = camera_object self.showing_error = False + self.is_running = True QThread.__init__(self, *args, **kwargs) def updateCam(self, camera_object): self.camera_object = camera_object def run(self): - while True: + while self.is_running: self.camera_refresh() self.msleep(self.delay) + + def stop(self): + self.is_running = False class RaddoseThread(QThread): From 4c8c30d998a9e4edec0bc11a05ad55c0e8e3f460 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Thu, 29 Jun 2023 11:16:29 -0400 Subject: [PATCH 27/39] use subprocess.Popen which has the cwd kwarg * os.popen does not --- top_view.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/top_view.py b/top_view.py index 4c3c732a..cd3bf9ed 100644 --- a/top_view.py +++ b/top_view.py @@ -6,6 +6,7 @@ import os import filecmp import logging +import subprocess from config_params import TOP_VIEW_CHECK logger = logging.getLogger(__name__) @@ -37,7 +38,7 @@ def wait90TopviewThread(gov_robot, prefix1,prefix90): if (not filecmp.cmp(getBlConfig("visitDirectory")+"/pinAlign/"+snapshot1Name,getBlConfig("visitDirectory")+"/pinAlign/"+snapshot2Name)): #this would mean something is wrong if true because the pictures are identical comm_s = os.environ["LSDCHOME"] + "/runPinAlign.py " + snapshot1Name + " " + snapshot2Name logger.info(comm_s) - lines = os.popen(comm_s, cwd=getBlConfig("visitDirectory")).readlines() + lines = subprocess.Popen(comm_s, cwd=getBlConfig("visitDirectory")).readlines() logger.info("printing lines right after popen ") logger.info(lines) logger.info(" done") From 29e5c7e31ebd1cb4a8ad39b711fb09aa2fa4a7c8 Mon Sep 17 00:00:00 2001 From: Shekar V Date: Thu, 29 Jun 2023 11:42:34 -0400 Subject: [PATCH 28/39] Reading lines of subprocess.Popen() --- top_view.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/top_view.py b/top_view.py index cd3bf9ed..c0312379 100644 --- a/top_view.py +++ b/top_view.py @@ -38,7 +38,8 @@ def wait90TopviewThread(gov_robot, prefix1,prefix90): if (not filecmp.cmp(getBlConfig("visitDirectory")+"/pinAlign/"+snapshot1Name,getBlConfig("visitDirectory")+"/pinAlign/"+snapshot2Name)): #this would mean something is wrong if true because the pictures are identical comm_s = os.environ["LSDCHOME"] + "/runPinAlign.py " + snapshot1Name + " " + snapshot2Name logger.info(comm_s) - lines = subprocess.Popen(comm_s, cwd=getBlConfig("visitDirectory")).readlines() + process = subprocess.Popen(comm_s, cwd=getBlConfig("visitDirectory"), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + lines = process.stdout.readlines() logger.info("printing lines right after popen ") logger.info(lines) logger.info(" done") From 4f424d88a96ca180c8ff6a37a97c6c89aaadfaf7 Mon Sep 17 00:00:00 2001 From: Shekar V Date: Thu, 29 Jun 2023 13:20:48 -0400 Subject: [PATCH 29/39] Using subprocess.run() instead of subprocess.Popen() in topview.py --- top_view.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/top_view.py b/top_view.py index c0312379..5e235d81 100644 --- a/top_view.py +++ b/top_view.py @@ -36,10 +36,10 @@ def wait90TopviewThread(gov_robot, prefix1,prefix90): snapshot1Name = prefix1+"_001.jpg" snapshot2Name = prefix90+"_001.jpg" if (not filecmp.cmp(getBlConfig("visitDirectory")+"/pinAlign/"+snapshot1Name,getBlConfig("visitDirectory")+"/pinAlign/"+snapshot2Name)): #this would mean something is wrong if true because the pictures are identical - comm_s = os.environ["LSDCHOME"] + "/runPinAlign.py " + snapshot1Name + " " + snapshot2Name + comm_s = [os.environ["LSDCHOME"] + "/runPinAlign.py", snapshot1Name, snapshot2Name] logger.info(comm_s) - process = subprocess.Popen(comm_s, cwd=getBlConfig("visitDirectory"), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - lines = process.stdout.readlines() + process = subprocess.run(comm_s, cwd=getBlConfig("visitDirectory"), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + lines = process.stdout.decode().strip().split('\n') logger.info("printing lines right after popen ") logger.info(lines) logger.info(" done") From b397f7290f0f34b5a21a9bf2f0290090affddb02 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Fri, 3 Mar 2023 12:12:39 -0500 Subject: [PATCH 30/39] remove homing pins after sample mount on FMX * due to unreliable gpy homing --- embl_robot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/embl_robot.py b/embl_robot.py index 653f0daf..bf34941f 100644 --- a/embl_robot.py +++ b/embl_robot.py @@ -262,9 +262,9 @@ def mount(self, gov_robot, puckPos,pinPos,sampID,**kwargs): else: robotStatus = beamline_support.get_any_epics_pv("SW:RobotState","VAL") if (robotStatus != "Ready"): - if (daq_utils.beamline == "fmx"): - daq_macros.homePins() - time.sleep(3.0) + #if (daq_utils.beamline == "fmx"): + # daq_macros.homePins() + # time.sleep(3.0) gov_status = gov_lib.setGovRobot(gov_robot, 'SE') if not gov_status.success: return MOUNT_FAILURE From 6a741839bf70dea655b6f2a547b83c03cf191737 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Thu, 6 Jul 2023 11:29:47 -0400 Subject: [PATCH 31/39] remove commented-out code for homing pins on FMX --- embl_robot.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/embl_robot.py b/embl_robot.py index bf34941f..35903b36 100644 --- a/embl_robot.py +++ b/embl_robot.py @@ -262,9 +262,6 @@ def mount(self, gov_robot, puckPos,pinPos,sampID,**kwargs): else: robotStatus = beamline_support.get_any_epics_pv("SW:RobotState","VAL") if (robotStatus != "Ready"): - #if (daq_utils.beamline == "fmx"): - # daq_macros.homePins() - # time.sleep(3.0) gov_status = gov_lib.setGovRobot(gov_robot, 'SE') if not gov_status.success: return MOUNT_FAILURE From 5f219dcd7875da99f0b69de0e4c5561a4b49f834 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Mon, 15 May 2023 13:25:19 -0400 Subject: [PATCH 32/39] move omega to 0 before SA->SE on FMX * do this right before governor SA -> SE to prevent spurious changes to omega during windback to 0 --- embl_robot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/embl_robot.py b/embl_robot.py index 35903b36..69b7ce2f 100644 --- a/embl_robot.py +++ b/embl_robot.py @@ -358,6 +358,8 @@ def preUnmount(self, gov_robot, puckPos,pinPos,sampID): #will somehow know where detDist = beamline_lib.motorPosFromDescriptor("detectorDist") if (detDist Date: Tue, 25 Jul 2023 11:41:24 -0400 Subject: [PATCH 34/39] Revert "comment out ISPyB calls to database" This reverts commit 807573e9b782eddaed9cb6f1a5b2c07a7a8e8f6b. --- ispybLib.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/ispybLib.py b/ispybLib.py index bc0fd2ed..567ef50c 100644 --- a/ispybLib.py +++ b/ispybLib.py @@ -20,7 +20,7 @@ #request_dicts = lsdb2.getColRequestsByTimeInterval('2018-02-14T00:00:00','2018-02-15T00:00:00') # Connect to ISPyB, get the relevant data area objects -conn = ispyb.open(credentials=conf_file) +conn = ispyb.open(conf_file) core = ispyb.factory.create_data_area(ispyb.factory.DataAreaType.CORE, conn) mxacquisition = ispyb.factory.create_data_area(ispyb.factory.DataAreaType.MXACQUISITION, conn) mxprocessing = ispyb.factory.create_data_area(ispyb.factory.DataAreaType.MXPROCESSING, conn) @@ -57,7 +57,6 @@ def maxVisitNumfromProposal(propNum): def createPerson(firstName,lastName,loginName): - return params = core.get_person_params() params['given_name'] = firstName params['family_name'] = lastName @@ -67,7 +66,6 @@ def createPerson(firstName,lastName,loginName): def createProposal(propNum,PI_login="boaty"): - return pid = personIdFromLogin(PI_login) if (pid == 0): createPerson("Not","Sure",PI_login) @@ -82,7 +80,6 @@ def createProposal(propNum,PI_login="boaty"): cnx.commit() #not sure why I needed to do this. Maybe mistake in stored proc? def createVisitName(propNum): # this is for the GUI to know what a datapath would be in row_clicked - return logger.info("creating visit Name for propnum " + str(propNum)) propID = proposalIdFromProposal(propNum) if (propID == 0): #proposal doesn't exist, just create and assign to boaty @@ -98,7 +95,6 @@ def createVisitName(propNum): # this is for the GUI to know what a datapath woul def createVisit(propNum): - return visitName, newVisitNum = createVisitName(propNum) personID = personIdFromProposal(propNum) params = core.get_session_for_proposal_code_number_params() @@ -167,7 +163,6 @@ def createVisit(propNum): return visitName def addPersonToProposal(personLogin,propNum): - return personID = personIdFromLogin(personLogin) if (personID == 0): createPerson("Not","Sure",personLogin) @@ -185,7 +180,6 @@ def addPersonToProposal(personLogin,propNum): def insertPlotResult(dc_id,imageNumber,spotTotal,goodBraggCandidates,method2Res,totalIntegratedSignal): - return params = mxprocessing.get_quality_indicators_params() params['datacollectionid'] = dc_id params['imageNumber'] = imageNumber @@ -198,7 +192,6 @@ def insertPlotResult(dc_id,imageNumber,spotTotal,goodBraggCandidates,method2Res, def insertResult(result,resultType,request,visitName,dc_id=None,xmlFileName=None): #xmlfilename for fastDP #keep in mind that request type can be standard and result type be fastDP - multiple results per req. - return try: sessionid = core.retrieve_visit_id(visitName) @@ -337,7 +330,6 @@ def insertResult(result,resultType,request,visitName,dc_id=None,xmlFileName=None def createDataCollection(directory, filePrefix, jpegImageFilename, params, request_obj, sessionid): - return params['starttime'] = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') params['endtime'] = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') dcg_id = mxacquisition.insert_data_collection_group(list(params.values())) @@ -388,7 +380,6 @@ def createDataCollection(directory, filePrefix, jpegImageFilename, params, reque # params['overlap'] = 89.0 def insertRasterResult(request,visitName): - return try: sessionid = core.retrieve_visit_id(visitName) except ISPyBNoResultException as e: From 3ee42283ef728caf33ea9aa2875e76d7e7459497 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Tue, 25 Jul 2023 11:01:16 -0400 Subject: [PATCH 35/39] Revert "fix for proposal name while ISPyB is down" This reverts commit ce27170e5d2e9bc511b17ac18760e507bb1013bd. --- daq_utils.py | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/daq_utils.py b/daq_utils.py index 6dca8948..169b4f26 100644 --- a/daq_utils.py +++ b/daq_utils.py @@ -364,27 +364,19 @@ def readPVDesc(): counter_inf = line.split() counter_dict[counter_inf[1]] = beamline_designation + counter_inf[0] -def createVisitNameRaw(proposalName, maxNumber=None): - if maxNumber: - number = maxNumber + 1 - else: - number = 1 - return f'mx{proposalName}-{number}', number - -def createVisitName(proposalName, maxNumber=None): - return createVisitNameRaw(proposalName, maxNumber) - -def setProposalID(proposalID,createVisit=True): # TODO JA proposalID implies a database ID, which it is not (just proposal number). Misleading - if (getProposalID() != proposalID): #proposalID changed - create a new visit. - logger.info("you changed proposals! " + str(proposalID)) - logger.info('ignoring createVisit for now - ISPyB required to properly account for visit numbers') - try: - visitName, visitNum = createVisitName(proposalID) - db_lib.setBeamlineConfigParam(beamline,"proposal",proposalID) - except Exception as e: - visitName = "999999-1234" - logger.error("error in set proposal. Error: %s" % e) - setVisitName(visitName) +def setProposalID(proposalID,createVisit=True): + if (getProposalID() != proposalID): #proposalID changed - create a new visit. + logger.info("you changed proposals! " + str(proposalID)) + try: + if (createVisit): + visitName = ispybLib.createVisit(proposalID) + db_lib.setBeamlineConfigParam(beamline,"proposal",proposalID) + else: + visitName, visitNum = ispybLib.createVisitName(proposalID) + except Exception as e: + visitName = "999999-1234" + logger.error("ispyb error in set proposal. Error: %s" % e) + setVisitName(visitName) def getProposalID(): return getBlConfig("proposal") From e736700509f5b06214aeec5c6e1c1241390ab71d Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Tue, 25 Jul 2023 11:08:05 -0400 Subject: [PATCH 36/39] use config from /etc/ispyb --- ispybLib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ispybLib.py b/ispybLib.py index 567ef50c..24eb03b2 100644 --- a/ispybLib.py +++ b/ispybLib.py @@ -14,7 +14,7 @@ #12/19 - I'm leaving all commented lines alone on this. Karl Levik, DLS, is an immense help with this. -conf_file = os.environ["CONFIGDIR"] + "ispybConfig.cfg" +conf_file = "/etc/ispyb/ispybConfig.cfg" visit = 'mx99999-1' # Get a list of request dicts #request_dicts = lsdb2.getColRequestsByTimeInterval('2018-02-14T00:00:00','2018-02-15T00:00:00') From f3653aeef411ec4e1ba21422b44a59e9bddac503 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Tue, 25 Jul 2023 11:21:37 -0400 Subject: [PATCH 37/39] use mysql connector object from ISPyB connection --- ispybLib.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ispybLib.py b/ispybLib.py index 24eb03b2..d0fb7a45 100644 --- a/ispybLib.py +++ b/ispybLib.py @@ -8,7 +8,6 @@ import db_lib import det_lib import time -import mysql.connector import logging logger = logging.getLogger(__name__) @@ -25,7 +24,7 @@ mxacquisition = ispyb.factory.create_data_area(ispyb.factory.DataAreaType.MXACQUISITION, conn) mxprocessing = ispyb.factory.create_data_area(ispyb.factory.DataAreaType.MXPROCESSING, conn) mxscreening = ispyb.factory.create_data_area(ispyb.factory.DataAreaType.MXSCREENING, conn) -cnx = mysql.connector.connect(user='ispyb_api', password=os.environ['ISPYB_PASSWORD'],host='ispyb-db.nsls2.bnl.local',database='ispyb') +cnx = conn.conn cursor = cnx.cursor() beamline = os.environ["BEAMLINE_ID"] From f1fd6226e6085aff6f2f3e50c350d038acbd644c Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Tue, 25 Jul 2023 14:35:35 -0400 Subject: [PATCH 38/39] fix missing parenthesis --- gui/control_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/control_main.py b/gui/control_main.py index 02ec4aaa..a233d85e 100644 --- a/gui/control_main.py +++ b/gui/control_main.py @@ -3111,7 +3111,7 @@ def takeRasterSnapshot(self, rasterReq): reqID=rasterReq["uid"], rasterHeatJpeg=jpegImageFilename, ) - self.send_to_server(f"ispybLib.insertRasterResult('{rasterReq['uid']}', '{visitName}'") + self.send_to_server(f"ispybLib.insertRasterResult('{rasterReq['uid']}', '{visitName}')") def reFillPolyRaster(self): rasterEvalOption = str(self.rasterEvalComboBox.currentText()) From 19ddc5b412b307f56f629447e42f14b7ebd31fe2 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Mon, 7 Aug 2023 10:38:04 -0400 Subject: [PATCH 39/39] Revert "all server and GUI instances will be run from lsdc__prod" This reverts commit deec17872792d9675a78e85cfc2b81a385300f4e. * un-doing my original plan to separate LSDC code into prod and dev locations. will keep running code from lsdc_ for now --- bin/lsdcGui_amx | 2 +- bin/lsdcGui_fmx | 2 +- bin/lsdcGui_nyx | 2 +- bin/lsdcRemote_amx.cmd | 2 +- bin/lsdcRemote_fmx.cmd | 2 +- bin/lsdcServer_amx.cmd | 2 +- bin/lsdcServer_fmx.cmd | 2 +- bin/lsdcServer_nyx.cmd | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/lsdcGui_amx b/bin/lsdcGui_amx index c684d14a..b39909a2 100755 --- a/bin/lsdcGui_amx +++ b/bin/lsdcGui_amx @@ -1,6 +1,6 @@ export PROJDIR=/nsls2/software/mx/daq/ export CONFIGDIR=${PROJDIR}bnlpx_config/ -export LSDCHOME=${PROJDIR}lsdc_amx_prod +export LSDCHOME=${PROJDIR}lsdc_amx export PATH=/usr/local/bin:/usr/bin:/bin export PYTHONPATH=".:${CONFIGDIR}:/opt/dectris/albula/4.0/python:${LSDCHOME}" diff --git a/bin/lsdcGui_fmx b/bin/lsdcGui_fmx index 159fdb4c..e028aee7 100755 --- a/bin/lsdcGui_fmx +++ b/bin/lsdcGui_fmx @@ -1,6 +1,6 @@ export PROJDIR=/nsls2/software/mx/daq/ export CONFIGDIR=${PROJDIR}bnlpx_config/ -export LSDCHOME=${PROJDIR}lsdc_fmx_prod +export LSDCHOME=${PROJDIR}lsdc_fmx export PATH=/usr/local/bin:/usr/bin:/bin export PYTHONPATH=".:${CONFIGDIR}:/opt/dectris/albula/4.0/python:${LSDCHOME}" diff --git a/bin/lsdcGui_nyx b/bin/lsdcGui_nyx index a0aa62ac..55d2122d 100755 --- a/bin/lsdcGui_nyx +++ b/bin/lsdcGui_nyx @@ -1,6 +1,6 @@ export PROJDIR=/nsls2/software/mx/daq/ export CONFIGDIR=${PROJDIR}bnlpx_config/ -export LSDCHOME=${PROJDIR}lsdc_nyx_prod +export LSDCHOME=${PROJDIR}lsdc_nyx export EPICS_CA_AUTO_ADDR_LIST=NO export EPICS_CA_ADDR_LIST=10.67.147.255 diff --git a/bin/lsdcRemote_amx.cmd b/bin/lsdcRemote_amx.cmd index ee8f0626..13f770b7 100755 --- a/bin/lsdcRemote_amx.cmd +++ b/bin/lsdcRemote_amx.cmd @@ -1,7 +1,7 @@ #!/bin/bash -l export PROJDIR=/nsls2/software/mx/daq/ export CONFIGDIR=${PROJDIR}bnlpx_config/ -export LSDCHOME=${PROJDIR}lsdc_amx_prod +export LSDCHOME=${PROJDIR}lsdc_amx export PYTHONPATH=".:${CONFIGDIR}:/usr/lib64/edna-mx/mxv1/src:/usr/lib64/edna-mx/kernel/src:${LSDCHOME}:${PROJDIR}/RobotControlLib" export PATH=/usr/local/bin:/usr/bin:/bin:${PROJDIR}/software/bin:/opt/ccp4/bin source ${CONFIGDIR}daq_env_amx.txt diff --git a/bin/lsdcRemote_fmx.cmd b/bin/lsdcRemote_fmx.cmd index a892611b..052d8241 100755 --- a/bin/lsdcRemote_fmx.cmd +++ b/bin/lsdcRemote_fmx.cmd @@ -1,7 +1,7 @@ #!/bin/bash -l export PROJDIR=/nsls2/software/mx/daq/ export CONFIGDIR=${PROJDIR}bnlpx_config/ -export LSDCHOME=${PROJDIR}lsdc_fmx_prod +export LSDCHOME=${PROJDIR}lsdc_fmx export PYTHONPATH=".:${CONFIGDIR}:/usr/lib64/edna-mx/mxv1/src:/usr/lib64/edna-mx/kernel/src:${LSDCHOME}:${PROJDIR}/RobotControlLib" export PATH=/usr/local/bin:/usr/bin:/bin:${PROJDIR}/software/bin:/opt/ccp4/bin source ${CONFIGDIR}daq_env.txt diff --git a/bin/lsdcServer_amx.cmd b/bin/lsdcServer_amx.cmd index 24fc70e6..64e6d4ed 100755 --- a/bin/lsdcServer_amx.cmd +++ b/bin/lsdcServer_amx.cmd @@ -1,7 +1,7 @@ #!/bin/bash -l export PROJDIR=/nsls2/software/mx/daq/ export CONFIGDIR=${PROJDIR}bnlpx_config/ -export LSDCHOME=${PROJDIR}lsdc_amx_prod +export LSDCHOME=${PROJDIR}lsdc_amx export PYTHONPATH=".:${CONFIGDIR}:/usr/lib64/edna-mx/mxv1/src:/usr/lib64/edna-mx/kernel/src:${LSDCHOME}:${PROJDIR}/RobotControlLib" export PATH=/usr/local/bin:/usr/bin:/bin:${PROJDIR}/software/bin:/opt/ccp4/bin source ${CONFIGDIR}daq_env_amx.txt diff --git a/bin/lsdcServer_fmx.cmd b/bin/lsdcServer_fmx.cmd index 096bdb41..59e63cfb 100755 --- a/bin/lsdcServer_fmx.cmd +++ b/bin/lsdcServer_fmx.cmd @@ -1,7 +1,7 @@ #!/bin/bash -l export PROJDIR=/nsls2/software/mx/daq/ export CONFIGDIR=${PROJDIR}bnlpx_config/ -export LSDCHOME=${PROJDIR}lsdc_fmx_prod +export LSDCHOME=${PROJDIR}lsdc_fmx export PYTHONPATH=".:${CONFIGDIR}:/usr/lib64/edna-mx/mxv1/src:/usr/lib64/edna-mx/kernel/src:${LSDCHOME}:${PROJDIR}/RobotControlLib" export PATH=/usr/local/bin:/usr/bin:/bin:${PROJDIR}/software/bin:/opt/ccp4/bin source ${CONFIGDIR}daq_env.txt diff --git a/bin/lsdcServer_nyx.cmd b/bin/lsdcServer_nyx.cmd index e46707bf..b06b291b 100755 --- a/bin/lsdcServer_nyx.cmd +++ b/bin/lsdcServer_nyx.cmd @@ -1,7 +1,7 @@ #!/bin/bash -l export PROJDIR=/nsls2/software/mx/daq/ export CONFIGDIR=${PROJDIR}bnlpx_config/ -export LSDCHOME=${PROJDIR}lsdc_nyx_prod +export LSDCHOME=${PROJDIR}lsdc_nyx export EPICS_CA_AUTO_ADDR_LIST=NO export PYTHONPATH=".:${CONFIGDIR}:/usr/lib64/edna-mx/mxv1/src:/usr/lib64/edna-mx/kernel/src:${LSDCHOME}:${PROJDIR}/RobotControlLib" export PATH=/usr/local/bin:/usr/bin:/bin:${PROJDIR}/software/bin:/opt/ccp4/bin