From 10ac902f725b6fdfa03667d5cbe322f3bef4fe4a Mon Sep 17 00:00:00 2001 From: bkieft-usa Date: Thu, 7 Nov 2024 09:34:28 -0800 Subject: [PATCH 1/2] Turn off recursion in rclone gdrive checker/uploader --- metatlas/untargeted/tools.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metatlas/untargeted/tools.py b/metatlas/untargeted/tools.py index 979cf196..63f81ed1 100644 --- a/metatlas/untargeted/tools.py +++ b/metatlas/untargeted/tools.py @@ -465,7 +465,7 @@ def upload_to_google_drive( logging.critical(tab_print("Warning! Google Drive upload failed on upload with overwrite=%s with exception on command %s"%(overwrite, upload_command), 3)) return False # Check that upload worked - check_upload_command = f'/global/cfs/cdirs/m342/USA/shared-envs/rclone/bin/rclone ls ben_lbl_gdrive:/untargeted_outputs | grep "{project_folder}"' + check_upload_command = f'/global/cfs/cdirs/m342/USA/shared-envs/rclone/bin/rclone ls ben_lbl_gdrive:/untargeted_outputs --max-depth 1 | grep "{project_folder}"' try: check_upload_out = subprocess.check_output(check_upload_command, shell=True) if check_upload_out.decode('utf-8').strip(): @@ -479,7 +479,7 @@ def upload_to_google_drive( return False if overwrite == False: # Overwrite is False, so first check if the project folder already exists on Google Drive - check_upload_command = f'/global/cfs/cdirs/m342/USA/shared-envs/rclone/bin/rclone ls ben_lbl_gdrive:/untargeted_outputs | grep "{project_folder}"' + check_upload_command = f'/global/cfs/cdirs/m342/USA/shared-envs/rclone/bin/rclone ls ben_lbl_gdrive:/untargeted_outputs --max-depth 1 | grep "{project_folder}"' try: check_upload_out = subprocess.check_output(check_upload_command, shell=True) if check_upload_out.decode('utf-8').strip(): @@ -490,7 +490,7 @@ def upload_to_google_drive( try: subprocess.check_output(upload_command, shell=True) try: - check_upload = f'/global/cfs/cdirs/m342/USA/shared-envs/rclone/bin/rclone ls ben_lbl_gdrive:/untargeted_outputs | grep "{project_folder}"' + check_upload = f'/global/cfs/cdirs/m342/USA/shared-envs/rclone/bin/rclone ls ben_lbl_gdrive:/untargeted_outputs --max-depth 1 | grep "{project_folder}"' check_upload_out = subprocess.check_output(check_upload, shell=True) if check_upload_out.decode('utf-8').strip(): logging.info(tab_print("Google Drive upload confirmed!", 3)) @@ -557,7 +557,7 @@ def get_untargeted_status( return if not df.empty: # Get all projects in gdrive - all_gdrive_projects_cmd = f'/global/cfs/cdirs/m342/USA/shared-envs/rclone/bin/rclone lsl ben_lbl_gdrive:/untargeted_outputs' + all_gdrive_projects_cmd = f'/global/cfs/cdirs/m342/USA/shared-envs/rclone/bin/rclone lsl ben_lbl_gdrive:/untargeted_outputs --max-depth 1' try: all_gdrive_projects = subprocess.check_output(all_gdrive_projects_cmd, shell=True) all_gdrive_projects_list = all_gdrive_projects.decode('utf-8').strip().split('\n') From 08dd2bde185330ce3a99cffcf88bc16bf5a5802d Mon Sep 17 00:00:00 2001 From: bkieft-usa Date: Thu, 7 Nov 2024 09:38:01 -0800 Subject: [PATCH 2/2] Update fbmn download skipper so it doesn't throw a critical error when the workflow is run when one polarity has finished and the other is in progress --- metatlas/untargeted/tools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metatlas/untargeted/tools.py b/metatlas/untargeted/tools.py index 63f81ed1..d67a332d 100644 --- a/metatlas/untargeted/tools.py +++ b/metatlas/untargeted/tools.py @@ -665,6 +665,8 @@ def download_fbmn_results( polarity_short = polarity[:3] if row['%s_%s_status'%(tasktype,polarity_short)] == '12 not relevant': continue + if row['%s_%s_status'%(tasktype,polarity_short)] != '07 complete' and row['%s_%s_status'%("mzmine",polarity_short)] != '07 complete': + continue # skip this polarity even if the other one is finished if row['%s_%s_status'%(tasktype,polarity_short)] == '09 error': logging.warning(tab_print("Warning! FBMN task for %s %s has error status. Not downloading files."%(project_name,polarity), 1)) continue