add source and destination prefix paths for globus, fix various bugs,…

… add further checks

cmd/cliutils/commonTransfer.go

@@ -17,7 +17,9 @@ type SshParams struct {
 type GlobusParams struct {
 	GlobusClient   globus.GlobusClient
 	SrcCollection  string
+	SrcPrefixPath  string
 	DestCollection string
+	DestPrefixPath string
 	Filelist       []string
 	IsSymlinkList  []bool
 }

cmd/cliutils/globusLogin.go

@@ -16,19 +16,21 @@ type globusConfig struct {
 	RedirectURL           string   `yaml:"redirect-url"`
 	Scopes                []string `yaml:"scopes,omitempty"`
 	SourceCollection      string   `yaml:"source-collection"`
-	DestinationCollection string   `yaml:"dest-collection"`
+	SourcePrefixPath      string   `yaml:"source-prefix-path,omitempty"`
+	DestinationCollection string   `yaml:"destination-collection"`
+	DestinationPrefixPath string   `yaml:"destination-prefix-path,omitempty"`
 }
 
-func GlobusLogin(confPath string) (gClient globus.GlobusClient, srcCollection string, destCollection string, err error) {
+func GlobusLogin(confPath string) (gClient globus.GlobusClient, srcCollection string, srcPrefixPath string, destCollection string, destPrefixPath string, err error) {
 	// read in config
 	data, err := os.ReadFile(confPath)
 	if err != nil {
-		return globus.GlobusClient{}, "", "", fmt.Errorf("can't read globus config: %v", err)
+		return globus.GlobusClient{}, "", "", "", "", fmt.Errorf("can't read globus config: %v", err)
 	}
 	var gConfig globusConfig
 	err = yaml.Unmarshal(data, &gConfig)
 	if err != nil {
-		return globus.GlobusClient{}, "", "", fmt.Errorf("can't unmarshal globus config: %v", err)
+		return globus.GlobusClient{}, "", "", "", "", fmt.Errorf("can't unmarshal globus config: %v", err)
 	}
 
 	// config setup
@@ -44,13 +46,13 @@ func GlobusLogin(confPath string) (gClient globus.GlobusClient, srcCollection st
 	// negotiate token and create client
 	var code string
 	if _, err := fmt.Scan(&code); err != nil {
-		return globus.GlobusClient{}, "", "", err
+		return globus.GlobusClient{}, "", "", "", "", err
 	}
 	tok, err := clientConfig.Exchange(ctx, code, oauth2.VerifierOption(verifier))
 	if err != nil {
-		return globus.GlobusClient{}, "", "", fmt.Errorf("oauth2 exchange failed: %v", err)
+		return globus.GlobusClient{}, "", "", "", "", fmt.Errorf("oauth2 exchange failed: %v", err)
 	}
 
 	// return globus client
-	return globus.HttpClientToGlobusClient(clientConfig.Client(ctx, tok)), gConfig.SourceCollection, gConfig.DestinationCollection, nil
+	return globus.HttpClientToGlobusClient(clientConfig.Client(ctx, tok)), gConfig.SourceCollection, gConfig.SourcePrefixPath, gConfig.DestinationCollection, gConfig.DestinationPrefixPath, nil
 }

cmd/cliutils/globusTransfer.go

@@ -12,18 +12,29 @@ func GlobusTransfer(params TransferParams) (archivable bool, err error) {
 	fileList := params.Filelist
 	isSymlinkList := params.IsSymlinkList
 	srcCollection := params.SrcCollection
+	srcPrefixPath := params.SrcPrefixPath
 	dsSourceFolder := params.DatasetSourceFolder
 
 	destCollection := params.DestCollection
+	destPrefixPath := params.DestPrefixPath
 	datasetId := params.DatasetId
 
 	archivable = false // the dataset is never archivable after a globus transfer request immediately
-	destFolder := "archive/" + strings.Split(datasetId, "/")[1] + dsSourceFolder
+	destFolder := destPrefixPath + "/archive/" + strings.Split(datasetId, "/")[1] + dsSourceFolder
+
+	for i := range fileList {
+		fileList[i] = srcPrefixPath + "/" + fileList[i]
+	}
 
 	// === copying files ===
 	log.Println("Syncing files to cache server...")
 	result, err := globusClient.TransferFileList(srcCollection, dsSourceFolder, destCollection, destFolder, fileList, isSymlinkList, true)
-	log.Printf("The transfer result response: \n=====\n%v\n=====\n", result)
+	log.Printf("The transfer result response: \n=====\n")
+	log.Printf("Task ID: %s\n", result.TaskId)
+	log.Printf("Code: %s\n", result.SubmissionId)
+	log.Printf("Message: %s\n", result.Message)
+	log.Printf("Resource: %s\n", result.Resource)
+	log.Printf("=====\n")
 	log.Println("Syncing files - STARTED")
 
 	// === return results ===

cmd/commands/datasetIngestor.go

@@ -91,7 +91,7 @@ For Windows you need instead to specify -user username:password on the command l
 
 		// globus specific vars (if needed)
 		var globusClient globus.GlobusClient
-		var srcCollection, destCollection string
+		var srcCollection, srcPrefixPath, destCollection, destPrefixPath string
 
 		switch transferType {
 		case Ssh:
@@ -106,10 +106,10 @@ For Windows you need instead to specify -user username:password on the command l
 				if err != nil {
 					log.Fatalln("can't find executable path:", err)
 				}
-				globusConfigPath = filepath.Join(execPath, "globus.yaml")
+				globusConfigPath = filepath.Join(filepath.Dir(execPath), "globus.yaml")
 			}
 
-			globusClient, srcCollection, destCollection, err = cliutils.GlobusLogin(globusConfigPath)
+			globusClient, srcCollection, srcPrefixPath, destCollection, destPrefixPath, err = cliutils.GlobusLogin(globusConfigPath)
 			if err != nil {
 				log.Fatalln("couldn't create globus client:", err)
 			}
@@ -450,7 +450,9 @@ For Windows you need instead to specify -user username:password on the command l
 						GlobusParams: cliutils.GlobusParams{
 							GlobusClient:   globusClient,
 							SrcCollection:  srcCollection,
+							SrcPrefixPath:  srcPrefixPath,
 							DestCollection: destCollection,
+							DestPrefixPath: destPrefixPath,
 							Filelist:       filePathList,
 							IsSymlinkList:  isSymlinkList,
 						},

cmd/commands/globusCheckTransfer.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/fatih/color"
@@ -46,22 +47,24 @@ For further help see "` + MANUAL + `"`,
 		markArchivable, _ := cmd.Flags().GetBool("mark-archivable")
 		dryRun, _ := cmd.Flags().GetBool("dry-run")
 		autoarchiveFlag, _ := cmd.Flags().GetBool("autoarchive")
+		skipDestPathCheck, _ := cmd.Flags().GetBool("skip-dest-path-check")
 		tapecopies, _ := cmd.Flags().GetInt("tapecopies")
 
 		if datasetUtils.TestFlags != nil {
 			datasetUtils.TestFlags(map[string]interface{}{
-				"testenv":         testenvFlag,
-				"devenv":          devenvFlag,
-				"localenv":        localenvFlag,
-				"tunnelenv":       tunnelenvFlag,
-				"user":            userpass,
-				"token":           token,
-				"version":         showVersion,
-				"globus-cfg":      globusCfgFlag,
-				"mark-archivable": globusCfgFlag,
-				"dry-run":         dryRun,
-				"autoarchive":     autoarchiveFlag,
-				"tapecopies":      tapecopies,
+				"testenv":              testenvFlag,
+				"devenv":               devenvFlag,
+				"localenv":             localenvFlag,
+				"tunnelenv":            tunnelenvFlag,
+				"user":                 userpass,
+				"token":                token,
+				"version":              showVersion,
+				"globus-cfg":           globusCfgFlag,
+				"mark-archivable":      globusCfgFlag,
+				"dry-run":              dryRun,
+				"autoarchive":          autoarchiveFlag,
+				"skip-dest-path-check": skipDestPathCheck,
+				"tapecopies":           tapecopies,
 			})
 			return
 		}
@@ -93,7 +96,7 @@ For further help see "` + MANUAL + `"`,
 			if err != nil {
 				log.Fatalln("can't find executable path:", err)
 			}
-			globusConfigPath = filepath.Join(execPath, "globus.yaml")
+			globusConfigPath = filepath.Join(filepath.Dir(execPath), "globus.yaml")
 		}
 
 		// environment overrides
@@ -133,7 +136,7 @@ For further help see "` + MANUAL + `"`,
 			user, _ = authenticate(RealAuthenticator{}, client, APIServer, userpass, token)
 		}
 
-		globusClient, _, _, err := cliutils.GlobusLogin(globusConfigPath)
+		globusClient, _, srcPrefixPath, _, _, err := cliutils.GlobusLogin(globusConfigPath)
 		if err != nil {
 			log.Fatalf("Couldn't create globus client: %v\n", err)
 		}
@@ -151,10 +154,23 @@ For further help see "` + MANUAL + `"`,
 			// if marking as archivable is requested and the transfer has succeded
 			if markArchivable && task.Status == "SUCCEEDED" {
 				if task.SourceBasePath == nil {
-					log.Printf("Can't get source base path for \"%s\". It will not be marked as archivable, but can be archived.\n", taskId)
+					log.Printf("Can't get source base path for task \"%s\". It will not be marked as archivable, but can probably be archived.\n", taskId)
 					continue
 				}
+
+				// get source and dest folders
 				sourceFolder := *task.SourceBasePath
+				sourceFolder = strings.TrimPrefix(sourceFolder, srcPrefixPath)
+				sourceFolder = strings.TrimSuffix(sourceFolder, "/")
+				var destFolder string
+				if !skipDestPathCheck {
+					if task.DestinationBasePath == nil {
+						log.Printf("Can't get destination base path for task \"%s\". It will not be marked as archivable, but can probably be archived.\n", taskId)
+						continue
+					}
+					destFolder = *task.DestinationBasePath
+				}
+
 				list, err := datasetIngestor.TestForExistingSourceFolder([]string{sourceFolder}, client, APIServer, user["accessToken"])
 
 				// error handling and exceptions
@@ -178,6 +194,18 @@ For further help see "` + MANUAL + `"`,
 				}
 
 				for _, result := range list {
+					if !skipDestPathCheck {
+						separatedPid := strings.Split(result.Pid, "/")
+						if len(separatedPid) != 2 {
+							log.Printf("\"%s\" dataset has irregular PID. Cannot check destFolder with it. Skipping...\n", result.Pid)
+							continue
+						}
+						shortPid := separatedPid[1]
+						if !strings.Contains(destFolder, shortPid) {
+							log.Printf("\"%s\" dataset's PID does not appear in the destination folder (\"%s\"). Cannot mark it as archivable. Skipping...\n", result.Pid, destFolder)
+							continue
+						}
+					}
 					log.Printf("%s dataset is being marked as archivable...\n", result.Pid)
 					err := datasetIngestor.MarkFilesReady(client, APIServer, result.Pid, user)
 					if err != nil {
@@ -223,6 +251,7 @@ func init() {
 	globusCheckTransfer.Flags().Bool("mark-archivable", false, "")
 	globusCheckTransfer.Flags().Bool("dry-run", false, "")
 	globusCheckTransfer.Flags().Bool("autoarchive", false, "")
+	globusCheckTransfer.Flags().Bool("skip-dest-path-check", false, "")
 	globusCheckTransfer.Flags().Int("tapecopies", 0, "Number of tapecopies to be used for archiving")
 
 	globusCheckTransfer.MarkFlagsMutuallyExclusive("testenv", "devenv", "localenv", "tunnelenv")

cmd/globus-config-example.yaml

@@ -0,0 +1,11 @@
+client-id: CLIENT_ID_HERE
+client-secret: (OPTIONAL)CLIENT_SECRET_HERE
+redirect-url: REDIRECT_URL_HERE
+scopes:
+ - (optional)
+ - scopes
+ - here
+source-collection: SOURCE_COLLECTION_UUID_HERE
+source-prefix-path: (OPTIONAL)SOURCE_PREFIX_PATH_HERE
+destination-collection: DESTINATION_COLLECTION_UUID_HERE
+destination-prefix-path: (OPTIONAL)DESTINATION_PREFIX_PATH_HERE

go.mod

@@ -6,7 +6,7 @@ toolchain go1.22.5
 
 require (
 	github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2
-	github.com/SwissOpenEM/globus v0.0.0-20240821075902-f0341ffd7619
+	github.com/SwissOpenEM/globus v0.0.0-20240822132653-119ec5e19eab
 	github.com/fatih/color v1.13.0
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2

go.sum

@@ -1,7 +1,7 @@
 github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s=
 github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
-github.com/SwissOpenEM/globus v0.0.0-20240821075902-f0341ffd7619 h1:FE3z8fPBL8mGsUcG9k8Yxf/wmIRsUfjbbM0t1IDVqB0=
-github.com/SwissOpenEM/globus v0.0.0-20240821075902-f0341ffd7619/go.mod h1:HiMwPdtUdztPpnA0TamNWBBRPGYjEJWXSRUIV5vjqXc=
+github.com/SwissOpenEM/globus v0.0.0-20240822132653-119ec5e19eab h1:Kn57UraLrMUdm/fXZM6qD60xh3vd3B9k1I42JIGvcqo=
+github.com/SwissOpenEM/globus v0.0.0-20240822132653-119ec5e19eab/go.mod h1:HiMwPdtUdztPpnA0TamNWBBRPGYjEJWXSRUIV5vjqXc=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI=
 github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=