Feature/v4backend

cmd/commands/transferType.go → cmd/cliutils/transferType.go

@@ -1,18 +1,18 @@
-package cmd
+package cliutils
 
 import (
 	"fmt"
 	"strings"
 )
 
-type transferType int
+type TransferType int
 
 const (
-	Ssh transferType = iota
+	Ssh TransferType = iota
 	Globus
 )
 
-func convertToTransferType(input string) (transferType, error) {
+func ConvertToTransferType(input string) (TransferType, error) {
 	input = strings.ToLower(input)
 	switch input {
 	case "ssh":

cmd/cliutils/transferType_test.go

@@ -0,0 +1,33 @@
+package cliutils
+
+import "testing"
+
+func TestConvertToTransferTypeSsh(t *testing.T) {
+	transfer := "ssh"
+	testSsh, err := ConvertToTransferType(transfer)
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	if testSsh != Ssh {
+		t.Errorf("Invalid transfer type: %v", testSsh)
+	}
+}
+
+func TestConvertToTransferTypeGlobus(t *testing.T) {
+	transfer := "globus"
+	testSsh, err := ConvertToTransferType(transfer)
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	if testSsh != Globus {
+		t.Errorf("Invalid transfer type: %v", testSsh)
+	}
+}
+
+func TestConvertToTransferWrongType(t *testing.T) {
+	transfer := "lolrandom"
+	testSsh, err := ConvertToTransferType(transfer)
+	if err == nil {
+		t.Errorf("Expected an error. Got: %v - %v", testSsh, err)
+	}
+}

cmd/commands/authenticate.go

@@ -13,17 +13,17 @@ import (
 
 // An interface with the methods so that we can mock them in tests
 type Authenticator interface {
-	AuthenticateUser(httpClient *http.Client, APIServer string, username string, password string) (map[string]string, []string)
-	GetUserInfoFromToken(httpClient *http.Client, APIServer string, token string) (map[string]string, []string)
+	AuthenticateUser(httpClient *http.Client, APIServer string, username string, password string) (map[string]string, []string, error)
+	GetUserInfoFromToken(httpClient *http.Client, APIServer string, token string) (map[string]string, []string, error)
 }
 
 type RealAuthenticator struct{}
 
-func (r RealAuthenticator) AuthenticateUser(httpClient *http.Client, APIServer string, username string, password string) (map[string]string, []string) {
+func (r RealAuthenticator) AuthenticateUser(httpClient *http.Client, APIServer string, username string, password string) (map[string]string, []string, error) {
 	return datasetUtils.AuthenticateUser(httpClient, APIServer, username, password)
 }
 
-func (r RealAuthenticator) GetUserInfoFromToken(httpClient *http.Client, APIServer string, token string) (map[string]string, []string) {
+func (r RealAuthenticator) GetUserInfoFromToken(httpClient *http.Client, APIServer string, token string) (map[string]string, []string, error) {
 	return datasetUtils.GetUserInfoFromToken(httpClient, APIServer, token)
 }
 
@@ -32,18 +32,21 @@ func (r RealAuthenticator) GetUserInfoFromToken(httpClient *http.Client, APIServ
 // and returning an authentication token if the credentials are valid.
 // This token can then be used for authenticated requests to the server.
 // If the credentials are not valid, the function returns an error.
-func authenticate(authenticator Authenticator, httpClient *http.Client, apiServer string, userpass string, token string, overrideFatalExit ...func(v ...any)) (map[string]string, []string) {
+func authenticate(authenticator Authenticator, httpClient *http.Client, apiServer string, userpass string, token string, overrideFatalExit ...func(v ...any)) (map[string]string, []string, error) {
 	fatalExit := log.Fatal // by default, call log fatal
 	if len(overrideFatalExit) == 1 {
 		fatalExit = overrideFatalExit[0]
 	}
 	if token != "" {
-		user, accessGroups := authenticator.GetUserInfoFromToken(httpClient, apiServer, token)
+		user, accessGroups, err := authenticator.GetUserInfoFromToken(httpClient, apiServer, token)
+		if err != nil {
+			return map[string]string{}, []string{}, err
+		}
 		uSplit := strings.Split(userpass, ":")
 		if len(uSplit) > 1 {
 			user["password"] = uSplit[1]
 		}
-		return user, accessGroups
+		return user, accessGroups, nil
 	}
 
 	if userpass != "" {

cmd/commands/authenticate_test.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
@@ -10,15 +11,15 @@ import (
 // Create a mock implementation of the interface
 type MockAuthenticator struct{}
 
-func (m *MockAuthenticator) AuthenticateUser(httpClient *http.Client, APIServer string, username string, password string) (map[string]string, []string) {
+func (m *MockAuthenticator) AuthenticateUser(httpClient *http.Client, APIServer string, username string, password string) (map[string]string, []string, error) {
 	if username == "" && password == "" {
-		return map[string]string{}, []string{}
+		return map[string]string{}, []string{}, fmt.Errorf("no username or password was provided")
 	}
-	return map[string]string{"username": "testuser", "password": "testpass"}, []string{"group1", "group2"}
+	return map[string]string{"username": "testuser", "password": "testpass"}, []string{"group1", "group2"}, nil
 }
 
-func (m *MockAuthenticator) GetUserInfoFromToken(httpClient *http.Client, APIServer string, token string) (map[string]string, []string) {
-	return map[string]string{"username": "tokenuser", "password": "tokenpass"}, []string{"group3", "group4"}
+func (m *MockAuthenticator) GetUserInfoFromToken(httpClient *http.Client, APIServer string, token string) (map[string]string, []string, error) {
+	return map[string]string{"username": "tokenuser", "password": "tokenpass"}, []string{"group3", "group4"}, nil
 }
 
 func TestAuthenticate(t *testing.T) {
@@ -82,7 +83,12 @@ func TestAuthenticate(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			httpClient := server.Client()
-			user, group := authenticate(auth, httpClient, server.URL, tt.userpass, tt.token, noExit)
+			user, group, err := authenticate(auth, httpClient, server.URL, tt.userpass, tt.token, noExit)
+			if err != nil {
+				if err.Error() != "no username or password was provided" {
+					t.Errorf("authenticate returned an error: %s", err.Error())
+				}
+			}
 
 			if !reflect.DeepEqual(user, tt.wantUser) {
 				t.Errorf("got %v, want %v", user, tt.wantUser)

cmd/commands/commonConstants.go

@@ -5,7 +5,7 @@ const MANUAL = "http://melanie.gitpages.psi.ch/SciCatPages"
 const PROD_API_SERVER string = "https://dacat.psi.ch/api/v3"
 const TEST_API_SERVER string = "https://dacat-qa.psi.ch/api/v3"
 const DEV_API_SERVER string = "https://dacat-development.psi.ch/api/v3"
-const LOCAL_API_SERVER string = "http://localhost:3000/api/v3"
+const LOCAL_API_SERVER string = "http://backend.localhost/api/v3"
 const TUNNEL_API_SERVER string = "https://dacat-development.psi.ch:5443/api/v3"
 
 const PROD_RSYNC_ARCHIVE_SERVER string = "pb-archive.psi.ch"

cmd/commands/datasetArchiver.go

@@ -7,7 +7,6 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/fatih/color"
@@ -26,7 +25,6 @@ Or you choose a (list of) datasetIds, in which case all archivable datasets
 of this list not yet archived will be archived. 
 
 For further help see "` + MANUAL + `"`,
-	Args: minArgsWithVersionException(1),
 	Run: func(cmd *cobra.Command, args []string) {
 		// consts & vars
 		var client = &http.Client{
@@ -47,6 +45,7 @@ For further help see "` + MANUAL + `"`,
 		localenvFlag, _ := cmd.Flags().GetBool("localenv")
 		devenvFlag, _ := cmd.Flags().GetBool("devenv")
 		nonInteractiveFlag, _ := cmd.Flags().GetBool("noninteractive")
+		ownergroupFlag, _ := cmd.Flags().GetString("ownergroup")
 		showVersion, _ := cmd.Flags().GetBool("version")
 
 		if datasetUtils.TestFlags != nil {
@@ -59,6 +58,7 @@ For further help see "` + MANUAL + `"`,
 				"devenv":         devenvFlag,
 				"noninteractive": nonInteractiveFlag,
 				"version":        showVersion,
+				"ownergroup":     ownergroupFlag,
 			})
 			return
 		}
@@ -89,25 +89,25 @@ For further help see "` + MANUAL + `"`,
 		log.Printf("You are about to archive dataset(s) to the === %s === data catalog environment...", env)
 		color.Unset()
 
-		ownerGroup := ""
-		inputdatasetList := make([]string, 0)
+		ownerGroup := ownergroupFlag
 
-		// argsWithoutProg := os.Args[1:]
-		if len(args) <= 0 {
-			log.Fatalln("invalid number of args")
-		}
-
-		if len(args) == 1 && !strings.Contains(args[0], "/") {
-			ownerGroup = args[0]
-		} else {
+		// optional list of dataset id's, if not specified, the full list of datasets of the ownergroup will be archived
+		var inputdatasetList []string
+		if len(args) > 0 {
 			inputdatasetList = args[0:]
 		}
 
-		user, _ := authenticate(RealAuthenticator{}, client, APIServer, userpass, token)
+		user, _, err := authenticate(RealAuthenticator{}, client, APIServer, userpass, token)
+		if err != nil {
+			log.Fatal(err)
+		}
 
-		archivableDatasets := datasetUtils.GetArchivableDatasets(client, APIServer, ownerGroup, inputdatasetList, user["accessToken"])
+		archivableDatasets, err := datasetUtils.GetArchivableDatasets(client, APIServer, ownerGroup, inputdatasetList, user["accessToken"])
+		if err != nil {
+			log.Fatalf("GetArchivableDatasets: %s\n", err.Error())
+		}
 		if len(archivableDatasets) <= 0 {
-			log.Fatalf("No archivable datasets remaining")
+			log.Fatalln("No archivable datasets remaining")
 		}
 
 		archive := ""
@@ -125,7 +125,7 @@ For further help see "` + MANUAL + `"`,
 
 		log.Printf("You chose to archive the new datasets\n")
 		log.Printf("Submitting Archive Job for the ingested datasets.\n")
-		jobId, err := datasetUtils.CreateArchivalJob(client, APIServer, user, archivableDatasets, &tapecopies)
+		jobId, err := datasetUtils.CreateArchivalJob(client, APIServer, user, ownerGroup, archivableDatasets, &tapecopies)
 		if err != nil {
 			log.Fatalf("Couldn't create a job: %s\n", err.Error())
 		}
@@ -141,6 +141,8 @@ func init() {
 	datasetArchiverCmd.Flags().Bool("localenv", false, "Use local environment (local) instead or production")
 	datasetArchiverCmd.Flags().Bool("devenv", false, "Use development environment instead or production")
 	datasetArchiverCmd.Flags().Bool("noninteractive", false, "Defines if no questions will be asked, just do it - make sure you know what you are doing")
+	datasetArchiverCmd.Flags().String("ownergroup", "", "Specifies to which owner group should the archival job belong. If no datasets id's are passed, all datasets belonging to this ownergroup that can also be marked as archivable will be included")
 
 	datasetArchiverCmd.MarkFlagsMutuallyExclusive("testenv", "localenv", "devenv")
+	datasetArchiverCmd.MarkFlagRequired("ownergroup")
 }

cmd/commands/datasetCleaner.go

@@ -94,7 +94,10 @@ For further help see "` + MANUAL + `"`,
 		}
 		pid := args[0]
 
-		user, _ := authenticate(RealAuthenticator{}, client, APIServer, userpass, token)
+		user, _, err := authenticate(RealAuthenticator{}, client, APIServer, userpass, token)
+		if err != nil {
+			log.Fatal(err)
+		}
 
 		if user["username"] != "archiveManager" {
 			log.Fatalf("You must be archiveManager to be allowed to delete datasets\n")

cmd/commands/datasetGetProposal.go

@@ -37,6 +37,7 @@ For further help see "` + MANUAL + `"`,
 		fieldname, _ := cmd.Flags().GetString("field")
 		testenvFlag, _ := cmd.Flags().GetBool("testenv")
 		devenvFlag, _ := cmd.Flags().GetBool("devenv")
+		localenvFlag, _ := cmd.Flags().GetBool("localenv")
 		showVersion, _ := cmd.Flags().GetBool("version")
 
 		if datasetUtils.TestFlags != nil {
@@ -60,6 +61,10 @@ For further help see "` + MANUAL + `"`,
 		// check for program version only if running interactively
 		datasetUtils.CheckForNewVersion(client, APP, VERSION)
 
+		if localenvFlag {
+			APIServer = LOCAL_API_SERVER
+			env = "local"
+		}
 		if devenvFlag {
 			APIServer = DEV_API_SERVER
 			env = "dev"
@@ -79,8 +84,11 @@ For further help see "` + MANUAL + `"`,
 		}
 		ownerGroup := args[0]
 
-		user, accessGroups := authenticate(RealAuthenticator{}, client, APIServer, userpass, token)
-		proposal, err := datasetUtils.GetProposal(client, APIServer, ownerGroup, user, accessGroups)
+		user, _, err := authenticate(RealAuthenticator{}, client, APIServer, userpass, token)
+		if err != nil {
+			log.Fatal(err)
+		}
+		proposal, err := datasetUtils.GetProposal(client, APIServer, ownerGroup, user)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -106,6 +114,7 @@ func init() {
 	datasetGetProposalCmd.Flags().String("field", "", "Defines optional field name , whose value should be returned instead of full information")
 	datasetGetProposalCmd.Flags().Bool("testenv", false, "Use test environment (qa) instead or production")
 	datasetGetProposalCmd.Flags().Bool("devenv", false, "Use development environment instead or production")
+	datasetGetProposalCmd.Flags().Bool("localenv", false, "Use local environment instead of production environment (developers only)")
 
-	datasetGetProposalCmd.MarkFlagsMutuallyExclusive("testenv", "devenv")
+	datasetGetProposalCmd.MarkFlagsMutuallyExclusive("testenv", "devenv", "localenv")
 }

cmd/commands/datasetIngestor.go

@@ -82,7 +82,7 @@ For Windows you need instead to specify -user username:password on the command l
 		// TODO: read in CFG!
 
 		// transfer type
-		transferType, err := convertToTransferType(transferTypeFlag)
+		transferType, err := cliutils.ConvertToTransferType(transferTypeFlag)
 		if err != nil {
 			log.Fatalln(err)
 		}
@@ -94,9 +94,9 @@ For Windows you need instead to specify -user username:password on the command l
 		var gConfig cliutils.GlobusConfig
 
 		switch transferType {
-		case Ssh:
+		case cliutils.Ssh:
 			transferFiles = cliutils.SshTransfer
-		case Globus:
+		case cliutils.Globus:
 			transferFiles = cliutils.GlobusTransfer
 			var globusConfigPath string
 			if cmd.Flags().Lookup("globus-cfg").Changed {
@@ -204,7 +204,10 @@ For Windows you need instead to specify -user username:password on the command l
 		log.Printf("You are about to add a dataset to the === %s === data catalog environment...", env)
 		color.Unset()
 
-		user, accessGroups := authenticate(RealAuthenticator{}, client, APIServer, userpass, token)
+		user, accessGroups, err := authenticate(RealAuthenticator{}, client, APIServer, userpass, token)
+		if err != nil {
+			log.Fatal(err)
+		}
 
 		/* TODO Add info about policy settings and that autoarchive will take place or not */
 		metaDataMap, metadataSourceFolder, beamlineAccount, err := datasetIngestor.ReadAndCheckMetadata(client, APIServer, metadatafile, user, accessGroups)
@@ -306,6 +309,10 @@ For Windows you need instead to specify -user username:password on the command l
 
 		// now everything is prepared, prepare to loop over all folders
 		var archivableDatasetList []string
+		archivableDatasetListOwnerGroup, ok := metaDataMap["ownerGroup"].(string)
+		if !ok {
+			log.Fatal("can't recover ownerGroup. This should normally be impossible as the checkMetadata function should've caught it already.")
+		}
 		for _, datasetSourceFolder := range datasetPaths {
 			log.Printf("===== Ingesting: \"%s\" =====\n", datasetSourceFolder)
 			// ignore empty lines
@@ -513,17 +520,19 @@ For Windows you need instead to specify -user username:password on the command l
 			os.Exit(1)
 		}
 
-		// === create archive job ===
+		// === create archive jobs ===
 		if autoarchiveFlag && ingestFlag {
 			log.Printf("Submitting Archive Job for the ingested datasets.\n")
 			// TODO: change param type from pointer to regular as it is unnecessary
 			//   for it to be passed as pointer
-			jobId, err := datasetUtils.CreateArchivalJob(client, APIServer, user, archivableDatasetList, &tapecopies)
+			jobId, err := datasetUtils.CreateArchivalJob(client, APIServer, user, archivableDatasetListOwnerGroup, archivableDatasetList, &tapecopies)
+
 			if err != nil {
 				color.Set(color.FgRed)
-				log.Printf("Could not create the archival job for the ingested datasets: %s", err.Error())
+				log.Printf("Could not create the archival job for the ingested datasets: %s\n", err.Error())
 				color.Unset()
 			}
+
 			log.Println("Submitted job:", jobId)
 		}