Refining use of paths during transfer copy

hack/ccp/go.mod

@@ -18,14 +18,13 @@ require (
 	github.com/gohugoio/hugo v0.126.1
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-retryablehttp v0.7.6
-	github.com/microsoft/kiota-abstractions-go v1.6.0
 	github.com/mikespook/gearman-go v0.0.0-20220520031403-2a518e866145
 	github.com/peterbourgon/ff/v3 v3.4.0
 	github.com/rs/cors v1.11.0
 	github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a
 	github.com/testcontainers/testcontainers-go v0.31.0
 	github.com/testcontainers/testcontainers-go/modules/mysql v0.31.0
-	go.artefactual.dev/ssclient v0.3.0
+	go.artefactual.dev/ssclient v0.4.0
 	go.artefactual.dev/tools v0.12.0
 	go.nhat.io/httpmock v0.11.0
 	go.starlark.net v0.0.0-20240510163022-f457c4c2b267
@@ -74,6 +73,7 @@ require (
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/microsoft/kiota-abstractions-go v1.6.0 // indirect
 	github.com/microsoft/kiota-http-go v1.4.1 // indirect
 	github.com/microsoft/kiota-serialization-form-go v1.0.0 // indirect
 	github.com/microsoft/kiota-serialization-json-go v1.0.7 // indirect

hack/ccp/go.sum

@@ -373,8 +373,8 @@ github.com/yuin/goldmark-emoji v1.0.2 h1:c/RgTShNgHTtc6xdz2KKI74jJr6rWi7FPgnP9GA
 github.com/yuin/goldmark-emoji v1.0.2/go.mod h1:RhP/RWpexdp+KHs7ghKnifRoIs/Bq4nDS7tRbCkOwKY=
 github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
 github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-go.artefactual.dev/ssclient v0.3.0 h1:1Pp996L08nTf4taY6k2KWQLQZw9Hlcn+jeGC5FvEAl4=
-go.artefactual.dev/ssclient v0.3.0/go.mod h1:ZtoXTqqbUM2+hKVcHwgEAP4t/AyfVGUCJOXAsgRl5dg=
+go.artefactual.dev/ssclient v0.4.0 h1:WkBzSdHYIroNQlS+pBC59Nqwfs1njNRJzb8uVmPp4Qo=
+go.artefactual.dev/ssclient v0.4.0/go.mod h1:ZtoXTqqbUM2+hKVcHwgEAP4t/AyfVGUCJOXAsgRl5dg=
 go.artefactual.dev/tools v0.12.0 h1:NxSnKoTcYEwnr91/8fOjC5gr20uhVlU5ouboIHAhT9M=
 go.artefactual.dev/tools v0.12.0/go.mod h1:exAc0MSKv1oXXb3FuSwkN+tg0n95HHGKpM18lxu4CKU=
 go.nhat.io/httpmock v0.11.0 h1:GSADjr4/sn1HXqnyluPr9PYpSmMh/h3ty0O7lEozD3c=

hack/ccp/integration/config_test.go

@@ -28,7 +28,7 @@ func getEnv(name, fallback string) string {
 
 func getEnvRequired(name string) string {
 	v := getEnv(name, "")
-	if v == "" {
+	if v == "" && enabled {
 		log.Fatalf("Required env %s_%s is empty.", prefix, name)
 	}
 	return v

hack/ccp/internal/controller/package.go

@@ -187,7 +187,6 @@ func NewTransferPackage(
 		// Copy into new location.
 		path, err := copyTransfer(ctx, ssclient, sharedDir, tmpDir, req.Name, req.Path[0])
 		if err != nil {
-			logger.Info("SHIT", "err", err)
 			return fmt.Errorf("copy transfer: %v", err)
 		}
 		pkg.UpdatePath(path)

hack/ccp/internal/controller/path.go

@@ -34,6 +34,9 @@ func joinPath(elem ...string) string {
 	if last == "" || strings.HasSuffix(last, sep) {
 		ret += sep
 	}
+	if last == "." {
+		ret += "/."
+	}
 
 	return ret
 }

hack/ccp/internal/controller/path_test.go

@@ -60,8 +60,20 @@ func TestJoinPath(t *testing.T) {
 			want: "a/b/c",
 		},
 		{
-			elem: []string{"a", "b/"},
-			want: "a/b/",
+			elem: []string{"a", "b", "c/"},
+			want: "a/b/c/",
+		},
+		{
+			elem: []string{"a", "b", "c", "/"},
+			want: "a/b/c/",
+		},
+		{
+			elem: []string{"a", "b", "c", ""},
+			want: "a/b/c/",
+		},
+		{
+			elem: []string{"a", "b", "c", "."},
+			want: "a/b/c/.",
 		},
 		{
 			elem: nil,

hack/ccp/internal/controller/source.go

@@ -39,9 +39,6 @@ func copyTransfer(ctx context.Context, ssclient ssclient.Client, sharedDir, tmpD
 		destAbs,
 		filepath.Join(joinPath(sharedDir, "currentlyProcessing")),
 	)
-	if err != nil {
-		return "", err
-	}
 
 	return final, err
 }
@@ -85,7 +82,7 @@ func determineTransferPaths(sharedDir, tmpDir, name, path string) (string, strin
 		_, p := locationPath(path)
 		destAbs = filepath.Join(tmpDir, filepath.Base(p))
 	} else {
-		path = joinPath(path, "") // Copy contents of dir but not dir.
+		path = joinPath(path, ".") // Copy contents of dir but not dir.
 		destAbs = filepath.Join(tmpDir, name)
 		transferDir = destAbs
 	}
@@ -104,10 +101,10 @@ func moveToInternalSharedDir(src, dst string) (_ string, err error) {
 		return "", errors.New("no path provided")
 	}
 	if strings.Contains(src, "..") {
-		return "", errors.New("illegal path")
+		return "", fmt.Errorf("illegal path: %q", src)
 	}
 	if _, err := os.Stat(src); os.IsNotExist(err) {
-		return "", errors.New("path does not exist")
+		return "", fmt.Errorf("path does not exist: %q", src)
 	}
 
 	var (
@@ -183,6 +180,7 @@ func copyFromTransferSources(ctx context.Context, c ssclient.Client, sharedDir s
 		}
 
 		dir := isDir(filepath.Join(sharedDir, "tmp", strings.TrimPrefix("/", destRel)))
+		fmt.Println(dir, sharedDir)
 
 		// Source relative to the transfer source path.
 		source := strings.Replace(path, ops.transferSource.Path, "", 1)
@@ -192,19 +190,29 @@ func copyFromTransferSources(ctx context.Context, c ssclient.Client, sharedDir s
 		// # a file, or the last folder if not. Keep the trailing / for folders.
 		//
 		// TODO: this is broken.
-		var lastSegment string
-		if dir {
-			lastSegment = joinPath(filepath.Base(filepath.Dir(source)), "")
-		} else {
-			lastSegment = filepath.Base(source)
-		}
-		destination := joinPath(currentlyProcessing.Path, destRel, lastSegment)
+		/*
+			var lastSegment string
+			if dir {
+				lastSegment = joinPath(filepath.Base(filepath.Dir(source)), "")
+			} else {
+				lastSegment = filepath.Base(source)
+			}
+		*/
+
+		destination := joinPath(currentlyProcessing.Path, destRel, "") + "."
 		destination = strings.Replace(destination, "%sharedPath%", "", 1)
 
-		// {
-		//   "archivematica/archivematica-sampledata/SampleTransfers/Images/pictures"
-		//   "/var/archivematica/sharedDirectory/tmp/3598350318/Prueba/Images/"
-		// }
+		// What SS expects must look like this:
+		//
+		//	[{
+		//		'source': 'archivematica/transfer/.',
+		//		'destination': '/var/archivematica/sharedDirectory/tmp/tmp9an4_1zv/20240521104109/.'
+		//	}]
+		//
+		//	[{
+		//		'source': 'archivematica/archivematica-sampledata/SampleTransfers/Images/pictures/.',
+		//		'destination': '/var/archivematica/sharedDirectory/tmp/tmpzwq0mg0r/20240521104254/.'
+		//	}]
 		ops.files = append(ops.files, [2]string{source, destination})
 	}
 

hack/ccp/internal/controller/source_test.go

@@ -1,13 +1,145 @@
 package controller
 
 import (
+	"context"
+	"os"
 	"path/filepath"
 	"testing"
 
+	"go.artefactual.dev/tools/mockutil"
+	"go.uber.org/mock/gomock"
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/fs"
+
+	"github.com/artefactual/archivematica/hack/ccp/internal/ssclient"
+	"github.com/artefactual/archivematica/hack/ccp/internal/ssclient/enums"
+	"github.com/artefactual/archivematica/hack/ccp/internal/ssclient/ssclientmock"
 )
 
+func TestCopyTransfer(t *testing.T) {
+	t.Parallel()
+
+	type args struct {
+		name     string
+		path     string
+		ssclient func(rec *ssclientmock.MockClientMockRecorder)
+	}
+
+	type want struct {
+		path     string
+		contents fs.Manifest
+		err      string
+	}
+
+	type test struct {
+		args args
+		want want
+	}
+
+	commonCalls := func(rec *ssclientmock.MockClientMockRecorder) {
+		rec.ReadDefaultLocation(
+			mockutil.Context(),
+			enums.LocationPurposeTS).
+			Return(
+				&ssclient.Location{
+					URI:     "/api/v2/location/440ec678-ef9f-463c-8725-b6222d44c66d/",
+					Purpose: enums.LocationPurposeTS,
+				},
+				nil,
+			).
+			Times(1)
+		rec.ReadProcessingLocation(mockutil.Context()).
+			Return(
+				&ssclient.Location{
+					URI:     "/api/v2/location/c72d6333-b8a8-45a8-846c-1fb9b57e3629/",
+					Purpose: enums.LocationPurposeCP,
+				},
+				nil,
+			).
+			Times(1)
+		rec.ListLocations(mockutil.Context(), "", enums.LocationPurposeTS).
+			Return(
+				[]*ssclient.Location{
+					{
+						URI:     "/api/v2/location/440ec678-ef9f-463c-8725-b6222d44c66d/",
+						Purpose: enums.LocationPurposeTS,
+					},
+				},
+				nil,
+			).
+			Times(1)
+	}
+
+	sharedDir := fs.NewDir(t, "ccp-shared",
+		fs.WithDir("tmp"),
+		fs.WithDir("currentlyProcessing"),
+	)
+
+	tests := map[string]test{
+		"Transfer1": {
+			args: args{
+				name: "Transfer1",
+				path: "/home/archivematica/transfer1",
+				ssclient: func(rec *ssclientmock.MockClientMockRecorder) {
+					commonCalls(rec)
+					rec.MoveFiles(
+						mockutil.Context(),
+						&ssclient.Location{
+							URI:     "/api/v2/location/440ec678-ef9f-463c-8725-b6222d44c66d/",
+							Purpose: enums.LocationPurposeTS,
+						},
+						&ssclient.Location{
+							URI:     "/api/v2/location/c72d6333-b8a8-45a8-846c-1fb9b57e3629/",
+							Purpose: enums.LocationPurposeCP,
+						},
+						[][2]string{
+							{
+								"home/archivematica/transfer1",
+								"/tmp/Transfer1/.",
+							},
+						},
+					).DoAndReturn(func(ctx context.Context, ts, cp *ssclient.Location, files [][2]string) error {
+						for _, item := range files {
+							os.MkdirAll(sharedDir.Join(item[1]), os.FileMode(0o770))
+						}
+						return nil
+					}).Times(1)
+				},
+			},
+			want: want{
+				path:     sharedDir.Join("currentlyProcessing/Transfer1"),
+				contents: fs.Expected(t, fs.MatchAnyFileMode),
+			},
+		},
+	}
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			ssclient := ssclientmock.NewMockClient(gomock.NewController(t))
+			tc.args.ssclient(ssclient.EXPECT())
+
+			path, err := copyTransfer(
+				context.Background(),
+				ssclient,
+				sharedDir.Path(),
+				sharedDir.Join("tmp"),
+				tc.args.name,
+				tc.args.path,
+			)
+
+			if tc.want.err != "" {
+				assert.Error(t, err, tc.want.err)
+				return
+			}
+
+			assert.NilError(t, err)
+			assert.Equal(t, path, tc.want.path)
+			assert.Assert(t, fs.Equal(tc.want.path, tc.want.contents))
+		})
+	}
+}
+
 func TestDetermineTransferPaths(t *testing.T) {
 	t.Parallel()
 
@@ -48,7 +180,7 @@ func TestDetermineTransferPaths(t *testing.T) {
 			want{
 				destRel: "/tmp/tmp.12345/Name2",
 				destAbs: "/var/archivematica/sharedDirectory/tmp/tmp.12345/Name2",
-				src:     "/var/source/transfer/",
+				src:     "/var/source/transfer/.",
 			},
 		},
 		{

hack/ccp/internal/ssclient/ssclient.go

@@ -1,7 +1,6 @@
 package ssclient
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -10,8 +9,6 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-	"github.com/hashicorp/go-retryablehttp"
-	"github.com/microsoft/kiota-abstractions-go/serialization"
 	ssclientlib "go.artefactual.dev/ssclient"
 	"go.artefactual.dev/ssclient/kiota/api"
 	"go.artefactual.dev/ssclient/kiota/models"
@@ -214,25 +211,7 @@ func (c *clientImpl) MoveFiles(ctx context.Context, src, dst *Location, files []
 	}
 	body.SetFiles(moves)
 
-	// TODO: why is this not working?
-	// _, err = c.client.Location().ByUuid(dst.ID.String()).Post(context.Background(), body, nil)
-
-	payload, err := serialization.SerializeToJson(body)
-	if err != nil {
-		return err
-	}
-	httpClient := retryablehttp.NewClient().StandardClient()
-	url := fmt.Sprintf("%s/api/v2/location/%s/", c.config.BaseURL, dst.ID.String())
-	req, _ := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload))
-	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("Authorization", fmt.Sprintf("ApiKey %s:%s", c.config.Username, c.config.Key))
-	resp, err := httpClient.Do(req)
-	if err != nil {
-		return err
-	}
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("request not ok: status code %d", resp.StatusCode)
-	}
+	_, err = c.client.Location().ByUuid(dst.ID.String()).Post(context.Background(), body, nil)
 
 	return err
 }