diff --git a/hack/ccp/go.mod b/hack/ccp/go.mod index e34c7f4e5..882416d06 100644 --- a/hack/ccp/go.mod +++ b/hack/ccp/go.mod @@ -18,14 +18,13 @@ require ( github.com/gohugoio/hugo v0.126.1 github.com/google/uuid v1.6.0 github.com/hashicorp/go-retryablehttp v0.7.6 - github.com/microsoft/kiota-abstractions-go v1.6.0 github.com/mikespook/gearman-go v0.0.0-20220520031403-2a518e866145 github.com/peterbourgon/ff/v3 v3.4.0 github.com/rs/cors v1.11.0 github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a github.com/testcontainers/testcontainers-go v0.31.0 github.com/testcontainers/testcontainers-go/modules/mysql v0.31.0 - go.artefactual.dev/ssclient v0.3.0 + go.artefactual.dev/ssclient v0.4.0 go.artefactual.dev/tools v0.12.0 go.nhat.io/httpmock v0.11.0 go.starlark.net v0.0.0-20240510163022-f457c4c2b267 @@ -74,6 +73,7 @@ require ( github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/microsoft/kiota-abstractions-go v1.6.0 // indirect github.com/microsoft/kiota-http-go v1.4.1 // indirect github.com/microsoft/kiota-serialization-form-go v1.0.0 // indirect github.com/microsoft/kiota-serialization-json-go v1.0.7 // indirect diff --git a/hack/ccp/go.sum b/hack/ccp/go.sum index 12151cb98..6902f8587 100644 --- a/hack/ccp/go.sum +++ b/hack/ccp/go.sum @@ -373,8 +373,8 @@ github.com/yuin/goldmark-emoji v1.0.2 h1:c/RgTShNgHTtc6xdz2KKI74jJr6rWi7FPgnP9GA github.com/yuin/goldmark-emoji v1.0.2/go.mod h1:RhP/RWpexdp+KHs7ghKnifRoIs/Bq4nDS7tRbCkOwKY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.artefactual.dev/ssclient v0.3.0 h1:1Pp996L08nTf4taY6k2KWQLQZw9Hlcn+jeGC5FvEAl4= -go.artefactual.dev/ssclient v0.3.0/go.mod h1:ZtoXTqqbUM2+hKVcHwgEAP4t/AyfVGUCJOXAsgRl5dg= +go.artefactual.dev/ssclient v0.4.0 h1:WkBzSdHYIroNQlS+pBC59Nqwfs1njNRJzb8uVmPp4Qo= +go.artefactual.dev/ssclient v0.4.0/go.mod h1:ZtoXTqqbUM2+hKVcHwgEAP4t/AyfVGUCJOXAsgRl5dg= go.artefactual.dev/tools v0.12.0 h1:NxSnKoTcYEwnr91/8fOjC5gr20uhVlU5ouboIHAhT9M= go.artefactual.dev/tools v0.12.0/go.mod h1:exAc0MSKv1oXXb3FuSwkN+tg0n95HHGKpM18lxu4CKU= go.nhat.io/httpmock v0.11.0 h1:GSADjr4/sn1HXqnyluPr9PYpSmMh/h3ty0O7lEozD3c= diff --git a/hack/ccp/integration/config_test.go b/hack/ccp/integration/config_test.go index 5a03aa599..002b76c37 100644 --- a/hack/ccp/integration/config_test.go +++ b/hack/ccp/integration/config_test.go @@ -28,7 +28,7 @@ func getEnv(name, fallback string) string { func getEnvRequired(name string) string { v := getEnv(name, "") - if v == "" { + if v == "" && enabled { log.Fatalf("Required env %s_%s is empty.", prefix, name) } return v diff --git a/hack/ccp/internal/controller/package.go b/hack/ccp/internal/controller/package.go index aff38346c..0c311c427 100644 --- a/hack/ccp/internal/controller/package.go +++ b/hack/ccp/internal/controller/package.go @@ -187,7 +187,6 @@ func NewTransferPackage( // Copy into new location. path, err := copyTransfer(ctx, ssclient, sharedDir, tmpDir, req.Name, req.Path[0]) if err != nil { - logger.Info("SHIT", "err", err) return fmt.Errorf("copy transfer: %v", err) } pkg.UpdatePath(path) diff --git a/hack/ccp/internal/controller/path.go b/hack/ccp/internal/controller/path.go index 267686368..d34a10044 100644 --- a/hack/ccp/internal/controller/path.go +++ b/hack/ccp/internal/controller/path.go @@ -34,6 +34,9 @@ func joinPath(elem ...string) string { if last == "" || strings.HasSuffix(last, sep) { ret += sep } + if last == "." { + ret += "/." + } return ret } diff --git a/hack/ccp/internal/controller/path_test.go b/hack/ccp/internal/controller/path_test.go index 6a7f0201c..8cb38cce2 100644 --- a/hack/ccp/internal/controller/path_test.go +++ b/hack/ccp/internal/controller/path_test.go @@ -60,8 +60,20 @@ func TestJoinPath(t *testing.T) { want: "a/b/c", }, { - elem: []string{"a", "b/"}, - want: "a/b/", + elem: []string{"a", "b", "c/"}, + want: "a/b/c/", + }, + { + elem: []string{"a", "b", "c", "/"}, + want: "a/b/c/", + }, + { + elem: []string{"a", "b", "c", ""}, + want: "a/b/c/", + }, + { + elem: []string{"a", "b", "c", "."}, + want: "a/b/c/.", }, { elem: nil, diff --git a/hack/ccp/internal/controller/source.go b/hack/ccp/internal/controller/source.go index 6f9bb1b41..6f926d54c 100644 --- a/hack/ccp/internal/controller/source.go +++ b/hack/ccp/internal/controller/source.go @@ -39,9 +39,6 @@ func copyTransfer(ctx context.Context, ssclient ssclient.Client, sharedDir, tmpD destAbs, filepath.Join(joinPath(sharedDir, "currentlyProcessing")), ) - if err != nil { - return "", err - } return final, err } @@ -85,7 +82,7 @@ func determineTransferPaths(sharedDir, tmpDir, name, path string) (string, strin _, p := locationPath(path) destAbs = filepath.Join(tmpDir, filepath.Base(p)) } else { - path = joinPath(path, "") // Copy contents of dir but not dir. + path = joinPath(path, ".") // Copy contents of dir but not dir. destAbs = filepath.Join(tmpDir, name) transferDir = destAbs } @@ -104,10 +101,10 @@ func moveToInternalSharedDir(src, dst string) (_ string, err error) { return "", errors.New("no path provided") } if strings.Contains(src, "..") { - return "", errors.New("illegal path") + return "", fmt.Errorf("illegal path: %q", src) } if _, err := os.Stat(src); os.IsNotExist(err) { - return "", errors.New("path does not exist") + return "", fmt.Errorf("path does not exist: %q", src) } var ( @@ -183,6 +180,7 @@ func copyFromTransferSources(ctx context.Context, c ssclient.Client, sharedDir s } dir := isDir(filepath.Join(sharedDir, "tmp", strings.TrimPrefix("/", destRel))) + fmt.Println(dir, sharedDir) // Source relative to the transfer source path. source := strings.Replace(path, ops.transferSource.Path, "", 1) @@ -192,19 +190,29 @@ func copyFromTransferSources(ctx context.Context, c ssclient.Client, sharedDir s // # a file, or the last folder if not. Keep the trailing / for folders. // // TODO: this is broken. - var lastSegment string - if dir { - lastSegment = joinPath(filepath.Base(filepath.Dir(source)), "") - } else { - lastSegment = filepath.Base(source) - } - destination := joinPath(currentlyProcessing.Path, destRel, lastSegment) + /* + var lastSegment string + if dir { + lastSegment = joinPath(filepath.Base(filepath.Dir(source)), "") + } else { + lastSegment = filepath.Base(source) + } + */ + + destination := joinPath(currentlyProcessing.Path, destRel, "") + "." destination = strings.Replace(destination, "%sharedPath%", "", 1) - // { - // "archivematica/archivematica-sampledata/SampleTransfers/Images/pictures" - // "/var/archivematica/sharedDirectory/tmp/3598350318/Prueba/Images/" - // } + // What SS expects must look like this: + // + // [{ + // 'source': 'archivematica/transfer/.', + // 'destination': '/var/archivematica/sharedDirectory/tmp/tmp9an4_1zv/20240521104109/.' + // }] + // + // [{ + // 'source': 'archivematica/archivematica-sampledata/SampleTransfers/Images/pictures/.', + // 'destination': '/var/archivematica/sharedDirectory/tmp/tmpzwq0mg0r/20240521104254/.' + // }] ops.files = append(ops.files, [2]string{source, destination}) } diff --git a/hack/ccp/internal/controller/source_test.go b/hack/ccp/internal/controller/source_test.go index 7f97c4963..1fb3d7dd7 100644 --- a/hack/ccp/internal/controller/source_test.go +++ b/hack/ccp/internal/controller/source_test.go @@ -1,13 +1,145 @@ package controller import ( + "context" + "os" "path/filepath" "testing" + "go.artefactual.dev/tools/mockutil" + "go.uber.org/mock/gomock" "gotest.tools/v3/assert" "gotest.tools/v3/fs" + + "github.com/artefactual/archivematica/hack/ccp/internal/ssclient" + "github.com/artefactual/archivematica/hack/ccp/internal/ssclient/enums" + "github.com/artefactual/archivematica/hack/ccp/internal/ssclient/ssclientmock" ) +func TestCopyTransfer(t *testing.T) { + t.Parallel() + + type args struct { + name string + path string + ssclient func(rec *ssclientmock.MockClientMockRecorder) + } + + type want struct { + path string + contents fs.Manifest + err string + } + + type test struct { + args args + want want + } + + commonCalls := func(rec *ssclientmock.MockClientMockRecorder) { + rec.ReadDefaultLocation( + mockutil.Context(), + enums.LocationPurposeTS). + Return( + &ssclient.Location{ + URI: "/api/v2/location/440ec678-ef9f-463c-8725-b6222d44c66d/", + Purpose: enums.LocationPurposeTS, + }, + nil, + ). + Times(1) + rec.ReadProcessingLocation(mockutil.Context()). + Return( + &ssclient.Location{ + URI: "/api/v2/location/c72d6333-b8a8-45a8-846c-1fb9b57e3629/", + Purpose: enums.LocationPurposeCP, + }, + nil, + ). + Times(1) + rec.ListLocations(mockutil.Context(), "", enums.LocationPurposeTS). + Return( + []*ssclient.Location{ + { + URI: "/api/v2/location/440ec678-ef9f-463c-8725-b6222d44c66d/", + Purpose: enums.LocationPurposeTS, + }, + }, + nil, + ). + Times(1) + } + + sharedDir := fs.NewDir(t, "ccp-shared", + fs.WithDir("tmp"), + fs.WithDir("currentlyProcessing"), + ) + + tests := map[string]test{ + "Transfer1": { + args: args{ + name: "Transfer1", + path: "/home/archivematica/transfer1", + ssclient: func(rec *ssclientmock.MockClientMockRecorder) { + commonCalls(rec) + rec.MoveFiles( + mockutil.Context(), + &ssclient.Location{ + URI: "/api/v2/location/440ec678-ef9f-463c-8725-b6222d44c66d/", + Purpose: enums.LocationPurposeTS, + }, + &ssclient.Location{ + URI: "/api/v2/location/c72d6333-b8a8-45a8-846c-1fb9b57e3629/", + Purpose: enums.LocationPurposeCP, + }, + [][2]string{ + { + "home/archivematica/transfer1", + "/tmp/Transfer1/.", + }, + }, + ).DoAndReturn(func(ctx context.Context, ts, cp *ssclient.Location, files [][2]string) error { + for _, item := range files { + os.MkdirAll(sharedDir.Join(item[1]), os.FileMode(0o770)) + } + return nil + }).Times(1) + }, + }, + want: want{ + path: sharedDir.Join("currentlyProcessing/Transfer1"), + contents: fs.Expected(t, fs.MatchAnyFileMode), + }, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + ssclient := ssclientmock.NewMockClient(gomock.NewController(t)) + tc.args.ssclient(ssclient.EXPECT()) + + path, err := copyTransfer( + context.Background(), + ssclient, + sharedDir.Path(), + sharedDir.Join("tmp"), + tc.args.name, + tc.args.path, + ) + + if tc.want.err != "" { + assert.Error(t, err, tc.want.err) + return + } + + assert.NilError(t, err) + assert.Equal(t, path, tc.want.path) + assert.Assert(t, fs.Equal(tc.want.path, tc.want.contents)) + }) + } +} + func TestDetermineTransferPaths(t *testing.T) { t.Parallel() @@ -48,7 +180,7 @@ func TestDetermineTransferPaths(t *testing.T) { want{ destRel: "/tmp/tmp.12345/Name2", destAbs: "/var/archivematica/sharedDirectory/tmp/tmp.12345/Name2", - src: "/var/source/transfer/", + src: "/var/source/transfer/.", }, }, { diff --git a/hack/ccp/internal/ssclient/ssclient.go b/hack/ccp/internal/ssclient/ssclient.go index 325f506f5..ff5b56ffd 100644 --- a/hack/ccp/internal/ssclient/ssclient.go +++ b/hack/ccp/internal/ssclient/ssclient.go @@ -1,7 +1,6 @@ package ssclient import ( - "bytes" "context" "errors" "fmt" @@ -10,8 +9,6 @@ import ( "time" "github.com/google/uuid" - "github.com/hashicorp/go-retryablehttp" - "github.com/microsoft/kiota-abstractions-go/serialization" ssclientlib "go.artefactual.dev/ssclient" "go.artefactual.dev/ssclient/kiota/api" "go.artefactual.dev/ssclient/kiota/models" @@ -214,25 +211,7 @@ func (c *clientImpl) MoveFiles(ctx context.Context, src, dst *Location, files [] } body.SetFiles(moves) - // TODO: why is this not working? - // _, err = c.client.Location().ByUuid(dst.ID.String()).Post(context.Background(), body, nil) - - payload, err := serialization.SerializeToJson(body) - if err != nil { - return err - } - httpClient := retryablehttp.NewClient().StandardClient() - url := fmt.Sprintf("%s/api/v2/location/%s/", c.config.BaseURL, dst.ID.String()) - req, _ := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload)) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", fmt.Sprintf("ApiKey %s:%s", c.config.Username, c.config.Key)) - resp, err := httpClient.Do(req) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("request not ok: status code %d", resp.StatusCode) - } + _, err = c.client.Location().ByUuid(dst.ID.String()).Post(context.Background(), body, nil) return err } diff --git a/hack/ccp/internal/ssclient/ssclient_test.go b/hack/ccp/internal/ssclient/ssclient_test.go index 709ea8b94..01238f7a6 100644 --- a/hack/ccp/internal/ssclient/ssclient_test.go +++ b/hack/ccp/internal/ssclient/ssclient_test.go @@ -31,11 +31,11 @@ func TestClient(t *testing.T) { "ReadPipeline reads a pipeline": { server: httpmock.New(func(s *httpmock.Server) { - s.ExpectGet("/api/v2/pipeline/8faae541-6124-471f-ade5-a6fe2099929d"). + s.ExpectGet("/api/v2/pipeline/8faae541-6124-471f-ade5-a6fe2099929d/"). ReturnHeader("Content-Type", "application/json"). ReturnJSON(map[string]any{ "uuid": "8faae541-6124-471f-ade5-a6fe2099929d", - "resource_uri": "/api/v2/pipeline/8faae541-6124-471f-ade5-a6fe2099929d", + "resource_uri": "/api/v2/pipeline/8faae541-6124-471f-ade5-a6fe2099929d/", }) }), client: func(t *testing.T, c ssclient.Client) { @@ -46,7 +46,7 @@ func TestClient(t *testing.T) { assert.NilError(t, err) assert.DeepEqual(t, ret, &ssclient.Pipeline{ ID: id, - URI: "/api/v2/pipeline/8faae541-6124-471f-ade5-a6fe2099929d", + URI: "/api/v2/pipeline/8faae541-6124-471f-ade5-a6fe2099929d/", }) }, }, @@ -72,7 +72,7 @@ func TestClient(t *testing.T) { }, server: httpmock.New(func(s *httpmock.Server) { // It looks up the pipeline details. - s.ExpectGet("/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05"). + s.ExpectGet("/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05/"). ReturnHeader("Content-Type", "application/json"). ReturnJSON(map[string]any{ "uuid": "fb2b8866-6f39-4616-b6cd-fa73193a3b05", @@ -123,14 +123,14 @@ func TestClient(t *testing.T) { }, server: httpmock.New(func(s *httpmock.Server) { // It looks up the pipeline details. - s.ExpectGet("/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05"). + s.ExpectGet("/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05/"). ReturnHeader("Content-Type", "application/json"). ReturnJSON(map[string]any{ "uuid": "fb2b8866-6f39-4616-b6cd-fa73193a3b05", "resource_uri": "/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05/", }) - s.ExpectGet("/api/v2/location?limit=100&pipeline__uuid=fb2b8866-6f39-4616-b6cd-fa73193a3b05&purpose=CP"). + s.ExpectGet("/api/v2/location/?limit=100&pipeline__uuid=fb2b8866-6f39-4616-b6cd-fa73193a3b05&purpose=CP"). ReturnHeader("Content-Type", "application/json"). Return(`{ "meta": { @@ -184,7 +184,7 @@ func TestClient(t *testing.T) { }, server: httpmock.New(func(s *httpmock.Server) { // It looks up the pipeline details. - s.ExpectGet("/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05"). + s.ExpectGet("/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05/"). ReturnHeader("Content-Type", "application/json"). ReturnJSON(map[string]any{ "uuid": "fb2b8866-6f39-4616-b6cd-fa73193a3b05", @@ -192,7 +192,7 @@ func TestClient(t *testing.T) { }) // It looks up the location list endpoint. - s.ExpectGet("/api/v2/location?limit=100&pipeline__uuid=fb2b8866-6f39-4616-b6cd-fa73193a3b05&purpose=DS"). + s.ExpectGet("/api/v2/location/?limit=100&pipeline__uuid=fb2b8866-6f39-4616-b6cd-fa73193a3b05&purpose=DS"). ReturnHeader("Content-Type", "application/json"). Return(`{ "meta": { @@ -247,7 +247,7 @@ func TestClient(t *testing.T) { }, server: httpmock.New(func(s *httpmock.Server) { // It looks up the pipeline details. - s.ExpectGet("/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05"). + s.ExpectGet("/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05/"). ReturnHeader("Content-Type", "application/json"). ReturnJSON(map[string]any{ "uuid": "fb2b8866-6f39-4616-b6cd-fa73193a3b05",