From 74b8033d8dcf7423e11fe7ae25977b20b9ce4ac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Garc=C3=ADa=20Crespo?= Date: Mon, 13 May 2024 08:54:14 +0000 Subject: [PATCH] Implement ssclient interface --- hack/ccp/go.mod | 15 +- hack/ccp/go.sum | 37 +++- hack/ccp/internal/controller/controller.go | 10 +- hack/ccp/internal/servercmd/cmd.go | 3 + hack/ccp/internal/servercmd/config.go | 2 + hack/ccp/internal/servercmd/server.go | 11 +- hack/ccp/internal/ssclient/config.go | 7 + hack/ccp/internal/ssclient/convert.go | 50 +++++ hack/ccp/internal/ssclient/ssclient.go | 157 ++++++++++++--- hack/ccp/internal/ssclient/ssclient_test.go | 205 +++++++++++++++++++- hack/docker-compose.yml | 3 + 11 files changed, 461 insertions(+), 39 deletions(-) create mode 100644 hack/ccp/internal/ssclient/config.go create mode 100644 hack/ccp/internal/ssclient/convert.go diff --git a/hack/ccp/go.mod b/hack/ccp/go.mod index cd7ec9f77..4adee8e08 100644 --- a/hack/ccp/go.mod +++ b/hack/ccp/go.mod @@ -17,14 +17,17 @@ require ( github.com/gohugoio/hugo v0.125.7 github.com/google/uuid v1.6.0 github.com/hashicorp/go-retryablehttp v0.7.6 + github.com/microsoft/kiota-abstractions-go v1.6.0 + github.com/microsoft/kiota-http-go v1.3.3 github.com/mikespook/gearman-go v0.0.0-20220520031403-2a518e866145 github.com/peterbourgon/ff/v3 v3.4.0 github.com/rs/cors v1.11.0 github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a github.com/testcontainers/testcontainers-go v0.30.0 github.com/testcontainers/testcontainers-go/modules/mysql v0.30.0 - go.artefactual.dev/ssclient v0.1.0 + go.artefactual.dev/ssclient v0.2.1 go.artefactual.dev/tools v0.10.0 + go.nhat.io/httpmock v0.11.0 go.starlark.net v0.0.0-20240411212711-9b43f0afd521 go.uber.org/goleak v1.3.0 go.uber.org/mock v0.4.0 @@ -45,6 +48,7 @@ require ( github.com/bep/godartsass v1.2.0 // indirect github.com/bep/godartsass/v2 v2.0.0 // indirect github.com/bep/golibsass v1.1.1 // indirect + github.com/bool64/shared v0.1.5 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cjlapao/common-go v0.0.39 // indirect github.com/cli/safeexec v1.0.1 // indirect @@ -66,12 +70,11 @@ require ( github.com/google/cel-go v0.20.1 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/iancoleman/orderedmap v0.2.0 // indirect github.com/klauspost/compress v1.16.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/microsoft/kiota-abstractions-go v1.6.0 // indirect - github.com/microsoft/kiota-http-go v1.3.3 // indirect github.com/microsoft/kiota-serialization-form-go v1.0.0 // indirect github.com/microsoft/kiota-serialization-json-go v1.0.7 // indirect github.com/microsoft/kiota-serialization-multipart-go v1.0.0 // indirect @@ -89,6 +92,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/sergi/go-diff v1.3.1 // indirect github.com/shirou/gopsutil/v3 v3.23.12 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.3 // indirect @@ -97,10 +101,15 @@ require ( github.com/std-uritemplate/std-uritemplate/go v0.0.55 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect github.com/stretchr/testify v1.9.0 // indirect + github.com/swaggest/assertjson v1.7.0 // indirect github.com/tdewolff/parse/v2 v2.7.13 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yudai/gojsondiff v1.0.0 // indirect + github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect + go.nhat.io/matcher/v2 v2.0.0 // indirect + go.nhat.io/wait v0.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel v1.24.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect diff --git a/hack/ccp/go.sum b/hack/ccp/go.sum index a59a9313b..bcd7d1900 100644 --- a/hack/ccp/go.sum +++ b/hack/ccp/go.sum @@ -56,6 +56,10 @@ github.com/bep/overlayfs v0.9.2 h1:qJEmFInsW12L7WW7dOTUhnMfyk/fN9OCDEO5Gr8HSDs= github.com/bep/overlayfs v0.9.2/go.mod h1:aYY9W7aXQsGcA7V9x/pzeR8LjEgIxbtisZm8Q7zPz40= github.com/bep/tmc v0.5.1 h1:CsQnSC6MsomH64gw0cT5f+EwQDcvZz4AazKunFwTpuI= github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0= +github.com/bool64/dev v0.2.17 h1:jE+T92oazAIV8fvMDJrKjsF1bzfr5XezZ8bM5GS1Cl0= +github.com/bool64/dev v0.2.17/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= +github.com/bool64/shared v0.1.5 h1:fp3eUhBsrSjNCQPcSdQqZxxh9bBwrYiZ+zOKFkM0/2E= +github.com/bool64/shared v0.1.5/go.mod h1:081yz68YC9jeFB3+Bbmno2RFWvGKv1lPKkMP6MHJlPs= github.com/bufbuild/protovalidate-go v0.6.2 h1:U/V3CGF0kPlR12v41rjO4DrYZtLcS4ZONLmWN+rJVCQ= github.com/bufbuild/protovalidate-go v0.6.2/go.mod h1:4BR3rKEJiUiTy+sqsusFn2ladOf0kYmA2Reo6BHSBgQ= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -192,6 +196,8 @@ github.com/hashicorp/go-retryablehttp v0.7.6 h1:TwRYfx2z2C4cLbXmT8I5PgP/xmuqASDy github.com/hashicorp/go-retryablehttp v0.7.6/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/iancoleman/orderedmap v0.2.0 h1:sq1N/TFpYH++aViPcaKjys3bDClUEU7s5B+z6jq8pNA= +github.com/iancoleman/orderedmap v0.2.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= github.com/jdkato/prose v1.2.1 h1:Fp3UnJmLVISmlc57BgKUzdjr0lOtjqTZicL3PaYy6cU= @@ -265,8 +271,14 @@ github.com/muesli/smartcrop v0.3.0 h1:JTlSkmxWg/oQ1TcLDoypuirdE8Y/jzNirQeLkxpA6O github.com/muesli/smartcrop v0.3.0/go.mod h1:i2fCI/UorTfgEpPPLWiFBv4pye+YAG78RwcQLUkocpI= github.com/niklasfasching/go-org v1.7.0 h1:vyMdcMWWTe/XmANk19F4k8XGBYg0GQ/gJGMimOjGMek= github.com/niklasfasching/go-org v1.7.0/go.mod h1:WuVm4d45oePiE0eX25GqTDQIt/qPW1T9DGkRscqLW5o= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.15.2 h1:l77YT15o814C2qVL47NOyjV/6RbaP7kKdrvZnxQ3Org= +github.com/onsi/ginkgo v1.15.2/go.mod h1:Dd6YFfwBW84ETqqtL0CPyPXillHgY6XhQH3uuCCTr/o= +github.com/onsi/gomega v1.11.0 h1:+CqWgvj0OZycCaqclBD1pxKHAU+tOkHmQIWvDHq2aug= +github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= @@ -296,6 +308,8 @@ github.com/rs/cors v1.11.0 h1:0B9GE/r9Bc2UxRMMtymBkHTenPkHDv0CW4Y98GBY+po= github.com/rs/cors v1.11.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= @@ -317,6 +331,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -324,6 +339,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/swaggest/assertjson v1.7.0 h1:SKw5Rn0LQs6UvmGrIdaKQbMR1R3ncXm5KNon+QJ7jtw= +github.com/swaggest/assertjson v1.7.0/go.mod h1:vxMJMehbSVJd+dDWFCKv3QRZKNTpy/ktZKTz9LOEDng= github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a h1:SJy1Pu0eH1C29XwJucQo73FrleVK6t4kYz4NVhp34Yw= github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a/go.mod h1:DFSS3NAGHthKo1gTlmEcSBiZrRJXi28rLNd/1udP1c8= github.com/tdewolff/minify/v2 v2.20.20 h1:vhULb+VsW2twkplgsawAoUY957efb+EdiZ7zu5fUhhk= @@ -340,6 +357,12 @@ github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFA github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.7.1 h1:3bajkSilaCbjdKVsKdZjZCLBNPL9pYzrCakKaf4U49U= @@ -348,10 +371,16 @@ github.com/yuin/goldmark-emoji v1.0.2 h1:c/RgTShNgHTtc6xdz2KKI74jJr6rWi7FPgnP9GA github.com/yuin/goldmark-emoji v1.0.2/go.mod h1:RhP/RWpexdp+KHs7ghKnifRoIs/Bq4nDS7tRbCkOwKY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.artefactual.dev/ssclient v0.1.0 h1:6fDwCOia9DFI8bZ2lZlCMSuoctwFY7P7XiMbJLmJB9E= -go.artefactual.dev/ssclient v0.1.0/go.mod h1:ImaAHtgGIbKlnrOUzczBMmltNVbhYkKZ7ujUjfBtUj8= +go.artefactual.dev/ssclient v0.2.1 h1:PKS7o8D7Q7XU+g1YpjWtGzD0ZknOj8ZPdh0KV9jI0cY= +go.artefactual.dev/ssclient v0.2.1/go.mod h1:ImaAHtgGIbKlnrOUzczBMmltNVbhYkKZ7ujUjfBtUj8= go.artefactual.dev/tools v0.10.0 h1:+LeZS5oHupAQBXvLQ4aGIuZyqf7zCpD7s3UpyDl9zn4= go.artefactual.dev/tools v0.10.0/go.mod h1:PIy0RtC45gC4sASb4r26g0aCU24kSWIp+mcV1p2gtpY= +go.nhat.io/httpmock v0.11.0 h1:GSADjr4/sn1HXqnyluPr9PYpSmMh/h3ty0O7lEozD3c= +go.nhat.io/httpmock v0.11.0/go.mod h1:276uIJ0K7BYfC8EW2WUK4S9PyEjiR71Ex0+43b3eNtk= +go.nhat.io/matcher/v2 v2.0.0 h1:W+rbHi0hKuZHtOQH4U5g+KwyKyfVioIxrxjoGRcUETE= +go.nhat.io/matcher/v2 v2.0.0/go.mod h1:cL5oYp0M9A4L8jEGqjmUfy+k7AXVDddoVt6aYIL1r5g= +go.nhat.io/wait v0.1.0 h1:aQ4YDzaOgFbypiJ9c/eAfOIB1G25VOv7Gd2QS8uz1gw= +go.nhat.io/wait v0.1.0/go.mod h1:+ijMghc9/9zXi+HDcs49HNReprvXOZha2Q3jTOtqJrE= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= @@ -474,9 +503,13 @@ google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGm google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/hack/ccp/internal/controller/controller.go b/hack/ccp/internal/controller/controller.go index 45eaf24d6..6cb27153e 100644 --- a/hack/ccp/internal/controller/controller.go +++ b/hack/ccp/internal/controller/controller.go @@ -13,6 +13,7 @@ import ( "golang.org/x/sync/errgroup" adminv1 "github.com/artefactual/archivematica/hack/ccp/internal/api/gen/archivematica/ccp/admin/v1beta1" + "github.com/artefactual/archivematica/hack/ccp/internal/ssclient" "github.com/artefactual/archivematica/hack/ccp/internal/store" "github.com/artefactual/archivematica/hack/ccp/internal/workflow" ) @@ -22,6 +23,10 @@ const maxConcurrentPackages = 2 type Controller struct { logger logr.Logger + // Archivematica Storage Service API client. + ssclient ssclient.Client + + // Application store. store store.Store // Embedded job server compatible with Gearman. @@ -30,8 +35,10 @@ type Controller struct { // wf is the workflow document. wf *workflow.Document + // Archivematica shared directory. sharedDir string + // Archivematica watched directory. watchedDir string // activePackages is the list of active packages. @@ -56,9 +63,10 @@ type Controller struct { closeOnce sync.Once } -func New(logger logr.Logger, store store.Store, gearman *gearmin.Server, wf *workflow.Document, sharedDir, watchedDir string) *Controller { +func New(logger logr.Logger, ssclient ssclient.Client, store store.Store, gearman *gearmin.Server, wf *workflow.Document, sharedDir, watchedDir string) *Controller { c := &Controller{ logger: logger, + ssclient: ssclient, store: store, gearman: gearman, wf: wf, diff --git a/hack/ccp/internal/servercmd/cmd.go b/hack/ccp/internal/servercmd/cmd.go index 0b95ab0af..48f76b613 100644 --- a/hack/ccp/internal/servercmd/cmd.go +++ b/hack/ccp/internal/servercmd/cmd.go @@ -33,6 +33,9 @@ func New(rootConfig *rootcmd.Config, out io.Writer) *ffcli.Command { fs.StringVar(&cfg.db.dsn, "db.dsn", "", "Database DSN") fs.StringVar(&cfg.api.admin.Addr, "api.admin.addr", "", "Admin API listen address") fs.StringVar(&cfg.gearmin.addr, "gearmin.addr", ":4730", "Gearmin job server listen address") + fs.StringVar(&cfg.ssclient.BaseURL, "ssclient.url", "", "Storage Service API base URL") + fs.StringVar(&cfg.ssclient.Username, "ssclient.username", "", "Storage Service API username") + fs.StringVar(&cfg.ssclient.Key, "ssclient.key", "", "Storage Service API key") rootConfig.RegisterFlags(fs) diff --git a/hack/ccp/internal/servercmd/config.go b/hack/ccp/internal/servercmd/config.go index bb642f19e..919d9837e 100644 --- a/hack/ccp/internal/servercmd/config.go +++ b/hack/ccp/internal/servercmd/config.go @@ -5,6 +5,7 @@ import ( "github.com/artefactual/archivematica/hack/ccp/internal/api/admin" "github.com/artefactual/archivematica/hack/ccp/internal/rootcmd" + "github.com/artefactual/archivematica/hack/ccp/internal/ssclient" ) type Config struct { @@ -15,6 +16,7 @@ type Config struct { db databaseConfig api apiConfig gearmin gearminConfig + ssclient ssclient.Config } type databaseConfig struct { diff --git a/hack/ccp/internal/servercmd/server.go b/hack/ccp/internal/servercmd/server.go index 7b55ccf80..6ed37e962 100644 --- a/hack/ccp/internal/servercmd/server.go +++ b/hack/ccp/internal/servercmd/server.go @@ -11,9 +11,11 @@ import ( "github.com/artefactual-labs/gearmin" "github.com/go-logr/logr" "github.com/gohugoio/hugo/watcher" + "github.com/hashicorp/go-retryablehttp" "github.com/artefactual/archivematica/hack/ccp/internal/api/admin" "github.com/artefactual/archivematica/hack/ccp/internal/controller" + "github.com/artefactual/archivematica/hack/ccp/internal/ssclient" "github.com/artefactual/archivematica/hack/ccp/internal/store" "github.com/artefactual/archivematica/hack/ccp/internal/workflow" ) @@ -106,8 +108,15 @@ func (s *Server) Run() error { s.gearman = gearmin.NewServer(ln) } + s.logger.V(1).Info("Creating ssclient.") + httpClient := retryablehttp.NewClient().StandardClient() + ssclient, err := ssclient.NewClient(httpClient, s.store, s.config.ssclient) + if err != nil { + return fmt.Errorf("error creating ssclient: %v", err) + } + s.logger.V(1).Info("Creating controller.") - s.controller = controller.New(s.logger.WithName("controller"), s.store, s.gearman, wf, s.config.sharedDir, watchedDir) + s.controller = controller.New(s.logger.WithName("controller"), ssclient, s.store, s.gearman, wf, s.config.sharedDir, watchedDir) if err := s.controller.Run(); err != nil { return fmt.Errorf("error creating controller: %v", err) } diff --git a/hack/ccp/internal/ssclient/config.go b/hack/ccp/internal/ssclient/config.go new file mode 100644 index 000000000..1410e33ba --- /dev/null +++ b/hack/ccp/internal/ssclient/config.go @@ -0,0 +1,7 @@ +package ssclient + +type Config struct { + BaseURL string + Username string + Key string +} diff --git a/hack/ccp/internal/ssclient/convert.go b/hack/ccp/internal/ssclient/convert.go new file mode 100644 index 000000000..d3feaa844 --- /dev/null +++ b/hack/ccp/internal/ssclient/convert.go @@ -0,0 +1,50 @@ +package ssclient + +import ( + "github.com/google/uuid" + "go.artefactual.dev/ssclient/kiota/models" + "go.artefactual.dev/tools/ref" + + "github.com/artefactual/archivematica/hack/ccp/internal/derrors" +) + +// TODO: why is kiota using ptrs for mandatory fields? + +func convertPipeline(m models.Pipelineable) (_ *Pipeline, err error) { + derrors.Add(&err, "convertPipeline") + + r := &Pipeline{} + + if uid, err := uuid.Parse(ref.DerefZero(m.GetUuid())); err != nil { + return nil, err + } else { + r.ID = uid + } + + r.URI = ref.DerefZero(m.GetResourceUri()) + + return r, nil +} + +func convertLocation(m models.Locationable) (_ *Location, err error) { + derrors.Add(&err, "convertLocation") + + r := &Location{} + + if uid, err := uuid.Parse(ref.DerefZero(m.GetUuid())); err != nil { + return nil, err + } else { + r.ID = uid + } + + r.URI = ref.DerefZero(m.GetResourceUri()) + r.Path = ref.DerefZero(m.GetPath()) + r.RelativePath = ref.DerefZero(m.GetRelativePath()) + r.Pipelines = m.GetPipeline() + + if ps := m.GetPurpose(); ps != nil { + r.Purpose = ps.String() + } + + return r, nil +} diff --git a/hack/ccp/internal/ssclient/ssclient.go b/hack/ccp/internal/ssclient/ssclient.go index 9cf685672..df3da827b 100644 --- a/hack/ccp/internal/ssclient/ssclient.go +++ b/hack/ccp/internal/ssclient/ssclient.go @@ -3,14 +3,21 @@ package ssclient import ( "context" "fmt" + "net/http" + "strings" "sync" "time" "github.com/google/uuid" - "github.com/hashicorp/go-retryablehttp" + kiotaabs "github.com/microsoft/kiota-abstractions-go" + kiotahttp "github.com/microsoft/kiota-http-go" ssclientlib "go.artefactual.dev/ssclient" "go.artefactual.dev/ssclient/kiota" + "go.artefactual.dev/ssclient/kiota/api" + "go.artefactual.dev/ssclient/kiota/models" + "go.artefactual.dev/tools/ref" + "github.com/artefactual/archivematica/hack/ccp/internal/derrors" "github.com/artefactual/archivematica/hack/ccp/internal/store" ) @@ -20,21 +27,25 @@ type Pipeline struct { } type Location struct { - ID uuid.UUID - Purpose string - Path string + ID uuid.UUID + URI string + Purpose string + Path string + RelativePath string + Pipelines []string } -// Client wraps go.artefactual.dev/ssclient-go. +// Client wraps go.artefactual.dev/ssclient-go. It provides additional +// functionality like awareness of the current pipeline identifier, the ability +// to page results and populate the default location. type Client interface { ReadPipeline(ctx context.Context, id uuid.UUID) (*Pipeline, error) - ReadLocation(ctx context.Context, purpose string) ([]*Location, error) ReadDefaultLocation(ctx context.Context, purpose string) (*Location, error) + ListLocations(ctx context.Context, path, purpose string) ([]*Location, error) CopyFiles(ctx context.Context, l *Location, files []string) error } -// clientImpl implements Client. It uses the store to read the pipeline ID and -// it caches the pipeline details to avoid hitting the server too often. +// clientImpl implements Client. type clientImpl struct { client *kiota.Client store store.Store @@ -47,9 +58,11 @@ type clientImpl struct { var _ Client = (*clientImpl)(nil) -func NewClient(store store.Store, baseURL, username, key string) (*clientImpl, error) { - stdClient := retryablehttp.NewClient().StandardClient() - k, err := ssclientlib.New(stdClient, baseURL, username, key) +func NewClient(httpClient *http.Client, store store.Store, config Config) (*clientImpl, error) { + if httpClient == nil { + httpClient = http.DefaultClient + } + k, err := ssclientlib.New(httpClient, config.BaseURL, config.Username, config.Key) if err != nil { return nil, err } @@ -59,45 +72,133 @@ func NewClient(store store.Store, baseURL, username, key string) (*clientImpl, e return c, nil } -func (c *clientImpl) ReadPipeline(ctx context.Context, id uuid.UUID) (*Pipeline, error) { +func (c *clientImpl) ReadPipeline(ctx context.Context, id uuid.UUID) (_ *Pipeline, err error) { + derrors.Add(&err, "ReadPipeline(%s)", id) + m, err := c.client.Api().V2().Pipeline().ByUuid(id.String()).Get(ctx, nil) if err != nil { return nil, err } - p := &Pipeline{ - URI: *m.GetResourceUri(), - } - - if id, err := uuid.Parse(*m.GetUuid()); err != nil { + p, err := convertPipeline(m) + if err != nil { return nil, err - } else { - p.ID = id } return p, nil } -func (c *clientImpl) ReadLocation(ctx context.Context, purpose string) ([]*Location, error) { +func (c *clientImpl) ListLocations(ctx context.Context, path, purpose string) (_ []*Location, err error) { + derrors.Add(&err, "ListLocations(%s, %s)", path, purpose) + p, err := c.pipeline(ctx) if err != nil { return nil, err } - fmt.Println(p.ID) - return nil, nil -} + reqConfig := &api.V2LocationRequestBuilderGetRequestConfiguration{ + QueryParameters: &api.V2LocationRequestBuilderGetQueryParameters{ + Pipeline__uuid: ref.New(p.ID.String()), + Limit: ref.New(int32(100)), + }, + } + + if path != "" { + reqConfig.QueryParameters.Relative_path = &path + } + + if purpose != "" { + ps, err := models.ParseLocationPurpose(purpose) + if err != nil { + return nil, err + } + if mps, ok := ps.(*models.LocationPurpose); ok { + reqConfig.QueryParameters.PurposeAsLocationPurpose = mps + } else { + return nil, fmt.Errorf("invalid purpose value: %v", ps) + } + } -func (c *clientImpl) ReadDefaultLocation(ctx context.Context, purpose string) (*Location, error) { - return nil, nil + list, err := c.client.Api().V2().Location().Get(ctx, reqConfig) + if err != nil { + return nil, err + } + + objects := list.GetObjects() + ret := make([]*Location, 0, len(objects)) + for _, obj := range objects { + l, err := convertLocation(obj) + if err != nil { + return nil, err + } + ret = append(ret, l) + } + + return ret, nil } -func (c *clientImpl) CopyFiles(ctx context.Context, l *Location, files []string) error { +func (c *clientImpl) ReadDefaultLocation(ctx context.Context, purpose string) (_ *Location, err error) { + derrors.Add(&err, "ReadDefaultLocation(%s)", purpose) + p, err := c.pipeline(ctx) + if err != nil { + return nil, err + } + + headerOptions := kiotahttp.NewHeadersInspectionOptions() + headerOptions.InspectResponseHeaders = true + + reqConfig := &api.V2LocationDefaultWithPurposeItemRequestBuilderGetRequestConfiguration{ + Options: []kiotaabs.RequestOption{headerOptions}, + } + if err := c.client.Api().V2().Location().DefaultEscaped().ByPurpose(purpose).Get(ctx, reqConfig); err != nil { + return nil, err + } + + uris := headerOptions.ResponseHeaders.Get("Location") + if len(uris) < 1 { + return nil, fmt.Errorf("location not available") + } + uri := uris[0] + if uri == "" { + return nil, fmt.Errorf("location not available") + } + + // Capture the UUID in the URI, e.g. "/api/v2/location/be68cfa8-d32a-44ba-a140-2ec5d6b903e0/". + id := strings.TrimSuffix(strings.TrimPrefix(uri, "/api/v2/location/"), "/") + + res, err := c.client.Api().V2().Location().ByUuid(id).Get(ctx, nil) + if err != nil { + return nil, err + } + + // Confirm that the default location has been made available to this pipeline. + var match bool + for _, item := range res.GetPipeline() { + if item == p.URI { + match = true + break + } + } + if !match { + return nil, fmt.Errorf("location not available") + } + + ret, err := convertLocation(res) + if err != nil { + return nil, err + } + + return ret, nil +} + +func (c *clientImpl) CopyFiles(ctx context.Context, l *Location, files []string) (err error) { + derrors.Add(&err, "CopyFiles()") + + _, err = c.pipeline(ctx) if err != nil { return err } - fmt.Println(p.URI) return nil } @@ -125,7 +226,7 @@ func (c *clientImpl) pipeline(ctx context.Context) (Pipeline, error) { c.mu.RLock() c.p = p c.ts = time.Now() - c.mu.Unlock() + c.mu.RUnlock() return *c.p, nil } diff --git a/hack/ccp/internal/ssclient/ssclient_test.go b/hack/ccp/internal/ssclient/ssclient_test.go index c97a98a94..a71deb1fd 100644 --- a/hack/ccp/internal/ssclient/ssclient_test.go +++ b/hack/ccp/internal/ssclient/ssclient_test.go @@ -1,10 +1,15 @@ package ssclient_test import ( + "context" "testing" + "github.com/google/uuid" + "go.artefactual.dev/tools/mockutil" + "go.nhat.io/httpmock" "go.uber.org/mock/gomock" "gotest.tools/v3/assert" + "gotest.tools/v3/assert/cmp" "github.com/artefactual/archivematica/hack/ccp/internal/ssclient" "github.com/artefactual/archivematica/hack/ccp/internal/store/fake" @@ -13,9 +18,201 @@ import ( func TestClient(t *testing.T) { t.Parallel() - store := fake.NewMockStore(gomock.NewController(t)) + tests := map[string]struct { + store func(rec *fake.MockStoreMockRecorder) + server httpmock.Mocker + client func(t *testing.T, c ssclient.Client) + }{ + // + // ReadPipeline + // - c, err := ssclient.NewClient(store, "bu", "u", "k") - assert.NilError(t, err) - assert.Assert(t, c != nil) + "ReadPipeline reads a pipeline": { + server: httpmock.New(func(s *httpmock.Server) { + s.ExpectGet("/api/v2/pipeline/8faae541-6124-471f-ade5-a6fe2099929d"). + ReturnHeader("Content-Type", "application/json"). + ReturnJSON(map[string]any{ + "uuid": "8faae541-6124-471f-ade5-a6fe2099929d", + "resource_uri": "/api/v2/pipeline/8faae541-6124-471f-ade5-a6fe2099929d", + }) + }), + client: func(t *testing.T, c ssclient.Client) { + id := uuid.MustParse("8faae541-6124-471f-ade5-a6fe2099929d") + + ret, err := c.ReadPipeline(context.Background(), id) + + assert.NilError(t, err) + assert.DeepEqual(t, ret, &ssclient.Pipeline{ + ID: id, + URI: "/api/v2/pipeline/8faae541-6124-471f-ade5-a6fe2099929d", + }) + }, + }, + "ReadPipeline fails if the context is canceled": { + client: func(t *testing.T, c ssclient.Client) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := c.ReadPipeline(ctx, uuid.Nil) + + assert.Assert(t, cmp.ErrorIs(err, context.Canceled)) + }, + }, + + // + // ReadDefaultLocation + // + + "ReadDefaultLocation returns the default AS location": { + store: func(rec *fake.MockStoreMockRecorder) { + // It looks up the pipeline ID in the store. + expectStoreReadPipelineID(rec) + }, + server: httpmock.New(func(s *httpmock.Server) { + // It looks up the pipeline details. + s.ExpectGet("/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05"). + ReturnHeader("Content-Type", "application/json"). + ReturnJSON(map[string]any{ + "uuid": "fb2b8866-6f39-4616-b6cd-fa73193a3b05", + "resource_uri": "/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05/", + }) + + // It looks up the default location for the given purpose. + s.ExpectGet("/api/v2/location/default/AS"). + ReturnHeader("Location", "/api/v2/location/be68cfa8-d32a-44ba-a140-2ec5d6b903e0/") + + // It looks up the location to confirm that is available to this pipeline. + s.ExpectGet("/api/v2/location/be68cfa8-d32a-44ba-a140-2ec5d6b903e0"). + ReturnHeader("Content-Type", "application/json"). + Return(`{ + "description": "Store AIP in standard Archivematica Directory", + "enabled": true, + "path": "/var/archivematica/sharedDirectory/www/AIPsStore", + "pipeline": ["/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05/"], + "purpose": "AS", + "quota": null, + "relative_path": "var/archivematica/sharedDirectory/www/AIPsStore", + "resource_uri": "/api/v2/location/be68cfa8-d32a-44ba-a140-2ec5d6b903e0/", + "space": "/api/v2/space/b4785c92-74c5-44d0-8d48-7f776fa55da7/", + "used": 0, + "uuid": "be68cfa8-d32a-44ba-a140-2ec5d6b903e0" + }`) + }), + client: func(t *testing.T, c ssclient.Client) { + ret, err := c.ReadDefaultLocation(context.Background(), "AS") + + assert.NilError(t, err) + assert.DeepEqual(t, ret, &ssclient.Location{ + ID: uuid.MustParse("be68cfa8-d32a-44ba-a140-2ec5d6b903e0"), + URI: "/api/v2/location/be68cfa8-d32a-44ba-a140-2ec5d6b903e0/", + Purpose: "AS", + Path: "/var/archivematica/sharedDirectory/www/AIPsStore", + RelativePath: "var/archivematica/sharedDirectory/www/AIPsStore", + Pipelines: []string{"/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05/"}, + }) + }, + }, + + // + // ListLocations + // + + "ListLocations returns a list of locations": { + store: func(rec *fake.MockStoreMockRecorder) { + // It looks up the pipeline ID in the store. + expectStoreReadPipelineID(rec) + }, + server: httpmock.New(func(s *httpmock.Server) { + // It looks up the pipeline details. + s.ExpectGet("/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05"). + ReturnHeader("Content-Type", "application/json"). + ReturnJSON(map[string]any{ + "uuid": "fb2b8866-6f39-4616-b6cd-fa73193a3b05", + "resource_uri": "/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05/", + }) + + // It looks up the location list endpoint. + s.ExpectGet("/api/v2/location?limit=100&pipeline__uuid=fb2b8866-6f39-4616-b6cd-fa73193a3b05&purpose=DS"). + ReturnHeader("Content-Type", "application/json"). + Return(`{ + "meta": { + "limit": 100, + "next": null, + "offset": 0, + "previous": null, + "total_count": 1 + }, + "objects": [ + { + "description": "Store DIP in standard Archivematica Directory", + "enabled": true, + "path": "/var/archivematica/sharedDirectory/www/DIPsStore", + "pipeline": ["/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05/"], + "purpose": "DS", + "quota": null, + "relative_path": "var/archivematica/sharedDirectory/www/DIPsStore", + "resource_uri": "/api/v2/location/18d6c0c4-afcd-4ee5-a9b0-19158cb199af/", + "space": "/api/v2/space/b4785c92-74c5-44d0-8d48-7f776fa55da7/", + "used": 0, + "uuid": "18d6c0c4-afcd-4ee5-a9b0-19158cb199af" + } + ] + }`) + }), + client: func(t *testing.T, c ssclient.Client) { + ret, err := c.ListLocations(context.Background(), "", "DS") + + assert.NilError(t, err) + assert.DeepEqual(t, ret, []*ssclient.Location{ + { + ID: uuid.MustParse("18d6c0c4-afcd-4ee5-a9b0-19158cb199af"), + URI: "/api/v2/location/18d6c0c4-afcd-4ee5-a9b0-19158cb199af/", + Purpose: "DS", + Path: "/var/archivematica/sharedDirectory/www/DIPsStore", + RelativePath: "var/archivematica/sharedDirectory/www/DIPsStore", + Pipelines: []string{"/api/v2/pipeline/fb2b8866-6f39-4616-b6cd-fa73193a3b05/"}, + }, + }) + }, + }, + + // + // CopyFiles + // + + "CopyFiles ...": { // TODO + server: nil, + client: func(t *testing.T, c ssclient.Client) {}, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + store := fake.NewMockStore(gomock.NewController(t)) + if tc.store != nil { + tc.store(store.EXPECT()) + } + + var srv *httpmock.Server + if tc.server == nil { + srv = httpmock.NewServer().WithTest(t) + } else { + srv = tc.server(t) + } + + config := ssclient.Config{srv.URL(), "username", "api-key"} + c, err := ssclient.NewClient(nil, store, config) + assert.NilError(t, err) + + tc.client(t, c) + }) + } +} + +func expectStoreReadPipelineID(rec *fake.MockStoreMockRecorder) { + rec. + ReadPipelineID(mockutil.Context()). + Return(uuid.MustParse("fb2b8866-6f39-4616-b6cd-fa73193a3b05"), nil). + Times(1) } diff --git a/hack/docker-compose.yml b/hack/docker-compose.yml index e4493398d..a77db79b2 100644 --- a/hack/docker-compose.yml +++ b/hack/docker-compose.yml @@ -65,6 +65,9 @@ services: - "CCP_DB_DRIVER=mysql" - "CCP_DB_DSN=root:12345@tcp(mysql:3306)/MCP" - "CCP_API_ADMIN_LISTEN=:8000" + - "CCP_SSCLIENT_URL=http://archivematica-storage-service:8000" + - "CCP_SSCLIENT_USERNAME=test" + - "CCP_SSCLIENT_KEY=test" volumes: - "archivematica_pipeline_data:/var/archivematica/sharedDirectory:rw" links: