From 88a78958acafbbddbc27bf78dc07f32687681c3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Garc=C3=ADa=20Crespo?= Date: Tue, 4 Jun 2024 11:23:51 +0000 Subject: [PATCH] Continue implementation of decision jobs --- hack/ccp/.golangci.yml | 9 + hack/ccp/buf.gen.yaml | 20 +- hack/ccp/buf.lock | 6 + hack/ccp/buf.work.yaml | 3 - hack/ccp/buf.yaml | 19 + hack/ccp/go.mod | 18 +- hack/ccp/go.sum | 34 +- hack/ccp/hack/integration.sh | 13 +- hack/ccp/hack/make/dep_buf.mk | 2 +- hack/ccp/integration/config_test.go | 23 + hack/ccp/integration/server_test.go | 67 +- hack/ccp/integration/support_test.go | 63 +- hack/ccp/internal/api/admin/admin.go | 23 +- .../ccp/admin/v1beta1/admin.pb.go | 546 +++++++++---- hack/ccp/internal/cmd/servercmd/cmd.go | 5 +- hack/ccp/internal/controller/chain.go | 40 + hack/ccp/internal/controller/controller.go | 194 +++-- hack/ccp/internal/controller/iterator.go | 253 +++--- hack/ccp/internal/controller/jobs.go | 745 +----------------- hack/ccp/internal/controller/jobs_client.go | 308 ++++++++ hack/ccp/internal/controller/jobs_decision.go | 497 ++++++++++++ .../internal/controller/jobs_decision_test.go | 62 ++ hack/ccp/internal/controller/jobs_local.go | 107 +++ .../internal/controller/jobs_local_test.go | 65 ++ hack/ccp/internal/controller/jobs_test.go | 64 ++ hack/ccp/internal/controller/package.go | 106 +-- hack/ccp/internal/controller/path.go | 4 + hack/ccp/internal/controller/task_test.go | 2 +- hack/ccp/internal/workflow/config.go | 390 ++------- hack/ccp/internal/workflow/config_builtins.go | 287 +++++++ .../internal/workflow/config_builtins_test.go | 31 + hack/ccp/internal/workflow/config_test.go | 41 +- hack/ccp/internal/workflow/workflow.go | 2 +- .../ccp/admin/v1beta1/admin.proto | 35 +- hack/ccp/proto/buf.lock | 8 - hack/ccp/proto/buf.yaml | 10 - .../ccp/admin/v1beta1/admin_pb.ts | 129 ++- 37 files changed, 2612 insertions(+), 1619 deletions(-) create mode 100644 hack/ccp/buf.lock delete mode 100644 hack/ccp/buf.work.yaml create mode 100644 hack/ccp/buf.yaml create mode 100644 hack/ccp/integration/config_test.go create mode 100644 hack/ccp/internal/controller/chain.go create mode 100644 hack/ccp/internal/controller/jobs_client.go create mode 100644 hack/ccp/internal/controller/jobs_decision.go create mode 100644 hack/ccp/internal/controller/jobs_decision_test.go create mode 100644 hack/ccp/internal/controller/jobs_local.go create mode 100644 hack/ccp/internal/controller/jobs_local_test.go create mode 100644 hack/ccp/internal/controller/jobs_test.go create mode 100644 hack/ccp/internal/workflow/config_builtins.go create mode 100644 hack/ccp/internal/workflow/config_builtins_test.go delete mode 100644 hack/ccp/proto/buf.lock delete mode 100644 hack/ccp/proto/buf.yaml diff --git a/hack/ccp/.golangci.yml b/hack/ccp/.golangci.yml index 8664dddfc..3937b2dfb 100644 --- a/hack/ccp/.golangci.yml +++ b/hack/ccp/.golangci.yml @@ -6,6 +6,7 @@ linters: - misspell - gofumpt - gci + - importas - unparam - gosec - unused @@ -39,3 +40,11 @@ linters-settings: confidence: low excludes: - G601 # does not apply to go1.22+ + importas: + no-unaliased: true + no-extra-aliases: false + alias: + - pkg: github.com/artefactual/archivematica/hack/ccp/internal/api/gen/archivematica/ccp/admin/v1beta1 + alias: adminv1 + - pkg: github.com/artefactual/archivematica/hack/ccp/internal/api/gen/archivematica/ccp/admin/v1beta1/adminv1beta1connect + alias: adminv1connect diff --git a/hack/ccp/buf.gen.yaml b/hack/ccp/buf.gen.yaml index e2e2309e5..5444a9a9c 100644 --- a/hack/ccp/buf.gen.yaml +++ b/hack/ccp/buf.gen.yaml @@ -1,20 +1,22 @@ -version: v1 +version: v2 managed: enabled: true - go_package_prefix: - default: github.com/artefactual/archivematica/hack/ccp/internal/api/gen - except: - - buf.build/bufbuild/protovalidate + disable: + - file_option: go_package + module: buf.build/bufbuild/protovalidate + override: + - file_option: go_package_prefix + value: github.com/artefactual/archivematica/hack/ccp/internal/api/gen plugins: - - plugin: buf.build/protocolbuffers/go + - remote: buf.build/protocolbuffers/go out: internal/api/gen opt: paths=source_relative - - plugin: buf.build/connectrpc/go + - remote: buf.build/connectrpc/go out: internal/api/gen opt: paths=source_relative - - plugin: buf.build/connectrpc/es + - remote: buf.build/connectrpc/es out: web/src/gen opt: target=ts - - plugin: buf.build/bufbuild/es + - remote: buf.build/bufbuild/es out: web/src/gen opt: target=ts diff --git a/hack/ccp/buf.lock b/hack/ccp/buf.lock new file mode 100644 index 000000000..c652fc7ed --- /dev/null +++ b/hack/ccp/buf.lock @@ -0,0 +1,6 @@ +# Generated by buf. DO NOT EDIT. +version: v2 +deps: + - name: buf.build/bufbuild/protovalidate + commit: 46a4cf4ba1094a34bcd89a6c67163b4b + digest: b5:2076a950fdf4a8047064d55fd1d20ef21e6d745bf56e3edf557071abd4488ed48c9466d60831d8a03489dc1fcc8ceaa073d197411b59ecd873e28b1328034e0b diff --git a/hack/ccp/buf.work.yaml b/hack/ccp/buf.work.yaml deleted file mode 100644 index 1878b341b..000000000 --- a/hack/ccp/buf.work.yaml +++ /dev/null @@ -1,3 +0,0 @@ -version: v1 -directories: - - proto diff --git a/hack/ccp/buf.yaml b/hack/ccp/buf.yaml new file mode 100644 index 000000000..f862da8f8 --- /dev/null +++ b/hack/ccp/buf.yaml @@ -0,0 +1,19 @@ +version: v2 +modules: + - path: proto + name: buf.build/artefactual/archivematica +deps: + - buf.build/bufbuild/protovalidate +lint: + use: + - DEFAULT + except: + - FIELD_NOT_REQUIRED + - PACKAGE_NO_IMPORT_CYCLE + disallow_comment_ignores: true +breaking: + use: + - FILE + except: + - EXTENSION_NO_DELETE + - FIELD_SAME_DEFAULT diff --git a/hack/ccp/go.mod b/hack/ccp/go.mod index 3e86f5833..b4b3304ce 100644 --- a/hack/ccp/go.mod +++ b/hack/ccp/go.mod @@ -1,6 +1,6 @@ module github.com/artefactual/archivematica/hack/ccp -go 1.22.3 +go 1.22.4 require ( buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.1-20240508200655-46a4cf4ba109.1 @@ -16,9 +16,10 @@ require ( github.com/fsnotify/fsnotify v1.7.0 github.com/go-logr/logr v1.4.2 github.com/go-sql-driver/mysql v1.8.1 - github.com/gohugoio/hugo v0.126.1 + github.com/gohugoio/hugo v0.127.0 + github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 - github.com/hashicorp/go-retryablehttp v0.7.6 + github.com/hashicorp/go-retryablehttp v0.7.7 github.com/mikespook/gearman-go v0.0.0-20220520031403-2a518e866145 github.com/otiai10/copy v1.14.0 github.com/peterbourgon/ff/v3 v3.4.0 @@ -33,7 +34,7 @@ require ( go.uber.org/goleak v1.3.0 go.uber.org/mock v0.4.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/net v0.25.0 + golang.org/x/net v0.26.0 golang.org/x/sync v0.7.0 google.golang.org/protobuf v1.34.1 gotest.tools/v3 v3.5.1 @@ -67,7 +68,6 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/cel-go v0.20.1 // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/iancoleman/orderedmap v0.2.0 // indirect github.com/klauspost/compress v1.16.0 // indirect @@ -117,11 +117,11 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.23.0 // indirect + golang.org/x/crypto v0.24.0 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - golang.org/x/tools v0.21.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/grpc v1.62.1 // indirect diff --git a/hack/ccp/go.sum b/hack/ccp/go.sum index 01e1a985d..727cca074 100644 --- a/hack/ccp/go.sum +++ b/hack/ccp/go.sum @@ -147,8 +147,10 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gohugoio/go-i18n/v2 v2.1.3-0.20230805085216-e63c13218d0e h1:QArsSubW7eDh8APMXkByjQWvuljwPGAGQpJEFn0F0wY= github.com/gohugoio/go-i18n/v2 v2.1.3-0.20230805085216-e63c13218d0e/go.mod h1:3Ltoo9Banwq0gOtcOwxuHG6omk+AwsQPADyw2vQYOJQ= -github.com/gohugoio/hugo v0.126.1 h1:jzs1VX6Ru/NR0luf4Z9ahKLVmYzQEox4Cxd/kyzgN9A= -github.com/gohugoio/hugo v0.126.1/go.mod h1:wo66RnKrp9Mx0WeeF22LJxPY6YB+v2weKdZpHa8fI/A= +github.com/gohugoio/httpcache v0.7.0 h1:ukPnn04Rgvx48JIinZvZetBfHaWE7I01JR2Q2RrQ3Vs= +github.com/gohugoio/httpcache v0.7.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI= +github.com/gohugoio/hugo v0.127.0 h1:7iKOa0NntxekHHkx2sc4BTsNUpTYE28jnpNM3vrvNr4= +github.com/gohugoio/hugo v0.127.0/go.mod h1:CH5I652/zeb3xwK7dOpd4GFn6ID2hOyvBXaJ2uNDRYQ= github.com/gohugoio/hugo-goldmark-extensions/extras v0.1.0 h1:YhxZNU8y2vxV6Ibr7QJzzUlpr8oHHWX/l+Q1R/a5Zao= github.com/gohugoio/hugo-goldmark-extensions/extras v0.1.0/go.mod h1:0cuvOnGKW7WeXA3i7qK6IS07FH1bgJ2XzOjQ7BMJYH4= github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.2.0 h1:PCtO5l++psZf48yen2LxQ3JiOXxaRC6v0594NeHvGZg= @@ -194,8 +196,8 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-retryablehttp v0.7.6 h1:TwRYfx2z2C4cLbXmT8I5PgP/xmuqASDyiVuGYfs9GZM= -github.com/hashicorp/go-retryablehttp v0.7.6/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/iancoleman/orderedmap v0.2.0 h1:sq1N/TFpYH++aViPcaKjys3bDClUEU7s5B+z6jq8pNA= @@ -420,8 +422,8 @@ golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= @@ -442,8 +444,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -464,14 +466,14 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -482,8 +484,8 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= -golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/hack/ccp/hack/integration.sh b/hack/ccp/hack/integration.sh index 2ff0b1af1..a047e01ef 100755 --- a/hack/ccp/hack/integration.sh +++ b/hack/ccp/hack/integration.sh @@ -1,12 +1,21 @@ #!/usr/bin/env bash +# USAGE: +# ./hack/integration.sh "TestServerCreatePackageWithUserDecision" + set -e readonly __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly __root="$(cd "$(dirname "${__dir}")" && pwd)" +runFlag="" +if [[ -n "$1" ]]; then + runFlag="-run=$1" +fi + env \ CCP_INTEGRATION_ENABLED=1 \ - CCP_INTEGRATION_ENABLE_LOGGING=no \ + CCP_INTEGRATION_ENABLE_LOGGING=yes \ CCP_INTEGRATION_ENABLE_TESTCONTAINERS_LOGGING=no \ - go test -count=1 -v ${__root}/integration/ -run=TestServerCreatePackage + CCP_INTEGRATION_ENABLE_MCPCLIENT_LOGGING=no \ + go test -count=1 -v ${__root}/integration/ $runFlag diff --git a/hack/ccp/hack/make/dep_buf.mk b/hack/ccp/hack/make/dep_buf.mk index a34854d20..9ee974d58 100644 --- a/hack/ccp/hack/make/dep_buf.mk +++ b/hack/ccp/hack/make/dep_buf.mk @@ -5,7 +5,7 @@ $(call _assert_var,UNAME_ARCH) $(call _assert_var,CACHE_VERSIONS) $(call _assert_var,CACHE_BIN) -BUF_VERSION ?= 1.31.0 +BUF_VERSION ?= 1.32.2 BUF := $(CACHE_VERSIONS)/buf/$(BUF_VERSION) $(BUF): diff --git a/hack/ccp/integration/config_test.go b/hack/ccp/integration/config_test.go new file mode 100644 index 000000000..bb899c739 --- /dev/null +++ b/hack/ccp/integration/config_test.go @@ -0,0 +1,23 @@ +package integration_test + +import "slices" + +// automatedConfigTransformations is the preferred list of transformations that +// we apply to the "automated" config in our tests. +var automatedConfigTransformations = []string{ + // Send SIP to backlog. + "bb194013-597c-4e4a-8493-b36d190f8717", "7065d256-2f47-4b7d-baec-2c4699626121", + // Virus scanning disabled. + "856d2d65-cd25-49fa-8da9-cabb78292894", "63767e4b-9ce8-4fe2-8724-65cc1f763de0", + "1dad74a2-95df-4825-bbba-dca8b91d2371", "697c0883-798d-4af7-b8b6-101c7f709cd5", + "7e81f94e-6441-4430-a12d-76df09181b66", "77355172-b437-4324-9dcc-e2607ad27cb1", + "390d6507-5029-4dae-bcd4-ce7178c9b560", "63be6081-bee8-4cf5-a453-91893e31940f", + "97a5ddc0-d4e0-43ac-a571-9722405a0a9b", "7f5244fe-590b-4e38-beaf-0cf1ccb9e71b", +} + +func configTransformations(processingConfigTransformations ...string) []string { + return slices.Concat( + automatedConfigTransformations, + processingConfigTransformations, + ) +} diff --git a/hack/ccp/integration/server_test.go b/hack/ccp/integration/server_test.go index 9dd7d51f0..fa17d0443 100644 --- a/hack/ccp/integration/server_test.go +++ b/hack/ccp/integration/server_test.go @@ -1,6 +1,7 @@ package integration_test import ( + "errors" "testing" "time" @@ -10,13 +11,17 @@ import ( "gotest.tools/v3/poll" adminv1 "github.com/artefactual/archivematica/hack/ccp/internal/api/gen/archivematica/ccp/admin/v1beta1" + "github.com/artefactual/archivematica/hack/ccp/internal/workflow" ) func TestServerCreatePackage(t *testing.T) { requireFlag(t) env := createEnv(t) - transferDir := env.createTransfer() + transferDir := env.createTransfer( + workflow.AutomatedConfig, + configTransformations()..., + ) cpResp, err := env.ccpClient.CreatePackage(env.ctx, &connect.Request[adminv1.CreatePackageRequest]{ Msg: &adminv1.CreatePackageRequest{ @@ -49,7 +54,65 @@ func TestServerCreatePackage(t *testing.T) { return poll.Continue("work is still ongoing") }, poll.WithDelay(time.Second/4), - poll.WithTimeout(time.Second*120), + poll.WithTimeout(time.Minute*2), + ) + + t.Log("Test completed successfully!") +} + +func TestServerCreatePackageWithUserDecision(t *testing.T) { + requireFlag(t) + env := createEnv(t) + + transferDir := env.createTransfer( + workflow.AutomatedConfig, + configTransformations( + // Remove "Assign UUIDs to directories" to trigger prompt. + "bd899573-694e-4d33-8c9b-df0af802437d", "", + )..., + ) + + cpResp, err := env.ccpClient.CreatePackage(env.ctx, &connect.Request[adminv1.CreatePackageRequest]{ + Msg: &adminv1.CreatePackageRequest{ + Name: "Foobar", + Path: []string{transferDir}, + AutoApprove: &wrapperspb.BoolValue{Value: true}, + }, + }) + assert.NilError(t, err) + + poll.WaitOn(t, + func(lt poll.LogT) poll.Result { + rpResp, err := env.ccpClient.ReadPackage(env.ctx, &connect.Request[adminv1.ReadPackageRequest]{ + Msg: &adminv1.ReadPackageRequest{ + Id: cpResp.Msg.Id, + }, + }) + if err != nil { + return poll.Error(err) + } + + pkg := rpResp.Msg.Pkg + if pkg.Status == adminv1.PackageStatus_PACKAGE_STATUS_AWAITING_DECISION { + switch rpResp.Msg.Decision.Name { + case "Assign UUIDs to directories?": + resolve(t, env.ctx, env.ccpClient, pkg, rpResp.Msg.Decision, "Yes") + return poll.Continue("decision resolved") + default: + return poll.Error(errors.New("unexpected decision to be resolved")) + } + } + if pkg.Status == adminv1.PackageStatus_PACKAGE_STATUS_FAILED { + return poll.Error(errors.New("package processing failed")) + } + if pkg.Status == adminv1.PackageStatus_PACKAGE_STATUS_DONE || pkg.Status == adminv1.PackageStatus_PACKAGE_STATUS_COMPLETED_SUCCESSFULLY { + return poll.Success() + } + + return poll.Continue("work is still ongoing") + }, + poll.WithDelay(time.Second/4), + poll.WithTimeout(time.Minute), ) t.Log("Test completed successfully!") diff --git a/hack/ccp/integration/support_test.go b/hack/ccp/integration/support_test.go index 6317af958..9383f1f43 100644 --- a/hack/ccp/integration/support_test.go +++ b/hack/ccp/integration/support_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "connectrpc.com/connect" "github.com/cenkalti/backoff/v4" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" @@ -29,9 +30,11 @@ import ( "gotest.tools/v3/assert" "github.com/artefactual/archivematica/hack/ccp/integration/storage" + adminv1 "github.com/artefactual/archivematica/hack/ccp/internal/api/gen/archivematica/ccp/admin/v1beta1" adminv1connect "github.com/artefactual/archivematica/hack/ccp/internal/api/gen/archivematica/ccp/admin/v1beta1/adminv1beta1connect" "github.com/artefactual/archivematica/hack/ccp/internal/cmd/rootcmd" "github.com/artefactual/archivematica/hack/ccp/internal/cmd/servercmd" + "github.com/artefactual/archivematica/hack/ccp/internal/workflow" ) const envPrefix = "CCP_INTEGRATION" @@ -40,6 +43,7 @@ var ( enableIntegration = getEnvBool("ENABLED", "no") enableLogging = getEnvBool("ENABLE_LOGGING", "yes") enableTestContainersLogging = getEnvBool("ENABLE_TESTCONTAINERS_LOGGING", "no") + enableMCPClientLogging = getEnvBool("ENABLE_MCPCLIENT_LOGGING", "no") ) func TestMain(m *testing.M) { @@ -144,6 +148,10 @@ func (e *env) runMySQL() { e.t.Log("Running MySQL server...") container, err := mysql.RunContainer(e.ctx, + mysql.WithDatabase("MCP"), + mysql.WithUsername("root"), + mysql.WithPassword("12345"), + mysql.WithScripts("data/mcp.sql.bz2"), testcontainers.WithImage("mysql:8.4.0"), testcontainers.CustomizeRequestOption(func(req *testcontainers.GenericContainerRequest) error { req.LogConsumerCfg = &testcontainers.LogConsumerConfig{ @@ -152,10 +160,6 @@ func (e *env) runMySQL() { } return nil }), - mysql.WithDatabase("MCP"), - mysql.WithUsername("root"), - mysql.WithPassword("12345"), - mysql.WithScripts("data/mcp.sql.bz2"), ) assert.NilError(e.t, err, "Failed to start container.") e.t.Cleanup(func() { @@ -241,7 +245,7 @@ func (e *env) runMCPClient() { assert.NilError(e.t, err) e.t.Cleanup(func() { - if e.t.Failed() { + if e.t.Failed() && enableMCPClientLogging { e.logContainerOutput(container) } @@ -301,6 +305,9 @@ func (e *env) runCCP() { e.t.Cleanup(func() { cancel() err := <-done + if errors.Is(err, context.Canceled) { + return + } assert.NilError(e.t, err) }) @@ -310,15 +317,36 @@ func (e *env) runCCP() { e.ccpClient = adminv1connect.NewAdminServiceClient(&http.Client{}, baseURL) } -// createTransfer creates a sample transfer in the transfer source directory. -func (e *env) createTransfer() string { +// createTransfer creates a sample transfer in the transfer source directory +// using the standard automated processing configuration. +func (e *env) createTransfer(config workflow.ProcessingConfig, processingConfigTransformations ...string) string { tmpDir, err := os.MkdirTemp(e.transferSourceDir, "transfer-*") assert.NilError(e.t, err) writeFile(e.t, filepath.Join(tmpDir, "f1.txt"), "") writeFile(e.t, filepath.Join(tmpDir, "f2.txt"), "") - err = os.Link("../hack/processingMCP.xml", filepath.Join(tmpDir, "processingMCP.xml")) + choices := config.Choices + for i := 0; i < len(processingConfigTransformations); i = i + 2 { + src, dst := processingConfigTransformations[i], processingConfigTransformations[i+1] + var remPos *int + for i, item := range choices { + if item.AppliesTo == src { + if dst == "" { + remPos = &i + } else { + item.GoToChain = dst + choices[i] = item + } + break + } + } + if remPos != nil { + slices.Delete(choices, *remPos, *remPos+1) + } + } + + err = workflow.SaveConfigFile(filepath.Join(tmpDir, "processingMCP.xml"), choices) assert.NilError(e.t, err) e.t.Logf("Created transfer: %s", tmpDir) @@ -436,3 +464,22 @@ func (c *logConsumer) Accept(l testcontainers.Log) { c.t.Logf("[%s] %s", c.container, content) } } + +// resolve a decision. +func resolve(t *testing.T, ctx context.Context, client adminv1connect.AdminServiceClient, pkg *adminv1.Package, decision *adminv1.Decision, choiceLabel string) { + var choice *adminv1.Choice + for _, c := range decision.Choice { + if c.Label == choiceLabel { + choice = c + } + } + assert.Assert(t, choice != nil, "choice %s not found", choiceLabel) + + _, err := client.ResolveAwaitingDecision(ctx, &connect.Request[adminv1.ResolveAwaitingDecisionRequest]{ + Msg: &adminv1.ResolveAwaitingDecisionRequest{ + Id: pkg.Id, + Choice: choice, + }, + }) + assert.NilError(t, err) +} diff --git a/hack/ccp/internal/api/admin/admin.go b/hack/ccp/internal/api/admin/admin.go index 01f858acd..56834aa04 100644 --- a/hack/ccp/internal/api/admin/admin.go +++ b/hack/ccp/internal/api/admin/admin.go @@ -132,14 +132,21 @@ func (s *Server) ReadPackage(ctx context.Context, req *connect.Request[adminv1.R return nil, connect.NewError(connect.CodeUnknown, nil) } - return connect.NewResponse(&adminv1.ReadPackageResponse{ + resp := &adminv1.ReadPackageResponse{ Pkg: &adminv1.Package{ Id: req.Msg.Id, Name: t.Name, Type: t.Type, Status: t.Status, }, - }), nil + } + + if decision, ok := s.ctrl.Decision(id); ok { + resp.Pkg.Status = adminv1.PackageStatus_PACKAGE_STATUS_AWAITING_DECISION + resp.Decision = decision + } + + return connect.NewResponse(resp), nil } func (s *Server) ApproveTransfer(ctx context.Context, req *connect.Request[adminv1.ApproveTransferRequest]) (*connect.Response[adminv1.ApproveTransferResponse], error) { @@ -167,6 +174,18 @@ func (s *Server) ListAwaitingDecisions(ctx context.Context, req *connect.Request } func (s *Server) ResolveAwaitingDecision(ctx context.Context, req *connect.Request[adminv1.ResolveAwaitingDecisionRequest]) (*connect.Response[adminv1.ResolveAwaitingDecisionResponse], error) { + if err := s.v.Validate(req.Msg); err != nil { + return nil, connect.NewError(connect.CodeInvalidArgument, err) + } + + id := uuid.MustParse(req.Msg.Id) + + err := s.ctrl.ResolveDecision(id, int(req.Msg.Choice.Id)) + if err != nil { + s.logger.Error(err, "Failed to resolve awaiting decision.", "id", id) + return nil, connect.NewError(connect.CodeUnknown, nil) + } + return connect.NewResponse(&adminv1.ResolveAwaitingDecisionResponse{}), nil } diff --git a/hack/ccp/internal/api/gen/archivematica/ccp/admin/v1beta1/admin.pb.go b/hack/ccp/internal/api/gen/archivematica/ccp/admin/v1beta1/admin.pb.go index b45780055..de501bcba 100644 --- a/hack/ccp/internal/api/gen/archivematica/ccp/admin/v1beta1/admin.pb.go +++ b/hack/ccp/internal/api/gen/archivematica/ccp/admin/v1beta1/admin.pb.go @@ -98,6 +98,8 @@ const ( PackageStatus_PACKAGE_STATUS_DONE PackageStatus = 2 PackageStatus_PACKAGE_STATUS_COMPLETED_SUCCESSFULLY PackageStatus = 3 PackageStatus_PACKAGE_STATUS_FAILED PackageStatus = 4 + // This is not found in the database but we can infer it at runtime. + PackageStatus_PACKAGE_STATUS_AWAITING_DECISION PackageStatus = 5 ) // Enum value maps for PackageStatus. @@ -108,6 +110,7 @@ var ( 2: "PACKAGE_STATUS_DONE", 3: "PACKAGE_STATUS_COMPLETED_SUCCESSFULLY", 4: "PACKAGE_STATUS_FAILED", + 5: "PACKAGE_STATUS_AWAITING_DECISION", } PackageStatus_value = map[string]int32{ "PACKAGE_STATUS_UNSPECIFIED": 0, @@ -115,6 +118,7 @@ var ( "PACKAGE_STATUS_DONE": 2, "PACKAGE_STATUS_COMPLETED_SUCCESSFULLY": 3, "PACKAGE_STATUS_FAILED": 4, + "PACKAGE_STATUS_AWAITING_DECISION": 5, } ) @@ -361,6 +365,9 @@ type ReadPackageResponse struct { unknownFields protoimpl.UnknownFields Pkg *Package `protobuf:"bytes,1,opt,name=pkg,proto3" json:"pkg,omitempty"` + // A decision that needs to be resolved. Must be set in the package status is + // PACKAGE_STATUS_AWAITING_DECISION. + Decision *Decision `protobuf:"bytes,2,opt,name=decision,proto3,oneof" json:"decision,omitempty"` } func (x *ReadPackageResponse) Reset() { @@ -402,6 +409,13 @@ func (x *ReadPackageResponse) GetPkg() *Package { return nil } +func (x *ReadPackageResponse) GetDecision() *Decision { + if x != nil { + return x.Decision + } + return nil +} + type ApproveTransferRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -550,6 +564,7 @@ type ListActivePackagesResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + // List of active packages, referred by their identifiers. Value []string `protobuf:"bytes,1,rep,name=value,proto3" json:"value,omitempty"` } @@ -635,6 +650,7 @@ type ListAwaitingDecisionsResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + // List of packages awaiting decisions, referred by their identifiers. Value []string `protobuf:"bytes,1,rep,name=value,proto3" json:"value,omitempty"` } @@ -681,6 +697,11 @@ type ResolveAwaitingDecisionRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + + // Identifier of the package as a string (UUIDv4). + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // The choice to be used to resolve the decision. + Choice *Choice `protobuf:"bytes,2,opt,name=choice,proto3" json:"choice,omitempty"` } func (x *ResolveAwaitingDecisionRequest) Reset() { @@ -715,6 +736,20 @@ func (*ResolveAwaitingDecisionRequest) Descriptor() ([]byte, []int) { return file_archivematica_ccp_admin_v1beta1_admin_proto_rawDescGZIP(), []int{10} } +func (x *ResolveAwaitingDecisionRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *ResolveAwaitingDecisionRequest) GetChoice() *Choice { + if x != nil { + return x.Choice + } + return nil +} + type ResolveAwaitingDecisionResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -828,6 +863,118 @@ func (x *Package) GetStatus() PackageStatus { return PackageStatus_PACKAGE_STATUS_UNSPECIFIED } +type Decision struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Name of the decision (prompt). + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // Choices available. + Choice []*Choice `protobuf:"bytes,2,rep,name=choice,proto3" json:"choice,omitempty"` +} + +func (x *Decision) Reset() { + *x = Decision{} + if protoimpl.UnsafeEnabled { + mi := &file_archivematica_ccp_admin_v1beta1_admin_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Decision) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Decision) ProtoMessage() {} + +func (x *Decision) ProtoReflect() protoreflect.Message { + mi := &file_archivematica_ccp_admin_v1beta1_admin_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Decision.ProtoReflect.Descriptor instead. +func (*Decision) Descriptor() ([]byte, []int) { + return file_archivematica_ccp_admin_v1beta1_admin_proto_rawDescGZIP(), []int{13} +} + +func (x *Decision) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Decision) GetChoice() []*Choice { + if x != nil { + return x.Choice + } + return nil +} + +type Choice struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id int32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Label string `protobuf:"bytes,2,opt,name=label,proto3" json:"label,omitempty"` +} + +func (x *Choice) Reset() { + *x = Choice{} + if protoimpl.UnsafeEnabled { + mi := &file_archivematica_ccp_admin_v1beta1_admin_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Choice) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Choice) ProtoMessage() {} + +func (x *Choice) ProtoReflect() protoreflect.Message { + mi := &file_archivematica_ccp_admin_v1beta1_admin_proto_msgTypes[14] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Choice.ProtoReflect.Descriptor instead. +func (*Choice) Descriptor() ([]byte, []int) { + return file_archivematica_ccp_admin_v1beta1_admin_proto_rawDescGZIP(), []int{14} +} + +func (x *Choice) GetId() int32 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *Choice) GetLabel() string { + if x != nil { + return x.Label + } + return "" +} + var File_archivematica_ccp_admin_v1beta1_admin_proto protoreflect.FileDescriptor var file_archivematica_ccp_admin_v1beta1_admin_proto_rawDesc = []byte{ @@ -866,158 +1013,185 @@ var file_archivematica_ccp_admin_v1beta1_admin_proto_rawDesc = []byte{ 0x72, 0x6f, 0x76, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x22, 0x27, 0x0a, 0x15, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x61, - 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x2e, 0x0a, 0x12, 0x52, 0x65, - 0x61, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x18, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xba, 0x48, - 0x05, 0x72, 0x03, 0xb0, 0x01, 0x01, 0x52, 0x02, 0x69, 0x64, 0x22, 0x51, 0x0a, 0x13, 0x52, 0x65, - 0x61, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x3a, 0x0a, 0x03, 0x70, 0x6b, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, - 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, - 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, - 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x52, 0x03, 0x70, 0x6b, 0x67, 0x22, 0x79, 0x0a, - 0x16, 0x41, 0x70, 0x70, 0x72, 0x6f, 0x76, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x41, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, - 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, - 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x54, 0x79, - 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x29, 0x0a, 0x17, 0x41, 0x70, 0x70, 0x72, - 0x6f, 0x76, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x02, 0x69, 0x64, 0x22, 0x1b, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x63, 0x74, 0x69, 0x76, - 0x65, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x22, 0x32, 0x0a, 0x1a, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, - 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x22, 0x1e, 0x0a, 0x1c, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x77, 0x61, 0x69, - 0x74, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x22, 0x35, 0x0a, 0x1d, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x77, 0x61, 0x69, - 0x74, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x20, 0x0a, 0x1e, 0x52, - 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x41, 0x77, 0x61, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x44, 0x65, - 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x21, 0x0a, - 0x1f, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x41, 0x77, 0x61, 0x69, 0x74, 0x69, 0x6e, 0x67, - 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0xcb, 0x01, 0x0a, 0x07, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x02, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xba, 0x48, 0x05, 0x72, 0x03, 0xb0, - 0x01, 0x01, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1b, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, 0xba, 0x48, 0x04, 0x72, 0x02, 0x10, 0x01, 0x52, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x12, 0x41, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x2d, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, - 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, - 0x74, 0x61, 0x31, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, - 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x46, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, + 0x67, 0x22, 0x31, 0x0a, 0x15, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x61, + 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xba, 0x48, 0x05, 0x72, 0x03, 0xb0, 0x01, 0x01, + 0x52, 0x02, 0x69, 0x64, 0x22, 0x2e, 0x0a, 0x12, 0x52, 0x65, 0x61, 0x64, 0x50, 0x61, 0x63, 0x6b, + 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xba, 0x48, 0x05, 0x72, 0x03, 0xb0, 0x01, 0x01, + 0x52, 0x02, 0x69, 0x64, 0x22, 0xaa, 0x01, 0x0a, 0x13, 0x52, 0x65, 0x61, 0x64, 0x50, 0x61, 0x63, + 0x6b, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x03, + 0x70, 0x6b, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x61, 0x72, 0x63, 0x68, + 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, + 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x50, 0x61, 0x63, 0x6b, + 0x61, 0x67, 0x65, 0x52, 0x03, 0x70, 0x6b, 0x67, 0x12, 0x4a, 0x0a, 0x08, 0x64, 0x65, 0x63, 0x69, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x61, 0x72, 0x63, + 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, + 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x44, 0x65, 0x63, + 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x08, 0x64, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, + 0x6e, 0x88, 0x01, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x64, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, + 0x6e, 0x22, 0x79, 0x0a, 0x16, 0x41, 0x70, 0x70, 0x72, 0x6f, 0x76, 0x65, 0x54, 0x72, 0x61, 0x6e, + 0x73, 0x66, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x64, + 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x41, 0x0a, 0x04, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, + 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, + 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, + 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x33, 0x0a, 0x17, + 0x41, 0x70, 0x70, 0x72, 0x6f, 0x76, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x42, 0x08, 0xba, 0x48, 0x05, 0x72, 0x03, 0xb0, 0x01, 0x01, 0x52, 0x02, 0x69, + 0x64, 0x22, 0x1b, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x50, + 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x32, + 0x0a, 0x1a, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x63, 0x6b, + 0x61, 0x67, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x22, 0x1e, 0x0a, 0x1c, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x77, 0x61, 0x69, 0x74, 0x69, + 0x6e, 0x67, 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x22, 0x35, 0x0a, 0x1d, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x77, 0x61, 0x69, 0x74, 0x69, + 0x6e, 0x67, 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x7b, 0x0a, 0x1e, 0x52, 0x65, 0x73, + 0x6f, 0x6c, 0x76, 0x65, 0x41, 0x77, 0x61, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x63, 0x69, + 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xba, 0x48, 0x05, 0x72, 0x03, 0xb0, 0x01, + 0x01, 0x52, 0x02, 0x69, 0x64, 0x12, 0x3f, 0x0a, 0x06, 0x63, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, + 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, + 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x43, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x52, 0x06, + 0x63, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x22, 0x21, 0x0a, 0x1f, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, + 0x65, 0x41, 0x77, 0x61, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xcb, 0x01, 0x0a, 0x07, 0x50, 0x61, + 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x42, 0x08, 0xba, 0x48, 0x05, 0x72, 0x03, 0xb0, 0x01, 0x01, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x1b, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, 0xba, + 0x48, 0x04, 0x72, 0x02, 0x10, 0x01, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x41, 0x0a, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x61, 0x72, 0x63, + 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, + 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, + 0x46, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x2e, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, + 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, + 0x31, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x72, 0x0a, 0x08, 0x44, 0x65, 0x63, 0x69, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x42, 0x07, 0xba, 0x48, 0x04, 0x72, 0x02, 0x10, 0x01, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x49, 0x0a, 0x06, 0x63, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x27, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, + 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, + 0x61, 0x31, 0x2e, 0x43, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x42, 0x08, 0xba, 0x48, 0x05, 0x92, 0x01, + 0x02, 0x08, 0x01, 0x52, 0x06, 0x63, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x22, 0x40, 0x0a, 0x06, 0x43, + 0x68, 0x6f, 0x69, 0x63, 0x65, 0x12, 0x17, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x05, 0x42, 0x07, 0xba, 0x48, 0x04, 0x1a, 0x02, 0x28, 0x00, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1d, + 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, 0xba, + 0x48, 0x04, 0x72, 0x02, 0x10, 0x01, 0x52, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x2a, 0x8d, 0x02, + 0x0a, 0x0c, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, + 0x0a, 0x19, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x45, 0x52, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1a, 0x0a, + 0x16, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x45, 0x52, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, + 0x54, 0x41, 0x4e, 0x44, 0x41, 0x52, 0x44, 0x10, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x54, 0x52, 0x41, + 0x4e, 0x53, 0x46, 0x45, 0x52, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x5a, 0x49, 0x50, 0x5f, 0x46, + 0x49, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x1e, 0x0a, 0x1a, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x45, + 0x52, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x5a, 0x49, 0x50, 0x50, 0x45, 0x44, 0x5f, + 0x42, 0x41, 0x47, 0x10, 0x03, 0x12, 0x1c, 0x0a, 0x18, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x45, + 0x52, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x5a, 0x49, 0x50, 0x50, 0x45, 0x44, 0x5f, 0x42, 0x41, + 0x47, 0x10, 0x04, 0x12, 0x18, 0x0a, 0x14, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x45, 0x52, 0x5f, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x53, 0x50, 0x41, 0x43, 0x45, 0x10, 0x05, 0x12, 0x19, 0x0a, + 0x15, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x45, 0x52, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4d, + 0x41, 0x49, 0x4c, 0x44, 0x49, 0x52, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x54, 0x52, 0x41, 0x4e, + 0x53, 0x46, 0x45, 0x52, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x52, 0x49, 0x4d, 0x10, 0x07, + 0x12, 0x1b, 0x0a, 0x17, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x45, 0x52, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x56, 0x45, 0x52, 0x53, 0x45, 0x10, 0x08, 0x2a, 0xd3, 0x01, + 0x0a, 0x0d, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, + 0x1e, 0x0a, 0x1a, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, + 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, + 0x1d, 0x0a, 0x19, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, + 0x53, 0x5f, 0x50, 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x17, + 0x0a, 0x13, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, + 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x02, 0x12, 0x29, 0x0a, 0x25, 0x50, 0x41, 0x43, 0x4b, 0x41, + 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, + 0x54, 0x45, 0x44, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x46, 0x55, 0x4c, 0x4c, 0x59, + 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, + 0x41, 0x54, 0x55, 0x53, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x24, 0x0a, + 0x20, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, + 0x41, 0x57, 0x41, 0x49, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x45, 0x43, 0x49, 0x53, 0x49, 0x4f, + 0x4e, 0x10, 0x05, 0x32, 0xe4, 0x06, 0x0a, 0x0c, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x80, 0x01, 0x0a, 0x0d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, + 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x35, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, - 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2a, 0x8d, - 0x02, 0x0a, 0x0c, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x1d, 0x0a, 0x19, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x45, 0x52, 0x5f, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1a, - 0x0a, 0x16, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x45, 0x52, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x53, 0x54, 0x41, 0x4e, 0x44, 0x41, 0x52, 0x44, 0x10, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x54, 0x52, - 0x41, 0x4e, 0x53, 0x46, 0x45, 0x52, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x5a, 0x49, 0x50, 0x5f, - 0x46, 0x49, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x1e, 0x0a, 0x1a, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, - 0x45, 0x52, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x5a, 0x49, 0x50, 0x50, 0x45, 0x44, - 0x5f, 0x42, 0x41, 0x47, 0x10, 0x03, 0x12, 0x1c, 0x0a, 0x18, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, - 0x45, 0x52, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x5a, 0x49, 0x50, 0x50, 0x45, 0x44, 0x5f, 0x42, - 0x41, 0x47, 0x10, 0x04, 0x12, 0x18, 0x0a, 0x14, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x45, 0x52, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x53, 0x50, 0x41, 0x43, 0x45, 0x10, 0x05, 0x12, 0x19, - 0x0a, 0x15, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x45, 0x52, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x4d, 0x41, 0x49, 0x4c, 0x44, 0x49, 0x52, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x54, 0x52, 0x41, - 0x4e, 0x53, 0x46, 0x45, 0x52, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x52, 0x49, 0x4d, 0x10, - 0x07, 0x12, 0x1b, 0x0a, 0x17, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x45, 0x52, 0x5f, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x56, 0x45, 0x52, 0x53, 0x45, 0x10, 0x08, 0x2a, 0xad, - 0x01, 0x0a, 0x0d, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x1e, 0x0a, 0x1a, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, - 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x1d, 0x0a, 0x19, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, - 0x55, 0x53, 0x5f, 0x50, 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, - 0x17, 0x0a, 0x13, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, - 0x53, 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x02, 0x12, 0x29, 0x0a, 0x25, 0x50, 0x41, 0x43, 0x4b, - 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x4c, - 0x45, 0x54, 0x45, 0x44, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x46, 0x55, 0x4c, 0x4c, - 0x59, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x5f, 0x53, - 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x32, 0xe4, - 0x06, 0x0a, 0x0c, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, - 0x80, 0x01, 0x0a, 0x0d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, - 0x65, 0x12, 0x35, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, - 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, - 0x74, 0x61, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x36, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, - 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, - 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x7a, 0x0a, 0x0b, 0x52, 0x65, 0x61, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, - 0x65, 0x12, 0x33, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, - 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, - 0x74, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x34, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, + 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, + 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x36, 0x2e, + 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, + 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, + 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7a, 0x0a, 0x0b, 0x52, 0x65, 0x61, 0x64, 0x50, + 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x33, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x50, 0x61, 0x63, - 0x6b, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x86, - 0x01, 0x0a, 0x0f, 0x41, 0x70, 0x70, 0x72, 0x6f, 0x76, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, - 0x65, 0x72, 0x12, 0x37, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, - 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, - 0x65, 0x74, 0x61, 0x31, 0x2e, 0x41, 0x70, 0x70, 0x72, 0x6f, 0x76, 0x65, 0x54, 0x72, 0x61, 0x6e, - 0x73, 0x66, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x38, 0x2e, 0x61, 0x72, + 0x6b, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x34, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, - 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x41, 0x70, - 0x70, 0x72, 0x6f, 0x76, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x8f, 0x01, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, - 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x12, 0x3a, - 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, - 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x61, - 0x67, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3b, 0x2e, 0x61, 0x72, 0x63, - 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, - 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x98, 0x01, 0x0a, 0x15, 0x4c, 0x69, - 0x73, 0x74, 0x41, 0x77, 0x61, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, - 0x6f, 0x6e, 0x73, 0x12, 0x3d, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, + 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x52, 0x65, + 0x61, 0x64, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x86, 0x01, 0x0a, 0x0f, 0x41, 0x70, 0x70, 0x72, 0x6f, 0x76, 0x65, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x12, 0x37, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, + 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, + 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x41, 0x70, 0x70, 0x72, 0x6f, 0x76, + 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x38, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, + 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, + 0x61, 0x31, 0x2e, 0x41, 0x70, 0x70, 0x72, 0x6f, 0x76, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, + 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x8f, 0x01, 0x0a, + 0x12, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x61, + 0x67, 0x65, 0x73, 0x12, 0x3a, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, - 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x77, 0x61, 0x69, 0x74, 0x69, - 0x6e, 0x67, 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x3e, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, - 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, - 0x65, 0x74, 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x77, 0x61, 0x69, 0x74, 0x69, 0x6e, - 0x67, 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x9e, 0x01, 0x0a, 0x17, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, + 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, + 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x3b, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, + 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, + 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x63, 0x6b, + 0x61, 0x67, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x98, + 0x01, 0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x77, 0x61, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x44, + 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x3d, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, + 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, + 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x41, + 0x77, 0x61, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3e, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, + 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, + 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x77, + 0x61, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x9e, 0x01, 0x0a, 0x17, 0x52, 0x65, + 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x41, 0x77, 0x61, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x63, + 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x3f, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, + 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, + 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x41, + 0x77, 0x61, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x40, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, + 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, + 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x41, 0x77, 0x61, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x3f, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0xbd, 0x02, 0x0a, 0x23, 0x63, + 0x6f, 0x6d, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, - 0x61, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x41, 0x77, 0x61, 0x69, 0x74, 0x69, - 0x6e, 0x67, 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x40, 0x2e, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, - 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, - 0x74, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x41, 0x77, 0x61, 0x69, 0x74, - 0x69, 0x6e, 0x67, 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0xbd, 0x02, 0x0a, 0x23, 0x63, 0x6f, 0x6d, 0x2e, 0x61, 0x72, - 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x63, 0x63, 0x70, 0x2e, - 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x42, 0x0a, 0x41, - 0x64, 0x6d, 0x69, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x6b, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x72, 0x74, 0x65, 0x66, 0x61, 0x63, 0x74, - 0x75, 0x61, 0x6c, 0x2f, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, - 0x61, 0x2f, 0x68, 0x61, 0x63, 0x6b, 0x2f, 0x63, 0x63, 0x70, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, 0x72, 0x63, 0x68, - 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2f, 0x63, 0x63, 0x70, 0x2f, 0x61, 0x64, - 0x6d, 0x69, 0x6e, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x3b, 0x61, 0x64, 0x6d, 0x69, - 0x6e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0xa2, 0x02, 0x03, 0x41, 0x43, 0x41, 0xaa, 0x02, - 0x1f, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2e, 0x43, - 0x63, 0x70, 0x2e, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x56, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, - 0xca, 0x02, 0x1f, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, - 0x5c, 0x43, 0x63, 0x70, 0x5c, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x5c, 0x56, 0x31, 0x62, 0x65, 0x74, - 0x61, 0x31, 0xe2, 0x02, 0x2b, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, - 0x63, 0x61, 0x5c, 0x43, 0x63, 0x70, 0x5c, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x5c, 0x56, 0x31, 0x62, - 0x65, 0x74, 0x61, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0xea, 0x02, 0x22, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, - 0x3a, 0x3a, 0x43, 0x63, 0x70, 0x3a, 0x3a, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x3a, 0x3a, 0x56, 0x31, - 0x62, 0x65, 0x74, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x31, 0x42, 0x0a, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, + 0x5a, 0x6b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x72, 0x74, + 0x65, 0x66, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x2f, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, + 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2f, 0x68, 0x61, 0x63, 0x6b, 0x2f, 0x63, 0x63, 0x70, 0x2f, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x67, 0x65, 0x6e, + 0x2f, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x2f, 0x63, + 0x63, 0x70, 0x2f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, + 0x3b, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0xa2, 0x02, 0x03, + 0x41, 0x43, 0x41, 0xaa, 0x02, 0x1f, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, 0x61, 0x74, + 0x69, 0x63, 0x61, 0x2e, 0x43, 0x63, 0x70, 0x2e, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x56, 0x31, + 0x62, 0x65, 0x74, 0x61, 0x31, 0xca, 0x02, 0x1f, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, + 0x61, 0x74, 0x69, 0x63, 0x61, 0x5c, 0x43, 0x63, 0x70, 0x5c, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x5c, + 0x56, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0xe2, 0x02, 0x2b, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, + 0x65, 0x6d, 0x61, 0x74, 0x69, 0x63, 0x61, 0x5c, 0x43, 0x63, 0x70, 0x5c, 0x41, 0x64, 0x6d, 0x69, + 0x6e, 0x5c, 0x56, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x22, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x6d, + 0x61, 0x74, 0x69, 0x63, 0x61, 0x3a, 0x3a, 0x43, 0x63, 0x70, 0x3a, 0x3a, 0x41, 0x64, 0x6d, 0x69, + 0x6e, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( @@ -1033,7 +1207,7 @@ func file_archivematica_ccp_admin_v1beta1_admin_proto_rawDescGZIP() []byte { } var file_archivematica_ccp_admin_v1beta1_admin_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_archivematica_ccp_admin_v1beta1_admin_proto_msgTypes = make([]protoimpl.MessageInfo, 13) +var file_archivematica_ccp_admin_v1beta1_admin_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_archivematica_ccp_admin_v1beta1_admin_proto_goTypes = []interface{}{ (TransferType)(0), // 0: archivematica.ccp.admin.v1beta1.TransferType (PackageStatus)(0), // 1: archivematica.ccp.admin.v1beta1.PackageStatus @@ -1050,34 +1224,39 @@ var file_archivematica_ccp_admin_v1beta1_admin_proto_goTypes = []interface{}{ (*ResolveAwaitingDecisionRequest)(nil), // 12: archivematica.ccp.admin.v1beta1.ResolveAwaitingDecisionRequest (*ResolveAwaitingDecisionResponse)(nil), // 13: archivematica.ccp.admin.v1beta1.ResolveAwaitingDecisionResponse (*Package)(nil), // 14: archivematica.ccp.admin.v1beta1.Package - (*wrapperspb.StringValue)(nil), // 15: google.protobuf.StringValue - (*wrapperspb.BoolValue)(nil), // 16: google.protobuf.BoolValue + (*Decision)(nil), // 15: archivematica.ccp.admin.v1beta1.Decision + (*Choice)(nil), // 16: archivematica.ccp.admin.v1beta1.Choice + (*wrapperspb.StringValue)(nil), // 17: google.protobuf.StringValue + (*wrapperspb.BoolValue)(nil), // 18: google.protobuf.BoolValue } var file_archivematica_ccp_admin_v1beta1_admin_proto_depIdxs = []int32{ 0, // 0: archivematica.ccp.admin.v1beta1.CreatePackageRequest.type:type_name -> archivematica.ccp.admin.v1beta1.TransferType - 15, // 1: archivematica.ccp.admin.v1beta1.CreatePackageRequest.metadata_set_id:type_name -> google.protobuf.StringValue - 16, // 2: archivematica.ccp.admin.v1beta1.CreatePackageRequest.auto_approve:type_name -> google.protobuf.BoolValue + 17, // 1: archivematica.ccp.admin.v1beta1.CreatePackageRequest.metadata_set_id:type_name -> google.protobuf.StringValue + 18, // 2: archivematica.ccp.admin.v1beta1.CreatePackageRequest.auto_approve:type_name -> google.protobuf.BoolValue 14, // 3: archivematica.ccp.admin.v1beta1.ReadPackageResponse.pkg:type_name -> archivematica.ccp.admin.v1beta1.Package - 0, // 4: archivematica.ccp.admin.v1beta1.ApproveTransferRequest.type:type_name -> archivematica.ccp.admin.v1beta1.TransferType - 0, // 5: archivematica.ccp.admin.v1beta1.Package.type:type_name -> archivematica.ccp.admin.v1beta1.TransferType - 1, // 6: archivematica.ccp.admin.v1beta1.Package.status:type_name -> archivematica.ccp.admin.v1beta1.PackageStatus - 2, // 7: archivematica.ccp.admin.v1beta1.AdminService.CreatePackage:input_type -> archivematica.ccp.admin.v1beta1.CreatePackageRequest - 4, // 8: archivematica.ccp.admin.v1beta1.AdminService.ReadPackage:input_type -> archivematica.ccp.admin.v1beta1.ReadPackageRequest - 6, // 9: archivematica.ccp.admin.v1beta1.AdminService.ApproveTransfer:input_type -> archivematica.ccp.admin.v1beta1.ApproveTransferRequest - 8, // 10: archivematica.ccp.admin.v1beta1.AdminService.ListActivePackages:input_type -> archivematica.ccp.admin.v1beta1.ListActivePackagesRequest - 10, // 11: archivematica.ccp.admin.v1beta1.AdminService.ListAwaitingDecisions:input_type -> archivematica.ccp.admin.v1beta1.ListAwaitingDecisionsRequest - 12, // 12: archivematica.ccp.admin.v1beta1.AdminService.ResolveAwaitingDecision:input_type -> archivematica.ccp.admin.v1beta1.ResolveAwaitingDecisionRequest - 3, // 13: archivematica.ccp.admin.v1beta1.AdminService.CreatePackage:output_type -> archivematica.ccp.admin.v1beta1.CreatePackageResponse - 5, // 14: archivematica.ccp.admin.v1beta1.AdminService.ReadPackage:output_type -> archivematica.ccp.admin.v1beta1.ReadPackageResponse - 7, // 15: archivematica.ccp.admin.v1beta1.AdminService.ApproveTransfer:output_type -> archivematica.ccp.admin.v1beta1.ApproveTransferResponse - 9, // 16: archivematica.ccp.admin.v1beta1.AdminService.ListActivePackages:output_type -> archivematica.ccp.admin.v1beta1.ListActivePackagesResponse - 11, // 17: archivematica.ccp.admin.v1beta1.AdminService.ListAwaitingDecisions:output_type -> archivematica.ccp.admin.v1beta1.ListAwaitingDecisionsResponse - 13, // 18: archivematica.ccp.admin.v1beta1.AdminService.ResolveAwaitingDecision:output_type -> archivematica.ccp.admin.v1beta1.ResolveAwaitingDecisionResponse - 13, // [13:19] is the sub-list for method output_type - 7, // [7:13] is the sub-list for method input_type - 7, // [7:7] is the sub-list for extension type_name - 7, // [7:7] is the sub-list for extension extendee - 0, // [0:7] is the sub-list for field type_name + 15, // 4: archivematica.ccp.admin.v1beta1.ReadPackageResponse.decision:type_name -> archivematica.ccp.admin.v1beta1.Decision + 0, // 5: archivematica.ccp.admin.v1beta1.ApproveTransferRequest.type:type_name -> archivematica.ccp.admin.v1beta1.TransferType + 16, // 6: archivematica.ccp.admin.v1beta1.ResolveAwaitingDecisionRequest.choice:type_name -> archivematica.ccp.admin.v1beta1.Choice + 0, // 7: archivematica.ccp.admin.v1beta1.Package.type:type_name -> archivematica.ccp.admin.v1beta1.TransferType + 1, // 8: archivematica.ccp.admin.v1beta1.Package.status:type_name -> archivematica.ccp.admin.v1beta1.PackageStatus + 16, // 9: archivematica.ccp.admin.v1beta1.Decision.choice:type_name -> archivematica.ccp.admin.v1beta1.Choice + 2, // 10: archivematica.ccp.admin.v1beta1.AdminService.CreatePackage:input_type -> archivematica.ccp.admin.v1beta1.CreatePackageRequest + 4, // 11: archivematica.ccp.admin.v1beta1.AdminService.ReadPackage:input_type -> archivematica.ccp.admin.v1beta1.ReadPackageRequest + 6, // 12: archivematica.ccp.admin.v1beta1.AdminService.ApproveTransfer:input_type -> archivematica.ccp.admin.v1beta1.ApproveTransferRequest + 8, // 13: archivematica.ccp.admin.v1beta1.AdminService.ListActivePackages:input_type -> archivematica.ccp.admin.v1beta1.ListActivePackagesRequest + 10, // 14: archivematica.ccp.admin.v1beta1.AdminService.ListAwaitingDecisions:input_type -> archivematica.ccp.admin.v1beta1.ListAwaitingDecisionsRequest + 12, // 15: archivematica.ccp.admin.v1beta1.AdminService.ResolveAwaitingDecision:input_type -> archivematica.ccp.admin.v1beta1.ResolveAwaitingDecisionRequest + 3, // 16: archivematica.ccp.admin.v1beta1.AdminService.CreatePackage:output_type -> archivematica.ccp.admin.v1beta1.CreatePackageResponse + 5, // 17: archivematica.ccp.admin.v1beta1.AdminService.ReadPackage:output_type -> archivematica.ccp.admin.v1beta1.ReadPackageResponse + 7, // 18: archivematica.ccp.admin.v1beta1.AdminService.ApproveTransfer:output_type -> archivematica.ccp.admin.v1beta1.ApproveTransferResponse + 9, // 19: archivematica.ccp.admin.v1beta1.AdminService.ListActivePackages:output_type -> archivematica.ccp.admin.v1beta1.ListActivePackagesResponse + 11, // 20: archivematica.ccp.admin.v1beta1.AdminService.ListAwaitingDecisions:output_type -> archivematica.ccp.admin.v1beta1.ListAwaitingDecisionsResponse + 13, // 21: archivematica.ccp.admin.v1beta1.AdminService.ResolveAwaitingDecision:output_type -> archivematica.ccp.admin.v1beta1.ResolveAwaitingDecisionResponse + 16, // [16:22] is the sub-list for method output_type + 10, // [10:16] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name } func init() { file_archivematica_ccp_admin_v1beta1_admin_proto_init() } @@ -1242,14 +1421,39 @@ func file_archivematica_ccp_admin_v1beta1_admin_proto_init() { return nil } } + file_archivematica_ccp_admin_v1beta1_admin_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Decision); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_archivematica_ccp_admin_v1beta1_admin_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Choice); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } + file_archivematica_ccp_admin_v1beta1_admin_proto_msgTypes[3].OneofWrappers = []interface{}{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_archivematica_ccp_admin_v1beta1_admin_proto_rawDesc, NumEnums: 2, - NumMessages: 13, + NumMessages: 15, NumExtensions: 0, NumServices: 1, }, diff --git a/hack/ccp/internal/cmd/servercmd/cmd.go b/hack/ccp/internal/cmd/servercmd/cmd.go index 9efe3374e..226b718a7 100644 --- a/hack/ccp/internal/cmd/servercmd/cmd.go +++ b/hack/ccp/internal/cmd/servercmd/cmd.go @@ -2,6 +2,7 @@ package servercmd import ( "context" + "errors" "flag" "io" "os" @@ -88,7 +89,9 @@ func (c *Config) Exec(ctx context.Context, args []string) error { <-ctx.Done() if err := s.Close(); err != nil { - logger.Error(err, "Failed to close server gracefully.") + if !errors.Is(err, context.Canceled) { + logger.Error(err, "Failed to close server gracefully.") + } return err } diff --git a/hack/ccp/internal/controller/chain.go b/hack/ccp/internal/controller/chain.go new file mode 100644 index 000000000..601554635 --- /dev/null +++ b/hack/ccp/internal/controller/chain.go @@ -0,0 +1,40 @@ +package controller + +import "github.com/artefactual/archivematica/hack/ccp/internal/workflow" + +// A chain is used for passing information between jobs. +// +// In Archivematica the workflow is structured around chains and links. +// A chain is a sequence of links used to accomplish a broader task or set of +// tasks, carrying local state relevant only for the duration of the chain. +// The output of a chain is placed in a watched directory to trigger the next +// chain. +// +// In MCPServer, `chain.jobChain` is implemented as an iterator, simplifying +// the process of moving through the jobs in a chain. When a chain completes, +// the queue manager checks the queues for ay work awaiting to be processed, +// which could be related to other packages. +// +// In a3m, chains and watched directories were removed, but it's hard to do it +// without introducing backward-incompatible changes given the reliance on it +// in some edge cases like reingest, etc. +type chain struct { + // The properties of the chain as described by the workflow document. + wc *workflow.Chain + + // A map of replacement variables for tasks. + // TODO: why are we not using replacementMappings instead? + context *packageContext + + // choices is a list of choices available from script output, e.g. available + // storage service locations. Choices are generated by outputClientScriptJob + // and presented as decision points using via outputDecisionJob. + choices []choice +} + +// update the context of the chain with a new map. +func (c *chain) update(kvs map[string]string) { + for k, v := range kvs { + c.context.Set(k, string(v)) + } +} diff --git a/hack/ccp/internal/controller/controller.go b/hack/ccp/internal/controller/controller.go index 293e18f3e..16cf42077 100644 --- a/hack/ccp/internal/controller/controller.go +++ b/hack/ccp/internal/controller/controller.go @@ -4,8 +4,8 @@ import ( "context" "errors" "fmt" + "io" "path/filepath" - "strings" "sync" "time" @@ -17,7 +17,6 @@ import ( adminv1 "github.com/artefactual/archivematica/hack/ccp/internal/api/gen/archivematica/ccp/admin/v1beta1" "github.com/artefactual/archivematica/hack/ccp/internal/ssclient" "github.com/artefactual/archivematica/hack/ccp/internal/store" - "github.com/artefactual/archivematica/hack/ccp/internal/store/enums" "github.com/artefactual/archivematica/hack/ccp/internal/workflow" ) @@ -50,6 +49,9 @@ type Controller struct { // queuedPackages is the list of queued packages, FIFO style. queuedPackages []*Package + // awaitingPackages is the list of packages awaiting a decision. + awaitingPackages map[uuid.UUID]*decision + // sync.RWMutex protects the internal Package slices. mu sync.RWMutex @@ -68,15 +70,16 @@ type Controller struct { func New(logger logr.Logger, ssclient ssclient.Client, store store.Store, gearman *gearmin.Server, wf *workflow.Document, sharedDir, watchedDir string) *Controller { c := &Controller{ - logger: logger, - ssclient: ssclient, - store: store, - gearman: gearman, - wf: wf, - sharedDir: sharedDir, - watchedDir: watchedDir, - activePackages: []*Package{}, - queuedPackages: []*Package{}, + logger: logger, + ssclient: ssclient, + store: store, + gearman: gearman, + wf: wf, + sharedDir: sharedDir, + watchedDir: watchedDir, + activePackages: []*Package{}, + queuedPackages: []*Package{}, + awaitingPackages: map[uuid.UUID]*decision{}, } c.groupCtx, c.groupCancel = context.WithCancel(context.Background()) @@ -179,31 +182,39 @@ func (c *Controller) pick() { return } - var current *Package + var pkg *Package if len(c.queuedPackages) > 0 { - current = c.queuedPackages[0] - c.activePackages = append(c.activePackages, current) + pkg = c.queuedPackages[0] + c.activePackages = append(c.activePackages, pkg) c.queuedPackages = c.queuedPackages[1:] } - if current == nil { + if pkg == nil { return } c.group.Go(func() error { - logger := c.logger.V(2).WithValues("package", current) - - defer c.deactivate(current) - + logger := c.logger.V(2).WithValues("package", pkg) logger.Info("Processing started.") - err := NewIterator(logger, c.gearman, c.wf, current).Process(c.groupCtx) // Block. - if err != nil { - logger.Info("Processing failed.", "err", err) - } else { - logger.Info("Processing completed successfully") - } + defer c.deactivate(pkg) - return err + iter := newJobIterator(c.groupCtx, logger, c.gearman, c.wf, pkg) + for { + err := iter.next() // Runs the next job. + + if errors.Is(err, errEnd) || errors.Is(err, io.EOF) { + return nil + } else if ew, ok := isErrWait(err); ok { + if err := c.await(iter, pkg, ew.decision); err != nil { + return err + } else { + continue + } + } else if err != nil { + logger.Error(err, "Processing failed.") + return err + } + } }) } @@ -220,26 +231,100 @@ func (c *Controller) deactivate(p *Package) { } } -type PackageStatus struct { - ID uuid.UUID - Status enums.PackageStatus +// await blocks until the awaiting package is resolved. +func (c *Controller) await(iter *jobIterator, pkg *Package, decision *decision) error { + _ = c.queueToAwait(pkg, decision) + defer c.dequeueFromAwait(pkg) + + next, err := decision.await(c.groupCtx) + c.logger.Info("Resolution of awaiting package completed.", "next", next, "err", err) + if err != nil { + return err + } + + iter.nextLink = next + + return nil } -// Package returns the status of an active package given its identifier. -func (c *Controller) Package(id uuid.UUID) *PackageStatus { +// queueToAwait moves an active package to the awaiting list. +func (c *Controller) queueToAwait(pkg *Package, decision *decision) error { + pkgID := pkg.id + + // Confirm that the package is in the active queue. + c.mu.RLock() + var index int + var found bool + for i, active := range c.activePackages { + if active.id == pkgID { + index = i + found = true + break + } + } + c.mu.RUnlock() + if !found { + return errors.New("package not found in the active list") + } + + // Move to the awaiting list. + c.mu.Lock() + c.activePackages = append(c.activePackages[:index], c.activePackages[index+1:]...) + c.awaitingPackages[pkgID] = decision + c.mu.Unlock() + + return nil +} + +func (c *Controller) dequeueFromAwait(pkg *Package) { + c.mu.Lock() + defer c.mu.Unlock() + + _, ok := c.awaitingPackages[pkg.id] + if !ok { + return + } + + delete(c.awaitingPackages, pkg.id) + c.activePackages = append(c.activePackages, pkg) +} + +func (c *Controller) IsPackageActive(id uuid.UUID) bool { c.mu.RLock() defer c.mu.RUnlock() - for _, pkg := range c.activePackages { - if id == pkg.id { - return &PackageStatus{ - ID: id, - Status: enums.PackageStatusProcessing, - } + for _, item := range c.activePackages { + if item.id == id { + return true } } - return nil + return false +} + +// Decision returns the decision of a Package given its identifier. +func (c *Controller) Decision(id uuid.UUID) (*adminv1.Decision, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + + decision, ok := c.awaitingPackages[id] + if !ok { + return nil, false + } + + ret := &adminv1.Decision{ + Name: decision.name, + Choice: make([]*adminv1.Choice, 0, len(decision.choices)), + } + for i, item := range decision.choices { + choice := &adminv1.Choice{ + Id: int32(i), + Label: item.label, + } + ret.Choice = append(ret.Choice, choice) + } + + return ret, true } // Active lists all active packages. @@ -257,22 +342,31 @@ func (c *Controller) Active() []uuid.UUID { // Decisions lists awaiting decisions for all active packages. func (c *Controller) Decisions() []string { - c.mu.Lock() - defer c.mu.Unlock() + c.mu.RLock() + defer c.mu.RUnlock() - ret := []string{} + // TODO: use c.awaitingDecisionsByPackage. - for _, item := range c.activePackages { - opts := item.Decision() - ln := len(opts) - if ln == 0 { - continue + return []string{} +} + +func (c *Controller) ResolveDecision(pkgID uuid.UUID, pos int) error { + c.mu.RLock() + defer c.mu.RUnlock() + + var match *decision + for id, decision := range c.awaitingPackages { + if id == pkgID { + match = decision + break } - ret = append(ret, fmt.Sprintf("package %s has an awaiting decision with %d options available", item, ln)) + } + if match == nil { + return errors.New("package is not awaiting") } - return ret + return match.resolve(pos) } func (c *Controller) Close() error { @@ -286,7 +380,3 @@ func (c *Controller) Close() error { return err } - -func trim(path string) string { - return strings.Trim(path, string(filepath.Separator)) -} diff --git a/hack/ccp/internal/controller/iterator.go b/hack/ccp/internal/controller/iterator.go index 4971cbe62..875dc0456 100644 --- a/hack/ccp/internal/controller/iterator.go +++ b/hack/ccp/internal/controller/iterator.go @@ -13,214 +13,137 @@ import ( "github.com/artefactual/archivematica/hack/ccp/internal/workflow" ) -var ( - errWait = errors.New("wait") - errEnd = errors.New("terminator") -) - -// A chain is used for passing information between jobs. -// -// In Archivematica the workflow is structured around chains and links. -// A chain is a sequence of links used to accomplish a broader task or set of -// tasks, carrying local state relevant only for the duration of the chain. -// The output of a chain is placed in a watched directory to trigger the next -// chain. -// -// In MCPServer, `chain.jobChain` is implemented as an iterator, simplifying -// the process of moving through the jobs in a chain. When a chain completes, -// the queue manager checks the queues for ay work awaiting to be processed, -// which could be related to other packages. -// -// In a3m, chains and watched directories were removed, but it's hard to do it -// without introducing backward-incompatible changes given the reliance on it -// in some edge cases like reingest, etc. -type chain struct { - // The properties of the chain as described by the workflow document. - wc *workflow.Chain - - // A map of replacement variables for tasks. - // TODO: why are we not using replacementMappings instead? - pCtx *packageContext - - // choices is a list of choices available from script output, e.g. available - // storage service locations. Choices are generated by outputClientScriptJob - // and presented as decision points using via outputDecisionJob. - choices map[string]outputClientScriptChoice +var errEnd = errors.New("terminator") + +type jobIterator struct { + ctx context.Context + logger logr.Logger + gearman *gearmin.Server + wf *workflow.Document + pkg *Package + nextLink uuid.UUID // Next workflow link or workflow chain link. + chain *chain // Current workflow chain } -// update the context of the chain with a new map. -func (c *chain) update(kvs map[string]string) { - for k, v := range kvs { - c.pCtx.Set(k, string(v)) +func newJobIterator(ctx context.Context, logger logr.Logger, gearman *gearmin.Server, wf *workflow.Document, pkg *Package) *jobIterator { + iter := &jobIterator{ + ctx: ctx, + logger: logger, + gearman: gearman, + wf: wf, + pkg: pkg, } -} - -// iterator carries a package through all its workflow. -type iterator struct { - logger logr.Logger - - gearman *gearmin.Server - - wf *workflow.Document - - p *Package - startAtChainID uuid.UUID - - startAtLinkID uuid.UUID + return iter +} - chain *chain +func (i *jobIterator) init() error { + i.logger.Info("Init iterator.") - waitCh chan waitSignal -} + err := i.pkg.markAsProcessing(i.ctx) + if err != nil { + return err + } -func NewIterator(logger logr.Logger, gearman *gearmin.Server, wf *workflow.Document, p *Package) *iterator { - iter := &iterator{ - logger: logger, - gearman: gearman, - wf: wf, - p: p, - startAtChainID: p.startAtChainID, - startAtLinkID: p.startAtLinkID, - waitCh: make(chan waitSignal, 10), + wc, ok := i.wf.Chains[i.pkg.startAtChainID] + if !ok { + return fmt.Errorf("can't process a job without a chain") + } else { + i.nextLink = wc.ID // Starting point. } - return iter + return nil } -func (i *iterator) Process(ctx context.Context) (err error) { - if err := i.p.markAsProcessing(ctx); err != nil { +func (i *jobIterator) next() error { + if err := i.ctx.Err(); err != nil { return err } - next := i.startAtChainID + i.logger.Info("Starting new iteration.", "nextLink", i.nextLink) - // The package indicates that we should start from a specific chain link - // within the chain. Needed by transfer submission with auto-approval - // enabled, otherwise uuid.Nil. - bypassLink := i.startAtLinkID - - for { - if err := ctx.Err(); err != nil { + // Only when we start the iterator. + if i.nextLink == uuid.Nil { + if err := i.init(); err != nil { return err } + } - // If we're starting a new chain. - if ch, ok := i.wf.Chains[next]; ok { - i.logger.Info("Starting new chain.", "id", ch.ID, "desc", ch.Description) - i.chain = &chain{wc: ch} - if pCtx, err := loadContext(ctx, i.p); err != nil { - return fmt.Errorf("load context: %v", err) - } else { - i.chain.pCtx = pCtx - } - if bypassLink != uuid.Nil { - next = bypassLink // Bypass in place. - bypassLink = uuid.Nil // Do this only once. - } else { - next = ch.LinkID - } - continue - } - - if i.chain == nil { - return fmt.Errorf("can't process a job without a chain") - } - - n, err := i.runJob(ctx, next) - - // End of chain. - if errors.Is(err, io.EOF) { - // TODO: can we have this iterator span across chains? - return nil - } - - // End of processing. - if errors.Is(err, errEnd) { - if markErr := i.p.markAsDone(ctx); err != nil { - return markErr - } - return nil - } - - // Prompt. - if errors.Is(err, errWait) { - choice, waitErr := i.wait(ctx) // puts the loop on hold. - if waitErr != nil { - return fmt.Errorf("wait: %v", waitErr) - } - next = choice - continue + if wc, ok := i.wf.Chains[i.nextLink]; ok { + i.logger.Info("Starting new chain.", "id", wc.ID, "desc", wc.Description) + i.chain = &chain{wc: wc} + if pCtx, err := loadContext(i.ctx, i.pkg); err != nil { + return fmt.Errorf("load context: %v", err) + } else { + i.chain.context = pCtx } - - if err != nil { - // TODO: mark as failed? - return fmt.Errorf("run job: %v", err) + // Special case where the next list is override with the bypass. + if wc.ID == i.pkg.startAtChainID && i.pkg.startAtLinkID != uuid.Nil { + i.nextLink = i.pkg.startAtLinkID + } else { + i.nextLink = wc.LinkID // Normal flow. } - next = n + return nil } -} - -// runJob runs a job given the identifier of a workflow chain or link. -func (i *iterator) runJob(ctx context.Context, id uuid.UUID) (uuid.UUID, error) { - wl, ok := i.wf.Links[id] - logger := i.logger.WithName("job").WithValues("type", "link", "linkID", id, "desc", wl.Description, "manager", wl.Manager) - logger.Info("Running job.") - if wl.End { - logger.Info("This job is a terminator.") + if i.chain == nil { + return fmt.Errorf("can't process a job without a chain") } + wl, ok := i.wf.Links[i.nextLink] if !ok { - return uuid.Nil, fmt.Errorf("link %s couldn't be found", id) + return fmt.Errorf("link not found in workflow document") } - s, err := i.buildJob(wl, logger) + j, err := i.buildJob(wl, i.logger.WithName("job")) if err != nil { - return uuid.Nil, fmt.Errorf("link %s couldn't be built: %v", id, err) + return fmt.Errorf("build job for link %s: %v", wl.ID, err) } - next, err := s.exec(ctx) + next, err := j.exec(i.ctx) + j.logger.Info("Job executed.", "name", j.wl.Description, "err", err) + if errors.Is(err, io.EOF) { if wl.End { - return uuid.Nil, errEnd + if err := j.pkg.markAsDone(i.ctx); err != nil { + j.logger.Error(err, "Failed to mark the package as done.") + } + return errEnd } else { - return uuid.Nil, err + // Signal end of this iterator. + // Workflow must continue using a watched directory. + // + // TODO: continue work in this iterator. + return io.EOF } - } - if err != nil { - return uuid.Nil, fmt.Errorf("link %s with manager %s (%s) couldn't be executed: %v", id, wl.Manager, wl.Description, err) + } else if _, ok := isErrWait(err); ok { + return err + } else if err != nil { + if err := j.pkg.markAsFailed(i.ctx); err != nil { + j.logger.Error(err, "Failed to mark the package as failed.") + } + return fmt.Errorf("exec job for link %s with manager %s (%s) : %v", wl.ID, wl.Manager, wl.Description, err) } - // Workflow needs to be reactivated by another watched directory. - if next == uuid.Nil { - return uuid.Nil, errWait - } + i.nextLink = next - return next, nil + return nil } // buildJob configures a workflow job given the workflow chain link definition. -func (i *iterator) buildJob(wl *workflow.Link, logger logr.Logger) (*job, error) { - j, err := newJob(logger, i.chain, i.p, i.gearman, wl, i.wf) +func (i *jobIterator) buildJob(wl *workflow.Link, logger logr.Logger) (*job, error) { + logger = logger.WithValues( + "type", "link", + "linkID", wl.ID, + "desc", wl.Description, + "manager", wl.Manager, + "terminator", wl.End, + ) + + j, err := newJob(logger, i.chain, i.pkg, i.gearman, wl, i.wf) if err != nil { return nil, fmt.Errorf("build job: %v", err) } return j, nil } - -func (i *iterator) wait(ctx context.Context) (uuid.UUID, error) { - i.logger.Info("Package is now on hold.") - - select { - case <-ctx.Done(): - return uuid.Nil, ctx.Err() - case s := <-i.waitCh: - return s.next, nil - } -} - -type waitSignal struct { - next uuid.UUID -} diff --git a/hack/ccp/internal/controller/jobs.go b/hack/ccp/internal/controller/jobs.go index b548dd6fd..10b867016 100644 --- a/hack/ccp/internal/controller/jobs.go +++ b/hack/ccp/internal/controller/jobs.go @@ -3,19 +3,14 @@ package controller import ( "context" "database/sql" - "encoding/json" - "errors" "fmt" - "io" - "maps" "time" "github.com/artefactual-labs/gearmin" "github.com/go-logr/logr" "github.com/google/uuid" - "github.com/artefactual/archivematica/hack/ccp/internal/python" - "github.com/artefactual/archivematica/hack/ccp/internal/store" + "github.com/artefactual/archivematica/hack/ccp/internal/derrors" "github.com/artefactual/archivematica/hack/ccp/internal/store/sqlcmysql" "github.com/artefactual/archivematica/hack/ccp/internal/workflow" ) @@ -48,12 +43,14 @@ type job struct { jobRunner } +// jobRunner is the interface that all jobs must implement. type jobRunner interface { exec(context.Context) (uuid.UUID, error) } func newJob(logger logr.Logger, chain *chain, pkg *Package, gearman *gearmin.Server, wl *workflow.Link, wf *workflow.Document) (*job, error) { j := &job{ + logger: logger, gearman: gearman, id: uuid.New(), createdAt: time.Now().UTC(), @@ -103,7 +100,15 @@ func newJob(logger logr.Logger, chain *chain, pkg *Package, gearman *gearmin.Ser return j, err } -func (j *job) save(ctx context.Context) error { +// save the job in the store. +func (j *job) save(ctx context.Context) (err error) { + defer derrors.Add(&err, "save") + + // Reload the package before creating the job. + if err := j.pkg.reload(ctx); err != nil { + return fmt.Errorf("reload package: %v", err) + } + return j.pkg.store.CreateJob(ctx, &sqlcmysql.CreateJobParams{ ID: j.id, Type: j.wl.Description.String(), @@ -158,725 +163,21 @@ func (j *job) updateStatusFromExitCode(ctx context.Context, code int) error { return nil } -// outputDecisionJob. -// -// A job that handles a workflow decision point, with choices based on script -// output. -// -// Manager: linkTaskManagerGetUserChoiceFromMicroserviceGeneratedList. -// Class: OutputDecisionJob(DecisionJob). -type outputDecisionJob struct { - j *job - config *workflow.LinkStandardTaskConfig -} - -var _ jobRunner = (*outputDecisionJob)(nil) - -func newOutputDecisionJob(j *job) (*outputDecisionJob, error) { - config, ok := j.wl.Config.(workflow.LinkStandardTaskConfig) - if !ok { - return nil, errors.New("invalid config") - } - - return &outputDecisionJob{ - j: j, - config: &config, - }, nil -} - -func (l *outputDecisionJob) exec(ctx context.Context) (uuid.UUID, error) { - if err := l.j.pkg.reload(ctx); err != nil { - return uuid.Nil, fmt.Errorf("reload: %v", err) - } - if err := l.j.save(ctx); err != nil { - return uuid.Nil, fmt.Errorf("save: %v", err) - } - - panic("not implemented") - - return uuid.Nil, nil // nolint: govet -} - -// nextChainDecisionJob. -// -// A type of workflow decision that determines the next chain to be executed, -// by UUID. -// -// Manager: linkTaskManagerChoice. -// Class: NextChainDecisionJob(DecisionJob). -type nextChainDecisionJob struct { - j *job - config *workflow.LinkMicroServiceChainChoice -} - -var _ jobRunner = (*nextChainDecisionJob)(nil) - -func newNextChainDecisionJob(j *job) (*nextChainDecisionJob, error) { - config, ok := j.wl.Config.(workflow.LinkMicroServiceChainChoice) - if !ok { - return nil, errors.New("invalid config") - } - - return &nextChainDecisionJob{ - j: j, - config: &config, - }, nil -} - -func (l *nextChainDecisionJob) exec(ctx context.Context) (_ uuid.UUID, err error) { - defer func() { - if err != nil { - err = fmt.Errorf("nextChainDecisionJob: %v", err) - return - } - if e := l.j.markComplete(ctx); e != nil { - err = e - } - }() - - if err := l.j.pkg.reload(ctx); err != nil { - return uuid.Nil, fmt.Errorf("reload: %v", err) - } - if err := l.j.save(ctx); err != nil { - return uuid.Nil, fmt.Errorf("save: %v", err) - } - - // Use a preconfigured choice if it validates. - chainID, err := l.j.pkg.PreconfiguredChoice(l.j.wl.ID) - if err != nil { - return uuid.Nil, err - } else if chainID != uuid.Nil { - // Fail if the choice is not available in workflow. - var matched bool - for _, cid := range l.config.Choices { - if _, ok := l.j.wf.Chains[cid]; ok { - matched = true - } - } - if !matched { - return uuid.Nil, fmt.Errorf("choice %s is not one of the available choices", chainID) - } - if err := l.j.markComplete(ctx); err != nil { - return uuid.Nil, err - } - return chainID, nil - } - - // Build decision point and await resolution. - opts := make([]option, len(l.config.Choices)) - for i, item := range l.config.Choices { - opts[i] = option(item.String()) - } - - return l.await(ctx, opts) -} - -func (l *nextChainDecisionJob) await(ctx context.Context, opts []option) (_ uuid.UUID, err error) { - defer func() { - if err != nil { - err = fmt.Errorf("await: %v", err) - return - } - }() - - if err := l.j.markAwaitingDecision(ctx); err != nil { - return uuid.Nil, err - } - - decision, err := l.j.pkg.AwaitDecision(ctx, opts) - if err != nil { - return uuid.Nil, err - } - - return decision.uuid(), nil -} - -// updateContextDecisionJob. -// -// A job that updates the job chain context based on a user choice. -// -// TODO: This type of job is mostly copied from the previous -// linkTaskManagerReplacementDicFromChoice, and it seems to have multiple ways -// of executing. It could use some cleanup. -// -// Manager: linkTaskManagerReplacementDicFromChoice. -// Class: UpdateContextDecisionJob(DecisionJob) (decisions.py). -type updateContextDecisionJob struct { - j *job - config *workflow.LinkMicroServiceChoiceReplacementDic -} - -var _ jobRunner = (*updateContextDecisionJob)(nil) - -// Maps decision point UUIDs and decision UUIDs to their "canonical" -// equivalents. This is useful for when there are multiple decision points which -// are effectively identical and a preconfigured decision for one should hold -// for all of the others as well. For example, there are 5 "Assign UUIDs to -// directories?" decision points and making a processing config decision for the -// designated canonical one, in this case -// 'bd899573-694e-4d33-8c9b-df0af802437d', should result in that decision taking -// effect for all of the others as well. This allows that. -// TODO: this should be defined in the workflow, not hardcoded here. -var updateContextDecisionJobChoiceMapping = map[uuid.UUID]uuid.UUID{ - // Decision point "Assign UUIDs to directories?". - uuid.MustParse("8882bad4-561c-4126-89c9-f7f0c083d5d7"): uuid.MustParse("bd899573-694e-4d33-8c9b-df0af802437d"), - uuid.MustParse("e10a31c3-56df-4986-af7e-2794ddfe8686"): uuid.MustParse("bd899573-694e-4d33-8c9b-df0af802437d"), - uuid.MustParse("d6f6f5db-4cc2-4652-9283-9ec6a6d181e5"): uuid.MustParse("bd899573-694e-4d33-8c9b-df0af802437d"), - uuid.MustParse("1563f22f-f5f7-4dfe-a926-6ab50d408832"): uuid.MustParse("bd899573-694e-4d33-8c9b-df0af802437d"), - // Decision "Yes" (for "Assign UUIDs to directories?"). - uuid.MustParse("7e4cf404-e62d-4dc2-8d81-6141e390f66f"): uuid.MustParse("2dc3f487-e4b0-4e07-a4b3-6216ed24ca14"), - uuid.MustParse("2732a043-b197-4cbc-81ab-4e2bee9b74d3"): uuid.MustParse("2dc3f487-e4b0-4e07-a4b3-6216ed24ca14"), - uuid.MustParse("aa793efa-1b62-498c-8f92-cab187a99a2a"): uuid.MustParse("2dc3f487-e4b0-4e07-a4b3-6216ed24ca14"), - uuid.MustParse("efd98ddb-80a6-4206-80bf-81bf00f84416"): uuid.MustParse("2dc3f487-e4b0-4e07-a4b3-6216ed24ca14"), - // Decision "No" (for "Assign UUIDs to directories?"). - uuid.MustParse("0053c670-3e61-4a3e-a188-3a2dd1eda426"): uuid.MustParse("891f60d0-1ba8-48d3-b39e-dd0934635d29"), - uuid.MustParse("8e93e523-86bb-47e1-a03a-4b33e13f8c5e"): uuid.MustParse("891f60d0-1ba8-48d3-b39e-dd0934635d29"), - uuid.MustParse("6dfbeff8-c6b1-435b-833a-ed764229d413"): uuid.MustParse("891f60d0-1ba8-48d3-b39e-dd0934635d29"), - uuid.MustParse("dc0ee6b6-ed5f-42a3-bc8f-c9c7ead03ed1"): uuid.MustParse("891f60d0-1ba8-48d3-b39e-dd0934635d29"), -} - -func newUpdateContextDecisionJob(j *job) (*updateContextDecisionJob, error) { - config, ok := j.wl.Config.(workflow.LinkMicroServiceChoiceReplacementDic) - if !ok { - return nil, errors.New("invalid config") - } - - return &updateContextDecisionJob{ - j: j, - config: &config, - }, nil -} - -func (l *updateContextDecisionJob) exec(ctx context.Context) (linkID uuid.UUID, err error) { - defer func() { - if err != nil { - err = fmt.Errorf("nextChainDecisionJob: %v", err) - return - } - if e := l.j.markComplete(ctx); e != nil { - err = e - return - } - if id := l.j.wl.ExitCodes[0].LinkID; id == nil || *id == uuid.Nil { - err = fmt.Errorf("nextChainDecisionJob: linkID undefined") - } else { - linkID = *id - } - }() - - if err := l.j.pkg.reload(ctx); err != nil { - return uuid.Nil, fmt.Errorf("reload: %v", err) - } - if err := l.j.save(ctx); err != nil { - return uuid.Nil, fmt.Errorf("save: %v", err) - } - - // Load new context from the database (DashboardSettings). - // TODO: split this out? Workflow items with no replacements configured - // seems like a different case. - if len(l.config.Replacements) == 0 { - if dict, err := l.loadDatabaseContext(ctx); err != nil { - return uuid.Nil, fmt.Errorf("load dict from db: %v", err) - } else if dict != nil { - l.j.chain.update(dict) - return uuid.Nil, nil - } - } - - // Load new context from processing configuration. - if dict, err := l.loadPreconfiguredContext(); err != nil { - return uuid.Nil, fmt.Errorf("load context with preconfigured choice: %v", err) - } else if dict != nil { - l.j.chain.update(dict) - return uuid.Nil, nil - } - - // Build decision point and await resolution. - opts := make([]option, len(l.config.Replacements)) - for i, item := range l.config.Replacements { - opts[i] = option(item.Description.String()) - } - - return l.await(ctx, opts) -} - -// loadDatabaseContext loads the context dictionary from the database. -func (l *updateContextDecisionJob) loadDatabaseContext(ctx context.Context) (map[string]string, error) { - ln, ok := l.j.wf.Links[l.j.wl.FallbackLinkID] - if !ok { - return nil, nil - } - cfg, ok := ln.Config.(workflow.LinkStandardTaskConfig) - if !ok { - return nil, nil - } - if cfg.Execute == "" { - return nil, nil - } - - ret, err := l.j.pkg.store.ReadDict(ctx, cfg.Execute) - if err != nil { - return nil, err - } - - return l.formatChoices(ret), nil -} - -// loadPreconfiguredContext loads the context dictionary from the workflow. -func (l *updateContextDecisionJob) loadPreconfiguredContext() (map[string]string, error) { - var normalizedChoice uuid.UUID - if v, ok := updateContextDecisionJobChoiceMapping[l.j.wl.ID]; ok { - normalizedChoice = v - } else { - normalizedChoice = l.j.wl.ID - } - - choices, err := l.j.pkg.parseProcessingConfig() - if err != nil { - return nil, err - } - - for _, choice := range choices { - if choice.AppliesTo != normalizedChoice.String() { - continue - } - desiredChoice, err := uuid.Parse(choice.GoToChain) - if err != nil { - return nil, err - } - if v, ok := updateContextDecisionJobChoiceMapping[desiredChoice]; ok { - desiredChoice = v - } - ln, ok := l.j.wf.Links[normalizedChoice] - if !ok { - return nil, nil // fmt.Errorf("desired choice not found: %s", desiredChoice) - } - config, ok := ln.Config.(workflow.LinkMicroServiceChoiceReplacementDic) - if !ok { - return nil, fmt.Errorf("desired choice doesn't have the expected type: %s", desiredChoice) - } - for _, replacement := range config.Replacements { - if replacement.ID == desiredChoice.String() { - choices := maps.Clone(replacement.Items) - return l.formatChoices(choices), nil - } - } - } - - return nil, nil -} - -func (l *updateContextDecisionJob) formatChoices(choices map[string]string) map[string]string { - for k, v := range choices { - delete(choices, k) - choices[fmt.Sprintf("%%%s%%", k)] = v - } - - return choices -} - -func (l *updateContextDecisionJob) await(ctx context.Context, opts []option) (_ uuid.UUID, err error) { - defer func() { - if err != nil { - err = fmt.Errorf("await: %v", err) - return - } - }() - - if err := l.j.markAwaitingDecision(ctx); err != nil { - return uuid.Nil, err - } - - decision, err := l.j.pkg.AwaitDecision(ctx, opts) // nolint: staticcheck - if err != nil { - return uuid.Nil, err - } - - // TODO: decision here should be an integer. - // https://github.com/artefactual/archivematica/blob/2dd5a2366bf0529c193a19a5546087ed9a0b5534/src/MCPServer/lib/server/jobs/decisions.py#L286-L298 - - panic("not implemented") - - return decision.uuid(), nil // nolint: govet -} - -// directoryClientScriptJob. -// -// Manager: linkTaskManagerDirectories. -// Class: DirectoryClientScriptJob(DecisionJob). -type directoryClientScriptJob struct { - j *job - config *workflow.LinkStandardTaskConfig -} - -var _ jobRunner = (*directoryClientScriptJob)(nil) - -func newDirectoryClientScriptJob(j *job) (*directoryClientScriptJob, error) { - config, ok := j.wl.Config.(workflow.LinkStandardTaskConfig) - if !ok { - return nil, errors.New("invalid config") - } - - return &directoryClientScriptJob{ - j: j, - config: &config, - }, nil -} - -func (l *directoryClientScriptJob) exec(ctx context.Context) (uuid.UUID, error) { - if err := l.j.pkg.reload(ctx); err != nil { - return uuid.Nil, fmt.Errorf("reload: %v", err) - } - if err := l.j.save(ctx); err != nil { - return uuid.Nil, fmt.Errorf("save: %v", err) - } - - taskResult, err := l.submitTasks(ctx) - if err != nil { - return uuid.Nil, fmt.Errorf("submit task: %v", err) - } - - if err := l.j.updateStatusFromExitCode(ctx, taskResult.ExitCode); err != nil { - return uuid.Nil, err - } - - if ec, ok := l.j.wl.ExitCodes[taskResult.ExitCode]; ok { - if ec.LinkID == nil { - return uuid.Nil, io.EOF // End of chain. - } - return *ec.LinkID, nil - } - - if l.j.wl.FallbackLinkID == uuid.Nil { - return uuid.Nil, io.EOF // End of chain. - } - - return l.j.wl.FallbackLinkID, nil -} - -func (l *directoryClientScriptJob) submitTasks(ctx context.Context) (*taskResult, error) { - rm := l.j.pkg.unit.replacements(l.config.FilterSubdir).update(l.j.chain.pCtx) - args := rm.replaceValues(l.config.Arguments) - stdout := rm.replaceValues(l.config.StdoutFile) - stderr := rm.replaceValues(l.config.StderrFile) - - taskBackend := newTaskBackend(l.j.logger, l.j, l.j.pkg.store, l.j.gearman, l.config) - if err := taskBackend.submit(ctx, rm, args, false, stdout, stderr); err != nil { - return nil, err - } - - results, err := taskBackend.wait(ctx) - if err != nil { - return nil, fmt.Errorf("wait: %v", err) - } - - ret := results.First() - if ret == nil { - return nil, errors.New("submit task: no results") - } - - return ret, nil -} - -// filesClientScriptJob. -// -// Manager: linkTaskManagerFiles. -// Class: FilesClientScriptJob(DecisionJob). -type filesClientScriptJob struct { - j *job - config *workflow.LinkStandardTaskConfig +type ConfigT interface { + workflow.LinkStandardTaskConfig | + workflow.LinkTaskConfigSetUnitVariable | + workflow.LinkTaskConfigUnitVariableLinkPull | + workflow.LinkMicroServiceChainChoice | + workflow.LinkMicroServiceChoiceReplacementDic } -var _ jobRunner = (*filesClientScriptJob)(nil) - -func newFilesClientScriptJob(j *job) (*filesClientScriptJob, error) { - config, ok := j.wl.Config.(workflow.LinkStandardTaskConfig) +func loadConfig[T ConfigT](wl *workflow.Link, dest *T) error { + config, ok := wl.Config.(T) if !ok { - return nil, errors.New("invalid config") - } - - return &filesClientScriptJob{ - j: j, - config: &config, - }, nil -} - -func (l *filesClientScriptJob) exec(ctx context.Context) (uuid.UUID, error) { - if err := l.j.pkg.reload(ctx); err != nil { - return uuid.Nil, fmt.Errorf("reload: %v", err) + return fmt.Errorf("config provided is not compatible with its type") } - if err := l.j.save(ctx); err != nil { - return uuid.Nil, fmt.Errorf("save: %v", err) - } - - filterSubDir, err := l.filterSubDir(ctx) - if err != nil { - return uuid.Nil, fmt.Errorf("look up filterSubDir: %v", err) - } - - taskResults, err := l.submitTasks(ctx, filterSubDir) - if err != nil { - return uuid.Nil, fmt.Errorf("submit task: %v", err) - } - exitCode := 0 - if taskResults != nil { - exitCode = taskResults.ExitCode() - } - - if err := l.j.updateStatusFromExitCode(ctx, exitCode); err != nil { - return uuid.Nil, err - } - - if ec, ok := l.j.wl.ExitCodes[exitCode]; ok { - if ec.LinkID == nil { - return uuid.Nil, io.EOF // End of chain. - } - return *ec.LinkID, nil - } - - if l.j.wl.FallbackLinkID == uuid.Nil { - return uuid.Nil, io.EOF // End of chain. - } - - return l.j.wl.FallbackLinkID, nil -} - -func (l *filesClientScriptJob) submitTasks(ctx context.Context, filterSubDir string) (*taskResults, error) { - rm := l.j.pkg.unit.replacements(filterSubDir).update(l.j.chain.pCtx) - taskBackend := newTaskBackend(l.j.logger, l.j, l.j.pkg.store, l.j.gearman, l.config) - files, err := l.j.pkg.Files(ctx, l.config.FilterFileEnd, filterSubDir) - if err != nil { - return nil, err - } - if len(files) == 0 { - return nil, nil // Nothing to do. - } - - for _, fileReplacements := range files { - rm = rm.with(fileReplacements) - args := rm.replaceValues(l.config.Arguments) - stdout := rm.replaceValues(l.config.StdoutFile) - stderr := rm.replaceValues(l.config.StderrFile) - - if err := taskBackend.submit(ctx, rm, args, false, stdout, stderr); err != nil { - return nil, err - } - } - - res, err := taskBackend.wait(ctx) - if err != nil { - return nil, fmt.Errorf("wait: %v", err) - } - - return res, nil -} - -// filterSubDir returns the directory to filter files on. This path is usually -// defined in the workflow but can be overridden per package in a UnitVariable, -// so we need to look that up. -func (l *filesClientScriptJob) filterSubDir(ctx context.Context) (string, error) { - filterSubDir := l.config.FilterSubdir - - // Check if filterSubDir has been overridden for this Transfer/SIP. - val, err := l.j.pkg.store.ReadUnitVar(ctx, l.j.pkg.id, l.j.pkg.packageType(), l.config.Execute) - if err != nil { - if errors.Is(err, store.ErrNotFound) { - return filterSubDir, nil - } - return "", err - } - - if val == "" { - return filterSubDir, nil - } - if m, err := python.EvalMap(val); err != nil { - if override, ok := m["filterSubDir"]; ok { - filterSubDir = override - } - } - - return filterSubDir, nil -} - -// outputClientScriptJob. -// -// Manager: linkTaskManagerGetMicroserviceGeneratedListInStdOut. -// Class: OutputClientScriptJob(DecisionJob). -type outputClientScriptJob struct { - j *job - config *workflow.LinkStandardTaskConfig -} - -var _ jobRunner = (*outputClientScriptJob)(nil) - -func newOutputClientScriptJob(j *job) (*outputClientScriptJob, error) { - config, ok := j.wl.Config.(workflow.LinkStandardTaskConfig) - if !ok { - return nil, errors.New("invalid config") - } - - return &outputClientScriptJob{ - j: j, - config: &config, - }, nil -} - -// The list of choices are represented using a dictionary as follows: -// -// { -// "default": {"description": "asdf", "uri": "asdf"}, -// "5c732a52-6cdb-4b50-ac2e-ae10361b019a": {"description": "asdf", "uri": "asdf"}, -// } -type outputClientScriptChoice struct { - Description string `json:"description"` - URI string `json:"uri"` -} + *dest = config -func (l *outputClientScriptJob) exec(ctx context.Context) (uuid.UUID, error) { - if err := l.j.pkg.reload(ctx); err != nil { - return uuid.Nil, fmt.Errorf("reload: %v", err) - } - if err := l.j.save(ctx); err != nil { - return uuid.Nil, fmt.Errorf("save: %v", err) - } - - taskResult, err := l.submitTasks(ctx) - if err != nil { - return uuid.Nil, fmt.Errorf("submit task: %v", err) - } - - choices := map[string]outputClientScriptChoice{} - if err := json.Unmarshal([]byte(taskResult.Stdout), &choices); err != nil { - l.j.logger.Error(err, "Unable to parse output: %s", taskResult.Stdout) - } else { - l.j.chain.choices = choices - } - - if err := l.j.updateStatusFromExitCode(ctx, taskResult.ExitCode); err != nil { - return uuid.Nil, err - } - - if ec, ok := l.j.wl.ExitCodes[taskResult.ExitCode]; ok { - if ec.LinkID == nil { - return uuid.Nil, io.EOF // End of chain. - } - return *ec.LinkID, nil - } - - if l.j.wl.FallbackLinkID == uuid.Nil { - return uuid.Nil, io.EOF // End of chain. - } - - return uuid.Nil, nil -} - -func (l *outputClientScriptJob) submitTasks(ctx context.Context) (*taskResult, error) { - rm := l.j.pkg.unit.replacements(l.config.FilterSubdir).update(l.j.chain.pCtx) - args := rm.replaceValues(l.config.Arguments) - stdout := rm.replaceValues(l.config.StdoutFile) - stderr := rm.replaceValues(l.config.StderrFile) - - taskBackend := newTaskBackend(l.j.logger, l.j, l.j.pkg.store, l.j.gearman, l.config) - if err := taskBackend.submit(ctx, rm, args, true, stdout, stderr); err != nil { - return nil, err - } - - results, err := taskBackend.wait(ctx) - if err != nil { - return nil, fmt.Errorf("wait: %v", err) - } - - ret := results.First() - if ret == nil { - return nil, errors.New("submit task: no results") - } - - return ret, nil -} - -// setUnitVarLinkJob is a local job that sets the unit variable configured in -// the workflow. -// -// Manager: linkTaskManagerSetUnitVariable. -// Class: SetUnitVarLinkJob(DecisionJob) (decisions.py). -type setUnitVarLinkJob struct { - j *job - config *workflow.LinkTaskConfigSetUnitVariable -} - -var _ jobRunner = (*setUnitVarLinkJob)(nil) - -func newSetUnitVarLinkJob(j *job) (*setUnitVarLinkJob, error) { - config, ok := j.wl.Config.(workflow.LinkTaskConfigSetUnitVariable) - if !ok { - return nil, errors.New("invalid config") - } - - return &setUnitVarLinkJob{ - j: j, - config: &config, - }, nil -} - -func (l *setUnitVarLinkJob) exec(ctx context.Context) (uuid.UUID, error) { - if err := l.j.pkg.saveLinkID(ctx, l.config.Variable, l.config.LinkID); err != nil { - return uuid.Nil, err - } - - if err := l.j.markComplete(ctx); err != nil { - return uuid.Nil, err - } - - return l.config.LinkID, nil -} - -// getUnitVarLinkJob is a local job that gets the next link in the chain from a -// UnitVariable. -// -// Manager: linkTaskManagerUnitVariableLinkPull. -// Class: GetUnitVarLinkJob(DecisionJob) (decisions.py). -type getUnitVarLinkJob struct { - j *job - config *workflow.LinkTaskConfigUnitVariableLinkPull -} - -var _ jobRunner = (*getUnitVarLinkJob)(nil) - -func newGetUnitVarLinkJob(j *job) (*getUnitVarLinkJob, error) { - config, ok := j.wl.Config.(workflow.LinkTaskConfigUnitVariableLinkPull) - if !ok { - return nil, errors.New("invalid config") - } - - return &getUnitVarLinkJob{ - j: j, - config: &config, - }, nil -} - -func (l *getUnitVarLinkJob) exec(ctx context.Context) (uuid.UUID, error) { - if err := l.j.pkg.reload(ctx); err != nil { - return uuid.Nil, fmt.Errorf("reload: %v", err) - } - if err := l.j.save(ctx); err != nil { - return uuid.Nil, fmt.Errorf("save: %v", err) - } - - linkID, err := l.j.pkg.store.ReadUnitLinkID(ctx, l.j.pkg.id, l.j.pkg.packageType(), l.config.Variable) - if errors.Is(err, store.ErrNotFound) { - return l.config.LinkID, nil - } - if err != nil { - return uuid.Nil, fmt.Errorf("read: %v", err) - } - if linkID == uuid.Nil { - linkID = l.config.LinkID - } - - if err := l.j.markComplete(ctx); err != nil { - return uuid.Nil, err - } - - return linkID, nil + return nil } diff --git a/hack/ccp/internal/controller/jobs_client.go b/hack/ccp/internal/controller/jobs_client.go new file mode 100644 index 000000000..0f46d9e08 --- /dev/null +++ b/hack/ccp/internal/controller/jobs_client.go @@ -0,0 +1,308 @@ +package controller + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + + "github.com/google/uuid" + + "github.com/artefactual/archivematica/hack/ccp/internal/python" + "github.com/artefactual/archivematica/hack/ccp/internal/store" + "github.com/artefactual/archivematica/hack/ccp/internal/workflow" +) + +// directoryClientScriptJob. +// +// Manager: linkTaskManagerDirectories. +// Class: DirectoryClientScriptJob(DecisionJob). +type directoryClientScriptJob struct { + j *job + config *workflow.LinkStandardTaskConfig +} + +var _ jobRunner = (*directoryClientScriptJob)(nil) + +func newDirectoryClientScriptJob(j *job) (*directoryClientScriptJob, error) { + ret := &directoryClientScriptJob{ + j: j, + config: &workflow.LinkStandardTaskConfig{}, + } + if err := loadConfig(j.wl, ret.config); err != nil { + return nil, err + } + + return ret, nil +} + +func (l *directoryClientScriptJob) exec(ctx context.Context) (uuid.UUID, error) { + if err := l.j.save(ctx); err != nil { + return uuid.Nil, err + } + + taskResult, err := l.submitTasks(ctx) + if err != nil { + return uuid.Nil, fmt.Errorf("submit task: %v", err) + } + + if err := l.j.updateStatusFromExitCode(ctx, taskResult.ExitCode); err != nil { + return uuid.Nil, err + } + + if ec, ok := l.j.wl.ExitCodes[taskResult.ExitCode]; ok { + if ec.LinkID == nil { + return uuid.Nil, io.EOF // End of chain. + } + return *ec.LinkID, nil + } + + if l.j.wl.FallbackLinkID == uuid.Nil { + return uuid.Nil, io.EOF // End of chain. + } + + return l.j.wl.FallbackLinkID, nil +} + +func (l *directoryClientScriptJob) submitTasks(ctx context.Context) (*taskResult, error) { + rm := l.j.pkg.unit.replacements(l.config.FilterSubdir).update(l.j.chain.context) + args := rm.replaceValues(l.config.Arguments) + stdout := rm.replaceValues(l.config.StdoutFile) + stderr := rm.replaceValues(l.config.StderrFile) + + taskBackend := newTaskBackend(l.j.logger, l.j, l.j.pkg.store, l.j.gearman, l.config) + if err := taskBackend.submit(ctx, rm, args, false, stdout, stderr); err != nil { + return nil, err + } + + results, err := taskBackend.wait(ctx) + if err != nil { + return nil, fmt.Errorf("wait: %v", err) + } + + ret := results.First() + if ret == nil { + return nil, errors.New("submit task: no results") + } + + return ret, nil +} + +// filesClientScriptJob. +// +// Manager: linkTaskManagerFiles. +// Class: FilesClientScriptJob(DecisionJob). +type filesClientScriptJob struct { + j *job + config *workflow.LinkStandardTaskConfig +} + +var _ jobRunner = (*filesClientScriptJob)(nil) + +func newFilesClientScriptJob(j *job) (*filesClientScriptJob, error) { + ret := &filesClientScriptJob{ + j: j, + config: &workflow.LinkStandardTaskConfig{}, + } + if err := loadConfig(j.wl, ret.config); err != nil { + return nil, err + } + + return ret, nil +} + +func (l *filesClientScriptJob) exec(ctx context.Context) (uuid.UUID, error) { + if err := l.j.save(ctx); err != nil { + return uuid.Nil, err + } + + filterSubDir, err := l.filterSubDir(ctx) + if err != nil { + return uuid.Nil, fmt.Errorf("look up filterSubDir: %v", err) + } + + taskResults, err := l.submitTasks(ctx, filterSubDir) + if err != nil { + return uuid.Nil, fmt.Errorf("submit task: %v", err) + } + exitCode := 0 + if taskResults != nil { + exitCode = taskResults.ExitCode() + } + + if err := l.j.updateStatusFromExitCode(ctx, exitCode); err != nil { + return uuid.Nil, err + } + + if ec, ok := l.j.wl.ExitCodes[exitCode]; ok { + if ec.LinkID == nil { + return uuid.Nil, io.EOF // End of chain. + } + return *ec.LinkID, nil + } + + if l.j.wl.FallbackLinkID == uuid.Nil { + return uuid.Nil, io.EOF // End of chain. + } + + return l.j.wl.FallbackLinkID, nil +} + +func (l *filesClientScriptJob) submitTasks(ctx context.Context, filterSubDir string) (*taskResults, error) { + rm := l.j.pkg.unit.replacements(filterSubDir).update(l.j.chain.context) + taskBackend := newTaskBackend(l.j.logger, l.j, l.j.pkg.store, l.j.gearman, l.config) + + files, err := l.j.pkg.Files(ctx, l.config.FilterFileEnd, filterSubDir) + if err != nil { + return nil, err + } + if len(files) == 0 { + return nil, nil // Nothing to do. + } + + for _, fileReplacements := range files { + rm = rm.with(fileReplacements) + args := rm.replaceValues(l.config.Arguments) + stdout := rm.replaceValues(l.config.StdoutFile) + stderr := rm.replaceValues(l.config.StderrFile) + + if err := taskBackend.submit(ctx, rm, args, false, stdout, stderr); err != nil { + return nil, err + } + } + + res, err := taskBackend.wait(ctx) + if err != nil { + return nil, fmt.Errorf("wait: %v", err) + } + + return res, nil +} + +// filterSubDir returns the directory to filter files on. This path is usually +// defined in the workflow but can be overridden per package in a UnitVariable, +// so we need to look that up. +func (l *filesClientScriptJob) filterSubDir(ctx context.Context) (string, error) { + filterSubDir := l.config.FilterSubdir + + // Check if filterSubDir has been overridden for this Transfer/SIP. + val, err := l.j.pkg.store.ReadUnitVar(ctx, l.j.pkg.id, l.j.pkg.packageType(), l.config.Execute) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + return filterSubDir, nil + } + return "", err + } + + if val == "" { + return filterSubDir, nil + } + if m, err := python.EvalMap(val); err != nil { + if override, ok := m["filterSubDir"]; ok { + filterSubDir = override + } + } + + return filterSubDir, nil +} + +// outputClientScriptJob. +// +// Manager: linkTaskManagerGetMicroserviceGeneratedListInStdOut. +// Class: OutputClientScriptJob(DecisionJob). +type outputClientScriptJob struct { + j *job + config *workflow.LinkStandardTaskConfig +} + +var _ jobRunner = (*outputClientScriptJob)(nil) + +func newOutputClientScriptJob(j *job) (*outputClientScriptJob, error) { + ret := &outputClientScriptJob{ + j: j, + config: &workflow.LinkStandardTaskConfig{}, + } + if err := loadConfig(j.wl, ret.config); err != nil { + return nil, err + } + + return ret, nil +} + +// The list of choices as encoded by the client script: +// +// { +// "default": {"description": "asdf", "uri": "asdf"}, +// "5c732a52-6cdb-4b50-ac2e-ae10361b019a": {"description": "asdf", "uri": "asdf"}, +// } +type outputClientScriptChoice struct { + Description string `json:"description"` + URI string `json:"uri"` +} + +func (l *outputClientScriptJob) exec(ctx context.Context) (uuid.UUID, error) { + if err := l.j.save(ctx); err != nil { + return uuid.Nil, err + } + + taskResult, err := l.submitTasks(ctx) + if err != nil { + return uuid.Nil, fmt.Errorf("submit task: %v", err) + } + + output := map[string]outputClientScriptChoice{} + if err := json.Unmarshal([]byte(taskResult.Stdout), &output); err != nil { + l.j.logger.Error(err, "Unable to parse output: %s", taskResult.Stdout) + } else { + choices := make([]choice, 0, len(output)) + for _, item := range output { + choices = append(choices, choice{ + label: item.Description, + value: [2]string{"", item.URI}, + }) + } + l.j.chain.choices = choices + } + + if err := l.j.updateStatusFromExitCode(ctx, taskResult.ExitCode); err != nil { + return uuid.Nil, err + } + + if ec, ok := l.j.wl.ExitCodes[taskResult.ExitCode]; ok { + if ec.LinkID == nil { + return uuid.Nil, io.EOF // End of chain. + } + return *ec.LinkID, nil + } + + if l.j.wl.FallbackLinkID == uuid.Nil { + return uuid.Nil, io.EOF // End of chain. + } + + return uuid.Nil, nil +} + +func (l *outputClientScriptJob) submitTasks(ctx context.Context) (*taskResult, error) { + rm := l.j.pkg.unit.replacements(l.config.FilterSubdir).update(l.j.chain.context) + args := rm.replaceValues(l.config.Arguments) + stdout := rm.replaceValues(l.config.StdoutFile) + stderr := rm.replaceValues(l.config.StderrFile) + + taskBackend := newTaskBackend(l.j.logger, l.j, l.j.pkg.store, l.j.gearman, l.config) + if err := taskBackend.submit(ctx, rm, args, true, stdout, stderr); err != nil { + return nil, err + } + + results, err := taskBackend.wait(ctx) + if err != nil { + return nil, fmt.Errorf("wait: %v", err) + } + + ret := results.First() + if ret == nil { + return nil, errors.New("submit task: no results") + } + + return ret, nil +} diff --git a/hack/ccp/internal/controller/jobs_decision.go b/hack/ccp/internal/controller/jobs_decision.go new file mode 100644 index 000000000..384df44da --- /dev/null +++ b/hack/ccp/internal/controller/jobs_decision.go @@ -0,0 +1,497 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "maps" + "sync" + + "github.com/google/uuid" + + "github.com/artefactual/archivematica/hack/ccp/internal/derrors" + "github.com/artefactual/archivematica/hack/ccp/internal/workflow" +) + +// jobDecider is a type of job that handles decisions. +// +// It is implemented by outputDecisionJob, updateContextDecisionJob, and +// nextChainDecisionJob. The decide method is executed by the controller to +// propagate the resolution so a job can react to it, e.g. update the context. +type jobDecider interface { + decide(ctx context.Context, c choice) error +} + +// errWait is used by decision jobs to signal the awaiting condition. The +// controller is expected to consume and resolve it. +type errWait struct { + decision *decision +} + +func (err errWait) Error() string { + return "errWait" +} + +func isErrWait(err error) (*errWait, bool) { + ew := &errWait{} + if ok := errors.As(err, &ew); ok { + return ew, true + } + return nil, false +} + +// createAwait returns an errWait with all the details needed for the controller to +// coordinate the decision and its resolution. +func createAwait(ctx context.Context, j *job, choices []choice) (_ uuid.UUID, err error) { + if err := j.markAwaitingDecision(ctx); err != nil { + return uuid.Nil, err + } + + jd, ok := j.jobRunner.(jobDecider) + if !ok { + return uuid.Nil, errors.New("impossible to await this job because it's not a decider") + } + + err = &errWait{ + decision: newDecision(j.wl.Description.String(), j.pkg, choices, jd), + } + + return uuid.Nil, err +} + +// choice is a single selectable user decision created by a job to be presented +// within a decision. +type choice struct { + // label is the string representation of this choice (mandatory). + label string // TODO: use the i18n field in the workflow package. + + // value is optional, not used by nextChainDecisionJob. + // - outputClientScriptJob populates a single value, e.g.: `[2]string{"", item.URI}`. + // - udpateContextDecisionJob populates a pair, e.g.: `[2]string{"AIPCompressionLevel", "1"}`. + value [2]string + + // nextLink indicates where to continue processing when the decision is + // resolved using this choice (mandatory). + nextLink uuid.UUID +} + +func (c choice) String() string { + return fmt.Sprintf("choice %s: %s", c.label, c.value) +} + +// A decision can be awaited until someone else resolves it. It provides the +// list of available choices and the resolution interface. +type decision struct { + name string // Name of the decision. + pkg *Package // Related package. + choices []choice // Ordered list of choices. + decider jobDecider // So we can call the decide callback. + res chan int // Resolution channel - receives the position of the choice. + resolved bool // Remembers if this decision is already resolved. + sync.RWMutex // Protects the decision from concurrent read-writes. +} + +func newDecision(name string, pkg *Package, choices []choice, job jobDecider) *decision { + return &decision{ + name: name, + pkg: pkg, + choices: choices, + decider: job, + + // The channel is buffered so the decision can be resolved even when + // there is no one awaiting. + res: make(chan int, 1), + } +} + +// resolve the decision given the position of one of the known choices. +func (d *decision) resolve(pos int) error { + d.RLock() + if d.resolved { + return errors.New("decision is not pending resolution") + } + d.RUnlock() + + d.Lock() + d.res <- pos + d.resolved = true + d.Unlock() + + return nil +} + +// await waits for the resolution. It returns the next workflow chain link, +// which will most likely be uuid.Nil unless indicated by nextChainDecisionJob. +func (d *decision) await(ctx context.Context) (uuid.UUID, error) { + select { + case <-ctx.Done(): + return uuid.Nil, ctx.Err() + case pos := <-d.res: + ln := len(d.choices) + if ln < 0 || ln > len(d.choices) { + return uuid.Nil, errors.New("unavailable choice") + } + choice := d.choices[pos] + if err := d.decider.decide(ctx, choice); err != nil { + return uuid.Nil, err + } + return choice.nextLink, nil + } +} + +// outputDecisionJob. +// +// A job that handles a workflow decision point, with choices based on script +// output. +// +// Manager: linkTaskManagerGetUserChoiceFromMicroserviceGeneratedList. +// Class: OutputDecisionJob(DecisionJob). +type outputDecisionJob struct { + j *job + config *workflow.LinkStandardTaskConfig +} + +var ( + _ jobRunner = (*outputDecisionJob)(nil) + _ jobDecider = (*outputDecisionJob)(nil) +) + +func newOutputDecisionJob(j *job) (*outputDecisionJob, error) { + ret := &outputDecisionJob{ + j: j, + config: &workflow.LinkStandardTaskConfig{}, + } + if err := loadConfig(j.wl, ret.config); err != nil { + return nil, err + } + + return ret, nil +} + +func (l *outputDecisionJob) exec(ctx context.Context) (_ uuid.UUID, err error) { + derrors.Add(&err, "outputDecisionJob") + + if err := l.j.save(ctx); err != nil { + return uuid.Nil, err + } + + var c *choice + locURI, err := l.j.pkg.PreconfiguredChoice(l.j.wl.ID) + if err != nil { + return uuid.Nil, err + } else if locURI != "" { + for _, item := range l.j.chain.choices { + v := item.value + if locURI == v[1] { + c = &item + } + } + } + if c != nil { + // TODO: choice was preconfigured, now we have to update the chain + // context, mark the job as complete and continue processing. + // Use exit_codes[0] if OK, or fallback_link_id but it's present in + // one of the two chain links only. + panic("not implemented") + } + + return createAwait(ctx, l.j, l.j.chain.choices) +} + +func (l *outputDecisionJob) decide(ctx context.Context, c choice) error { + // Pass the choice to the next job. This case is only used to select an AIP + // store URI, and the value of execute (script_name here) is a replacement + // string (e.g. %AIPsStore%). + l.j.chain.context.Set(l.config.Execute, c.value[1]) + + return nil +} + +// nextChainDecisionJob. +// +// A type of workflow decision that determines the next chain to be executed, +// by UUID. +// +// Manager: linkTaskManagerChoice. +// Class: NextChainDecisionJob(DecisionJob). +type nextChainDecisionJob struct { + j *job + config *workflow.LinkMicroServiceChainChoice +} + +var ( + _ jobRunner = (*nextChainDecisionJob)(nil) + _ jobDecider = (*outputDecisionJob)(nil) +) + +func newNextChainDecisionJob(j *job) (*nextChainDecisionJob, error) { + ret := &nextChainDecisionJob{ + j: j, + config: &workflow.LinkMicroServiceChainChoice{}, + } + if err := loadConfig(j.wl, ret.config); err != nil { + return nil, err + } + + return ret, nil +} + +func (l *nextChainDecisionJob) exec(ctx context.Context) (_ uuid.UUID, err error) { + derrors.Add(&err, "nextChainDecisionJob") + defer func() { + if e := l.j.markComplete(ctx); e != nil { + err = e + } + }() + + if err := l.j.save(ctx); err != nil { + return uuid.Nil, err + } + + // Use a preconfigured choice if it validates. + chainID, err := l.j.pkg.PreconfiguredChoice(l.j.wl.ID) + if err != nil { + return uuid.Nil, err + } else if chainID != "" { + cid, err := uuid.Parse(chainID) + if err != nil { + return uuid.Nil, err + } + + // Fail if the choice is not available in workflow. + var matched bool + for _, item := range l.config.Choices { + if _, ok := l.j.wf.Chains[item]; ok { + matched = true + } + } + if !matched { + return uuid.Nil, fmt.Errorf("choice %s is not one of the available choices", chainID) + } + if err := l.j.markComplete(ctx); err != nil { + return uuid.Nil, err + } + return cid, nil + } + + // Build choices. + choices := make([]choice, len(l.config.Choices)) + for i, item := range l.config.Choices { + c := &choices[i] + + var label string + if ch, ok := l.j.wf.Chains[item]; ok { + label = ch.Description.String() + } else { + label = fmt.Sprintf("Chain: %s (not found)", item) + } + c.label = label + + c.nextLink = item + } + + return createAwait(ctx, l.j, choices) +} + +func (l *nextChainDecisionJob) decide(ctx context.Context, c choice) error { //nolint: unparam + return l.j.markComplete(ctx) +} + +// updateContextDecisionJob. +// +// A job that updates the job chain context based on a user choice. +// +// TODO: This type of job is mostly copied from the previous +// linkTaskManagerReplacementDicFromChoice, and it seems to have multiple ways +// of executing. It could use some cleanup. +// +// Manager: linkTaskManagerReplacementDicFromChoice. +// Class: UpdateContextDecisionJob(DecisionJob) (decisions.py). +type updateContextDecisionJob struct { + j *job + config *workflow.LinkMicroServiceChoiceReplacementDic +} + +var ( + _ jobRunner = (*updateContextDecisionJob)(nil) + _ jobDecider = (*outputDecisionJob)(nil) +) + +// Maps decision point UUIDs and decision UUIDs to their "canonical" +// equivalents. This is useful for when there are multiple decision points which +// are effectively identical and a preconfigured decision for one should hold +// for all of the others as well. For example, there are 5 "Assign UUIDs to +// directories?" decision points and making a processing config decision for the +// designated canonical one, in this case +// 'bd899573-694e-4d33-8c9b-df0af802437d', should result in that decision taking +// effect for all of the others as well. This allows that. +// TODO: this should be defined in the workflow, not hardcoded here. +var updateContextDecisionJobChoiceMapping = map[uuid.UUID]uuid.UUID{ + // Decision point "Assign UUIDs to directories?". + uuid.MustParse("8882bad4-561c-4126-89c9-f7f0c083d5d7"): uuid.MustParse("bd899573-694e-4d33-8c9b-df0af802437d"), + uuid.MustParse("e10a31c3-56df-4986-af7e-2794ddfe8686"): uuid.MustParse("bd899573-694e-4d33-8c9b-df0af802437d"), + uuid.MustParse("d6f6f5db-4cc2-4652-9283-9ec6a6d181e5"): uuid.MustParse("bd899573-694e-4d33-8c9b-df0af802437d"), + uuid.MustParse("1563f22f-f5f7-4dfe-a926-6ab50d408832"): uuid.MustParse("bd899573-694e-4d33-8c9b-df0af802437d"), + // Decision "Yes" (for "Assign UUIDs to directories?"). + uuid.MustParse("7e4cf404-e62d-4dc2-8d81-6141e390f66f"): uuid.MustParse("2dc3f487-e4b0-4e07-a4b3-6216ed24ca14"), + uuid.MustParse("2732a043-b197-4cbc-81ab-4e2bee9b74d3"): uuid.MustParse("2dc3f487-e4b0-4e07-a4b3-6216ed24ca14"), + uuid.MustParse("aa793efa-1b62-498c-8f92-cab187a99a2a"): uuid.MustParse("2dc3f487-e4b0-4e07-a4b3-6216ed24ca14"), + uuid.MustParse("efd98ddb-80a6-4206-80bf-81bf00f84416"): uuid.MustParse("2dc3f487-e4b0-4e07-a4b3-6216ed24ca14"), + // Decision "No" (for "Assign UUIDs to directories?"). + uuid.MustParse("0053c670-3e61-4a3e-a188-3a2dd1eda426"): uuid.MustParse("891f60d0-1ba8-48d3-b39e-dd0934635d29"), + uuid.MustParse("8e93e523-86bb-47e1-a03a-4b33e13f8c5e"): uuid.MustParse("891f60d0-1ba8-48d3-b39e-dd0934635d29"), + uuid.MustParse("6dfbeff8-c6b1-435b-833a-ed764229d413"): uuid.MustParse("891f60d0-1ba8-48d3-b39e-dd0934635d29"), + uuid.MustParse("dc0ee6b6-ed5f-42a3-bc8f-c9c7ead03ed1"): uuid.MustParse("891f60d0-1ba8-48d3-b39e-dd0934635d29"), +} + +func newUpdateContextDecisionJob(j *job) (*updateContextDecisionJob, error) { + ret := &updateContextDecisionJob{ + j: j, + config: &workflow.LinkMicroServiceChoiceReplacementDic{}, + } + if err := loadConfig(j.wl, ret.config); err != nil { + return nil, err + } + + return ret, nil +} + +func (l *updateContextDecisionJob) exec(ctx context.Context) (linkID uuid.UUID, err error) { + derrors.Add(&err, "nextChainDecisionJob") + defer func() { + if e := l.j.markComplete(ctx); e != nil { + err = e + return + } + if id := l.j.wl.ExitCodes[0].LinkID; id == nil || *id == uuid.Nil { + err = fmt.Errorf("updateContextDecisionJob: linkID undefined") + } else { + linkID = *id + } + }() + + if err := l.j.save(ctx); err != nil { + return uuid.Nil, err + } + + // Load new context from the database (DashboardSettings). + // We have two chain links in workflow with no replacements configured: + // "7f975ba6" and "a0db8294". This feels like a different case where we are + // loading the replacements from the application database. + // TODO: split this out? + if len(l.config.Replacements) == 0 { + if dict, err := l.loadDatabaseContext(ctx); err != nil { + return uuid.Nil, fmt.Errorf("load dict from db: %v", err) + } else if dict != nil { + l.j.chain.update(dict) + return uuid.Nil, nil + } + } + + // Load new context from processing configuration. + if dict, err := l.loadPreconfiguredContext(); err != nil { + return uuid.Nil, fmt.Errorf("load context with preconfigured choice: %v", err) + } else if dict != nil { + l.j.chain.update(dict) + return uuid.Nil, nil + } + + // Build choices. + choices := make([]choice, len(l.config.Replacements)) + for i, item := range l.config.Replacements { + c := &choices[i] + c.label = item.Description.String() + c.nextLink = *l.j.wl.ExitCodes[0].LinkID + for k, v := range item.Items { + c.value = [2]string{k, v} + break + } + } + + return createAwait(ctx, l.j, choices) +} + +// loadDatabaseContext loads the context dictionary from the database. +func (l *updateContextDecisionJob) loadDatabaseContext(ctx context.Context) (map[string]string, error) { + // We're looking for the "execute" parameter of the next link, e.g.: + // "upload-archivesspace_v0.0" or "upload-qubit_v0.0". + ln, ok := l.j.wf.Links[l.j.wl.FallbackLinkID] + if !ok { + return nil, nil + } + cfg, ok := ln.Config.(workflow.LinkStandardTaskConfig) + if !ok { + return nil, nil + } + if cfg.Execute == "" { + return nil, nil + } + + ret, err := l.j.pkg.store.ReadDict(ctx, cfg.Execute) + if err != nil { + return nil, err + } + + return l.formatChoices(ret), nil +} + +// loadPreconfiguredContext loads the context dictionary from the workflow. +func (l *updateContextDecisionJob) loadPreconfiguredContext() (map[string]string, error) { + var normalizedChoice uuid.UUID + if v, ok := updateContextDecisionJobChoiceMapping[l.j.wl.ID]; ok { + normalizedChoice = v + } else { + normalizedChoice = l.j.wl.ID + } + + choices, err := l.j.pkg.parseProcessingConfig() + if err != nil { + return nil, err + } + + for _, choice := range choices { + if choice.AppliesTo != normalizedChoice.String() { + continue + } + desiredChoice, err := uuid.Parse(choice.GoToChain) + if err != nil { + return nil, err + } + if v, ok := updateContextDecisionJobChoiceMapping[desiredChoice]; ok { + desiredChoice = v + } + ln, ok := l.j.wf.Links[normalizedChoice] + if !ok { + return nil, nil // fmt.Errorf("desired choice not found: %s", desiredChoice) + } + config, ok := ln.Config.(workflow.LinkMicroServiceChoiceReplacementDic) + if !ok { + return nil, fmt.Errorf("desired choice doesn't have the expected type: %s", desiredChoice) + } + for _, replacement := range config.Replacements { + if replacement.ID == desiredChoice { + choices := maps.Clone(replacement.Items) + return l.formatChoices(choices), nil + } + } + } + + return nil, nil +} + +func (l *updateContextDecisionJob) formatChoices(choices map[string]string) map[string]string { + for k, v := range choices { + delete(choices, k) + choices[fmt.Sprintf("%%%s%%", k)] = v + } + + return choices +} + +func (l *updateContextDecisionJob) decide(ctx context.Context, c choice) error { + if c.value[0] != "" { + l.j.chain.context.Set(c.value[0], c.value[1]) // "AssignUUIDsToDirectories" => "True" + } + + return l.j.markComplete(ctx) +} diff --git a/hack/ccp/internal/controller/jobs_decision_test.go b/hack/ccp/internal/controller/jobs_decision_test.go new file mode 100644 index 000000000..503b074dc --- /dev/null +++ b/hack/ccp/internal/controller/jobs_decision_test.go @@ -0,0 +1,62 @@ +package controller + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "go.uber.org/mock/gomock" + "gotest.tools/v3/assert" +) + +func TestOutputDecisionJob(t *testing.T) { + t.Parallel() +} + +func TestNextChainDecisionJob(t *testing.T) { + t.Parallel() + + t.Run("Honours preconfigured choices", func(t *testing.T) { + t.Parallel() + }) + + t.Run("Creates a decision", func(t *testing.T) { + t.Parallel() + + job, store := createJob(t, "56eebd45-5600-4768-a8c2-ec0114555a3d") + + store.EXPECT().UpdateJobStatus(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + store.EXPECT().CreateJob(gomock.Any(), gomock.Any()).Return(nil).Times(1) + + id, err := job.exec(context.Background()) + assert.Equal(t, id, uuid.Nil) + + decision := assertErrWait(t, err, "Generate transfer structure report", []choice{ + {label: "Yes", nextLink: uuid.MustParse("df54fec1-dae1-4ea6-8d17-a839ee7ac4a7")}, + {label: "No", nextLink: uuid.MustParse("e9eaef1e-c2e0-4e3b-b942-bfb537162795")}, + }) + + decision.resolve(0) + nextLink, err := decision.await(context.Background()) + assert.NilError(t, err) + assert.Equal(t, nextLink, uuid.MustParse("df54fec1-dae1-4ea6-8d17-a839ee7ac4a7")) + }) +} + +func TestUpdateContextDecisionJob(t *testing.T) { + t.Parallel() +} + +func assertErrWait(t *testing.T, err error, name string, choices []choice) *decision { + t.Helper() + + ew := &errWait{} + assert.Equal(t, errors.As(err, &ew), true) + + assert.Equal(t, ew.decision.name, name) + assert.DeepEqual(t, ew.decision.choices, choices, cmpopts.EquateComparable(choice{})) + + return ew.decision +} diff --git a/hack/ccp/internal/controller/jobs_local.go b/hack/ccp/internal/controller/jobs_local.go new file mode 100644 index 000000000..cba558bec --- /dev/null +++ b/hack/ccp/internal/controller/jobs_local.go @@ -0,0 +1,107 @@ +package controller + +import ( + "context" + "errors" + "fmt" + + "github.com/google/uuid" + + "github.com/artefactual/archivematica/hack/ccp/internal/store" + "github.com/artefactual/archivematica/hack/ccp/internal/workflow" +) + +// setUnitVarLinkJob is a local job that sets the unit variable configured in +// the workflow. +// +// Manager: linkTaskManagerSetUnitVariable. +// Class: SetUnitVarLinkJob(DecisionJob) (decisions.py). +type setUnitVarLinkJob struct { + j *job + config *workflow.LinkTaskConfigSetUnitVariable +} + +var _ jobRunner = (*setUnitVarLinkJob)(nil) + +func newSetUnitVarLinkJob(j *job) (*setUnitVarLinkJob, error) { + ret := &setUnitVarLinkJob{ + j: j, + config: &workflow.LinkTaskConfigSetUnitVariable{}, + } + if err := loadConfig(j.wl, ret.config); err != nil { + return nil, err + } + + return ret, nil +} + +func (l *setUnitVarLinkJob) exec(ctx context.Context) (_ uuid.UUID, err error) { + defer func() { + if err == nil { + if markErr := l.j.markComplete(ctx); markErr != nil { + err = markErr + } + } + }() + + if err := l.j.save(ctx); err != nil { + return uuid.Nil, err + } + + if err := l.j.pkg.saveLinkID(ctx, l.config.Variable, l.config.LinkID); err != nil { + return uuid.Nil, err + } + + return l.config.LinkID, nil +} + +// getUnitVarLinkJob is a local job that gets the next link in the chain from a +// UnitVariable. +// +// Manager: linkTaskManagerUnitVariableLinkPull. +// Class: GetUnitVarLinkJob(DecisionJob) (decisions.py). +type getUnitVarLinkJob struct { + j *job + config *workflow.LinkTaskConfigUnitVariableLinkPull +} + +var _ jobRunner = (*getUnitVarLinkJob)(nil) + +func newGetUnitVarLinkJob(j *job) (*getUnitVarLinkJob, error) { + ret := &getUnitVarLinkJob{ + j: j, + config: &workflow.LinkTaskConfigUnitVariableLinkPull{}, + } + if err := loadConfig(j.wl, ret.config); err != nil { + return nil, err + } + + return ret, nil +} + +func (l *getUnitVarLinkJob) exec(ctx context.Context) (_ uuid.UUID, err error) { + defer func() { + if err == nil { + if markErr := l.j.markComplete(ctx); markErr != nil { + err = markErr + } + } + }() + + if err := l.j.save(ctx); err != nil { + return uuid.Nil, err + } + + linkID, err := l.j.pkg.store.ReadUnitLinkID(ctx, l.j.pkg.id, l.j.pkg.packageType(), l.config.Variable) + if errors.Is(err, store.ErrNotFound) { + return l.config.LinkID, nil + } + if err != nil { + return uuid.Nil, fmt.Errorf("read: %v", err) + } + if linkID == uuid.Nil { + linkID = l.config.LinkID + } + + return linkID, nil +} diff --git a/hack/ccp/internal/controller/jobs_local_test.go b/hack/ccp/internal/controller/jobs_local_test.go new file mode 100644 index 000000000..6a51764d9 --- /dev/null +++ b/hack/ccp/internal/controller/jobs_local_test.go @@ -0,0 +1,65 @@ +package controller + +import ( + "context" + "testing" + + "github.com/google/uuid" + "go.artefactual.dev/tools/mockutil" + "go.uber.org/mock/gomock" + "gotest.tools/v3/assert" + + "github.com/artefactual/archivematica/hack/ccp/internal/store/enums" + "github.com/artefactual/archivematica/hack/ccp/internal/store/sqlcmysql" +) + +func TestSetUnitVarLinkJob(t *testing.T) { + t.Parallel() + + t.Run("Stores the linkID in the database", func(t *testing.T) { + t.Parallel() + + job, st := createJob(t, "b33c9544-145c-4525-8a80-d686b4d1c3fa") + + st.EXPECT().CreateJob(mockutil.Context(), gomock.AssignableToTypeOf(&sqlcmysql.CreateJobParams{})).Return(nil).Times(1) + st.EXPECT().CreateUnitVar(mockutil.Context(), job.pkg.id, enums.PackageTypeTransfer, "normalizationThumbnailProcessing", "", uuid.MustParse("180ae3d0-aa6c-4ed4-ab94-d0a2121e7f21"), true).Times(1) + st.EXPECT().UpdateJobStatus(mockutil.Context(), job.id, "STATUS_COMPLETED_SUCCESSFULLY").Return(nil).Times(1) + + linkID, err := job.exec(context.Background()) + assert.NilError(t, err) + assert.Equal(t, linkID, uuid.MustParse("180ae3d0-aa6c-4ed4-ab94-d0a2121e7f21")) + }) +} + +func TestGetUnitVarLinkJob(t *testing.T) { + t.Parallel() + + t.Run("Returns the linkID stored in the database", func(t *testing.T) { + t.Parallel() + + nextLinkID := uuid.MustParse("0ce7ab48-fd48-4abe-9150-f682499e7cf0") + job, st := createJob(t, "6e5126be-76ac-4c8f-9754-fc25a234a751") + + st.EXPECT().CreateJob(mockutil.Context(), gomock.AssignableToTypeOf(&sqlcmysql.CreateJobParams{})).Return(nil).Times(1) + st.EXPECT().UpdateJobStatus(mockutil.Context(), job.id, "STATUS_COMPLETED_SUCCESSFULLY").Return(nil).Times(1) + st.EXPECT().ReadUnitLinkID(mockutil.Context(), job.pkg.id, enums.PackageTypeTransfer, "normalizationThumbnailProcessing").Return(nextLinkID, nil).Times(1) + + linkID, err := job.exec(context.Background()) + assert.NilError(t, err) + assert.Equal(t, linkID, nextLinkID) + }) + + t.Run("Returns the linkID defined in workflow", func(t *testing.T) { + t.Parallel() + + job, st := createJob(t, "b04e9232-2aea-49fc-9560-27349c8eba4e") + + st.EXPECT().CreateJob(mockutil.Context(), gomock.AssignableToTypeOf(&sqlcmysql.CreateJobParams{})).Return(nil).Times(1) + st.EXPECT().UpdateJobStatus(mockutil.Context(), job.id, "STATUS_COMPLETED_SUCCESSFULLY").Return(nil).Times(1) + st.EXPECT().ReadUnitLinkID(mockutil.Context(), job.pkg.id, enums.PackageTypeTransfer, "loadOptionsToCreateSIP").Return(uuid.MustParse("bb194013-597c-4e4a-8493-b36d190f8717"), nil).Times(1) + + linkID, err := job.exec(context.Background()) + assert.NilError(t, err) + assert.Equal(t, linkID, uuid.MustParse("bb194013-597c-4e4a-8493-b36d190f8717")) + }) +} diff --git a/hack/ccp/internal/controller/jobs_test.go b/hack/ccp/internal/controller/jobs_test.go new file mode 100644 index 000000000..53c2f3712 --- /dev/null +++ b/hack/ccp/internal/controller/jobs_test.go @@ -0,0 +1,64 @@ +package controller + +import ( + "context" + "testing" + + "github.com/artefactual-labs/gearmin/gearmintest" + "github.com/go-logr/logr" + "github.com/google/uuid" + "github.com/mikespook/gearman-go/worker" + "go.uber.org/mock/gomock" + "gotest.tools/v3/assert" + "gotest.tools/v3/fs" + + "github.com/artefactual/archivematica/hack/ccp/internal/store/enums" + "github.com/artefactual/archivematica/hack/ccp/internal/store/storemock" + "github.com/artefactual/archivematica/hack/ccp/internal/workflow" +) + +func createJob(t *testing.T, linkID string) (*job, *storemock.MockStore) { + t.Helper() + + gearmin := gearmintest.Server(t, map[string]gearmintest.Handler{"hello": func(job worker.Job) ([]byte, error) { return []byte("hi!"), nil }}) + wf, _ := workflow.Default() + ln := wf.Links[uuid.MustParse(linkID)] + tmpDir := fs.NewDir(t, "", fs.WithDir("sharedDir")) + store := storemock.NewMockStore(gomock.NewController(t)) + chain := &chain{} + + pkg := newPackage(logr.Discard(), store, tmpDir.Join("sharedDir")) + pkg.id = uuid.New() + pkg.unit = &noUnit{} + + job, err := newJob(logr.Discard(), chain, pkg, gearmin, ln, wf) + assert.NilError(t, err) + + return job, store +} + +type noUnit struct{} + +func (u *noUnit) hydrate(ctx context.Context, path, watchedDir string) error { + return nil +} + +func (u *noUnit) reload(ctx context.Context) error { + return nil +} + +func (u *noUnit) replacements(filterSubdirPath string) replacementMapping { + return nil +} + +func (u *noUnit) replacementPath() string { + return "" +} + +func (u *noUnit) packageType() enums.PackageType { + return enums.PackageTypeTransfer +} + +func (u *noUnit) jobUnitType() string { + return "" +} diff --git a/hack/ccp/internal/controller/package.go b/hack/ccp/internal/controller/package.go index 5b8fc7dd8..f00b40c90 100644 --- a/hack/ccp/internal/controller/package.go +++ b/hack/ccp/internal/controller/package.go @@ -9,8 +9,6 @@ import ( "os" "path/filepath" "strings" - "sync" - "sync/atomic" "github.com/elliotchance/orderedmap/v2" "github.com/go-logr/logr" @@ -48,9 +46,6 @@ type Package struct { // Identifier of the link where the iterator must start processing. startAtLinkID uuid.UUID - - // User decisinon manager - decision decision } func newPackage(logger logr.Logger, store store.Store, sharedDir string) *Package { @@ -256,18 +251,18 @@ func (p *Package) parseProcessingConfig() ([]workflow.Choice, error) { // PreconfiguredChoice looks up a pre-configured choice in the processing // configuration file that is part of the package. -func (p *Package) PreconfiguredChoice(linkID uuid.UUID) (uuid.UUID, error) { +func (p *Package) PreconfiguredChoice(linkID uuid.UUID) (string, error) { // TODO: auto-approval should only happen if requested by the user, but // this is convenient during initial development. if chainID := Transfers.Decide(linkID); chainID != uuid.Nil { - return chainID, nil + return chainID.String(), nil } choices, err := p.parseProcessingConfig() if err != nil { - return uuid.Nil, err + return "", err } else if len(choices) == 0 { - return uuid.Nil, nil + return "", nil } var chainID uuid.UUID @@ -281,7 +276,7 @@ func (p *Package) PreconfiguredChoice(linkID uuid.UUID) (uuid.UUID, error) { // Resort to automated config. // TODO: allow user to choose the system processing config to use. if chainID == uuid.Nil { - for _, choice := range workflow.AutomatedConfig.Choices.Choices { + for _, choice := range workflow.AutomatedConfig.Choices { if choice.LinkID() == linkID { chainID = choice.ChainID() break @@ -289,31 +284,7 @@ func (p *Package) PreconfiguredChoice(linkID uuid.UUID) (uuid.UUID, error) { } } - return chainID, nil -} - -// Decide resolves an awaiting decision. -func (p *Package) Decide(opt option) error { - return p.decision.resolve(opt) -} - -// AwaitDecision builds a new decision and waits for its resolution. -func (p *Package) AwaitDecision(ctx context.Context, opts []option) (option, error) { - p.decision.build(opts...) - - for { - select { - case d := <-p.decision.recv: - return d, nil - case <-ctx.Done(): - return option(""), ctx.Err() - } - } -} - -// Decision provides the current awaiting decision. -func (p *Package) Decision() []option { - return p.decision.decision() + return chainID.String(), nil } // Files iterates over all files associated with the package or that should be @@ -412,6 +383,10 @@ func (p *Package) markAsDone(ctx context.Context) error { return p.store.UpdatePackageStatus(ctx, p.id, p.packageType(), enums.PackageStatusDone) } +func (p *Package) markAsFailed(ctx context.Context) error { + return p.store.UpdatePackageStatus(ctx, p.id, p.packageType(), enums.PackageStatusFailed) +} + func (p *Package) updateActiveAgent(ctx context.Context, userID string) error { return nil // TODO: we have not implemented auth yet! } @@ -662,67 +637,6 @@ func (u *DIP) jobUnitType() string { return "unitDIP" } -type decision struct { - opts []option - recv chan option - unsolved atomic.Bool - sync.Mutex -} - -func (pd *decision) build(opts ...option) { - pd.Lock() - pd.opts = opts - pd.recv = make(chan option) // is this ok? - pd.Unlock() - - pd.unsolved.Store(true) -} - -func (pd *decision) resolve(opt option) error { - if !pd.unsolved.Load() { - return errors.New("decision is not pending resolution") - } - - select { - case pd.recv <- opt: - pd.unsolved.Store(false) - default: - return errors.New("resolve can't proceed because nobody is listening") - } - - return nil -} - -func (pd *decision) decision() []option { - if !pd.unsolved.Load() { - return nil - } - - var opts []option - if pd.unsolved.Load() { - pd.Lock() - opts = make([]option, len(pd.opts)) - copy(opts, pd.opts) - pd.Unlock() - } - - return opts -} - -// option is a single selectable decision choice. -// -// In most cases, an option is the UUID of a workflow item, but there is one -// exception: "Store DIP location", containing a location path. -type option string - -func (do option) uuid() uuid.UUID { - id, err := uuid.Parse(string(do)) - if err != nil { - return uuid.Nil - } - return id -} - func dirBasename(path string) string { abs, _ := filepath.Abs(path) return filepath.Base(abs) diff --git a/hack/ccp/internal/controller/path.go b/hack/ccp/internal/controller/path.go index d34a10044..4e8cc24c4 100644 --- a/hack/ccp/internal/controller/path.go +++ b/hack/ccp/internal/controller/path.go @@ -65,3 +65,7 @@ func isDir(path string) bool { return info.IsDir() } + +func trim(path string) string { + return strings.Trim(path, string(filepath.Separator)) +} diff --git a/hack/ccp/internal/controller/task_test.go b/hack/ccp/internal/controller/task_test.go index 189c52595..8cfbfd930 100644 --- a/hack/ccp/internal/controller/task_test.go +++ b/hack/ccp/internal/controller/task_test.go @@ -53,7 +53,7 @@ func workerHandler(t *testing.T, job worker.Job) ([]byte, error) { func TestTaskBackend(t *testing.T) { t.Parallel() - t.Skip("Needs to be fixed: https://github.com/artefactual-labs/gearmin/issues/3.") + t.Skip("Fails interminttently in (github.com/mikespook/gearman-go).") batchSize = 128 fnName := "do" diff --git a/hack/ccp/internal/workflow/config.go b/hack/ccp/internal/workflow/config.go index 089a8f71d..f2fb905f3 100644 --- a/hack/ccp/internal/workflow/config.go +++ b/hack/ccp/internal/workflow/config.go @@ -1,304 +1,57 @@ package workflow import ( + "bytes" "encoding/xml" - "errors" "fmt" "io" "os" - "path/filepath" - "strings" "github.com/google/uuid" ) -var builtinConfigs = map[string]ProcessingConfig{ - "default": DefaultConfig, - "automated": AutomatedConfig, +const ( + xmlPrefix = "" + xmlIndent = " " +) + +type ProcessingConfig struct { + XMLName xml.Name `xml:"processingMCP"` + Choices Choices `xml:"preconfiguredChoices>preconfiguredChoice"` } -func InstallBuiltinConfigs(path string) error { - var errs error - for name, config := range builtinConfigs { - path := filepath.Join(path, fmt.Sprintf("%sProcessingMCP.xml", name)) - blob, err := xml.MarshalIndent(config, "", " ") - if err != nil { - errs = errors.Join(errs, fmt.Errorf("cannot encode %s: %v", path, err)) - continue +type Choices []Choice + +// MarshalXML encodes the comment of each preconfiguredChoice. +// +// For example: +// +// +// +// 5e58066d-e113-4383-b20b-f301ed4d751c +// 8d29eb3d-a8a8-4347-806e-3d8227ed44a1 +// +func (c Choices) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + indent := xml.CharData(fmt.Sprintf("\n%s%s", xmlIndent, xmlIndent)) + for _, item := range c { + if err := e.EncodeToken(indent); err != nil { + return err } - err = os.WriteFile(path, blob, os.FileMode(0o660)) - if err != nil { - errs = errors.Join(errs, fmt.Errorf("cannot write %s: %v", path, err)) - continue + if err := e.EncodeToken(xml.Comment(fmt.Sprintf(" %s ", item.Comment))); err != nil { + return err + } + if err := e.Encode(item); err != nil { + return err } } - return errs -} - -var DefaultConfig = ProcessingConfig{ - Choices: Choices{ - Choices: []Choice{ - { - Comment: "Select compression level", - AppliesTo: "01c651cb-c174-4ba4-b985-1d87a44d6754", - GoToChain: "414da421-b83f-4648-895f-a34840e3c3f5", - }, - { - Comment: "Perform file format identification (Submission documentation & metadata)", - AppliesTo: "087d27be-c719-47d8-9bbb-9a7d8b609c44", - GoToChain: "4dec164b-79b0-4459-8505-8095af9655b5", - }, - { - Comment: "Bind PIDs", - AppliesTo: "a2ba5278-459a-4638-92d9-38eb1588717d", - GoToChain: "44a7c397-8187-4fd2-b8f7-c61737c4df49", - }, - { - Comment: "Generate transfer structure report", - AppliesTo: "56eebd45-5600-4768-a8c2-ec0114555a3d", - GoToChain: "df54fec1-dae1-4ea6-8d17-a839ee7ac4a7", - }, - { - Comment: "Perform policy checks on originals", - AppliesTo: "70fc7040-d4fb-4d19-a0e6-792387ca1006", - GoToChain: "3e891cc4-39d2-4989-a001-5107a009a223", - }, - { - Comment: "Generate thumbnails", - AppliesTo: "498f7a6d-1b8c-431a-aa5d-83f14f3c5e65", - GoToChain: "c318b224-b718-4535-a911-494b1af6ff26", - }, - { - Comment: "Select compression algorithm", - AppliesTo: "01d64f58-8295-4b7b-9cab-8f1b153a504f", - GoToChain: "9475447c-9889-430c-9477-6287a9574c5b", - }, - { - Comment: "Perform policy checks on access derivatives", - AppliesTo: "8ce07e94-6130-4987-96f0-2399ad45c5c2", - GoToChain: "76befd52-14c3-44f9-838f-15a4e01624b0", - }, - { - Comment: "Perform file format identification (Ingest)", - AppliesTo: "7a024896-c4f7-4808-a240-44c87c762bc5", - GoToChain: "3c1faec7-7e1e-4cdd-b3bd-e2f05f4baa9b", - }, - { - Comment: "Perform policy checks on preservation derivatives", - AppliesTo: "153c5f41-3cfb-47ba-9150-2dd44ebc27df", - GoToChain: "b7ce05f0-9d94-4b3e-86cc-d4b2c6dba546", - }, - { - Comment: "Assign UUIDs to directories", - AppliesTo: "bd899573-694e-4d33-8c9b-df0af802437d", - GoToChain: "891f60d0-1ba8-48d3-b39e-dd0934635d29", - }, - { - Comment: "Document empty directories", - AppliesTo: "d0dfa5fc-e3c2-4638-9eda-f96eea1070e0", - GoToChain: "65273f18-5b4e-4944-af4f-09be175a88e8", - }, - { - Comment: "Virus scanning: Yes", - AppliesTo: "856d2d65-cd25-49fa-8da9-cabb78292894", - GoToChain: "6e431096-c403-4cbf-a59a-a26e86be54a8", - }, - { - Comment: "Virus scanning: Yes", - AppliesTo: "1dad74a2-95df-4825-bbba-dca8b91d2371", - GoToChain: "1ac7d792-b63f-46e0-9945-d48d9e5c02c9", - }, - { - Comment: "Virus scanning: Yes", - AppliesTo: "7e81f94e-6441-4430-a12d-76df09181b66", - GoToChain: "97be337c-ff27-4869-bf63-ef1abc9df15d", - }, - { - Comment: "Virus scanning: Yes", - AppliesTo: "390d6507-5029-4dae-bcd4-ce7178c9b560", - GoToChain: "34944d4f-762e-4262-8c79-b9fd48521ca0", - }, - { - Comment: "Virus scanning: Yes", - AppliesTo: "97a5ddc0-d4e0-43ac-a571-9722405a0a9b", - GoToChain: "3e8c0c39-3f30-4c9b-a449-85eef1b2a458", - }, - }, - }, -} - -var AutomatedConfig = ProcessingConfig{ - Choices: Choices{ - Choices: []Choice{ - { - Comment: "Store DIP", - AppliesTo: "5e58066d-e113-4383-b20b-f301ed4d751c", - GoToChain: "8d29eb3d-a8a8-4347-806e-3d8227ed44a1", - }, - { - Comment: "Select compression level", - AppliesTo: "01c651cb-c174-4ba4-b985-1d87a44d6754", - GoToChain: "414da421-b83f-4648-895f-a34840e3c3f5", - }, - { - Comment: "Examine contents", - AppliesTo: "accea2bf-ba74-4a3a-bb97-614775c74459", - GoToChain: "e0a39199-c62a-4a2f-98de-e9d1116460a8", - }, - { - Comment: "Perform file format identification (Submission documentation & metadata)", - AppliesTo: "087d27be-c719-47d8-9bbb-9a7d8b609c44", - GoToChain: "4dec164b-79b0-4459-8505-8095af9655b5", - }, - { - Comment: `Normalize (match 1 for "Normalize for preservation")`, - AppliesTo: "cb8e5706-e73f-472f-ad9b-d1236af8095f", - GoToChain: "612e3609-ce9a-4df6-a9a3-63d634d2d934", - }, - { - Comment: `Normalize (match 2 for "Normalize for preservation")`, - AppliesTo: "7509e7dc-1e1b-4dce-8d21-e130515fce73", - GoToChain: "612e3609-ce9a-4df6-a9a3-63d634d2d934", - }, - { - Comment: "Bind PIDs", - AppliesTo: "a2ba5278-459a-4638-92d9-38eb1588717d", - GoToChain: "44a7c397-8187-4fd2-b8f7-c61737c4df49", - }, - { - Comment: "Create SIP(s)", - AppliesTo: "bb194013-597c-4e4a-8493-b36d190f8717", - GoToChain: "61cfa825-120e-4b17-83e6-51a42b67d969", - }, - { - Comment: "Delete packages after extraction", - AppliesTo: "f19926dd-8fb5-4c79-8ade-c83f61f55b40", - GoToChain: "85b1e45d-8f98-4cae-8336-72f40e12cbef", - }, - { - Comment: "Transcribe files (OCR)", - AppliesTo: "82ee9ad2-2c74-4c7c-853e-e4eaf68fc8b6", - GoToChain: "0a24787c-00e3-4710-b324-90e792bfb484", - }, - { - Comment: "Perform file format identification (Transfer)", - AppliesTo: "f09847c2-ee51-429a-9478-a860477f6b8d", - GoToChain: "d97297c7-2b49-4cfe-8c9f-0613d63ed763", - }, - { - Comment: "Store DIP location", - AppliesTo: "cd844b6e-ab3c-4bc6-b34f-7103f88715de", - GoToChain: "/api/v2/location/default/DS/", - }, - { - Comment: "Generate transfer structure report", - AppliesTo: "56eebd45-5600-4768-a8c2-ec0114555a3d", - GoToChain: "e9eaef1e-c2e0-4e3b-b942-bfb537162795", - }, - { - Comment: "Perform policy checks on originals", - AppliesTo: "70fc7040-d4fb-4d19-a0e6-792387ca1006", - GoToChain: "3e891cc4-39d2-4989-a001-5107a009a223", - }, - { - Comment: "Reminder: add metadata if desired", - AppliesTo: "eeb23509-57e2-4529-8857-9d62525db048", - GoToChain: "5727faac-88af-40e8-8c10-268644b0142d", - }, - { - Comment: "Generate thumbnails", - AppliesTo: "498f7a6d-1b8c-431a-aa5d-83f14f3c5e65", - GoToChain: "972fce6c-52c8-4c00-99b9-d6814e377974", - }, - { - Comment: "Select compression algorithm", - AppliesTo: "01d64f58-8295-4b7b-9cab-8f1b153a504f", - GoToChain: "9475447c-9889-430c-9477-6287a9574c5b", - }, - { - Comment: "Store AIP", - AppliesTo: "2d32235c-02d4-4686-88a6-96f4d6c7b1c3", - GoToChain: "9efab23c-31dc-4cbd-a39d-bb1665460cbe", - }, - { - Comment: "Perform policy checks on access derivatives", - AppliesTo: "8ce07e94-6130-4987-96f0-2399ad45c5c2", - GoToChain: "76befd52-14c3-44f9-838f-15a4e01624b0", - }, - { - Comment: "Perform file format identification (Ingest)", - AppliesTo: "7a024896-c4f7-4808-a240-44c87c762bc5", - GoToChain: "5b3c8268-5b33-4b70-b1aa-0e4540fe03d1", - }, - { - Comment: "Perform policy checks on preservation derivatives", - AppliesTo: "153c5f41-3cfb-47ba-9150-2dd44ebc27df", - GoToChain: "b7ce05f0-9d94-4b3e-86cc-d4b2c6dba546", - }, - { - Comment: "Assign UUIDs to directories", - AppliesTo: "bd899573-694e-4d33-8c9b-df0af802437d", - GoToChain: "2dc3f487-e4b0-4e07-a4b3-6216ed24ca14", - }, - { - Comment: "Store AIP location", - AppliesTo: "b320ce81-9982-408a-9502-097d0daa48fa", - GoToChain: "/api/v2/location/default/AS/", - }, - { - Comment: "Document empty directories", - AppliesTo: "d0dfa5fc-e3c2-4638-9eda-f96eea1070e0", - GoToChain: "65273f18-5b4e-4944-af4f-09be175a88e8", - }, - { - Comment: "Extract packages", - AppliesTo: "dec97e3c-5598-4b99-b26e-f87a435a6b7f", - GoToChain: "01d80b27-4ad1-4bd1-8f8d-f819f18bf685", - }, - { - Comment: "Approve normalization", - AppliesTo: "de909a42-c5b5-46e1-9985-c031b50e9d30", - GoToChain: "1e0df175-d56d-450d-8bee-7df1dc7ae815", - }, - { - Comment: "Upload DIP", - AppliesTo: "92879a29-45bf-4f0b-ac43-e64474f0f2f9", - GoToChain: "6eb8ebe7-fab3-4e4c-b9d7-14de17625baa", - }, - { - Comment: "Virus scanning: Yes", - AppliesTo: "856d2d65-cd25-49fa-8da9-cabb78292894", - GoToChain: "6e431096-c403-4cbf-a59a-a26e86be54a8", - }, - { - Comment: "Virus scanning: Yes", - AppliesTo: "1dad74a2-95df-4825-bbba-dca8b91d2371", - GoToChain: "1ac7d792-b63f-46e0-9945-d48d9e5c02c9", - }, - { - Comment: "Virus scanning: Yes", - AppliesTo: "7e81f94e-6441-4430-a12d-76df09181b66", - GoToChain: "97be337c-ff27-4869-bf63-ef1abc9df15d", - }, - { - Comment: "Virus scanning: Yes", - AppliesTo: "390d6507-5029-4dae-bcd4-ce7178c9b560", - GoToChain: "34944d4f-762e-4262-8c79-b9fd48521ca0", - }, - { - Comment: "Virus scanning: Yes", - AppliesTo: "97a5ddc0-d4e0-43ac-a571-9722405a0a9b", - GoToChain: "3e8c0c39-3f30-4c9b-a449-85eef1b2a458", - }, - }, - }, + return e.Flush() } type Choice struct { XMLName xml.Name `xml:"preconfiguredChoice"` Comment string `xml:"-"` - AppliesTo string `xml:"appliesTo"` - GoToChain string `xml:"goToChain"` + AppliesTo string `xml:"appliesTo"` // UUID. + GoToChain string `xml:"goToChain"` // UUID or URI. } func (c Choice) LinkID() uuid.UUID { @@ -319,57 +72,13 @@ func (c Choice) Value() string { return c.GoToChain } -type Choices struct { - XMLName xml.Name `xml:"preconfiguredChoices"` - Choices []Choice `xml:"preconfiguredChoice"` -} - -func (c *Choices) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { - var comment string - for { - t, err := d.Token() - if err != nil { - if err == io.EOF { - break - } - return err - } - switch se := t.(type) { - case xml.Comment: - comment = string(se) - case xml.StartElement: - if se.Name.Local == "preconfiguredChoice" { - var p Choice - err := d.DecodeElement(&p, &se) - if err != nil { - return err - } - p.Comment = strings.TrimSpace(comment) - c.Choices = append(c.Choices, p) - } - } - } - return nil -} - -func (pc Choices) MarshalXML(e *xml.Encoder, start xml.StartElement) error { - for _, p := range pc.Choices { - if err := e.EncodeToken(xml.CharData("\n ")); err != nil { - return err - } - if err := e.EncodeToken(xml.Comment(fmt.Sprintf(" %s ", p.Comment))); err != nil { - return err - } - if err := e.Encode(p); err != nil { - return err - } +func ParseConfigFile(path string) ([]Choice, error) { + blob, err := os.ReadFile(path) + if err != nil { + return nil, err } - return e.Flush() -} -type ProcessingConfig struct { - XMLName xml.Name `xml:"processingMCP"` - Choices Choices `xml:"preconfiguredChoices"` + return ParseConfig(bytes.NewReader(blob)) } func ParseConfig(reader io.Reader) ([]Choice, error) { @@ -384,5 +93,26 @@ func ParseConfig(reader io.Reader) ([]Choice, error) { return nil, err } - return config.Choices.Choices, nil + return config.Choices, nil +} + +func SaveConfigFile(path string, choices []Choice) error { + config := ProcessingConfig{ + Choices: choices, + } + + blob, err := xml.MarshalIndent(config, xmlPrefix, xmlIndent) + if err != nil { + return err + } + + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return err + } + defer f.Close() + + _, err = f.Write(blob) + + return err } diff --git a/hack/ccp/internal/workflow/config_builtins.go b/hack/ccp/internal/workflow/config_builtins.go new file mode 100644 index 000000000..2152aca53 --- /dev/null +++ b/hack/ccp/internal/workflow/config_builtins.go @@ -0,0 +1,287 @@ +package workflow + +import ( + "encoding/xml" + "errors" + "fmt" + "os" + "path/filepath" +) + +var builtinConfigs = map[string]ProcessingConfig{ + "default": DefaultConfig, + "automated": AutomatedConfig, +} + +func InstallBuiltinConfigs(path string) error { + var errs error + for name, config := range builtinConfigs { + path := filepath.Join(path, fmt.Sprintf("%sProcessingMCP.xml", name)) + blob, err := xml.MarshalIndent(config, "", " ") + if err != nil { + errs = errors.Join(errs, fmt.Errorf("encode %s: %v", path, err)) + continue + } + err = os.WriteFile(path, blob, os.FileMode(0o660)) + if err != nil { + errs = errors.Join(errs, fmt.Errorf("write %s: %v", path, err)) + continue + } + } + return errs +} + +var DefaultConfig = ProcessingConfig{ + Choices: []Choice{ + { + Comment: "Select compression level", + AppliesTo: "01c651cb-c174-4ba4-b985-1d87a44d6754", + GoToChain: "414da421-b83f-4648-895f-a34840e3c3f5", + }, + { + Comment: "Perform file format identification (Submission documentation & metadata)", + AppliesTo: "087d27be-c719-47d8-9bbb-9a7d8b609c44", + GoToChain: "4dec164b-79b0-4459-8505-8095af9655b5", + }, + { + Comment: "Bind PIDs", + AppliesTo: "a2ba5278-459a-4638-92d9-38eb1588717d", + GoToChain: "44a7c397-8187-4fd2-b8f7-c61737c4df49", + }, + { + Comment: "Generate transfer structure report", + AppliesTo: "56eebd45-5600-4768-a8c2-ec0114555a3d", + GoToChain: "df54fec1-dae1-4ea6-8d17-a839ee7ac4a7", + }, + { + Comment: "Perform policy checks on originals", + AppliesTo: "70fc7040-d4fb-4d19-a0e6-792387ca1006", + GoToChain: "3e891cc4-39d2-4989-a001-5107a009a223", + }, + { + Comment: "Generate thumbnails", + AppliesTo: "498f7a6d-1b8c-431a-aa5d-83f14f3c5e65", + GoToChain: "c318b224-b718-4535-a911-494b1af6ff26", + }, + { + Comment: "Select compression algorithm", + AppliesTo: "01d64f58-8295-4b7b-9cab-8f1b153a504f", + GoToChain: "9475447c-9889-430c-9477-6287a9574c5b", + }, + { + Comment: "Perform policy checks on access derivatives", + AppliesTo: "8ce07e94-6130-4987-96f0-2399ad45c5c2", + GoToChain: "76befd52-14c3-44f9-838f-15a4e01624b0", + }, + { + Comment: "Perform file format identification (Ingest)", + AppliesTo: "7a024896-c4f7-4808-a240-44c87c762bc5", + GoToChain: "3c1faec7-7e1e-4cdd-b3bd-e2f05f4baa9b", + }, + { + Comment: "Perform policy checks on preservation derivatives", + AppliesTo: "153c5f41-3cfb-47ba-9150-2dd44ebc27df", + GoToChain: "b7ce05f0-9d94-4b3e-86cc-d4b2c6dba546", + }, + { + Comment: "Assign UUIDs to directories", + AppliesTo: "bd899573-694e-4d33-8c9b-df0af802437d", + GoToChain: "891f60d0-1ba8-48d3-b39e-dd0934635d29", + }, + { + Comment: "Document empty directories", + AppliesTo: "d0dfa5fc-e3c2-4638-9eda-f96eea1070e0", + GoToChain: "65273f18-5b4e-4944-af4f-09be175a88e8", + }, + { + Comment: "Virus scanning: Yes", + AppliesTo: "856d2d65-cd25-49fa-8da9-cabb78292894", + GoToChain: "6e431096-c403-4cbf-a59a-a26e86be54a8", + }, + { + Comment: "Virus scanning: Yes", + AppliesTo: "1dad74a2-95df-4825-bbba-dca8b91d2371", + GoToChain: "1ac7d792-b63f-46e0-9945-d48d9e5c02c9", + }, + { + Comment: "Virus scanning: Yes", + AppliesTo: "7e81f94e-6441-4430-a12d-76df09181b66", + GoToChain: "97be337c-ff27-4869-bf63-ef1abc9df15d", + }, + { + Comment: "Virus scanning: Yes", + AppliesTo: "390d6507-5029-4dae-bcd4-ce7178c9b560", + GoToChain: "34944d4f-762e-4262-8c79-b9fd48521ca0", + }, + { + Comment: "Virus scanning: Yes", + AppliesTo: "97a5ddc0-d4e0-43ac-a571-9722405a0a9b", + GoToChain: "3e8c0c39-3f30-4c9b-a449-85eef1b2a458", + }, + }, +} + +var AutomatedConfig = ProcessingConfig{ + Choices: []Choice{ + { + Comment: "Store DIP", + AppliesTo: "5e58066d-e113-4383-b20b-f301ed4d751c", + GoToChain: "8d29eb3d-a8a8-4347-806e-3d8227ed44a1", + }, + { + Comment: "Select compression level", + AppliesTo: "01c651cb-c174-4ba4-b985-1d87a44d6754", + GoToChain: "414da421-b83f-4648-895f-a34840e3c3f5", + }, + { + Comment: "Examine contents", + AppliesTo: "accea2bf-ba74-4a3a-bb97-614775c74459", + GoToChain: "e0a39199-c62a-4a2f-98de-e9d1116460a8", + }, + { + Comment: "Perform file format identification (Submission documentation & metadata)", + AppliesTo: "087d27be-c719-47d8-9bbb-9a7d8b609c44", + GoToChain: "4dec164b-79b0-4459-8505-8095af9655b5", + }, + { + Comment: `Normalize (match 1 for "Normalize for preservation")`, + AppliesTo: "cb8e5706-e73f-472f-ad9b-d1236af8095f", + GoToChain: "612e3609-ce9a-4df6-a9a3-63d634d2d934", + }, + { + Comment: `Normalize (match 2 for "Normalize for preservation")`, + AppliesTo: "7509e7dc-1e1b-4dce-8d21-e130515fce73", + GoToChain: "612e3609-ce9a-4df6-a9a3-63d634d2d934", + }, + { + Comment: "Bind PIDs", + AppliesTo: "a2ba5278-459a-4638-92d9-38eb1588717d", + GoToChain: "44a7c397-8187-4fd2-b8f7-c61737c4df49", + }, + { + Comment: "Create SIP(s)", + AppliesTo: "bb194013-597c-4e4a-8493-b36d190f8717", + GoToChain: "61cfa825-120e-4b17-83e6-51a42b67d969", + }, + { + Comment: "Delete packages after extraction", + AppliesTo: "f19926dd-8fb5-4c79-8ade-c83f61f55b40", + GoToChain: "85b1e45d-8f98-4cae-8336-72f40e12cbef", + }, + { + Comment: "Transcribe files (OCR)", + AppliesTo: "82ee9ad2-2c74-4c7c-853e-e4eaf68fc8b6", + GoToChain: "0a24787c-00e3-4710-b324-90e792bfb484", + }, + { + Comment: "Perform file format identification (Transfer)", + AppliesTo: "f09847c2-ee51-429a-9478-a860477f6b8d", + GoToChain: "d97297c7-2b49-4cfe-8c9f-0613d63ed763", + }, + { + Comment: "Store DIP location", + AppliesTo: "cd844b6e-ab3c-4bc6-b34f-7103f88715de", + GoToChain: "/api/v2/location/default/DS/", + }, + { + Comment: "Generate transfer structure report", + AppliesTo: "56eebd45-5600-4768-a8c2-ec0114555a3d", + GoToChain: "e9eaef1e-c2e0-4e3b-b942-bfb537162795", + }, + { + Comment: "Perform policy checks on originals", + AppliesTo: "70fc7040-d4fb-4d19-a0e6-792387ca1006", + GoToChain: "3e891cc4-39d2-4989-a001-5107a009a223", + }, + { + Comment: "Reminder: add metadata if desired", + AppliesTo: "eeb23509-57e2-4529-8857-9d62525db048", + GoToChain: "5727faac-88af-40e8-8c10-268644b0142d", + }, + { + Comment: "Generate thumbnails", + AppliesTo: "498f7a6d-1b8c-431a-aa5d-83f14f3c5e65", + GoToChain: "972fce6c-52c8-4c00-99b9-d6814e377974", + }, + { + Comment: "Select compression algorithm", + AppliesTo: "01d64f58-8295-4b7b-9cab-8f1b153a504f", + GoToChain: "9475447c-9889-430c-9477-6287a9574c5b", + }, + { + Comment: "Store AIP", + AppliesTo: "2d32235c-02d4-4686-88a6-96f4d6c7b1c3", + GoToChain: "9efab23c-31dc-4cbd-a39d-bb1665460cbe", + }, + { + Comment: "Perform policy checks on access derivatives", + AppliesTo: "8ce07e94-6130-4987-96f0-2399ad45c5c2", + GoToChain: "76befd52-14c3-44f9-838f-15a4e01624b0", + }, + { + Comment: "Perform file format identification (Ingest)", + AppliesTo: "7a024896-c4f7-4808-a240-44c87c762bc5", + GoToChain: "5b3c8268-5b33-4b70-b1aa-0e4540fe03d1", + }, + { + Comment: "Perform policy checks on preservation derivatives", + AppliesTo: "153c5f41-3cfb-47ba-9150-2dd44ebc27df", + GoToChain: "b7ce05f0-9d94-4b3e-86cc-d4b2c6dba546", + }, + { + Comment: "Assign UUIDs to directories", + AppliesTo: "bd899573-694e-4d33-8c9b-df0af802437d", + GoToChain: "2dc3f487-e4b0-4e07-a4b3-6216ed24ca14", + }, + { + Comment: "Store AIP location", + AppliesTo: "b320ce81-9982-408a-9502-097d0daa48fa", + GoToChain: "/api/v2/location/default/AS/", + }, + { + Comment: "Document empty directories", + AppliesTo: "d0dfa5fc-e3c2-4638-9eda-f96eea1070e0", + GoToChain: "65273f18-5b4e-4944-af4f-09be175a88e8", + }, + { + Comment: "Extract packages", + AppliesTo: "dec97e3c-5598-4b99-b26e-f87a435a6b7f", + GoToChain: "01d80b27-4ad1-4bd1-8f8d-f819f18bf685", + }, + { + Comment: "Approve normalization", + AppliesTo: "de909a42-c5b5-46e1-9985-c031b50e9d30", + GoToChain: "1e0df175-d56d-450d-8bee-7df1dc7ae815", + }, + { + Comment: "Upload DIP", + AppliesTo: "92879a29-45bf-4f0b-ac43-e64474f0f2f9", + GoToChain: "6eb8ebe7-fab3-4e4c-b9d7-14de17625baa", + }, + { + Comment: "Virus scanning: Yes", + AppliesTo: "856d2d65-cd25-49fa-8da9-cabb78292894", + GoToChain: "6e431096-c403-4cbf-a59a-a26e86be54a8", + }, + { + Comment: "Virus scanning: Yes", + AppliesTo: "1dad74a2-95df-4825-bbba-dca8b91d2371", + GoToChain: "1ac7d792-b63f-46e0-9945-d48d9e5c02c9", + }, + { + Comment: "Virus scanning: Yes", + AppliesTo: "7e81f94e-6441-4430-a12d-76df09181b66", + GoToChain: "97be337c-ff27-4869-bf63-ef1abc9df15d", + }, + { + Comment: "Virus scanning: Yes", + AppliesTo: "390d6507-5029-4dae-bcd4-ce7178c9b560", + GoToChain: "34944d4f-762e-4262-8c79-b9fd48521ca0", + }, + { + Comment: "Virus scanning: Yes", + AppliesTo: "97a5ddc0-d4e0-43ac-a571-9722405a0a9b", + GoToChain: "3e8c0c39-3f30-4c9b-a449-85eef1b2a458", + }, + }, +} diff --git a/hack/ccp/internal/workflow/config_builtins_test.go b/hack/ccp/internal/workflow/config_builtins_test.go new file mode 100644 index 000000000..3b9948c83 --- /dev/null +++ b/hack/ccp/internal/workflow/config_builtins_test.go @@ -0,0 +1,31 @@ +package workflow_test + +import ( + "testing" + + "gotest.tools/v3/assert" + "gotest.tools/v3/fs" + + "github.com/artefactual/archivematica/hack/ccp/internal/workflow" +) + +func TestInstallBuiltinConfigs(t *testing.T) { + tmpDir := fs.NewDir(t, "") + + err := workflow.InstallBuiltinConfigs(tmpDir.Path()) + assert.NilError(t, err) + + choices, err := workflow.ParseConfigFile(tmpDir.Join("defaultProcessingMCP.xml")) + assert.NilError(t, err) + assert.Equal(t, len(choices), 17, "unexpected number of preconfigured choices found in the default config") + + choices, err = workflow.ParseConfigFile(tmpDir.Join("automatedProcessingMCP.xml")) + assert.NilError(t, err) + assert.Equal(t, len(choices), 32, "unexpected number of preconfigured choices found in the automated config") + + expected := fs.Expected(t, + fs.WithFile("automatedProcessingMCP.xml", "", fs.MatchAnyFileContent, fs.MatchAnyFileMode), + fs.WithFile("defaultProcessingMCP.xml", "", fs.MatchAnyFileContent, fs.MatchAnyFileMode), + ) + assert.Assert(t, fs.Equal(tmpDir.Path(), expected)) +} diff --git a/hack/ccp/internal/workflow/config_test.go b/hack/ccp/internal/workflow/config_test.go index 4a53823c2..0964fb18b 100644 --- a/hack/ccp/internal/workflow/config_test.go +++ b/hack/ccp/internal/workflow/config_test.go @@ -1,20 +1,49 @@ package workflow_test import ( - "os" "testing" + "github.com/google/uuid" "gotest.tools/v3/assert" + "gotest.tools/v3/fs" "github.com/artefactual/archivematica/hack/ccp/internal/workflow" ) -func TestParseConfig(t *testing.T) { - f, err := os.Open("../../hack/processingMCP.xml") +func TestParseConfigFile(t *testing.T) { + choices, err := workflow.ParseConfigFile("../../hack/processingMCP.xml") assert.NilError(t, err) - t.Cleanup(func() { f.Close() }) - config, err := workflow.ParseConfig(f) + assert.Equal(t, len(choices), 32, "unexpected number of preconfigured choices found") + + assert.Equal(t, choices[0].Comment, "") // TODO: we're not preserving the comment yet. + assert.Equal(t, choices[0].LinkID(), uuid.MustParse("5e58066d-e113-4383-b20b-f301ed4d751c")) + assert.Equal(t, choices[0].ChainID(), uuid.MustParse("8d29eb3d-a8a8-4347-806e-3d8227ed44a1")) + assert.Equal(t, choices[0].Value(), "8d29eb3d-a8a8-4347-806e-3d8227ed44a1") +} + +func TestSaveConfigFile(t *testing.T) { + dir := fs.NewDir(t, "") + + err := workflow.SaveConfigFile(dir.Join("processingMCP.xml"), []workflow.Choice{ + { + Comment: "Store DIP", + AppliesTo: "5e58066d-e113-4383-b20b-f301ed4d751c", + GoToChain: "8d29eb3d-a8a8-4347-806e-3d8227ed44a1", + }, + }) assert.NilError(t, err) - assert.Equal(t, len(config), 32) + + expected := fs.Expected(t, + fs.WithFile("processingMCP.xml", ` + + + + 5e58066d-e113-4383-b20b-f301ed4d751c + 8d29eb3d-a8a8-4347-806e-3d8227ed44a1 + + +`), + ) + assert.Assert(t, fs.Equal(dir.Path(), expected)) } diff --git a/hack/ccp/internal/workflow/workflow.go b/hack/ccp/internal/workflow/workflow.go index 4ab587f17..c48df49a6 100644 --- a/hack/ccp/internal/workflow/workflow.go +++ b/hack/ccp/internal/workflow/workflow.go @@ -202,7 +202,7 @@ type LinkMicroServiceChainChoice struct { } type ConfigReplacement struct { - ID string `json:"id"` + ID uuid.UUID `json:"id"` Description I18nField `json:"description"` Items map[string]string `json:"items"` } diff --git a/hack/ccp/proto/archivematica/ccp/admin/v1beta1/admin.proto b/hack/ccp/proto/archivematica/ccp/admin/v1beta1/admin.proto index d4be9f0b4..70b42c2ee 100644 --- a/hack/ccp/proto/archivematica/ccp/admin/v1beta1/admin.proto +++ b/hack/ccp/proto/archivematica/ccp/admin/v1beta1/admin.proto @@ -103,7 +103,7 @@ message CreatePackageRequest { message CreatePackageResponse { // Identifier of the package as a string (UUIDv4). - string id = 1; + string id = 1 [(buf.validate.field).string.uuid = true]; } message ReadPackageRequest { @@ -113,6 +113,10 @@ message ReadPackageRequest { message ReadPackageResponse { Package pkg = 1; + + // A decision that needs to be resolved. Must be set in the package status is + // PACKAGE_STATUS_AWAITING_DECISION. + optional Decision decision = 2; } message ApproveTransferRequest { @@ -125,22 +129,30 @@ message ApproveTransferRequest { message ApproveTransferResponse { // Identifier of the package as a string (UUIDv4). - string id = 1; + string id = 1 [(buf.validate.field).string.uuid = true]; } message ListActivePackagesRequest {} message ListActivePackagesResponse { + // List of active packages, referred by their identifiers. repeated string value = 1; } message ListAwaitingDecisionsRequest {} message ListAwaitingDecisionsResponse { + // List of packages awaiting decisions, referred by their identifiers. repeated string value = 1; } -message ResolveAwaitingDecisionRequest {} +message ResolveAwaitingDecisionRequest { + // Identifier of the package as a string (UUIDv4). + string id = 1 [(buf.validate.field).string.uuid = true]; + + // The choice to be used to resolve the decision. + Choice choice = 2; +} message ResolveAwaitingDecisionResponse {} @@ -163,6 +175,9 @@ enum PackageStatus { PACKAGE_STATUS_DONE = 2; PACKAGE_STATUS_COMPLETED_SUCCESSFULLY = 3; PACKAGE_STATUS_FAILED = 4; + + // This is not found in the database but we can infer it at runtime. + PACKAGE_STATUS_AWAITING_DECISION = 5; } message Package { @@ -178,3 +193,17 @@ message Package { // Status of the package. PackageStatus status = 4; } + +message Decision { + // Name of the decision (prompt). + string name = 1 [(buf.validate.field).string.min_len = 1]; + + // Choices available. + repeated Choice choice = 2 [(buf.validate.field).repeated = {min_items: 1}]; +} + +message Choice { + int32 id = 1 [(buf.validate.field).int32 = {gte: 0}]; + + string label = 2 [(buf.validate.field).string.min_len = 1]; +} diff --git a/hack/ccp/proto/buf.lock b/hack/ccp/proto/buf.lock deleted file mode 100644 index 53f951294..000000000 --- a/hack/ccp/proto/buf.lock +++ /dev/null @@ -1,8 +0,0 @@ -# Generated by buf. DO NOT EDIT. -version: v1 -deps: - - remote: buf.build - owner: bufbuild - repository: protovalidate - commit: 46a4cf4ba1094a34bcd89a6c67163b4b - digest: shake256:436ce453801917c11bc7b21d66bcfae87da2aceb804a041487be1e51dc9fbc219e61ea6a552db7a7aa6d63bb5efd0f3ed5fe3d4c42d4f750d0eb35f14144e3b6 diff --git a/hack/ccp/proto/buf.yaml b/hack/ccp/proto/buf.yaml deleted file mode 100644 index 2cd71fef7..000000000 --- a/hack/ccp/proto/buf.yaml +++ /dev/null @@ -1,10 +0,0 @@ -version: v1 -name: buf.build/artefactual/archivematica -deps: - - buf.build/bufbuild/protovalidate -lint: - use: - - DEFAULT -breaking: - use: - - FILE diff --git a/hack/ccp/web/src/gen/archivematica/ccp/admin/v1beta1/admin_pb.ts b/hack/ccp/web/src/gen/archivematica/ccp/admin/v1beta1/admin_pb.ts index 62881f437..b5f078fc4 100644 --- a/hack/ccp/web/src/gen/archivematica/ccp/admin/v1beta1/admin_pb.ts +++ b/hack/ccp/web/src/gen/archivematica/ccp/admin/v1beta1/admin_pb.ts @@ -1,4 +1,4 @@ -// @generated by protoc-gen-es v1.9.0 with parameter "target=ts" +// @generated by protoc-gen-es v1.10.0 with parameter "target=ts" // @generated from file archivematica/ccp/admin/v1beta1/admin.proto (package archivematica.ccp.admin.v1beta1, syntax proto3) /* eslint-disable */ // @ts-nocheck @@ -98,6 +98,13 @@ export enum PackageStatus { * @generated from enum value: PACKAGE_STATUS_FAILED = 4; */ FAILED = 4, + + /** + * This is not found in the database but we can infer it at runtime. + * + * @generated from enum value: PACKAGE_STATUS_AWAITING_DECISION = 5; + */ + AWAITING_DECISION = 5, } // Retrieve enum metadata with: proto3.getEnumType(PackageStatus) proto3.util.setEnumType(PackageStatus, "archivematica.ccp.admin.v1beta1.PackageStatus", [ @@ -106,6 +113,7 @@ proto3.util.setEnumType(PackageStatus, "archivematica.ccp.admin.v1beta1.PackageS { no: 2, name: "PACKAGE_STATUS_DONE" }, { no: 3, name: "PACKAGE_STATUS_COMPLETED_SUCCESSFULLY" }, { no: 4, name: "PACKAGE_STATUS_FAILED" }, + { no: 5, name: "PACKAGE_STATUS_AWAITING_DECISION" }, ]); /** @@ -293,6 +301,14 @@ export class ReadPackageResponse extends Message { */ pkg?: Package; + /** + * A decision that needs to be resolved. Must be set in the package status is + * PACKAGE_STATUS_AWAITING_DECISION. + * + * @generated from field: optional archivematica.ccp.admin.v1beta1.Decision decision = 2; + */ + decision?: Decision; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -302,6 +318,7 @@ export class ReadPackageResponse extends Message { static readonly typeName = "archivematica.ccp.admin.v1beta1.ReadPackageResponse"; static readonly fields: FieldList = proto3.util.newFieldList(() => [ { no: 1, name: "pkg", kind: "message", T: Package }, + { no: 2, name: "decision", kind: "message", T: Decision, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): ReadPackageResponse { @@ -443,6 +460,8 @@ export class ListActivePackagesRequest extends Message { /** + * List of active packages, referred by their identifiers. + * * @generated from field: repeated string value = 1; */ value: string[] = []; @@ -511,6 +530,8 @@ export class ListAwaitingDecisionsRequest extends Message { /** + * List of packages awaiting decisions, referred by their identifiers. + * * @generated from field: repeated string value = 1; */ value: string[] = []; @@ -547,6 +568,20 @@ export class ListAwaitingDecisionsResponse extends Message { + /** + * Identifier of the package as a string (UUIDv4). + * + * @generated from field: string id = 1; + */ + id = ""; + + /** + * The choice to be used to resolve the decision. + * + * @generated from field: archivematica.ccp.admin.v1beta1.Choice choice = 2; + */ + choice?: Choice; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -555,6 +590,8 @@ export class ResolveAwaitingDecisionRequest extends Message [ + { no: 1, name: "id", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 2, name: "choice", kind: "message", T: Choice }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): ResolveAwaitingDecisionRequest { @@ -668,3 +705,93 @@ export class Package extends Message { } } +/** + * @generated from message archivematica.ccp.admin.v1beta1.Decision + */ +export class Decision extends Message { + /** + * Name of the decision (prompt). + * + * @generated from field: string name = 1; + */ + name = ""; + + /** + * Choices available. + * + * @generated from field: repeated archivematica.ccp.admin.v1beta1.Choice choice = 2; + */ + choice: Choice[] = []; + + constructor(data?: PartialMessage) { + super(); + proto3.util.initPartial(data, this); + } + + static readonly runtime: typeof proto3 = proto3; + static readonly typeName = "archivematica.ccp.admin.v1beta1.Decision"; + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: "name", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 2, name: "choice", kind: "message", T: Choice, repeated: true }, + ]); + + static fromBinary(bytes: Uint8Array, options?: Partial): Decision { + return new Decision().fromBinary(bytes, options); + } + + static fromJson(jsonValue: JsonValue, options?: Partial): Decision { + return new Decision().fromJson(jsonValue, options); + } + + static fromJsonString(jsonString: string, options?: Partial): Decision { + return new Decision().fromJsonString(jsonString, options); + } + + static equals(a: Decision | PlainMessage | undefined, b: Decision | PlainMessage | undefined): boolean { + return proto3.util.equals(Decision, a, b); + } +} + +/** + * @generated from message archivematica.ccp.admin.v1beta1.Choice + */ +export class Choice extends Message { + /** + * @generated from field: int32 id = 1; + */ + id = 0; + + /** + * @generated from field: string label = 2; + */ + label = ""; + + constructor(data?: PartialMessage) { + super(); + proto3.util.initPartial(data, this); + } + + static readonly runtime: typeof proto3 = proto3; + static readonly typeName = "archivematica.ccp.admin.v1beta1.Choice"; + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: "id", kind: "scalar", T: 5 /* ScalarType.INT32 */ }, + { no: 2, name: "label", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + ]); + + static fromBinary(bytes: Uint8Array, options?: Partial): Choice { + return new Choice().fromBinary(bytes, options); + } + + static fromJson(jsonValue: JsonValue, options?: Partial): Choice { + return new Choice().fromJson(jsonValue, options); + } + + static fromJsonString(jsonString: string, options?: Partial): Choice { + return new Choice().fromJsonString(jsonString, options); + } + + static equals(a: Choice | PlainMessage | undefined, b: Choice | PlainMessage | undefined): boolean { + return proto3.util.equals(Choice, a, b); + } +} +