diff --git a/config/dr-cluster/rbac/role.yaml b/config/dr-cluster/rbac/role.yaml index 47ef42d6ae..c294be8032 100644 --- a/config/dr-cluster/rbac/role.yaml +++ b/config/dr-cluster/rbac/role.yaml @@ -235,6 +235,26 @@ rules: - get - list - watch +- apiGroups: + - replication.storage.openshift.io + resources: + - volumegroupreplicationclasses + verbs: + - get + - list + - watch +- apiGroups: + - replication.storage.openshift.io + resources: + - volumegroupreplications + verbs: + - create + - update + - delete + - get + - list + - watch + - patch - apiGroups: - storage.k8s.io resources: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6646480582..eb0de6094f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -460,6 +460,26 @@ rules: - get - patch - update +- apiGroups: + - replication.storage.openshift.io + resources: + - volumegroupreplicationclasses + verbs: + - get + - list + - watch +- apiGroups: + - replication.storage.openshift.io + resources: + - volumegroupreplications + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - replication.storage.openshift.io resources: diff --git a/e2e/go.mod b/e2e/go.mod index 51957ff19b..fb0103364a 100644 --- a/e2e/go.mod +++ b/e2e/go.mod @@ -1,6 +1,8 @@ module github.com/ramendr/ramen/e2e -go 1.21 +go 1.22 + +toolchain go1.22.5 require ( github.com/go-logr/logr v1.4.1 diff --git a/go.mod b/go.mod index 06e004234e..2aa5933871 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/ramendr/ramen -go 1.21 +go 1.22.0 + +toolchain go1.22.5 // This replace should always be here for ease of development. replace github.com/ramendr/ramen/api => ./api @@ -8,58 +10,58 @@ replace github.com/ramendr/ramen/api => ./api require ( github.com/aws/aws-sdk-go v1.44.289 github.com/backube/volsync v0.7.1 - github.com/csi-addons/kubernetes-csi-addons v0.8.0 - github.com/go-logr/logr v1.3.0 - github.com/google/uuid v1.3.1 + github.com/csi-addons/kubernetes-csi-addons v0.8.1-0.20240709090842-7d3f7a5e96f0 + github.com/go-logr/logr v1.4.2 + github.com/google/uuid v1.6.0 github.com/kubernetes-csi/external-snapshotter/client/v7 v7.0.0 - github.com/onsi/ginkgo/v2 v2.13.0 - github.com/onsi/gomega v1.30.0 + github.com/onsi/ginkgo/v2 v2.17.2 + github.com/onsi/gomega v1.33.1 github.com/open-cluster-management-io/api v0.0.0-00010101000000-000000000000 github.com/open-cluster-management/api v0.0.0-20210527013639-a6845f2ebcb1 github.com/operator-framework/api v0.17.6 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.16.0 + github.com/prometheus/client_golang v1.18.0 github.com/ramendr/ramen/api v0.0.0-20240117171503-e11c56eac24d github.com/ramendr/recipe v0.0.0-20230817160432-729dc7fd8932 github.com/stolostron/multicloud-operators-foundation v0.0.0-20220824091202-e9cd9710d009 github.com/stolostron/multicloud-operators-placementrule v1.2.4-1-20220311-8eedb3f.0.20230828200208-cd3c119a7fa0 github.com/vmware-tanzu/velero v1.9.1 - go.uber.org/zap v1.26.0 + go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 golang.org/x/time v0.3.0 - k8s.io/api v0.29.0 - k8s.io/apiextensions-apiserver v0.28.3 - k8s.io/apimachinery v0.29.0 + k8s.io/api v0.30.2 + k8s.io/apiextensions-apiserver v0.30.1 + k8s.io/apimachinery v0.30.2 k8s.io/client-go v12.0.0+incompatible - k8s.io/component-base v0.29.0 - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 + k8s.io/component-base v0.30.1 + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 open-cluster-management.io/config-policy-controller v0.12.0 open-cluster-management.io/governance-policy-propagator v0.12.0 - sigs.k8s.io/controller-runtime v0.16.3 - sigs.k8s.io/yaml v1.3.0 + sigs.k8s.io/controller-runtime v0.18.4 + sigs.k8s.io/yaml v1.4.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/csi-addons/spec v0.2.1-0.20230606140122-d20966d2e444 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/csi-addons/spec v0.2.1-0.20240619103729-12c61f25a2a5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch/v5 v5.6.0 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/zapr v1.2.4 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20230510103437-eeec1cb781c3 // indirect + github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -68,15 +70,15 @@ require ( github.com/kubernetes-csi/external-snapshotter/client/v6 v6.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/afero v1.9.3 // indirect @@ -86,22 +88,21 @@ require ( github.com/spf13/viper v1.15.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.18.0 // indirect - golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/sys v0.14.0 // indirect - golang.org/x/term v0.14.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.12.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/oauth2 v0.20.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/tools v0.20.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect - google.golang.org/grpc v1.59.0 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/grpc v1.65.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/klog/v2 v2.110.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect open-cluster-management.io/api v0.11.1-0.20230905055724-cf1ead467a83 // indirect open-cluster-management.io/multicloud-operators-subscription v0.12.0 // indirect @@ -110,6 +111,6 @@ require ( ) // replace directives to accommodate for stolostron -replace k8s.io/client-go v12.0.0+incompatible => k8s.io/client-go v0.29.0 +replace k8s.io/client-go v12.0.0+incompatible => k8s.io/client-go v0.30.2 replace github.com/open-cluster-management-io/api => open-cluster-management.io/api v0.10.0 diff --git a/go.sum b/go.sum index dc11988d71..8d1db0e546 100644 --- a/go.sum +++ b/go.sum @@ -55,14 +55,13 @@ github.com/aws/aws-sdk-go v1.44.289 h1:5CVEjiHFvdiVlKPBzv0rjG4zH/21W/onT18R5AH/q github.com/aws/aws-sdk-go v1.44.289/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/backube/volsync v0.7.1 h1:GK3MEY9qtZ99ykP/3wl1shVYHKW20nSIfWR3pC3F3RE= github.com/backube/volsync v0.7.1/go.mod h1:IIFJXj+R8h1fodOF2VlVhCEYDE/g4NjvF1D5KrjnCaI= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -71,10 +70,10 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/csi-addons/kubernetes-csi-addons v0.8.0 h1:zvYGp4DM6KdQzEX3dQSYKykqJdLZlxpVBJjtpbaqFjs= -github.com/csi-addons/kubernetes-csi-addons v0.8.0/go.mod h1:dvinzoiXlqdOGDpKkYx8Jxl507BzVEEEO+SI0OmBaRI= -github.com/csi-addons/spec v0.2.1-0.20230606140122-d20966d2e444 h1:hWVCrZWVHctpWt6cQxV1I6dW3wpBDMg3Vrvu9uAuUxw= -github.com/csi-addons/spec v0.2.1-0.20230606140122-d20966d2e444/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI= +github.com/csi-addons/kubernetes-csi-addons v0.8.1-0.20240709090842-7d3f7a5e96f0 h1:h+Fdpkho5VdhqKPXGssZsqZbx9H/p//EI4xNQI+YEbQ= +github.com/csi-addons/kubernetes-csi-addons v0.8.1-0.20240709090842-7d3f7a5e96f0/go.mod h1:emzFhYlYqZMrDZeZBrd1rUs1Bc+B7uD1Z9wndjXsFw8= +github.com/csi-addons/spec v0.2.1-0.20240619103729-12c61f25a2a5 h1:/pXa+X+YKDPRI2JG8WEnxGKk6PcVZRhcLqdPks+bQa8= +github.com/csi-addons/spec v0.2.1-0.20240619103729-12c61f25a2a5/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -95,26 +94,25 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= -github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= -github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -128,8 +126,8 @@ github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -195,13 +193,13 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20230510103437-eeec1cb781c3 h1:2XF1Vzq06X+inNqgJ9tRnGuw+ZVCB3FazXODD6JE1R8= -github.com/google/pprof v0.0.0-20230510103437-eeec1cb781c3/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= @@ -217,7 +215,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -255,8 +252,8 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -278,12 +275,12 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= -github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/ginkgo/v2 v2.17.2 h1:7eMhcy3GimbsA3hEnVKdw/PQM9XN9krpKVXsZdph0/g= +github.com/onsi/ginkgo/v2 v2.17.2/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= -github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/open-cluster-management/api v0.0.0-20210527013639-a6845f2ebcb1 h1:AaFycHD9YOfFXe9C5VsYxKf4LKCXKSLZgK2DnFdHY4M= github.com/open-cluster-management/api v0.0.0-20210527013639-a6845f2ebcb1/go.mod h1:ot+A1DWq+v1IV+e1S7nhIteYAmNByFgtazvzpoeAfRQ= github.com/openshift/build-machinery-go v0.0.0-20210115170933-e575b44a7a94/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE= @@ -292,21 +289,20 @@ github.com/operator-framework/api v0.17.6/go.mod h1:l/cuwtPxkVUY7fzYgdust2m9tlmb github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/ramendr/recipe v0.0.0-20230817160432-729dc7fd8932 h1:n89W9K2gDa0XwdIVuWyg53hPgaR97DfGVi9o2V0WcWA= github.com/ramendr/recipe v0.0.0-20230817160432-729dc7fd8932/go.mod h1:QHVQXKgNId8EfvNd+Y6JcTrsXwTImtSFkV4IsiOkwCw= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= @@ -345,8 +341,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/vmware-tanzu/velero v1.9.1 h1:uZhNMq1Pn8AZjT7HLtKseTq47EeHeIuUxvGPFFp/+Vs= @@ -355,7 +351,6 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -363,16 +358,12 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -418,10 +409,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -457,11 +445,10 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -471,8 +458,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -483,7 +470,6 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -521,22 +507,19 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -545,8 +528,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -606,10 +589,9 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= +golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -641,7 +623,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -679,8 +660,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -697,8 +678,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -709,8 +690,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -740,29 +721,29 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.20.0/go.mod h1:HyLC5l5eoS/ygQYl1BXBgFzWNlkHiAuyNAbevIn+FKg= -k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A= -k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA= -k8s.io/apiextensions-apiserver v0.28.3 h1:Od7DEnhXHnHPZG+W9I97/fSQkVpVPQx2diy+2EtmY08= -k8s.io/apiextensions-apiserver v0.28.3/go.mod h1:NE1XJZ4On0hS11aWWJUTNkmVB03j9LM7gJSisbRt8Lc= +k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= +k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= +k8s.io/apiextensions-apiserver v0.30.1 h1:4fAJZ9985BmpJG6PkoxVRpXv9vmPUOVzl614xarePws= +k8s.io/apiextensions-apiserver v0.30.1/go.mod h1:R4GuSrlhgq43oRY9sF2IToFh7PVlF1JjfWdoG3pixk4= k8s.io/apimachinery v0.20.0/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= -k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= +k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= +k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= k8s.io/client-go v0.20.0/go.mod h1:4KWh/g+Ocd8KkCwKF8vUNnmqgv+EVnQDK4MBF4oB5tY= -k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8= -k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38= +k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50= +k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs= k8s.io/code-generator v0.20.0/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= -k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s= -k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M= +k8s.io/component-base v0.30.1 h1:bvAtlPh1UrdaZL20D9+sWxsJljMi0QZ3Lmw+kmZAaxQ= +k8s.io/component-base v0.30.1/go.mod h1:e/X9kDiOebwlI41AvBHuWdqFriSRrX50CdwA9TFaHLI= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/kubectl v0.26.1 h1:K8A0Jjlwg8GqrxOXxAbjY5xtmXYeYjLU96cHp2WMQ7s= k8s.io/kubectl v0.26.1/go.mod h1:miYFVzldVbdIiXMrHZYmL/EDWwJKM+F0sSsdxsATFPo= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -781,8 +762,8 @@ open-cluster-management.io/multicloud-operators-subscription v0.12.0/go.mod h1:+ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= -sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= +sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= +sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= @@ -790,5 +771,5 @@ sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+s sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/hack/test/replication.storage.openshift.io_volumegroupreplicationclasses.yaml b/hack/test/replication.storage.openshift.io_volumegroupreplicationclasses.yaml new file mode 100644 index 0000000000..b7e6b1bf4e --- /dev/null +++ b/hack/test/replication.storage.openshift.io_volumegroupreplicationclasses.yaml @@ -0,0 +1,76 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: volumegroupreplicationclasses.replication.storage.openshift.io +spec: + group: replication.storage.openshift.io + names: + kind: VolumeGroupReplicationClass + listKind: VolumeGroupReplicationClassList + plural: volumegroupreplicationclasses + singular: volumegroupreplicationclass + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: VolumeGroupReplicationClass is the Schema for the volumegroupreplicationclasses + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + VolumeGroupReplicationClassSpec specifies parameters that an underlying storage system uses + when creating a volumegroup replica. A specific VolumeGroupReplicationClass is used by specifying + its name in a VolumeGroupReplication object. + properties: + parameters: + additionalProperties: + type: string + description: |- + Parameters is a key-value map with storage provisioner specific configurations for + creating volume group replicas + type: object + x-kubernetes-validations: + - message: parameters are immutable + rule: self == oldSelf + provisioner: + description: Provisioner is the name of storage provisioner + type: string + x-kubernetes-validations: + - message: provisioner is immutable + rule: self == oldSelf + required: + - provisioner + type: object + x-kubernetes-validations: + - message: parameters are immutable + rule: has(self.parameters) == has(oldSelf.parameters) + status: + description: VolumeGroupReplicationClassStatus defines the observed state + of VolumeGroupReplicationClass + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/hack/test/replication.storage.openshift.io_volumegroupreplicationcontents.yaml b/hack/test/replication.storage.openshift.io_volumegroupreplicationcontents.yaml new file mode 100644 index 0000000000..be67a9e331 --- /dev/null +++ b/hack/test/replication.storage.openshift.io_volumegroupreplicationcontents.yaml @@ -0,0 +1,176 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: volumegroupreplicationcontents.replication.storage.openshift.io +spec: + group: replication.storage.openshift.io + names: + kind: VolumeGroupReplicationContent + listKind: VolumeGroupReplicationContentList + plural: volumegroupreplicationcontents + singular: volumegroupreplicationcontent + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: VolumeGroupReplicationContent is the Schema for the volumegroupreplicationcontents + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: VolumeGroupReplicationContentSpec defines the desired state + of VolumeGroupReplicationContent + properties: + provisioner: + description: |- + provisioner is the name of the CSI driver used to create the physical + volume group on + the underlying storage system. + This MUST be the same as the name returned by the CSI GetPluginName() call for + that driver. + Required. + type: string + source: + description: |- + Source specifies whether the snapshot is (or should be) dynamically provisioned + or already exists, and just requires a Kubernetes object representation. + This field is immutable after creation. + Required. + properties: + volumeHandles: + description: |- + VolumeHandles is a list of volume handles on the backend to be grouped + and replicated. + items: + type: string + type: array + required: + - volumeHandles + type: object + volumeGroupReplicationClassName: + description: |- + VolumeGroupReplicationClassName is the name of the VolumeGroupReplicationClass from + which this group replication was (or will be) created. + type: string + volumeGroupReplicationHandle: + description: |- + VolumeGroupReplicationHandle is a unique id returned by the CSI driver + to identify the VolumeGroupReplication on the storage system. + type: string + volumeGroupReplicationRef: + description: |- + VolumeGroupreplicationRef specifies the VolumeGroupReplication object to which this + VolumeGroupReplicationContent object is bound. + VolumeGroupReplication.Spec.VolumeGroupReplicationContentName field must reference to + this VolumeGroupReplicationContent's name for the bidirectional binding to be valid. + For a pre-existing VolumeGroupReplicationContent object, name and namespace of the + VolumeGroupReplication object MUST be provided for binding to happen. + This field is immutable after creation. + Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: both volumeGroupReplicationRef.name and volumeGroupReplicationRef.namespace + must be set + rule: has(self.name) && has(self.__namespace__) + - message: volumeGroupReplicationRef is immutable + rule: self == oldSelf + required: + - provisioner + - source + - volumeGroupReplicationHandle + - volumeGroupReplicationRef + type: object + status: + description: VolumeGroupReplicationContentStatus defines the status of + VolumeGroupReplicationContent + properties: + persistentVolumeRefList: + description: |- + PersistentVolumeRefList is the list of of PV for the group replication + The maximum number of allowed PV in the group is 100. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/hack/test/replication.storage.openshift.io_volumegroupreplications.yaml b/hack/test/replication.storage.openshift.io_volumegroupreplications.yaml new file mode 100644 index 0000000000..d8f5757cb2 --- /dev/null +++ b/hack/test/replication.storage.openshift.io_volumegroupreplications.yaml @@ -0,0 +1,257 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: volumegroupreplications.replication.storage.openshift.io +spec: + group: replication.storage.openshift.io + names: + kind: VolumeGroupReplication + listKind: VolumeGroupReplicationList + plural: volumegroupreplications + singular: volumegroupreplication + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: VolumeGroupReplication is the Schema for the volumegroupreplications + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: VolumeGroupReplicationSpec defines the desired state of VolumeGroupReplication + properties: + autoResync: + default: false + description: |- + AutoResync represents the group to be auto resynced when + ReplicationState is "secondary" + type: boolean + replicationState: + description: |- + ReplicationState represents the replication operation to be performed on the group. + Supported operations are "primary", "secondary" and "resync" + enum: + - primary + - secondary + - resync + type: string + source: + description: |- + Source specifies where a group replications will be created from. + This field is immutable after creation. + Required. + properties: + selector: + description: |- + Selector is a label query over persistent volume claims that are to be + grouped together for replication. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: selector is immutable + rule: self == oldSelf + type: object + x-kubernetes-validations: + - message: source is immutable + rule: self == oldSelf + volumeGroupReplicationClassName: + description: volumeGroupReplicationClassName is the volumeGroupReplicationClass + name for this VolumeGroupReplication resource + type: string + x-kubernetes-validations: + - message: volumeGroupReplicationClassName is immutable + rule: self == oldSelf + volumeGroupReplicationContentName: + description: Name of the VolumeGroupReplicationContent object created + for this volumeGroupReplication + type: string + x-kubernetes-validations: + - message: volumeGroupReplicationContentName is immutable + rule: self == oldSelf + volumeReplicationClassName: + description: volumeReplicationClassName is the volumeReplicationClass + name for VolumeReplication object + type: string + x-kubernetes-validations: + - message: volumReplicationClassName is immutable + rule: self == oldSelf + volumeReplicationName: + description: Name of the VolumeReplication object created for this + volumeGroupReplication + type: string + x-kubernetes-validations: + - message: volumeReplicationName is immutable + rule: self == oldSelf + required: + - autoResync + - replicationState + - source + - volumeGroupReplicationClassName + - volumeReplicationClassName + type: object + status: + description: VolumeGroupReplicationStatus defines the observed state of + VolumeGroupReplication + properties: + conditions: + description: Conditions are the list of conditions and their status. + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + lastCompletionTime: + format: date-time + type: string + lastStartTime: + format: date-time + type: string + lastSyncBytes: + format: int64 + type: integer + lastSyncDuration: + type: string + lastSyncTime: + format: date-time + type: string + message: + type: string + observedGeneration: + description: observedGeneration is the last generation change the + operator has dealt with + format: int64 + type: integer + state: + description: State captures the latest state of the replication operation. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/internal/controller/volumereplicationgroup_controller.go b/internal/controller/volumereplicationgroup_controller.go index e238cabab1..fc2122dcc0 100644 --- a/internal/controller/volumereplicationgroup_controller.go +++ b/internal/controller/volumereplicationgroup_controller.go @@ -98,7 +98,8 @@ func (r *VolumeReplicationGroupReconciler) SetupWithManager( builder.WithPredicates(rmnutil.CreateOrDeleteOrResourceVersionUpdatePredicate{}), ). Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(r.configMapFun)). - Owns(&volrep.VolumeReplication{}) + Owns(&volrep.VolumeReplication{}). + Owns(&volrep.VolumeGroupReplication{}) if !ramenConfig.VolSync.Disabled { r.Log.Info("VolSync enabled; adding owns and watches") @@ -363,6 +364,8 @@ func filterPVC(reader client.Reader, pvc *corev1.PersistentVolumeClaim, log logr // +kubebuilder:rbac:groups=ramendr.openshift.io,resources=volumereplicationgroups/finalizers,verbs=update // +kubebuilder:rbac:groups=replication.storage.openshift.io,resources=volumereplications,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=replication.storage.openshift.io,resources=volumereplicationclasses,verbs=get;list;watch +// +kubebuilder:rbac:groups=replication.storage.openshift.io,resources=volumegroupreplications,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=replication.storage.openshift.io,resources=volumegroupreplicationclasses,verbs=get;list;watch // +kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;list;watch;create;update // +kubebuilder:rbac:groups=storage.k8s.io,resources=volumeattachments,verbs=get;list;watch // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch @@ -404,6 +407,7 @@ func (r *VolumeReplicationGroupReconciler) Reconcile(ctx context.Context, req ct volRepPVCs: []corev1.PersistentVolumeClaim{}, volSyncPVCs: []corev1.PersistentVolumeClaim{}, replClassList: &volrep.VolumeReplicationClassList{}, + grpReplClassList: &volrep.VolumeGroupReplicationClassList{}, namespacedName: req.NamespacedName.String(), objectStorers: make(map[string]cachedObjectStorer), storageClassCache: make(map[string]*storagev1.StorageClass), @@ -474,6 +478,7 @@ type VRGInstance struct { volRepPVCs []corev1.PersistentVolumeClaim volSyncPVCs []corev1.PersistentVolumeClaim replClassList *volrep.VolumeReplicationClassList + grpReplClassList *volrep.VolumeGroupReplicationClassList storageClassCache map[string]*storagev1.StorageClass vrgObjectProtected *metav1.Condition kubeObjectsProtected *metav1.Condition @@ -689,7 +694,7 @@ func (v *VRGInstance) updatePVCList() error { return nil } - if len(v.replClassList.Items) == 0 { + if len(v.replClassList.Items) == 0 && len(v.grpReplClassList.Items) == 0 { v.volSyncPVCs = make([]corev1.PersistentVolumeClaim, len(pvcList.Items)) numCopied := copy(v.volSyncPVCs, pvcList.Items) v.log.Info("No VolumeReplicationClass available. Using all PVCs with VolSync", "pvcCount", numCopied) @@ -761,17 +766,28 @@ func (v *VRGInstance) updateReplicationClassList() error { client.MatchingLabels(labelSelector.MatchLabels), } - if err := v.reconciler.List(v.ctx, v.replClassList, listOptions...); err != nil { - v.log.Error(err, "Failed to list Replication Classes", - "labeled", labels.Set(labelSelector.MatchLabels)) + if !rmnutil.IsCGEnabled(v.instance.GetAnnotations()) { + if err := v.reconciler.List(v.ctx, v.replClassList, listOptions...); err != nil { + v.log.Error(err, "Failed to list Replication Classes", + "labeled", labels.Set(labelSelector.MatchLabels)) + + return fmt.Errorf("failed to list Replication Classes, %w", err) + } + + v.log.Info("Number of Replication Classes", "count", len(v.replClassList.Items)) + } else { + if err := v.reconciler.List(v.ctx, v.grpReplClassList, listOptions...); err != nil { + v.log.Error(err, "Failed to list Group Replication Classes", + "labeled", labels.Set(labelSelector.MatchLabels)) + + return fmt.Errorf("failed to list Group Replication Classes, %w", err) + } - return fmt.Errorf("failed to list Replication Classes, %w", err) + v.log.Info("Number of Group Replication Classes", "count", len(v.grpReplClassList.Items)) } v.vrcUpdated = true - v.log.Info("Number of Replication Classes", "count", len(v.replClassList.Items)) - return nil } @@ -791,6 +807,7 @@ func (v *VRGInstance) separatePVCsUsingVRGStatus(pvcList *corev1.PersistentVolum } } +//nolint:gocognit,cyclop func (v *VRGInstance) separatePVCsUsingStorageClassProvisioner(pvcList *corev1.PersistentVolumeClaimList) error { for idx := range pvcList.Items { pvc := &pvcList.Items[idx] @@ -809,12 +826,23 @@ func (v *VRGInstance) separatePVCsUsingStorageClassProvisioner(pvcList *corev1.P replicationClassMatchFound := false - for _, replicationClass := range v.replClassList.Items { - if storageClass.Provisioner == replicationClass.Spec.Provisioner { - v.volRepPVCs = append(v.volRepPVCs, *pvc) - replicationClassMatchFound = true + if rmnutil.IsCGEnabled(v.instance.GetAnnotations()) { + for _, replicationClass := range v.grpReplClassList.Items { + if storageClass.Provisioner == replicationClass.Spec.Provisioner { + v.volRepPVCs = append(v.volRepPVCs, *pvc) + replicationClassMatchFound = true - break + break + } + } + } else { + for _, replicationClass := range v.replClassList.Items { + if storageClass.Provisioner == replicationClass.Spec.Provisioner { + v.volRepPVCs = append(v.volRepPVCs, *pvc) + replicationClassMatchFound = true + + break + } } } diff --git a/internal/controller/vrg_volrep.go b/internal/controller/vrg_volrep.go index 23deb8a68c..9335cd1d7c 100644 --- a/internal/controller/vrg_volrep.go +++ b/internal/controller/vrg_volrep.go @@ -19,6 +19,7 @@ import ( storagev1 "k8s.io/api/storage/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -67,7 +68,7 @@ func (v *VRGInstance) reconcileVolRepsAsPrimary() { } // If VR did not reach primary state, it is fine to still upload the PV and continue processing - requeueResult, _, err := v.processVRAsPrimary(pvcNamespacedName, log) + requeueResult, _, err := v.processVRAsPrimary(pvcNamespacedName, pvc, log) if requeueResult { v.requeue() } @@ -159,7 +160,7 @@ func (v *VRGInstance) reconcileVRAsSecondary(pvc *corev1.PersistentVolumeClaim, pvcNamespacedName := types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace} - requeueResult, ready, err := v.processVRAsSecondary(pvcNamespacedName, log) + requeueResult, ready, err := v.processVRAsSecondary(pvcNamespacedName, pvc, log) if err != nil { log.Info("Failure in getting or creating VolumeReplication resource for PersistentVolumeClaim", "errorValue", err) @@ -258,7 +259,7 @@ func (v *VRGInstance) updateProtectedPVCs(pvc *corev1.PersistentVolumeClaim) err func setPVCStorageIdentifiers( protectedPVC *ramendrv1alpha1.ProtectedPVC, storageClass *storagev1.StorageClass, - volumeReplicationClass *volrep.VolumeReplicationClass, + volumeReplicationClass client.Object, ) { protectedPVC.StorageIdentifiers.StorageProvisioner = storageClass.Provisioner @@ -269,9 +270,9 @@ func setPVCStorageIdentifiers( } } - if value, ok := volumeReplicationClass.Labels[VolumeReplicationIDLabel]; ok { + if value, ok := volumeReplicationClass.GetLabels()[VolumeReplicationIDLabel]; ok { protectedPVC.StorageIdentifiers.ReplicationID.ID = value - if modes, ok := volumeReplicationClass.Labels[MModesLabel]; ok { + if modes, ok := volumeReplicationClass.GetLabels()[MModesLabel]; ok { protectedPVC.StorageIdentifiers.ReplicationID.Modes = MModesFromCSV(modes) } } @@ -846,7 +847,7 @@ func (v *VRGInstance) reconcileVRForDeletion(pvc *corev1.PersistentVolumeClaim, return !requeue } } else { - requeueResult, ready, err := v.processVRAsPrimary(pvcNamespacedName, log) + requeueResult, ready, err := v.processVRAsPrimary(pvcNamespacedName, pvc, log) switch { case err != nil: log.Info("Requeuing due to failure in getting or creating VolumeReplication resource for PersistentVolumeClaim", @@ -868,7 +869,7 @@ func (v *VRGInstance) undoPVCFinalizersAndPVRetention(pvc *corev1.PersistentVolu pvcNamespacedName := types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace} - if err := v.deleteVR(pvcNamespacedName, log); err != nil { + if err := v.deleteVR(pvcNamespacedName, pvc, log); err != nil { log.Info("Requeuing due to failure in finalizing VolumeReplication resource for PersistentVolumeClaim", "errorValue", err) @@ -905,10 +906,11 @@ func (v *VRGInstance) reconcileMissingVR(pvc *corev1.PersistentVolumeClaim, log return !vrMissing, !requeue } - volRep := &volrep.VolumeReplication{} + var volRep client.Object + vrNamespacedName := types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace} - err := v.reconciler.Get(v.ctx, vrNamespacedName, volRep) + err := v.getVolumeReplication(pvc, vrNamespacedName, &volRep) if err == nil { if rmnutil.ResourceIsDeleted(volRep) { log.Info("Requeuing due to processing a VR under deletion") @@ -937,6 +939,86 @@ func (v *VRGInstance) reconcileMissingVR(pvc *corev1.PersistentVolumeClaim, log return vrMissing, !requeue } +func (v *VRGInstance) buildVolumeGroupReplicationName(pvc *corev1.PersistentVolumeClaim) (string, error) { + storageID, ok := pvc.GetLabels()[ConsistencyGroupLabel] + if !ok { + v.log.Info("Missing storageID for PVC %s/%s", pvc.GetNamespace(), pvc.GetName()) + + return "", fmt.Errorf("missing storageID for PVC %s/%s", pvc.GetNamespace(), pvc.GetName()) + } + + vgrName := storageID + v.instance.Name + + return vgrName, nil +} + +func (v *VRGInstance) getVolumeReplication(pvc *corev1.PersistentVolumeClaim, + vrNamespacedName types.NamespacedName, volRep *client.Object, +) error { + if rmnutil.IsCGEnabled(v.instance.GetAnnotations()) { + vgrName, err := v.buildVolumeGroupReplicationName(pvc) + if err != nil { + return err + } + + vrNamespacedName.Name = vgrName + + *volRep = &volrep.VolumeGroupReplication{} + } else { + *volRep = &volrep.VolumeReplication{} + } + + return v.reconciler.Get(v.ctx, vrNamespacedName, *volRep) +} + +func (v *VRGInstance) createVolumeReplication(vrNamespacedName types.NamespacedName, + volumeReplicationClass client.Object, state volrep.ReplicationState, +) client.Object { + volRep := &volrep.VolumeReplication{ + ObjectMeta: metav1.ObjectMeta{ + Name: vrNamespacedName.Name, + Namespace: vrNamespacedName.Namespace, + Labels: rmnutil.OwnerLabels(v.instance), + }, + Spec: volrep.VolumeReplicationSpec{ + DataSource: corev1.TypedLocalObjectReference{ + Kind: "PersistentVolumeClaim", + Name: vrNamespacedName.Name, + APIGroup: new(string), + }, + ReplicationState: state, + VolumeReplicationClass: volumeReplicationClass.GetName(), + AutoResync: v.autoResync(state), + }, + } + + return volRep +} + +func (v *VRGInstance) createVolumeGroupReplication(storageID string, vrNamespacedName types.NamespacedName, + volumeReplicationClass client.Object, state volrep.ReplicationState, +) client.Object { + selector := metav1.AddLabelToSelector(&v.recipeElements.PvcSelector.LabelSelector, + ConsistencyGroupLabel, storageID) + + volRep := &volrep.VolumeGroupReplication{ + ObjectMeta: metav1.ObjectMeta{ + Name: storageID + v.instance.Name, + Namespace: vrNamespacedName.Namespace, + Labels: rmnutil.OwnerLabels(v.instance), + }, + Spec: volrep.VolumeGroupReplicationSpec{ + ReplicationState: state, + VolumeGroupReplicationClassName: volumeReplicationClass.GetName(), + Source: volrep.VolumeGroupReplicationSource{ + Selector: selector, + }, + }, + } + + return volRep +} + func (v *VRGInstance) deleteClusterDataInS3Stores(log logr.Logger) error { log.Info("Delete cluster data in", "s3Profiles", v.instance.Spec.S3Profiles) @@ -1010,9 +1092,11 @@ func (v *VRGInstance) s3StoreDo(do func(ObjectStorer) error, msg, s3ProfileName // - a boolean indicating if a reconcile requeue is required // - a boolean indicating if VR is already at the desired state // - any errors during processing -func (v *VRGInstance) processVRAsPrimary(vrNamespacedName types.NamespacedName, log logr.Logger) (bool, bool, error) { +func (v *VRGInstance) processVRAsPrimary(vrNamespacedName types.NamespacedName, + pvc *corev1.PersistentVolumeClaim, log logr.Logger, +) (bool, bool, error) { if v.instance.Spec.Async != nil { - return v.createOrUpdateVR(vrNamespacedName, volrep.Primary, log) + return v.createOrUpdateVR(vrNamespacedName, pvc, volrep.Primary, log) } // TODO: createOrUpdateVR does two things. It modifies the VR and also @@ -1042,9 +1126,11 @@ func (v *VRGInstance) processVRAsPrimary(vrNamespacedName types.NamespacedName, // - a boolean indicating if a reconcile requeue is required // - a boolean indicating if VR is already at the desired state // - any errors during processing -func (v *VRGInstance) processVRAsSecondary(vrNamespacedName types.NamespacedName, log logr.Logger) (bool, bool, error) { +func (v *VRGInstance) processVRAsSecondary(vrNamespacedName types.NamespacedName, + pvc *corev1.PersistentVolumeClaim, log logr.Logger, +) (bool, bool, error) { if v.instance.Spec.Async != nil { - return v.createOrUpdateVR(vrNamespacedName, volrep.Secondary, log) + return v.createOrUpdateVR(vrNamespacedName, pvc, volrep.Secondary, log) } // TODO: createOrUpdateVR does two things. It modifies the VR and also @@ -1081,13 +1167,13 @@ func (v *VRGInstance) processVRAsSecondary(vrNamespacedName types.NamespacedName // - a boolean indicating if VR is already at the desired state // - any errors during processing func (v *VRGInstance) createOrUpdateVR(vrNamespacedName types.NamespacedName, - state volrep.ReplicationState, log logr.Logger, + pvc *corev1.PersistentVolumeClaim, state volrep.ReplicationState, log logr.Logger, ) (bool, bool, error) { const requeue = true - volRep := &volrep.VolumeReplication{} + var volRep client.Object - err := v.reconciler.Get(v.ctx, vrNamespacedName, volRep) + err := v.getVolumeReplication(pvc, vrNamespacedName, &volRep) if err != nil { if !k8serrors.IsNotFound(err) { log.Error(err, "Failed to get VolumeReplication resource", "resource", vrNamespacedName) @@ -1105,7 +1191,7 @@ func (v *VRGInstance) createOrUpdateVR(vrNamespacedName types.NamespacedName, } // Create VR for PVC - if err = v.createVR(vrNamespacedName, state); err != nil { + if err = v.createVR(vrNamespacedName, pvc, state); err != nil { log.Error(err, "Failed to create VolumeReplication resource", "resource", vrNamespacedName) rmnutil.ReportIfNotPresent(v.reconciler.eventRecorder, v.instance, corev1.EventTypeWarning, rmnutil.EventReasonVRCreateFailed, err.Error()) @@ -1125,7 +1211,7 @@ func (v *VRGInstance) createOrUpdateVR(vrNamespacedName types.NamespacedName, return !requeue, false, nil } - return v.updateVR(volRep, state, log) + return v.updateVR(pvc, volRep, state, log) } func (v *VRGInstance) autoResync(state volrep.ReplicationState) bool { @@ -1144,70 +1230,92 @@ func (v *VRGInstance) autoResync(state volrep.ReplicationState) bool { // - a boolean indicating if a reconcile requeue is required // - a boolean indicating if VR is already at the desired state // - any errors during the process of updating the resource -func (v *VRGInstance) updateVR(volRep *volrep.VolumeReplication, +func (v *VRGInstance) updateVR(pvc *corev1.PersistentVolumeClaim, volRep client.Object, state volrep.ReplicationState, log logr.Logger, ) (bool, bool, error) { const requeue = true - // If state is already as desired, check the status - if volRep.Spec.ReplicationState == state && volRep.Spec.AutoResync == v.autoResync(state) { - log.Info("VolumeReplication and VolumeReplicationGroup state and autoresync match. Proceeding to status check") + if rmnutil.IsCGEnabled(v.instance.GetAnnotations()) { + log.Info("Update VolumeGroupReplication for PVC %s/%s", pvc.GetNamespace(), pvc.GetName()) - return !requeue, v.checkVRStatus(volRep), nil - } + if volRep.(*volrep.VolumeGroupReplication).Spec.ReplicationState == state && + volRep.(*volrep.VolumeGroupReplication).Spec.AutoResync == v.autoResync(state) { + log.Info("VolumeGroupReplication and VolumeReplicationGroup state match. Proceeding to status check") + + return !requeue, + v.checkVRStatus(pvc, volRep, &volRep.(*volrep.VolumeGroupReplication).Status.VolumeReplicationStatus), + nil + } + + volRep.(*volrep.VolumeGroupReplication).Spec.ReplicationState = state + volRep.(*volrep.VolumeGroupReplication).Spec.AutoResync = v.autoResync(state) + } else { + log.Info("Update VolumeReplication for PVC %s/%s", pvc.GetNamespace(), pvc.GetName()) - volRep.Spec.ReplicationState = state - volRep.Spec.AutoResync = v.autoResync(state) + // If state is already as desired, check the status + if volRep.(*volrep.VolumeReplication).Spec.ReplicationState == state && + volRep.(*volrep.VolumeReplication).Spec.AutoResync == v.autoResync(state) { + log.Info("VolumeReplication and VolumeReplicationGroup state and autoresync match. Proceeding to status check") + + return !requeue, v.checkVRStatus(pvc, volRep, &volRep.(*volrep.VolumeReplication).Status), nil + } + + volRep.(*volrep.VolumeReplication).Spec.ReplicationState = state + volRep.(*volrep.VolumeReplication).Spec.AutoResync = v.autoResync(state) + } if err := v.reconciler.Update(v.ctx, volRep); err != nil { log.Error(err, "Failed to update VolumeReplication resource", - "name", volRep.Name, "namespace", volRep.Namespace, + "name", volRep.GetName(), "namespace", volRep.GetNamespace(), "state", state) rmnutil.ReportIfNotPresent(v.reconciler.eventRecorder, v.instance, corev1.EventTypeWarning, rmnutil.EventReasonVRUpdateFailed, err.Error()) msg := "Failed to update VolumeReplication resource" - v.updatePVCDataReadyCondition(volRep.Namespace, volRep.Name, VRGConditionReasonError, msg) + v.updatePVCDataReadyCondition(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonError, msg) return requeue, false, fmt.Errorf("failed to update VolumeReplication resource"+ " (%s/%s) as %s, belonging to VolumeReplicationGroup (%s/%s), %w", - volRep.Namespace, volRep.Name, state, + volRep.GetNamespace(), volRep.GetName(), state, v.instance.Namespace, v.instance.Name, err) } log.Info(fmt.Sprintf("Updated VolumeReplication resource (%s/%s) with state %s", - volRep.Name, volRep.Namespace, state)) + volRep.GetName(), volRep.GetNamespace(), state)) // Just updated the state of the VolRep. Mark it as progressing. msg := "Updated VolumeReplication resource for PVC" - v.updatePVCDataReadyCondition(volRep.Namespace, volRep.Name, VRGConditionReasonProgressing, msg) + v.updatePVCDataReadyCondition(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonProgressing, msg) return !requeue, false, nil } // createVR creates a VolumeReplication CR with a PVC as its data source. -func (v *VRGInstance) createVR(vrNamespacedName types.NamespacedName, state volrep.ReplicationState) error { +func (v *VRGInstance) createVR(vrNamespacedName types.NamespacedName, + pvc *corev1.PersistentVolumeClaim, state volrep.ReplicationState, +) error { volumeReplicationClass, err := v.selectVolumeReplicationClass(vrNamespacedName) if err != nil { return fmt.Errorf("failed to find the appropriate VolumeReplicationClass (%s) %w", v.instance.Name, err) } - volRep := &volrep.VolumeReplication{ - ObjectMeta: metav1.ObjectMeta{ - Name: vrNamespacedName.Name, - Namespace: vrNamespacedName.Namespace, - Labels: rmnutil.OwnerLabels(v.instance), - }, - Spec: volrep.VolumeReplicationSpec{ - DataSource: corev1.TypedLocalObjectReference{ - Kind: "PersistentVolumeClaim", - Name: vrNamespacedName.Name, - APIGroup: new(string), - }, - ReplicationState: state, - VolumeReplicationClass: volumeReplicationClass.GetName(), - AutoResync: v.autoResync(state), - }, + var volRep client.Object + + if rmnutil.IsCGEnabled(v.instance.GetAnnotations()) { + v.log.Info("Create VolumeGroupReplication for PVC %s/%s", pvc.GetNamespace(), pvc.GetName()) + + storageID, ok := pvc.GetLabels()[ConsistencyGroupLabel] + if !ok { + v.log.Info("Missing storageID for PVC %s/%s", pvc.GetNamespace(), pvc.GetName()) + + return fmt.Errorf("missing storageID for PVC %s/%s", pvc.GetNamespace(), pvc.GetName()) + } + + volRep = v.createVolumeGroupReplication(storageID, vrNamespacedName, volumeReplicationClass, state) + } else { + v.log.Info("Create VolumeReplication for PVC %s/%s", pvc.GetNamespace(), pvc.GetName()) + + volRep = v.createVolumeReplication(vrNamespacedName, volumeReplicationClass, state) } if !vrgInAdminNamespace(v.instance, v.ramenConfig) { @@ -1216,14 +1324,15 @@ func (v *VRGInstance) createVR(vrNamespacedName types.NamespacedName, state volr // when VRG is not in the admin namespace. if err := ctrl.SetControllerReference(v.instance, volRep, v.reconciler.Scheme); err != nil { return fmt.Errorf("failed to set owner reference to VolumeReplication resource (%s/%s), %w", - volRep.Name, volRep.Namespace, err) + volRep.GetName(), volRep.GetNamespace(), err) } } v.log.Info("Creating VolumeReplication resource", "resource", volRep) if err := v.reconciler.Create(v.ctx, volRep); err != nil { - return fmt.Errorf("failed to create VolumeReplication resource (%s), %w", vrNamespacedName, err) + return fmt.Errorf("failed to create VolumeReplication resource (%s/%s), %w", + volRep.GetName(), volRep.GetNamespace(), err) } return nil @@ -1234,16 +1343,18 @@ func (v *VRGInstance) createVR(vrNamespacedName types.NamespacedName, state volr // VolumeReplicationGroup has the same name as pvc. But in future if it changes // functions to be changed would be processVRAsPrimary(), processVRAsSecondary() // to either receive pvc NamespacedName or pvc itself as an additional argument. +// +//nolint:funlen,cyclop,gocognit,nestif,gocyclo func (v *VRGInstance) selectVolumeReplicationClass( namespacedName types.NamespacedName, -) (*volrep.VolumeReplicationClass, error) { +) (client.Object, error) { if err := v.updateReplicationClassList(); err != nil { v.log.Error(err, "Failed to get VolumeReplicationClass list") return nil, fmt.Errorf("failed to get VolumeReplicationClass list") } - if len(v.replClassList.Items) == 0 { + if len(v.replClassList.Items) == 0 && len(v.grpReplClassList.Items) == 0 { v.log.Info("No VolumeReplicationClass available") return nil, fmt.Errorf("no VolumeReplicationClass available") @@ -1258,20 +1369,37 @@ func (v *VRGInstance) selectVolumeReplicationClass( namespacedName, err) } - matchingReplicationClassList := []*volrep.VolumeReplicationClass{} + matchingReplicationClassList := []client.Object{} - for index := range v.replClassList.Items { - replicationClass := &v.replClassList.Items[index] - schedulingInterval, found := replicationClass.Spec.Parameters["schedulingInterval"] + if !rmnutil.IsCGEnabled(v.instance.GetAnnotations()) { + for index := range v.replClassList.Items { + replicationClass := &v.replClassList.Items[index] + schedulingInterval, found := replicationClass.Spec.Parameters["schedulingInterval"] - if storageClass.Provisioner != replicationClass.Spec.Provisioner || !found { - // skip this replication class if provisioner does not match or if schedule not found - continue + if storageClass.Provisioner != replicationClass.Spec.Provisioner || !found { + // skip this replication class if provisioner does not match or if schedule not found + continue + } + + // ReplicationClass that matches both VRG schedule and pvc provisioner + if schedulingInterval == v.instance.Spec.Async.SchedulingInterval { + matchingReplicationClassList = append(matchingReplicationClassList, replicationClass) + } } + } else { + for index := range v.grpReplClassList.Items { + replicationClass := &v.grpReplClassList.Items[index] + schedulingInterval, found := replicationClass.Spec.Parameters["schedulingInterval"] - // ReplicationClass that matches both VRG schedule and pvc provisioner - if schedulingInterval == v.instance.Spec.Async.SchedulingInterval { - matchingReplicationClassList = append(matchingReplicationClassList, replicationClass) + if storageClass.Provisioner != replicationClass.Spec.Provisioner || !found { + // skip this replication class if provisioner does not match or if schedule not found + continue + } + + // ReplicationClass that matches both VRG schedule and pvc provisioner + if schedulingInterval == v.instance.Spec.Async.SchedulingInterval { + matchingReplicationClassList = append(matchingReplicationClassList, replicationClass) + } } } @@ -1294,14 +1422,14 @@ func (v *VRGInstance) selectVolumeReplicationClass( // filterDefaultVRC filters the VRC list to return VRCs with default annotation // if the list contains more than one VRC. func (v *VRGInstance) filterDefaultVRC( - replicationClassList []*volrep.VolumeReplicationClass, -) (*volrep.VolumeReplicationClass, error) { + replicationClassList []client.Object, +) (client.Object, error) { v.log.Info("Found multiple matching VolumeReplicationClasses, filtering with default annotation") - filteredVRCs := []*volrep.VolumeReplicationClass{} + filteredVRCs := []client.Object{} for index := range replicationClassList { - if replicationClassList[index].Annotations[defaultVRCAnnotationKey] == "true" { + if replicationClassList[index].GetAnnotations()[defaultVRCAnnotationKey] == "true" { filteredVRCs = append( filteredVRCs, replicationClassList[index]) @@ -1310,8 +1438,8 @@ func (v *VRGInstance) filterDefaultVRC( switch len(filteredVRCs) { case 0: - v.log.Info(fmt.Sprintf("Multiple VolumeReplicationClass found, with no default annotation (%s/%s)", - replicationClassList[0].Spec.Provisioner, v.instance.Spec.Async.SchedulingInterval)) + v.log.Info(fmt.Sprintf("Multiple VolumeReplicationClass found, with no default annotation (%s)", + defaultVRCAnnotationKey)) return nil, fmt.Errorf("multiple VolumeReplicationClass found, with no default annotation, %s", defaultVRCAnnotationKey) @@ -1373,30 +1501,32 @@ func (v *VRGInstance) getStorageClass(namespacedName types.NamespacedName) (*sto // checkVRStatus checks if the VolumeReplication resource has the desired status for the // current generation and returns true if so, false otherwise -func (v *VRGInstance) checkVRStatus(volRep *volrep.VolumeReplication) bool { +func (v *VRGInstance) checkVRStatus(pvc *corev1.PersistentVolumeClaim, volRep client.Object, + status *volrep.VolumeReplicationStatus, +) bool { // When the generation in the status is updated, VRG would get a reconcile // as it owns VolumeReplication resource. - if volRep.Generation != volRep.Status.ObservedGeneration { + if volRep.GetGeneration() != status.ObservedGeneration { v.log.Info(fmt.Sprintf("Generation mismatch in status for VolumeReplication resource (%s/%s)", - volRep.Name, volRep.Namespace)) + volRep.GetName(), volRep.GetNamespace())) msg := "VolumeReplication generation not updated in status" - v.updatePVCDataReadyCondition(volRep.Namespace, volRep.Name, VRGConditionReasonProgressing, msg) + v.updatePVCDataReadyCondition(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonProgressing, msg) return false } switch { case v.instance.Spec.ReplicationState == ramendrv1alpha1.Primary: - return v.validateVRStatus(volRep, ramendrv1alpha1.Primary) + return v.validateVRStatus(pvc, volRep, ramendrv1alpha1.Primary, status) case v.instance.Spec.ReplicationState == ramendrv1alpha1.Secondary: - return v.validateVRStatus(volRep, ramendrv1alpha1.Secondary) + return v.validateVRStatus(pvc, volRep, ramendrv1alpha1.Secondary, status) default: v.log.Info(fmt.Sprintf("invalid Replication State %s for VolumeReplicationGroup (%s:%s)", string(v.instance.Spec.ReplicationState), v.instance.Name, v.instance.Namespace)) msg := "VolumeReplicationGroup state invalid" - v.updatePVCDataReadyCondition(volRep.Namespace, volRep.Name, VRGConditionReasonError, msg) + v.updatePVCDataReadyCondition(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonError, msg) return false } @@ -1407,7 +1537,9 @@ func (v *VRGInstance) checkVRStatus(volRep *volrep.VolumeReplication) bool { // - When replication state is Primary, only Completed condition is checked. // - When replication state is Secondary, all 3 conditions for Completed/Degraded/Resyncing is // checked and ensured healthy. -func (v *VRGInstance) validateVRStatus(volRep *volrep.VolumeReplication, state ramendrv1alpha1.ReplicationState) bool { +func (v *VRGInstance) validateVRStatus(pvc *corev1.PersistentVolumeClaim, volRep client.Object, + state ramendrv1alpha1.ReplicationState, status *volrep.VolumeReplicationStatus, +) bool { var ( stateString string action string @@ -1423,33 +1555,33 @@ func (v *VRGInstance) validateVRStatus(volRep *volrep.VolumeReplication, state r } // it should be completed - conditionMet, msg := isVRConditionMet(volRep, volrepController.ConditionCompleted, metav1.ConditionTrue) + conditionMet, msg := isVRConditionMet(volRep, status, volrepController.ConditionCompleted, metav1.ConditionTrue) if !conditionMet { defaultMsg := fmt.Sprintf("VolumeReplication resource for pvc not %s to %s", action, stateString) - v.updatePVCDataReadyConditionHelper(volRep.Namespace, volRep.Name, VRGConditionReasonError, msg, + v.updatePVCDataReadyConditionHelper(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonError, msg, defaultMsg) - v.updatePVCDataProtectedConditionHelper(volRep.Namespace, volRep.Name, VRGConditionReasonError, msg, + v.updatePVCDataProtectedConditionHelper(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonError, msg, defaultMsg) - v.log.Info(fmt.Sprintf("%s (VolRep: %s/%s)", defaultMsg, volRep.Name, volRep.Namespace)) + v.log.Info(fmt.Sprintf("%s (VolRep: %s/%s)", defaultMsg, volRep.GetName(), volRep.GetNamespace())) return false } // if primary, all checks are completed if state == ramendrv1alpha1.Secondary { - return v.validateAdditionalVRStatusForSecondary(volRep) + return v.validateAdditionalVRStatusForSecondary(pvc, volRep, status) } msg = "PVC in the VolumeReplicationGroup is ready for use" - v.updatePVCDataReadyCondition(volRep.Namespace, volRep.Name, VRGConditionReasonReady, msg) - v.updatePVCDataProtectedCondition(volRep.Namespace, volRep.Name, VRGConditionReasonReady, msg) - v.updatePVCLastSyncTime(volRep.Namespace, volRep.Name, volRep.Status.LastSyncTime) - v.updatePVCLastSyncDuration(volRep.Namespace, volRep.Name, volRep.Status.LastSyncDuration) - v.updatePVCLastSyncBytes(volRep.Namespace, volRep.Name, volRep.Status.LastSyncBytes) - v.log.Info(fmt.Sprintf("VolumeReplication resource %s/%s is ready for use", volRep.Name, - volRep.Namespace)) + v.updatePVCDataReadyCondition(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonReady, msg) + v.updatePVCDataProtectedCondition(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonReady, msg) + v.updatePVCLastSyncTime(pvc.GetNamespace(), pvc.GetName(), status.LastSyncTime) + v.updatePVCLastSyncDuration(pvc.GetNamespace(), pvc.GetName(), status.LastSyncDuration) + v.updatePVCLastSyncBytes(pvc.GetNamespace(), pvc.GetName(), status.LastSyncBytes) + v.log.Info(fmt.Sprintf("VolumeReplication resource %s/%s is ready for use", volRep.GetName(), + volRep.GetNamespace())) return true } @@ -1471,92 +1603,96 @@ func (v *VRGInstance) validateVRStatus(volRep *volrep.VolumeReplication, state r // With 2nd condition being met, // ProtectedPVC.Conditions[DataReady] = True // ProtectedPVC.Conditions[DataProtected] = True -func (v *VRGInstance) validateAdditionalVRStatusForSecondary(volRep *volrep.VolumeReplication) bool { - v.updatePVCLastSyncTime(volRep.Namespace, volRep.Name, nil) - v.updatePVCLastSyncDuration(volRep.Namespace, volRep.Name, nil) - v.updatePVCLastSyncBytes(volRep.Namespace, volRep.Name, nil) - - conditionMet, _ := isVRConditionMet(volRep, volrepController.ConditionResyncing, metav1.ConditionTrue) +func (v *VRGInstance) validateAdditionalVRStatusForSecondary(pvc *corev1.PersistentVolumeClaim, volRep client.Object, + status *volrep.VolumeReplicationStatus, +) bool { + v.updatePVCLastSyncTime(pvc.GetNamespace(), pvc.GetName(), nil) + v.updatePVCLastSyncDuration(pvc.GetNamespace(), pvc.GetName(), nil) + v.updatePVCLastSyncBytes(pvc.GetNamespace(), pvc.GetName(), nil) + + conditionMet, _ := isVRConditionMet(volRep, status, volrepController.ConditionResyncing, metav1.ConditionTrue) if !conditionMet { - return v.checkResyncCompletionAsSecondary(volRep) + return v.checkResyncCompletionAsSecondary(pvc, volRep, status) } - conditionMet, msg := isVRConditionMet(volRep, volrepController.ConditionDegraded, metav1.ConditionTrue) + conditionMet, msg := isVRConditionMet(volRep, status, volrepController.ConditionDegraded, metav1.ConditionTrue) if !conditionMet { - v.updatePVCDataProtectedConditionHelper(volRep.Namespace, volRep.Name, VRGConditionReasonError, msg, + v.updatePVCDataProtectedConditionHelper(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonError, msg, "VolumeReplication resource for pvc is not in Degraded condition while resyncing") - v.updatePVCDataReadyConditionHelper(volRep.Namespace, volRep.Name, VRGConditionReasonError, msg, + v.updatePVCDataReadyConditionHelper(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonError, msg, "VolumeReplication resource for pvc is not in Degraded condition while resyncing") v.log.Info(fmt.Sprintf("VolumeReplication resource is not in degraded condition while"+ - " resyncing is true (%s/%s)", volRep.Name, volRep.Namespace)) + " resyncing is true (%s/%s)", volRep.GetName(), volRep.GetNamespace())) return false } msg = "VolumeReplication resource for the pvc is syncing as Secondary" - v.updatePVCDataReadyCondition(volRep.Namespace, volRep.Name, VRGConditionReasonReplicating, msg) - v.updatePVCDataProtectedCondition(volRep.Namespace, volRep.Name, VRGConditionReasonReplicating, msg) + v.updatePVCDataReadyCondition(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonReplicating, msg) + v.updatePVCDataProtectedCondition(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonReplicating, msg) v.log.Info(fmt.Sprintf("VolumeReplication resource for the pvc is syncing as Secondary (%s/%s)", - volRep.Name, volRep.Namespace)) + volRep.GetName(), volRep.GetNamespace())) return true } // checkResyncCompletionAsSecondary returns true if resync status is complete as secondary, false otherwise -func (v *VRGInstance) checkResyncCompletionAsSecondary(volRep *volrep.VolumeReplication) bool { - conditionMet, msg := isVRConditionMet(volRep, volrepController.ConditionResyncing, metav1.ConditionFalse) +func (v *VRGInstance) checkResyncCompletionAsSecondary(pvc *corev1.PersistentVolumeClaim, volRep client.Object, + status *volrep.VolumeReplicationStatus, +) bool { + conditionMet, msg := isVRConditionMet(volRep, status, volrepController.ConditionResyncing, metav1.ConditionFalse) if !conditionMet { defaultMsg := "VolumeReplication resource for pvc not syncing as Secondary" - v.updatePVCDataReadyConditionHelper(volRep.Namespace, volRep.Name, VRGConditionReasonError, msg, + v.updatePVCDataReadyConditionHelper(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonError, msg, defaultMsg) - v.updatePVCDataProtectedConditionHelper(volRep.Namespace, volRep.Name, VRGConditionReasonError, msg, + v.updatePVCDataProtectedConditionHelper(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonError, msg, defaultMsg) - v.log.Info(fmt.Sprintf("%s (VolRep: %s/%s)", defaultMsg, volRep.Name, volRep.Namespace)) + v.log.Info(fmt.Sprintf("%s (VolRep: %s/%s)", defaultMsg, volRep.GetName(), volRep.GetNamespace())) return false } - conditionMet, msg = isVRConditionMet(volRep, volrepController.ConditionDegraded, metav1.ConditionFalse) + conditionMet, msg = isVRConditionMet(volRep, status, volrepController.ConditionDegraded, metav1.ConditionFalse) if !conditionMet { defaultMsg := "VolumeReplication resource for pvc is not syncing and is degraded as Secondary" - v.updatePVCDataReadyConditionHelper(volRep.Namespace, volRep.Name, VRGConditionReasonError, msg, + v.updatePVCDataReadyConditionHelper(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonError, msg, defaultMsg) - v.updatePVCDataProtectedConditionHelper(volRep.Namespace, volRep.Name, VRGConditionReasonError, msg, + v.updatePVCDataProtectedConditionHelper(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonError, msg, defaultMsg) - v.log.Info(fmt.Sprintf("%s (VolRep: %s/%s)", defaultMsg, volRep.Name, volRep.Namespace)) + v.log.Info(fmt.Sprintf("%s (VolRep: %s/%s)", defaultMsg, volRep.GetName(), volRep.GetNamespace())) return false } msg = "VolumeReplication resource for the pvc as Secondary is in sync with Primary" - v.updatePVCDataReadyCondition(volRep.Namespace, volRep.Name, VRGConditionReasonReplicated, msg) - v.updatePVCDataProtectedCondition(volRep.Namespace, volRep.Name, VRGConditionReasonDataProtected, msg) + v.updatePVCDataReadyCondition(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonReplicated, msg) + v.updatePVCDataProtectedCondition(pvc.GetNamespace(), pvc.GetName(), VRGConditionReasonDataProtected, msg) v.log.Info(fmt.Sprintf("data sync completed as both degraded and resyncing are false for"+ - " secondary VolRep (%s/%s)", volRep.Name, volRep.Namespace)) + " secondary VolRep (%s/%s)", volRep.GetName(), volRep.GetNamespace())) return true } -func isVRConditionMet(volRep *volrep.VolumeReplication, +func isVRConditionMet(volRep client.Object, status *volrep.VolumeReplicationStatus, conditionType string, desiredStatus metav1.ConditionStatus, ) (bool, string) { - volRepCondition := findCondition(volRep.Status.Conditions, conditionType) + volRepCondition := findCondition(status.Conditions, conditionType) if volRepCondition == nil { msg := fmt.Sprintf("Failed to get the %s condition from status of VolumeReplication resource.", conditionType) return false, msg } - if volRep.Generation != volRepCondition.ObservedGeneration { + if volRep.GetGeneration() != volRepCondition.ObservedGeneration { msg := fmt.Sprintf("Stale generation for condition %s from status of VolumeReplication resource.", conditionType) return false, msg @@ -1751,9 +1887,9 @@ func setPVCClusterDataProtectedCondition(protectedPVC *ramendrv1alpha1.Protected // ensureVRDeletedFromAPIServer adds an additional step to ensure that we wait for volumereplication deletion // from API server before moving ahead with vrg finalizer removal. -func (v *VRGInstance) ensureVRDeletedFromAPIServer(vrNamespacedName types.NamespacedName, log logr.Logger) error { - volRep := &volrep.VolumeReplication{} - +func (v *VRGInstance) ensureVRDeletedFromAPIServer(vrNamespacedName types.NamespacedName, + volRep client.Object, log logr.Logger, +) error { err := v.reconciler.APIReader.Get(v.ctx, vrNamespacedName, volRep) if err == nil { log.Info("Found VolumeReplication resource pending delete", "vr", volRep) @@ -1773,12 +1909,38 @@ func (v *VRGInstance) ensureVRDeletedFromAPIServer(vrNamespacedName types.Namesp } // deleteVR deletes a VolumeReplication instance if found -func (v *VRGInstance) deleteVR(vrNamespacedName types.NamespacedName, log logr.Logger) error { - cr := &volrep.VolumeReplication{ - ObjectMeta: metav1.ObjectMeta{ - Name: vrNamespacedName.Name, - Namespace: vrNamespacedName.Namespace, - }, +func (v *VRGInstance) deleteVR(vrNamespacedName types.NamespacedName, + pvc *corev1.PersistentVolumeClaim, log logr.Logger, +) error { + var cr client.Object + + if rmnutil.IsCGEnabled(v.instance.GetAnnotations()) { + log.Info("Delete VolumeGroupReplication for PVC %s/%s", pvc.GetNamespace(), pvc.GetName()) + + vgrName, err := v.buildVolumeGroupReplicationName(pvc) + if err != nil { + return err + } + + vrNamespacedName.Name = vgrName + + cr, err = v.reconcileVolumeGroupReplicationForDeletion(vrNamespacedName, pvc, log) + if err != nil { + return err + } + + if cr == nil { + return nil + } + } else { + log.Info("Delete VolumeReplication for PVC %s/%s", pvc.GetNamespace(), pvc.GetName()) + + cr = &volrep.VolumeReplication{ + ObjectMeta: metav1.ObjectMeta{ + Name: vrNamespacedName.Name, + Namespace: vrNamespacedName.Namespace, + }, + } } err := v.reconciler.Delete(v.ctx, cr) @@ -1793,7 +1955,57 @@ func (v *VRGInstance) deleteVR(vrNamespacedName types.NamespacedName, log logr.L return nil } - return v.ensureVRDeletedFromAPIServer(vrNamespacedName, log) + return v.ensureVRDeletedFromAPIServer(vrNamespacedName, cr, log) +} + +func (v *VRGInstance) reconcileVolumeGroupReplicationForDeletion(vrNamespacedName types.NamespacedName, + pvc *corev1.PersistentVolumeClaim, log logr.Logger, +) (client.Object, error) { + volRep := &volrep.VolumeGroupReplication{} + + err := v.reconciler.Get(v.ctx, vrNamespacedName, volRep) + if err != nil { + if !k8serrors.IsNotFound(err) { + log.Error(err, "Failed to get VolumeGroupReplication resource") + + return nil, fmt.Errorf("failed to get VolumeGroupReplication resource (%s/%s), %w", + vrNamespacedName.Namespace, vrNamespacedName.Name, err) + } + + return nil, nil + } + + pvcLabelSelector := volRep.Spec.Source.Selector + + // Found VGR, if there is only 1 PVC protected by it, we can delete + pvcList, err := rmnutil.ListPVCsByPVCSelector(v.ctx, v.reconciler.Client, v.log, + *pvcLabelSelector, + []string{vrNamespacedName.Namespace}, + v.instance.Spec.VolSync.Disabled, + ) + if err != nil { + return nil, err + } + + if len(pvcList.Items) > 1 { + log.Error(err, "VolumeGroupReplication resource is in use and cannot be deleted yet") + + return nil, nil + } + + selector, err := metav1.LabelSelectorAsSelector(pvcLabelSelector) + if err != nil { + return nil, err + } + + labelMatch := selector.Matches(labels.Set(pvc.GetLabels())) + if !labelMatch { + log.Info(fmt.Sprintf("PVC %s does not match VolumeGroupReplication label selector %v", pvc.Name, selector)) + + return nil, fmt.Errorf("PVC %s does not match VolumeGroupReplication label selector %v", pvc.Name, selector) + } + + return volRep, nil } func (v *VRGInstance) addProtectedAnnotationForPVC(pvc *corev1.PersistentVolumeClaim, log logr.Logger) error { diff --git a/internal/controller/vrg_volrep_test.go b/internal/controller/vrg_volrep_test.go index 65fe501ae9..c52b43aa6a 100644 --- a/internal/controller/vrg_volrep_test.go +++ b/internal/controller/vrg_volrep_test.go @@ -578,6 +578,7 @@ var _ = Describe("VolumeReplicationGroupVolRepController", func() { vrgVRDeleteEnsureTestCase.promoteVolReps() vrgVRDeleteEnsureTestCase.verifyVRGStatusExpectation(true, vrgController.VRGConditionReasonReady) }) + //nolint:dupl It("ensures orderly cleanup post VolumeReplication deletion", func() { By("Protecting the VolumeReplication resources from deletion") vrgVRDeleteEnsureTestCase.protectDeletionOfVolReps() @@ -615,6 +616,149 @@ var _ = Describe("VolumeReplicationGroupVolRepController", func() { }) }) + // Test VRG finalizer removal during deletion is deferred till VGR is deleted + var vrgVGRDeleteEnsureTestCase *vrgTest + Context("in primary state", func() { + createTestTemplate := &template{ + ClaimBindInfo: corev1.ClaimBound, + VolumeBindInfo: corev1.VolumeBound, + schedulingInterval: "1h", + storageClassName: "manual", + replicationClassName: "test-replicationclass", + vrcProvisioner: "manual.storage.com", + scProvisioner: "manual.storage.com", + replicationClassLabels: map[string]string{"protection": "ramen"}, + } + It("sets up PVCs, PVs and VRGs (with s3 stores that fail uploads)", func() { + createTestTemplate.s3Profiles = []string{s3Profiles[vrgS3ProfileNumber].S3ProfileName} + vrgVGRDeleteEnsureTestCase = newVRGTestCaseCreate(1, createTestTemplate, true, false) + vrgVGRDeleteEnsureTestCase.repGroup = true + vrgVGRDeleteEnsureTestCase.VRGTestCaseStart() + }) + It("waits for VRG to create a VGR for all PVCs", func() { + expectedVRCount := 1 + vrgVGRDeleteEnsureTestCase.waitForVGRCountToMatch(expectedVRCount) + }) + It("waits for VRG status to match", func() { + vrgVGRDeleteEnsureTestCase.promoteVolGroupReps() + vrgVGRDeleteEnsureTestCase.verifyVRGStatusExpectation(true, vrgController.VRGConditionReasonReady) + }) + //nolint:dupl + It("ensures orderly cleanup post VolumeGroupReplication deletion", func() { + By("Protecting the VolumeGroupReplication resources from deletion") + vrgVGRDeleteEnsureTestCase.protectDeletionOfVolGroupReps() + + By("Starting the VRG deletion process") + vrgVGRDeleteEnsureTestCase.cleanupPVCs(pvcProtectedVerify, vrAndPvcDeletionTimestampsRecentVerify) + vrg := vrgVGRDeleteEnsureTestCase.getVRG() + Expect(k8sClient.Delete(context.TODO(), vrg)).To(Succeed()) + + By("Ensuring VRG is not deleted till VGR is present") + Consistently(apiReader.Get, vrgtimeout, vrginterval). + WithArguments(context.TODO(), vrgVGRDeleteEnsureTestCase.vrgNamespacedName(), vrg). + Should(Succeed(), "while waiting for VRG %v to remain undeleted", + vrgVGRDeleteEnsureTestCase.vrgNamespacedName()) + + By("Un-protecting the VolumeReplication resources to ensure their deletion") + vrgVGRDeleteEnsureTestCase.unprotectDeletionOfVolGroupReps() + + By("Ensuring VRG is deleted eventually as a result") + var i int + Eventually(func() error { + i++ + + return apiReader.Get(context.TODO(), vrgVGRDeleteEnsureTestCase.vrgNamespacedName(), vrg) + }, vrgtimeout*2, vrginterval). + Should(MatchError(errors.NewNotFound(schema.GroupResource{ + Group: ramendrv1alpha1.GroupVersion.Group, + Resource: "volumereplicationgroups", + }, vrgVGRDeleteEnsureTestCase.vrgName)), + "polled %d times for VRG to be garbage collected\n"+format.Object(*vrg, 1), i) + + vrgVGRDeleteEnsureTestCase.cleanupNamespace() + vrgVGRDeleteEnsureTestCase.cleanupSC() + vrgVGRDeleteEnsureTestCase.cleanupVGRC() + }) + }) + + // Try the simple case of creating VRG, PVC, PV and + // check whether VolGroupRep resources are created or not + var vrgCreateVGRTestCase *vrgTest + Context("in primary state", func() { + createTestTemplate := &template{ + ClaimBindInfo: corev1.ClaimBound, + VolumeBindInfo: corev1.VolumeBound, + schedulingInterval: "1h", + storageClassName: "manual", + replicationClassName: "test-replicationclass", + vrcProvisioner: "manual.storage.com", + scProvisioner: "manual.storage.com", + replicationClassLabels: map[string]string{"protection": "ramen"}, + } + It("sets up PVCs, PVs and VRGs", func() { + createTestTemplate.s3Profiles = []string{s3Profiles[vrgS3ProfileNumber].S3ProfileName} + vrgCreateVGRTestCase = newVRGTestCaseCreate(3, createTestTemplate, true, false) + vrgCreateVGRTestCase.repGroup = true + vrgCreateVGRTestCase.VRGTestCaseStart() + }) + It("waits for VRG to create a VGR for all PVCs", func() { + expectedVGRCount := 1 + vrgCreateVGRTestCase.waitForVGRCountToMatch(expectedVGRCount) + }) + It("waits for VRG status to match", func() { + vrgCreateVGRTestCase.promoteVolGroupReps() + vrgCreateVGRTestCase.verifyVRGStatusExpectation(true, vrgController.VRGConditionReasonReady) + }) + It("cleans up after testing", func() { + vrgCreateVGRTestCase.cleanupProtected() + }) + }) + + // Creates VRG. PVCs and PV are created with Status.Phase + // set to pending and VolGroupRep should not be created until + // all the PVCs and PVs are bound. So, these tests then + // change the Status.Phase of PVCs and PVs to bound state, + // and then checks whether VolGroupRep + // resource have been created or not. + var vrgPVCnotBoundVGRTestCase *vrgTest + Context("in primary state", func() { + createTestTemplate := &template{ + ClaimBindInfo: corev1.ClaimPending, + VolumeBindInfo: corev1.VolumePending, + schedulingInterval: "1h", + storageClassName: "manual", + replicationClassName: "test-replicationclass", + vrcProvisioner: "manual.storage.com", + scProvisioner: "manual.storage.com", + replicationClassLabels: map[string]string{"protection": "ramen"}, + } + It("sets up PVCs, PVs and VRGs", func() { + createTestTemplate.s3Profiles = []string{s3Profiles[vrgS3ProfileNumber].S3ProfileName} + vrgPVCnotBoundVGRTestCase = newVRGTestCaseCreate(3, createTestTemplate, false, false) + vrgPVCnotBoundVGRTestCase.repGroup = true + vrgPVCnotBoundVGRTestCase.VRGTestCaseStart() + }) + It("expect no VR to be created as PVC not bound", func() { + expectedVGRCount := 0 + vrgPVCnotBoundVGRTestCase.waitForVGRCountToMatch(expectedVGRCount) + }) + It("bind each pv to corresponding pvc", func() { + vrgPVCnotBoundVGRTestCase.bindPVAndPVC() + vrgPVCnotBoundVGRTestCase.verifyPVCBindingToPV(true) + }) + It("waits for VRG to create one VGR resource for all PVCs", func() { + expectedVGRCount := 1 + vrgPVCnotBoundVGRTestCase.waitForVGRCountToMatch(expectedVGRCount) + }) + It("waits for VRG status to match", func() { + vrgPVCnotBoundVGRTestCase.promoteVolGroupReps() + vrgPVCnotBoundVGRTestCase.verifyVRGStatusExpectation(true, vrgController.VRGConditionReasonReady) + }) + It("cleans up after testing", func() { + vrgPVCnotBoundVGRTestCase.cleanupProtected() + }) + }) + // Try the simple case of creating VRG, PVC, PV and // check whether VolRep resources are created or not var vrgTestCases []*vrgTest @@ -1150,6 +1294,7 @@ type vrgTest struct { skipCreationPVandPVC bool checkBind bool vrgFirst bool + repGroup bool template *template } @@ -1215,7 +1360,12 @@ func (v *vrgTest) VRGTestCaseStart() { By("Creating namespace " + v.namespace) v.createNamespace() v.createSC(v.template) - v.createVRC(v.template) + + if v.repGroup { + v.createVGRC(v.template) + } else { + v.createVRC(v.template) + } if v.vrgFirst { v.createVRG() @@ -1491,11 +1641,14 @@ func (v *vrgTest) bindPVAndPVC() { i := i // capture i for use in closure // Bind PVC - pvc := getPVC(v.pvcNames[i]) - pvc.Status.Phase = corev1.ClaimBound - err = k8sClient.Status().Update(context.TODO(), pvc) - Expect(err).To(BeNil(), - "failed to update status of PVC %s", v.pvcNames[i]) + retryErr := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + pvc := getPVC(v.pvcNames[i]) + pvc.Status.Phase = corev1.ClaimBound + + return k8sClient.Status().Update(context.TODO(), pvc) + }) + + Expect(retryErr).NotTo(HaveOccurred()) } } @@ -1527,6 +1680,15 @@ func (v *vrgTest) createVRG() { S3Profiles: v.template.s3Profiles, }, } + + if v.repGroup { + if vrg.ObjectMeta.Annotations == nil { + vrg.ObjectMeta.Annotations = map[string]string{} + } + + vrg.ObjectMeta.Annotations[util.IsCGEnabledAnnotation] = "true" + } + err := k8sClient.Create(context.TODO(), vrg) expectedErr := errors.NewAlreadyExists( schema.GroupResource{ @@ -1606,6 +1768,45 @@ func (v *vrgTest) createVRC(testTemplate *template) { } } +func (v *vrgTest) createVGRC(testTemplate *template) { + defaultAnnotations := map[string]string{} + defaultAnnotations["replication.storage.openshift.io/is-default-class"] = "true" + + By("creating VGRC " + v.replicationClass) + + parameters := make(map[string]string) + + if testTemplate.schedulingInterval != "" { + parameters["schedulingInterval"] = testTemplate.schedulingInterval + } + + vrc := &volrep.VolumeGroupReplicationClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: v.replicationClass, + Namespace: v.namespace, + Annotations: defaultAnnotations, + }, + Spec: volrep.VolumeGroupReplicationClassSpec{ + Provisioner: testTemplate.vrcProvisioner, + Parameters: parameters, + }, + } + + if len(testTemplate.replicationClassLabels) > 0 { + vrc.ObjectMeta.Labels = testTemplate.replicationClassLabels + } + + err := k8sClient.Create(context.TODO(), vrc) + if err != nil { + if errors.IsAlreadyExists(err) { + err = k8sClient.Get(context.TODO(), types.NamespacedName{Name: v.replicationClass}, vrc) + } + } + + Expect(err).NotTo(HaveOccurred(), + "failed to create/get VolumeGroupReplicationClass %s/%s", v.replicationClass, v.vrgName) +} + func (v *vrgTest) createSC(testTemplate *template) { By("creating StorageClass " + v.storageClass) @@ -1616,6 +1817,9 @@ func (v *vrgTest) createSC(testTemplate *template) { sc := &storagev1.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: v.storageClass, + Labels: map[string]string{ + vrgController.StorageIDLabel: "test-storageid", + }, }, Provisioner: testTemplate.scProvisioner, } @@ -1885,6 +2089,7 @@ func (v *vrgTest) cleanup( v.cleanupNamespace() v.cleanupSC() v.cleanupVRC() + v.cleanupVGRC() } func (v *vrgTest) cleanupPVCs( @@ -2132,6 +2337,26 @@ func (v *vrgTest) cleanupVRC() { "failed to delete replicationClass %s", v.replicationClass) } +func (v *vrgTest) cleanupVGRC() { + key := types.NamespacedName{ + Name: v.replicationClass, + Namespace: v.namespace, + } + + vgrc := &volrep.VolumeGroupReplicationClass{} + + err := k8sClient.Get(context.TODO(), key, vgrc) + if err != nil { + if errors.IsNotFound(err) { + return + } + } + + err = k8sClient.Delete(context.TODO(), vgrc) + Expect(err).To(BeNil(), + "failed to delete replicationClass %s", v.replicationClass) +} + func (v *vrgTest) cleanupNamespace() { By("deleting namespace " + v.namespace) @@ -2164,6 +2389,24 @@ func (v *vrgTest) waitForVRCountToMatch(vrCount int) { vrCount, v.vrgName, v.namespace) } +func (v *vrgTest) waitForVGRCountToMatch(vgrCount int) { + By("Waiting for VRs count to match " + v.namespace) + + Eventually(func() int { + listOptions := &client.ListOptions{ + Namespace: v.namespace, + } + volGroupRepList := &volrep.VolumeGroupReplicationList{} + err := k8sClient.List(context.TODO(), volGroupRepList, listOptions) + Expect(err).NotTo(HaveOccurred(), + "failed to get a list of VGRs in namespace %s", v.namespace) + + return len(volGroupRepList.Items) + }, timeout, interval).Should(BeNumerically("==", vgrCount), + "while waiting for VGR count of %d in VRG %s of namespace %s", + vgrCount, v.vrgName, v.namespace) +} + func (v *vrgTest) promoteVolReps() { v.promoteVolRepsAndDo(func(index, count int) { // VRG should not be ready until last VolRep is ready. @@ -2171,10 +2414,18 @@ func (v *vrgTest) promoteVolReps() { }) } +func (v *vrgTest) promoteVolGroupReps() { + v.promoteVolGroupRepsAndDo(func(index, count int) { + // VRG should not be ready until last VolRep is ready. + v.verifyVRGStatusExpectation(index == count-1, vrgController.VRGConditionReasonReady) + }) +} + func (v *vrgTest) promoteVolRepsWithoutVrgStatusCheck() { v.promoteVolRepsAndDo(func(index, count int) {}) } +//nolint:dupl func (v *vrgTest) promoteVolRepsAndDo(do func(int, int)) { By("Promoting VolumeReplication resources " + v.namespace) @@ -2231,6 +2482,65 @@ func (v *vrgTest) promoteVolRepsAndDo(do func(int, int)) { } } +// nolint: dupl +func (v *vrgTest) promoteVolGroupRepsAndDo(do func(int, int)) { + By("Promoting VolumeGroupReplication resources " + v.namespace) + + volGroupRepList := &volrep.VolumeGroupReplicationList{} + listOptions := &client.ListOptions{ + Namespace: v.namespace, + } + err := k8sClient.List(context.TODO(), volGroupRepList, listOptions) + Expect(err).NotTo(HaveOccurred(), "failed to get a list of VRs in namespace %s", v.namespace) + + for index := range volGroupRepList.Items { + volGroup := volGroupRepList.Items[index] + + volGroupRepStatus := volrep.VolumeGroupReplicationStatus{ + VolumeReplicationStatus: volrep.VolumeReplicationStatus{ + Conditions: []metav1.Condition{ + { + Type: volrepController.ConditionCompleted, + Reason: volrepController.Promoted, + ObservedGeneration: volGroup.Generation, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Now()), + }, + { + Type: volrepController.ConditionDegraded, + Reason: volrepController.Healthy, + ObservedGeneration: volGroup.Generation, + Status: metav1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Now()), + }, + { + Type: volrepController.ConditionResyncing, + Reason: volrepController.NotResyncing, + ObservedGeneration: volGroup.Generation, + Status: metav1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Now()), + }, + }, + }, + } + volGroupRepStatus.ObservedGeneration = volGroup.Generation + volGroupRepStatus.State = volrep.PrimaryState + volGroupRepStatus.Message = "volume is marked primary" + volGroup.Status = volGroupRepStatus + + err = k8sClient.Status().Update(context.TODO(), &volGroup) + Expect(err).NotTo(HaveOccurred(), "failed to update the status of VolGroupRep %s", volGroup.Name) + + volrepKey := types.NamespacedName{ + Name: volGroup.Name, + Namespace: volGroup.Namespace, + } + v.waitForVolGroupRepPromotion(volrepKey) + + do(index, len(volGroupRepList.Items)) + } +} + func (v *vrgTest) protectDeletionOfVolReps() { By("Adding a finalizer to protect VolumeReplication resources being deleted " + v.namespace) @@ -2250,6 +2560,25 @@ func (v *vrgTest) protectDeletionOfVolReps() { } } +func (v *vrgTest) protectDeletionOfVolGroupReps() { + By("Adding a finalizer to protect VolumeGroupReplication resources being deleted " + v.namespace) + + volGroupRepList := &volrep.VolumeGroupReplicationList{} + listOptions := &client.ListOptions{ + Namespace: v.namespace, + } + err := apiReader.List(context.TODO(), volGroupRepList, listOptions) + Expect(err).NotTo(HaveOccurred(), "failed to get a list of VGRs in namespace %s", v.namespace) + + for index := range volGroupRepList.Items { + volGroupRep := volGroupRepList.Items[index] + if controllerutil.AddFinalizer(client.Object(&volGroupRep), "testDeleteProtected") { + err = k8sClient.Update(context.TODO(), &volGroupRep) + Expect(err).NotTo(HaveOccurred(), "failed to add finalizer to VolGroupRep %s", volGroupRep.Name) + } + } +} + func (v *vrgTest) unprotectDeletionOfVolReps() { By("Removing finalizer that protects VolumeReplication resources from being deleted " + v.namespace) @@ -2269,6 +2598,25 @@ func (v *vrgTest) unprotectDeletionOfVolReps() { } } +func (v *vrgTest) unprotectDeletionOfVolGroupReps() { + By("Removing finalizer that protects VolumeGroupReplication resources from being deleted " + v.namespace) + + volGroupRepList := &volrep.VolumeGroupReplicationList{} + listOptions := &client.ListOptions{ + Namespace: v.namespace, + } + err := apiReader.List(context.TODO(), volGroupRepList, listOptions) + Expect(err).NotTo(HaveOccurred(), "failed to get a list of VGRs in namespace %s", v.namespace) + + for index := range volGroupRepList.Items { + volGroupRep := volGroupRepList.Items[index] + if controllerutil.RemoveFinalizer(client.Object(&volGroupRep), "testDeleteProtected") { + err = k8sClient.Update(context.TODO(), &volGroupRep) + Expect(err).NotTo(HaveOccurred(), "failed to remove finalizer to VolGroupRep %s", volGroupRep.Name) + } + } +} + func (v *vrgTest) waitForVolRepPromotion(vrNamespacedName types.NamespacedName) { updatedVolRep := volrep.VolumeReplication{} @@ -2297,6 +2645,55 @@ func (v *vrgTest) waitForVolRepPromotion(vrNamespacedName types.NamespacedName) "while waiting for protected pvc condition %s/%s", updatedVolRep.Namespace, updatedVolRep.Name) } +func (v *vrgTest) waitForVolGroupRepPromotion(vrNamespacedName types.NamespacedName) { + updatedVolGroupRep := volrep.VolumeGroupReplication{} + + Eventually(func() bool { + err := k8sClient.Get(context.TODO(), vrNamespacedName, &updatedVolGroupRep) + + return err == nil && len(updatedVolGroupRep.Status.Conditions) == 3 + }, vrgtimeout, vrginterval).Should(BeTrue(), + "failed to wait for volRep condition type to change to 'ConditionCompleted' (%d)", + len(updatedVolGroupRep.Status.Conditions)) + + Eventually(func() bool { + vrg := v.getVRG() + + pvcLabelSelector := updatedVolGroupRep.Spec.Source.Selector + + pvcSelector, err := metav1.LabelSelectorAsSelector(pvcLabelSelector) + if err != nil { + return false + } + listOptions := []client.ListOption{ + client.MatchingLabelsSelector{ + Selector: pvcSelector, + }, + } + + pvcList := &corev1.PersistentVolumeClaimList{} + if err := k8sClient.List(context.TODO(), pvcList, listOptions...); err != nil { + return false + } + + protected := false + for idx := range pvcList.Items { + pvc := pvcList.Items[idx] + protectedPVC := vrgController.FindProtectedPVC(vrg, pvc.Namespace, pvc.Name) + if protectedPVC == nil { + continue + } + protected = v.checkProtectedPVCSuccess(vrg, protectedPVC) + if !protected { + return false + } + } + + return protected + }, vrgtimeout, vrginterval).Should(BeTrue(), + "while waiting for protected pvc condition %s/%s", updatedVolGroupRep.Namespace, updatedVolGroupRep.Name) +} + func (v *vrgTest) checkProtectedPVCSuccess(vrg *ramendrv1alpha1.VolumeReplicationGroup, protectedPVC *ramendrv1alpha1.ProtectedPVC, ) bool { diff --git a/test/addons/rook-pool/storage-class.yaml b/test/addons/rook-pool/storage-class.yaml index 36a077b7bd..79069d9653 100644 --- a/test/addons/rook-pool/storage-class.yaml +++ b/test/addons/rook-pool/storage-class.yaml @@ -6,6 +6,8 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: rook-ceph-block + labels: + ramendr.openshift.io/storageid: rook-ceph-storage-id provisioner: rook-ceph.rbd.csi.ceph.com parameters: clusterID: rook-ceph