From f2d9dbf3f68b09f138379545784a6baae48a789b Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Mon, 13 May 2024 13:15:40 -0400 Subject: [PATCH 01/11] drpc conditions in wide output Signed-off-by: Raghavendra Talur --- api/v1alpha1/drplacementcontrol_types.go | 2 ++ .../bases/ramendr.openshift.io_drplacementcontrols.yaml | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/api/v1alpha1/drplacementcontrol_types.go b/api/v1alpha1/drplacementcontrol_types.go index b4cb8d04a..4c4141097 100644 --- a/api/v1alpha1/drplacementcontrol_types.go +++ b/api/v1alpha1/drplacementcontrol_types.go @@ -246,7 +246,9 @@ type DRPlacementControlStatus struct { // +kubebuilder:printcolumn:JSONPath=".status.progression",name=progression,type=string,priority=2 // +kubebuilder:printcolumn:JSONPath=".status.actionStartTime",name=start time,type=string,priority=2 // +kubebuilder:printcolumn:JSONPath=".status.actionDuration",name=duration,type=string,priority=2 +// +kubebuilder:printcolumn:JSONPath=".status.conditions[0].status",name=available,type=string,priority=2 // +kubebuilder:printcolumn:JSONPath=".status.conditions[1].status",name=peer ready,type=string,priority=2 +// +kubebuilder:printcolumn:JSONPath=".status.conditions[2].status",name=protected,type=string,priority=2 // +kubebuilder:resource:shortName=drpc // DRPlacementControl is the Schema for the drplacementcontrols API diff --git a/config/crd/bases/ramendr.openshift.io_drplacementcontrols.yaml b/config/crd/bases/ramendr.openshift.io_drplacementcontrols.yaml index 6b255d74f..acc513fb8 100644 --- a/config/crd/bases/ramendr.openshift.io_drplacementcontrols.yaml +++ b/config/crd/bases/ramendr.openshift.io_drplacementcontrols.yaml @@ -44,10 +44,18 @@ spec: name: duration priority: 2 type: string + - jsonPath: .status.conditions[0].status + name: available + priority: 2 + type: string - jsonPath: .status.conditions[1].status name: peer ready priority: 2 type: string + - jsonPath: .status.conditions[2].status + name: protected + priority: 2 + type: string name: v1alpha1 schema: openAPIV3Schema: From efbf62064eeff687ccbde4bc9762534e283ff90b Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Fri, 7 Jun 2024 03:51:12 -0400 Subject: [PATCH 02/11] makefile support for x86_64 Signed-off-by: Raghavendra Talur --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 0de1cdf16..ca6df308b 100644 --- a/Makefile +++ b/Makefile @@ -217,6 +217,9 @@ run-dr-cluster: generate manifests ## Run DR manager controller from your host. docker-build: ## Build docker image with the manager. $(DOCKERCMD) build -t ${IMG} . +docker-build-x86-64: ## Build docker image with the manager. + $(DOCKERCMD) build -t ${IMG} --platform linux/amd64 . + docker-push: ## Push docker image with the manager. $(DOCKERCMD) push ${IMG} From 5322b06056b22bfc3149bb81ca2d3378d5533e3d Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Tue, 16 Apr 2024 23:33:12 -0400 Subject: [PATCH 03/11] my env Signed-off-by: Raghavendra Talur --- hack/dev-env.sh | 4 ++-- test/envs/regional-dr.yaml | 14 ++++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/hack/dev-env.sh b/hack/dev-env.sh index 8c74c907e..f02f66d02 100755 --- a/hack/dev-env.sh +++ b/hack/dev-env.sh @@ -20,9 +20,9 @@ cd "$script_dir"/.. cd test if [[ $1 == "create" ]]; then - drenv start --name-prefix "${RDR_NAME_PREFIX}"- envs/regional-dr.yaml + drenv start -v --name-prefix "${RDR_NAME_PREFIX}"- envs/regional-dr.yaml fi if [[ $1 == "destroy" ]]; then - drenv delete --name-prefix "${RDR_NAME_PREFIX}"- envs/regional-dr.yaml + drenv delete -v --name-prefix "${RDR_NAME_PREFIX}"- envs/regional-dr.yaml fi diff --git a/test/envs/regional-dr.yaml b/test/envs/regional-dr.yaml index 89378e04d..ae86310c8 100644 --- a/test/envs/regional-dr.yaml +++ b/test/envs/regional-dr.yaml @@ -14,11 +14,16 @@ ramen: templates: - name: "dr-cluster" + external: true driver: "$vm" container_runtime: containerd + containerd: + plugins: + io.containerd.grpc.v1.cri: + device_ownership_from_security_context: true network: "$network" - cpus: 4 - memory: "6g" + cpus: 8 + memory: "16g" extra_disks: 1 disk_size: "50g" feature_gates: @@ -42,11 +47,12 @@ templates: - name: minio - name: velero - name: "hub-cluster" + external: true driver: "$vm" container_runtime: containerd network: "$network" - cpus: 2 - memory: "4g" + cpus: 4 + memory: "16g" workers: - addons: - name: ocm-hub From 6dbc5c18f875335fdb266550b8b106b13ffd822f Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Fri, 10 May 2024 10:51:02 -0400 Subject: [PATCH 04/11] my makefile vars Signed-off-by: Raghavendra Talur --- Makefile | 2 ++ config/manager/manager.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ca6df308b..2d9241a9a 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,8 @@ DEFAULT_CHANNEL := alpha endif BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) +include developer.mk + IMAGE_REGISTRY ?= quay.io IMAGE_REPOSITORY ?= ramendr IMAGE_NAME ?= ramen diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index bdafc52a4..5fce4d297 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -30,7 +30,7 @@ spec: - command: - /manager image: controller - imagePullPolicy: IfNotPresent + imagePullPolicy: Always name: manager env: - name: POD_NAMESPACE From 8df829d6c10217d1fda5e7ef2addaacb240f1b60 Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Tue, 13 Aug 2024 15:55:56 -0400 Subject: [PATCH 05/11] my config for e2e Signed-off-by: Raghavendra Talur --- e2e/config.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/e2e/config.yaml b/e2e/config.yaml index d078d7734..529b1ee42 100644 --- a/e2e/config.yaml +++ b/e2e/config.yaml @@ -7,3 +7,10 @@ pvcspecs: accessmodes: ReadWriteMany - storageclassname: rook-ceph-block accessmodes: ReadWriteOnce +Clusters: + c1: + kubeconfigpath: /Users/rtalur/.config/drenv/rdr-rdr/kubeconfigs/rdr-dr1 + c2: + kubeconfigpath: /Users/rtalur/.config/drenv/rdr-rdr/kubeconfigs/rdr-dr2 + hub: + kubeconfigpath: /Users/rtalur/.config/drenv/rdr-rdr/kubeconfigs/rdr-hub From f49ca9494a3b2548ded391842f33b6c88a1346e4 Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Fri, 15 Mar 2024 03:59:50 -0400 Subject: [PATCH 06/11] tests: keep all suite cleanup functions in suite_test.go Signed-off-by: Raghavendra Talur --- internal/controller/ramenconfig_test.go | 24 ++++++++++++++++++++++-- internal/controller/suite_test.go | 1 + 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/internal/controller/ramenconfig_test.go b/internal/controller/ramenconfig_test.go index 03401ffe8..9ca218075 100644 --- a/internal/controller/ramenconfig_test.go +++ b/internal/controller/ramenconfig_test.go @@ -6,11 +6,11 @@ package controllers_test import ( "context" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ramen "github.com/ramendr/ramen/api/v1alpha1" controllers "github.com/ramendr/ramen/internal/controller" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" "sigs.k8s.io/yaml" @@ -26,10 +26,30 @@ func configMapCreate(ramenConfig *ramen.RamenConfig) { configMap, err := controllers.ConfigMapNew(ramenNamespace, configMapName, ramenConfig) Expect(err).NotTo(HaveOccurred()) Expect(k8sClient.Create(context.TODO(), configMap)).To(Succeed()) - DeferCleanup(k8sClient.Delete, context.TODO(), configMap) } } +func configMapDelete() error { + for _, configMapName := range configMapNames { + cm := &corev1.ConfigMap{} + + err := k8sClient.Get(context.TODO(), types.NamespacedName{ + Namespace: ramenNamespace, + Name: configMapName, + }, cm) + if err != nil && !errors.IsNotFound(err) { + return err + } + + err = k8sClient.Delete(context.TODO(), cm) + if err != nil && !errors.IsNotFound(err) { + return err + } + } + + return nil +} + func configMapUpdate() { ramenConfigYaml, err := yaml.Marshal(ramenConfig) Expect(err).NotTo(HaveOccurred()) diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index ed7329a25..265c0a17b 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -242,6 +242,7 @@ var _ = BeforeSuite(func() { ramenConfig.DrClusterOperator.S3SecretDistributionEnabled = true ramenConfig.MultiNamespace.FeatureEnabled = true configMapCreate(ramenConfig) + DeferCleanup(configMapDelete) s3Secrets[0] = corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Namespace: ramenNamespace, Name: "s3secret0"}, From 4244b77163903f7be233acfcde84fdcea0f17288 Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Fri, 15 Mar 2024 03:28:48 -0400 Subject: [PATCH 07/11] tests: dump the kubeconfig for the testenv server This is going to be useful to inspect the state of the cluster after a test fails. Set the env variable SKIP_CLEANUP to "true" or "1" to skip the cleanup of the environment to make use of the kubeconfig. The kubeconfig is dumped in the testbin directory. After debugging you will have to kill the testEnv. The two binaries that are used by the testEnv are etcd and kube-apiserver. Check for the running process with a command like ps aux | grep -e kube-apiserver -e etcd Signed-off-by: Raghavendra Talur --- internal/controller/suite_test.go | 46 ++++++++++++++++++- internal/controller/testutils.go | 46 +++++++++++++++++++ .../controller/volsync/volsync_suite_test.go | 28 ++++++++++- 3 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 internal/controller/testutils.go diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 265c0a17b..eb19decd9 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -5,6 +5,7 @@ package controllers_test import ( "context" + "log" "os" "path/filepath" "testing" @@ -89,6 +90,8 @@ var ( objectStorers [2]ramencontrollers.ObjectStorer ramenNamespace = "ns-envtest" + + skipCleanup bool ) func TestAPIs(t *testing.T) { @@ -146,6 +149,11 @@ var _ = BeforeSuite(func() { ramenNamespace = rNs } + skipCleanupEnv, set := os.LookupEnv("SKIP_CLEANUP") + if set && (skipCleanupEnv == "true" || skipCleanupEnv == "1") { + skipCleanup = true + } + By("bootstrapping test environment") testEnv = &envtest.Environment{ CRDDirectoryPaths: []string{ @@ -164,10 +172,25 @@ var _ = BeforeSuite(func() { defer GinkgoRecover() cfg, err = testEnv.Start() DeferCleanup(func() error { + if skipCleanup { + By("skipping cleanup of the test environment") + + return nil + } By("tearing down the test environment") return testEnv.Stop() }) + kubeConfigContent, err := ramencontrollers.ConvertRestConfigToKubeConfig(cfg) + if err != nil { + log.Fatalf("Failed to convert rest.Config to kubeconfig: %v", err) + } + + filePath := "../../testbin/kubeconfig.yaml" + if err := ramencontrollers.WriteKubeConfigToFile(kubeConfigContent, filePath); err != nil { + log.Fatalf("Failed to write kubeconfig file: %v", err) + } + close(done) }() Eventually(done).WithTimeout(time.Minute).Should(BeClosed()) @@ -242,7 +265,15 @@ var _ = BeforeSuite(func() { ramenConfig.DrClusterOperator.S3SecretDistributionEnabled = true ramenConfig.MultiNamespace.FeatureEnabled = true configMapCreate(ramenConfig) - DeferCleanup(configMapDelete) + DeferCleanup(func() error { + if skipCleanup { + By("skipping cleanup of the test environment") + + return nil + } + + return configMapDelete() + }) s3Secrets[0] = corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Namespace: ramenNamespace, Name: "s3secret0"}, @@ -385,7 +416,18 @@ var _ = BeforeSuite(func() { Expect(err).ToNot(HaveOccurred()) ctx, cancel = context.WithCancel(context.TODO()) - DeferCleanup(cancel) + DeferCleanup(func() error { + if skipCleanup { + By("skipping cleanup of the test environment") + + return nil + } + + cancel() + + return nil + }) + go func() { err = k8sManager.Start(ctx) Expect(err).ToNot(HaveOccurred()) diff --git a/internal/controller/testutils.go b/internal/controller/testutils.go new file mode 100644 index 000000000..12409da1c --- /dev/null +++ b/internal/controller/testutils.go @@ -0,0 +1,46 @@ +package controllers + +import ( + "os" + + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/clientcmd/api" +) + +const ownerReadWritePermission = 0o600 + +// WriteKubeConfigToFile writes the kubeconfig content to a specified file path. +func WriteKubeConfigToFile(kubeConfigContent, filePath string) error { + return os.WriteFile(filePath, []byte(kubeConfigContent), ownerReadWritePermission) +} + +// ConvertRestConfigToKubeConfig converts a rest.Config to a kubeconfig string. +func ConvertRestConfigToKubeConfig(restConfig *rest.Config) (string, error) { + kubeConfig := api.NewConfig() + + cluster := api.NewCluster() + cluster.Server = restConfig.Host + cluster.CertificateAuthorityData = restConfig.CAData + + user := api.NewAuthInfo() + user.ClientCertificateData = restConfig.CertData + user.ClientKeyData = restConfig.KeyData + user.Token = restConfig.BearerToken + + context := api.NewContext() + context.Cluster = "cluster" + context.AuthInfo = "user" + + kubeConfig.Clusters["cluster"] = cluster + kubeConfig.AuthInfos["user"] = user + kubeConfig.Contexts["test"] = context + kubeConfig.CurrentContext = "test" + + kubeConfigContent, err := clientcmd.Write(*kubeConfig) + if err != nil { + return "", err + } + + return string(kubeConfigContent), nil +} diff --git a/internal/controller/volsync/volsync_suite_test.go b/internal/controller/volsync/volsync_suite_test.go index 5f567aae4..d3cca10b1 100644 --- a/internal/controller/volsync/volsync_suite_test.go +++ b/internal/controller/volsync/volsync_suite_test.go @@ -5,6 +5,7 @@ package volsync_test import ( "context" + "log" "os" "path/filepath" "testing" @@ -19,6 +20,7 @@ import ( storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" cfgpolicyv1 "open-cluster-management.io/config-policy-controller/api/v1" policyv1 "open-cluster-management.io/governance-policy-propagator/api/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -29,6 +31,7 @@ import ( metrics "sigs.k8s.io/controller-runtime/pkg/metrics/server" volsyncv1alpha1 "github.com/backube/volsync/api/v1alpha1" + controllers "github.com/ramendr/ramen/internal/controller" "github.com/ramendr/ramen/internal/controller/util" ) @@ -56,6 +59,9 @@ var ( volumeSnapshotClassB *snapv1.VolumeSnapshotClass totalVolumeSnapshotClassCount = 0 + + skipCleanup bool + cfg *rest.Config ) func TestVolsync(t *testing.T) { @@ -64,6 +70,8 @@ func TestVolsync(t *testing.T) { } var _ = BeforeSuite(func() { + var err error + logger = zap.New(zap.UseFlagOptions(&zap.Options{ Development: true, DestWriter: GinkgoWriter, @@ -85,6 +93,11 @@ var _ = BeforeSuite(func() { Expect(os.Setenv("KUBEBUILDER_ASSETS", string(content))).To(Succeed()) } + skipCleanupEnv, set := os.LookupEnv("SKIP_CLEANUP") + if set && (skipCleanupEnv == "true" || skipCleanupEnv == "1") { + skipCleanup = true + } + By("bootstrapping test environment") testEnv = &envtest.Environment{ CRDDirectoryPaths: []string{ @@ -93,7 +106,7 @@ var _ = BeforeSuite(func() { }, } - cfg, err := testEnv.Start() + cfg, err = testEnv.Start() Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) @@ -217,6 +230,19 @@ var _ = BeforeSuite(func() { }) var _ = AfterSuite(func() { + if skipCleanup { + kubeConfigContent, err := controllers.ConvertRestConfigToKubeConfig(cfg) + if err != nil { + log.Fatalf("Failed to convert rest.Config to kubeconfig: %v", err) + } + + filePath := "../../../testbin/kubeconfig.yaml" + if err := controllers.WriteKubeConfigToFile(kubeConfigContent, filePath); err != nil { + log.Fatalf("Failed to write kubeconfig file: %v", err) + } + + return + } cancel() By("tearing down the test environment") err := testEnv.Stop() From 59d0ece6a898a7b4670ef96081c7de1779fa8a3f Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Tue, 23 Apr 2024 00:20:11 -0400 Subject: [PATCH 08/11] hack: customize script for OS type Signed-off-by: Raghavendra Talur --- hack/check-drenv-prereqs.sh | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/hack/check-drenv-prereqs.sh b/hack/check-drenv-prereqs.sh index e417dd7f7..46470097a 100755 --- a/hack/check-drenv-prereqs.sh +++ b/hack/check-drenv-prereqs.sh @@ -1,13 +1,20 @@ #!/bin/bash set -e -if ! groups "$USER" | grep -qw libvirt; then - echo "Error: User is not part of the libvirt group." - exit 1 +os=$(uname -s) + +if [[ "$os" == "Linux" ]] +then + if ! groups "$USER" | grep -qw libvirt; then + echo "Error: User is not part of the libvirt group." + exit 1 + fi fi commands=("minikube" "kubectl" "clusteradm" "subctl" "velero" "helm" "virtctl" -"virt-host-validate" "kustomize" "mc") +"kustomize" "mc") + +linux_only_commands=("virt-host-validate") for cmd in "${commands[@]}"; do if ! command -v "$cmd" &> /dev/null; then @@ -16,9 +23,23 @@ for cmd in "${commands[@]}"; do fi done -if ! virt-host-validate qemu -q; then - echo "Error: 'virt-host-validate qemu' did not return 0." - exit 1 + +if [[ "$os" == "Linux" ]] +then + for cmd in "${linux_only_commands[@]}"; do + if ! command -v "$cmd" &> /dev/null; then + echo "Error: $cmd could not be found in PATH." + exit 1 + fi + done +fi + +if [[ "$os" == "Linux" ]] +then + if ! virt-host-validate qemu -q; then + echo "Error: 'virt-host-validate qemu' did not return 0." + exit 1 + fi fi echo "All prerequisites met for drenv" From bc6922d9216ad97ff475601839857c12441df714 Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Mon, 18 Sep 2023 23:55:19 -0400 Subject: [PATCH 09/11] drenv: turn off check for broker certs On Mac, the check for certs is more strict and it fails for submariner service. Turning off the check for certs. More info: https://github.com/golang/go/issues/51991 Signed-off-by: Raghavendra Talur --- test/addons/submariner/start | 1 + test/drenv/subctl.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/test/addons/submariner/start b/test/addons/submariner/start index 324f6e6fe..a62f0f41a 100755 --- a/test/addons/submariner/start +++ b/test/addons/submariner/start @@ -60,6 +60,7 @@ def join_cluster(cluster, broker_info): clusterid=cluster, cable_driver="vxlan", version=VERSION, + check_broker_certificate=False, ) diff --git a/test/drenv/subctl.py b/test/drenv/subctl.py index 0b925d38b..8aadf2d8d 100644 --- a/test/drenv/subctl.py +++ b/test/drenv/subctl.py @@ -27,11 +27,20 @@ def deploy_broker(context, globalnet=False, broker_info=None, version=None, log= shutil.move(BROKER_INFO, broker_info) -def join(broker_info, context, clusterid, cable_driver=None, version=None, log=print): +def join( + broker_info, + context, + clusterid, + cable_driver=None, + version=None, + check_broker_certificate=True, + log=print, +): """ Run subctl join ... logging progress messages. """ args = ["join", broker_info, "--context", context, "--clusterid", clusterid] + args.append(f"--check-broker-certificate={check_broker_certificate}") if cable_driver: args.extend(("--cable-driver", cable_driver)) if version: From 5712c858ec3aa9e9a6307f8cb34ba4c45ca9d253 Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Tue, 27 Aug 2024 03:23:34 -0400 Subject: [PATCH 10/11] vrg: remove the use of captureInProgressStatusUpdate captureInProgressStatusUpdate was a function pointer that was an empty function when the VRG was being reconciled as Primary and a call to VRGConditionReasonUploading with status false when reconciling as Secondary in the relocate case. Simplify the code by removing the function pointer. * Remove the captureInProgressStatusUpdate parameter from kubeObjectsProtect kubeObjectsCaptureStartOrResumeOrDelay kubeObjectsCaptureStartOrResume kubeObjectsGroupCapture. Signed-off-by: Raghavendra Talur --- internal/controller/vrg_kubeobjects.go | 29 ++++++++------------------ 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/internal/controller/vrg_kubeobjects.go b/internal/controller/vrg_kubeobjects.go index f13c1e106..c7c1079ac 100644 --- a/internal/controller/vrg_kubeobjects.go +++ b/internal/controller/vrg_kubeobjects.go @@ -57,28 +57,18 @@ func kubeObjectsRecoverName(prefix string, groupNumber int) string { } func (v *VRGInstance) kubeObjectsProtectPrimary(result *ctrl.Result) { - v.kubeObjectsProtect(result, kubeObjectsCaptureStartConditionallyPrimary, - func() {}, - ) + v.kubeObjectsProtect(result, kubeObjectsCaptureStartConditionallyPrimary) } func (v *VRGInstance) kubeObjectsProtectSecondary(result *ctrl.Result) { - v.kubeObjectsProtect(result, kubeObjectsCaptureStartConditionallySecondary, - func() { - v.kubeObjectsCaptureStatusFalse(VRGConditionReasonUploading, "Kube objects capture for relocate in-progress") - }, - ) + v.kubeObjectsProtect(result, kubeObjectsCaptureStartConditionallySecondary) } -type ( - captureStartConditionally func(*VRGInstance, *ctrl.Result, int64, time.Duration, time.Duration, func()) - captureInProgressStatusUpdate func() -) +type captureStartConditionally func(*VRGInstance, *ctrl.Result, int64, time.Duration, time.Duration, func()) func (v *VRGInstance) kubeObjectsProtect( result *ctrl.Result, captureStartConditionally captureStartConditionally, - captureInProgressStatusUpdate captureInProgressStatusUpdate, ) { if v.kubeObjectProtectionDisabled("capture") { return @@ -106,7 +96,6 @@ func (v *VRGInstance) kubeObjectsProtect( v.kubeObjectsCaptureStartOrResumeOrDelay(result, captureStartConditionally, - captureInProgressStatusUpdate, captureToRecoverFrom, ) } @@ -114,7 +103,6 @@ func (v *VRGInstance) kubeObjectsProtect( func (v *VRGInstance) kubeObjectsCaptureStartOrResumeOrDelay( result *ctrl.Result, captureStartConditionally captureStartConditionally, - captureInProgressStatusUpdate captureInProgressStatusUpdate, captureToRecoverFrom *ramen.KubeObjectsCaptureIdentifier, ) { veleroNamespaceName := v.veleroNamespaceName() @@ -141,7 +129,6 @@ func (v *VRGInstance) kubeObjectsCaptureStartOrResumeOrDelay( log.Info("Kube objects capture "+startOrResume, "generation", generation) v.kubeObjectsCaptureStartOrResume(result, captureStartConditionally, - captureInProgressStatusUpdate, number, pathName, capturePathName, namePrefix, veleroNamespaceName, interval, labels, generation, kubeobjects.RequestsMapKeyedByName(requests), @@ -230,7 +217,6 @@ const ( func (v *VRGInstance) kubeObjectsCaptureStartOrResume( result *ctrl.Result, captureStartConditionally captureStartConditionally, - captureInProgressStatusUpdate captureInProgressStatusUpdate, captureNumber int64, pathName, capturePathName, namePrefix, veleroNamespaceName string, interval time.Duration, @@ -248,7 +234,6 @@ func (v *VRGInstance) kubeObjectsCaptureStartOrResume( log1 := log.WithValues("group", groupNumber, "name", captureGroup.Name) requestsCompletedCount += v.kubeObjectsGroupCapture( result, captureGroup, pathName, capturePathName, namePrefix, veleroNamespaceName, - captureInProgressStatusUpdate, labels, annotations, requests, log, ) requestsProcessedCount += len(v.s3StoreAccessors) @@ -278,7 +263,6 @@ func (v *VRGInstance) kubeObjectsGroupCapture( result *ctrl.Result, captureGroup kubeobjects.CaptureSpec, pathName, capturePathName, namePrefix, veleroNamespaceName string, - captureInProgressStatusUpdate captureInProgressStatusUpdate, labels, annotations map[string]string, requests map[string]kubeobjects.Request, log logr.Logger, ) (requestsCompletedCount int) { @@ -328,7 +312,12 @@ func (v *VRGInstance) kubeObjectsGroupCapture( return } - captureInProgressStatusUpdate() + if v.instance.Spec.ReplicationState == ramen.Secondary && + v.instance.Spec.Action == ramen.VRGActionRelocate { + v.kubeObjectsCaptureStatusFalse(VRGConditionReasonUploading, + "Kube objects capture for relocate in-progress") + } + } return requestsCompletedCount From 36ad6eed6ae8739bf5314bd5e0a9add47fc9f6ef Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Tue, 27 Aug 2024 04:16:04 -0400 Subject: [PATCH 11/11] vrg: remove the use of captureStartConditionally captureStartConditionally was a function pointer that was * capturing kube objects one last time when the vrg was secondary * capturing kube objects if time since last capture was greater than the capture interval Refactored into one function that performs the capture based on the vrg spec. Signed-off-by: Raghavendra Talur --- internal/controller/vrg_kubeobjects.go | 53 ++++++++++---------------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/internal/controller/vrg_kubeobjects.go b/internal/controller/vrg_kubeobjects.go index c7c1079ac..1340c2f77 100644 --- a/internal/controller/vrg_kubeobjects.go +++ b/internal/controller/vrg_kubeobjects.go @@ -57,18 +57,15 @@ func kubeObjectsRecoverName(prefix string, groupNumber int) string { } func (v *VRGInstance) kubeObjectsProtectPrimary(result *ctrl.Result) { - v.kubeObjectsProtect(result, kubeObjectsCaptureStartConditionallyPrimary) + v.kubeObjectsProtect(result) } func (v *VRGInstance) kubeObjectsProtectSecondary(result *ctrl.Result) { - v.kubeObjectsProtect(result, kubeObjectsCaptureStartConditionallySecondary) + v.kubeObjectsProtect(result) } -type captureStartConditionally func(*VRGInstance, *ctrl.Result, int64, time.Duration, time.Duration, func()) - func (v *VRGInstance) kubeObjectsProtect( result *ctrl.Result, - captureStartConditionally captureStartConditionally, ) { if v.kubeObjectProtectionDisabled("capture") { return @@ -95,14 +92,12 @@ func (v *VRGInstance) kubeObjectsProtect( } v.kubeObjectsCaptureStartOrResumeOrDelay(result, - captureStartConditionally, captureToRecoverFrom, ) } func (v *VRGInstance) kubeObjectsCaptureStartOrResumeOrDelay( result *ctrl.Result, - captureStartConditionally captureStartConditionally, captureToRecoverFrom *ramen.KubeObjectsCaptureIdentifier, ) { veleroNamespaceName := v.veleroNamespaceName() @@ -128,7 +123,6 @@ func (v *VRGInstance) kubeObjectsCaptureStartOrResumeOrDelay( captureStartOrResume := func(generation int64, startOrResume string) { log.Info("Kube objects capture "+startOrResume, "generation", generation) v.kubeObjectsCaptureStartOrResume(result, - captureStartConditionally, number, pathName, capturePathName, namePrefix, veleroNamespaceName, interval, labels, generation, kubeobjects.RequestsMapKeyedByName(requests), @@ -142,7 +136,7 @@ func (v *VRGInstance) kubeObjectsCaptureStartOrResumeOrDelay( return } - captureStartConditionally( + kubeObjectsCaptureStartConditionally( v, result, captureToRecoverFrom.StartGeneration, time.Since(captureToRecoverFrom.StartTime.Time), interval, func() { if v.kubeObjectsCapturesDelete(result, number, capturePathName) != nil { @@ -154,7 +148,7 @@ func (v *VRGInstance) kubeObjectsCaptureStartOrResumeOrDelay( ) } -func kubeObjectsCaptureStartConditionallySecondary( +func kubeObjectsCaptureStartConditionally( v *VRGInstance, result *ctrl.Result, captureStartGeneration int64, captureStartTimeSince, captureStartInterval time.Duration, captureStart func(), @@ -162,26 +156,26 @@ func kubeObjectsCaptureStartConditionallySecondary( generation := v.instance.Generation log := v.log.WithValues("generation", generation) - if captureStartGeneration == generation { - log.Info("Kube objects capture for relocate complete") + // Capture one last time for secondary before relocating + if v.instance.Spec.ReplicationState == ramen.Secondary && + v.instance.Spec.Action != ramen.VRGActionRelocate { + if captureStartGeneration == generation { + log.Info("Kube objects capture for relocate complete") - return + return + } + + v.kubeObjectsCaptureStatusFalse(VRGConditionReasonUploading, "Kube objects capture for relocate pending") } - v.kubeObjectsCaptureStatusFalse(VRGConditionReasonUploading, "Kube objects capture for relocate pending") - captureStart() -} + // Capture for primary is time since last capture is greater than interval + if v.instance.Spec.ReplicationState == ramen.Primary { + if delay := captureStartInterval - captureStartTimeSince; delay > 0 { + v.log.Info("Kube objects capture start delay", "delay", delay, "interval", captureStartInterval) + delaySetIfLess(result, delay, v.log) -func kubeObjectsCaptureStartConditionallyPrimary( - v *VRGInstance, result *ctrl.Result, - captureStartGeneration int64, captureStartTimeSince, captureStartInterval time.Duration, - captureStart func(), -) { - if delay := captureStartInterval - captureStartTimeSince; delay > 0 { - v.log.Info("Kube objects capture start delay", "delay", delay, "interval", captureStartInterval) - delaySetIfLess(result, delay, v.log) - - return + return + } } captureStart() @@ -216,7 +210,6 @@ const ( func (v *VRGInstance) kubeObjectsCaptureStartOrResume( result *ctrl.Result, - captureStartConditionally captureStartConditionally, captureNumber int64, pathName, capturePathName, namePrefix, veleroNamespaceName string, interval time.Duration, @@ -249,7 +242,6 @@ func (v *VRGInstance) kubeObjectsCaptureStartOrResume( v.kubeObjectsCaptureComplete( result, - captureStartConditionally, captureNumber, veleroNamespaceName, interval, @@ -344,7 +336,6 @@ func (v *VRGInstance) kubeObjectsCaptureDeleteAndLog( func (v *VRGInstance) kubeObjectsCaptureComplete( result *ctrl.Result, - captureStartConditionally captureStartConditionally, captureNumber int64, veleroNamespaceName string, interval time.Duration, labels map[string]string, startTime metav1.Time, annotations map[string]string, ) { @@ -371,7 +362,6 @@ func (v *VRGInstance) kubeObjectsCaptureComplete( func() { v.kubeObjectsCaptureIdentifierUpdateComplete( result, - captureStartConditionally, *captureToRecoverFromIdentifier, veleroNamespaceName, interval, @@ -386,7 +376,6 @@ func (v *VRGInstance) kubeObjectsCaptureComplete( func (v *VRGInstance) kubeObjectsCaptureIdentifierUpdateComplete( result *ctrl.Result, - captureStartConditionally captureStartConditionally, captureToRecoverFromIdentifier *ramen.KubeObjectsCaptureIdentifier, veleroNamespaceName string, interval time.Duration, @@ -409,7 +398,7 @@ func (v *VRGInstance) kubeObjectsCaptureIdentifierUpdateComplete( captureStartTimeSince := time.Since(captureToRecoverFromIdentifier.StartTime.Time) v.log.Info("Kube objects captured", "recovery point", captureToRecoverFromIdentifier, "duration", captureStartTimeSince) - captureStartConditionally( + kubeObjectsCaptureStartConditionally( v, result, captureToRecoverFromIdentifier.StartGeneration, captureStartTimeSince, interval, func() { v.log.Info("Kube objects capture schedule to run immediately")