From 708e7b02a6056d9f43573df08da50c43f2665b60 Mon Sep 17 00:00:00 2001 From: Leela Venkaiah G Date: Tue, 15 Oct 2024 06:28:58 +0000 Subject: [PATCH 1/7] remove extra deep copy of object from cache controller-runtime accepts the pointer to the memory which is reserved by application and runtime deepcopies the object in it's cache to the memory provided by application. this commit removes extra copy of driver spec which isn't required. Signed-off-by: Leela Venkaiah G --- internal/controller/driver_controller.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/controller/driver_controller.go b/internal/controller/driver_controller.go index 75a82b86..a6ca04df 100644 --- a/internal/controller/driver_controller.go +++ b/internal/controller/driver_controller.go @@ -312,9 +312,6 @@ func (r *driverReconcile) LoadAndValidateDesiredState() error { r.images = maps.Clone(imageDefaults) if opConfig.Spec.DriverSpecDefaults != nil { - // Creating a copy of the driver spec, making sure any local changes will not effect the object residing - // in the client's cache - r.driver.Spec = *r.driver.Spec.DeepCopy() mergeDriverSpecs(&r.driver.Spec, opConfig.Spec.DriverSpecDefaults) // If provided, load an imageset from configmap to overwrite default images From 9645b44c42a48b6e2c985f86907aa6b6f22d2a32 Mon Sep 17 00:00:00 2001 From: Leela Venkaiah G Date: Tue, 15 Oct 2024 11:37:11 +0000 Subject: [PATCH 2/7] provide ability to adjust controller plugin deployment strategy Signed-off-by: Leela Venkaiah G --- api/v1alpha1/driver_types.go | 5 + api/v1alpha1/zz_generated.deepcopy.go | 5 + config/crd/bases/csi.ceph.io_drivers.yaml | 52 +++++++++ .../bases/csi.ceph.io_operatorconfigs.yaml | 52 +++++++++ deploy/all-in-one/install.yaml | 104 ++++++++++++++++++ deploy/multifile/crd.yaml | 104 ++++++++++++++++++ 6 files changed, 322 insertions(+) diff --git a/api/v1alpha1/driver_types.go b/api/v1alpha1/driver_types.go index 5fa10638..b562eeaa 100644 --- a/api/v1alpha1/driver_types.go +++ b/api/v1alpha1/driver_types.go @@ -213,6 +213,11 @@ type ControllerPluginSpec struct { // Embedded common pods spec PodCommonSpec `json:",inline"` + // DeploymentStrategy describes how to replace existing pods with new ones + // Default value is RollingUpdate with MaxUnavailable and MaxSurege as 25% (kubernetes default) + //+kubebuilder:validation:Optional + DeploymentStrategy *appsv1.DeploymentStrategy `json:"deploymentStrategy,omitempty"` + // Set replicas for controller plugin's deployment. Defaults to 2 //+kubebuilder:validation:Optional //+kubebuilder:validation:Minimum:=1 diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index cfc03cf8..a8542df9 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -438,6 +438,11 @@ func (in *ControllerPluginResourcesSpec) DeepCopy() *ControllerPluginResourcesSp func (in *ControllerPluginSpec) DeepCopyInto(out *ControllerPluginSpec) { *out = *in in.PodCommonSpec.DeepCopyInto(&out.PodCommonSpec) + if in.DeploymentStrategy != nil { + in, out := &in.DeploymentStrategy, &out.DeploymentStrategy + *out = new(appsv1.DeploymentStrategy) + (*in).DeepCopyInto(*out) + } if in.Replicas != nil { in, out := &in.Replicas, &out.Replicas *out = new(int32) diff --git a/config/crd/bases/csi.ceph.io_drivers.yaml b/config/crd/bases/csi.ceph.io_drivers.yaml index aa0d2148..7fe11209 100644 --- a/config/crd/bases/csi.ceph.io_drivers.yaml +++ b/config/crd/bases/csi.ceph.io_drivers.yaml @@ -999,6 +999,58 @@ spec: type: string description: Pod's annotations type: object + deploymentStrategy: + description: |- + DeploymentStrategy describes how to replace existing pods with new ones + Default value is RollingUpdate with MaxUnavailable and MaxSurege as 25% (kubernetes default) + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or "RollingUpdate". + Default is RollingUpdate. + type: string + type: object imagePullPolicy: description: To indicate the image pull policy to be applied to all the containers in the csi driver pods. diff --git a/config/crd/bases/csi.ceph.io_operatorconfigs.yaml b/config/crd/bases/csi.ceph.io_operatorconfigs.yaml index db633aff..9e4884a3 100644 --- a/config/crd/bases/csi.ceph.io_operatorconfigs.yaml +++ b/config/crd/bases/csi.ceph.io_operatorconfigs.yaml @@ -1008,6 +1008,58 @@ spec: type: string description: Pod's annotations type: object + deploymentStrategy: + description: |- + DeploymentStrategy describes how to replace existing pods with new ones + Default value is RollingUpdate with MaxUnavailable and MaxSurege as 25% (kubernetes default) + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or + "RollingUpdate". Default is RollingUpdate. + type: string + type: object imagePullPolicy: description: To indicate the image pull policy to be applied to all the containers in the csi driver pods. diff --git a/deploy/all-in-one/install.yaml b/deploy/all-in-one/install.yaml index 0e58d129..83b5f7d7 100644 --- a/deploy/all-in-one/install.yaml +++ b/deploy/all-in-one/install.yaml @@ -1249,6 +1249,58 @@ spec: type: string description: Pod's annotations type: object + deploymentStrategy: + description: |- + DeploymentStrategy describes how to replace existing pods with new ones + Default value is RollingUpdate with MaxUnavailable and MaxSurege as 25% (kubernetes default) + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or "RollingUpdate". + Default is RollingUpdate. + type: string + type: object imagePullPolicy: description: To indicate the image pull policy to be applied to all the containers in the csi driver pods. @@ -8057,6 +8109,58 @@ spec: type: string description: Pod's annotations type: object + deploymentStrategy: + description: |- + DeploymentStrategy describes how to replace existing pods with new ones + Default value is RollingUpdate with MaxUnavailable and MaxSurege as 25% (kubernetes default) + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or + "RollingUpdate". Default is RollingUpdate. + type: string + type: object imagePullPolicy: description: To indicate the image pull policy to be applied to all the containers in the csi driver pods. diff --git a/deploy/multifile/crd.yaml b/deploy/multifile/crd.yaml index 3155f0a4..09b88da6 100644 --- a/deploy/multifile/crd.yaml +++ b/deploy/multifile/crd.yaml @@ -1240,6 +1240,58 @@ spec: type: string description: Pod's annotations type: object + deploymentStrategy: + description: |- + DeploymentStrategy describes how to replace existing pods with new ones + Default value is RollingUpdate with MaxUnavailable and MaxSurege as 25% (kubernetes default) + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or "RollingUpdate". + Default is RollingUpdate. + type: string + type: object imagePullPolicy: description: To indicate the image pull policy to be applied to all the containers in the csi driver pods. @@ -8048,6 +8100,58 @@ spec: type: string description: Pod's annotations type: object + deploymentStrategy: + description: |- + DeploymentStrategy describes how to replace existing pods with new ones + Default value is RollingUpdate with MaxUnavailable and MaxSurege as 25% (kubernetes default) + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or + "RollingUpdate". Default is RollingUpdate. + type: string + type: object imagePullPolicy: description: To indicate the image pull policy to be applied to all the containers in the csi driver pods. From 8632ff8a8ed23bed826830f78688fb5bedf5395d Mon Sep 17 00:00:00 2001 From: Leela Venkaiah G Date: Tue, 15 Oct 2024 11:37:40 +0000 Subject: [PATCH 3/7] configure controller plugin default deployment strategy Signed-off-by: Leela Venkaiah G --- internal/controller/defaults.go | 10 +++++++++- internal/controller/driver_controller.go | 3 ++- .../ceph-csi-operator/api/v1alpha1/driver_types.go | 5 +++++ .../api/v1alpha1/zz_generated.deepcopy.go | 5 +++++ 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/internal/controller/defaults.go b/internal/controller/defaults.go index 19ed50a4..d5b46360 100644 --- a/internal/controller/defaults.go +++ b/internal/controller/defaults.go @@ -50,13 +50,21 @@ var defaultLeaderElection = csiv1a1.LeaderElectionSpec{ RetryPeriod: 26, } -var defautUpdateStrategy = appsv1.DaemonSetUpdateStrategy{ +var defaultDaemonSetUpdateStrategy = appsv1.DaemonSetUpdateStrategy{ Type: appsv1.RollingUpdateDaemonSetStrategyType, RollingUpdate: &appsv1.RollingUpdateDaemonSet{ MaxUnavailable: ptr.To(intstr.FromInt(1)), }, } +var defaultDeploymentStrategy = appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: ptr.To(intstr.FromString("25%")), + MaxUnavailable: ptr.To(intstr.FromString("25%")), + }, +} + var operatorNamespace = utils.Call(func() string { namespace := os.Getenv("OPERATOR_NAMESPACE") if namespace == "" { diff --git a/internal/controller/driver_controller.go b/internal/controller/driver_controller.go index 75a82b86..fd182644 100644 --- a/internal/controller/driver_controller.go +++ b/internal/controller/driver_controller.go @@ -564,6 +564,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { deploy.Spec = appsv1.DeploymentSpec{ Replicas: pluginSpec.Replicas, Selector: &appSelector, + Strategy: ptr.Deref(pluginSpec.DeploymentStrategy, defaultDeploymentStrategy), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: utils.Call(func() map[string]string { @@ -957,7 +958,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"app": appName}, }, - UpdateStrategy: ptr.Deref(pluginSpec.UpdateStrategy, defautUpdateStrategy), + UpdateStrategy: ptr.Deref(pluginSpec.UpdateStrategy, defaultDaemonSetUpdateStrategy), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: utils.Call(func() map[string]string { diff --git a/vendor/github.com/ceph/ceph-csi-operator/api/v1alpha1/driver_types.go b/vendor/github.com/ceph/ceph-csi-operator/api/v1alpha1/driver_types.go index 5fa10638..b562eeaa 100644 --- a/vendor/github.com/ceph/ceph-csi-operator/api/v1alpha1/driver_types.go +++ b/vendor/github.com/ceph/ceph-csi-operator/api/v1alpha1/driver_types.go @@ -213,6 +213,11 @@ type ControllerPluginSpec struct { // Embedded common pods spec PodCommonSpec `json:",inline"` + // DeploymentStrategy describes how to replace existing pods with new ones + // Default value is RollingUpdate with MaxUnavailable and MaxSurege as 25% (kubernetes default) + //+kubebuilder:validation:Optional + DeploymentStrategy *appsv1.DeploymentStrategy `json:"deploymentStrategy,omitempty"` + // Set replicas for controller plugin's deployment. Defaults to 2 //+kubebuilder:validation:Optional //+kubebuilder:validation:Minimum:=1 diff --git a/vendor/github.com/ceph/ceph-csi-operator/api/v1alpha1/zz_generated.deepcopy.go b/vendor/github.com/ceph/ceph-csi-operator/api/v1alpha1/zz_generated.deepcopy.go index cfc03cf8..a8542df9 100644 --- a/vendor/github.com/ceph/ceph-csi-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/vendor/github.com/ceph/ceph-csi-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -438,6 +438,11 @@ func (in *ControllerPluginResourcesSpec) DeepCopy() *ControllerPluginResourcesSp func (in *ControllerPluginSpec) DeepCopyInto(out *ControllerPluginSpec) { *out = *in in.PodCommonSpec.DeepCopyInto(&out.PodCommonSpec) + if in.DeploymentStrategy != nil { + in, out := &in.DeploymentStrategy, &out.DeploymentStrategy + *out = new(appsv1.DeploymentStrategy) + (*in).DeepCopyInto(*out) + } if in.Replicas != nil { in, out := &in.Replicas, &out.Replicas *out = new(int32) From 282fc547e97e5bfb79d34b31472bfa944433dae0 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Thu, 26 Sep 2024 09:17:46 +0200 Subject: [PATCH 4/7] doc: add quick-start guide added a quick start guide for the minimal installation Signed-off-by: Madhu Rajanna --- docs/quick-start.md | 158 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 docs/quick-start.md diff --git a/docs/quick-start.md b/docs/quick-start.md new file mode 100644 index 00000000..b82a19c0 --- /dev/null +++ b/docs/quick-start.md @@ -0,0 +1,158 @@ +- [Quick Start Guide for Ceph-CSI-Operator](#quick-start-guide-for-ceph-csi-operator) + - [1. Prerequisites](#1-prerequisites) + - [2. Install the Ceph-CSI Operator](#2-install-the-ceph-csi-operator) + - [3. Deploy Ceph-CSI Drivers](#3-deploy-ceph-csi-drivers) + - [3.1 Deploy the RBD Driver](#31-deploy-the-rbd-driver) + - [3.2 Deploy the CephFS Driver](#32-deploy-the-cephfs-driver) + - [3.3 Deploy the Ceph-NFS Driver](#33-deploy-the-ceph-nfs-driver) + - [4. Verify Installation](#4-verify-installation) + - [5. Create CephConnection](#5-create-cephconnection) + - [6. Create ClientProfile](#6-create-clientprofile) + +# Quick Start Guide for Ceph-CSI-Operator + +## 1. Prerequisites + +Before deploying the Ceph-CSI-Operator, ensure the following requirements are met: + +- A Kubernetes cluster ([supported version](https://kubernetes.io/releases/) recommended) +- Ceph cluster ([supported version](https://docs.ceph.com/en/latest/releases/) recommended) +- `kubectl` CLI installed + +**Note:** In this guide, we will use minimal configurations to deploy the Ceph-CSI-Operator and drivers. You can customize the configurations as per your requirements. + +## 2. Install the Ceph-CSI Operator + +```console +kubectl create -f deploy/all-in-one/install.yaml +``` + +verify the installation: + +```bash +kubectl get pods -n ceph-csi-operator-system +NAME READY STATUS RESTARTS AGE +ceph-csi-operator-controller-manager-67d45fd9ff-zgst7 2/2 Running 0 40s +``` + +### 3. Deploy Ceph-CSI Drivers + +Once the operator is installed, deploy the Ceph-CSI drivers: + +#### 3.1 Deploy the RBD Driver + +```console +echo ' +apiVersion: csi.ceph.io/v1alpha1 +kind: Driver +metadata: + name: rbd.csi.ceph.com + namespace: ceph-csi-operator-system +' | kubectl create -f - +``` + +#### 3.2 Deploy the CephFS Driver + +```console +echo ' +apiVersion: csi.ceph.io/v1alpha1 +kind: Driver +metadata: + name: cephfs.csi.ceph.com + namespace: ceph-csi-operator-system +' | kubectl create -f - +``` + +### 3.3 Deploy the Ceph-NFS Driver + +```console +echo ' +apiVersion: csi.ceph.io/v1alpha1 +kind: Driver +metadata: + name: nfs.csi.ceph.com + namespace: ceph-csi-operator-system +' | kubectl create -f - +``` + +## 4. Verify Installation + +To verify the installation, check the status of the Ceph-CSI components: + +```bash +kubectl get pod -nceph-csi-operator-system +NAME READY STATUS RESTARTS AGE +ceph-csi-operator-controller-manager-744dc99cb5-scxxh 2/2 Running 0 45s +cephfs.csi.ceph.com-ctrlplugin-5847c998b5-xf85m 5/5 Running 0 27s +cephfs.csi.ceph.com-nodeplugin-r6pkt 2/2 Running 0 27s +nfs.csi.ceph.com-ctrlplugin-76fd4f5b4c-smk2g 5/5 Running 0 27s +nfs.csi.ceph.com-nodeplugin-kbzms 2/2 Running 0 27s +rbd.csi.ceph.com-ctrlplugin-6965dcfdb8-w88kn 5/5 Running 0 4m35s +rbd.csi.ceph.com-nodeplugin-lnm4n 2/2 Running 0 4m35s +``` + +## 5. Create CephConnection + +Create a CephConnection CR to connect to the Ceph cluster: + +```console +echo ' +apiVersion: csi.ceph.io/v1alpha1 +kind: CephConnection +metadata: + name: ceph-connection + namespace: ceph-csi-operator-system +spec: + monitors: + - 10.98.44.171:6789 +' | kubectl create -f - +``` + +## 6. Create ClientProfile + +Create a ClientProfile CR to define the client configuration which points to +the CephConnection CR and the CephFS and RBD configurations: + +```console +echo ' +apiVersion: csi.ceph.io/v1alpha1 +kind: ClientProfile +metadata: + name: storage + namespace: ceph-csi-operator-system +spec: + cephConnectionRef: + name: ceph-connection + cephFs: + subVolumeGroup: csi +' | kubectl create -f - +``` + +Use the ClientProfile Name as the ClusterID in the required classes (StrorageClass,VolumeSnapshotClass etc). + +```console + +## 7. Clean Up Resources + +To clean up the resources, delete the cepconnection, clientprofile and drivers: + +```console +kubectl delete cephconnection ceph-connection -n ceph-csi-operator-system +kubectl delete clientprofile storage -n ceph-csi-operator-system +kubectl delete driver rbd.csi.ceph.com -n ceph-csi-operator-system +kubectl delete driver cephfs.csi.ceph.com -n ceph-csi-operator-system +kubectl delete driver nfs.csi.ceph.com -n ceph-csi-operator-system +``` + +To uninstall the Ceph-CSI-Operator, delete the operator: + +```console +kubectl delete -f deploy/all-in-one/install.yaml +``` + +verify the deletion: + +```bash +kubectl get pods -n ceph-csi-operator-system +No resources found in ceph-csi-operator-system namespace. +``` From 1ba697c237d4530075fdd4d3673b4ff91c47f94a Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 6 Nov 2024 08:34:32 +0100 Subject: [PATCH 5/7] csiaddons: add new RBAC csiaddons required new RBAC in the next release to create/update the csiaddonsnode object based on the owner deployment/daemonset names of the pods its running with. Signed-off-by: Madhu Rajanna --- config/csi-rbac/cephfs_ctrlplugin_role.yaml | 8 +++- config/csi-rbac/rbd_ctrlplugin_role.yaml | 8 +++- config/csi-rbac/rbd_nodeplugin_role.yaml | 8 +++- deploy/all-in-one/install.yaml | 42 +++++++++++++++++++++ deploy/multifile/csi-rbac.yaml | 42 +++++++++++++++++++++ 5 files changed, 105 insertions(+), 3 deletions(-) diff --git a/config/csi-rbac/cephfs_ctrlplugin_role.yaml b/config/csi-rbac/cephfs_ctrlplugin_role.yaml index 69c45dce..79937014 100644 --- a/config/csi-rbac/cephfs_ctrlplugin_role.yaml +++ b/config/csi-rbac/cephfs_ctrlplugin_role.yaml @@ -8,4 +8,10 @@ rules: verbs: ["get", "watch", "list", "delete", "update", "create"] - apiGroups: ["csiaddons.openshift.io"] resources: ["csiaddonsnodes"] - verbs: ["create"] + verbs: ["get", "create", "update"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get"] + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get"] diff --git a/config/csi-rbac/rbd_ctrlplugin_role.yaml b/config/csi-rbac/rbd_ctrlplugin_role.yaml index 34eed43b..ab8b74e7 100644 --- a/config/csi-rbac/rbd_ctrlplugin_role.yaml +++ b/config/csi-rbac/rbd_ctrlplugin_role.yaml @@ -8,4 +8,10 @@ rules: verbs: ["get", "watch", "list", "delete", "update", "create"] - apiGroups: ["csiaddons.openshift.io"] resources: ["csiaddonsnodes"] - verbs: ["create"] + verbs: ["get", "create", "update"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get"] + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get"] diff --git a/config/csi-rbac/rbd_nodeplugin_role.yaml b/config/csi-rbac/rbd_nodeplugin_role.yaml index 417f2eb3..436bfcd6 100644 --- a/config/csi-rbac/rbd_nodeplugin_role.yaml +++ b/config/csi-rbac/rbd_nodeplugin_role.yaml @@ -5,4 +5,10 @@ metadata: rules: - apiGroups: ["csiaddons.openshift.io"] resources: ["csiaddonsnodes"] - verbs: ["create"] + verbs: ["get", "create", "update"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get"] + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get"] diff --git a/deploy/all-in-one/install.yaml b/deploy/all-in-one/install.yaml index aa81e3e3..2ecb670a 100644 --- a/deploy/all-in-one/install.yaml +++ b/deploy/all-in-one/install.yaml @@ -13993,7 +13993,21 @@ rules: resources: - csiaddonsnodes verbs: + - get - create + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resources: + - replicasets + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -14058,7 +14072,21 @@ rules: resources: - csiaddonsnodes verbs: + - get - create + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resources: + - replicasets + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -14071,7 +14099,21 @@ rules: resources: - csiaddonsnodes verbs: + - get - create + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resources: + - replicasets + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/deploy/multifile/csi-rbac.yaml b/deploy/multifile/csi-rbac.yaml index bbe62d35..7a644ee6 100644 --- a/deploy/multifile/csi-rbac.yaml +++ b/deploy/multifile/csi-rbac.yaml @@ -56,7 +56,21 @@ rules: resources: - csiaddonsnodes verbs: + - get - create + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resources: + - replicasets + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -80,7 +94,21 @@ rules: resources: - csiaddonsnodes verbs: + - get - create + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resources: + - replicasets + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -93,7 +121,21 @@ rules: resources: - csiaddonsnodes verbs: + - get - create + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resources: + - replicasets + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole From b7ea7890441bc9087d20e8c2e2a3a7a60fd7fbd1 Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Thu, 7 Nov 2024 13:15:51 +0100 Subject: [PATCH 6/7] Fix broken links in the PR template Links were broken due to using a nonexistent devel branch. Signed-off-by: Michael Adam --- .github/pull_request_template.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 55045f96..b916f371 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -38,11 +38,11 @@ functionality, but are work items that can be taken up subsequently. * [ ] **Commit Message Formatting**: Commit titles and messages follow guidelines in the [developer - guide](https://github.com/ceph/ceph-csi-operator/blob/devel/docs/development-guide.md#commit-messages). + guide](https://github.com/ceph/ceph-csi-operator/blob/main/docs/development-guide.md#commit-messages). * [ ] Reviewed the developer guide on [Submitting a Pull - Request](https://github.com/ceph/ceph-csi-operator/blob/devel/docs/development-guide.md#development-workflow) + Request](https://github.com/ceph/ceph-csi-operator/blob/main/docs/development-guide.md#development-workflow) * [ ] [Pending release - notes](https://github.com/ceph/ceph-csi-operator/blob/devel/PendingReleaseNotes.md) + notes](https://github.com/ceph/ceph-csi-operator/blob/main/PendingReleaseNotes.md) updated with breaking and/or notable changes for the next major release. * [ ] Documentation has been updated, if necessary. * [ ] Unit tests have been added, if necessary. From 93f5bbcc703f19010ce2e255e7105f0b3a9cfde3 Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Thu, 7 Nov 2024 12:39:47 +0100 Subject: [PATCH 7/7] docs: clarify the operator design doc a bit I the operator design document, custom resource definition(CRD) seems to have been used as a synonym for custom resource(CR) which is usually not correct. This change makes the wording a little clearer by using CRD only when technically correct and using CR when appropriate. Signed-off-by: Michael Adam Update docs/design/operator.md Co-authored-by: nb-ohad --- docs/design/operator.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/design/operator.md b/docs/design/operator.md index 1d4f00d7..01c1681e 100644 --- a/docs/design/operator.md +++ b/docs/design/operator.md @@ -54,12 +54,12 @@ graph TD In this diagram: -- **CRD Changes**: Represents changes made to Custom Resource Definitions, +- **CR Changes**: Represents changes made to Custom Resources (CRs), which trigger actions in the operator. -- **Operator**: Listens for changes in CRDs and initiates the installation and +- **Operator**: Listens for changes in CRs and initiates the installation and configuration of CSI drivers. - **Configure CephFS, NFS, RBD**: Actions performed by the operator to install - and configure the respective CSI drivers based on the CRD changes. + and configure the respective CSI drivers based on the configuration presented on the updated CR. ## CRDs for ceph-csi-operator