From 43080158f725859f6f8191a6fc00784738b7e872 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Fri, 22 May 2020 11:11:50 +0530 Subject: [PATCH] Fix TLS cert creation when Zones are added. (#115) This PR revamps the main controller to look for changes in MinIOInstance fields to respond to changes like addition of zones, image updates. Fixes #97 --- README.md | 20 +- docs/adding-zones.md | 44 ++ examples/minioinstance-kes.yaml | 11 +- examples/minioinstance-mcs.yaml | 2 - examples/minioinstance.yaml | 21 - examples/patch.yaml | 4 +- pkg/controller/cluster/csr.go | 4 +- pkg/controller/cluster/main-controller.go | 432 ++++++++++-------- .../statefulsets/minio-statefulset.go | 11 +- 9 files changed, 305 insertions(+), 244 deletions(-) create mode 100644 docs/adding-zones.md diff --git a/README.md b/README.md index e8d06ca0821..6dbe0d97019 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ MinIO-Operator brings native MinIO, [MCS](https://github.com/minio/mcs), [KES](h | Feature | Reference Document | |-------------------------|--------------------| | Create and delete highly available distributed MinIO clusters | [Create a MinIO Instance](https://github.com/minio/minio-operator#create-a-minio-instance). | -| Expand an existing MinIO cluster | [Expand a MinIO Cluster](https://github.com/minio/minio-operator#expand-a-minio-cluster). | | Automatic TLS for MinIO | [Automatic TLS for MinIO Instance](https://github.com/minio/minio-operator/blob/master/docs/tls.md#automatic-csr-generation). | +| Expand an existing MinIO cluster | [Expand a MinIO Cluster](https://github.com/minio/minio-operator/blob/master/docs/adding-zones.md). | | Deploy MCS with MinIO cluster | [Deploy MinIO Instance with MCS](https://github.com/minio/minio-operator/blob/master/docs/mcs.md). | | Deploy KES with MinIO cluster | [Deploy MinIO Instance with KES](https://github.com/minio/minio-operator/blob/master/docs/kes.md). | | Deploy mc mirror | [Deploy Mirror Instance](https://github.com/minio/minio-operator/blob/master/docs/mirror.md). | @@ -44,24 +44,6 @@ Once MinIO-Operator deployment is running, you can create MinIO instances using kubectl apply -f https://raw.githubusercontent.com/minio/minio-operator/master/examples/minioinstance.yaml ``` -### Expand a MinIO cluster - -After you have a distributed MinIO Cluster running (zones.server >= 4), you can expand the MinIO cluster using - -``` -kubectl patch minioinstances.operator.min.io minio --patch "$(cat examples/patch.yaml)" --type=merge -``` - -You can expand an existing cluster by adding new zones to the `patch.yaml` and run the above `kubectl-patch` command. - -**NOTE**: Important point to consider _before_ using cluster expansion: - -During cluster expansion, MinIO Operator removes the existing StatefulSet and creates a new StatefulSet with required number of Pods. This means, there is a short downtime during expansion, as the pods are terminated and created again. - -As existing StatefulSet pods are terminated, its PVCs are also deleted. It is _very important_ to ensure PVs bound to MinIO StatefulSet PVCs are not deleted at this time to avoid data loss. We recommend configuring every PV with reclaim policy [`retain`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#retain), to ensure the PV is not deleted. - -If you attempt cluster expansion while the PV reclaim policy is set to something else, it may lead to data loss. If you have the reclaim policy set to something else, change it as explained in [Kubernetes documents](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy/). - ### Access MinIOInstance via Service Add an [external service](https://kubernetes.io/docs/concepts/services-networking/service/) in MinIOInstance definition to enable Service based access to the MinIOInstance pods. Refer [the example here](https://github.com/minio/minio-operator/blob/master/examples/minioinstance.yaml?raw=true) for details on how to setup service based access for MinIOInstance pods. diff --git a/docs/adding-zones.md b/docs/adding-zones.md new file mode 100644 index 00000000000..7758544766a --- /dev/null +++ b/docs/adding-zones.md @@ -0,0 +1,44 @@ +# Adding Zones to a MinIO Cluster + +[![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) +[![Docker Pulls](https://img.shields.io/docker/pulls/minio/k8s-operator.svg?maxAge=604800)](https://hub.docker.com/r/minio/k8s-operator) + +This document explains how to add zones to an existing MinIO Cluster with Operator. This document is only applicable to a MinIO Cluster created by MinIO Operator. + +Read more about MinIO Zones design in [MinIO Docs](https://github.com/minio/minio/blob/master/docs/distributed). + +## Getting Started + +Assuming you have a MinIO cluster with single zone, `zone-0` with 4 drives (as shown in [examples](https://github.com/minio/minio-operator/tree/master/examples)). You can dd a new zone `zone-1` with 4 drives using `kubectl patch` command. + +``` +kubectl patch minioinstances.operator.min.io minio --patch "$(cat examples/patch.yaml)" --type=merge +``` + +If you're using a custom configuration (e.g. multiple zones or higher number of drives per zone), make sure to change `patch.yaml` accordingly. + +**NOTE**: Important points to consider _before_ using cluster expansion: + +- During cluster expansion, MinIO Operator removes the existing StatefulSet and creates a new StatefulSet with required number of Pods. This means, there is a short downtime during expansion, as the pods are terminated and created again. As existing StatefulSet pods are terminated, its PVCs are also deleted. It is _very important_ to ensure PVs bound to MinIO StatefulSet PVCs are not deleted at this time to avoid data loss. We recommend configuring every PV with reclaim policy [`retain`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#retain), to ensure the PV is not deleted. If you attempt cluster expansion while the PV reclaim policy is set to something else, it may lead to data loss. If you have the reclaim policy set to something else, change it as explained in [Kubernetes documents](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy/). + +- MinIO server currently doesn't support zone removal. So, please ensure to not remove zones in patch.yaml file while applying the patch. It can have unintended consequences including missing data or failure of MinIO cluster. + +## Rules of Adding Zones + +Each zone is a self contained entity with same SLA's (read/write quorum) for each object as original cluster. By using the existing namespace for lookup validation MinIO ensures conflicting objects are not created. When no such object exists then MinIO simply uses the least used zone. There are no limits on how many zones can be combined. + +There is only one requirement, i.e. based on initial zone's erasure set count (say `n`), new zones are expected to have a minimum of `n` drives to match the original cluster SLA or it should be in multiples of `n`. For example if initial set count is 4, new zones should have at least 4 or multiple of 4 drives. + +Read more about MinIO Zones design in [MinIO Docs](https://github.com/minio/minio/blob/master/docs/distributed). + +## Effects on KES/TLS Enabled Instance + +If your MinIO Operator configuration has [KES](https://github.com/minio/minio-operator/blob/master/docs/kes.md) or [Automatic TLS](https://github.com/minio/minio-operator/blob/master/docs/tls.md#automatic-csr-generation) enabled, there are additional considerations: + +- When new zones are added, Operator invalidates older self signed TLS certificates and the related secrets. Operator then creates new certificate signing requests (CSR). This is because there are new MinIO nodes that must be added in certificate DNS names. The administrator must approve these CSRs for MinIO server to be deployed again. Unless the CSR are approved, Operator will not create MinIO StatefulSet pods. + +- If you're using your own certificates, as explained [here](https://github.com/minio/minio-operator/blob/master/docs/tls.md#pass-certificate-secret-to-minioinstance), please ensure to use/update proper certificates that allow older and new MinIO nodes. + +## Downtime + +Since Operator deletes existing StatefulSet and related CSR / Secrets (for TLS enabled setups), before re-creating a new StatefulSet and other resources, there is a downtime involved when adding zones. This downtime is generally few minutes, assuming CSRs are approved quickly and resources are available for new StatefulSet to be created. diff --git a/examples/minioinstance-kes.yaml b/examples/minioinstance-kes.yaml index f0000fe89a0..278fbbe9527 100644 --- a/examples/minioinstance-kes.yaml +++ b/examples/minioinstance-kes.yaml @@ -13,10 +13,11 @@ metadata: name: minio-mcs-secret type: Opaque data: - mcshmacjwt: WU9VUkpXVFNJR05JTkdTRUNSRVQ= # base 64 encoded "YOURJWTSIGNINGSECRET" (echo -n 'YOURJWTSIGNINGSECRET' | base64) - mcspbkdfpassphrase: U0VDUkVU # base 64 encoded "SECRET" (echo -n 'SECRET' | base64) - mcspbkdfsalt: U0VDUkVU # base 64 encoded "SECRET" (echo -n 'SECRET' | base64) - mcssecretkey: WU9VUk1DU1NFQ1JFVA== # base 64 encoded "YOURMCSSECRET" (echo -n 'YOURMCSSECRET' | base64) + MCS_HMAC_JWT_SECRET: WU9VUkpXVFNJR05JTkdTRUNSRVQ= # base 64 encoded "YOURJWTSIGNINGSECRET" (echo -n 'YOURJWTSIGNINGSECRET' | base64) + MCS_PBKDF_PASSPHRASE: U0VDUkVU # base 64 encoded "SECRET" (echo -n 'SECRET' | base64) + MCS_PBKDF_SALT: U0VDUkVU # base 64 encoded "SECRET" (echo -n 'SECRET' | base64) + MCS_ACCESS_KEY: WU9VUk1DU0FDQ0VTUw== # base 64 encoded "YOURMCSACCESS" (echo -n 'YOURMCSACCESS' | base64) + MCS_SECRET_KEY: WU9VUk1DU1NFQ1JFVA== # base 64 encoded "YOURMCSSECRET" (echo -n 'YOURMCSSECRET' | base64) --- apiVersion: v1 kind: Service @@ -148,7 +149,6 @@ spec: ## restarts the pods if liveness checks fail. liveness: httpGet: - scheme: HTTPS path: /minio/health/live port: 9000 initialDelaySeconds: 120 @@ -160,7 +160,6 @@ spec: ## Disable this check if you're setting PodManagementPolicy to "OrderedReady". readiness: httpGet: - scheme: HTTPS path: /minio/health/ready port: 9000 initialDelaySeconds: 120 diff --git a/examples/minioinstance-mcs.yaml b/examples/minioinstance-mcs.yaml index d3f1a9786e1..7e547793969 100644 --- a/examples/minioinstance-mcs.yaml +++ b/examples/minioinstance-mcs.yaml @@ -137,7 +137,6 @@ spec: ## restarts the pods if liveness checks fail. liveness: httpGet: - scheme: HTTPS path: /minio/health/live port: 9000 initialDelaySeconds: 120 @@ -149,7 +148,6 @@ spec: ## Disable this check if you're setting PodManagementPolicy to "OrderedReady". readiness: httpGet: - scheme: HTTPS path: /minio/health/ready port: 9000 initialDelaySeconds: 120 diff --git a/examples/minioinstance.yaml b/examples/minioinstance.yaml index f2d238616ab..889ceb28ff7 100644 --- a/examples/minioinstance.yaml +++ b/examples/minioinstance.yaml @@ -8,17 +8,6 @@ data: secretkey: bWluaW8xMjM= # based 64 encoded "minio123" (echo -n 'minio123' | base64) --- apiVersion: v1 -kind: Secret -metadata: - name: minio-mcs-secret -type: Opaque -data: - mcshmacjwt: WU9VUkpXVFNJR05JTkdTRUNSRVQ= # base 64 encoded "YOURJWTSIGNINGSECRET" (echo -n 'YOURJWTSIGNINGSECRET' | base64) - mcspbkdfpassphrase: U0VDUkVU # base 64 encoded "SECRET" (echo -n 'SECRET' | base64) - mcspbkdfsalt: U0VDUkVU # base 64 encoded "SECRET" (echo -n 'SECRET' | base64) - mcssecretkey: WU9VUk1DU1NFQ1JFVA== # base 64 encoded "YOURMCSSECRET" (echo -n 'YOURMCSSECRET' | base64) ---- -apiVersion: v1 kind: Service metadata: name: minio-service @@ -106,16 +95,6 @@ spec: # key: dedicated # operator: Equal # value: storage - ## Define configuration for mcs (A graphical user interface for MinIO) - mcs: - image: minio/mcs:v0.0.4 - replicas: 2 - mcsSecret: - name: minio-mcs-secret - ## Optionally metadata can be attached to the mcs pod - # metadata: - # labels: - # extra: labels ## Add environment variables to be set in MinIO container (https://github.com/minio/minio/tree/master/docs/config) env: - name: MINIO_BROWSER diff --git a/examples/patch.yaml b/examples/patch.yaml index 1dd0537c6c2..6968c0b9b09 100644 --- a/examples/patch.yaml +++ b/examples/patch.yaml @@ -5,8 +5,8 @@ metadata: spec: zones: - name: "zone-0" - ## Number of MinIO servers/pods in this zone. + ## Number of MinIO servers/pods in zone-0. servers: 4 - name: "zone-1" - ## Number of MinIO servers/pods in this zone. + ## Number of MinIO servers/pods in zone-1. servers: 4 diff --git a/pkg/controller/cluster/csr.go b/pkg/controller/cluster/csr.go index c4f304b1900..d7ef32c0984 100644 --- a/pkg/controller/cluster/csr.go +++ b/pkg/controller/cluster/csr.go @@ -254,9 +254,7 @@ func (c *Controller) createSecret(ctx context.Context, mi *miniov1.MinIOInstance "public.crt": certBytes, }, } - cOpts := metav1.CreateOptions{} - _, err := c.kubeClientSet.CoreV1().Secrets(mi.Namespace).Create(ctx, secret, cOpts) - if err != nil { + if _, err := c.kubeClientSet.CoreV1().Secrets(mi.Namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { return err } return nil diff --git a/pkg/controller/cluster/main-controller.go b/pkg/controller/cluster/main-controller.go index 3cec21a1a26..833adec0915 100644 --- a/pkg/controller/cluster/main-controller.go +++ b/pkg/controller/cluster/main-controller.go @@ -316,14 +316,11 @@ func (c *Controller) syncHandler(key string) error { // Convert the namespace/name string into a distinct namespace and name namespace, name, err := cache.SplitMetaNamespaceKey(key) - klog.V(2).Infof("Key after splitting, namespace: %s, name: %s", namespace, name) if err != nil { runtime.HandleError(fmt.Errorf("Invalid resource key: %s", key)) return nil } - nsName := types.NamespacedName{Namespace: namespace, Name: name} - // Get the MinIOInstance resource with this namespace/name mi, err := c.minioInstancesLister.MinIOInstances(namespace).Get(name) if err != nil { @@ -335,42 +332,11 @@ func (c *Controller) syncHandler(key string) error { return err } + // Set any required default values and init Global variables + nsName := types.NamespacedName{Namespace: namespace, Name: name} mi.EnsureDefaults() miniov1.InitGlobals(mi) - svc, err := c.serviceLister.Services(mi.Namespace).Get(mi.MinIOCIServiceName()) - // If the service doesn't exist, we'll create it - if apierrors.IsNotFound(err) { - currentState := "Provisioning Service" - mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, 0) - if err != nil { - return err - } - klog.V(2).Infof("Creating a new Cluster IP Service for cluster %q", nsName) - svc = services.NewClusterIPForMinIO(mi) - _, err = c.kubeClientSet.CoreV1().Services(mi.Namespace).Create(ctx, svc, cOpts) - } - - hlSvc, err := c.serviceLister.Services(mi.Namespace).Get(mi.MinIOHLServiceName()) - // If the headless service doesn't exist, we'll create it - if apierrors.IsNotFound(err) { - currentState := "Provisining Headless Service" - mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, 0) - if err != nil { - return err - } - klog.V(2).Infof("Creating a new Headless Service for cluster %q", nsName) - hlSvc = services.NewHeadlessForMinIO(mi) - _, err = c.kubeClientSet.CoreV1().Services(mi.Namespace).Create(ctx, hlSvc, cOpts) - } - - // If an error occurs during Get/Create, we'll requeue the item so we can - // attempt processing again later. This could have been caused by a - // temporary network failure, or any other transient reason. - if err != nil { - return err - } - // check if both auto certificate creation and external secret with certificate is passed, // this is an error as only one of this is allowed in one MinIOInstance if mi.RequiresAutoCertSetup() && mi.RequiresExternalCertSetup() { @@ -379,129 +345,97 @@ func (c *Controller) syncHandler(key string) error { return fmt.Errorf(msg) } - if mi.RequiresAutoCertSetup() { + // TLS is mandatory if KES is enabled + if mi.HasKESEnabled() && !(mi.RequiresAutoCertSetup() || mi.RequiresExternalCertSetup()) { + msg := "KES Setup Requires MinIO to be configured with TLS" + klog.V(2).Infof(msg) + return fmt.Errorf(msg) + } - _, err = c.certClient.CertificateSigningRequests().Get(ctx, mi.MinIOCSRName(), metav1.GetOptions{}) - if err != nil { - // If the CSR doesn't exist, we'll create it - if apierrors.IsNotFound(err) { - klog.V(2).Infof("Creating a new Certificate Signing Request for cluster %q", nsName) - currentState := "Requesting Certificate" - mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, 0) - if err != nil { - return err - } - // create CSR here - c.createCSR(ctx, mi) - } else { + // Handle the Internal ClusterIP Service for MinIOInstance + svc, err := c.serviceLister.Services(mi.Namespace).Get(mi.MinIOCIServiceName()) + if err != nil { + if apierrors.IsNotFound(err) { + currentState := "Provisioning Service" + mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, 0) + if err != nil { return err } + klog.V(2).Infof("Creating a new Cluster IP Service for cluster %q", nsName) + svc = services.NewClusterIPForMinIO(mi) + _, err = c.kubeClientSet.CoreV1().Services(mi.Namespace).Create(ctx, svc, cOpts) + if err != nil { + return err + } + } else { + return err } - if mi.HasKESEnabled() { - _, err := c.certClient.CertificateSigningRequests().Get(ctx, mi.KESCSRName(), metav1.GetOptions{}) + } + + // Handle the Internal Headless Service for MinIOInstance StatefulSet + hlSvc, err := c.serviceLister.Services(mi.Namespace).Get(mi.MinIOHLServiceName()) + if err != nil { + if apierrors.IsNotFound(err) { + currentState := "Provisining Headless Service" + mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, 0) if err != nil { - // If the CSR doesn't exist, we'll create it - if apierrors.IsNotFound(err) { - currentState := "Creating TLS CSR for KES" - mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, 0) - if err != nil { - return err - } - klog.V(2).Infof("Creating a new Certificate Signing Request for cluster %q", nsName) - err = c.createKESTLSCSR(ctx, mi) - if err != nil { - return err - } - } else { - return err - } + return err } - _, err = c.certClient.CertificateSigningRequests().Get(ctx, mi.MinIOClientCSRName(), metav1.GetOptions{}) + klog.V(2).Infof("Creating a new Headless Service for cluster %q", nsName) + hlSvc = services.NewHeadlessForMinIO(mi) + _, err = c.kubeClientSet.CoreV1().Services(mi.Namespace).Create(ctx, hlSvc, cOpts) if err != nil { - // If the CSR doesn't exist, we'll create it - if apierrors.IsNotFound(err) { - currentState := "Creating Client TLS CSR" - mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, 0) - if err != nil { - return err - } - klog.V(2).Infof("Creating a new Certificate Signing Request for cluster %q", nsName) - err = c.createMinIOClientTLSCSR(ctx, mi) - if err != nil { - return err - } - } else { - return err - } + return err } + } else { + return err } } // Get the StatefulSet with the name specified in MinIOInstance.spec ss, err := c.statefulSetLister.StatefulSets(mi.Namespace).Get(mi.Name) - // If the resource doesn't exist, we'll create it - if apierrors.IsNotFound(err) { - currentState := "Provisioning Statefulset" - mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, 0) - if err != nil { - return err - } - ss = statefulsets.NewForMinIO(mi, hlSvc.Name) - ss, err = c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Create(ctx, ss, cOpts) - } - - // If an error occurs during Get/Create, we'll requeue the item so we can - // attempt processing again later. This could have been caused by a - // temporary network failure, or any other transient reason. if err != nil { - return err - } - - if mi.HasMCSEnabled() { - // Get the Deployment with the name specified in MirrorInstace.spec - d, err := c.deploymentLister.Deployments(mi.Namespace).Get(mi.MCSDeploymentName()) - // If the resource doesn't exist, we'll create it if apierrors.IsNotFound(err) { - if !mi.HasCredsSecret() || !mi.HasMCSSecret() { - msg := "Please set the credentials" - klog.V(2).Infof(msg) - return fmt.Errorf(msg) - } - - minioSecretName := mi.Spec.CredsSecret.Name - minioSecret, sErr := c.kubeClientSet.CoreV1().Secrets(mi.Namespace).Get(ctx, minioSecretName, gOpts) - if sErr != nil { - return sErr - } - - mcsSecretName := mi.Spec.MCS.MCSSecret.Name - mcsSecret, sErr := c.kubeClientSet.CoreV1().Secrets(mi.Namespace).Get(ctx, mcsSecretName, gOpts) - if sErr != nil { - return sErr - } - // Check if any one replica is READY - if ss.Status.ReadyReplicas > 0 { - currentState := "Provisioning MCS" - mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, ss.Status.Replicas) - if err != nil { + if mi.RequiresAutoCertSetup() { + if err = c.checkAndCreateMinIOCSR(ctx, nsName, mi); err != nil { return err } - if pErr := mi.CreateMCSUser(minioSecret.Data, mcsSecret.Data); pErr != nil { - klog.V(2).Infof(pErr.Error()) - return pErr + } + if mi.HasKESEnabled() { + if err = c.checkAndCreateKESCSR(ctx, nsName, mi); err != nil { + return err } - // Create MCS Deployment - d = deployments.NewForMCS(mi) - d, err = c.kubeClientSet.AppsV1().Deployments(mi.Namespace).Create(ctx, d, cOpts) - if err != nil { - klog.V(2).Infof(err.Error()) + } + currentState := "Provisioning Statefulset" + mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, 0) + if err != nil { + return err + } + ss = statefulsets.NewForMinIO(mi, hlSvc.Name) + ss, err = c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Create(ctx, ss, cOpts) + if err != nil { + return err + } + } else { + return err + } + } else { + // If this number of the replicas on the MinIOInstance resource is specified, and the + // number does not equal the current desired replicas on the StatefulSet, we + // should update the StatefulSet resource. + if mi.MinIOReplicas() != *ss.Spec.Replicas { + klog.V(2).Infof("MinIOInstance %s replicas: %d, StatefulSet replicas: %d", name, mi.MinIOReplicas(), *ss.Spec.Replicas) + + // If this was a TLS enabled Setup, we create new CSR because change in number of replicas means the new endpoints + // need to be added in the CSR + if mi.RequiresAutoCertSetup() { + klog.V(2).Infof("Removing the existing MinIO CSRs and related secrets") + if err := c.removeMinIOCSRAndSecrets(ctx, mi); err != nil { return err } - // Create MCS service - mcsSvc := services.NewClusterIPForMCS(mi) - _, err = c.kubeClientSet.CoreV1().Services(svc.Namespace).Create(ctx, mcsSvc, cOpts) - if err != nil { - klog.V(2).Infof(err.Error()) + + klog.V(2).Infof("Creating required MinIO CSRs and related secrets") + if err := c.checkAndCreateMinIOCSR(ctx, nsName, mi); err != nil { return err } } else { @@ -511,20 +445,97 @@ func (c *Controller) syncHandler(key string) error { return err } } - } else { - return err + + ss = statefulsets.NewForMinIO(mi, hlSvc.Name) + klog.V(2).Infof("Removing the existing StatefulSet %s with replicas: %d", name, *ss.Spec.Replicas) + if err := c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Delete(ctx, ss.Name, metav1.DeleteOptions{}); err != nil { + return err + } + + currentState := "Provisioning Statefulset" + mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, 0) + if err != nil { + return err + } + klog.V(2).Infof("Creating a new StatefulSet %s with replicas: %d", name, mi.MinIOReplicas()) + if _, err := c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Create(ctx, ss, cOpts); err != nil { + return err + } + } + + // If this container version on the MinIOInstance resource is specified, and the + // version does not equal the current desired version in the StatefulSet, we + // should update the StatefulSet resource. + if mi.Spec.Image != ss.Spec.Template.Spec.Containers[0].Image { + mi, err = c.updateMinIOInstanceStatus(ctx, mi, "Updating MinIO Version", ss.Status.Replicas) + if err != nil { + return err + } + klog.V(4).Infof("Updating MinIOInstance %s MinIO server version %s, to: %s", name, mi.Spec.Image, ss.Spec.Template.Spec.Containers[0].Image) + ss = statefulsets.NewForMinIO(mi, hlSvc.Name) + if _, err := c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Update(ctx, ss, uOpts); err != nil { + return err + } } } - if mi.HasKESEnabled() && !(mi.RequiresAutoCertSetup() || mi.RequiresExternalCertSetup()) { - currentState := "waiting on TLS for KES" - mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, ss.Status.Replicas) - if err != nil { - return err + if mi.HasMCSEnabled() { + // Get the Deployment with the name specified in MirrorInstace.spec + if d, err := c.deploymentLister.Deployments(mi.Namespace).Get(mi.MCSDeploymentName()); err != nil { + if apierrors.IsNotFound(err) { + if !mi.HasCredsSecret() || !mi.HasMCSSecret() { + msg := "Please set the credentials" + klog.V(2).Infof(msg) + return fmt.Errorf(msg) + } + + minioSecretName := mi.Spec.CredsSecret.Name + minioSecret, sErr := c.kubeClientSet.CoreV1().Secrets(mi.Namespace).Get(ctx, minioSecretName, gOpts) + if sErr != nil { + return sErr + } + + mcsSecretName := mi.Spec.MCS.MCSSecret.Name + mcsSecret, sErr := c.kubeClientSet.CoreV1().Secrets(mi.Namespace).Get(ctx, mcsSecretName, gOpts) + if sErr != nil { + return sErr + } + // Check if any one replica is READY + if ss.Status.ReadyReplicas > 0 { + currentState := "Provisioning MCS" + mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, ss.Status.Replicas) + if err != nil { + return err + } + if pErr := mi.CreateMCSUser(minioSecret.Data, mcsSecret.Data); pErr != nil { + klog.V(2).Infof(pErr.Error()) + return pErr + } + // Create MCS Deployment + d = deployments.NewForMCS(mi) + d, err = c.kubeClientSet.AppsV1().Deployments(mi.Namespace).Create(ctx, d, cOpts) + if err != nil { + klog.V(2).Infof(err.Error()) + return err + } + // Create MCS service + mcsSvc := services.NewClusterIPForMCS(mi) + _, err = c.kubeClientSet.CoreV1().Services(svc.Namespace).Create(ctx, mcsSvc, cOpts) + if err != nil { + klog.V(2).Infof(err.Error()) + return err + } + } else { + currentState := "Waiting for Pods to be ready" + mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, ss.Status.Replicas) + if err != nil { + return err + } + } + } else { + return err + } } - msg := "KES Setup Requires MinIO to be configured with TLS" - klog.V(2).Infof(msg) - return fmt.Errorf(msg) } if mi.HasKESEnabled() && (mi.RequiresAutoCertSetup() || mi.RequiresExternalCertSetup()) { @@ -540,6 +551,11 @@ func (c *Controller) syncHandler(key string) error { // Get the StatefulSet with the name specified in spec ks, err := c.statefulSetLister.StatefulSets(mi.Namespace).Get(mi.KESStatefulSetName()) if apierrors.IsNotFound(err) { + currentState := "Provisioning KES StatefulSet" + mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, 0) + if err != nil { + return err + } ks = statefulsets.NewForKES(mi, svc.Name) klog.V(2).Infof("Creating a new StatefulSet for cluster %q", nsName) ks, err = c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Create(ctx, ks, cOpts) @@ -578,42 +594,6 @@ func (c *Controller) syncHandler(key string) error { return fmt.Errorf(msg) } - // If this number of the replicas on the MinIOInstance resource is specified, and the - // number does not equal the current desired replicas on the StatefulSet, we - // should update the StatefulSet resource. - if mi.MinIOReplicas() != *ss.Spec.Replicas { - klog.V(4).Infof("MinIOInstance %s replicas: %d, StatefulSet replicas: %d", name, mi.MinIOReplicas(), *ss.Spec.Replicas) - ss = statefulsets.NewForMinIO(mi, hlSvc.Name) - err = c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Delete(ctx, ss.Name, metav1.DeleteOptions{}) - if err != nil { - return err - } - _, err = c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Create(ctx, ss, cOpts) - if err != nil { - return err - } - } - - // If this container version on the MinIOInstance resource is specified, and the - // version does not equal the current desired version in the StatefulSet, we - // should update the StatefulSet resource. - if mi.Spec.Image != ss.Spec.Template.Spec.Containers[0].Image { - mi, err = c.updateMinIOInstanceStatus(ctx, mi, "Updating MinIO Version", ss.Status.Replicas) - if err != nil { - return err - } - klog.V(4).Infof("Updating MinIOInstance %s MinIO server version %s, to: %s", name, mi.Spec.Image, ss.Spec.Template.Spec.Containers[0].Image) - ss = statefulsets.NewForMinIO(mi, hlSvc.Name) - _, err = c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Update(ctx, ss, uOpts) - } - - // If an error occurs during Update, we'll requeue the item so we can - // attempt processing again later. This could have been caused by a - // temporary network failure, or any other transient reason. - if err != nil { - return err - } - if mi.HasMCSEnabled() && d != nil && mi.Spec.MCS.Image != d.Spec.Template.Spec.Containers[0].Image { mi, err = c.updateMinIOInstanceStatus(ctx, mi, "Updating MCS Version", ss.Status.Replicas) if err != nil { @@ -639,6 +619,80 @@ func (c *Controller) syncHandler(key string) error { return nil } +func (c *Controller) removeMinIOCSRAndSecrets(ctx context.Context, mi *miniov1.MinIOInstance) error { + if err := c.certClient.CertificateSigningRequests().Delete(ctx, mi.MinIOCSRName(), metav1.DeleteOptions{}); err != nil { + return err + } + if err := c.kubeClientSet.CoreV1().Secrets(mi.Namespace).Delete(ctx, mi.MinIOTLSSecretName(), metav1.DeleteOptions{}); err != nil { + return err + } + + if mi.HasKESEnabled() { + if err := c.certClient.CertificateSigningRequests().Delete(ctx, mi.MinIOClientCSRName(), metav1.DeleteOptions{}); err != nil { + return err + } + if err := c.kubeClientSet.CoreV1().Secrets(mi.Namespace).Delete(ctx, mi.MinIOClientTLSSecretName(), metav1.DeleteOptions{}); err != nil { + return err + } + } + return nil +} + +func (c *Controller) checkAndCreateMinIOCSR(ctx context.Context, nsName types.NamespacedName, mi *miniov1.MinIOInstance) error { + if _, err := c.certClient.CertificateSigningRequests().Get(ctx, mi.MinIOCSRName(), metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + currentState := "Requesting Certificate" + mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, 0) + if err != nil { + return err + } + klog.V(2).Infof("Creating a new Certificate Signing Request for MinIO Server Certs, cluster %q", nsName) + if err = c.createCSR(ctx, mi); err != nil { + return err + } + } else { + return err + } + } + if mi.HasKESEnabled() { + if _, err := c.certClient.CertificateSigningRequests().Get(ctx, mi.MinIOClientCSRName(), metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + currentState := "Creating Client TLS CSR" + mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, 0) + if err != nil { + return err + } + klog.V(2).Infof("Creating a new Certificate Signing Request for MinIO Client Certs, cluster %q", nsName) + if err = c.createMinIOClientTLSCSR(ctx, mi); err != nil { + return err + } + } else { + return err + } + } + } + return nil +} + +func (c *Controller) checkAndCreateKESCSR(ctx context.Context, nsName types.NamespacedName, mi *miniov1.MinIOInstance) error { + if _, err := c.certClient.CertificateSigningRequests().Get(ctx, mi.KESCSRName(), metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + currentState := "Creating TLS CSR for KES" + mi, err = c.updateMinIOInstanceStatus(ctx, mi, currentState, 0) + if err != nil { + return err + } + klog.V(2).Infof("Creating a new Certificate Signing Request for KES Server Certs, cluster %q", nsName) + if err = c.createKESTLSCSR(ctx, mi); err != nil { + return err + } + } else { + return err + } + } + return nil +} + func (c *Controller) updateMinIOInstanceStatus(ctx context.Context, minioInstance *miniov1.MinIOInstance, currentState string, availableReplicas int32) (*miniov1.MinIOInstance, error) { opts := metav1.UpdateOptions{} // NEVER modify objects from the store. It's a read-only, local cache. @@ -647,10 +701,10 @@ func (c *Controller) updateMinIOInstanceStatus(ctx context.Context, minioInstanc minioInstanceCopy := minioInstance.DeepCopy() minioInstanceCopy.Status.AvailableReplicas = availableReplicas minioInstanceCopy.Status.CurrentState = currentState - // If the CustomResourceSubresources feature gate is not enabled, - // we must use Update instead of UpdateStatus to update the Status block of the MinIOInstance resource. - // UpdateStatus will not allow changes to the Spec of the resource, - // which is ideal for ensuring nothing other than resource status has been updated. + // If the CustomResourceSubresources feature gate is not enabled, // If the CustomResourceSubresources feature gate is not enabled, + // we must use Update instead of UpdateStatus to update the Status block of the MinIOInstance resource. // we must use Update instead of UpdateStatus to update the Status block of the MinIOInstance resource. + // UpdateStatus will not allow changes to the Spec of the resource, // UpdateStatus will not allow changes to the Spec of the resource, + // which is ideal for ensuring nothing other than resource status has been updated. // which is ideal for ensuring nothing other than resource status has been updated. mi, err := c.minioClientSet.OperatorV1().MinIOInstances(minioInstance.Namespace).UpdateStatus(ctx, minioInstanceCopy, opts) time.Sleep(time.Second * 2) return mi, err diff --git a/pkg/resources/statefulsets/minio-statefulset.go b/pkg/resources/statefulsets/minio-statefulset.go index 9bfcee72bfd..59db57c9f1b 100644 --- a/pkg/resources/statefulsets/minio-statefulset.go +++ b/pkg/resources/statefulsets/minio-statefulset.go @@ -21,6 +21,7 @@ import ( "fmt" "net" "strconv" + "strings" miniov1 "github.com/minio/minio-operator/pkg/apis/operator.min.io/v1" appsv1 "k8s.io/api/apps/v1" @@ -166,6 +167,12 @@ func minioServerContainer(mi *miniov1.MinIOInstance, serviceName string) corev1. } } + readyProbe := mi.Spec.Readiness + readyProbe.HTTPGet.Scheme = corev1.URIScheme(strings.ToUpper(miniov1.Scheme)) + + liveProbe := mi.Spec.Liveness + liveProbe.HTTPGet.Scheme = corev1.URIScheme(strings.ToUpper(miniov1.Scheme)) + return corev1.Container{ Name: miniov1.MinIOServerName, Image: mi.Spec.Image, @@ -179,8 +186,8 @@ func minioServerContainer(mi *miniov1.MinIOInstance, serviceName string) corev1. Args: args, Env: minioEnvironmentVars(mi), Resources: mi.Spec.Resources, - LivenessProbe: mi.Spec.Liveness, - ReadinessProbe: mi.Spec.Readiness, + LivenessProbe: liveProbe, + ReadinessProbe: readyProbe, } }