Add support for bucket expansion (#80)

README.md

@@ -48,11 +48,22 @@ Once MinIO-Operator deployment is running, you can create MinIO instances using
 kubectl create -f https://raw.githubusercontent.com/minio/minio-operator/master/examples/minioinstance.yaml
 ```
 
+### Expand a MinIO cluster
+
+After you have a distributed MinIO Cluster running (zones.server > 3), you can expand the MinIO cluster using
+
+```
+kubectl patch minioinstances.miniocontroller.min.io minio --patch "$(cat examples/patch.yaml)" --type=merge
+```
+
+You can further keep adding new zones in the `patch.yaml` file and apply the patch, to add new nodes to existing cluster. 
+
 ## Features
 
 MinIO-Operator currently supports following features:
 
 - Create and delete highly available distributed MinIO clusters.
+- Expand an existing MinIO cluster.
 - Upgrading existing distributed MinIO clusters.
 
 Refer [`minioinstance.yaml`](https://raw.githubusercontent.com/minio/minio-operator/master/examples/minioinstance.yaml) for details on how to pass supported fields to the operator.

docs/operator-fields.md

@@ -23,6 +23,10 @@ MinIO Operator creates native Kubernetes resources within the cluster. If the Mi
 
 - `spec.image`: Set the container registry and image tag for MinIO server to be used in the MinIOInstance.
 
+- `spec.zones`: Set the number of servers per MinIO Zone. Add a new Zone field to expand the MinIO cluster. Read more on [MinIO zones here](https://github.com/minio/minio/blob/master/docs/distributed/DESIGN.md).
+
+- `spec.volumesPerServer`: Set the number of volume mounts per MinIO node. For example if you set `spec.zones[0].Servers = 4`, `spec.zones[1].Servers = 8` and `spec.volumesPerServer = 4`, then you'll have total 12 MinIO Pods, with 4 volume mounts on each Pod. Note that `volumesPerServer` is static per cluster, expanding a cluster will add new nodes.
+
 - `spec.imagePullSecret`: Defines the secret to be used for pull image from a private Docker image.
 
 - `spec.credsSecret`: Use this secret to assign custom credentials (access key and secret key) to MinIOInstance.

examples/minioinstance-with-external-service.yaml

@@ -41,30 +41,54 @@ spec:
       prometheus.io/port: "9000"
       prometheus.io/scrape: "true"
   ## Registry location and Tag to download MinIO Server image
-  image: minio/minio:RELEASE.2020-01-03T19-12-21Z
+  image: minio/minio:RELEASE.2020-04-15T00-39-01Z
+  zones:
+    - name: "zone-0"
+      ## Number of MinIO servers/pods in this zone.
+      ## For standalone mode, supply 1. For distributed mode, supply 4 or more.
+      ## Note that the operator does not support upgrading from standalone to distributed mode.
+      servers: 4
+  ## Supply number of volumes to be mounted per MinIO server instance.
+  volumesPerServer: 1
+  ## Mount path where PV will be mounted inside container(s). Defaults to "/export".
+  mountPath: /export
+  ## Sub path inside Mount path where MinIO starts. Defaults to "".
+  # subPath: /data
+  ## This VolumeClaimTemplate is used across all the volumes provisioned for MinIO cluster.
+  ## Please do not change the volumeClaimTemplate field while expanding the cluster, this may
+  ## lead to unbound PVCs and missing data
+  volumeClaimTemplate:
+    metadata:
+      name: data
+    spec:
+      accessModes:
+        - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Gi
   ## Secret with credentials to be used by MinIO instance.
   credsSecret:
     name: minio-creds-secret
-  ## Supply number of replicas.
-  ## For standalone mode, supply 1. For distributed mode, supply 4 or more (should be even).
-  ## Note that the operator does not support upgrading from standalone to distributed mode.
-  replicas: 4
   ## PodManagement policy for pods created by StatefulSet. Can be "OrderedReady" or "Parallel"
   ## Refer https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#pod-management-policy
   ## for details. Defaults to "Parallel"
   ## If set to "OrderedReady", then disable Readiness checks below. Readiness check will only
   ## work if PodManagementPolicy is set to "Parallel".
   podManagementPolicy: Parallel
+  ## Secret with certificates to configure TLS for MinIO certs. Create secrets as explained
+  ## here: https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret
+  # externalCertSecret:
+  # name: tls-ssl-minio
   ## Enable Kubernetes based certificate generation and signing as explained in
   ## https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster
   requestAutoCert: false
   ## Used when "requestAutoCert" is set to true. Set CommonName for the auto-generated certificate.
   ## Internal DNS name for the pod will be used if CommonName is not provided.
+  ## DNS name format is minio-{0...3}.minio.default.svc.cluster.local
   certConfig:
     commonName: ""
     organizationName: []
     dnsNames: []
-
   ## Used to specify a toleration for a pod
   # tolerations:
   #  - effect: NoSchedule
@@ -111,20 +135,3 @@ spec:
   ## Affinity settings for MinIO pods. Read more about affinity
   ## here: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity.
   # affinity:
-  ## Secret with certificates to configure TLS for MinIO certs. Create secrets as explained
-  ## here: https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret
-  # externalCertSecret:
-  # name: tls-ssl-minio
-  ## Mountpath where PV will be mounted inside container(s). Defaults to "/export".
-  # mountPath: /export
-  ## Subpath inside Mountpath where MinIO starts. Defaults to "".
-  # subPath: /data
-  volumeClaimTemplate:
-    metadata:
-      name: data
-    spec:
-      accessModes:
-        - ReadWriteOnce
-      resources:
-        requests:
-          storage: 10Gi

examples/minioinstance.yaml

@@ -28,30 +28,54 @@ spec:
       prometheus.io/port: "9000"
       prometheus.io/scrape: "true"
   ## Registry location and Tag to download MinIO Server image
-  image: minio/minio:RELEASE.2020-01-03T19-12-21Z
+  image: minio/minio:RELEASE.2020-04-15T00-39-01Z
+  zones:
+    - name: "zone-0"
+      ## Number of MinIO servers/pods in this zone.
+      ## For standalone mode, supply 1. For distributed mode, supply 4 or more.
+      ## Note that the operator does not support upgrading from standalone to distributed mode.
+      servers: 4
+  ## Supply number of volumes to be mounted per MinIO server instance.
+  volumesPerServer: 1
+  ## Mount path where PV will be mounted inside container(s). Defaults to "/export".
+  mountPath: /export
+  ## Sub path inside Mount path where MinIO starts. Defaults to "".
+  # subPath: /data
+  ## This VolumeClaimTemplate is used across all the volumes provisioned for MinIO cluster.
+  ## Please do not change the volumeClaimTemplate field while expanding the cluster, this may
+  ## lead to unbound PVCs and missing data
+  volumeClaimTemplate:
+    metadata:
+      name: data
+    spec:
+      accessModes:
+        - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Gi
   ## Secret with credentials to be used by MinIO instance.
   credsSecret:
     name: minio-creds-secret
-  ## Supply number of replicas.
-  ## For standalone mode, supply 1. For distributed mode, supply 4 or more (should be even).
-  ## Note that the operator does not support upgrading from standalone to distributed mode.
-  replicas: 4
   ## PodManagement policy for pods created by StatefulSet. Can be "OrderedReady" or "Parallel"
   ## Refer https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#pod-management-policy
   ## for details. Defaults to "Parallel"
   ## If set to "OrderedReady", then disable Readiness checks below. Readiness check will only
   ## work if PodManagementPolicy is set to "Parallel".
   podManagementPolicy: Parallel
+  ## Secret with certificates to configure TLS for MinIO certs. Create secrets as explained
+  ## here: https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret
+  # externalCertSecret:
+  # name: tls-ssl-minio
   ## Enable Kubernetes based certificate generation and signing as explained in
   ## https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster
   requestAutoCert: false
   ## Used when "requestAutoCert" is set to true. Set CommonName for the auto-generated certificate.
   ## Internal DNS name for the pod will be used if CommonName is not provided.
+  ## DNS name format is minio-{0...3}.minio.default.svc.cluster.local
   certConfig:
     commonName: ""
     organizationName: []
     dnsNames: []
-
   ## Used to specify a toleration for a pod
   # tolerations:
   #  - effect: NoSchedule
@@ -98,20 +122,4 @@ spec:
   ## Affinity settings for MinIO pods. Read more about affinity
   ## here: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity.
   # affinity:
-  ## Secret with certificates to configure TLS for MinIO certs. Create secrets as explained
-  ## here: https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret
-  # externalCertSecret:
-  # name: tls-ssl-minio
-  ## Mountpath where PV will be mounted inside container(s). Defaults to "/export".
-  # mountPath: /export
-  ## Subpath inside Mountpath where MinIO starts. Defaults to "".
-  # subPath: /data
-  volumeClaimTemplate:
-    metadata:
-      name: data
-    spec:
-      accessModes:
-        - ReadWriteOnce
-      resources:
-        requests:
-          storage: 10Gi
+

examples/patch.yaml

@@ -0,0 +1,16 @@
+apiVersion: miniocontroller.min.io/v1beta1
+kind: MinIOInstance
+metadata:
+  name: minio
+## If specified, MinIOInstance pods will be dispatched by specified scheduler.
+## If not specified, the pod will be dispatched by default scheduler.
+# scheduler:
+#  name: my-custom-scheduler
+spec:
+  zones:
+    - name: "zone-0"
+      ## Number of MinIO servers/pods in this zone.
+      servers: 4
+    - name: "zone-1"
+      ## Number of MinIO servers/pods in this zone.
+      servers: 4

go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/eapache/go-resiliency v1.2.0 // indirect
 	github.com/evanphx/json-patch v4.5.0+incompatible // indirect
 	github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4 // indirect
 	github.com/gogo/protobuf v0.0.0-20170702163824-dda3e8acadcc // indirect

go.sum

@@ -2,6 +2,8 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
+github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=

pkg/apis/miniocontroller/v1beta1/helper.go

@@ -66,22 +66,37 @@ func (mi *MinIOInstance) RequiresAutoCertSetup() bool {
 	return mi.Spec.RequestAutoCert == true
 }
 
-// PodLabels retuns the default labels
+// PodLabels returns the default labels
 func (mi *MinIOInstance) PodLabels() map[string]string {
 	m := make(map[string]string, 1)
 	m[constants.InstanceLabel] = mi.Name
 	return m
 }
 
+// GetVolumesPath returns the paths for MinIO mounts based on
+// total number of volumes per MinIO server
+func (mi *MinIOInstance) GetVolumesPath() string {
+	if mi.Spec.VolumesPerServer == 1 {
+		return path.Join(mi.Spec.Mountpath, mi.Spec.Subpath)
+	}
+	return path.Join(mi.Spec.Mountpath+"{0..."+strconv.Itoa((mi.Spec.VolumesPerServer)-1)+"}", mi.Spec.Subpath)
+}
+
+// GetReplicas returns the number of total replicas
+// required for this cluster
+func (mi *MinIOInstance) GetReplicas() int32 {
+	var replicas int32
+	for _, z := range mi.Spec.Zones {
+		replicas = replicas + z.Servers
+	}
+	return replicas
+}
+
 // EnsureDefaults will ensure that if a user omits and fields in the
 // spec that are required, we set some sensible defaults.
 // For example a user can choose to omit the version
 // and number of members.
 func (mi *MinIOInstance) EnsureDefaults() *MinIOInstance {
-	if mi.Spec.Replicas == 0 {
-		mi.Spec.Replicas = constants.DefaultReplicas
-	}
-
 	if mi.Spec.PodManagementPolicy == "" || (mi.Spec.PodManagementPolicy != appsv1.OrderedReadyPodManagement &&
 		mi.Spec.PodManagementPolicy != appsv1.ParallelPodManagement) {
 		mi.Spec.PodManagementPolicy = constants.DefaultPodManagementPolicy
@@ -91,18 +106,22 @@ func (mi *MinIOInstance) EnsureDefaults() *MinIOInstance {
 		mi.Spec.Image = constants.DefaultMinIOImage
 	}
 
+	for _, z := range mi.Spec.Zones {
+		if z.Servers == 0 {
+			z.Servers = constants.DefaultServers
+		}
+	}
+
+	if mi.Spec.VolumesPerServer == 0 {
+		mi.Spec.VolumesPerServer = constants.DefaultVolumesPerServer
+	}
+
 	if mi.Spec.Mountpath == "" {
 		mi.Spec.Mountpath = constants.MinIOVolumeMountPath
-	} else {
-		// Ensure there is no trailing `/`
-		mi.Spec.Mountpath = path.Clean(mi.Spec.Mountpath)
 	}
 
 	if mi.Spec.Subpath == "" {
 		mi.Spec.Subpath = constants.MinIOVolumeSubPath
-	} else {
-		// Ensure there is no `/` in beginning
-		mi.Spec.Subpath = path.Clean(mi.Spec.Subpath)
 	}
 
 	if mi.RequiresAutoCertSetup() == true {
@@ -138,10 +157,16 @@ func (mi *MinIOInstance) EnsureDefaults() *MinIOInstance {
 // current MinIOInstance
 func (mi *MinIOInstance) GetHosts() []string {
 	hosts := make([]string, 0)
-	// append all the MinIOInstance replica URLs
+	var max int32
+	// Create the ellipses style URL
 	// mi.Name is the headless service name
-	for i := 0; i < int(mi.Spec.Replicas); i++ {
-		hosts = append(hosts, fmt.Sprintf("%s-"+strconv.Itoa(i)+".%s.%s.svc.cluster.local", mi.Name, mi.GetHeadlessServiceName(), mi.Namespace))
+	for i, z := range mi.Spec.Zones {
+		max = max + z.Servers
+		if i == 0 {
+			hosts = append(hosts, fmt.Sprintf("%s-{0..."+strconv.Itoa(int(max)-1)+"}.%s.%s.svc.cluster.local", mi.Name, mi.GetHeadlessServiceName(), mi.Namespace))
+		} else {
+			hosts = append(hosts, fmt.Sprintf("%s-{"+strconv.Itoa(int(mi.Spec.Zones[i-1].Servers))+"..."+strconv.Itoa(int(max)-1)+"}.%s.%s.svc.cluster.local", mi.Name, mi.GetHeadlessServiceName(), mi.Namespace))
+		}
 	}
 	return hosts
 }

pkg/apis/miniocontroller/v1beta1/helper_test.go

@@ -13,7 +13,6 @@ func TestEnsureDefaults(t *testing.T) {
 	mi.EnsureDefaults()
 
 	t.Run("defaults", func(t *testing.T) {
-		assert.Equal(t, mi.Spec.Replicas, int32(constants.DefaultReplicas))
 		assert.Equal(t, mi.Spec.Image, constants.DefaultMinIOImage)
 		assert.Equal(t, mi.Spec.Mountpath, constants.MinIOVolumeMountPath)
 		assert.Equal(t, mi.Spec.Subpath, constants.MinIOVolumeSubPath)
@@ -38,13 +37,9 @@ func TestEnsureDefaults(t *testing.T) {
 
 	t.Run("defaults don't override", func(t *testing.T) {
 		newImage := "minio/minio:latest"
-		newReplicas := int32(99)
 		mi.Spec.Image = newImage
-		mi.Spec.Replicas = newReplicas
-
 		mi.EnsureDefaults()
 
 		assert.Equal(t, newImage, mi.Spec.Image)
-		assert.Equal(t, newReplicas, mi.Spec.Replicas)
 	})
 }

pkg/apis/miniocontroller/v1beta1/types.go

@@ -49,11 +49,18 @@ type CertificateConfig struct {
 	DNSNames         []string `json:"dnsNames,omitempty"`
 }
 
+// LocalCertificateReference defines the spec for a local certificate
 type LocalCertificateReference struct {
 	Name string `json:"name"`
 	Type string `json:"type,omitempty"`
 }
 
+// Zone defines the spec for a MinIO Zone
+type Zone struct {
+	Name    string `json:"name"`
+	Servers int32  `json:"servers"`
+}
+
 // MinIOInstanceSpec is the spec for a MinIOInstance resource
 type MinIOInstanceSpec struct {
 	// Image defines the MinIOInstance Docker image.
@@ -62,9 +69,6 @@ type MinIOInstanceSpec struct {
 	// ImagePullSecret defines the secret to be used for pull image from a private Docker image.
 	// +optional
 	ImagePullSecret corev1.LocalObjectReference `json:"imagePullSecret,omitempty"`
-	// Replicas defines the number of MinIO instances in a MinIOInstance resource
-	// +optional
-	Replicas int32 `json:"replicas,omitempty"`
 	// Pod Management Policy for pod created by StatefulSet
 	// +optional
 	PodManagementPolicy appsv1.PodManagementPolicyType `json:"podManagementPolicy,omitempty"`
@@ -80,6 +84,9 @@ type MinIOInstanceSpec struct {
 	// If provided, use these requests and limit for cpu/memory resource allocation
 	// +optional
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
+	// VolumesPerServer allows a user to specify how many volumes per MinIO Server/Pod instance
+	// +optional
+	VolumesPerServer int `json:"volumesPerServer"`
 	// VolumeClaimTemplate allows a user to specify how volumes inside a MinIOInstance
 	// +optional
 	VolumeClaimTemplate *corev1.PersistentVolumeClaim `json:"volumeClaimTemplate,omitempty"`
@@ -119,6 +126,9 @@ type MinIOInstanceSpec struct {
 	// Tolerations allows users to set entries like effect, key, operator, value.
 	// +optional
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+	// Definition for Cluster in given MinIO cluster
+	// +optional
+	Zones []Zone `json:"zones"`
 }
 
 // MinIOInstanceStatus is the status for a MinIOInstance resource