Skip to content

Commit

Permalink
Merge branch 'minio:master' into resize_pvc
Browse files Browse the repository at this point in the history
  • Loading branch information
jiuker authored Jan 4, 2024
2 parents 9ce7534 + 64cb15e commit 3a620d5
Show file tree
Hide file tree
Showing 13 changed files with 726 additions and 415 deletions.
2 changes: 0 additions & 2 deletions api/configure_operator.go
Original file line number Diff line number Diff line change
Expand Up @@ -181,10 +181,8 @@ func setupGlobalMiddleware(handler http.Handler) http.Handler {
BrowserXssFilter: GetSecureBrowserXSSFilter(),
ContentSecurityPolicy: GetSecureContentSecurityPolicy(),
ContentSecurityPolicyReportOnly: GetSecureContentSecurityPolicyReportOnly(),
PublicKey: GetSecurePublicKey(),
ReferrerPolicy: GetSecureReferrerPolicy(),
FeaturePolicy: GetSecureFeaturePolicy(),
ExpectCTHeader: GetSecureExpectCTHeader(),
IsDevelopment: false,
}
secureMiddleware := secure.New(secureOptions)
Expand Down
7 changes: 3 additions & 4 deletions api/operator_subnet.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
"github.com/minio/operator/api/operations/operator_api"
"github.com/minio/operator/models"
miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2"
v2 "github.com/minio/operator/pkg/apis/minio.min.io/v2"
xhttp "github.com/minio/operator/pkg/http"
"github.com/minio/operator/pkg/subnet"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -154,7 +153,7 @@ func getTenantsToRegister(ctx context.Context, session *models.Principal, k8sCli
if err != nil {
return nil, err
}
tenantStructs := make([]tenantInterface, len(tenantList.Items))
var tenantStructs []tenantInterface
for _, tenant := range tenantList.Items {
svcURL := tenant.GetTenantServiceURL()
mAdmin, err := getTenantAdminClient(ctx, k8sClient, &tenant, svcURL)
Expand All @@ -168,7 +167,7 @@ func getTenantsToRegister(ctx context.Context, session *models.Principal, k8sCli

func registerTenants(ctx context.Context, k8sClient K8sClientI, tenants []tenantInterface, apiKey string) (*models.OperatorSubnetRegisterAPIKeyResponse, *models.Error) {
for _, tenant := range tenants {
if err := registerTenant(ctx, k8sClient, tenant.mAdminClient, tenant.tenant, apiKey); err != nil {
if err := registerTenant(ctx, tenant.mAdminClient, apiKey); err != nil {
return nil, ErrorWithContext(ctx, err)
}
}
Expand Down Expand Up @@ -202,7 +201,7 @@ func SubnetRegisterWithAPIKey(ctx context.Context, minioClient MinioAdmin, apiKe
return true, nil
}

func registerTenant(ctx context.Context, k8sClient K8sClientI, adminClient MinioAdmin, tenant v2.Tenant, apiKey string) error {
func registerTenant(ctx context.Context, adminClient MinioAdmin, apiKey string) error {
_, err := SubnetRegisterWithAPIKey(ctx, adminClient, apiKey)
return err
}
Expand Down
2 changes: 1 addition & 1 deletion api/pool-handlers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ func (suite *TenantTestSuite) TestUpdateTenantPoolsWithoutError() {
Pools: []miniov2.Pool{{
VolumeClaimTemplate: &corev1.PersistentVolumeClaim{
Spec: corev1.PersistentVolumeClaimSpec{
Resources: corev1.ResourceRequirements{
Resources: corev1.VolumeResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceStorage: resource.MustParse("1Gi"),
},
Expand Down
2 changes: 1 addition & 1 deletion api/tenant-handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -864,7 +864,7 @@ func parseTenantPoolRequest(poolParams *models.Pool) (*miniov2.Pool, error) {
AccessModes: []corev1.PersistentVolumeAccessMode{
corev1.ReadWriteOnce,
},
Resources: corev1.ResourceRequirements{
Resources: corev1.VolumeResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceStorage: *volumeSize,
},
Expand Down
2 changes: 1 addition & 1 deletion api/tenant-handlers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -397,7 +397,7 @@ func Test_TenantInfo(t *testing.T) {
VolumesPerServer: 4,
VolumeClaimTemplate: &corev1.PersistentVolumeClaim{
Spec: corev1.PersistentVolumeClaimSpec{
Resources: corev1.ResourceRequirements{
Resources: corev1.VolumeResourceRequirements{
Requests: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceStorage: resource.MustParse("1Mi"),
},
Expand Down
12 changes: 6 additions & 6 deletions docs/expansion.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@ MinIO expansion is done in terms of MinIO pools, read more about the design in [

## Getting Started

You can add capacity to the tenant using `kubectl minio` plugin.
You can add capacity to the tenant using `kubectl minio` plugin, or using the MinIO Operator Console.

```
kubectl minio tenant volume add --name TENANT_NAME --servers SERVERS --volumes TOTAL_VOLUMES --capacity TOTAL_RAW_CAPACITY
kubectl minio tenant expand TENANT_NAME --servers SERVERS --volumes TOTAL_VOLUMES --capacity TOTAL_RAW_CAPACITY
```

Remember to replace `TENANT_NAME` with tenant name where you want to add volumes, `SERVERS` with new servers to be added to the tenant, `TOTAL_VOLUMES` with total new volumes to be added to tenant and `TOTAL_RAW_CAPACITY` with total new raw capacity to be added to the tenant.
Remember to replace `TENANT_NAME` with tenant name where you want to add volumes, `SERVERS` with the number of servers to be added to the tenant, `TOTAL_VOLUMES` with the total number of volumes to be added to tenant and `TOTAL_RAW_CAPACITY` with the total raw capacity to be added to the tenant.

**NOTE**: Important points to consider _before_ using Tenant expansion:

Expand All @@ -24,17 +24,17 @@ Remember to replace `TENANT_NAME` with tenant name where you want to add volumes

### What are MinIO pools

A MinIO pool is a self contained entity with same SLA's (read/write quorum) for each object. There are no limits on how many pools can be combined. After adding of a pool, MinIO simply uses the least used pool. All pools are for all purposes are invisible to an any application, and MinIO handles the pools internally.
A MinIO pool is a self-contained entity with same SLA's (read/write quorum) for each object. There are no limits on how many pools can be combined. After adding of a pool, MinIO simply uses the least used pool. All pools are for all purposes are invisible to an any application, and MinIO handles the pools internally.

### Rules of Adding pools

There is only one requirement, i.e. based on initial pool's erasure set count (say `n`), new pools are expected to have a minimum of `n` drives to match the original Tenant SLA or it should be in multiples of `n`. For example if initial set count is 4, new pools should have at least 4 or multiple of 4 drives.
There is only one requirement, i.e. based on initial pool's erasure set count (say `n`), new pools are expected to have a minimum of `n` drives to match the original Tenant SLA, or it should be in multiples of `n`. For example if initial set count is 4, new pools should have at least 4 or multiple of 4 drives.

### Effects on KES/TLS Enabled Instance

If your MinIO Operator configuration has [KES](https://github.com/minio/operator/blob/master/docs/kes.md) or [Automatic TLS](https://github.com/minio/operator/blob/master/docs/tls.md#automatic-csr-generation) enabled, there are additional considerations:

- When new pools are added, Operator invalidates older self signed TLS certificates and the related secrets. Operator then creates new certificate signing requests (CSR). This is because there are new MinIO nodes that must be added in certificate DNS names. The administrator must approve these CSRs for MinIO server to be deployed again. Unless the CSR are approved, Operator will not create MinIO StatefulSet pods.
- When new pools are added, Operator invalidates older self-signed TLS certificates and the related secrets. Operator then creates new certificate signing requests (CSR). This is because there are new MinIO nodes that must be added in certificate DNS names. The administrator must approve these CSRs for MinIO server to be deployed again. Unless the CSR are approved, Operator will not create MinIO StatefulSet pods.

- If you're using your own certificates, as explained [here](https://github.com/minio/operator/blob/master/docs/tls.md#pass-certificate-secret-to-tenant), please ensure to use/update proper certificates that allow older and new MinIO nodes.

Expand Down
105 changes: 53 additions & 52 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,27 +7,27 @@ require (
github.com/docker/cli v24.0.7+incompatible
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fatih/color v1.16.0
github.com/go-openapi/errors v0.20.4
github.com/go-openapi/loads v0.21.2
github.com/go-openapi/runtime v0.26.0
github.com/go-openapi/spec v0.20.11
github.com/go-openapi/strfmt v0.21.8
github.com/go-openapi/swag v0.22.4
github.com/go-openapi/validate v0.22.3
github.com/go-openapi/errors v0.21.0
github.com/go-openapi/loads v0.21.5
github.com/go-openapi/runtime v0.26.2
github.com/go-openapi/spec v0.20.13
github.com/go-openapi/strfmt v0.21.10
github.com/go-openapi/swag v0.22.7
github.com/go-openapi/validate v0.22.6
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/golang-jwt/jwt/v4 v4.5.0
github.com/google/go-containerregistry v0.17.0
github.com/google/uuid v1.4.0
github.com/google/uuid v1.5.0
github.com/gorilla/mux v1.8.1
github.com/hashicorp/go-version v1.6.0
github.com/jessevdk/go-flags v1.5.0
github.com/klauspost/compress v1.17.4
github.com/miekg/dns v1.1.57
github.com/minio/cli v1.24.2
github.com/minio/highwayhash v1.0.2
github.com/minio/madmin-go/v3 v3.0.36
github.com/minio/mc v0.0.0-20231202112410-d920e2b34b22
github.com/minio/minio-go/v7 v7.0.65
github.com/minio/madmin-go/v3 v3.0.38
github.com/minio/mc v0.0.0-20231226180728-176f657e538d
github.com/minio/minio-go/v7 v7.0.66
github.com/minio/pkg v1.7.5
github.com/minio/selfupdate v0.6.0 // indirect
github.com/minio/websocket v1.6.0
Expand All @@ -38,21 +38,21 @@ require (
github.com/secure-io/sio-go v0.3.1
github.com/stretchr/testify v1.8.4
github.com/tidwall/gjson v1.17.0
github.com/unrolled/secure v1.13.0
golang.org/x/crypto v0.16.0
github.com/unrolled/secure v1.14.0
golang.org/x/crypto v0.17.0
golang.org/x/net v0.19.0
golang.org/x/oauth2 v0.15.0
// Added to include security fix for
// https://github.com/golang/go/issues/56152
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.5.0
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.28.4
k8s.io/apimachinery v0.28.4
k8s.io/client-go v0.28.4
k8s.io/code-generator v0.28.4
k8s.io/api v0.29.0
k8s.io/apimachinery v0.29.0
k8s.io/client-go v0.29.0
k8s.io/code-generator v0.29.0
k8s.io/klog/v2 v2.110.1
k8s.io/kubectl v0.28.4
k8s.io/kubectl v0.29.0
k8s.io/utils v0.0.0-20231127182322-b307cd553661
sigs.k8s.io/structured-merge-diff/v4 v4.4.1
)
Expand All @@ -66,16 +66,16 @@ require (

require (
aead.dev/mem v0.2.0 // indirect
aead.dev/minisign v0.2.0 // indirect
aead.dev/minisign v0.2.1 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/charmbracelet/bubbles v0.16.1 // indirect
github.com/charmbracelet/bubbletea v0.24.2 // indirect
github.com/charmbracelet/lipgloss v0.8.0 // indirect
github.com/charmbracelet/bubbles v0.17.1 // indirect
github.com/charmbracelet/bubbletea v0.25.0 // indirect
github.com/charmbracelet/lipgloss v0.9.1 // indirect
github.com/cheggaaa/pb v1.0.29 // indirect
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
Expand All @@ -92,12 +92,12 @@ require (
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/gdamore/encoding v1.0.0 // indirect
github.com/gdamore/tcell/v2 v2.6.0 // indirect
github.com/gdamore/tcell/v2 v2.7.0 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-openapi/analysis v0.21.4 // indirect
github.com/go-openapi/jsonpointer v0.20.0 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/analysis v0.22.0 // indirect
github.com/go-openapi/jsonpointer v0.20.2 // indirect
github.com/go-openapi/jsonreference v0.20.4 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
Expand All @@ -110,30 +110,31 @@ require (
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jedib0t/go-pretty/v6 v6.4.7 // indirect
github.com/jedib0t/go-pretty/v6 v6.4.9 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/juju/ratelimit v1.0.2 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
github.com/lestrrat-go/blackmagic v1.0.2 // indirect
github.com/lestrrat-go/httpcc v1.0.1 // indirect
github.com/lestrrat-go/iter v1.0.2 // indirect
github.com/lestrrat-go/jwx v1.2.27 // indirect
github.com/lestrrat-go/option v1.0.1 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect
github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-ieproxy v0.0.11 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
github.com/minio/colorjson v1.0.6 // indirect
github.com/minio/filepath v1.0.0 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/pkg/v2 v2.0.4 // indirect
github.com/minio/pkg/v2 v2.0.7 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
Expand All @@ -143,7 +144,7 @@ require (
github.com/muesli/reflow v0.3.0 // indirect
github.com/muesli/termenv v0.15.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/navidys/tvxwidgets v0.3.0 // indirect
github.com/navidys/tvxwidgets v0.4.1 // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
Expand All @@ -154,42 +155,42 @@ require (
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/posener/complete v1.2.3 // indirect
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
github.com/prometheus/client_golang v1.16.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/prometheus/client_golang v1.17.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.45.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/prometheus/prom2json v1.3.3 // indirect
github.com/rivo/tview v0.0.0-20230909130259-ba6a2a345459 // indirect
github.com/rivo/tview v0.0.0-20231206124440-5f078138442e // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rjeczalik/notify v0.9.3 // indirect
github.com/safchain/ethtool v0.3.0 // indirect
github.com/shirou/gopsutil/v3 v3.23.8 // indirect
github.com/shirou/gopsutil/v3 v3.23.11 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tinylib/msgp v1.1.8 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/tinylib/msgp v1.1.9 // indirect
github.com/tklauser/go-sysconf v0.3.13 // indirect
github.com/tklauser/numcpus v0.7.0 // indirect
github.com/vbatts/tar-split v0.11.3 // indirect
github.com/vbauerster/mpb/v8 v8.6.2 // indirect
github.com/vbauerster/mpb/v8 v8.7.1 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
go.etcd.io/etcd/api/v3 v3.5.9 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
go.etcd.io/etcd/client/v3 v3.5.9 // indirect
go.mongodb.org/mongo-driver v1.12.0 // indirect
go.etcd.io/etcd/api/v3 v3.5.11 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.11 // indirect
go.etcd.io/etcd/client/v3 v3.5.11 // indirect
go.mongodb.org/mongo-driver v1.13.1 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.25.0 // indirect
golang.org/x/sync v0.4.0 // indirect
go.uber.org/zap v1.26.0 // indirect
golang.org/x/sync v0.5.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/term v0.15.0 // indirect
golang.org/x/tools v0.13.0 // indirect
golang.org/x/tools v0.14.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb // indirect
google.golang.org/grpc v1.58.3 // indirect
google.golang.org/genproto v0.0.0-20231212172506-995d672761c0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 // indirect
google.golang.org/grpc v1.60.1 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/h2non/filetype.v1 v1.0.5 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
Expand Down
Loading

0 comments on commit 3a620d5

Please sign in to comment.