From d3d7c5e86bffdcf8fa567bca50fe82b95f893580 Mon Sep 17 00:00:00 2001 From: d-kuro Date: Wed, 28 Aug 2024 13:48:32 +0900 Subject: [PATCH 1/2] Add mysql-configmap-history-limit flags Signed-off-by: d-kuro --- cmd/moco-controller/cmd/root.go | 36 ++++++----- cmd/moco-controller/cmd/run.go | 25 ++++---- controllers/mysqlcluster_controller.go | 37 +++++++---- controllers/mysqlcluster_controller_test.go | 62 ++++++++++++------ docs/moco-controller.md | 71 +++++++++++---------- 5 files changed, 137 insertions(+), 94 deletions(-) diff --git a/cmd/moco-controller/cmd/root.go b/cmd/moco-controller/cmd/root.go index 39694fc85..5c0b83e94 100644 --- a/cmd/moco-controller/cmd/root.go +++ b/cmd/moco-controller/cmd/root.go @@ -23,23 +23,24 @@ var ( ) var config struct { - metricsAddr string - probeAddr string - pprofAddr string - leaderElectionID string - webhookAddr string - certDir string - grpcCertDir string - agentImage string - backupImage string - fluentBitImage string - exporterImage string - pvcSyncAnnotationKeys []string - pvcSyncLabelKeys []string - interval time.Duration - maxConcurrentReconciles int - qps int - zapOpts zap.Options + metricsAddr string + probeAddr string + pprofAddr string + leaderElectionID string + webhookAddr string + certDir string + grpcCertDir string + agentImage string + backupImage string + fluentBitImage string + exporterImage string + pvcSyncAnnotationKeys []string + pvcSyncLabelKeys []string + interval time.Duration + maxConcurrentReconciles int + mySQLConfigMapHistoryLimit int + qps int + zapOpts zap.Options } func init() { @@ -109,6 +110,7 @@ func init() { fs.StringSliceVar(&config.pvcSyncLabelKeys, "pvc-sync-label-keys", []string{}, "The keys of labels from MySQLCluster's volumeClaimTemplates to be synced to the PVC") fs.DurationVar(&config.interval, "check-interval", 1*time.Minute, "Interval of cluster maintenance") fs.IntVar(&config.maxConcurrentReconciles, "max-concurrent-reconciles", 8, "The maximum number of concurrent reconciles which can be run") + fs.IntVar(&config.mySQLConfigMapHistoryLimit, "mysql-configmap-history-limit", 10, "The maximum number of MySQLConfigMap's history to be kept") // The default QPS is 20. // https://github.com/kubernetes-sigs/controller-runtime/blob/a26de2d610c3cf4b2a02688534aaf5a65749c743/pkg/client/config/config.go#L84-L85 fs.IntVar(&config.qps, "apiserver-qps-throttle", 20, "The maximum QPS to the API server.") diff --git a/cmd/moco-controller/cmd/run.go b/cmd/moco-controller/cmd/run.go index ecbd9a5dc..dec0425c6 100644 --- a/cmd/moco-controller/cmd/run.go +++ b/cmd/moco-controller/cmd/run.go @@ -102,18 +102,19 @@ func subMain(ns, addr string, port int) error { defer clusterMgr.StopAll() if err = (&controllers.MySQLClusterReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor("moco-controller"), - AgentImage: config.agentImage, - BackupImage: config.backupImage, - FluentBitImage: config.fluentBitImage, - ExporterImage: config.exporterImage, - SystemNamespace: ns, - PVCSyncAnnotationKeys: config.pvcSyncAnnotationKeys, - PVCSyncLabelKeys: config.pvcSyncLabelKeys, - ClusterManager: clusterMgr, - MaxConcurrentReconciles: config.maxConcurrentReconciles, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("moco-controller"), + AgentImage: config.agentImage, + BackupImage: config.backupImage, + FluentBitImage: config.fluentBitImage, + ExporterImage: config.exporterImage, + SystemNamespace: ns, + PVCSyncAnnotationKeys: config.pvcSyncAnnotationKeys, + PVCSyncLabelKeys: config.pvcSyncLabelKeys, + ClusterManager: clusterMgr, + MaxConcurrentReconciles: config.maxConcurrentReconciles, + MySQLConfigMapHistoryLimit: config.mySQLConfigMapHistoryLimit, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "MySQLCluster") return err diff --git a/controllers/mysqlcluster_controller.go b/controllers/mysqlcluster_controller.go index 07a5cbba5..361b0f188 100644 --- a/controllers/mysqlcluster_controller.go +++ b/controllers/mysqlcluster_controller.go @@ -8,6 +8,7 @@ import ( "hash/fnv" "os" "path/filepath" + "sort" "strings" "time" @@ -136,17 +137,18 @@ func apply[S any, T clientObjectConstraint[S], U any](ctx context.Context, r cli // MySQLClusterReconciler reconciles a MySQLCluster object type MySQLClusterReconciler struct { client.Client - Scheme *runtime.Scheme - Recorder record.EventRecorder - AgentImage string - BackupImage string - FluentBitImage string - ExporterImage string - SystemNamespace string - PVCSyncAnnotationKeys []string - PVCSyncLabelKeys []string - ClusterManager clustering.ClusterManager - MaxConcurrentReconciles int + Scheme *runtime.Scheme + Recorder record.EventRecorder + AgentImage string + BackupImage string + FluentBitImage string + ExporterImage string + SystemNamespace string + PVCSyncAnnotationKeys []string + PVCSyncLabelKeys []string + ClusterManager clustering.ClusterManager + MaxConcurrentReconciles int + MySQLConfigMapHistoryLimit int } //+kubebuilder:rbac:groups=moco.cybozu.com,resources=mysqlclusters,verbs=get;list;watch;update;patch @@ -506,11 +508,22 @@ func (r *MySQLClusterReconciler) reconcileV1MyCnf(ctx context.Context, req ctrl. if err := r.List(ctx, cms, client.InNamespace(cluster.Namespace)); err != nil { return nil, err } - for _, old := range cms.Items { + + // Sort the ConfigMapList by creation timestamp in descending order + sort.Slice(cms.Items, func(i, j int) bool { + return cms.Items[i].CreationTimestamp.Time.After(cms.Items[j].CreationTimestamp.Time) + }) + + for i, old := range cms.Items { + if i < r.MySQLConfigMapHistoryLimit { + continue + } + if strings.HasPrefix(old.Name, prefix) && old.Name != cmName { if err := r.Delete(ctx, &old); err != nil { return nil, fmt.Errorf("failed to delete old my.cnf configmap %s/%s: %w", old.Namespace, old.Name, err) } + log.Info("deleted old my.cnf configmap", "configMapName", old.Name) } } diff --git a/controllers/mysqlcluster_controller_test.go b/controllers/mysqlcluster_controller_test.go index 1efffdfaf..138f4f4ef 100644 --- a/controllers/mysqlcluster_controller_test.go +++ b/controllers/mysqlcluster_controller_test.go @@ -118,15 +118,16 @@ var _ = Describe("MySQLCluster reconciler", func() { clusters: make(map[string]struct{}), } mysqlr := &MySQLClusterReconciler{ - Client: mgr.GetClient(), - Scheme: scheme, - Recorder: mgr.GetEventRecorderFor("moco-controller"), - SystemNamespace: testMocoSystemNamespace, - ClusterManager: mockMgr, - AgentImage: testAgentImage, - BackupImage: testBackupImage, - FluentBitImage: testFluentBitImage, - ExporterImage: testExporterImage, + Client: mgr.GetClient(), + Scheme: scheme, + Recorder: mgr.GetEventRecorderFor("moco-controller"), + SystemNamespace: testMocoSystemNamespace, + ClusterManager: mockMgr, + AgentImage: testAgentImage, + BackupImage: testBackupImage, + FluentBitImage: testFluentBitImage, + ExporterImage: testExporterImage, + MySQLConfigMapHistoryLimit: 2, } err = mysqlr.SetupWithManager(mgr) Expect(err).ToNot(HaveOccurred()) @@ -418,19 +419,28 @@ var _ = Describe("MySQLCluster reconciler", func() { err = k8sClient.Update(ctx, cluster) Expect(err).NotTo(HaveOccurred()) - oldName := cm.Name Eventually(func() error { cms := &corev1.ConfigMapList{} if err := k8sClient.List(ctx, cms, client.InNamespace("test")); err != nil { return err } - var mycnfCMs []*corev1.ConfigMap - for i, cm := range cms.Items { - if cm.Name == oldName { + var mycnfCount int + prefix := cluster.PrefixedName() + "." + for _, cm := range cms.Items { + if !strings.HasPrefix(cm.Name, prefix) { continue } - if strings.HasPrefix(cm.Name, "moco-test.") { + mycnfCount++ + } + + if mycnfCount != 2 { + return fmt.Errorf("the number of config maps is not history limits: %d", len(cms.Items)) + } + + var mycnfCMs []*corev1.ConfigMap + for i, cm := range cms.Items { + if strings.HasPrefix(cm.Name, "moco-test.") && strings.Contains(cm.Data["my.cnf"], "foo = bar") { mycnfCMs = append(mycnfCMs, &cms.Items[i]) } } @@ -454,19 +464,28 @@ var _ = Describe("MySQLCluster reconciler", func() { err = k8sClient.Update(ctx, userCM) Expect(err).NotTo(HaveOccurred()) - oldName = cm.Name Eventually(func() error { cms := &corev1.ConfigMapList{} if err := k8sClient.List(ctx, cms, client.InNamespace("test")); err != nil { return err } - var mycnfCMs []*corev1.ConfigMap - for i, cm := range cms.Items { - if cm.Name == oldName { + var mycnfCount int + prefix := cluster.PrefixedName() + "." + for _, cm := range cms.Items { + if !strings.HasPrefix(cm.Name, prefix) { continue } - if strings.HasPrefix(cm.Name, "moco-test.") { + mycnfCount++ + } + + if mycnfCount != 2 { + return fmt.Errorf("the number of config maps is not history limits: %d", len(cms.Items)) + } + + var mycnfCMs []*corev1.ConfigMap + for i, cm := range cms.Items { + if strings.HasPrefix(cm.Name, "moco-test.") && strings.Contains(cm.Data["my.cnf"], "foo = baz") { mycnfCMs = append(mycnfCMs, &cms.Items[i]) } } @@ -479,6 +498,11 @@ var _ = Describe("MySQLCluster reconciler", func() { return nil }).Should(Succeed()) + cms := &corev1.ConfigMapList{} + if err := k8sClient.List(ctx, cms, client.InNamespace("test")); err != nil { + panic(err) + } + Expect(cm.Data["my.cnf"]).To(ContainSubstring("foo = baz")) }) diff --git a/docs/moco-controller.md b/docs/moco-controller.md index 06603fff1..f54f38bd1 100644 --- a/docs/moco-controller.md +++ b/docs/moco-controller.md @@ -12,38 +12,41 @@ ``` Flags: - --add_dir_header If true, adds the file directory to the header of the log messages - --agent-image string The image of moco-agent sidecar container - --alsologtostderr log to standard error as well as files (no effect when -logtostderr=true) - --apiserver-qps-throttle int The maximum QPS to the API server. (default 20) - --backup-image string The image of moco-backup container - --cert-dir string webhook certificate directory - --check-interval duration Interval of cluster maintenance (default 1m0s) - --fluent-bit-image string The image of fluent-bit sidecar container - --grpc-cert-dir string gRPC certificate directory (default "/grpc-cert") - --health-probe-addr string Listen address for health probes (default ":8081") - -h, --help help for moco-controller - --leader-election-id string ID for leader election by controller-runtime (default "moco") - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory (no effect when -logtostderr=true) - --log_file string If non-empty, use this log file (no effect when -logtostderr=true) - --log_file_max_size uint Defines the maximum size a log file can grow to (no effect when -logtostderr=true). Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) - --logtostderr log to standard error instead of files (default true) - --max-concurrent-reconciles int The maximum number of concurrent reconciles which can be run (default 8) - --metrics-addr string Listen address for metric endpoint (default ":8080") - --mysqld-exporter-image string The image of mysqld_exporter sidecar container - --one_output If true, only write logs to their native severity level (vs also writing to each lower severity level; no effect when -logtostderr=true) - --pprof-addr string Listen address for pprof endpoints. pprof is disabled by default - --skip_headers If true, avoid header prefixes in the log messages - --skip_log_headers If true, avoid headers when opening log files (no effect when -logtostderr=true) - --stderrthreshold severity logs at or above this threshold go to stderr when writing to files and stderr (no effect when -logtostderr=true or -alsologtostderr=false) (default 2) - -v, --v Level number for the log level verbosity - --version version for moco-controller - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --webhook-addr string Listen address for the webhook endpoint (default ":9443") - --zap-devel Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) - --zap-encoder encoder Zap log encoding (one of 'json' or 'console') - --zap-log-level level Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', or any integer value > 0 which corresponds to custom debug levels of increasing verbosity - --zap-stacktrace-level level Zap Level at and above which stacktraces are captured (one of 'info', 'error', 'panic'). - --zap-time-encoding time-encoding Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano'). Defaults to 'epoch'. + --add_dir_header If true, adds the file directory to the header of the log messages + --agent-image string The image of moco-agent sidecar container (default "ghcr.io/cybozu-go/moco-agent:0.12.1") + --alsologtostderr log to standard error as well as files (no effect when -logtostderr=true) + --apiserver-qps-throttle int The maximum QPS to the API server. (default 20) + --backup-image string The image of moco-backup container (default "ghcr.io/cybozu-go/moco-backup:0.23.2") + --cert-dir string webhook certificate directory + --check-interval duration Interval of cluster maintenance (default 1m0s) + --fluent-bit-image string The image of fluent-bit sidecar container (default "ghcr.io/cybozu-go/moco/fluent-bit:3.0.2.1") + --grpc-cert-dir string gRPC certificate directory (default "/grpc-cert") + --health-probe-addr string Listen address for health probes (default ":8081") + -h, --help help for moco-controller + --leader-election-id string ID for leader election by controller-runtime (default "moco") + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory (no effect when -logtostderr=true) + --log_file string If non-empty, use this log file (no effect when -logtostderr=true) + --log_file_max_size uint Defines the maximum size a log file can grow to (no effect when -logtostderr=true). Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) + --logtostderr log to standard error instead of files (default true) + --max-concurrent-reconciles int The maximum number of concurrent reconciles which can be run (default 8) + --metrics-addr string Listen address for metric endpoint (default ":8080") + --mysql-configmap-history-limit int The maximum number of MySQLConfigMap's history to be kept (default 10) + --mysqld-exporter-image string The image of mysqld_exporter sidecar container (default "ghcr.io/cybozu-go/moco/mysqld_exporter:0.15.1.2") + --one_output If true, only write logs to their native severity level (vs also writing to each lower severity level; no effect when -logtostderr=true) + --pprof-addr string Listen address for pprof endpoints. pprof is disabled by default + --pvc-sync-annotation-keys strings The keys of annotations from MySQLCluster's volumeClaimTemplates to be synced to the PVC + --pvc-sync-label-keys strings The keys of labels from MySQLCluster's volumeClaimTemplates to be synced to the PVC + --skip_headers If true, avoid header prefixes in the log messages + --skip_log_headers If true, avoid headers when opening log files (no effect when -logtostderr=true) + --stderrthreshold severity logs at or above this threshold go to stderr when writing to files and stderr (no effect when -logtostderr=true or -alsologtostderr=true) (default 2) + -v, --v Level number for the log level verbosity + --version version for moco-controller + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --webhook-addr string Listen address for the webhook endpoint (default ":9443") + --zap-devel Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) + --zap-encoder encoder Zap log encoding (one of 'json' or 'console') + --zap-log-level level Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', or any integer value > 0 which corresponds to custom debug levels of increasing verbosity + --zap-stacktrace-level level Zap Level at and above which stacktraces are captured (one of 'info', 'error', 'panic'). + --zap-time-encoding time-encoding Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano'). Defaults to 'epoch'. ``` From 7b3d5a25328e92c149bfc9da6247b008b8aaeb89 Mon Sep 17 00:00:00 2001 From: Masayuki Ishii Date: Wed, 18 Sep 2024 15:30:10 +0900 Subject: [PATCH 2/2] Update controllers/mysqlcluster_controller_test.go --- controllers/mysqlcluster_controller_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/controllers/mysqlcluster_controller_test.go b/controllers/mysqlcluster_controller_test.go index 138f4f4ef..22a0c272a 100644 --- a/controllers/mysqlcluster_controller_test.go +++ b/controllers/mysqlcluster_controller_test.go @@ -498,11 +498,6 @@ var _ = Describe("MySQLCluster reconciler", func() { return nil }).Should(Succeed()) - cms := &corev1.ConfigMapList{} - if err := k8sClient.List(ctx, cms, client.InNamespace("test")); err != nil { - panic(err) - } - Expect(cm.Data["my.cnf"]).To(ContainSubstring("foo = baz")) })