From 9b1f463b779ccd2161117160e907aeb113a67b6f Mon Sep 17 00:00:00 2001
From: xiongww <xiongww@chinatelecom.cn>
Date: Tue, 23 Jan 2024 16:58:04 +0800
Subject: [PATCH 01/15] keep chrony config file clean and consistency when
 install and uninstall cluster in the same nodes

---
 cmd/kk/pkg/bootstrap/os/tasks.go | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/cmd/kk/pkg/bootstrap/os/tasks.go b/cmd/kk/pkg/bootstrap/os/tasks.go
index 1f3467610..d663dc296 100644
--- a/cmd/kk/pkg/bootstrap/os/tasks.go
+++ b/cmd/kk/pkg/bootstrap/os/tasks.go
@@ -504,15 +504,33 @@ func (n *NodeConfigureNtpServer) Execute(runtime connector.Runtime) error {
 		chronyService = "chrony.service"
 	}
 
+	clearOldServerCmd := fmt.Sprintf(`sed -i '/^server/d' %s`, chronyConfigFile)
+	if _, err := runtime.GetRunner().SudoCmd(clearOldServerCmd, false); err != nil {
+		return errors.Wrapf(err, "delete old servers failed, please check file %s", chronyConfigFile)
+	}
+
 	// if NtpServers was configured
 	for _, server := range n.KubeConf.Cluster.System.NtpServers {
 
 		serverAddr := strings.Trim(server, " \"")
+		fmt.Printf("ntpserver: %s, current host: %s\n", serverAddr, currentHost.GetName())
 		if serverAddr == currentHost.GetName() || serverAddr == currentHost.GetInternalAddress() {
-			allowClientCmd := fmt.Sprintf(`sed -i '/#allow/ a\allow 0.0.0.0/0' %s`, chronyConfigFile)
+			deleteAllowCmd := fmt.Sprintf(`sed -i '/^allow/d' %s`, chronyConfigFile)
+			if _, err := runtime.GetRunner().SudoCmd(deleteAllowCmd, false); err != nil {
+				return errors.Wrapf(err, "delete allow failed, please check file %s", chronyConfigFile)
+			}
+			allowClientCmd := fmt.Sprintf(`echo 'allow 0.0.0.0/0' >> %s`, chronyConfigFile)
 			if _, err := runtime.GetRunner().SudoCmd(allowClientCmd, false); err != nil {
 				return errors.Wrapf(err, "change host:%s chronyd conf failed, please check file %s", serverAddr, chronyConfigFile)
 			}
+			deleteLocalCmd := fmt.Sprintf(`sed -i '/^local/d' %s`, chronyConfigFile)
+			if _, err := runtime.GetRunner().SudoCmd(deleteLocalCmd, false); err != nil {
+				return errors.Wrapf(err, "delete local stratum failed, please check file %s", chronyConfigFile)
+			}
+			AddLocalCmd := fmt.Sprintf(`echo 'local stratum 10' >> %s`, chronyConfigFile)
+			if _, err := runtime.GetRunner().SudoCmd(AddLocalCmd, false); err != nil {
+				return errors.Wrapf(err, "Add local stratum 10 conf failed, please check file %s", chronyConfigFile)
+			}
 		}
 
 		// use internal ip to client chronyd server

From ace1ce7ceb02a86df33b4777be843ed10e58c4e8 Mon Sep 17 00:00:00 2001
From: stark <stark@yunify.com>
Date: Wed, 31 Jan 2024 10:47:38 +0800
Subject: [PATCH 02/15] Support use docker for version >= v1.24.0

---
 cmd/kk/apis/kubekey/v1alpha2/default.go       |  9 ++-
 cmd/kk/pkg/binaries/kubernetes.go             |  8 ++-
 cmd/kk/pkg/bootstrap/confirm/tasks.go         | 33 +++++-----
 cmd/kk/pkg/common/common.go                   |  1 +
 cmd/kk/pkg/container/docker.go                | 59 ++++++++++++++++-
 cmd/kk/pkg/container/module.go                | 56 +++++++++++++++++
 cmd/kk/pkg/container/prepares.go              | 18 ++++++
 .../templates/cri_dockerd_service.go          | 63 +++++++++++++++++++
 cmd/kk/pkg/files/file.go                      |  6 ++
 cmd/kk/pkg/kubernetes/tasks.go                |  8 ++-
 cmd/kk/pkg/version/kubernetes/version_enum.go | 13 ++++
 version/components.json                       |  8 +++
 12 files changed, 259 insertions(+), 23 deletions(-)
 create mode 100644 cmd/kk/pkg/container/templates/cri_dockerd_service.go

diff --git a/cmd/kk/apis/kubekey/v1alpha2/default.go b/cmd/kk/apis/kubekey/v1alpha2/default.go
index eede3ac42..f979e5bef 100644
--- a/cmd/kk/apis/kubekey/v1alpha2/default.go
+++ b/cmd/kk/apis/kubekey/v1alpha2/default.go
@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
 )
 
 const (
@@ -42,6 +43,7 @@ const (
 	DefaultEtcdVersion             = "v3.5.6"
 	DefaultEtcdPort                = "2379"
 	DefaultDockerVersion           = "24.0.6"
+	DefaultCriDockerdVersion        = "0.3.9"
 	DefaultContainerdVersion       = "1.7.12"
 	DefaultRuncVersion             = "v1.1.11"
 	DefaultCrictlVersion           = "v1.29.0"
@@ -68,6 +70,7 @@ const (
 	DefaultProxyMode               = "ipvs"
 	DefaultCrioEndpoint            = "unix:///var/run/crio/crio.sock"
 	DefaultContainerdEndpoint      = "unix:///run/containerd/containerd.sock"
+	DefaultCriDockerdEndpoint      = "unix:///var/run/cri-dockerd.sock"
 	DefaultIsulaEndpoint           = "unix:///var/run/isulad.sock"
 	Etcd                           = "etcd"
 	Master                         = "master"
@@ -315,7 +318,11 @@ func SetDefaultClusterCfg(cfg *ClusterSpec) Kubernetes {
 	if cfg.Kubernetes.ContainerRuntimeEndpoint == "" {
 		switch cfg.Kubernetes.ContainerManager {
 		case Docker:
-			cfg.Kubernetes.ContainerRuntimeEndpoint = ""
+			if kubernetes.IsAtLeastV124(cfg.Kubernetes.Version){
+				cfg.Kubernetes.ContainerRuntimeEndpoint = DefaultCriDockerdEndpoint
+			} else {
+				cfg.Kubernetes.ContainerRuntimeEndpoint = ""
+			}
 		case Crio:
 			cfg.Kubernetes.ContainerRuntimeEndpoint = DefaultCrioEndpoint
 		case Containerd:
diff --git a/cmd/kk/pkg/binaries/kubernetes.go b/cmd/kk/pkg/binaries/kubernetes.go
index 4e381305a..df87b25a2 100644
--- a/cmd/kk/pkg/binaries/kubernetes.go
+++ b/cmd/kk/pkg/binaries/kubernetes.go
@@ -20,14 +20,14 @@ import (
 	"fmt"
 	"os/exec"
 
-	"github.com/pkg/errors"
-
 	kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
+	"github.com/pkg/errors"
 )
 
 // K8sFilesDownloadHTTP defines the kubernetes' binaries that need to be downloaded in advance and downloads them.
@@ -40,6 +40,7 @@ func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string,
 	kubecni := files.NewKubeBinary("kubecni", arch, kubekeyapiv1alpha2.DefaultCniVersion, path, kubeConf.Arg.DownloadCommand)
 	helm := files.NewKubeBinary("helm", arch, kubekeyapiv1alpha2.DefaultHelmVersion, path, kubeConf.Arg.DownloadCommand)
 	docker := files.NewKubeBinary("docker", arch, kubekeyapiv1alpha2.DefaultDockerVersion, path, kubeConf.Arg.DownloadCommand)
+	criDockerd := files.NewKubeBinary("cri-dockerd", arch, kubekeyapiv1alpha2.DefaultCriDockerdVersion, path, kubeConf.Arg.DownloadCommand)
 	crictl := files.NewKubeBinary("crictl", arch, kubekeyapiv1alpha2.DefaultCrictlVersion, path, kubeConf.Arg.DownloadCommand)
 	containerd := files.NewKubeBinary("containerd", arch, kubekeyapiv1alpha2.DefaultContainerdVersion, path, kubeConf.Arg.DownloadCommand)
 	runc := files.NewKubeBinary("runc", arch, kubekeyapiv1alpha2.DefaultRuncVersion, path, kubeConf.Arg.DownloadCommand)
@@ -49,6 +50,9 @@ func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string,
 
 	if kubeConf.Cluster.Kubernetes.ContainerManager == kubekeyapiv1alpha2.Docker {
 		binaries = append(binaries, docker)
+		if kubernetes.IsAtLeastV124(kubeConf.Cluster.Kubernetes.Version) && kubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
+			binaries = append(binaries, criDockerd)
+		}
 	} else if kubeConf.Cluster.Kubernetes.ContainerManager == kubekeyapiv1alpha2.Containerd {
 		binaries = append(binaries, containerd, runc)
 	}
diff --git a/cmd/kk/pkg/bootstrap/confirm/tasks.go b/cmd/kk/pkg/bootstrap/confirm/tasks.go
index 06926da07..59edc41f2 100644
--- a/cmd/kk/pkg/bootstrap/confirm/tasks.go
+++ b/cmd/kk/pkg/bootstrap/confirm/tasks.go
@@ -23,16 +23,16 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/mitchellh/mapstructure"
-	"github.com/modood/table"
-	"github.com/pkg/errors"
-	versionutil "k8s.io/apimachinery/pkg/util/version"
-
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
+	"github.com/mitchellh/mapstructure"
+	"github.com/modood/table"
+	"github.com/pkg/errors"
+	versionutil "k8s.io/apimachinery/pkg/util/version"
 )
 
 // PreCheckResults defines the items to be checked.
@@ -107,18 +107,17 @@ func (i *InstallationConfirm) Execute(runtime connector.Runtime) error {
 	fmt.Println("https://github.com/kubesphere/kubekey#requirements-and-recommendations")
 	fmt.Println("")
 
-	if k8sVersion, err := versionutil.ParseGeneric(i.KubeConf.Cluster.Kubernetes.Version); err == nil {
-		if k8sVersion.AtLeast(versionutil.MustParseSemantic("v1.24.0")) && i.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
-			fmt.Println("[Notice]")
-			fmt.Println("Incorrect runtime. Please specify a container runtime other than Docker to install Kubernetes v1.24 or later.")
-			fmt.Println("You can set \"spec.kubernetes.containerManager\" in the configuration file to \"containerd\" or add \"--container-manager containerd\" to the \"./kk create cluster\" command.")
-			fmt.Println("For more information, see:")
-			fmt.Println("https://github.com/kubesphere/kubekey/blob/master/docs/commands/kk-create-cluster.md")
-			fmt.Println("https://kubernetes.io/docs/setup/production-environment/container-runtimes/#container-runtimes")
-			fmt.Println("https://kubernetes.io/blog/2022/02/17/dockershim-faq/")
-			fmt.Println("")
-			stopFlag = true
-		}
+	if kubernetes.IsAtLeastV124(i.KubeConf.Cluster.Kubernetes.Version) && i.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker &&
+		i.KubeConf.Cluster.Kubernetes.Type != common.Kubernetes {
+		fmt.Println("[Notice]")
+		fmt.Println("Incorrect runtime. Please specify a container runtime other than Docker to install Kubernetes v1.24 or later.")
+		fmt.Println("You can set \"spec.kubernetes.containerManager\" in the configuration file to \"containerd\" or add \"--container-manager containerd\" to the \"./kk create cluster\" command.")
+		fmt.Println("For more information, see:")
+		fmt.Println("https://github.com/kubesphere/kubekey/blob/master/docs/commands/kk-create-cluster.md")
+		fmt.Println("https://kubernetes.io/docs/setup/production-environment/container-runtimes/#container-runtimes")
+		fmt.Println("https://kubernetes.io/blog/2022/02/17/dockershim-faq/")
+		fmt.Println("")
+		stopFlag = true
 	}
 
 	if stopFlag {
diff --git a/cmd/kk/pkg/common/common.go b/cmd/kk/pkg/common/common.go
index 4b24d7a3e..aeaa69f9c 100644
--- a/cmd/kk/pkg/common/common.go
+++ b/cmd/kk/pkg/common/common.go
@@ -62,6 +62,7 @@ const (
 	Hybridnet = "hybridnet"
 
 	Docker     = "docker"
+	CriDockerd = "cri-dockerd"
 	Crictl     = "crictl"
 	Containerd = "containerd"
 	Crio       = "crio"
diff --git a/cmd/kk/pkg/container/docker.go b/cmd/kk/pkg/container/docker.go
index f32807d8e..8be4612a3 100644
--- a/cmd/kk/pkg/container/docker.go
+++ b/cmd/kk/pkg/container/docker.go
@@ -21,14 +21,14 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/pkg/errors"
-
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container/templates"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
+	"github.com/pkg/errors"
 )
 
 type SyncDockerBinaries struct {
@@ -64,6 +64,39 @@ func (s *SyncDockerBinaries) Execute(runtime connector.Runtime) error {
 	return nil
 }
 
+type SyncCriDockerdBinaries struct {
+	common.KubeAction
+}
+
+func (s *SyncCriDockerdBinaries) Execute(runtime connector.Runtime) error {
+	if err := utils.ResetTmpDir(runtime); err != nil {
+		return err
+	}
+
+	binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch())
+	if !ok {
+		return errors.New("get KubeBinary by pipeline cache failed")
+	}
+	binariesMap := binariesMapObj.(map[string]*files.KubeBinary)
+
+	criDockerd, ok := binariesMap[common.CriDockerd]
+	if !ok {
+		return errors.New("get KubeBinary key cri-dockerd by pipeline cache failed")
+	}
+
+	dst := filepath.Join(common.TmpDir, criDockerd.FileName)
+	if err := runtime.GetRunner().Scp(criDockerd.Path(), dst); err != nil {
+		return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync cri-dockerd binaries failed"))
+	}
+
+	if _, err := runtime.GetRunner().SudoCmd(
+		fmt.Sprintf("mkdir -p /usr/bin && tar -zxf %s && mv cri-dockerd/* /usr/bin && rm -rf cri-dockerd", dst),
+		false); err != nil {
+		return errors.Wrap(errors.WithStack(err), fmt.Sprintf("install container runtime cri-dockerd binaries failed"))
+	}
+	return nil
+}
+
 type EnableContainerdForDocker struct {
 	common.KubeAction
 }
@@ -90,6 +123,19 @@ func (e *EnableDocker) Execute(runtime connector.Runtime) error {
 	return nil
 }
 
+type EnableCriDockerd struct {
+	common.KubeAction
+}
+
+func (e *EnableCriDockerd) Execute(runtime connector.Runtime) error {
+	if _, err := runtime.GetRunner().SudoCmd(
+		"systemctl daemon-reload && systemctl enable cri-docker && systemctl start cri-docker",
+		false); err != nil {
+		return errors.Wrap(errors.WithStack(err), fmt.Sprintf("enable and start cri-docker failed"))
+	}
+	return nil
+}
+
 type DockerLoginRegistry struct {
 	common.KubeAction
 }
@@ -141,6 +187,15 @@ func (d *DisableDocker) Execute(runtime connector.Runtime) error {
 		filepath.Join("/etc/systemd/system", templates.DockerService.Name()),
 		filepath.Join("/etc/docker", templates.DockerConfig.Name()),
 	}
+
+	if kubernetes.IsAtLeastV124(d.KubeConf.Cluster.Kubernetes.Version) && d.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
+		if _, err := runtime.GetRunner().SudoCmd("systemctl disable cri-docker && systemctl stop cri-docker",
+			false); err != nil {
+			return errors.Wrap(errors.WithStack(err), fmt.Sprintf("disable and stop cri-docker failed"))
+		}
+		files = append(files, filepath.Join("/etc/systemd/system", templates.CriDockerService.Name()))
+	}
+
 	if d.KubeConf.Cluster.Registry.DataRoot != "" {
 		files = append(files, d.KubeConf.Cluster.Registry.DataRoot)
 	} else {
diff --git a/cmd/kk/pkg/container/module.go b/cmd/kk/pkg/container/module.go
index ba8a8b688..61a7bfa5d 100644
--- a/cmd/kk/pkg/container/module.go
+++ b/cmd/kk/pkg/container/module.go
@@ -30,6 +30,7 @@ import (
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry"
+	versionk8s "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
 )
 
 type InstallContainerModule struct {
@@ -73,6 +74,19 @@ func InstallDocker(m *InstallContainerModule) []task.Interface {
 		Retry:    2,
 	}
 
+	syncCriDockerdBinaries := &task.RemoteTask{
+		Name:  "SyncCriDockerdBinaries",
+		Desc:  "Sync cri-dockerd binaries",
+		Hosts: m.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			&kubernetes.NodeInCluster{Not: true},
+			&CriDockerdExist{Not: true},
+		},
+		Action:   new(SyncCriDockerdBinaries),
+		Parallel: true,
+		Retry:    2,
+	}
+
 	generateContainerdService := &task.RemoteTask{
 		Name:  "GenerateContainerdService",
 		Desc:  "Generate containerd service",
@@ -161,6 +175,48 @@ func InstallDocker(m *InstallContainerModule) []task.Interface {
 		Parallel: true,
 	}
 
+	generateCriDockerdService := &task.RemoteTask{
+		Name:  "GenerateCriDockerdService",
+		Desc:  "Generate cri-dockerd service",
+		Hosts: m.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			&kubernetes.NodeInCluster{Not: true},
+			&CriDockerdExist{Not: true},
+		},
+		Action: &action.Template{
+			Template: templates.CriDockerService,
+			Dst:      filepath.Join("/etc/systemd/system", templates.CriDockerService.Name()),
+		},
+		Parallel: true,
+	}
+
+	enableCriDockerd := &task.RemoteTask{
+		Name:  "EnableCriDockerd",
+		Desc:  "Enable cri-dockerd",
+		Hosts: m.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			&kubernetes.NodeInCluster{Not: true},
+			&CriDockerdExist{Not: true},
+		},
+		Action:   new(EnableCriDockerd),
+		Parallel: true,
+	}
+
+	if versionk8s.IsAtLeastV124(m.KubeConf.Cluster.Kubernetes.Version) && m.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
+		return []task.Interface{
+			syncBinaries,
+			syncCriDockerdBinaries,
+			generateContainerdService,
+			generateDockerService,
+			generateDockerConfig,
+			enableContainerdForDocker,
+			enableDocker,
+			dockerLoginRegistry,
+			generateCriDockerdService,
+			enableCriDockerd,
+		}
+	}
+
 	return []task.Interface{
 		syncBinaries,
 		generateContainerdService,
diff --git a/cmd/kk/pkg/container/prepares.go b/cmd/kk/pkg/container/prepares.go
index fd12032f8..d0f0ba829 100644
--- a/cmd/kk/pkg/container/prepares.go
+++ b/cmd/kk/pkg/container/prepares.go
@@ -41,6 +41,24 @@ func (d *DockerExist) PreCheck(runtime connector.Runtime) (bool, error) {
 	return !d.Not, nil
 }
 
+type CriDockerdExist struct {
+	common.KubePrepare
+	Not bool
+}
+
+func (d *CriDockerdExist) PreCheck(runtime connector.Runtime) (bool, error) {
+	output, err := runtime.GetRunner().SudoCmd("if [ -z $(command -v cri-dockerd) ] || [ ! -e /var/run/cri-dockerd.sock ]; "+
+		"then echo 'not exist'; "+
+		"fi", false)
+	if err != nil {
+		return false, err
+	}
+	if strings.Contains(output, "not exist") {
+		return d.Not, nil
+	}
+	return !d.Not, nil
+}
+
 type CrictlExist struct {
 	common.KubePrepare
 	Not bool
diff --git a/cmd/kk/pkg/container/templates/cri_dockerd_service.go b/cmd/kk/pkg/container/templates/cri_dockerd_service.go
new file mode 100644
index 000000000..ed5ca4196
--- /dev/null
+++ b/cmd/kk/pkg/container/templates/cri_dockerd_service.go
@@ -0,0 +1,63 @@
+/*
+ Copyright 2021 The KubeSphere Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package templates
+
+import (
+	"text/template"
+
+	"github.com/lithammer/dedent"
+)
+
+var CriDockerService = template.Must(template.New("cri-docker.service").Parse(
+	dedent.Dedent(`[Unit]
+Description=CRI Interface for Docker Application Container Engine
+Documentation=https://docs.mirantis.com
+
+[Service]
+Type=notify
+ExecStart=/usr/bin/cri-dockerd --pod-infra-container-image docker.io/kubesphere/pause:3.8
+ExecReload=/bin/kill -s HUP $MAINPID
+TimeoutSec=0
+RestartSec=2
+Restart=always
+
+# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
+# Both the old, and new location are accepted by systemd 229 and up, so using the old location
+# to make them work for either version of systemd.
+StartLimitBurst=3
+
+# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
+# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
+# this option work for either version of systemd.
+StartLimitInterval=60s
+
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=infinity
+LimitNPROC=infinity
+LimitCORE=infinity
+
+# Comment TasksMax if your systemd version does not support it.
+# Only systemd 226 and above support this option.
+TasksMax=infinity
+Delegate=yes
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target
+
+    `)))
diff --git a/cmd/kk/pkg/files/file.go b/cmd/kk/pkg/files/file.go
index bba319b6c..a3a8cc0d4 100644
--- a/cmd/kk/pkg/files/file.go
+++ b/cmd/kk/pkg/files/file.go
@@ -44,6 +44,7 @@ const (
 	k3s        = "k3s"
 	k8e        = "k8e"
 	docker     = "docker"
+	cridockerd = "cri-dockerd"
 	crictl     = "crictl"
 	registry   = "registry"
 	harbor     = "harbor"
@@ -58,6 +59,7 @@ const (
 	CNI        = "cni"
 	CRICTL     = "crictl"
 	DOCKER     = "docker"
+	CRIDOCKERD = "cri-dockerd"
 	ETCD       = "etcd"
 	HELM       = "helm"
 	KUBE       = "kube"
@@ -147,6 +149,10 @@ func NewKubeBinary(name, arch, version, prePath string, getCmd func(path, url st
 		if component.Zone == "cn" {
 			component.Url = fmt.Sprintf("https://mirrors.aliyun.com/docker-ce/linux/static/stable/%s/docker-%s.tgz", util.ArchAlias(arch), version)
 		}
+	case cridockerd:
+		component.Type = CRIDOCKERD
+		component.FileName = fmt.Sprintf("cri-dockerd-%s.tgz", version)
+		component.Url = fmt.Sprintf("https://github.com/Mirantis/cri-dockerd/releases/download/v%s/cri-dockerd-%s.%s.tgz", version, version, arch)
 	case crictl:
 		component.Type = CRICTL
 		component.FileName = fmt.Sprintf("crictl-%s-linux-%s.tar.gz", version, arch)
diff --git a/cmd/kk/pkg/kubernetes/tasks.go b/cmd/kk/pkg/kubernetes/tasks.go
index fce0e8cfb..aaeda50fd 100644
--- a/cmd/kk/pkg/kubernetes/tasks.go
+++ b/cmd/kk/pkg/kubernetes/tasks.go
@@ -46,6 +46,7 @@ import (
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes/templates"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
 )
 
 type GetClusterStatus struct {
@@ -715,10 +716,15 @@ func (u *UpgradeKubeMaster) Execute(runtime connector.Runtime) error {
 		return errors.Wrap(errors.WithStack(err), fmt.Sprintf("stop kubelet failed: %s", host.GetName()))
 	}
 
-	if versionutil.MustParseSemantic(u.KubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.24.0")) {
+	if kubernetes.IsAtLeastV124(u.KubeConf.Cluster.Kubernetes.Version){
 		if _, err := runtime.GetRunner().SudoCmd("sed -i 's/ --network-plugin=cni / /g' /var/lib/kubelet/kubeadm-flags.env", false); err != nil {
 			return errors.Wrap(errors.WithStack(err), fmt.Sprintf("update kubelet config failed: %s", host.GetName()))
 		}
+		if u.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
+			if _, err := runtime.GetRunner().SudoCmd("sed -i 's/ --container-runtime=remote / /g' /var/lib/kubelet/kubeadm-flags.env", false); err != nil {
+				return errors.Wrap(errors.WithStack(err), fmt.Sprintf("update kubelet config failed: %s", host.GetName()))
+			}
+		}
 	}
 
 	if err := SetKubeletTasks(runtime, u.KubeAction); err != nil {
diff --git a/cmd/kk/pkg/version/kubernetes/version_enum.go b/cmd/kk/pkg/version/kubernetes/version_enum.go
index e0fffdbbf..730466d35 100644
--- a/cmd/kk/pkg/version/kubernetes/version_enum.go
+++ b/cmd/kk/pkg/version/kubernetes/version_enum.go
@@ -103,3 +103,16 @@ func SupportedK8sVersionList() []string {
 
 	return versionsList
 }
+
+func IsAtLeastV124(clusterVersion string) bool {
+	parsedVersion, err := versionutil.ParseGeneric(clusterVersion)
+	if err != nil {
+		return false
+	}
+
+	if parsedVersion.AtLeast(versionutil.MustParseSemantic("v1.24.0")) {
+		return true
+	}
+
+	return false
+}
\ No newline at end of file
diff --git a/version/components.json b/version/components.json
index 2de3b3cee..a95d94024 100644
--- a/version/components.json
+++ b/version/components.json
@@ -1023,6 +1023,14 @@
             "24.0.6": "d9f58aecc42451503e82e6e0562cafa1812b334c92186a7f486e111e70a0f5bd"
         }
     },
+    "cri-dockerd": {
+        "amd64": {
+            "0.3.9": "a6d9b4b796e9eff830311a2349d259507302cb3955dd07b78296b91e40e8b433"
+        },
+        "arm64": {
+            "0.3.9": "f5051002b4f95b0e8fe7fbd5f8de4493350e010834d2a8b647f2b26c45c6c203"
+        }
+    },
     "containerd": {
         "amd64": {
             "1.6.2": "3d94f887de5f284b0d6ee61fa17ba413a7d60b4bb27d756a402b713a53685c6a",

From 45abd0984a86a061466f206a87a0a6474bb3c34f Mon Sep 17 00:00:00 2001
From: wangxingrui <wangxingrui@leinao.ai>
Date: Wed, 31 Jan 2024 15:52:33 +0800
Subject: [PATCH 03/15] When deleting a cluster, also delete the /pki/etcd
 directory in the current working directory, otherwise when the host's
 internal ip is changed, the redeployment of the etcd certificate will still
 use the certificate generated by the previous ip, which will result in the
 failure of the normal connection of etcd

---
 cmd/kk/pkg/bootstrap/os/tasks.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cmd/kk/pkg/bootstrap/os/tasks.go b/cmd/kk/pkg/bootstrap/os/tasks.go
index 1f3467610..e8e53b11b 100644
--- a/cmd/kk/pkg/bootstrap/os/tasks.go
+++ b/cmd/kk/pkg/bootstrap/os/tasks.go
@@ -258,6 +258,9 @@ func (r *RemoveFiles) Execute(runtime connector.Runtime) error {
 	for _, file := range clusterFiles {
 		_, _ = runtime.GetRunner().SudoCmd(fmt.Sprintf("rm -rf %s", file), true)
 	}
+	// remove pki/etcd Path if it exists, otherwise it will cause the etcd reinstallation to fail if ip change
+	pkiPath := fmt.Sprintf("%s/pki/etcd", runtime.GetWorkDir())
+	_, _ = runtime.GetRunner().SudoCmd(fmt.Sprintf("rm -rf %s", pkiPath), true)
 	return nil
 }
 

From 17f70af41f9012c270054379735a296e7053b48e Mon Sep 17 00:00:00 2001
From: Xiao Liu <46879761+liangzai006@users.noreply.github.com>
Date: Sun, 4 Feb 2024 10:11:30 +0800
Subject: [PATCH 04/15] support 'cluster-info dump' command

---
 .../kubekey/v1alpha2/multicluster_types.go    | 162 ++++++++++++++
 cmd/kk/cmd/clusterinfo/clusterinfo.go         |  18 ++
 cmd/kk/cmd/clusterinfo/clusterinfo_dump.go    |  60 +++++
 cmd/kk/cmd/root.go                            |   3 +-
 cmd/kk/pkg/clusterinfo/clusterinfo_dump.go    | 121 ++++++++++
 cmd/kk/pkg/clusterinfo/dump_file.go           | 143 ++++++++++++
 cmd/kk/pkg/clusterinfo/dump_resources.go      | 210 ++++++++++++++++++
 cmd/kk/pkg/clusterinfo/dump_tar.go            |  34 +++
 cmd/kk/pkg/utils/client.go                    |  43 +++-
 go.mod                                        |  23 +-
 go.sum                                        |  67 +++++-
 11 files changed, 857 insertions(+), 27 deletions(-)
 create mode 100644 cmd/kk/apis/kubekey/v1alpha2/multicluster_types.go
 create mode 100644 cmd/kk/cmd/clusterinfo/clusterinfo.go
 create mode 100644 cmd/kk/cmd/clusterinfo/clusterinfo_dump.go
 create mode 100644 cmd/kk/pkg/clusterinfo/clusterinfo_dump.go
 create mode 100644 cmd/kk/pkg/clusterinfo/dump_file.go
 create mode 100644 cmd/kk/pkg/clusterinfo/dump_resources.go
 create mode 100644 cmd/kk/pkg/clusterinfo/dump_tar.go

diff --git a/cmd/kk/apis/kubekey/v1alpha2/multicluster_types.go b/cmd/kk/apis/kubekey/v1alpha2/multicluster_types.go
new file mode 100644
index 000000000..c6028ee3f
--- /dev/null
+++ b/cmd/kk/apis/kubekey/v1alpha2/multicluster_types.go
@@ -0,0 +1,162 @@
+package v1alpha2
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+type MultiClusterSpec struct {
+	// Join cluster as a kubefed cluster
+	JoinFederation bool `json:"joinFederation,omitempty"`
+
+	// Desired state of the cluster
+	Enable bool `json:"enable,omitempty"`
+
+	// Provider of the cluster, this field is just for description
+	Provider string `json:"provider,omitempty"`
+
+	// Connection holds info to connect to the member cluster
+	Connection Connection `json:"connection,omitempty"`
+
+	// ExternalKubeAPIEnabled export kubeapiserver to public use a lb type service if connection type is proxy
+	ExternalKubeAPIEnabled bool `json:"externalKubeAPIEnabled,omitempty"`
+}
+
+type ConnectionType string
+
+const (
+	ConnectionTypeDirect ConnectionType = "direct"
+	ConnectionTypeProxy  ConnectionType = "proxy"
+)
+
+type Connection struct {
+
+	// type defines how host cluster will connect to host cluster
+	// ConnectionTypeDirect means direct connection, this requires
+	//   kubeconfig and kubesphere apiserver endpoint provided
+	// ConnectionTypeProxy means using kubesphere proxy, no kubeconfig
+	//   or kubesphere apiserver endpoint required
+	Type ConnectionType `json:"type,omitempty"`
+
+	// KubeSphere API Server endpoint. Example: http://10.10.0.11:8080
+	// Should provide this field explicitly if connection type is direct.
+	// Will be populated by ks-apiserver if connection type is proxy.
+	KubeSphereAPIEndpoint string `json:"kubesphereAPIEndpoint,omitempty"`
+
+	// Kubernetes API Server endpoint. Example: https://10.10.0.1:6443
+	// Should provide this field explicitly if connection type is direct.
+	// Will be populated by ks-apiserver if connection type is proxy.
+	KubernetesAPIEndpoint string `json:"kubernetesAPIEndpoint,omitempty"`
+
+	// External Kubernetes API Server endpoint
+	// Will be populated by ks-apiserver if connection type is proxy and ExternalKubeAPIEnabled is true.
+	ExternalKubernetesAPIEndpoint string `json:"externalKubernetesAPIEndpoint,omitempty"`
+
+	// KubeConfig content used to connect to cluster api server
+	// Should provide this field explicitly if connection type is direct.
+	// Will be populated by ks-proxy if connection type is proxy.
+	KubeConfig []byte `json:"kubeconfig,omitempty"`
+
+	// Token used by agents of member cluster to connect to host cluster proxy.
+	// This field is populated by apiserver only if connection type is proxy.
+	Token string `json:"token,omitempty"`
+
+	// KubeAPIServerPort is the port which listens for forwarding kube-apiserver traffic
+	// Only applicable when connection type is proxy.
+	KubernetesAPIServerPort uint16 `json:"kubernetesAPIServerPort,omitempty"`
+
+	// KubeSphereAPIServerPort is the port which listens for forwarding kubesphere apigateway traffic
+	// Only applicable when connection type is proxy.
+	KubeSphereAPIServerPort uint16 `json:"kubesphereAPIServerPort,omitempty"`
+}
+type MultiClusterStatus struct {
+
+	// Represents the latest available observations of a cluster's current state.
+	Conditions []ClusterCondition `json:"conditions,omitempty"`
+
+	// GitVersion of the kubernetes cluster, this field is populated by cluster controller
+	KubernetesVersion string `json:"kubernetesVersion,omitempty"`
+
+	// GitVersion of the /kapis/version api response, this field is populated by cluster controller
+	KubeSphereVersion string `json:"kubeSphereVersion,omitempty"`
+
+	// Count of the kubernetes cluster nodes
+	// This field may not reflect the instant status of the cluster.
+	NodeCount int `json:"nodeCount,omitempty"`
+
+	// Zones are the names of availability zones in which the nodes of the cluster exist, e.g. 'us-east1-a'.
+	// +optional
+	Zones []string `json:"zones,omitempty"`
+
+	// Region is the name of the region in which all of the nodes in the cluster exist.  e.g. 'us-east1'.
+	// +optional
+	Region *string `json:"region,omitempty"`
+
+	// Configz is status of components enabled in the member cluster. This is synchronized with member cluster
+	// every amount of time, like 5 minutes.
+	// +optional
+	Configz map[string]bool `json:"configz,omitempty"`
+
+	// UID is the kube-system namespace UID of the cluster, which represents the unique ID of the cluster.
+	UID types.UID `json:"uid,omitempty"`
+}
+type ClusterConditionType string
+
+const (
+	// Cluster agent is initialized and waiting for connecting
+	ClusterInitialized ClusterConditionType = "Initialized"
+
+	// Cluster agent is available
+	ClusterAgentAvailable ClusterConditionType = "AgentAvailable"
+
+	// Cluster has been one of federated clusters
+	ClusterFederated ClusterConditionType = "Federated"
+
+	// Cluster external access ready
+	ClusterExternalAccessReady ClusterConditionType = "ExternalAccessReady"
+
+	// Cluster is all available for requests
+	ClusterReady ClusterConditionType = "Ready"
+
+	// Openpitrix runtime is created
+	ClusterOpenPitrixRuntimeReady ClusterConditionType = "OpenPitrixRuntimeReady"
+
+	// ClusterKubeConfigCertExpiresInSevenDays indicates that the cluster certificate is about to expire.
+	ClusterKubeConfigCertExpiresInSevenDays ClusterConditionType = "KubeConfigCertExpiresInSevenDays"
+)
+
+type ClusterCondition struct {
+	// Type of the condition
+	Type ClusterConditionType `json:"type"`
+	// Status of the condition, one of True, False, Unknown.
+	Status corev1.ConditionStatus `json:"status"`
+	// The last time this condition was updated.
+	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
+	// Last time the condition transitioned from one status to another.
+	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
+	// The reason for the condition's last transition.
+	Reason string `json:"reason,omitempty"`
+	// A human readable message indicating details about the transition.
+	Message string `json:"message,omitempty"`
+}
+type MultiCluster struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   MultiClusterSpec   `json:"spec,omitempty"`
+	Status MultiClusterStatus `json:"status,omitempty"`
+}
+
+type MultiClusterList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []MultiCluster `json:"items"`
+}
+
+const HostClusterLabel = "cluster-role.kubesphere.io/host"
+
+func (m MultiCluster) IsHostCluster() bool {
+	_, host := m.Labels[HostClusterLabel]
+	return host
+}
diff --git a/cmd/kk/cmd/clusterinfo/clusterinfo.go b/cmd/kk/cmd/clusterinfo/clusterinfo.go
new file mode 100644
index 000000000..c7e51db53
--- /dev/null
+++ b/cmd/kk/cmd/clusterinfo/clusterinfo.go
@@ -0,0 +1,18 @@
+package clusterinfo
+
+import (
+	"github.com/spf13/cobra"
+)
+
+// NewCmdClusterInfo creates a new clusterinfo command
+func NewCmdClusterInfo() *cobra.Command {
+
+	cmd := &cobra.Command{
+		Use:   "cluster-info",
+		Short: "display cluster information",
+	}
+
+	cmd.AddCommand(NewCmdClusterInfoDump())
+
+	return cmd
+}
diff --git a/cmd/kk/cmd/clusterinfo/clusterinfo_dump.go b/cmd/kk/cmd/clusterinfo/clusterinfo_dump.go
new file mode 100644
index 000000000..d9322d7cf
--- /dev/null
+++ b/cmd/kk/cmd/clusterinfo/clusterinfo_dump.go
@@ -0,0 +1,60 @@
+package clusterinfo
+
+import (
+	"fmt"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/cmd/util"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/clusterinfo"
+	"github.com/spf13/cobra"
+)
+
+type ClusterInfoDumpOptions struct {
+	Options clusterinfo.DumpOption
+}
+
+func NewClusterInfoDumpOptions() *ClusterInfoDumpOptions {
+	return &ClusterInfoDumpOptions{}
+}
+
+func NewCmdClusterInfoDump() *cobra.Command {
+	o := NewClusterInfoDumpOptions()
+	cmd := &cobra.Command{
+		Use:   "dump",
+		Short: "Dumping key cluster configurations and files",
+		Run: func(cmd *cobra.Command, args []string) {
+
+			util.CheckErr(o.Validate())
+			util.CheckErr(o.Run())
+
+		},
+	}
+
+	o.addFlag(cmd)
+	return cmd
+}
+
+func (o *ClusterInfoDumpOptions) Validate() error {
+	switch o.Options.Type {
+	case "yaml", "YAML", "json", "JSON":
+	default:
+		return fmt.Errorf("unsupport output content  format [%s]", o.Options.Type)
+	}
+	return nil
+}
+
+func (o *ClusterInfoDumpOptions) addFlag(cmd *cobra.Command) {
+	DefaultDumpNamespaces := []string{"kubesphere-system", "kubesphere-logging-system", "kubesphere-monitoring-system", "openpitrix-system", "kube-system", "istio-system", "kubesphere-devops-system", "porter-system"}
+	cmd.Flags().StringArrayVar(&o.Options.Namespace, "namespaces", DefaultDumpNamespaces, "Namespaces to be dumped, separated by commas.")
+	cmd.Flags().StringVar(&o.Options.KubeConfig, "kube-config", "", "Path to the kube-config file")
+	cmd.Flags().BoolVarP(&o.Options.AllNamespaces, "all-namespaces", "A", false, "dump all namespaces.")
+	cmd.Flags().StringVar(&o.Options.OutputDir, "output-dir", "", "output the dump result to the specified directory directory.")
+	cmd.Flags().StringVarP(&o.Options.Type, "output", "o", "json", "output file content format. support in json,yaml")
+	cmd.Flags().BoolVarP(&o.Options.Tar, "tar", "t", false, "build the dump result into a tar")
+	cmd.Flags().IntVar(&o.Options.Queue, "queue", 5, "dump queue size")
+	cmd.Flags().BoolVar(&o.Options.Logger, "log", false, "output the dump result to the log console")
+}
+
+func (o *ClusterInfoDumpOptions) Run() error {
+	fmt.Println("dumping cluster info...")
+	return clusterinfo.Dump(o.Options)
+
+}
diff --git a/cmd/kk/cmd/root.go b/cmd/kk/cmd/root.go
index 3ca697a9b..003354c65 100755
--- a/cmd/kk/cmd/root.go
+++ b/cmd/kk/cmd/root.go
@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/cmd/clusterinfo"
 	"os"
 	"os/exec"
 	"runtime"
@@ -120,7 +121,7 @@ func NewKubeKeyCommand(o KubeKeyOptions) *cobra.Command {
 
 	cmds.AddCommand(completion.NewCmdCompletion())
 	cmds.AddCommand(version.NewCmdVersion())
-
+	cmds.AddCommand(clusterinfo.NewCmdClusterInfo())
 	return cmds
 }
 
diff --git a/cmd/kk/pkg/clusterinfo/clusterinfo_dump.go b/cmd/kk/pkg/clusterinfo/clusterinfo_dump.go
new file mode 100644
index 000000000..7cbac9146
--- /dev/null
+++ b/cmd/kk/pkg/clusterinfo/clusterinfo_dump.go
@@ -0,0 +1,121 @@
+package clusterinfo
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils"
+	"github.com/xuri/excelize/v2"
+)
+
+type DumpOption struct {
+	Namespace     []string
+	KubeConfig    string
+	AllNamespaces bool
+	OutputDir     string
+	Type          string
+	Tar           bool
+	Queue         int
+	Logger        bool
+}
+
+func Dump(option DumpOption) error {
+	client, err := utils.NewClient(option.KubeConfig)
+	if err != nil {
+		return err
+	}
+	dump := NewDumpOption(client)
+
+	cluster, err := dump.GetMultiCluster()
+	if err != nil {
+		return err
+	}
+
+	fileChan := NewFileChan(option, excelize.NewFile())
+	queue := make(chan struct{}, option.Queue)
+	go func() {
+		err = fileChan.WriteFile()
+		if err != nil {
+			fmt.Printf("failed to write file %s", err.Error())
+			fileChan.WaitGroup.Done()
+		}
+	}()
+
+	for _, multiCluster := range cluster {
+		fileChan.WaitGroup.Add(1)
+		queue <- struct{}{}
+		go func(multiCluster v1alpha2.MultiCluster) {
+			defer func() {
+				<-queue
+			}()
+			if multiCluster.IsHostCluster() {
+				fileChan.ReadData(dump, multiCluster.Name)
+			} else {
+				clusterClient, err := utils.NewClientForCluster(multiCluster.Spec.Connection.KubeConfig)
+				if err != nil {
+					fmt.Printf("failed to create cluster %s", multiCluster.Name)
+					fileChan.WaitGroup.Done()
+					return
+				}
+				fileChan.ReadData(NewDumpOption(clusterClient), multiCluster.Name)
+			}
+		}(multiCluster)
+	}
+
+	defer func() {
+		if err := fileChan.Excel.Close(); err != nil {
+			fmt.Println(err)
+		}
+		close(fileChan.OutFileChan)
+		close(queue)
+	}()
+
+	fileChan.WaitGroup.Wait()
+
+	fileChan.Excel.DeleteSheet("Sheet1")
+	fileChan.Excel.SaveAs(fmt.Sprintf("%s/%s", option.GetOutputDir(), "cluster_dump.xlsx"))
+
+	if option.Tar {
+		err = NewTar(option).Run()
+		if err != nil {
+			fmt.Printf("failed to tar file %s", err.Error())
+			return err
+		}
+	}
+
+	return nil
+}
+
+func resourcesClassification(resources interface{}) map[string][]interface{} {
+
+	var resourcesMap []map[string]interface{}
+	if marshal, err := json.Marshal(resources); err != nil {
+		fmt.Println(err, "marshal resources error")
+		return nil
+	} else {
+		decoder := json.NewDecoder(bytes.NewReader(marshal))
+		if err = decoder.Decode(&resourcesMap); err != nil {
+			fmt.Println(err, "Decode resources error")
+			return nil
+		}
+	}
+
+	var completeMap = make(map[string][]interface{})
+	for _, m := range resourcesMap {
+		namespace, ok := m["metadata"].(map[string]interface{})["namespace"]
+		if ok {
+			completeMap[namespace.(string)] = append(completeMap[namespace.(string)], m)
+		} else {
+			completeMap[""] = append(completeMap[""], m)
+		}
+	}
+	return completeMap
+}
+
+func (c *DumpOption) GetOutputDir() string {
+	if c.OutputDir == "" {
+		return "cluster_dump"
+	}
+	return c.OutputDir
+}
diff --git a/cmd/kk/pkg/clusterinfo/dump_file.go b/cmd/kk/pkg/clusterinfo/dump_file.go
new file mode 100644
index 000000000..d1ae91d6a
--- /dev/null
+++ b/cmd/kk/pkg/clusterinfo/dump_file.go
@@ -0,0 +1,143 @@
+package clusterinfo
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/xuri/excelize/v2"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/utils/strings/slices"
+	"os"
+	"path/filepath"
+	"sigs.k8s.io/yaml"
+	"strings"
+	"sync"
+)
+
+type ChanData struct {
+	FilePath string
+	Data     interface{}
+}
+type DumpFileChan struct {
+	DumpOption
+	OutFileChan chan ChanData
+	Excel       *excelize.File
+	WaitGroup   sync.WaitGroup
+}
+
+type DumpFile interface {
+	CreateFile(string) (*os.File, error)
+	GetWireData(ChanData) []byte
+	WriteFile() error
+	SendData(ChanData, bool)
+	ReadData(*ClientSet, string)
+}
+
+func NewFileChan(option DumpOption, excel *excelize.File) *DumpFileChan {
+	return &DumpFileChan{
+		OutFileChan: make(chan ChanData),
+		WaitGroup:   sync.WaitGroup{},
+		DumpOption:  option,
+		Excel:       excel,
+	}
+}
+
+func (c *DumpFileChan) CreateFile(path string) (*os.File, error) {
+	if err := os.MkdirAll(filepath.Dir(path), 0775); err != nil {
+		return nil, err
+	}
+
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0775)
+	if err != nil {
+		return nil, err
+	}
+	return file, nil
+}
+
+func (c *DumpFileChan) GetWireData(o ChanData) []byte {
+	switch strings.ToLower(c.Type) {
+	case "yaml":
+		marshal, err := yaml.Marshal(o.Data)
+		if err != nil {
+			fmt.Printf("marshal data error %s\n", err.Error())
+			return nil
+		}
+		return marshal
+	default:
+		marshal, err := json.Marshal(o.Data)
+		if err != nil {
+			fmt.Printf("marshal data error %s\n", err.Error())
+			return nil
+		}
+		return marshal
+	}
+}
+
+func (c *DumpFileChan) WriteFile() error {
+
+	for o := range c.OutFileChan {
+		file, err := c.CreateFile(o.FilePath)
+		if err != nil {
+			fmt.Println(err, "create file error")
+			c.WaitGroup.Done()
+			return err
+		}
+		data := c.GetWireData(o)
+		if _, err = file.Write(data); err != nil {
+			fmt.Println(err, "write file error")
+			c.WaitGroup.Done()
+			return err
+		}
+		if c.Logger {
+			fmt.Println(string(data))
+		}
+	}
+	return nil
+}
+
+func (c *DumpFileChan) SendData(data ChanData, isContinue bool) {
+	if isContinue || c.AllNamespaces {
+		c.OutFileChan <- data
+	}
+}
+
+func (c *DumpFileChan) ReadData(client *ClientSet, clusterName string) {
+
+	_, err := c.Excel.NewSheet(clusterName)
+	if err != nil {
+		fmt.Println(err, "create sheet error")
+		return
+	}
+
+	index, err := c.Excel.NewStyle(&excelize.Style{
+		Font: &excelize.Font{
+			Bold: true,
+			Size: 15,
+		},
+	})
+	resources := client.GetClusterResources(corev1.NamespaceAll)
+	cellTag := []string{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"}
+	cellTagIndex := 0
+	for resourceKey, resource := range resources {
+		for kind, instance := range resource {
+			c.Excel.SetCellValue(clusterName, fmt.Sprintf("%s1", cellTag[cellTagIndex]), kind)
+			rowIndex := 2
+			for namespace, data := range instance {
+				for _, datum := range data {
+					name := datum.(map[string]interface{})["metadata"].(map[string]interface{})["name"]
+					c.Excel.SetCellValue(clusterName, fmt.Sprintf("%s%d", cellTag[cellTagIndex], rowIndex), name)
+					c.SendData(ChanData{
+						FilePath: filepath.Join(c.GetOutputDir(), clusterName, resourceKey, namespace, kind, fmt.Sprintf("%s.%s", name, strings.ToLower(c.Type))),
+						Data:     datum,
+					}, slices.Contains(c.Namespace, namespace) || namespace == "")
+					rowIndex++
+				}
+			}
+			cellTagIndex++
+		}
+	}
+
+	c.Excel.SetCellStyle(clusterName, "A1", fmt.Sprintf("%s1", cellTag[cellTagIndex-1]), index)
+	c.Excel.SetColWidth(clusterName, "A", fmt.Sprintf("%s", cellTag[cellTagIndex-1]), 30)
+	c.WaitGroup.Done()
+
+}
diff --git a/cmd/kk/pkg/clusterinfo/dump_resources.go b/cmd/kk/pkg/clusterinfo/dump_resources.go
new file mode 100644
index 000000000..7222d0eee
--- /dev/null
+++ b/cmd/kk/pkg/clusterinfo/dump_resources.go
@@ -0,0 +1,210 @@
+package clusterinfo
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2"
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+type ClientSet struct {
+	Client *kubernetes.Clientset
+}
+
+type DumpResources interface {
+	GetNodes() []corev1.Node
+	GetNamespaces() []corev1.Namespace
+	GetConfigMap(namespace string) []corev1.ConfigMap
+	GetServices(namespace string) []corev1.Service
+	GetSecrets(namespace string) []corev1.Secret
+	GetDeployment(namespace string) []appsv1.Deployment
+	GetStatefulSets(namespace string) []appsv1.StatefulSet
+	GetPods(namespace string) []corev1.Pod
+	GetPersistentVolumeClaims(namespace string) []corev1.PersistentVolumeClaim
+	GetPersistentVolumes() []corev1.PersistentVolume
+	GetStorageClasses() []interface{}
+	GetJobs(namespace string) []batchv1.Job
+	GetCronJobs(namespace string) []batchv1.CronJob
+	GetCRD() []interface{}
+	GetClusterResources(namespace string) map[string]map[string]map[string][]interface{}
+	GetMultiCluster() ([]v1alpha2.MultiCluster, error)
+}
+
+func NewDumpOption(client *kubernetes.Clientset) *ClientSet {
+	return &ClientSet{
+		Client: client,
+	}
+}
+
+func (c *ClientSet) GetClusterResources(namespace string) map[string]map[string]map[string][]interface{} {
+
+	clusterResources := map[string]map[string][]interface{}{}
+	clusterResources["nodes"] = resourcesClassification(c.GetNodes())
+	clusterResources["namespaces"] = resourcesClassification(c.GetNamespaces())
+	clusterResources["persistentvolumes"] = resourcesClassification(c.GetPersistentVolumes())
+	clusterResources["storageclasses"] = resourcesClassification(c.GetStorageClasses())
+	namespaceResources := map[string]map[string][]interface{}{}
+	namespaceResources["crds"] = resourcesClassification(c.GetCRD())
+	namespaceResources["deployments"] = resourcesClassification(c.GetDeployment(namespace))
+	namespaceResources["statefulsets"] = resourcesClassification(c.GetStatefulSets(namespace))
+	namespaceResources["pods"] = resourcesClassification(c.GetPods(namespace))
+	namespaceResources["services"] = resourcesClassification(c.GetServices(namespace))
+	namespaceResources["configmaps"] = resourcesClassification(c.GetConfigMap(namespace))
+	namespaceResources["secrets"] = resourcesClassification(c.GetSecrets(namespace))
+	namespaceResources["persistentvolumeclaims"] = resourcesClassification(c.GetPersistentVolumeClaims(namespace))
+	namespaceResources["jobs"] = resourcesClassification(c.GetJobs(namespace))
+	namespaceResources["cronjobs"] = resourcesClassification(c.GetCronJobs(namespace))
+
+	return map[string]map[string]map[string][]interface{}{"clusterResources": clusterResources, "namespaceResources": namespaceResources}
+
+}
+
+func (c *ClientSet) GetCRD() []interface{} {
+	raw, err := c.Client.CoreV1().RESTClient().Get().AbsPath("/apis/installer.kubesphere.io/v1alpha1/clusterconfigurations").DoRaw(context.Background())
+	if err != nil {
+		return nil
+	}
+	var crd map[string]interface{}
+	err = json.Unmarshal(raw, &crd)
+	if err != nil {
+		return nil
+	}
+	return crd["items"].([]interface{})
+}
+
+func (c *ClientSet) GetNodes() []corev1.Node {
+	list, err := c.Client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return nil
+	}
+
+	return list.Items
+}
+
+func (c *ClientSet) GetNamespaces() []corev1.Namespace {
+
+	list, err := c.Client.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return nil
+	}
+	return list.Items
+}
+
+func (c *ClientSet) GetConfigMap(namespace string) []corev1.ConfigMap {
+	list, err := c.Client.CoreV1().ConfigMaps(namespace).List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return nil
+	}
+	return list.Items
+}
+
+func (c *ClientSet) GetMultiCluster() ([]v1alpha2.MultiCluster, error) {
+	raw, err := c.Client.CoreV1().RESTClient().Get().AbsPath("/apis/cluster.kubesphere.io/v1alpha1/clusters").DoRaw(context.TODO())
+	if err != nil {
+		fmt.Println(err, "failed to get cluster config")
+		return nil, err
+	}
+	var cluster v1alpha2.MultiClusterList
+	err = json.Unmarshal(raw, &cluster)
+	if err != nil {
+		fmt.Println(err, "failed to unmarshal cluster config")
+		return nil, err
+	}
+
+	return cluster.Items, nil
+}
+func (c *ClientSet) GetDeployment(namespace string) []appsv1.Deployment {
+
+	list, err := c.Client.AppsV1().Deployments(namespace).List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return nil
+	}
+
+	return list.Items
+}
+
+func (c *ClientSet) GetStatefulSets(namespace string) []appsv1.StatefulSet {
+
+	list, err := c.Client.AppsV1().StatefulSets(namespace).List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return nil
+	}
+	return list.Items
+}
+
+func (c *ClientSet) GetPods(namespace string) []corev1.Pod {
+	list, err := c.Client.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return nil
+	}
+	return list.Items
+}
+
+func (c *ClientSet) GetServices(namespace string) []corev1.Service {
+	list, err := c.Client.CoreV1().Services(namespace).List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return nil
+	}
+	return list.Items
+}
+
+func (c *ClientSet) GetSecrets(namespace string) []corev1.Secret {
+	list, err := c.Client.CoreV1().Secrets(namespace).List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return nil
+	}
+	return list.Items
+}
+
+func (c *ClientSet) GetPersistentVolumeClaims(namespace string) []corev1.PersistentVolumeClaim {
+
+	list, err := c.Client.CoreV1().PersistentVolumeClaims(namespace).List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return nil
+	}
+
+	return list.Items
+}
+
+func (c *ClientSet) GetPersistentVolumes() []corev1.PersistentVolume {
+	list, err := c.Client.CoreV1().PersistentVolumes().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return nil
+	}
+	return list.Items
+}
+
+func (c *ClientSet) GetStorageClasses() []interface{} {
+
+	raw, err := c.Client.CoreV1().RESTClient().Get().AbsPath("/apis/storage.k8s.io/v1/storageclasses").DoRaw(context.Background())
+	if err != nil {
+		return nil
+	}
+	var storageClass map[string]interface{}
+	err = json.Unmarshal(raw, &storageClass)
+	if err != nil {
+		return nil
+	}
+	return storageClass["items"].([]interface{})
+}
+
+func (c *ClientSet) GetJobs(namespace string) []batchv1.Job {
+	list, err := c.Client.BatchV1().Jobs(namespace).List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		return nil
+	}
+	return list.Items
+}
+
+func (c *ClientSet) GetCronJobs(namespace string) []batchv1.CronJob {
+	list, err := c.Client.BatchV1().CronJobs(namespace).List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		return nil
+	}
+	return list.Items
+}
diff --git a/cmd/kk/pkg/clusterinfo/dump_tar.go b/cmd/kk/pkg/clusterinfo/dump_tar.go
new file mode 100644
index 000000000..fd4066404
--- /dev/null
+++ b/cmd/kk/pkg/clusterinfo/dump_tar.go
@@ -0,0 +1,34 @@
+package clusterinfo
+
+import (
+	"fmt"
+	"os/exec"
+)
+
+const tarFileName = "cluster_dump.tar"
+
+type TarOptions struct {
+	DumpOption
+}
+
+func NewTar(option DumpOption) *TarOptions {
+	return &TarOptions{
+		DumpOption: option,
+	}
+}
+
+func (t TarOptions) Run() error {
+
+	sourceDir := t.GetOutputDir()
+	cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("tar -czvf %s %s", tarFileName, sourceDir))
+	output, err := cmd.Output()
+	if err != nil {
+		return err
+	}
+
+	if t.Logger {
+		fmt.Println(string(output))
+	}
+
+	return nil
+}
diff --git a/cmd/kk/pkg/utils/client.go b/cmd/kk/pkg/utils/client.go
index 98f22cfa8..af92b6933 100644
--- a/cmd/kk/pkg/utils/client.go
+++ b/cmd/kk/pkg/utils/client.go
@@ -17,12 +17,12 @@
 package utils
 
 import (
-	"os"
-	"path/filepath"
-
 	"github.com/pkg/errors"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"os"
+	"path/filepath"
 )
 
 func NewClient(config string) (*kubernetes.Clientset, error) {
@@ -41,13 +41,8 @@ func NewClient(config string) (*kubernetes.Clientset, error) {
 	if err != nil {
 		return nil, err
 	}
-	// create the clientset
-	clientset, err := kubernetes.NewForConfig(configCluster)
-	if err != nil {
-		return nil, err
-	}
 
-	return clientset, nil
+	return newForClient(configCluster)
 }
 
 func homeDir() string {
@@ -56,3 +51,33 @@ func homeDir() string {
 	}
 	return os.Getenv("USERPROFILE")
 }
+
+func KubeConfigFormByte(data []byte) (*rest.Config, error) {
+	ClientConfig, err := clientcmd.NewClientConfigFromBytes(data)
+	if err != nil {
+		return nil, err
+	}
+	restConfig, err := ClientConfig.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+	return restConfig, nil
+}
+
+func NewClientForCluster(kubeConfig []byte) (*kubernetes.Clientset, error) {
+
+	forClientConfig, err := KubeConfigFormByte(kubeConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return newForClient(forClientConfig)
+}
+
+func newForClient(config *rest.Config) (*kubernetes.Clientset, error) {
+	client, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}
diff --git a/go.mod b/go.mod
index 0b0055c8f..5434899a8 100644
--- a/go.mod
+++ b/go.mod
@@ -33,7 +33,8 @@ require (
 	github.com/spf13/cobra v1.5.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.12.0
-	golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0
+	github.com/xuri/excelize/v2 v2.8.0
+	golang.org/x/crypto v0.12.0
 	gopkg.in/yaml.v3 v3.0.1
 	helm.sh/helm/v3 v3.9.4
 	k8s.io/api v0.25.4
@@ -96,6 +97,7 @@ require (
 	github.com/coredns/corefile-migration v1.0.17 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/daviddengcn/go-colortext v1.0.0 // indirect
 	github.com/deislabs/oras v0.9.0 // indirect
 	github.com/docker/cli v20.10.17+incompatible // indirect
 	github.com/docker/distribution v2.8.1+incompatible // indirect
@@ -180,9 +182,11 @@ require (
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/runc v1.1.4 // indirect
@@ -197,6 +201,8 @@ require (
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/richardlehane/mscfb v1.0.4 // indirect
+	github.com/richardlehane/msoleps v1.0.3 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rubenv/sql-migrate v1.1.1 // indirect
 	github.com/russross/blackfriday v1.6.0 // indirect
@@ -219,17 +225,19 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/xlab/treeprint v1.1.0 // indirect
+	github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca // indirect
+	github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
 	go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
-	golang.org/x/net v0.1.0 // indirect
+	golang.org/x/net v0.14.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 // indirect
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
-	golang.org/x/sys v0.1.0 // indirect
-	golang.org/x/term v0.1.0 // indirect
-	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/term v0.11.0 // indirect
+	golang.org/x/text v0.12.0 // indirect
 	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
@@ -243,10 +251,13 @@ require (
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/component-helpers v0.25.4 // indirect
 	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
+	k8s.io/metrics v0.25.4 // indirect
 	oras.land/oras-go v1.2.0 // indirect
 	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 	sigs.k8s.io/kustomize/api v0.12.1 // indirect
+	sigs.k8s.io/kustomize/kustomize/v4 v4.5.7 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )
diff --git a/go.sum b/go.sum
index a64458c9f..4d00eeb82 100644
--- a/go.sum
+++ b/go.sum
@@ -395,6 +395,8 @@ github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7h
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/daviddengcn/go-colortext v1.0.0 h1:ANqDyC0ys6qCSvuEK7l3g5RaehL/Xck9EX8ATG8oKsE=
+github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c=
 github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4=
 github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo=
 github.com/deislabs/oras v0.9.0 h1:R6PRN3bTruUjHcGKgdteurzbpsCxwf3XJCLsxLFyBuU=
@@ -600,6 +602,11 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho=
+github.com/golangplus/bytes v1.0.0/go.mod h1:AdRaCFwmc/00ZzELMWb01soso6W1R/++O1XL80yAn+A=
+github.com/golangplus/fmt v1.0.0/go.mod h1:zpM0OfbMCjPtd2qkTD/jX2MgiFCqklhSUFyDW44gVQE=
+github.com/golangplus/testing v1.0.0 h1:+ZeeiKZENNOMkTTELoSySazi+XaEhVO0mb+eanrSEUQ=
+github.com/golangplus/testing v1.0.0/go.mod h1:ZDreixUV3YzhoVraIDyOzHrr76p6NUh6k/pPg/Q3gYA=
 github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -943,6 +950,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/modood/table v0.0.0-20220527013332-8d47e76dad33 h1:T5IbS9C1G2zeHb6eBy6OfIvj5tfQB23kGFpewCJuGDg=
 github.com/modood/table v0.0.0-20220527013332-8d47e76dad33/go.mod h1:41qyXVI5QH9/ObyPj27CGCVau5v/njfc3Gjj7yzr0HQ=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
@@ -953,6 +962,7 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -1096,6 +1106,11 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM=
+github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
+github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
+github.com/richardlehane/msoleps v1.0.3 h1:aznSZzrwYRl3rLKRT3gUk9am7T/mLNSnJINvN0AQoVM=
+github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
 github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo=
 github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
@@ -1255,12 +1270,19 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
 github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk=
 github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca h1:uvPMDVyP7PXMMioYdyPH+0O+Ta/UO1WFfNYMO3Wz0eg=
+github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
+github.com/xuri/excelize/v2 v2.8.0 h1:Vd4Qy809fupgp1v7X+nCS/MioeQmYVVzi495UCTqB7U=
+github.com/xuri/excelize/v2 v2.8.0/go.mod h1:6iA2edBTKxKbZAa7X5bDhcCg51xdOn1Ar5sfoXRGrQg=
+github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a h1:Mw2VNrNNNjDtw68VsEj2+st+oCSn4Uz7vZw6TbhcV1o=
+github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 h1:p7OofyZ509h8DmPLh8Hn+EIIZm/xYhdZHJ9GnXHdr6U=
 github.com/yvasiyarov/gorelic v0.0.7 h1:4DTF1WOM2ZZS/xMOkTFBOcb6XiHu/PKn3rVo6dbewQE=
 github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 h1:AsFN8kXcCVkUFHyuzp1FtYbzp1nCO/H6+1uPSGEyPzM=
@@ -1356,8 +1378,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 h1:a5Yg6ylndHHYJqIPrdq0AhvR6KTvDTAvgBtaidhEevY=
-golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
+golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1370,6 +1392,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo=
+golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1393,6 +1417,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1459,9 +1485,12 @@ golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
-golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
-golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
+golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1498,8 +1527,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1622,17 +1652,22 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
-golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
+golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1642,8 +1677,10 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
-golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
+golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1722,6 +1759,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -2025,6 +2064,8 @@ k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMY
 k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI=
 k8s.io/component-base v0.25.4 h1:n1bjg9Yt+G1C0WnIDJmg2fo6wbEU1UGMRiQSjmj7hNQ=
 k8s.io/component-base v0.25.4/go.mod h1:nnZJU8OP13PJEm6/p5V2ztgX2oyteIaAGKGMYb2L2cY=
+k8s.io/component-helpers v0.25.4 h1:FIwp5ZzjaA7p0lekHQM28E7qTd9Aetx5O9e7r4n2Kqo=
+k8s.io/component-helpers v0.25.4/go.mod h1:X4KJ8SsJ/onWcDQkRhcE2WRG/iNMufCl7RsNSYtguJg=
 k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM=
 k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
 k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
@@ -2049,6 +2090,8 @@ k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHU
 k8s.io/kubectl v0.25.4 h1:O3OA1z4V1ZyvxCvScjq0pxAP7ABgznr8UvnVObgI6Dc=
 k8s.io/kubectl v0.25.4/go.mod h1:CKMrQ67Bn2YCP26tZStPQGq62zr9pvzEf65A0navm8k=
 k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
+k8s.io/metrics v0.25.4 h1:Kq2vLaeKkksyYCuvEjg5kJbTb/BAawUgci3xasfL+nA=
+k8s.io/metrics v0.25.4/go.mod h1:cFxN3gbdb0nld4IGHHM51qKHUCcXvzkKh3z1g2YriL8=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
@@ -2075,6 +2118,8 @@ sigs.k8s.io/kind v0.14.0 h1:cNmI3jGBvp7UegEGbC5we8plDtCUmaNRL+bod7JoSCE=
 sigs.k8s.io/kind v0.14.0/go.mod h1:UrFRPHG+2a5j0Q7qiR4gtJ4rEyn8TuMQwuOPf+m4oHg=
 sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM=
 sigs.k8s.io/kustomize/api v0.12.1/go.mod h1:y3JUhimkZkR6sbLNwfJHxvo1TCLwuwm14sCYnkH6S1s=
+sigs.k8s.io/kustomize/kustomize/v4 v4.5.7 h1:cDW6AVMl6t/SLuQaezMET8hgnadZGIAr8tUrxFVOrpg=
+sigs.k8s.io/kustomize/kustomize/v4 v4.5.7/go.mod h1:VSNKEH9D9d9bLiWEGbS6Xbg/Ih0tgQalmPvntzRxZ/Q=
 sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk=
 sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ85/nOXac4=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=

From 877c9e47b367d2b39b4c2aa5e93b0c0ae55aca24 Mon Sep 17 00:00:00 2001
From: Xiao Liu <46879761+liangzai006@users.noreply.github.com>
Date: Sun, 4 Feb 2024 10:17:50 +0800
Subject: [PATCH 05/15] support 'cluster-info dump' command

---
 go.mod |  5 -----
 go.sum | 14 --------------
 2 files changed, 19 deletions(-)

diff --git a/go.mod b/go.mod
index 5434899a8..fab48a89f 100644
--- a/go.mod
+++ b/go.mod
@@ -97,7 +97,6 @@ require (
 	github.com/coredns/corefile-migration v1.0.17 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/daviddengcn/go-colortext v1.0.0 // indirect
 	github.com/deislabs/oras v0.9.0 // indirect
 	github.com/docker/cli v20.10.17+incompatible // indirect
 	github.com/docker/distribution v2.8.1+incompatible // indirect
@@ -186,7 +185,6 @@ require (
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/runc v1.1.4 // indirect
@@ -251,13 +249,10 @@ require (
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/component-helpers v0.25.4 // indirect
 	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
-	k8s.io/metrics v0.25.4 // indirect
 	oras.land/oras-go v1.2.0 // indirect
 	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 	sigs.k8s.io/kustomize/api v0.12.1 // indirect
-	sigs.k8s.io/kustomize/kustomize/v4 v4.5.7 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )
diff --git a/go.sum b/go.sum
index 4d00eeb82..e15f0f228 100644
--- a/go.sum
+++ b/go.sum
@@ -395,8 +395,6 @@ github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7h
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/daviddengcn/go-colortext v1.0.0 h1:ANqDyC0ys6qCSvuEK7l3g5RaehL/Xck9EX8ATG8oKsE=
-github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c=
 github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4=
 github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo=
 github.com/deislabs/oras v0.9.0 h1:R6PRN3bTruUjHcGKgdteurzbpsCxwf3XJCLsxLFyBuU=
@@ -602,11 +600,6 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho=
-github.com/golangplus/bytes v1.0.0/go.mod h1:AdRaCFwmc/00ZzELMWb01soso6W1R/++O1XL80yAn+A=
-github.com/golangplus/fmt v1.0.0/go.mod h1:zpM0OfbMCjPtd2qkTD/jX2MgiFCqklhSUFyDW44gVQE=
-github.com/golangplus/testing v1.0.0 h1:+ZeeiKZENNOMkTTELoSySazi+XaEhVO0mb+eanrSEUQ=
-github.com/golangplus/testing v1.0.0/go.mod h1:ZDreixUV3YzhoVraIDyOzHrr76p6NUh6k/pPg/Q3gYA=
 github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -962,7 +955,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -2064,8 +2056,6 @@ k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMY
 k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI=
 k8s.io/component-base v0.25.4 h1:n1bjg9Yt+G1C0WnIDJmg2fo6wbEU1UGMRiQSjmj7hNQ=
 k8s.io/component-base v0.25.4/go.mod h1:nnZJU8OP13PJEm6/p5V2ztgX2oyteIaAGKGMYb2L2cY=
-k8s.io/component-helpers v0.25.4 h1:FIwp5ZzjaA7p0lekHQM28E7qTd9Aetx5O9e7r4n2Kqo=
-k8s.io/component-helpers v0.25.4/go.mod h1:X4KJ8SsJ/onWcDQkRhcE2WRG/iNMufCl7RsNSYtguJg=
 k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM=
 k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
 k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
@@ -2090,8 +2080,6 @@ k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHU
 k8s.io/kubectl v0.25.4 h1:O3OA1z4V1ZyvxCvScjq0pxAP7ABgznr8UvnVObgI6Dc=
 k8s.io/kubectl v0.25.4/go.mod h1:CKMrQ67Bn2YCP26tZStPQGq62zr9pvzEf65A0navm8k=
 k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
-k8s.io/metrics v0.25.4 h1:Kq2vLaeKkksyYCuvEjg5kJbTb/BAawUgci3xasfL+nA=
-k8s.io/metrics v0.25.4/go.mod h1:cFxN3gbdb0nld4IGHHM51qKHUCcXvzkKh3z1g2YriL8=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
@@ -2118,8 +2106,6 @@ sigs.k8s.io/kind v0.14.0 h1:cNmI3jGBvp7UegEGbC5we8plDtCUmaNRL+bod7JoSCE=
 sigs.k8s.io/kind v0.14.0/go.mod h1:UrFRPHG+2a5j0Q7qiR4gtJ4rEyn8TuMQwuOPf+m4oHg=
 sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM=
 sigs.k8s.io/kustomize/api v0.12.1/go.mod h1:y3JUhimkZkR6sbLNwfJHxvo1TCLwuwm14sCYnkH6S1s=
-sigs.k8s.io/kustomize/kustomize/v4 v4.5.7 h1:cDW6AVMl6t/SLuQaezMET8hgnadZGIAr8tUrxFVOrpg=
-sigs.k8s.io/kustomize/kustomize/v4 v4.5.7/go.mod h1:VSNKEH9D9d9bLiWEGbS6Xbg/Ih0tgQalmPvntzRxZ/Q=
 sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk=
 sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ85/nOXac4=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=

From f47d36a2d86a3fdd5c6896e0846622ff2dbabf69 Mon Sep 17 00:00:00 2001
From: wenwutang <1218040628@qq.com>
Date: Mon, 5 Feb 2024 15:55:35 +0800
Subject: [PATCH 06/15] add flag skip-dependency-check when upgrade cluster

---
 cmd/kk/cmd/upgrade/upgrade.go           | 35 ++++++++++++++-----------
 cmd/kk/pkg/bootstrap/precheck/module.go | 29 ++++++++++++++------
 cmd/kk/pkg/common/kube_runtime.go       |  1 +
 cmd/kk/pkg/pipelines/upgrade_cluster.go |  2 +-
 4 files changed, 42 insertions(+), 25 deletions(-)

diff --git a/cmd/kk/cmd/upgrade/upgrade.go b/cmd/kk/cmd/upgrade/upgrade.go
index 4e9f60381..22382d2e7 100644
--- a/cmd/kk/cmd/upgrade/upgrade.go
+++ b/cmd/kk/cmd/upgrade/upgrade.go
@@ -31,14 +31,15 @@ import (
 )
 
 type UpgradeOptions struct {
-	CommonOptions    *options.CommonOptions
-	ClusterCfgFile   string
-	Kubernetes       string
-	EnableKubeSphere bool
-	KubeSphere       string
-	SkipPullImages   bool
-	DownloadCmd      string
-	Artifact         string
+	CommonOptions       *options.CommonOptions
+	ClusterCfgFile      string
+	Kubernetes          string
+	EnableKubeSphere    bool
+	KubeSphere          string
+	SkipPullImages      bool
+	SkipDependencyCheck bool
+	DownloadCmd         string
+	Artifact            string
 }
 
 func NewUpgradeOptions() *UpgradeOptions {
@@ -80,14 +81,15 @@ func (o *UpgradeOptions) Complete(cmd *cobra.Command, args []string) error {
 
 func (o *UpgradeOptions) Run() error {
 	arg := common.Argument{
-		FilePath:          o.ClusterCfgFile,
-		KubernetesVersion: o.Kubernetes,
-		KsEnable:          o.EnableKubeSphere,
-		KsVersion:         o.KubeSphere,
-		SkipPullImages:    o.SkipPullImages,
-		Debug:             o.CommonOptions.Verbose,
-		SkipConfirmCheck:  o.CommonOptions.SkipConfirmCheck,
-		Artifact:          o.Artifact,
+		FilePath:            o.ClusterCfgFile,
+		KubernetesVersion:   o.Kubernetes,
+		KsEnable:            o.EnableKubeSphere,
+		KsVersion:           o.KubeSphere,
+		SkipPullImages:      o.SkipPullImages,
+		Debug:               o.CommonOptions.Verbose,
+		SkipConfirmCheck:    o.CommonOptions.SkipConfirmCheck,
+		Artifact:            o.Artifact,
+		SkipDependencyCheck: o.SkipDependencyCheck,
 	}
 	return pipelines.UpgradeCluster(arg, o.DownloadCmd)
 }
@@ -100,6 +102,7 @@ func (o *UpgradeOptions) AddFlags(cmd *cobra.Command) {
 	cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s",
 		`The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`)
 	cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact")
+	cmd.Flags().BoolVarP(&o.SkipDependencyCheck, "skip-dependency-check", "", false, "Skip kubernetes and kubesphere dependency version check")
 }
 
 func completionSetting(cmd *cobra.Command) (err error) {
diff --git a/cmd/kk/pkg/bootstrap/precheck/module.go b/cmd/kk/pkg/bootstrap/precheck/module.go
index df6583cdd..d38a515d7 100644
--- a/cmd/kk/pkg/bootstrap/precheck/module.go
+++ b/cmd/kk/pkg/bootstrap/precheck/module.go
@@ -88,6 +88,7 @@ func (n *NodePreCheckModule) Init() {
 
 type ClusterPreCheckModule struct {
 	common.KubeModule
+	SkipDependencyCheck bool
 }
 
 func (c *ClusterPreCheckModule) Init() {
@@ -159,13 +160,25 @@ func (c *ClusterPreCheckModule) Init() {
 		Parallel: true,
 	}
 
-	c.Tasks = []task.Interface{
-		getKubeConfig,
-		getAllNodesK8sVersion,
-		calculateMinK8sVersion,
-		checkDesiredK8sVersion,
-		ksVersionCheck,
-		dependencyCheck,
-		getKubernetesNodesStatus,
+	if !c.SkipDependencyCheck {
+		c.Tasks = []task.Interface{
+			getKubeConfig,
+			getAllNodesK8sVersion,
+			calculateMinK8sVersion,
+			checkDesiredK8sVersion,
+			ksVersionCheck,
+			dependencyCheck,
+			getKubernetesNodesStatus,
+		}
+	} else {
+		c.Tasks = []task.Interface{
+			getKubeConfig,
+			getAllNodesK8sVersion,
+			calculateMinK8sVersion,
+			checkDesiredK8sVersion,
+			ksVersionCheck,
+			getKubernetesNodesStatus,
+		}
 	}
+
 }
diff --git a/cmd/kk/pkg/common/kube_runtime.go b/cmd/kk/pkg/common/kube_runtime.go
index 970e924e9..e983fe735 100644
--- a/cmd/kk/pkg/common/kube_runtime.go
+++ b/cmd/kk/pkg/common/kube_runtime.go
@@ -39,6 +39,7 @@ type Argument struct {
 	IgnoreErr           bool
 	SkipPullImages      bool
 	SkipPushImages      bool
+	SkipDependencyCheck bool
 	SecurityEnhancement bool
 	DeployLocalStorage  *bool
 	DownloadCommand     func(path, url string) string
diff --git a/cmd/kk/pkg/pipelines/upgrade_cluster.go b/cmd/kk/pkg/pipelines/upgrade_cluster.go
index 8aafd4f0f..918f5303c 100644
--- a/cmd/kk/pkg/pipelines/upgrade_cluster.go
+++ b/cmd/kk/pkg/pipelines/upgrade_cluster.go
@@ -40,7 +40,7 @@ func NewUpgradeClusterPipeline(runtime *common.KubeRuntime) error {
 	m := []module.Module{
 		&precheck.GreetingsModule{},
 		&precheck.NodePreCheckModule{},
-		&precheck.ClusterPreCheckModule{},
+		&precheck.ClusterPreCheckModule{SkipDependencyCheck: runtime.Arg.SkipDependencyCheck},
 		&confirm.UpgradeConfirmModule{Skip: runtime.Arg.SkipConfirmCheck},
 		&artifact.UnArchiveModule{Skip: noArtifact},
 		&kubernetes.SetUpgradePlanModule{Step: kubernetes.ToV121},

From bbfd7481ad26885be2fc61e1721b0f38e9b0588f Mon Sep 17 00:00:00 2001
From: "allcontributors[bot]"
 <46447321+allcontributors[bot]@users.noreply.github.com>
Date: Tue, 13 Feb 2024 11:12:09 +0000
Subject: [PATCH 07/15] update README.md [skip ci]

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index fac0829d5..3550f7b7c 100644
--- a/README.md
+++ b/README.md
@@ -397,6 +397,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://dashen.tech"><img src="https://avatars.githubusercontent.com/u/15921519?v=4?s=100" width="100px;" alt="cui fliter"/><br /><sub><b>cui fliter</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=cuishuang" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/liuxu623"><img src="https://avatars.githubusercontent.com/u/9653438?v=4?s=100" width="100px;" alt="刘旭"/><br /><sub><b>刘旭</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=liuxu623" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/yzxiu"><img src="https://avatars.githubusercontent.com/u/13790023?v=4?s=100" width="100px;" alt="yuyu"/><br /><sub><b>yuyu</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=yzxiu" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/chilianyi"><img src="https://avatars.githubusercontent.com/u/5917832?v=4?s=100" width="100px;" alt="chilianyi"/><br /><sub><b>chilianyi</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=chilianyi" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

From 8295a7e5ec76264086d76b76b1d5020699f5975e Mon Sep 17 00:00:00 2001
From: "allcontributors[bot]"
 <46447321+allcontributors[bot]@users.noreply.github.com>
Date: Tue, 13 Feb 2024 11:12:10 +0000
Subject: [PATCH 08/15] update CONTRIBUTORS.md [skip ci]

---
 CONTRIBUTORS.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
index 5b0ea4e1e..dc5212d9a 100644
--- a/CONTRIBUTORS.md
+++ b/CONTRIBUTORS.md
@@ -113,6 +113,7 @@ Contributions of any kind are welcome! Thanks goes to these wonderful contributo
       <td align="center" valign="top" width="14.28%"><a href="https://dashen.tech"><img src="https://avatars.githubusercontent.com/u/15921519?v=4?s=100" width="100px;" alt="cui fliter"/><br /><sub><b>cui fliter</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=cuishuang" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/liuxu623"><img src="https://avatars.githubusercontent.com/u/9653438?v=4?s=100" width="100px;" alt="刘旭"/><br /><sub><b>刘旭</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=liuxu623" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/yzxiu"><img src="https://avatars.githubusercontent.com/u/13790023?v=4?s=100" width="100px;" alt="yuyu"/><br /><sub><b>yuyu</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=yzxiu" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/chilianyi"><img src="https://avatars.githubusercontent.com/u/5917832?v=4?s=100" width="100px;" alt="chilianyi"/><br /><sub><b>chilianyi</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=chilianyi" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

From 5d78dd327d05d432fa4d44eadbf8dca4d209342f Mon Sep 17 00:00:00 2001
From: "allcontributors[bot]"
 <46447321+allcontributors[bot]@users.noreply.github.com>
Date: Tue, 13 Feb 2024 11:12:11 +0000
Subject: [PATCH 09/15] update README_zh-CN.md [skip ci]

---
 README_zh-CN.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README_zh-CN.md b/README_zh-CN.md
index 4e16a4604..e562317b0 100644
--- a/README_zh-CN.md
+++ b/README_zh-CN.md
@@ -414,6 +414,7 @@ kubectl completion bash >/etc/bash_completion.d/kubectl
       <td align="center" valign="top" width="14.28%"><a href="https://dashen.tech"><img src="https://avatars.githubusercontent.com/u/15921519?v=4?s=100" width="100px;" alt="cui fliter"/><br /><sub><b>cui fliter</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=cuishuang" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/liuxu623"><img src="https://avatars.githubusercontent.com/u/9653438?v=4?s=100" width="100px;" alt="刘旭"/><br /><sub><b>刘旭</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=liuxu623" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/yzxiu"><img src="https://avatars.githubusercontent.com/u/13790023?v=4?s=100" width="100px;" alt="yuyu"/><br /><sub><b>yuyu</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=yzxiu" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/chilianyi"><img src="https://avatars.githubusercontent.com/u/5917832?v=4?s=100" width="100px;" alt="chilianyi"/><br /><sub><b>chilianyi</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=chilianyi" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

From 75ce547fca2850dc50d5f450803bb63692698ca6 Mon Sep 17 00:00:00 2001
From: "allcontributors[bot]"
 <46447321+allcontributors[bot]@users.noreply.github.com>
Date: Tue, 13 Feb 2024 11:12:12 +0000
Subject: [PATCH 10/15] update .all-contributorsrc [skip ci]

---
 .all-contributorsrc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.all-contributorsrc b/.all-contributorsrc
index b1ecabf5e..725ad7a3b 100644
--- a/.all-contributorsrc
+++ b/.all-contributorsrc
@@ -769,6 +769,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "chilianyi",
+      "name": "chilianyi",
+      "avatar_url": "https://avatars.githubusercontent.com/u/5917832?v=4",
+      "profile": "https://github.com/chilianyi",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

From 93b169d3a44c9bdd168662fba3ff2ebc61b918d7 Mon Sep 17 00:00:00 2001
From: "allcontributors[bot]"
 <46447321+allcontributors[bot]@users.noreply.github.com>
Date: Tue, 13 Feb 2024 11:16:46 +0000
Subject: [PATCH 11/15] update README.md [skip ci]

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index 3550f7b7c..3a2c29d06 100644
--- a/README.md
+++ b/README.md
@@ -399,6 +399,9 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/yzxiu"><img src="https://avatars.githubusercontent.com/u/13790023?v=4?s=100" width="100px;" alt="yuyu"/><br /><sub><b>yuyu</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=yzxiu" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/chilianyi"><img src="https://avatars.githubusercontent.com/u/5917832?v=4?s=100" width="100px;" alt="chilianyi"/><br /><sub><b>chilianyi</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=chilianyi" title="Code">💻</a></td>
     </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/xrwang8"><img src="https://avatars.githubusercontent.com/u/68765051?v=4?s=100" width="100px;" alt="Ronald Fletcher"/><br /><sub><b>Ronald Fletcher</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=xrwang8" title="Code">💻</a></td>
+    </tr>
   </tbody>
 </table>
 

From e95e55e2294e1482b94ef3e32be06d8792cbe4cf Mon Sep 17 00:00:00 2001
From: "allcontributors[bot]"
 <46447321+allcontributors[bot]@users.noreply.github.com>
Date: Tue, 13 Feb 2024 11:16:47 +0000
Subject: [PATCH 12/15] update CONTRIBUTORS.md [skip ci]

---
 CONTRIBUTORS.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
index dc5212d9a..1a75af7e8 100644
--- a/CONTRIBUTORS.md
+++ b/CONTRIBUTORS.md
@@ -115,6 +115,9 @@ Contributions of any kind are welcome! Thanks goes to these wonderful contributo
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/yzxiu"><img src="https://avatars.githubusercontent.com/u/13790023?v=4?s=100" width="100px;" alt="yuyu"/><br /><sub><b>yuyu</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=yzxiu" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/chilianyi"><img src="https://avatars.githubusercontent.com/u/5917832?v=4?s=100" width="100px;" alt="chilianyi"/><br /><sub><b>chilianyi</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=chilianyi" title="Code">💻</a></td>
     </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/xrwang8"><img src="https://avatars.githubusercontent.com/u/68765051?v=4?s=100" width="100px;" alt="Ronald Fletcher"/><br /><sub><b>Ronald Fletcher</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=xrwang8" title="Code">💻</a></td>
+    </tr>
   </tbody>
 </table>
 

From b584c070cd4994828408c50784a74975ee74b088 Mon Sep 17 00:00:00 2001
From: "allcontributors[bot]"
 <46447321+allcontributors[bot]@users.noreply.github.com>
Date: Tue, 13 Feb 2024 11:16:48 +0000
Subject: [PATCH 13/15] update README_zh-CN.md [skip ci]

---
 README_zh-CN.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README_zh-CN.md b/README_zh-CN.md
index e562317b0..62c4844f2 100644
--- a/README_zh-CN.md
+++ b/README_zh-CN.md
@@ -416,6 +416,9 @@ kubectl completion bash >/etc/bash_completion.d/kubectl
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/yzxiu"><img src="https://avatars.githubusercontent.com/u/13790023?v=4?s=100" width="100px;" alt="yuyu"/><br /><sub><b>yuyu</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=yzxiu" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/chilianyi"><img src="https://avatars.githubusercontent.com/u/5917832?v=4?s=100" width="100px;" alt="chilianyi"/><br /><sub><b>chilianyi</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=chilianyi" title="Code">💻</a></td>
     </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/xrwang8"><img src="https://avatars.githubusercontent.com/u/68765051?v=4?s=100" width="100px;" alt="Ronald Fletcher"/><br /><sub><b>Ronald Fletcher</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=xrwang8" title="Code">💻</a></td>
+    </tr>
   </tbody>
 </table>
 

From 579326186a19179be77249a2ad3db82971a24a30 Mon Sep 17 00:00:00 2001
From: "allcontributors[bot]"
 <46447321+allcontributors[bot]@users.noreply.github.com>
Date: Tue, 13 Feb 2024 11:16:49 +0000
Subject: [PATCH 14/15] update .all-contributorsrc [skip ci]

---
 .all-contributorsrc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.all-contributorsrc b/.all-contributorsrc
index 725ad7a3b..ea9bf6fe9 100644
--- a/.all-contributorsrc
+++ b/.all-contributorsrc
@@ -778,6 +778,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "xrwang8",
+      "name": "Ronald Fletcher",
+      "avatar_url": "https://avatars.githubusercontent.com/u/68765051?v=4",
+      "profile": "https://github.com/xrwang8",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

From 4213761bda1603762b367d82c82c3ca5f23c6a64 Mon Sep 17 00:00:00 2001
From: pixiake <guofeng@yunify.com>
Date: Wed, 21 Feb 2024 16:06:09 +0800
Subject: [PATCH 15/15] update default docker version to 24.0.9

Signed-off-by: pixiake <guofeng@yunify.com>
---
 cmd/kk/apis/kubekey/v1alpha2/default.go |  6 +--
 version/components.json                 | 54 ++-----------------------
 2 files changed, 7 insertions(+), 53 deletions(-)

diff --git a/cmd/kk/apis/kubekey/v1alpha2/default.go b/cmd/kk/apis/kubekey/v1alpha2/default.go
index f979e5bef..b29348836 100644
--- a/cmd/kk/apis/kubekey/v1alpha2/default.go
+++ b/cmd/kk/apis/kubekey/v1alpha2/default.go
@@ -42,8 +42,8 @@ const (
 	DefaultSSHTimeout              = 30
 	DefaultEtcdVersion             = "v3.5.6"
 	DefaultEtcdPort                = "2379"
-	DefaultDockerVersion           = "24.0.6"
-	DefaultCriDockerdVersion        = "0.3.9"
+	DefaultDockerVersion           = "24.0.9"
+	DefaultCriDockerdVersion       = "0.3.9"
 	DefaultContainerdVersion       = "1.7.12"
 	DefaultRuncVersion             = "v1.1.11"
 	DefaultCrictlVersion           = "v1.29.0"
@@ -318,7 +318,7 @@ func SetDefaultClusterCfg(cfg *ClusterSpec) Kubernetes {
 	if cfg.Kubernetes.ContainerRuntimeEndpoint == "" {
 		switch cfg.Kubernetes.ContainerManager {
 		case Docker:
-			if kubernetes.IsAtLeastV124(cfg.Kubernetes.Version){
+			if kubernetes.IsAtLeastV124(cfg.Kubernetes.Version) {
 				cfg.Kubernetes.ContainerRuntimeEndpoint = DefaultCriDockerdEndpoint
 			} else {
 				cfg.Kubernetes.ContainerRuntimeEndpoint = ""
diff --git a/version/components.json b/version/components.json
index a95d94024..ff58f9af3 100644
--- a/version/components.json
+++ b/version/components.json
@@ -969,58 +969,12 @@
     },
     "docker": {
         "amd64": {
-            "20.10.2": "97017e32a8ecbdd1826bb3c7b1424303ee0dea3f900d33591b1df5e394ed4eed",
-            "20.10.3": "47065a47f0692cd5af03073c7386fe090d9ef5ac88a7d8455a884d8e15809be5",
-            "20.10.4": "6ec28b6a251e093f5cf32569c4bfce4821eda02923b33c060694e6ca2c851daa",
-            "20.10.5": "3f18edc66e1faae607d428349e77f9800bdea554528521f0f6c49fc3f1de6abf",
-            "20.10.6": "e3b6c3b11518281a51fb0eee73138482b83041e908f01adf8abd3a24b34ea21e",
-            "20.10.7": "34ad50146fce29b28e5115a1e8510dd5232459c9a4a9f28f65909f92cca314d9",
-            "20.10.8": "7ea11ecb100fdc085dbfd9ab1ff380e7f99733c890ed815510a5952e5d6dd7e0",
-            "20.10.9": "caf74e54b58c0b38bb4d96c8f87665f29b684371c9a325562a3904b8c389995e",
-            "20.10.10": "1719446f99cd56e87d0c67019996af4ea859f11891bfd89de2252d6c916ccaaa",
-            "20.10.11": "dd6ff72df1edfd61ae55feaa4aadb88634161f0aa06dbaaf291d1be594099ff3",
-            "20.10.12": "ee9b5be14e54bf92f48c82c2e6a83fbdd1c5329e8f247525a9ed2fe90d9f89a5",
-            "20.10.13": "39edf7c8d773939ff5e4d318ae565691a9c7e754ed768e172757e58898fb7079",
-            "20.10.14": "7ca4aeeed86619909ae584ce3405da3766d495f98904ffbd9d859add26b83af5",
-            "20.10.15": "9ccfc39305ae1d8882d18c9c431544fca82913d6df717409ac2244ac58c4f070",
-            "20.10.16": "b43ac6c4d2f0b64e445c6564860e4fccd6331f4a61815a60642c7748b53c59ff",
-            "20.10.17": "969210917b5548621a2b541caf00f86cc6963c6cf0fb13265b9731c3b98974d9",
-            "20.10.18": "0629b063fa3aa5660f3fb96f67edb0e20e92d5050b82403f95faf1c142177401",
-            "20.10.19": "ddcd732baaa03958cc8f326a5dca09bcd8f348bb7d2737aaf67bbdd7d80302d1",
-            "20.10.20": "a303cee9125c89abbbb6c4f044b3e2c01c7895e373b90d8de16a7ed25bb2530a",
-            "20.10.21": "2582bed8772b283bda9d4565c0af76ee653c93d93dc6b8d0aad795d731a1bb81",
-            "20.10.22": "945c3a3ddcb79ee7307496c2f39eb3d8372466e8654e63d60bbb462e4a3c1427",
-            "20.10.23": "0ee39f72cc434137d294c14d30897826bad6e24979e421f51a252769ad37e6d1",
-            "23.0.0": "6a03bbda96845b7451be2f6aba69c3816c60a97de318e83fd1b39d1be262d8af",
-            "23.0.1": "ec8a71e79125d3ca76f7cc295f35eea225f4450e0ffe0775f103e2952ff580f6",
-            "24.0.6": "99792dec613df93169a118b05312a722a63604b868e4c941b1b436abcf3bb70f"
+            "24.0.6": "99792dec613df93169a118b05312a722a63604b868e4c941b1b436abcf3bb70f",
+            "24.0.9": "692ecfc28333485d184f628b74c25b2894cee9495a51a5418ba60ef95bf733ca"
         },
         "arm64": {
-            "20.10.2": "9ea59f249ae92bbaa9831a22f2affa2edc9e824f9daaba831ca51d6d22ef2df5",
-            "20.10.3": "4dcd105f721297f314bb53622e67dd981a743d72f4b2bfe4f42a8790e0892c82",
-            "20.10.4": "bd9fb2f770eb508b3273237c5604266b6cd93789d048069bd6b16efbf8919fca",
-            "20.10.5": "83157b92d7469117c2720fc44074749e080b3c510ae35b8a57c66a016cf07dd5",
-            "20.10.6": "998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910",
-            "20.10.7": "be8c9a5a06ebec8fb1d36e867cd00fb5777746a9812a0cae2966778ff899c525",
-            "20.10.8": "4eb9d5e2adf718cd7ee59f6951715f3113c9c4ee49c75c9efb9747f2c3457b2b",
-            "20.10.9": "0259f8b6572f02cf0dafd7388ca0e4adfdbbfaba81cfb1b7443e89fccbed22c7",
-            "20.10.10": "8db47cdcd7ac6e082c9ce83347d8fb99eaa01e04b0c8d94851e8d58f350a3633",
-            "20.10.11": "87a4219c54552797ffd38790b72832372a90eceb7c8e451c36a682093d57dae6",
-            "20.10.12": "e1f964e9a7a824bbe2164560c0eb335fab9cc7ee9eb90da36c250c073459cf17",
-            "20.10.13": "debed306ed9a4e70dcbcb228a0b3898f9730099e324f34bb0e76abbaddf7a6a7",
-            "20.10.14": "ea971edc1179088bfd25edd04a0c12848143d15cb8202ebb93a6a08973464fd0",
-            "20.10.15": "46102273fab8d6b8a7cf248a928ebaa4bee43114001c593b0d07092a34a439e1",
-            "20.10.16": "2f35d8d422b63a59279084c159c9092b63b6d974a7fcd868167aee4cc5f79f3b",
-            "20.10.17": "249244024b507a6599084522cc73e73993349d13264505b387593f2b2ed603e6",
-            "20.10.18": "aa2b2da571fb9160df87fd5a831f203fb97655e35fb9c4e8d46e72078ae16acf",
-            "20.10.19": "a04414b3fcf537f0cff17cf01e2b7cb3e39013c10d12e7959547f11aaf71f63c",
-            "20.10.20": "ef69a2a8ddb87026a8b19e240b2ae3087764b7285860df7faee24e04024f2eb7",
-            "20.10.21": "b4ceb6151d4dd1bfc7557f5fe0317e29cfcac91f798c34fae7dee891a811f8ee",
-            "20.10.22": "2c75cd6c3dc9b81cb5bde664c882e4339a2054e09cf09606f9f7dd6970e7f078",
-            "20.10.23": "5c40bb7dcd1aad94be49ad75d24e7fd409119ed0eaad04f5d13c4fddfb397c8a",
-            "23.0.0": "2919ff3448187d4f13cfbe2332707cff3f6dcf2baaac42a34bea8dd21f434f4a",
-            "23.0.1": "3865f837dbd951b19eeb5f7d87aada2e865b2017e9462fe389f0e5d9a438324d",
-            "24.0.6": "d9f58aecc42451503e82e6e0562cafa1812b334c92186a7f486e111e70a0f5bd"
+            "24.0.6": "d9f58aecc42451503e82e6e0562cafa1812b334c92186a7f486e111e70a0f5bd",
+            "24.0.9": "7e999590330a15469de20ac37051407d222ea73c71c10c05d61666d42c5922d1"
         }
     },
     "cri-dockerd": {