diff --git a/.github/workflows/gen-repository-iso.yaml b/.github/workflows/gen-repository-iso.yaml index 27fc7d193..c97562754 100644 --- a/.github/workflows/gen-repository-iso.yaml +++ b/.github/workflows/gen-repository-iso.yaml @@ -66,7 +66,7 @@ jobs: - name: Release and upload packages if: startsWith(github.ref, 'refs/tags/') - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/issue_comment_webhook.yml b/.github/workflows/issue_comment_webhook.yml new file mode 100644 index 000000000..189cb9569 --- /dev/null +++ b/.github/workflows/issue_comment_webhook.yml @@ -0,0 +1,45 @@ +name: Issue Comment WeCom Webhook + +on: + issues: + types: [opened, edited] + issue_comment: + types: [created, edited] + +jobs: + send_to_webhook: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-node@v4 + with: + node-version: "20.x" + - run: npm install axios + - name: Send issue/comment to WeCom webhook + uses: actions/github-script@v7 + env: + WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }} + with: + script: | + console.log(context); + if (context.payload.sender.login === "ks-ci-bot") return; + const axios = require('axios'); + const issue = context.payload.issue; + const comment = context.payload.comment; + var subject = {}; + var action = ''; + if (comment) { + action = "comment"; + subject = comment; + } else { + action = "issue"; + subject = issue; + }; + const payload = { + msgtype: 'markdown', + markdown: { + content: `[${context.payload.sender.login}](${context.payload.sender.html_url}) ${context.payload.action} ${action} [${issue.title}](${subject.html_url})\n${subject.body}`, + }, + }; + const formattedPayload = JSON.stringify(payload, null, 2); + console.log(formattedPayload); + await axios.post(process.env.WEBHOOK_URL, payload); diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 205c60fe9..accbf0a51 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -24,7 +24,7 @@ jobs: run: | make release - name: Release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: draft: true files: out/* diff --git a/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go b/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go index 2a3cfeddb..b5b3bc2af 100644 --- a/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go +++ b/cmd/kk/apis/kubekey/v1alpha2/cluster_types.go @@ -146,7 +146,7 @@ func (cfg *ClusterSpec) GenerateCertSANs() []string { if InternalIPv4Address != host.Address && InternalIPv4Address != cfg.ControlPlaneEndpoint.Address { extraCertSANs = append(extraCertSANs, InternalIPv4Address) } - if len(nodeAddresses)==2 { + if len(nodeAddresses) == 2 { InternalIPv6Address := nodeAddresses[1] extraCertSANs = append(extraCertSANs, InternalIPv6Address) } @@ -310,3 +310,10 @@ func (c *ControlPlaneEndpoint) EnableExternalDNS() bool { } return *c.ExternalDNS } + +func (r *RegistryConfig) GetHost() string { + if r.PrivateRegistry == "" { + return "" + } + return strings.Split(r.PrivateRegistry, "/")[0] +} diff --git a/cmd/kk/pkg/binaries/kubernetes.go b/cmd/kk/pkg/binaries/kubernetes.go index 98eeafe99..1f43fda86 100644 --- a/cmd/kk/pkg/binaries/kubernetes.go +++ b/cmd/kk/pkg/binaries/kubernetes.go @@ -101,18 +101,25 @@ func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string, return nil } -func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path, arch, k8sVersion string) error { +func KubernetesComponentBinariesDownload(manifest *common.ArtifactManifest, path, arch string) error { m := manifest.Spec + var binaries []*files.KubeBinary - etcd := files.NewKubeBinary("etcd", arch, m.Components.ETCD.Version, path, manifest.Arg.DownloadCommand) - kubeadm := files.NewKubeBinary("kubeadm", arch, k8sVersion, path, manifest.Arg.DownloadCommand) - kubelet := files.NewKubeBinary("kubelet", arch, k8sVersion, path, manifest.Arg.DownloadCommand) - kubectl := files.NewKubeBinary("kubectl", arch, k8sVersion, path, manifest.Arg.DownloadCommand) - kubecni := files.NewKubeBinary("kubecni", arch, m.Components.CNI.Version, path, manifest.Arg.DownloadCommand) - helm := files.NewKubeBinary("helm", arch, m.Components.Helm.Version, path, manifest.Arg.DownloadCommand) - crictl := files.NewKubeBinary("crictl", arch, m.Components.Crictl.Version, path, manifest.Arg.DownloadCommand) - calicoctl := files.NewKubeBinary("calicoctl", arch, m.Components.Calicoctl.Version, path, manifest.Arg.DownloadCommand) - binaries := []*files.KubeBinary{kubeadm, kubelet, kubectl, helm, kubecni, etcd, calicoctl} + if m.Components.ETCD.Version != "" { + binaries = append(binaries, files.NewKubeBinary("etcd", arch, m.Components.ETCD.Version, path, manifest.Arg.DownloadCommand)) + } + if m.Components.CNI.Version != "" { + binaries = append(binaries, files.NewKubeBinary("kubecni", arch, m.Components.CNI.Version, path, manifest.Arg.DownloadCommand)) + } + if m.Components.Helm.Version != "" { + binaries = append(binaries, files.NewKubeBinary("helm", arch, m.Components.Helm.Version, path, manifest.Arg.DownloadCommand)) + } + if m.Components.Crictl.Version != "" { + binaries = append(binaries, files.NewKubeBinary("crictl", arch, m.Components.Crictl.Version, path, manifest.Arg.DownloadCommand)) + } + if m.Components.Calicoctl.Version != "" { + binaries = append(binaries, files.NewKubeBinary("calicoctl", arch, m.Components.Calicoctl.Version, path, manifest.Arg.DownloadCommand)) + } containerManagerArr := make([]*files.KubeBinary, 0, 0) containerManagerVersion := make(map[string]struct{}) @@ -128,11 +135,36 @@ func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path, } } - binaries = append(binaries, containerManagerArr...) - if m.Components.Crictl.Version != "" { - binaries = append(binaries, crictl) + for _, binary := range binaries { + if err := binary.CreateBaseDir(); err != nil { + return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) + } + + logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) + + if util.IsExist(binary.Path()) { + // download it again if it's incorrect + if err := binary.SHA256Check(); err != nil { + _ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", binary.Path())).Run() + } else { + continue + } + } + + if err := binary.Download(); err != nil { + return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err) + } } + return nil +} + +func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path, arch, k8sVersion string) error { + kubeadm := files.NewKubeBinary("kubeadm", arch, k8sVersion, path, manifest.Arg.DownloadCommand) + kubelet := files.NewKubeBinary("kubelet", arch, k8sVersion, path, manifest.Arg.DownloadCommand) + kubectl := files.NewKubeBinary("kubectl", arch, k8sVersion, path, manifest.Arg.DownloadCommand) + binaries := []*files.KubeBinary{kubeadm, kubelet, kubectl} + for _, binary := range binaries { if err := binary.CreateBaseDir(); err != nil { return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName) diff --git a/cmd/kk/pkg/binaries/tasks.go b/cmd/kk/pkg/binaries/tasks.go index 7d829cb79..1c6b3620c 100644 --- a/cmd/kk/pkg/binaries/tasks.go +++ b/cmd/kk/pkg/binaries/tasks.go @@ -168,6 +168,10 @@ func (a *ArtifactDownload) Execute(runtime connector.Runtime) error { } } + if err := KubernetesComponentBinariesDownload(a.Manifest, basePath, arch); err != nil { + return err + } + if err := RegistryBinariesDownload(a.Manifest, basePath, arch); err != nil { return err } diff --git a/cmd/kk/pkg/bootstrap/os/templates/init_script.go b/cmd/kk/pkg/bootstrap/os/templates/init_script.go index 09e14c878..ca671010c 100644 --- a/cmd/kk/pkg/bootstrap/os/templates/init_script.go +++ b/cmd/kk/pkg/bootstrap/os/templates/init_script.go @@ -249,9 +249,9 @@ func GenerateHosts(runtime connector.ModuleRuntime, kubeConf *common.KubeConf) [ if len(runtime.GetHostsByRole(common.Registry)) > 0 { if kubeConf.Cluster.Registry.PrivateRegistry != "" { - hostsList = append(hostsList, fmt.Sprintf("%s %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalIPv4Address(), kubeConf.Cluster.Registry.PrivateRegistry)) + hostsList = append(hostsList, fmt.Sprintf("%s %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalIPv4Address(), kubeConf.Cluster.Registry.GetHost())) if runtime.GetHostsByRole(common.Registry)[0].GetInternalIPv6Address() != "" { - hostsList = append(hostsList, fmt.Sprintf("%s %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalIPv6Address(), kubeConf.Cluster.Registry.PrivateRegistry)) + hostsList = append(hostsList, fmt.Sprintf("%s %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalIPv6Address(), kubeConf.Cluster.Registry.GetHost())) } } else { diff --git a/cmd/kk/pkg/bootstrap/registry/certs.go b/cmd/kk/pkg/bootstrap/registry/certs.go index 690cb4bfa..883a05a81 100644 --- a/cmd/kk/pkg/bootstrap/registry/certs.go +++ b/cmd/kk/pkg/bootstrap/registry/certs.go @@ -105,7 +105,7 @@ func (g *GenerateCerts) Execute(runtime connector.Runtime) error { var altName cert.AltNames - dnsList := []string{"localhost", g.KubeConf.Cluster.Registry.PrivateRegistry} + dnsList := []string{"localhost", g.KubeConf.Cluster.Registry.GetHost()} ipList := []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback} for _, h := range runtime.GetHostsByRole(common.Registry) { @@ -115,13 +115,13 @@ func (g *GenerateCerts) Execute(runtime connector.Runtime) error { altName.DNSNames = dnsList altName.IPs = ipList - files := []string{"ca.pem", "ca-key.pem", fmt.Sprintf("%s.pem", g.KubeConf.Cluster.Registry.PrivateRegistry), fmt.Sprintf("%s-key.pem", g.KubeConf.Cluster.Registry.PrivateRegistry)} + files := []string{"ca.pem", "ca-key.pem", fmt.Sprintf("%s.pem", g.KubeConf.Cluster.Registry.GetHost()), fmt.Sprintf("%s-key.pem", g.KubeConf.Cluster.Registry.GetHost())} // CA certsList := []*certs.KubekeyCert{KubekeyCertRegistryCA()} // Certs - certsList = append(certsList, KubekeyCertRegistryServer(g.KubeConf.Cluster.Registry.PrivateRegistry, &altName)) + certsList = append(certsList, KubekeyCertRegistryServer(g.KubeConf.Cluster.Registry.GetHost(), &altName)) var lastCACert *certs.KubekeyCert for _, c := range certsList { diff --git a/cmd/kk/pkg/bootstrap/registry/module.go b/cmd/kk/pkg/bootstrap/registry/module.go index 284b88bac..2b38a1be1 100644 --- a/cmd/kk/pkg/bootstrap/registry/module.go +++ b/cmd/kk/pkg/bootstrap/registry/module.go @@ -131,8 +131,8 @@ func InstallRegistry(i *InstallRegistryModule) []task.Interface { Template: templates.RegistryConfigTempl, Dst: "/etc/kubekey/registry/config.yaml", Data: util.Data{ - "Certificate": fmt.Sprintf("%s.pem", i.KubeConf.Cluster.Registry.PrivateRegistry), - "Key": fmt.Sprintf("%s-key.pem", i.KubeConf.Cluster.Registry.PrivateRegistry), + "Certificate": fmt.Sprintf("%s.pem", i.KubeConf.Cluster.Registry.GetHost()), + "Key": fmt.Sprintf("%s-key.pem", i.KubeConf.Cluster.Registry.GetHost()), }, }, Parallel: true, @@ -170,6 +170,20 @@ func InstallHarbor(i *InstallRegistryModule) []task.Interface { Retry: 2, } + generateContainerdService := &task.RemoteTask{ + Name: "GenerateContainerdService", + Desc: "Generate containerd service", + Hosts: i.Runtime.GetHostsByRole(common.K8s), + Prepare: &prepare.PrepareCollection{ + &container.ContainerdExist{Not: true}, + }, + Action: &action.Template{ + Template: docker_template.ContainerdService, + Dst: filepath.Join("/etc/systemd/system", docker_template.ContainerdService.Name()), + }, + Parallel: true, + } + generateDockerService := &task.RemoteTask{ Name: "GenerateDockerService", Desc: "Generate docker service", @@ -202,6 +216,17 @@ func InstallHarbor(i *InstallRegistryModule) []task.Interface { Parallel: true, } + enableContainerdForDocker := &task.RemoteTask{ + Name: "EnableContainerd", + Desc: "Enable containerd", + Hosts: i.Runtime.GetHostsByRole(common.K8s), + Prepare: &prepare.PrepareCollection{ + &container.ContainerdExist{Not: true}, + }, + Action: new(container.EnableContainerdForDocker), + Parallel: true, + } + enableDocker := &task.RemoteTask{ Name: "EnableDocker", Desc: "Enable docker", @@ -250,10 +275,10 @@ func InstallHarbor(i *InstallRegistryModule) []task.Interface { } generateHarborConfig := &task.RemoteTask{ - Name: "GenerateHarborConfig", - Desc: "Generate harbor config", - Hosts: i.Runtime.GetHostsByRole(common.Registry), - Action: new(GenerateHarborConfig), + Name: "GenerateHarborConfig", + Desc: "Generate harbor config", + Hosts: i.Runtime.GetHostsByRole(common.Registry), + Action: new(GenerateHarborConfig), Parallel: true, Retry: 1, } @@ -269,8 +294,10 @@ func InstallHarbor(i *InstallRegistryModule) []task.Interface { return []task.Interface{ syncBinaries, + generateContainerdService, generateDockerService, generateDockerConfig, + enableContainerdForDocker, enableDocker, installDockerCompose, syncHarborPackage, diff --git a/cmd/kk/pkg/bootstrap/registry/tasks.go b/cmd/kk/pkg/bootstrap/registry/tasks.go index 1185cb11e..c216c5e71 100644 --- a/cmd/kk/pkg/bootstrap/registry/tasks.go +++ b/cmd/kk/pkg/bootstrap/registry/tasks.go @@ -89,7 +89,7 @@ func (s *SyncCertsToAllNodes) Execute(runtime connector.Runtime) error { } } - if err := runtime.GetRunner().SudoScp(filepath.Join(dir, fileName), filepath.Join(filepath.Join("/etc/docker/certs.d", s.KubeConf.Cluster.Registry.PrivateRegistry), dstFileName)); err != nil { + if err := runtime.GetRunner().SudoScp(filepath.Join(dir, fileName), filepath.Join(filepath.Join("/etc/docker/certs.d", s.KubeConf.Cluster.Registry.GetHost()), dstFileName)); err != nil { return errors.Wrap(errors.WithStack(err), "scp registry certs file to /etc/docker/certs.d/ failed") } @@ -144,7 +144,7 @@ func (g *StartRegistryService) Execute(runtime connector.Runtime) error { } fmt.Println() - fmt.Println(fmt.Sprintf("Local image registry created successfully. Address: %s", g.KubeConf.Cluster.Registry.PrivateRegistry)) + fmt.Println(fmt.Sprintf("Local image registry created successfully. Address: %s", g.KubeConf.Cluster.Registry.GetHost())) fmt.Println() return nil @@ -221,7 +221,7 @@ type GenerateHarborConfig struct { } func (g *GenerateHarborConfig) Execute(runtime connector.Runtime) error { - registryDomain := g.KubeConf.Cluster.Registry.PrivateRegistry + registryDomain := g.KubeConf.Cluster.Registry.GetHost() if g.KubeConf.Cluster.Registry.Type == "harbor-ha" { host := runtime.RemoteHost() @@ -233,9 +233,9 @@ func (g *GenerateHarborConfig) Execute(runtime connector.Runtime) error { Dst: "/opt/harbor/harbor.yml", Data: util.Data{ "Domain": registryDomain, - "Certificate": fmt.Sprintf("%s.pem", g.KubeConf.Cluster.Registry.PrivateRegistry), - "Key": fmt.Sprintf("%s-key.pem", g.KubeConf.Cluster.Registry.PrivateRegistry), - "Password": templates.Password(g.KubeConf, g.KubeConf.Cluster.Registry.PrivateRegistry), + "Certificate": fmt.Sprintf("%s.pem", g.KubeConf.Cluster.Registry.GetHost()), + "Key": fmt.Sprintf("%s-key.pem", g.KubeConf.Cluster.Registry.GetHost()), + "Password": templates.Password(g.KubeConf, g.KubeConf.Cluster.Registry.GetHost()), }, } templateAction.Init(nil, nil) @@ -256,7 +256,7 @@ func (g *StartHarbor) Execute(runtime connector.Runtime) error { } fmt.Println() - fmt.Println(fmt.Sprintf("Local image registry created successfully. Address: %s", g.KubeConf.Cluster.Registry.PrivateRegistry)) + fmt.Println(fmt.Sprintf("Local image registry created successfully. Address: %s", g.KubeConf.Cluster.Registry.GetHost())) fmt.Println() return nil diff --git a/cmd/kk/pkg/container/module.go b/cmd/kk/pkg/container/module.go index 5c732b02a..09c109370 100644 --- a/cmd/kk/pkg/container/module.go +++ b/cmd/kk/pkg/container/module.go @@ -281,6 +281,10 @@ type InstallCriDockerdModule struct { Skip bool } +func (m *InstallCriDockerdModule) IsSkip() bool { + return m.Skip +} + func (m *InstallCriDockerdModule) Init() { m.Name = "InstallCriDockerdModule" m.Desc = "Install cri-dockerd" diff --git a/cmd/kk/pkg/images/tasks.go b/cmd/kk/pkg/images/tasks.go index fee9a2862..33b938cd4 100644 --- a/cmd/kk/pkg/images/tasks.go +++ b/cmd/kk/pkg/images/tasks.go @@ -289,7 +289,7 @@ func (c *CopyImagesToRegistry) Execute(runtime connector.Runtime) error { } auth := new(registry.DockerRegistryEntry) - if config, ok := auths[c.KubeConf.Cluster.Registry.PrivateRegistry]; ok { + if config, ok := auths[c.KubeConf.Cluster.Registry.GetHost()]; ok { auth = config } diff --git a/cmd/kk/pkg/pipelines/artifact_export.go b/cmd/kk/pkg/pipelines/artifact_export.go index 6e730deb5..a47db6857 100644 --- a/cmd/kk/pkg/pipelines/artifact_export.go +++ b/cmd/kk/pkg/pipelines/artifact_export.go @@ -117,7 +117,7 @@ func ArtifactExport(args common.ArtifactArgument, downloadCmd string) error { } if len(runtime.Spec.KubernetesDistributions) == 0 { - return errors.New("the length of kubernetes distributions can't be 0") + return NewArtifactExportPipeline(runtime) } pre := runtime.Spec.KubernetesDistributions[0].Type diff --git a/hack/gen-repository-iso/dockerfile.ubuntu2204 b/hack/gen-repository-iso/dockerfile.ubuntu2204 index 7a92912a6..42ebb4d94 100644 --- a/hack/gen-repository-iso/dockerfile.ubuntu2204 +++ b/hack/gen-repository-iso/dockerfile.ubuntu2204 @@ -1,5 +1,6 @@ FROM ubuntu:22.04 as ubuntu2204 ARG TARGETARCH +ARG DISTRO=ubuntu2204 ARG OS_RELEASE=jammy ARG DIR=ubuntu-22.04-${TARGETARCH}-debs ARG PKGS=.common[],.debs[],.ubuntu[],.ubuntu2204[] @@ -15,6 +16,19 @@ RUN apt update -qq \ && echo "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu ${OS_RELEASE} stable" > /etc/apt/sources.list.d/docker.list\ && apt update -qq +# install NVIDIA CUDA +RUN if [ "${TARGETARCH}" = "amd64" ]; then \ + ARCH=x86_64; \ + else \ + ARCH=${TARGETARCH}; \ + fi \ + && wget https://developer.download.nvidia.com/compute/cuda/repos/${DISTRO}/${ARCH}/cuda-archive-keyring.gpg \ + && mv cuda-archive-keyring.gpg /usr/share/keyrings/cuda-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/${DISTRO}/${ARCH}/ /" | tee /etc/apt/sources.list.d/cuda-${DISTRO}-${ARCH}.list \ + && wget https://developer.download.nvidia.com/compute/cuda/repos/${DISTRO}/${ARCH}/cuda-${DISTRO}.pin \ + && mv cuda-${DISTRO}.pin /etc/apt/preferences.d/cuda-repository-pin-600 \ + && apt-get update + WORKDIR /package COPY packages.yaml . diff --git a/hack/gen-repository-iso/packages.yaml b/hack/gen-repository-iso/packages.yaml index 65d89a3eb..4708bd564 100644 --- a/hack/gen-repository-iso/packages.yaml +++ b/hack/gen-repository-iso/packages.yaml @@ -39,6 +39,9 @@ debs: - openssh-server - software-properties-common - sudo + - cuda-toolkit-12-4 + - nvidia-driver-550-open + - cuda-drivers-550 centos: - containerd.io diff --git a/version/components.json b/version/components.json index 185ebf316..8cdb05ac6 100644 --- a/version/components.json +++ b/version/components.json @@ -979,10 +979,10 @@ }, "cri-dockerd": { "amd64": { - "0.3.10": "46bc9594c634e1bae9b4c7dfaba1b4b34917b407c3275c08425f93d193024dab" + "0.3.10": "3e19ef525e02d2d1dfd42e8d661ee45b4bc8a49a6dcafd8baa578bdb3a23aeb6" }, "arm64": { - "0.3.10": "8684425e696dc52c3a10086af34af1ce6a7412de0ef819c3994f11e0e17d7fb9" + "0.3.10": "24d2d9cdbb4ed4bda4b0838edb52104ac7a4e2212a0ee05b177de0ae5b6a4a9a" } }, "containerd": {