From 3a20093929f4515bd3a822216e0bd3483dbb5ba6 Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Wed, 5 Jul 2023 22:42:41 +0800 Subject: [PATCH] feat: support create kubernetes cluster (#3680) Co-authored-by: sophon-zt --- cmd/cli.go | 3 + cmd/infrastructure/builder/builder.go | 83 ++++++ .../template/containerd.config.toml.tpl | 48 ++++ .../builder/template/containerd.service.tpl | 26 ++ .../builder/template/crictl.yaml.tpl | 5 + .../builder/template/init_os.sh.tpl | 167 ++++++++++++ .../builder/template/kubekey_cluster.tpl | 64 +++++ cmd/infrastructure/builder/template_task.go | 55 ++++ cmd/infrastructure/cluster.go | 251 ++++++++++++++++++ cmd/infrastructure/constant/const.go | 50 ++++ cmd/infrastructure/create.go | 224 ++++++++++++++++ cmd/infrastructure/create_test.go | 100 +++++++ cmd/infrastructure/delete.go | 117 ++++++++ cmd/infrastructure/infras.go | 36 +++ cmd/infrastructure/infras_test.go | 43 +++ cmd/infrastructure/loader_adapter.go | 72 +++++ cmd/infrastructure/loader_adapter_test.go | 82 ++++++ cmd/infrastructure/pipeline.go | 74 ++++++ cmd/infrastructure/suit_test.go | 32 +++ cmd/infrastructure/tasks/addon.go | 73 +++++ cmd/infrastructure/tasks/dependent.go | 217 +++++++++++++++ cmd/infrastructure/tasks/download.go | 98 +++++++ cmd/infrastructure/tasks/kubeconfig.go | 89 +++++++ cmd/infrastructure/tasks/kubernetes.go | 217 +++++++++++++++ cmd/infrastructure/tasks/kubernetes_test.go | 20 ++ cmd/infrastructure/tasks/pipeline_runner.go | 103 +++++++ cmd/infrastructure/tasks/tasks.go | 132 +++++++++ cmd/infrastructure/types/features.go | 27 ++ cmd/infrastructure/types/type.go | 215 +++++++++++++++ cmd/infrastructure/utils/checksum.go | 81 ++++++ cmd/infrastructure/utils/command_wrapper.go | 56 ++++ cmd/infrastructure/utils/helm_wrapper.go | 92 +++++++ cmd/infrastructure/utils/install.go | 24 ++ cmd/infrastructure/utils/yaml_wrapper.go | 34 +++ util/helm/errors.go | 2 +- util/helm/helm.go | 2 +- util/helm/helm_test.go | 2 +- util/util.go | 4 +- util/util_test.go | 2 +- 39 files changed, 3016 insertions(+), 6 deletions(-) create mode 100644 cmd/infrastructure/builder/builder.go create mode 100644 cmd/infrastructure/builder/template/containerd.config.toml.tpl create mode 100644 cmd/infrastructure/builder/template/containerd.service.tpl create mode 100644 cmd/infrastructure/builder/template/crictl.yaml.tpl create mode 100644 cmd/infrastructure/builder/template/init_os.sh.tpl create mode 100644 cmd/infrastructure/builder/template/kubekey_cluster.tpl create mode 100644 cmd/infrastructure/builder/template_task.go create mode 100644 cmd/infrastructure/cluster.go create mode 100644 cmd/infrastructure/constant/const.go create mode 100644 cmd/infrastructure/create.go create mode 100644 cmd/infrastructure/create_test.go create mode 100644 cmd/infrastructure/delete.go create mode 100644 cmd/infrastructure/infras.go create mode 100644 cmd/infrastructure/infras_test.go create mode 100644 cmd/infrastructure/loader_adapter.go create mode 100644 cmd/infrastructure/loader_adapter_test.go create mode 100644 cmd/infrastructure/pipeline.go create mode 100644 cmd/infrastructure/suit_test.go create mode 100644 cmd/infrastructure/tasks/addon.go create mode 100644 cmd/infrastructure/tasks/dependent.go create mode 100644 cmd/infrastructure/tasks/download.go create mode 100644 cmd/infrastructure/tasks/kubeconfig.go create mode 100644 cmd/infrastructure/tasks/kubernetes.go create mode 100644 cmd/infrastructure/tasks/kubernetes_test.go create mode 100644 cmd/infrastructure/tasks/pipeline_runner.go create mode 100644 cmd/infrastructure/tasks/tasks.go create mode 100644 cmd/infrastructure/types/features.go create mode 100644 cmd/infrastructure/types/type.go create mode 100644 cmd/infrastructure/utils/checksum.go create mode 100644 cmd/infrastructure/utils/command_wrapper.go create mode 100644 cmd/infrastructure/utils/helm_wrapper.go create mode 100644 cmd/infrastructure/utils/install.go create mode 100644 cmd/infrastructure/utils/yaml_wrapper.go diff --git a/cmd/cli.go b/cmd/cli.go index fe99c0c7df4..1740343410c 100644 --- a/cmd/cli.go +++ b/cmd/cli.go @@ -36,6 +36,8 @@ import ( utilcomp "k8s.io/kubectl/pkg/util/completion" "k8s.io/kubectl/pkg/util/templates" + infras "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure" + "github.com/apecloud/kubeblocks/internal/cli/cmd/addon" "github.com/apecloud/kubeblocks/internal/cli/cmd/alert" "github.com/apecloud/kubeblocks/internal/cli/cmd/bench" @@ -183,6 +185,7 @@ A Command Line Interface for KubeBlocks`, fault.NewFaultCmd(f, ioStreams), builder.NewBuilderCmd(f, ioStreams), report.NewReportCmd(f, ioStreams), + infras.NewInfraCmd(ioStreams), ) filters := []string{"options"} diff --git a/cmd/infrastructure/builder/builder.go b/cmd/infrastructure/builder/builder.go new file mode 100644 index 00000000000..203c2a5ebc8 --- /dev/null +++ b/cmd/infrastructure/builder/builder.go @@ -0,0 +1,83 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package builder + +import ( + "bufio" + "embed" + "encoding/json" + "strings" + + "github.com/leaanthony/debme" + "k8s.io/apimachinery/pkg/util/yaml" + + cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + "github.com/apecloud/kubeblocks/internal/gotemplate" +) + +var ( + //go:embed template/* + cueTemplate embed.FS +) + +func newBuildTemplate(templateName string) (string, error) { + tmplFs, _ := debme.FS(cueTemplate, "template") + if tmlBytes, err := tmplFs.ReadFile(templateName); err != nil { + return "", err + } else { + return string(tmlBytes), nil + } +} + +func BuildFromTemplate(values *gotemplate.TplValues, templateName string) (string, error) { + tpl, err := newBuildTemplate(templateName) + if err != nil { + return "", err + } + + engine := gotemplate.NewTplEngine(values, nil, templateName, nil, nil) + rendered, err := engine.Render(tpl) + if err != nil { + return "", err + } + return rendered, nil +} + +func BuildResourceFromYaml[T any](obj T, bYaml string) (*T, error) { + var ret map[string]interface{} + + content, err := yaml.NewYAMLReader(bufio.NewReader(strings.NewReader(bYaml))).Read() + if err != nil { + return nil, cfgcore.WrapError(err, "failed to read the cluster yaml") + } + err = yaml.Unmarshal(content, &ret) + if err != nil { + return nil, cfgcore.WrapError(err, "failed to unmarshal the cluster yaml") + } + + contentToJSON, err := yaml.ToJSON(content) + if err != nil { + return nil, cfgcore.WrapError(err, "Unable to convert configuration to json") + } + if err := json.Unmarshal(contentToJSON, &obj); err != nil { + return nil, cfgcore.WrapError(err, "failed to unmarshal the cluster") + } + return &obj, nil +} diff --git a/cmd/infrastructure/builder/template/containerd.config.toml.tpl b/cmd/infrastructure/builder/template/containerd.config.toml.tpl new file mode 100644 index 00000000000..b4af2c0bf37 --- /dev/null +++ b/cmd/infrastructure/builder/template/containerd.config.toml.tpl @@ -0,0 +1,48 @@ +version = 2 +{{- if .DataRoot }} +root = {{ .DataRoot }} +{{ else }} +root = "/var/lib/containerd" +{{- end }} +state = "/run/containerd" + +[grpc] + address = "/run/containerd/containerd.sock" + uid = 0 + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + +[debug] + level = "info" + +[metrics] + address = "" + grpc_histogram = false + +[timeouts] + "io.containerd.timeout.shim.cleanup" = "5s" + "io.containerd.timeout.shim.load" = "5s" + "io.containerd.timeout.shim.shutdown" = "3s" + "io.containerd.timeout.task.state" = "2s" + +[plugins."io.containerd.grpc.v1.cri".containerd] +default_runtime_name = "runc" + +[plugins."io.containerd.grpc.v1.cri"] +stream_server_address = "127.0.0.1" +max_container_log_line_size = 262144 +sandbox_image = "{{ .SandBoxImage }}" + +[plugins."io.containerd.grpc.v1.cri".registry] +config_path = "/etc/containerd/certs.d:/etc/docker/certs.d" + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] +runtime_type = "io.containerd.runc.v2" + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] +SystemdCgroup = true + +[plugins."io.containerd.grpc.v1.cri".cni] +bin_dir = "/opt/cni/bin" +conf_dir = "/etc/cni/net.d" diff --git a/cmd/infrastructure/builder/template/containerd.service.tpl b/cmd/infrastructure/builder/template/containerd.service.tpl new file mode 100644 index 00000000000..d5af236df2f --- /dev/null +++ b/cmd/infrastructure/builder/template/containerd.service.tpl @@ -0,0 +1,26 @@ +[Unit] +Description=containerd container runtime +Documentation=https://containerd.io +After=network.target local-fs.target + +[Service] +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/bin/containerd + +Type=notify +Delegate=yes +KillMode=process +Restart=always +RestartSec=5 +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNPROC=infinity +LimitCORE=infinity +LimitNOFILE=infinity +# Comment TasksMax if your systemd version does not supports it. +# Only systemd 226 and above support this version. +TasksMax=infinity +OOMScoreAdjust=-999 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/cmd/infrastructure/builder/template/crictl.yaml.tpl b/cmd/infrastructure/builder/template/crictl.yaml.tpl new file mode 100644 index 00000000000..236bb88f5fd --- /dev/null +++ b/cmd/infrastructure/builder/template/crictl.yaml.tpl @@ -0,0 +1,5 @@ +runtime-endpoint: {{ .Endpoint }} +image-endpoint: {{ .Endpoint }} +timeout: 5 +debug: false +pull-image-on-create: false \ No newline at end of file diff --git a/cmd/infrastructure/builder/template/init_os.sh.tpl b/cmd/infrastructure/builder/template/init_os.sh.tpl new file mode 100644 index 00000000000..3a016ae43a8 --- /dev/null +++ b/cmd/infrastructure/builder/template/init_os.sh.tpl @@ -0,0 +1,167 @@ +#!/usr/bin/env bash + +function swap_off() { + echo "swap off..." + swapoff -a + sed -i /^[^#]*swap*/s/^/\#/g /etc/fstab + + # clean cache + echo 3 > /proc/sys/vm/drop_caches + echo +} + +function selinux_off() { + echo "selinux off..." + setenforce 0 + echo "enforce: $(getenforce)" + + if [ -f /etc/selinux/config ]; then + sed -ri 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config + fi + echo +} + +function firewalld_off() { + echo "firewall off..." + systemctl stop firewalld.service 1>/dev/null 2>&1 + systemctl disable firewalld.service 1>/dev/null 2>&1 + systemctl stop ufw 1>/dev/null 2>&1 + systemctl disable ufw 1>/dev/null 2>&1 + echo +} + +function replace_in_file() { + local filename="${1:?filename is required}" + local match_regex="${2:?match regex is required}" + local substitute_regex="${3:?substitute regex is required}" + local posix_regex=${4:-true} + + local result + local -r del=$'\001' + if [[ $posix_regex = true ]]; then + result="$(sed -E "s${del}${match_regex}${del}${substitute_regex}${del}g" "$filename")" + else + result="$(sed "s${del}${match_regex}${del}${substitute_regex}${del}g" "$filename")" + fi + echo "$result" > "$filename" +} + +function sysctl_set_keyvalue() { + local -r key="${1:?missing key}" + local -r value="${2:?missing value}" + local -r conf_file="${3:-"/etc/sysctl.conf"}" + if grep -qE "^#*\s*${key}" "$conf_file" >/dev/null; then + replace_in_file "$conf_file" "^#*\s*${key}\s*=.*" "${key} = ${value}" + else + echo "${key} = ${value}" >>"$conf_file" + fi +} + +function set_network() { + echo "set network..." + + sysctl_set_keyvalue "net.ipv4.tcp_tw_recycle" "0" + sysctl_set_keyvalue "net.ipv4.ip_forward" "1" + sysctl_set_keyvalue "net.bridge.bridge-nf-call-arptables" "1" + sysctl_set_keyvalue "net.bridge.bridge-nf-call-ip6tables" "1" + sysctl_set_keyvalue "net.bridge.bridge-nf-call-iptables" "1" + # for node port + sysctl_set_keyvalue "net.ipv4.ip_local_reserved_ports" "30000-32767" + + echo +} + +function install_hugepage() { + local -r hugepage="${1:?missing key}" + + echo "install hugepage..." + hpg_sz=$(grep Hugepagesize /proc/meminfo | awk '{print $2}') + ## convert to KB + re_hpg_num=$(echo $hugepage | awk '{sub("GB", "*1024*1024*1024", $1) || sub("MB", "*1024*1024", $1) || sub("KB", "*1024", $1); printf $1 "+"} END {print 0}' | bc) + hpg_num=$(echo "$re_hpg_num / ( $hpg_sz * 1024 )" | bc) + + sysctl_set_keyvalue "vm.nr_hugepages" "$hpg_num" + echo +} + +function common_os_setting() { + swap_off + selinux_off + firewalld_off + set_network +} + +function install_hosts() { + sed -i ':a;$!{N;ba};s@# kubeblocks hosts BEGIN.*# kubeblocks hosts END@@' /etc/hosts + sed -i '/^$/N;/\n$/N;//D' /etc/hosts + + cat >>/etc/hosts< /dev/null 2>&1 + if [ $? -eq 0 ]; then + modprobe br_netfilter + mkdir -p /etc/modules-load.d + echo 'br_netfilter' > /etc/modules-load.d/kubekey-br_netfilter.conf + fi + + modinfo overlay > /dev/null 2>&1 + if [ $? -eq 0 ]; then + modprobe overlay + echo 'overlay' >> /etc/modules-load.d/kubekey-br_netfilter.conf + fi +} + +function install_ipvs() { + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + cat > /etc/modules-load.d/kube_proxy-ipvs.conf << EOF +ip_vs +ip_vs_rr +ip_vs_wrr +ip_vs_sh +EOF + + modprobe nf_conntrack_ipv4 1>/dev/null 2>/dev/null + if [ $? -eq 0 ]; then + echo 'nf_conntrack_ipv4' > /etc/modules-load.d/kube_proxy-ipvs.conf + else + modprobe nf_conntrack + echo 'nf_conntrack' > /etc/modules-load.d/kube_proxy-ipvs.conf + fi +} + +{{- if $.Options.HugePageFeature }} +install_hugepage {{ $.Options.HugePageFeature.HugePageSize }} +{{- end }} + +install_netfilter +install_ipvs +install_hosts +common_os_setting + +# for es/opensearch +sysctl_set_keyvalue "vm.max_map_count" "262144" +# for kubeblocks +sysctl_set_keyvalue "fs.inotify.max_user_watches" "524288" +sysctl_set_keyvalue "fs.inotify.max_user_instances" "524288" +sysctl -p + +# Make sure the iptables utility doesn't use the nftables backend. +update-alternatives --set iptables /usr/sbin/iptables-legacy >/dev/null 2>&1 || true +update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy >/dev/null 2>&1 || true +update-alternatives --set arptables /usr/sbin/arptables-legacy >/dev/null 2>&1 || true +update-alternatives --set ebtables /usr/sbin/ebtables-legacy >/dev/null 2>&1 || true + +ulimit -u 65535 +ulimit -n 65535 diff --git a/cmd/infrastructure/builder/template/kubekey_cluster.tpl b/cmd/infrastructure/builder/template/kubekey_cluster.tpl new file mode 100644 index 00000000000..19395439517 --- /dev/null +++ b/cmd/infrastructure/builder/template/kubekey_cluster.tpl @@ -0,0 +1,64 @@ +apiVersion: kubekey.kubesphere.io/v1alpha2 +kind: Cluster +metadata: + name: {{ $.Name }} +spec: + hosts: + {{- range $.Hosts }} + - name: {{ .Name }} + address: {{ .Address }} + internalAddress: {{ .InternalAddress }} + user: {{ $.User.Name }} + password: {{ $.User.Password }} + privateKey: {{ $.User.PrivateKey | quote }} + timeout: {{ $.Timeout }} + {{- end }} + roleGroups: + {{- $roles := keys $.RoleGroups }} + {{- range $roles }} + {{- $nodes := get $.RoleGroups . }} + {{ . }}: + {{- range $nodes }} + - {{ . }} + {{- end }} + {{- end }} + controlPlaneEndpoint: + domain: {{ $.Kubernetes.ControlPlaneEndpoint.Domain }} + {{- $address := ""}} + {{- if hasKey $.RoleGroups "master" }} + {{- $mName := index (get $.RoleGroups "master") 0 }} + {{- range $.Hosts }} + {{- if eq .Name $mName }} + {{- $address = .InternalAddress }} + {{- end }} + {{- end }} + {{- end }} + {{- if eq $address "" }} + {{- failed "require control address." }} + {{- end }} + address: {{ $address }} + port: {{ $.Kubernetes.ControlPlaneEndpoint.Port }} + kubernetes: + nodelocaldns: false + dnsDomain: {{ $.Kubernetes.Networking.DNSDomain }} + version: {{ $.Version }} + clusterName: {{ $.Name }} + {{- $criType := "containerd" }} + nodeCidrMaskSize: 24 + proxyMode: {{ $.Kubernetes.ProxyMode }} + containerRuntimeEndpoint: {{ $.Kubernetes.CRI.ContainerRuntimeEndpoint }} + containerManager: {{ $.Kubernetes.CRI.ContainerRuntimeType }} + etcd: + backupDir: /var/backups/kube_etcd + backupPeriod: 1800 + keepBackupNumber: 6 + backupScript: /usr/local/bin/kube-scripts + {{- if hasKey $.RoleGroups "etcd" }} + type: kubekey + {{- else }} + type: kubeadm + {{- end }} + network: + plugin: {{ $.Kubernetes.Networking.Plugin }} + kubePodsCIDR: {{ $.Kubernetes.Networking.PodSubnet }} + kubeServiceCIDR: {{ $.Kubernetes.Networking.ServiceSubnet }} \ No newline at end of file diff --git a/cmd/infrastructure/builder/template_task.go b/cmd/infrastructure/builder/template_task.go new file mode 100644 index 00000000000..eaf7d5d1cc9 --- /dev/null +++ b/cmd/infrastructure/builder/template_task.go @@ -0,0 +1,55 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package builder + +import ( + "path/filepath" + + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" + + cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + "github.com/apecloud/kubeblocks/internal/gotemplate" +) + +type Template struct { + action.BaseAction + Template string + Dst string + Values gotemplate.TplValues +} + +func (t *Template) Execute(runtime connector.Runtime) error { + templateStr, err := BuildFromTemplate(&t.Values, t.Template) + if err != nil { + return cfgcore.WrapError(err, "failed to render template %s", t.Template) + } + + fileName := filepath.Join(runtime.GetHostWorkDir(), t.Template) + if err := util.WriteFile(fileName, []byte(templateStr)); err != nil { + return cfgcore.WrapError(err, "failed to write file %s", fileName) + } + + if err := runtime.GetRunner().SudoScp(fileName, t.Dst); err != nil { + return cfgcore.WrapError(err, "failed to scp file %s to remote %s", fileName, t.Dst) + } + return nil +} diff --git a/cmd/infrastructure/cluster.go b/cmd/infrastructure/cluster.go new file mode 100644 index 00000000000..90c77690da7 --- /dev/null +++ b/cmd/infrastructure/cluster.go @@ -0,0 +1,251 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package infrastructure + +import ( + "fmt" + "os" + "os/user" + "path/filepath" + "strings" + + "github.com/StudioSol/set" + kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" + "github.com/spf13/cobra" + "golang.org/x/exp/slices" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/cli-runtime/pkg/genericclioptions" + + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/builder" + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/types" + "github.com/apecloud/kubeblocks/internal/cli/printer" + "github.com/apecloud/kubeblocks/internal/cli/util/prompt" + cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + cfgutil "github.com/apecloud/kubeblocks/internal/configuration/util" +) + +type clusterOptions struct { + types.Cluster + IOStreams genericclioptions.IOStreams + + clusterConfig string + clusterName string + timeout int64 + nodes []string +} + +func buildCommonFlags(cmd *cobra.Command, o *clusterOptions) { + cmd.Flags().StringVarP(&o.clusterConfig, "config", "c", "", "Specify infra cluster config file. [option]") + cmd.Flags().StringVarP(&o.clusterName, "name", "", "", "Specify kubernetes cluster name") + cmd.Flags().StringSliceVarP(&o.nodes, "nodes", "", nil, "List of machines on which kubernetes is installed. [require]") + + // for user + cmd.Flags().StringVarP(&o.User.Name, "user", "u", "", "Specify the account to access the remote server. [require]") + cmd.Flags().Int64VarP(&o.timeout, "timeout", "t", 30, "Specify the ssh timeout.[option]") + cmd.Flags().StringVarP(&o.User.Password, "password", "p", "", "Specify the password for the account to execute sudo. [option]") + cmd.Flags().StringVarP(&o.User.PrivateKey, "private-key", "", "", "The PrimaryKey for ssh to the remote machine. [option]") + cmd.Flags().StringVarP(&o.User.PrivateKeyPath, "private-key-path", "", "", "Specify the file PrimaryKeyPath of ssh to the remote machine. default ~/.ssh/id_rsa.") + + cmd.Flags().StringSliceVarP(&o.RoleGroup.ETCD, "etcd", "", nil, "Specify etcd nodes") + cmd.Flags().StringSliceVarP(&o.RoleGroup.Master, "master", "", nil, "Specify master nodes") + cmd.Flags().StringSliceVarP(&o.RoleGroup.Worker, "worker", "", nil, "Specify worker nodes") +} + +func (o *clusterOptions) Complete() error { + if o.clusterName == "" && o.clusterConfig == "" { + o.clusterName = "kubeblocks-" + rand.String(6) + fmt.Printf("The cluster name is not set, auto generate cluster name: %s\n", o.clusterName) + } + + if o.clusterConfig != "" { + return o.fillClusterConfig(o.clusterConfig) + } + + if o.User.Name == "" { + currentUser, err := user.Current() + if err != nil { + return err + } + o.User.Name = currentUser.Username + fmt.Printf("The user is not set, use current user %s\n", o.User.Name) + } + if o.User.Password == "" && o.User.PrivateKey == "" && o.User.PrivateKeyPath == "" { + home, err := os.UserHomeDir() + if err != nil { + return err + } + o.User.PrivateKeyPath = filepath.Join(home, ".ssh", "id_rsa") + } + if len(o.nodes) == 0 { + return cfgcore.MakeError("The list of machines where kubernetes is installed must be specified.") + } + o.Nodes = make([]types.ClusterNode, len(o.nodes)) + for i, node := range o.nodes { + fields := strings.SplitN(node, ":", 3) + if len(fields) < 2 { + return cfgcore.MakeError("The node format is incorrect, require: [name:address:internalAddress].") + } + n := types.ClusterNode{ + Name: fields[0], + Address: fields[1], + InternalAddress: fields[1], + } + if len(fields) == 3 { + n.InternalAddress = fields[2] + } + o.Nodes[i] = n + } + return nil +} + +func (o *clusterOptions) Validate() error { + if o.User.Name == "" { + return cfgcore.MakeError("user name is empty") + } + if o.clusterName == "" { + return cfgcore.MakeError("kubernetes name is empty") + } + if err := validateUser(o); err != nil { + return err + } + if !o.RoleGroup.IsValidate() { + return cfgcore.MakeError("etcd, master or worker is empty") + } + if err := o.checkReplicaNode(o.RoleGroup.ETCD); err != nil { + return err + } + if err := o.checkReplicaNode(o.RoleGroup.Master); err != nil { + return err + } + if err := o.checkReplicaNode(o.RoleGroup.Worker); err != nil { + return err + } + return nil +} + +func (o *clusterOptions) checkReplicaNode(nodes []string) error { + sets := cfgutil.NewSet() + for _, node := range nodes { + if !o.hasNode(node) { + return cfgcore.MakeError("node %s is not exist!", node) + } + if sets.InArray(node) { + return cfgcore.MakeError("node %s is repeat!", node) + } + sets.Add(node) + } + return nil +} + +func (o *clusterOptions) hasNode(n string) bool { + for _, node := range o.Nodes { + if node.Name == n { + return true + } + } + return false +} + +func (o *clusterOptions) confirm(promptStr string) (bool, error) { + const yesStr = "yes" + const noStr = "no" + + confirmStr := []string{yesStr, noStr} + printer.Warning(o.IOStreams.Out, promptStr) + input, err := prompt.NewPrompt("Please type [yes/No] to confirm:", + func(input string) error { + if !slices.Contains(confirmStr, strings.ToLower(input)) { + return fmt.Errorf("typed \"%s\" does not match \"%s\"", input, confirmStr) + } + return nil + }, o.IOStreams.In).Run() + if err != nil { + return false, err + } + return strings.ToLower(input) == yesStr, nil +} + +func (o *clusterOptions) fillClusterConfig(configFile string) error { + _, err := os.Stat(configFile) + if err != nil { + return err + } + b, err := os.ReadFile(configFile) + if err != nil { + return err + } + + c, err := builder.BuildResourceFromYaml(o.Cluster, string(b)) + if err != nil { + return err + } + o.Cluster = *c + o.clusterName = c.Name + return nil +} + +func checkAndUpdateHomeDir(user *types.ClusterUser) error { + if !strings.HasPrefix(user.PrivateKeyPath, "~/") { + return nil + } + home, err := os.UserHomeDir() + if err != nil { + return err + } + user.PrivateKeyPath = filepath.Join(home, user.PrivateKeyPath[2:]) + return nil +} + +func validateUser(o *clusterOptions) error { + if o.User.Password != "" || o.User.PrivateKey != "" { + return nil + } + if o.User.PrivateKey == "" && o.User.PrivateKeyPath != "" { + if err := checkAndUpdateHomeDir(&o.User); err != nil { + return err + } + if _, err := os.Stat(o.User.PrivateKeyPath); err != nil { + return err + } + b, err := os.ReadFile(o.User.PrivateKeyPath) + if err != nil { + return err + } + o.User.PrivateKey = string(b) + } + return nil +} + +func syncClusterNodeRole(cluster *kubekeyapiv1alpha2.ClusterSpec, runtime *common.KubeRuntime) { + hostSet := set.NewLinkedHashSetString() + for _, role := range cluster.GroupHosts() { + for _, host := range role { + if host.IsRole(common.Master) || host.IsRole(common.Worker) { + host.SetRole(common.K8s) + } + if !hostSet.InArray(host.GetName()) { + hostSet.Add(host.GetName()) + runtime.BaseRuntime.AppendHost(host) + runtime.BaseRuntime.AppendRoleMap(host) + } + } + } +} diff --git a/cmd/infrastructure/constant/const.go b/cmd/infrastructure/constant/const.go new file mode 100644 index 00000000000..18b3e44bbb5 --- /dev/null +++ b/cmd/infrastructure/constant/const.go @@ -0,0 +1,50 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package constant + +const ( + ContainerdService = "containerd.service.tpl" + ContainerdConfig = "containerd.config.toml.tpl" + CRICtlConfig = "crictl.yaml.tpl" + ConfigureOSScripts = "init_os.sh.tpl" + + ContainerdServiceInstallPath = "/etc/systemd/system/containerd.service" + ContainerdConfigInstallPath = "/etc/containerd/config.toml" + CRICtlConfigInstallPath = "/etc/crictl.yaml" + + DefaultSandBoxImage = "k8s.gcr.io/pause:3.8" + + DefaultK8sVersion = "v1.26.5" // https://github.com/kubernetes/kubernetes/releases/tag/v1.26.5 + DefaultEtcdVersion = "v3.4.26" // https://github.com/etcd-io/etcd/releases/tag/v3.4.26 + DefaultCRICtlVersion = "v1.26.0" // https://github.com/kubernetes-sigs/cri-tools/releases/tag/v1.26.0 + DefaultHelmVersion = "v3.12.0" // https://github.com/helm/helm/releases + DefaultRuncVersion = "v1.1.7" // https://github.com/opencontainers/runc/releases + DefaultCniVersion = "v1.3.0" // https://github.com/containernetworking/plugins/releases + DefaultContainerdVersion = "1.7.2" // https://github.com/containerd/containerd/releases +) + +const ( + DefaultK8sDNSDomain = "cluster.local" + DefaultAPIDNSDomain = "lb.kubeblocks.local" + DefaultK8sProxyMode = "ipvs" + DefaultAPIServerPort = 6443 + + DefaultNetworkPlugin = "cilium" +) diff --git a/cmd/infrastructure/create.go b/cmd/infrastructure/create.go new file mode 100644 index 00000000000..2d432a9148f --- /dev/null +++ b/cmd/infrastructure/create.go @@ -0,0 +1,224 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package infrastructure + +import ( + "fmt" + "os" + + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" + "github.com/spf13/cobra" + versionutil "k8s.io/apimachinery/pkg/util/version" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/constant" + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/tasks" + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/types" + "github.com/apecloud/kubeblocks/internal/cli/util" + cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + "github.com/apecloud/kubeblocks/internal/configuration/container" +) + +type createOptions struct { + clusterOptions + version types.InfraVersionInfo + + criType string + debug bool + sandBoxImage string + + securityEnhancement bool + outputKubeconfig string +} + +var createExamples = templates.Examples(` + # Create kubernetes cluster with specified config yaml + kbcli infra create -c cluster.yaml + + # example cluster.yaml + cat cluster.yaml +metadata: + name: kb-k8s-test-cluster +user: + name: user1 + privateKeyPath: ~/.ssh/test.pem +nodes: + - name: kb-infra-node-0 + address: 1.1.1.1 + internalAddress: 10.128.0.19 + - name: kb-infra-node-1 + address: 1.1.1.2 + internalAddress: 10.128.0.20 + - name: kb-infra-node-2 + address: 1.1.1.3 + internalAddress: 10.128.0.21 + options: + hugePageFeature: + hugePageSize: 10GB +roleGroup: + etcd: + - kb-infra-node-0 + - kb-infra-node-1 + - kb-infra-node-2 + master: + - kb-infra-node-0 + worker: + - kb-infra-node-1 + - kb-infra-node-2 + +kubernetes: + containerManager: containerd + # apis/kubeadm/types.Networking + networking: + plugin: cilium + dnsDomain: cluster.local + podSubnet: 10.233.64.0/18 + serviceSubnet: 10.233.0.0/18 + controlPlaneEndpoint: + domain: lb.kubeblocks.local + port: 6443 + cri: + containerRuntimeType: "containerd" + containerRuntimeEndpoint: "unix:///run/containerd/containerd.sock" + sandBoxImage: "k8s.gcr.io/pause:3.8" +addons: + - name: openebs + namespace: kube-blocks + sources: + chart: + name: openebs + version: 3.7.0 + repo: https://openebs.github.io/charts + options: + values: + - "localprovisioner.basePath=/mnt/disks" + - "localprovisioner.hostpathClass.isDefaultClass=true" +`) + +func (o *createOptions) Run() error { + const minKubernetesVersion = "v1.24.0" + + v, err := versionutil.ParseSemantic(o.version.KubernetesVersion) + if err != nil { + return err + } + c, err := v.Compare(minKubernetesVersion) + if err != nil { + return err + } + if c < 0 { + return cfgcore.MakeError("kubernetes version must be greater than %s", minKubernetesVersion) + } + + o.Cluster.Kubernetes.AutoDefaultFill() + o.version = o.Version + o.checkAndSetDefaultVersion() + cluster, err := createClusterWithOptions(buildTemplateParams(o)) + if err != nil { + return err + } + + yes, err := o.confirm(fmt.Sprintf("install kubernetes using version: %v", o.version.KubernetesVersion)) + if err != nil { + return err + } + if !yes { + return nil + } + + runtime := &common.KubeRuntime{ + BaseRuntime: connector.NewBaseRuntime(o.clusterName, connector.NewDialer(), o.debug, false), + Cluster: cluster, + ClusterName: o.clusterName, + } + + syncClusterNodeRole(cluster, runtime) + checkAndUpdateZone() + pipelineRunner := tasks.NewPipelineRunner("CreateCluster", NewCreatePipeline(o), runtime) + if err := pipelineRunner.Do(o.IOStreams.Out); err != nil { + return err + } + fmt.Fprintf(o.IOStreams.Out, "Kubernetes Installation is complete.\n\n") + return nil +} + +func checkAndUpdateZone() { + const ZoneName = "KKZONE" + if location, _ := util.GetIPLocation(); location == "CN" { + os.Setenv(ZoneName, "cn") + } + fmt.Printf("current zone: %s\n", os.Getenv(ZoneName)) +} + +func NewCreateKubernetesCmd(streams genericclioptions.IOStreams) *cobra.Command { + o := &createOptions{ + clusterOptions: clusterOptions{ + IOStreams: streams, + }} + o.checkAndSetDefaultVersion() + cmd := &cobra.Command{ + Use: "create", + Short: "create kubernetes cluster.", + Example: createExamples, + Run: func(cmd *cobra.Command, args []string) { + util.CheckErr(o.Complete()) + util.CheckErr(o.Validate()) + util.CheckErr(o.Run()) + }, + } + o.buildCreateInfraFlags(cmd) + o.Version = o.version + return cmd +} + +func (o *createOptions) buildCreateInfraFlags(cmd *cobra.Command) { + buildCommonFlags(cmd, &o.clusterOptions) + cmd.Flags().StringVarP(&o.version.KubernetesVersion, "version", "", o.version.KubernetesVersion, fmt.Sprintf("Specify install kubernetes version. default version is %s", o.version.KubernetesVersion)) + cmd.Flags().StringVarP(&o.sandBoxImage, "sandbox-image", "", constant.DefaultSandBoxImage, "Specified sandbox-image will not be used by the cri. [option]") + cmd.Flags().StringVarP(&o.criType, "container-runtime", "", string(container.ContainerdType), "Specify kubernetes container runtime. default is containerd") + cmd.Flags().BoolVarP(&o.debug, "debug", "", false, "set debug mode") + cmd.Flags().StringVarP(&o.outputKubeconfig, "output-kubeconfig", "", tasks.GetDefaultConfig(), "Specified output kubeconfig. [option]") +} + +func (o *createOptions) checkAndSetDefaultVersion() { + if o.version.KubernetesVersion == "" { + o.version.KubernetesVersion = constant.DefaultK8sVersion + } + if o.version.EtcdVersion == "" { + o.version.EtcdVersion = constant.DefaultEtcdVersion + } + if o.version.ContainerVersion == "" { + o.version.ContainerVersion = constant.DefaultContainerdVersion + } + if o.version.HelmVersion == "" { + o.version.HelmVersion = constant.DefaultHelmVersion + } + if o.version.CRICtlVersion == "" { + o.version.CRICtlVersion = constant.DefaultCRICtlVersion + } + if o.version.CniVersion == "" { + o.version.CniVersion = constant.DefaultCniVersion + } + if o.version.RuncVersion == "" { + o.version.RuncVersion = constant.DefaultRuncVersion + } +} diff --git a/cmd/infrastructure/create_test.go b/cmd/infrastructure/create_test.go new file mode 100644 index 00000000000..85593f929bb --- /dev/null +++ b/cmd/infrastructure/create_test.go @@ -0,0 +1,100 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package infrastructure + +import ( + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + + "github.com/apecloud/kubeblocks/internal/cli/testing" + "github.com/apecloud/kubeblocks/test/testdata" +) + +var _ = Describe("infra create test", func() { + + var ( + tf *cmdtesting.TestFactory + streams genericclioptions.IOStreams + ) + + BeforeEach(func() { + streams, _, _, _ = genericclioptions.NewTestIOStreams() + tf = cmdtesting.NewTestFactory().WithNamespace(testing.Namespace) + }) + + AfterEach(func() { + tf.Cleanup() + }) + + mockPrivateKeyFile := func(tmpDir string) string { + privateKeyFile := filepath.Join(tmpDir, "id_rsa.pem") + Expect(os.WriteFile(privateKeyFile, []byte("private key"), os.ModePerm)).Should(Succeed()) + return privateKeyFile + } + + It("test create k8s cluster with config file", func() { + tmpDir, _ := os.MkdirTemp(os.TempDir(), "test-") + defer os.RemoveAll(tmpDir) + + By("Create cluster with config file") + o := &createOptions{ + clusterOptions: clusterOptions{ + IOStreams: streams, + }} + o.checkAndSetDefaultVersion() + o.clusterConfig = testdata.SubTestDataPath("infrastructure/infra-cluster.yaml") + Expect(o.Complete()).To(Succeed()) + o.Cluster.User.PrivateKeyPath = mockPrivateKeyFile(tmpDir) + Expect(o.Validate()).To(Succeed()) + }) + + It("test create k8s cluster with params", func() { + tmpDir, _ := os.MkdirTemp(os.TempDir(), "test-") + defer os.RemoveAll(tmpDir) + + By("Create cluster with config file") + o := &createOptions{ + clusterOptions: clusterOptions{ + IOStreams: streams, + }} + o.checkAndSetDefaultVersion() + + o.nodes = []string{ + "node0:1.1.1.1:10.128.0.1", + "node1:1.1.1.2:10.128.0.2", + "node2:1.1.1.3:10.128.0.3", + } + o.Cluster.User.PrivateKeyPath = mockPrivateKeyFile(tmpDir) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).ShouldNot(Succeed()) + + o.Cluster.RoleGroup.Master = []string{"node0"} + o.Cluster.RoleGroup.ETCD = []string{"node0"} + o.Cluster.RoleGroup.Worker = []string{"node1", "node2"} + Expect(o.Validate()).Should(Succeed()) + }) + +}) diff --git a/cmd/infrastructure/delete.go b/cmd/infrastructure/delete.go new file mode 100644 index 00000000000..2c0f27c6cd0 --- /dev/null +++ b/cmd/infrastructure/delete.go @@ -0,0 +1,117 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package infrastructure + +import ( + "fmt" + + "github.com/apecloud/kubeblocks/internal/gotemplate" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +type deleteOptions struct { + clusterOptions + + deleteCRI bool + debug bool +} + +var deleteExamples = templates.Examples(` + # delete kubernetes cluster with specified config yaml + kbcli infra delete -c cluster.yaml +`) + +func (o *deleteOptions) Run() error { + o.Cluster.Kubernetes.AutoDefaultFill() + cluster, err := createClusterWithOptions(&gotemplate.TplValues{ + builtinClusterNameObject: o.clusterName, + builtinClusterVersionObject: "0.0.0", + builtinUserObject: o.User, + builtinHostsObject: o.Nodes, + builtinTimeoutObject: o.timeout, + builtinKubernetesObject: o.Cluster.Kubernetes, + builtinRoleGroupsObject: gotemplate.TplValues{ + common.ETCD: o.RoleGroup.ETCD, + common.Master: o.RoleGroup.Master, + common.Worker: o.RoleGroup.Worker, + }, + }) + if err != nil { + return err + } + + yes, err := o.confirm(fmt.Sprintf("delete kubernetes: %s", o.clusterName)) + if err != nil { + return err + } + if !yes { + return nil + } + + runtime := &common.KubeRuntime{ + BaseRuntime: connector.NewBaseRuntime(o.clusterName, connector.NewDialer(), o.debug, false), + Cluster: cluster, + ClusterName: o.clusterName, + } + syncClusterNodeRole(cluster, runtime) + + pipeline := pipeline.Pipeline{ + Name: "DeleteCluster", + Modules: NewDeletePipeline(o), + Runtime: runtime, + } + if err := pipeline.Start(); err != nil { + return err + } + fmt.Fprintf(o.IOStreams.Out, "Kubernetes deletion is complete.\n\n") + return nil +} + +func (o *deleteOptions) buildDeleteInfraFlags(cmd *cobra.Command) { + buildCommonFlags(cmd, &o.clusterOptions) + cmd.Flags().BoolVarP(&o.debug, "debug", "", false, "set debug mode") + cmd.Flags().BoolVarP(&o.deleteCRI, "delete-cri", "", false, "delete cri") +} + +func NewDeleteKubernetesCmd(streams genericclioptions.IOStreams) *cobra.Command { + o := &deleteOptions{ + clusterOptions: clusterOptions{ + IOStreams: streams, + }} + cmd := &cobra.Command{ + Use: "delete", + Short: "delete kubernetes cluster.", + Example: deleteExamples, + Run: func(cmd *cobra.Command, args []string) { + util.CheckErr(o.Complete()) + util.CheckErr(o.Validate()) + util.CheckErr(o.Run()) + }, + } + o.buildDeleteInfraFlags(cmd) + return cmd +} diff --git a/cmd/infrastructure/infras.go b/cmd/infrastructure/infras.go new file mode 100644 index 00000000000..f47465f4ee3 --- /dev/null +++ b/cmd/infrastructure/infras.go @@ -0,0 +1,36 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package infrastructure + +import ( + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" +) + +// NewInfraCmd for builder functions +func NewInfraCmd(streams genericclioptions.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "infra", + Short: "infra command", + } + cmd.AddCommand(NewCreateKubernetesCmd(streams)) + cmd.AddCommand(NewDeleteKubernetesCmd(streams)) + return cmd +} diff --git a/cmd/infrastructure/infras_test.go b/cmd/infrastructure/infras_test.go new file mode 100644 index 00000000000..e7307b7e189 --- /dev/null +++ b/cmd/infrastructure/infras_test.go @@ -0,0 +1,43 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package infrastructure + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/cli-runtime/pkg/genericclioptions" +) + +var _ = Describe("kubeblock infra", func() { + var streams genericclioptions.IOStreams + + BeforeEach(func() { + streams, _, _, _ = genericclioptions.NewTestIOStreams() + }) + + AfterEach(func() { + }) + + It("command should succeed", func() { + cmd := NewInfraCmd(streams) + Expect(cmd).ShouldNot(BeNil()) + }) +}) diff --git a/cmd/infrastructure/loader_adapter.go b/cmd/infrastructure/loader_adapter.go new file mode 100644 index 00000000000..fe2149106b0 --- /dev/null +++ b/cmd/infrastructure/loader_adapter.go @@ -0,0 +1,72 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package infrastructure + +import ( + kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" + + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/builder" + "github.com/apecloud/kubeblocks/internal/gotemplate" +) + +var ReplicaSetSignature = func(_ kubekeyapiv1alpha2.Cluster, _ any) {} + +func createClusterWithOptions(values *gotemplate.TplValues) (*kubekeyapiv1alpha2.ClusterSpec, error) { + const tplFile = "kubekey_cluster.tpl" + rendered, err := builder.BuildFromTemplate(values, tplFile) + if err != nil { + return nil, err + } + + cluster, err := builder.BuildResourceFromYaml(kubekeyapiv1alpha2.Cluster{}, rendered) + if err != nil { + return nil, err + } + return &cluster.Spec, nil +} + +const ( + builtinClusterNameObject = "Name" + builtinTimeoutObject = "Timeout" + builtinClusterVersionObject = "Version" + builtinCRITypeObject = "CRIType" + builtinUserObject = "User" + builtinHostsObject = "Hosts" + builtinRoleGroupsObject = "RoleGroups" + builtinKubernetesObject = "Kubernetes" +) + +func buildTemplateParams(o *createOptions) *gotemplate.TplValues { + return &gotemplate.TplValues{ + builtinClusterNameObject: o.clusterName, + builtinClusterVersionObject: o.version.KubernetesVersion, + builtinCRITypeObject: o.criType, + builtinUserObject: o.User, + builtinHostsObject: o.Nodes, + builtinTimeoutObject: o.timeout, + builtinKubernetesObject: o.Cluster.Kubernetes, + builtinRoleGroupsObject: gotemplate.TplValues{ + common.ETCD: o.RoleGroup.ETCD, + common.Master: o.RoleGroup.Master, + common.Worker: o.RoleGroup.Worker, + }, + } +} diff --git a/cmd/infrastructure/loader_adapter_test.go b/cmd/infrastructure/loader_adapter_test.go new file mode 100644 index 00000000000..144d1265950 --- /dev/null +++ b/cmd/infrastructure/loader_adapter_test.go @@ -0,0 +1,82 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package infrastructure + +import ( + "testing" + + "github.com/apecloud/kubeblocks/internal/configuration/container" + kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" + + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/types" +) + +func TestCreateClusterWithOptions(t *testing.T) { + tests := []struct { + name string + args *createOptions + want *kubekeyapiv1alpha2.ClusterSpec + wantErr bool + }{{ + name: "generateClusterTest", + args: &createOptions{ + version: types.InfraVersionInfo{}, + criType: string(container.ContainerdType), + clusterOptions: clusterOptions{ + clusterName: "for_test", + Cluster: types.Cluster{ + Nodes: []types.ClusterNode{ + { + Name: "node1", + Address: "127.0.0.1", + InternalAddress: "127.0.0.1", + }, { + Name: "node2", + Address: "127.0.0.2", + InternalAddress: "127.0.0.2", + }, + }, + User: types.ClusterUser{ + Name: "test", + Password: "test", + }, + RoleGroup: types.RoleGroup{ + ETCD: []string{"node1"}, + Master: []string{"node1"}, + Worker: []string{"node2"}, + }, + }, + }, + }, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.args.Cluster.Kubernetes.AutoDefaultFill() + got, err := createClusterWithOptions(buildTemplateParams(tt.args)) + if (err != nil) != tt.wantErr { + t.Errorf("createClusterWithOptions() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got == nil { + t.Errorf("createClusterWithOptions() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/infrastructure/pipeline.go b/cmd/infrastructure/pipeline.go new file mode 100644 index 00000000000..810391e5e6b --- /dev/null +++ b/cmd/infrastructure/pipeline.go @@ -0,0 +1,74 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package infrastructure + +import ( + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/certs" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/etcd" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/dns" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/network" + + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/tasks" +) + +func NewCreatePipeline(o *createOptions) []module.Module { + return []module.Module{ + &precheck.GreetingsModule{}, + &tasks.CheckNodeArchitectureModule{}, + &precheck.NodePreCheckModule{}, + &tasks.InstallDependenciesModule{}, + &tasks.PrepareK8sBinariesModule{BinaryVersion: o.version}, + &tasks.ConfigureNodeOSModule{Nodes: o.Nodes}, + &kubernetes.StatusModule{}, + &tasks.InstallCRIModule{SandBoxImage: o.Cluster.Kubernetes.CRI.SandBoxImage}, + &etcd.PreCheckModule{}, + &etcd.CertsModule{}, + &etcd.InstallETCDBinaryModule{}, + &etcd.ConfigureModule{}, + &etcd.BackupModule{}, + &kubernetes.InstallKubeBinariesModule{}, + &kubernetes.InitKubernetesModule{}, + &dns.ClusterDNSModule{}, + &kubernetes.StatusModule{}, + &tasks.SaveKubeConfigModule{OutputKubeconfig: o.outputKubeconfig}, + &kubernetes.JoinNodesModule{}, + &network.DeployNetworkPluginModule{}, + &kubernetes.ConfigureKubernetesModule{}, + &filesystem.ChownModule{}, + &kubernetes.SecurityEnhancementModule{Skip: !o.securityEnhancement}, + &tasks.AddonsInstaller{Addons: o.Addons, Kubeconfig: o.outputKubeconfig}, + } +} + +func NewDeletePipeline(o *deleteOptions) []module.Module { + return []module.Module{ + &precheck.GreetingsModule{}, + &kubernetes.ResetClusterModule{}, + &container.UninstallContainerModule{Skip: !o.deleteCRI}, + &os.ClearOSEnvironmentModule{}, + &certs.UninstallAutoRenewCertsModule{}, + } +} diff --git a/cmd/infrastructure/suit_test.go b/cmd/infrastructure/suit_test.go new file mode 100644 index 00000000000..10f1ace770b --- /dev/null +++ b/cmd/infrastructure/suit_test.go @@ -0,0 +1,32 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package infrastructure + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestAppp(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Infra Cmd Test Suite") +} diff --git a/cmd/infrastructure/tasks/addon.go b/cmd/infrastructure/tasks/addon.go new file mode 100644 index 00000000000..6b7c1bcbbb0 --- /dev/null +++ b/cmd/infrastructure/tasks/addon.go @@ -0,0 +1,73 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package tasks + +import ( + cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" + + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/types" + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/utils" +) + +type AddonsInstaller struct { + common.KubeModule + + Addons []types.PluginMeta + Kubeconfig string +} + +type KBAddonsInstall struct { + common.KubeAction + + Addons []types.PluginMeta + Kubeconfig string +} + +func (a *AddonsInstaller) Init() { + a.Name = "AddonsInstaller" + a.Desc = "Install helm addons" + a.Tasks = []task.Interface{ + &task.LocalTask{ + Name: "AddonsInstaller", + Desc: "Install helm addons", + Action: &KBAddonsInstall{Addons: a.Addons, Kubeconfig: a.Kubeconfig}, + }} +} + +func (i *KBAddonsInstall) Execute(runtime connector.Runtime) error { + var installer utils.Installer + for _, addon := range i.Addons { + switch { + case addon.Sources.Chart != nil: + installer = utils.NewHelmInstaller(*addon.Sources.Chart, i.Kubeconfig) + case addon.Sources.Yaml != nil: + installer = utils.NewYamlInstaller(*addon.Sources.Yaml, i.Kubeconfig) + default: + return cfgcore.MakeError("addon source not supported: addon: %v", addon) + } + if err := installer.Install(addon.Name, addon.Namespace); err != nil { + return err + } + } + return nil +} diff --git a/cmd/infrastructure/tasks/dependent.go b/cmd/infrastructure/tasks/dependent.go new file mode 100644 index 00000000000..30204fa604d --- /dev/null +++ b/cmd/infrastructure/tasks/dependent.go @@ -0,0 +1,217 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package tasks + +import ( + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/confirm" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container/templates" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/prepare" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" + "github.com/mitchellh/mapstructure" + + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/builder" + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/constant" + "github.com/apecloud/kubeblocks/internal/gotemplate" +) + +type InstallDependenciesModule struct { + common.KubeModule +} + +func (i *InstallDependenciesModule) isNotReadyHosts() []connector.Host { + hosts := make([]connector.Host, 0) + for _, host := range i.Runtime.GetAllHosts() { + var result confirm.PreCheckResults + if v, ok := host.GetCache().Get(common.NodePreCheck); ok { + _ = mapstructure.Decode(v, &result) + if result.Socat == "" || + result.Conntrack == "" || + result.Curl == "" || + result.Ipset == "" || + result.Chronyd == "" || + result.Ipvsadm == "" || + result.Ebtables == "" { + hosts = append(hosts, host) + } + } + } + return hosts +} + +// Init install dependencies module +func (i *InstallDependenciesModule) Init() { + i.Name = "InstallDependenciesModule" + i.Desc = "install dependencies" + + hosts := i.isNotReadyHosts() + if len(hosts) == 0 { + logger.Log.Info("All hosts are ready, skip install dependencies") + return + } + + i.Tasks = []task.Interface{ + &task.RemoteTask{ + Name: "GetOSData", + Desc: "Get OS release", + Hosts: i.Runtime.GetAllHosts(), + Action: new(os.GetOSData), + Parallel: true, + }, + &task.RemoteTask{ + Name: "InstallDependenciesModule", + Desc: "check and install dependencies", + Hosts: hosts, + Action: new(InstallDependenciesTask), + Parallel: true, + }, + } +} + +type CheckNodeArchitectureModule struct { + common.KubeModule +} + +// Init install dependencies module +func (i *CheckNodeArchitectureModule) Init() { + i.Name = "CheckNodeArch" + i.Desc = "check and update host arch" + i.Tasks = []task.Interface{ + &task.RemoteTask{ + Name: "CheckNodeArch", + Desc: "check and update node arch", + Hosts: i.Runtime.GetAllHosts(), + Action: new(UpdateNodeTask), + Parallel: true, + }, + } +} + +type InstallCRIModule struct { + common.KubeModule + SandBoxImage string +} + +func (i *InstallCRIModule) Init() { + i.Name = "InstallContainerModule" + i.Desc = "Install container manager" + + syncContainerd := &task.RemoteTask{ + Name: "SyncContainerd", + Desc: "Sync containerd binaries", + Hosts: i.Runtime.GetHostsByRole(common.K8s), + Prepare: &prepare.PrepareCollection{ + &kubernetes.NodeInCluster{Not: true}, + &container.ContainerdExist{Not: true}, + }, + Action: new(container.SyncContainerd), + Parallel: true, + Retry: 2, + } + + syncCrictlBinaries := &task.RemoteTask{ + Name: "SyncCrictlBinaries", + Desc: "Sync crictl binaries", + Hosts: i.Runtime.GetHostsByRole(common.K8s), + Prepare: &prepare.PrepareCollection{ + &kubernetes.NodeInCluster{Not: true}, + &container.CrictlExist{Not: true}, + }, + Action: new(container.SyncCrictlBinaries), + Parallel: true, + Retry: 2, + } + + generateContainerdService := &task.RemoteTask{ + Name: "GenerateContainerdService", + Desc: "Generate containerd service", + Hosts: i.Runtime.GetHostsByRole(common.K8s), + Prepare: &prepare.PrepareCollection{ + &kubernetes.NodeInCluster{Not: true}, + &container.ContainerdExist{Not: true}, + }, + Action: &builder.Template{ + Template: constant.ContainerdService, + Dst: constant.ContainerdServiceInstallPath, + }, + Parallel: true, + } + + generateContainerdConfig := &task.RemoteTask{ + Name: "GenerateContainerdConfig", + Desc: "Generate containerd config", + Hosts: i.Runtime.GetHostsByRole(common.K8s), + Prepare: &prepare.PrepareCollection{ + &kubernetes.NodeInCluster{Not: true}, + &container.ContainerdExist{Not: true}, + }, + Action: &builder.Template{ + Template: constant.ContainerdConfig, + Dst: constant.ContainerdConfigInstallPath, + Values: gotemplate.TplValues{ + "SandBoxImage": i.SandBoxImage, + "DataRoot": templates.DataRoot(i.KubeConf), + }}, + Parallel: true, + } + + generateCrictlConfig := &task.RemoteTask{ + Name: "GenerateCrictlConfig", + Desc: "Generate crictl config", + Hosts: i.Runtime.GetHostsByRole(common.K8s), + Prepare: &prepare.PrepareCollection{ + &kubernetes.NodeInCluster{Not: true}, + &container.ContainerdExist{Not: true}, + }, + Action: &builder.Template{ + Template: constant.CRICtlConfig, + Dst: constant.CRICtlConfigInstallPath, + Values: gotemplate.TplValues{ + "Endpoint": i.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, + }}, + Parallel: true, + } + + enableContainerd := &task.RemoteTask{ + Name: "EnableContainerd", + Desc: "Enable containerd", + Hosts: i.Runtime.GetHostsByRole(common.K8s), + Prepare: &prepare.PrepareCollection{ + &kubernetes.NodeInCluster{Not: true}, + &container.ContainerdExist{Not: true}, + }, + Action: new(container.EnableContainerd), + Parallel: true, + } + + i.Tasks = []task.Interface{ + syncContainerd, + syncCrictlBinaries, + generateContainerdService, + generateContainerdConfig, + generateCrictlConfig, + enableContainerd, + } +} diff --git a/cmd/infrastructure/tasks/download.go b/cmd/infrastructure/tasks/download.go new file mode 100644 index 00000000000..83b6cc8aada --- /dev/null +++ b/cmd/infrastructure/tasks/download.go @@ -0,0 +1,98 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package tasks + +import ( + "fmt" + "os" + "os/exec" + + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" + + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/types" + kbutils "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/utils" + cfgcore "github.com/apecloud/kubeblocks/internal/configuration" +) + +const ( + CurlDownloadURLFormat = "curl -L -o %s %s" + WgetDownloadURLFormat = "wget -O %s %s" + + defaultDownloadURL = CurlDownloadURLFormat +) + +func downloadKubernetesBinaryWithArch(downloadPath string, arch string, binaryVersion types.InfraVersionInfo) (map[string]*files.KubeBinary, error) { + downloadCommand := func(path, url string) string { + return fmt.Sprintf(defaultDownloadURL, path, url) + } + + binaries := []*files.KubeBinary{ + files.NewKubeBinary("etcd", arch, binaryVersion.EtcdVersion, downloadPath, downloadCommand), + files.NewKubeBinary("kubeadm", arch, binaryVersion.KubernetesVersion, downloadPath, downloadCommand), + files.NewKubeBinary("kubelet", arch, binaryVersion.KubernetesVersion, downloadPath, downloadCommand), + files.NewKubeBinary("kubectl", arch, binaryVersion.KubernetesVersion, downloadPath, downloadCommand), + files.NewKubeBinary("kubecni", arch, binaryVersion.CniVersion, downloadPath, downloadCommand), + files.NewKubeBinary("helm", arch, binaryVersion.HelmVersion, downloadPath, downloadCommand), + // for containerd + files.NewKubeBinary("crictl", arch, binaryVersion.CRICtlVersion, downloadPath, downloadCommand), + files.NewKubeBinary("containerd", arch, binaryVersion.ContainerVersion, downloadPath, downloadCommand), + files.NewKubeBinary("runc", arch, binaryVersion.RuncVersion, downloadPath, downloadCommand), + } + + binariesMap := make(map[string]*files.KubeBinary) + for _, binary := range binaries { + if err := binary.CreateBaseDir(); err != nil { + return nil, cfgcore.WrapError(err, "failed to create file %s base dir.", binary.FileName) + } + logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version) + binariesMap[binary.ID] = binary + if checkDownloadBinary(binary) { + continue + } + if err := download(binary); err != nil { + return nil, cfgcore.WrapError(err, "failed to download %s binary: %s", binary.ID, binary.GetCmd()) + } + } + return binariesMap, nil +} + +func checkDownloadBinary(binary *files.KubeBinary) bool { + if !util.IsExist(binary.Path()) { + return false + } + err := kbutils.CheckSha256sum(binary) + if err != nil { + logger.Log.Messagef(common.LocalHost, "failed to check %s sha256, error: %v", binary.ID, err) + _ = os.Remove(binary.Path()) + return false + } + logger.Log.Messagef(common.LocalHost, "%s is existed", binary.ID) + return true +} + +func download(binary *files.KubeBinary) error { + if err := kbutils.RunCommand(exec.Command("/bin/sh", "-c", binary.GetCmd())); err != nil { + return err + } + return kbutils.WriteSha256sum(binary) +} diff --git a/cmd/infrastructure/tasks/kubeconfig.go b/cmd/infrastructure/tasks/kubeconfig.go new file mode 100644 index 00000000000..2776c6bd052 --- /dev/null +++ b/cmd/infrastructure/tasks/kubeconfig.go @@ -0,0 +1,89 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package tasks + +import ( + "fmt" + "os" + "time" + + "github.com/apecloud/kubeblocks/internal/cli/util" + cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +func kubeconfigMerge(newKubeConfig *clientcmdapi.Config, existingKubeConfig *clientcmdapi.Config, outPath string) error { + // merge clusters + for k, v := range newKubeConfig.Clusters { + if _, ok := existingKubeConfig.Clusters[k]; ok { + return cfgcore.MakeError("Cluster '%s' already exists in target KubeConfig", k) + } + existingKubeConfig.Clusters[k] = v + } + + // merge auth + for k, v := range newKubeConfig.AuthInfos { + if _, ok := existingKubeConfig.AuthInfos[k]; ok { + return cfgcore.MakeError("AuthInfo '%s' already exists in target KubeConfig", k) + } + existingKubeConfig.AuthInfos[k] = v + } + + // merge contexts + for k, v := range newKubeConfig.Contexts { + if _, ok := existingKubeConfig.Contexts[k]; ok { + return cfgcore.MakeError("Context '%s' already exists in target KubeConfig", k) + } + existingKubeConfig.Contexts[k] = v + } + + existingKubeConfig.CurrentContext = newKubeConfig.CurrentContext + return kubeconfigWrite(existingKubeConfig, outPath) +} + +func kubeconfigWrite(config *clientcmdapi.Config, path string) error { + tempPath := fmt.Sprintf("%s.kb_%s", path, time.Now().Format("20060102_150405.000000")) + if err := clientcmd.WriteToFile(*config, tempPath); err != nil { + return cfgcore.WrapError(err, "failed to write merged kubeconfig to temporary file '%s'", tempPath) + } + + // Move temporary file over existing KubeConfig + if err := os.Rename(tempPath, path); err != nil { + return cfgcore.WrapError(err, "failed to overwrite existing KubeConfig '%s' with new kubeconfig '%s'", path, tempPath) + } + return nil +} + +func GetDefaultConfig() string { + defaultKubeConfigLoadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + kcFile := defaultKubeConfigLoadingRules.GetDefaultFilename() + if kcFile == "" { + kcFile = util.GetKubeconfigDir() + } + return kcFile +} + +func kubeconfigLoad(kcFile string) (*clientcmdapi.Config, error) { + if _, err := os.Stat(kcFile); err == nil { + return clientcmd.LoadFromFile(kcFile) + } + return nil, nil +} diff --git a/cmd/infrastructure/tasks/kubernetes.go b/cmd/infrastructure/tasks/kubernetes.go new file mode 100644 index 00000000000..8c14780308a --- /dev/null +++ b/cmd/infrastructure/tasks/kubernetes.go @@ -0,0 +1,217 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package tasks + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/StudioSol/set" + kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os/templates" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/task" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/builder" + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/constant" + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/types" + cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + "github.com/apecloud/kubeblocks/internal/gotemplate" +) + +type PrepareK8sBinariesModule struct { + common.KubeModule + + // kubernetes version + BinaryVersion types.InfraVersionInfo +} + +type ConfigureNodeOSModule struct { + common.KubeModule + Nodes []types.ClusterNode +} + +type SaveKubeConfigModule struct { + common.KubeModule + + OutputKubeconfig string +} + +func (p *PrepareK8sBinariesModule) Init() { + p.Name = "PrepareK8sBinariesModule" + p.Desc = "Download installation binaries for kubernetes" + + p.Tasks = []task.Interface{ + &task.LocalTask{ + Name: "PrepareK8sBinaries", + Desc: "Download installation binaries", + Action: &DownloadKubernetesBinary{BinaryVersion: p.BinaryVersion}, + }} +} + +func (c *ConfigureNodeOSModule) Init() { + c.Name = "ConfigureNodeOSModule" + c.Desc = "Init os dependencies" + c.Tasks = []task.Interface{ + &task.RemoteTask{ + Name: "GetOSData", + Desc: "Get OS release", + Hosts: c.Runtime.GetAllHosts(), + Action: new(os.GetOSData), + Parallel: true, + }, + &task.RemoteTask{ + Name: "SetHostName", + Desc: "Prepare to init OS", + Hosts: c.Runtime.GetAllHosts(), + Action: new(os.NodeConfigureOS), + Parallel: true, + }, + &task.RemoteTask{ + Name: "GenerateScript", + Desc: "Generate init os script", + Hosts: c.Runtime.GetAllHosts(), + Action: &NodeScriptGenerator{ + Hosts: templates.GenerateHosts(c.Runtime, c.KubeConf), + Nodes: c.Nodes, + }, + Parallel: true, + }, + &task.RemoteTask{ + Name: "ExecScript", + Desc: "Exec init os script", + Hosts: c.Runtime.GetAllHosts(), + Action: new(os.NodeExecScript), + Parallel: true, + }} +} + +func (p *SaveKubeConfigModule) Init() { + p.Name = "SaveKubeConfigModule" + p.Desc = "Save kube config to local file" + + p.Tasks = []task.Interface{ + &task.LocalTask{ + Name: "SaveKubeConfig", + Desc: "Save kube config to local file", + Action: &SaveKubeConfig{outputKubeconfig: p.OutputKubeconfig}, + }} +} + +type DownloadKubernetesBinary struct { + common.KubeAction + BinaryVersion types.InfraVersionInfo +} + +func (d *DownloadKubernetesBinary) Execute(runtime connector.Runtime) error { + archSet := set.NewLinkedHashSetString() + for _, host := range runtime.GetAllHosts() { + archSet.Add(host.GetArch()) + } + + for _, arch := range archSet.AsSlice() { + binariesMap, err := downloadKubernetesBinaryWithArch(runtime.GetWorkDir(), arch, d.BinaryVersion) + if err != nil { + return err + } + d.PipelineCache.Set(common.KubeBinaries+"-"+arch, binariesMap) + } + return nil +} + +type SaveKubeConfig struct { + common.KubeAction + + outputKubeconfig string +} + +func (c *SaveKubeConfig) Execute(runtime connector.Runtime) error { + master := runtime.GetHostsByRole(common.Master)[0] + + status, ok := c.PipelineCache.Get(common.ClusterStatus) + if !ok { + return cfgcore.MakeError("failed to get kubernetes status.") + } + cluster := status.(*kubernetes.KubernetesStatus) + kubeConfigStr := cluster.KubeConfig + kc, err := clientcmd.Load([]byte(kubeConfigStr)) + if err != nil { + return err + } + updateClusterAPIServer(kc, master, c.KubeConf.Cluster.ControlPlaneEndpoint) + kcFile := GetDefaultConfig() + existingKC, err := kubeconfigLoad(kcFile) + if err != nil { + return err + } + if c.outputKubeconfig == "" { + c.outputKubeconfig = kcFile + } + if existingKC != nil { + return kubeconfigMerge(kc, existingKC, c.outputKubeconfig) + } + return kubeconfigWrite(kc, c.outputKubeconfig) +} + +type NodeScriptGenerator struct { + common.KubeAction + + Nodes []types.ClusterNode + Hosts []string +} + +func (c *NodeScriptGenerator) Execute(runtime connector.Runtime) error { + foundHostOptions := func(nodes []types.ClusterNode, host connector.Host) types.NodeOptions { + for _, node := range nodes { + switch { + default: + return types.NodeOptions{} + case node.Name != host.GetName(): + case node.NodeOptions != nil: + return *node.NodeOptions + } + } + return types.NodeOptions{} + } + + scriptsTemplate := builder.Template{ + Template: constant.ConfigureOSScripts, + Dst: filepath.Join(common.KubeScriptDir, "initOS.sh"), + Values: gotemplate.TplValues{ + "Hosts": c.Hosts, + "Options": foundHostOptions(c.Nodes, runtime.RemoteHost()), + }} + return scriptsTemplate.Execute(runtime) +} + +func updateClusterAPIServer(kc *clientcmdapi.Config, master connector.Host, endpoint kubekeyapiv1alpha2.ControlPlaneEndpoint) { + cpePrefix := fmt.Sprintf("https://%s:", endpoint.Domain) + for _, cluster := range kc.Clusters { + if strings.HasPrefix(cluster.Server, cpePrefix) { + cluster.Server = fmt.Sprintf("https://%s:%d", master.GetAddress(), endpoint.Port) + } + } +} diff --git a/cmd/infrastructure/tasks/kubernetes_test.go b/cmd/infrastructure/tasks/kubernetes_test.go new file mode 100644 index 00000000000..c26042f49b6 --- /dev/null +++ b/cmd/infrastructure/tasks/kubernetes_test.go @@ -0,0 +1,20 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package tasks diff --git a/cmd/infrastructure/tasks/pipeline_runner.go b/cmd/infrastructure/tasks/pipeline_runner.go new file mode 100644 index 00000000000..d4200c571cd --- /dev/null +++ b/cmd/infrastructure/tasks/pipeline_runner.go @@ -0,0 +1,103 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package tasks + +import ( + "fmt" + "io" + + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/ending" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline" + + cfgcore "github.com/apecloud/kubeblocks/internal/configuration" +) + +type PipelineWrapper struct { + pipeline.Pipeline +} + +func NewPipelineRunner(name string, modules []module.Module, runtime connector.Runtime) *PipelineWrapper { + return &PipelineWrapper{ + Pipeline: pipeline.Pipeline{ + Name: name, + Modules: modules, + Runtime: runtime, + PipelineCache: cache.NewCache(), + SpecHosts: len(runtime.GetAllHosts()), + }, + } +} + +func (w *PipelineWrapper) Do(output io.Writer) error { + defer func() { + w.PipelineCache.Clean() + w.releaseHostsConnector() + }() + + for i := range w.Modules { + m := w.Modules[i] + if m.IsSkip() { + continue + } + if res := w.safeRunModule(m); res.IsFailed() { + return cfgcore.WrapError(res.CombineResult, "failed to execute module: %s", getModuleName(m)) + } + } + fmt.Fprintf(output, "succeed to execute all modules in the pipeline[%s]", w.Name) + return nil +} + +func (w *PipelineWrapper) safeRunModule(m module.Module) *ending.ModuleResult { + newCache := func() *cache.Cache { + if moduleCache, ok := w.ModuleCachePool.Get().(*cache.Cache); ok { + return moduleCache + } + return cache.NewCache() + } + releaseCache := func(cache *cache.Cache) { + cache.Clean() + w.ModuleCachePool.Put(cache) + } + + moduleCache := newCache() + defer releaseCache(moduleCache) + m.Default(w.Runtime, w.PipelineCache, moduleCache) + m.AutoAssert() + m.Init() + return w.RunModule(m) +} + +func (w *PipelineWrapper) releaseHostsConnector() { + for _, host := range w.Runtime.GetAllHosts() { + if connector := w.Runtime.GetConnector(); connector != nil { + connector.Close(host) + } + } +} + +func getModuleName(m module.Module) string { + if b, ok := m.(*module.BaseModule); ok { + return b.Name + } + return "" +} diff --git a/cmd/infrastructure/tasks/tasks.go b/cmd/infrastructure/tasks/tasks.go new file mode 100644 index 00000000000..4cc6a41ef9f --- /dev/null +++ b/cmd/infrastructure/tasks/tasks.go @@ -0,0 +1,132 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package tasks + +import ( + "fmt" + "strings" + + kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" + "github.com/kubesphere/kubekey/v3/util/osrelease" + + cfgcore "github.com/apecloud/kubeblocks/internal/configuration" +) + +type InstallDependenciesTask struct { + common.KubeAction + pkg []string +} + +var dependenciesPkg = []string{"socat", "conntrack", "ipset", "ebtables", "chrony", "iptables", "curl", "ipvsadm"} + +func (i *InstallDependenciesTask) Execute(runtime connector.Runtime) (err error) { + host := runtime.RemoteHost() + release, ok := host.GetCache().Get(os.Release) + if !ok { + return cfgcore.MakeError("failed to get os release.") + } + + r := release.(*osrelease.Data) + installCommand, err := checkRepositoryInstallerCommand(r, runtime) + if err != nil { + return err + } + depPkg := strings.Join(append(i.pkg, dependenciesPkg...), " ") + stdout, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("%s %s", installCommand, depPkg), true) + logger.Log.Info(stdout) + return err +} + +type UpdateNodeTask struct { + common.KubeAction +} + +func (i *UpdateNodeTask) Execute(runtime connector.Runtime) (err error) { + host := runtime.RemoteHost() + if host.GetArch() != "" { + return nil + } + + stdout, err := runtime.GetRunner().Cmd("uname -m", false) + if err != nil { + return err + } + host.SetArch(parseNodeArchitecture(stdout)) + updateClusterSpecHost(i.KubeConf.Cluster, host) + return nil +} + +func updateClusterSpecHost(clusterSpec *kubekeyapiv1alpha2.ClusterSpec, host connector.Host) { + for i := range clusterSpec.Hosts { + h := &clusterSpec.Hosts[i] + if h.Name == host.GetName() { + h.Arch = host.GetArch() + } + } +} + +func parseNodeArchitecture(stdout string) string { + switch strings.TrimSpace(stdout) { + default: + return "amd64" + case "x86_64": + return "amd64" + case "arm64", "arm": + return "arm64" + case "aarch64", "aarch32": + return "arm" + } +} + +func checkRepositoryInstallerCommand(osData *osrelease.Data, runtime connector.Runtime) (string, error) { + const ( + debianCommand = "apt install -y" + rhelCommand = "yum install -y" + ) + + isDebianCore := func() bool { + checkDeb, err := runtime.GetRunner().SudoCmd("which apt", false) + return err == nil && strings.Contains(checkDeb, "bin") + } + isRhelCore := func() bool { + checkDeb, err := runtime.GetRunner().SudoCmd("which yum", false) + return err == nil && strings.Contains(checkDeb, "bin") + } + + switch strings.ToLower(osData.ID) { + case "ubuntu", "debian": + return debianCommand, nil + case "centos", "rhel": + return rhelCommand, nil + } + + switch { + default: + return "", cfgcore.MakeError("failed to check apt or yum.") + case isRhelCore(): + return rhelCommand, nil + case isDebianCore(): + return debianCommand, nil + } +} diff --git a/cmd/infrastructure/types/features.go b/cmd/infrastructure/types/features.go new file mode 100644 index 00000000000..e79d61063f0 --- /dev/null +++ b/cmd/infrastructure/types/features.go @@ -0,0 +1,27 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package types + +type HugePageFeature struct { + HugePageSize string `json:"hugePageSize"` + + // HugeSizeWithUnit is the huge page size with unit, such as 2Mi, 1Gi. + HugeSizeWithUnit string `json:"hugeSizeWithUnit"` +} diff --git a/cmd/infrastructure/types/type.go b/cmd/infrastructure/types/type.go new file mode 100644 index 00000000000..e73ebd1aaeb --- /dev/null +++ b/cmd/infrastructure/types/type.go @@ -0,0 +1,215 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package types + +import ( + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/constant" + kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2" + "helm.sh/helm/v3/pkg/cli/values" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type InfraVersionInfo struct { + KubernetesVersion string `json:"kubernetesVersion"` + EtcdVersion string `json:"etcdVersion"` + ContainerVersion string `json:"containerVersion"` + CRICtlVersion string `json:"crictlVersion"` + RuncVersion string `json:"runcVersion"` + CniVersion string `json:"cniVersion"` + HelmVersion string `json:"helmVersion"` +} + +type PluginMeta struct { + Name string `json:"name,omitempty"` + Namespace string `json:"namespace,omitempty"` + Sources PluginSources `json:"sources,omitempty"` +} + +type PluginSources struct { + Chart *HelmChart `json:"chart"` + Yaml *Yaml `json:"yaml"` +} + +type HelmChart struct { + Name string `json:"name"` + Version string `json:"version"` + Repo string `json:"repo"` + Path string `json:"path"` + ValueOptions values.Options `json:"options"` +} + +type Yaml struct { + Path []string `json:"path,omitempty"` +} + +type Cluster struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + User ClusterUser `json:"user"` + Nodes []ClusterNode `json:"nodes"` + + RoleGroup RoleGroup `json:"roleGroup"` + Addons []PluginMeta `json:"addons"` + + // for kubeadm configuration + Kubernetes Kubernetes `json:"kubernetes"` + + Version InfraVersionInfo `json:"version"` +} + +type RoleGroup struct { + ETCD []string `json:"etcd"` + Master []string `json:"master"` + Worker []string `json:"worker"` +} + +type ClusterNode struct { + Name string `json:"name"` + Address string `json:"address"` + InternalAddress string `json:"internalAddress"` + NodeOptions *NodeOptions `json:"options"` +} + +type NodeOptions struct { + HugePageFeature *HugePageFeature `json:"hugePageFeature"` +} + +type Kubernetes struct { + // ClusterName string `json:"clusterName"` + // DNSDomain string `json:"dnsDomain"` + ProxyMode string `json:"proxyMode"` + + Networking Networking `json:"networking"` + CRI ContainerRuntime `json:"cri"` + + ControlPlaneEndpoint ControlPlaneEndpoint `json:"controlPlaneEndpoint"` + APIServer APIServer `json:"apiServer"` + Scheduler Scheduler `json:"scheduler"` +} + +type Networking struct { + // using network plugin, default is calico + Plugin string `json:"plugin"` + + // apis/kubeadm/types.Networking + ServiceSubnet string `json:"serviceSubnet"` + PodSubnet string `json:"podSubnet"` + DNSDomain string `json:"dnsDomain"` +} + +type ContainerRuntime struct { + ContainerRuntimeType string `json:"containerRuntimeType"` + ContainerRuntimeEndpoint string `json:"containerRuntimeEndpoint"` + SandBoxImage string `json:"sandBoxImage"` +} + +type ControlPlaneComponent struct { + // apiserver extra args + ExtraArgs map[string]string `json:"extraArgs"` +} + +type APIServer struct { + ControlPlaneComponent `json:",inline"` +} + +type Scheduler struct { + ControlPlaneComponent `json:",inline"` +} + +type ControlPlaneEndpoint struct { + Domain string `json:"domain"` + Port int `json:"port"` + + // TODO support apiserver loadbalancer + LoadBalancer string `json:"loadBalancer"` +} + +type ClusterUser struct { + // user name + Name string `json:"name"` + // sudo password + Password string `json:"password"` + // ssh privateKey + PrivateKey string `json:"privateKey"` + PrivateKeyPath string `json:"privateKeyPath"` +} + +func (g *RoleGroup) IsValidate() bool { + return len(g.ETCD) > 0 && len(g.Master) > 0 && len(g.Worker) > 0 +} + +func (k *Kubernetes) AutoDefaultFill() { + // if k.ClusterName == "" { + // k.ClusterName = constant.DefaultK8sClusterName + // } + // if k.DNSDomain == "" { + // k.DNSDomain = constant.DefaultK8sDNSDomain + // } + if k.ProxyMode == "" { + k.ProxyMode = constant.DefaultK8sProxyMode + } + + fillNetworkField(&k.Networking) + fillContainerRuntimeField(&k.CRI) + fillAPIServerField(&k.APIServer) + fillSchedulerField(&k.Scheduler) + fillControlPlaneField(&k.ControlPlaneEndpoint) +} + +func fillContainerRuntimeField(c *ContainerRuntime) { + if c.ContainerRuntimeType == "" { + c.ContainerRuntimeType = kubekeyapiv1alpha2.Conatinerd + c.ContainerRuntimeEndpoint = kubekeyapiv1alpha2.DefaultContainerdEndpoint + } + if c.SandBoxImage == "" { + c.SandBoxImage = constant.DefaultSandBoxImage + } +} + +func fillControlPlaneField(c *ControlPlaneEndpoint) { + if c.Port == 0 { + c.Port = constant.DefaultAPIServerPort + } + if c.Domain == "" { + c.Domain = constant.DefaultAPIDNSDomain + } +} + +func fillSchedulerField(s *Scheduler) { +} + +func fillAPIServerField(a *APIServer) { +} + +func fillNetworkField(n *Networking) { + if n.Plugin == "" { + n.Plugin = constant.DefaultNetworkPlugin + } + if n.DNSDomain == "" { + n.DNSDomain = constant.DefaultK8sDNSDomain + } + if n.ServiceSubnet == "" { + n.ServiceSubnet = kubekeyapiv1alpha2.DefaultServiceCIDR + } + if n.PodSubnet == "" { + n.PodSubnet = kubekeyapiv1alpha2.DefaultPodsCIDR + } +} diff --git a/cmd/infrastructure/utils/checksum.go b/cmd/infrastructure/utils/checksum.go new file mode 100644 index 00000000000..3305293198e --- /dev/null +++ b/cmd/infrastructure/utils/checksum.go @@ -0,0 +1,81 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package utils + +import ( + "crypto/sha256" + "fmt" + "io" + "os" + "strings" + + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files" + + cfgcore "github.com/apecloud/kubeblocks/internal/configuration" +) + +func getSha256sumFile(binary *files.KubeBinary) string { + return fmt.Sprintf("%s.sum.%s", binary.Path(), "sha256") +} + +func CheckSha256sum(binary *files.KubeBinary) error { + checksumFile := getSha256sumFile(binary) + if !util.IsExist(checksumFile) { + return cfgcore.MakeError("checksum file %s is not exist", checksumFile) + } + + checksum, err := calSha256sum(binary.Path()) + if err != nil { + return err + } + + data, err := os.ReadFile(checksumFile) + if err != nil { + return err + } + if strings.TrimSpace(string(data)) == checksum { + return nil + } + return cfgcore.MakeError("checksum of %s is not match, [%s] vs [%s]", binary.ID, string(data), checksum) +} + +func calSha256sum(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + + data, err := io.ReadAll(file) + if err != nil { + return "", err + } + return fmt.Sprintf("%x", sha256.Sum256(data)), nil +} + +func WriteSha256sum(binary *files.KubeBinary) error { + checksumFile := getSha256sumFile(binary) + sum, err := calSha256sum(binary.Path()) + if err != nil { + return err + } + return util.WriteFile(checksumFile, []byte(sum)) +} diff --git a/cmd/infrastructure/utils/command_wrapper.go b/cmd/infrastructure/utils/command_wrapper.go new file mode 100644 index 00000000000..a732a47621c --- /dev/null +++ b/cmd/infrastructure/utils/command_wrapper.go @@ -0,0 +1,56 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package utils + +import ( + "errors" + "fmt" + "io" + "os/exec" + + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/common" + "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger" +) + +func RunCommand(cmd *exec.Cmd) error { + logger.Log.Messagef(common.LocalHost, "Running: %s", cmd.String()) + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + cmd.Stderr = cmd.Stdout + if err = cmd.Start(); err != nil { + return err + } + + // read from stdout + for { + tmp := make([]byte, 1024) + _, err := stdout.Read(tmp) + fmt.Print(string(tmp)) + if errors.Is(err, io.EOF) { + break + } else if err != nil { + logger.Log.Errorln(err) + break + } + } + return cmd.Wait() +} diff --git a/cmd/infrastructure/utils/helm_wrapper.go b/cmd/infrastructure/utils/helm_wrapper.go new file mode 100644 index 00000000000..0bbaa213fd0 --- /dev/null +++ b/cmd/infrastructure/utils/helm_wrapper.go @@ -0,0 +1,92 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package utils + +import ( + "fmt" + "path/filepath" + + "helm.sh/helm/v3/pkg/repo" + "k8s.io/klog/v2" + + "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/types" + "github.com/apecloud/kubeblocks/internal/cli/util/helm" +) + +type HelmInstallHelper struct { + types.HelmChart + kubeconfig string +} + +func (h *HelmInstallHelper) Install(name, ns string) error { + if err := h.addRepo(); err != nil { + return err + } + helmConfig := helm.NewConfig(ns, h.kubeconfig, "", klog.V(1).Enabled()) + installOpts := h.buildChart(name, ns) + output, err := installOpts.Install(helmConfig) + if err != nil && helm.ReleaseNotFound(err) { + fmt.Println(output) + return nil + } + return err +} + +func NewHelmInstaller(chart types.HelmChart, kubeconfig string) Installer { + installer := HelmInstallHelper{ + HelmChart: chart, + kubeconfig: kubeconfig} + return &installer +} + +func (h *HelmInstallHelper) buildChart(name, ns string) *helm.InstallOpts { + return &helm.InstallOpts{ + Name: name, + Chart: h.getChart(name), + Wait: true, + Version: h.Version, + Namespace: ns, + ValueOpts: &h.ValueOptions, + TryTimes: 3, + CreateNamespace: true, + Atomic: true, + } +} + +func (h *HelmInstallHelper) getChart(name string) string { + if h.Name == "" { + return "" + } + // install helm package form local path + if h.Repo != "" { + return filepath.Join(h.Name, name) + } + if h.Path != "" { + return filepath.Join(h.Path, name) + } + return "" +} + +func (h *HelmInstallHelper) addRepo() error { + if h.Repo == "" { + return nil + } + return helm.AddRepo(&repo.Entry{Name: h.Name, URL: h.Repo}) +} diff --git a/cmd/infrastructure/utils/install.go b/cmd/infrastructure/utils/install.go new file mode 100644 index 00000000000..6da5dce9719 --- /dev/null +++ b/cmd/infrastructure/utils/install.go @@ -0,0 +1,24 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package utils + +type Installer interface { + Install(name, ns string) error +} diff --git a/cmd/infrastructure/utils/yaml_wrapper.go b/cmd/infrastructure/utils/yaml_wrapper.go new file mode 100644 index 00000000000..aaf400c4d75 --- /dev/null +++ b/cmd/infrastructure/utils/yaml_wrapper.go @@ -0,0 +1,34 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package utils + +import "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure/types" + +type YamlInstallHelper struct { +} + +func (y *YamlInstallHelper) Install(name, ns string) error { + // TODO implement me + panic("implement me") +} + +func NewYamlInstaller(yaml types.Yaml, kubeconfig string) Installer { + return &YamlInstallHelper{} +} diff --git a/util/helm/errors.go b/util/helm/errors.go index 687872499b6..9f0b7d3e9ea 100644 --- a/util/helm/errors.go +++ b/util/helm/errors.go @@ -34,7 +34,7 @@ import ( var ErrReleaseNotDeployed = fmt.Errorf("release: not in deployed status") -func releaseNotFound(err error) bool { +func ReleaseNotFound(err error) bool { if err == nil { return false } diff --git a/util/helm/helm.go b/util/helm/helm.go index df995d48973..474c770bf83 100644 --- a/util/helm/helm.go +++ b/util/helm/helm.go @@ -198,7 +198,7 @@ func (i *InstallOpts) tryInstall(cfg *action.Configuration) (*release.Release, e if released != nil { return released, nil } - if err != nil && !releaseNotFound(err) { + if err != nil && !ReleaseNotFound(err) { return nil, err } } diff --git a/util/helm/helm_test.go b/util/helm/helm_test.go index 56addc920e1..62a1364f794 100644 --- a/util/helm/helm_test.go +++ b/util/helm/helm_test.go @@ -119,7 +119,7 @@ var _ = Describe("helm util", func() { }) It("should fail when release is not found", func() { - Expect(releaseNotFound(o.Upgrade(cfg))).Should(BeTrue()) + Expect(ReleaseNotFound(o.Upgrade(cfg))).Should(BeTrue()) Expect(o.Uninstall(cfg)).Should(HaveOccurred()) // release not found }) diff --git a/util/util.go b/util/util.go index d6692151d22..8039c140e8f 100644 --- a/util/util.go +++ b/util/util.go @@ -559,7 +559,7 @@ func IsSupportReconfigureParams(tpl appsv1alpha1.ComponentConfigSpec, values map return true, nil } -func getIPLocation() (string, error) { +func GetIPLocation() (string, error) { client := &http.Client{Timeout: 10 * time.Second} req, err := http.NewRequest("GET", "https://ifconfig.io/country_code", nil) if err != nil { @@ -595,7 +595,7 @@ func GetHelmChartRepoURL() string { // if helm repo url is not specified, choose one from GitHub and GitLab based on the IP location // if location is CN, or we can not get location, use GitLab helm chart repo repo := types.KubeBlocksChartURL - location, _ := getIPLocation() + location, _ := GetIPLocation() if location == "CN" || location == "" { repo = types.GitLabHelmChartRepo } diff --git a/util/util_test.go b/util/util_test.go index 5c5708853be..0531d5c6761 100644 --- a/util/util_test.go +++ b/util/util_test.go @@ -252,7 +252,7 @@ var _ = Describe("util", func() { }) It("get IP location", func() { - _, _ = getIPLocation() + _, _ = GetIPLocation() }) It("get helm chart repo url", func() {