feat: support create kubernetes cluster (#3680)

Co-authored-by: sophon-zt <sophon-zt@users.noreply.github.com>

cmd/cli.go

@@ -36,6 +36,8 @@ import (
 	utilcomp "k8s.io/kubectl/pkg/util/completion"
 	"k8s.io/kubectl/pkg/util/templates"
 
+	infras "github.com/apecloud/kubeblocks/internal/cli/cmd/infrastructure"
+
 	"github.com/apecloud/kubeblocks/internal/cli/cmd/addon"
 	"github.com/apecloud/kubeblocks/internal/cli/cmd/alert"
 	"github.com/apecloud/kubeblocks/internal/cli/cmd/bench"
@@ -183,6 +185,7 @@ A Command Line Interface for KubeBlocks`,
 		fault.NewFaultCmd(f, ioStreams),
 		builder.NewBuilderCmd(f, ioStreams),
 		report.NewReportCmd(f, ioStreams),
+		infras.NewInfraCmd(ioStreams),
 	)
 
 	filters := []string{"options"}

cmd/infrastructure/builder/builder.go

@@ -0,0 +1,83 @@
+/*
+Copyright (C) 2022-2023 ApeCloud Co., Ltd
+
+This file is part of KubeBlocks project
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package builder
+
+import (
+	"bufio"
+	"embed"
+	"encoding/json"
+	"strings"
+
+	"github.com/leaanthony/debme"
+	"k8s.io/apimachinery/pkg/util/yaml"
+
+	cfgcore "github.com/apecloud/kubeblocks/internal/configuration"
+	"github.com/apecloud/kubeblocks/internal/gotemplate"
+)
+
+var (
+	//go:embed template/*
+	cueTemplate embed.FS
+)
+
+func newBuildTemplate(templateName string) (string, error) {
+	tmplFs, _ := debme.FS(cueTemplate, "template")
+	if tmlBytes, err := tmplFs.ReadFile(templateName); err != nil {
+		return "", err
+	} else {
+		return string(tmlBytes), nil
+	}
+}
+
+func BuildFromTemplate(values *gotemplate.TplValues, templateName string) (string, error) {
+	tpl, err := newBuildTemplate(templateName)
+	if err != nil {
+		return "", err
+	}
+
+	engine := gotemplate.NewTplEngine(values, nil, templateName, nil, nil)
+	rendered, err := engine.Render(tpl)
+	if err != nil {
+		return "", err
+	}
+	return rendered, nil
+}
+
+func BuildResourceFromYaml[T any](obj T, bYaml string) (*T, error) {
+	var ret map[string]interface{}
+
+	content, err := yaml.NewYAMLReader(bufio.NewReader(strings.NewReader(bYaml))).Read()
+	if err != nil {
+		return nil, cfgcore.WrapError(err, "failed to read the cluster yaml")
+	}
+	err = yaml.Unmarshal(content, &ret)
+	if err != nil {
+		return nil, cfgcore.WrapError(err, "failed to unmarshal the cluster yaml")
+	}
+
+	contentToJSON, err := yaml.ToJSON(content)
+	if err != nil {
+		return nil, cfgcore.WrapError(err, "Unable to convert configuration to json")
+	}
+	if err := json.Unmarshal(contentToJSON, &obj); err != nil {
+		return nil, cfgcore.WrapError(err, "failed to unmarshal the cluster")
+	}
+	return &obj, nil
+}

cmd/infrastructure/builder/template/containerd.config.toml.tpl

@@ -0,0 +1,48 @@
+version = 2
+{{- if .DataRoot }}
+root = {{ .DataRoot }}
+{{ else }}
+root = "/var/lib/containerd"
+{{- end }}
+state = "/run/containerd"
+
+[grpc]
+  address = "/run/containerd/containerd.sock"
+  uid = 0
+  gid = 0
+  max_recv_message_size = 16777216
+  max_send_message_size = 16777216
+
+[debug]
+  level = "info"
+
+[metrics]
+  address = ""
+  grpc_histogram = false
+
+[timeouts]
+  "io.containerd.timeout.shim.cleanup" = "5s"
+  "io.containerd.timeout.shim.load" = "5s"
+  "io.containerd.timeout.shim.shutdown" = "3s"
+  "io.containerd.timeout.task.state" = "2s"
+
+[plugins."io.containerd.grpc.v1.cri".containerd]
+default_runtime_name = "runc"
+
+[plugins."io.containerd.grpc.v1.cri"]
+stream_server_address = "127.0.0.1"
+max_container_log_line_size = 262144
+sandbox_image = "{{ .SandBoxImage }}"
+
+[plugins."io.containerd.grpc.v1.cri".registry]
+config_path = "/etc/containerd/certs.d:/etc/docker/certs.d"
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+runtime_type = "io.containerd.runc.v2"
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+SystemdCgroup = true
+
+[plugins."io.containerd.grpc.v1.cri".cni]
+bin_dir = "/opt/cni/bin"
+conf_dir = "/etc/cni/net.d"

cmd/infrastructure/builder/template/containerd.service.tpl

@@ -0,0 +1,26 @@
+[Unit]
+Description=containerd container runtime
+Documentation=https://containerd.io
+After=network.target local-fs.target
+
+[Service]
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart=/usr/bin/containerd
+
+Type=notify
+Delegate=yes
+KillMode=process
+Restart=always
+RestartSec=5
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNPROC=infinity
+LimitCORE=infinity
+LimitNOFILE=infinity
+# Comment TasksMax if your systemd version does not supports it.
+# Only systemd 226 and above support this version.
+TasksMax=infinity
+OOMScoreAdjust=-999
+
+[Install]
+WantedBy=multi-user.target

cmd/infrastructure/builder/template/crictl.yaml.tpl

@@ -0,0 +1,5 @@
+runtime-endpoint: {{ .Endpoint }}
+image-endpoint: {{ .Endpoint }}
+timeout: 5
+debug: false
+pull-image-on-create: false

cmd/infrastructure/builder/template/init_os.sh.tpl

@@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+
+function swap_off() {
+  echo "swap off..."
+  swapoff -a
+  sed -i /^[^#]*swap*/s/^/\#/g /etc/fstab
+
+  # clean cache
+  echo 3 > /proc/sys/vm/drop_caches
+  echo
+}
+
+function selinux_off() {
+  echo "selinux off..."
+  setenforce 0
+  echo "enforce: $(getenforce)"
+
+  if [ -f /etc/selinux/config ]; then
+    sed -ri 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
+  fi
+  echo
+}
+
+function firewalld_off() {
+  echo "firewall off..."
+  systemctl stop firewalld.service 1>/dev/null 2>&1
+  systemctl disable firewalld.service 1>/dev/null 2>&1
+  systemctl stop ufw 1>/dev/null 2>&1
+  systemctl disable ufw 1>/dev/null 2>&1
+  echo
+}
+
+function replace_in_file() {
+    local filename="${1:?filename is required}"
+    local match_regex="${2:?match regex is required}"
+    local substitute_regex="${3:?substitute regex is required}"
+    local posix_regex=${4:-true}
+
+    local result
+    local -r del=$'\001'
+    if [[ $posix_regex = true ]]; then
+        result="$(sed -E "s${del}${match_regex}${del}${substitute_regex}${del}g" "$filename")"
+    else
+        result="$(sed "s${del}${match_regex}${del}${substitute_regex}${del}g" "$filename")"
+    fi
+    echo "$result" > "$filename"
+}
+
+function sysctl_set_keyvalue() {
+    local -r key="${1:?missing key}"
+    local -r value="${2:?missing value}"
+    local -r conf_file="${3:-"/etc/sysctl.conf"}"
+    if grep -qE "^#*\s*${key}" "$conf_file" >/dev/null; then
+        replace_in_file "$conf_file" "^#*\s*${key}\s*=.*" "${key} = ${value}"
+    else
+        echo "${key} = ${value}" >>"$conf_file"
+    fi
+}
+
+function set_network() {
+  echo "set network..."
+
+  sysctl_set_keyvalue "net.ipv4.tcp_tw_recycle" "0"
+  sysctl_set_keyvalue "net.ipv4.ip_forward" "1"
+  sysctl_set_keyvalue "net.bridge.bridge-nf-call-arptables" "1"
+  sysctl_set_keyvalue "net.bridge.bridge-nf-call-ip6tables" "1"
+  sysctl_set_keyvalue "net.bridge.bridge-nf-call-iptables" "1"
+  # for node port
+  sysctl_set_keyvalue "net.ipv4.ip_local_reserved_ports" "30000-32767"
+
+  echo
+}
+
+function install_hugepage() {
+  local -r hugepage="${1:?missing key}"
+
+  echo "install hugepage..."
+  hpg_sz=$(grep Hugepagesize /proc/meminfo | awk '{print $2}')
+  ## convert to KB
+  re_hpg_num=$(echo $hugepage | awk '{sub("GB", "*1024*1024*1024", $1) || sub("MB", "*1024*1024", $1) || sub("KB", "*1024", $1); printf $1 "+"} END {print 0}' | bc)
+  hpg_num=$(echo "$re_hpg_num / ( $hpg_sz * 1024 )" | bc)
+
+  sysctl_set_keyvalue "vm.nr_hugepages" "$hpg_num"
+  echo
+}
+
+function common_os_setting() {
+  swap_off
+  selinux_off
+  firewalld_off
+  set_network
+}
+
+function install_hosts() {
+  sed -i ':a;$!{N;ba};s@# kubeblocks hosts BEGIN.*# kubeblocks hosts END@@' /etc/hosts
+  sed -i '/^$/N;/\n$/N;//D' /etc/hosts
+
+  cat >>/etc/hosts<<EOF
+# kubeblocks hosts BEGIN
+{{- range .Hosts }}
+{{ . }}
+{{- end }}
+# kubeblocks hosts END
+EOF
+}
+
+function install_netfilter() {
+  modinfo br_netfilter > /dev/null 2>&1
+  if [ $? -eq 0 ]; then
+     modprobe br_netfilter
+     mkdir -p /etc/modules-load.d
+     echo 'br_netfilter' > /etc/modules-load.d/kubekey-br_netfilter.conf
+  fi
+
+  modinfo overlay > /dev/null 2>&1
+  if [ $? -eq 0 ]; then
+     modprobe overlay
+     echo 'overlay' >> /etc/modules-load.d/kubekey-br_netfilter.conf
+  fi
+}
+
+function install_ipvs() {
+  modprobe ip_vs
+  modprobe ip_vs_rr
+  modprobe ip_vs_wrr
+  modprobe ip_vs_sh
+
+  cat > /etc/modules-load.d/kube_proxy-ipvs.conf << EOF
+ip_vs
+ip_vs_rr
+ip_vs_wrr
+ip_vs_sh
+EOF
+
+  modprobe nf_conntrack_ipv4 1>/dev/null 2>/dev/null
+  if [ $? -eq 0 ]; then
+     echo 'nf_conntrack_ipv4' > /etc/modules-load.d/kube_proxy-ipvs.conf
+  else
+     modprobe nf_conntrack
+     echo 'nf_conntrack' > /etc/modules-load.d/kube_proxy-ipvs.conf
+  fi
+}
+
+{{- if $.Options.HugePageFeature }}
+install_hugepage {{ $.Options.HugePageFeature.HugePageSize }}
+{{- end }}
+
+install_netfilter
+install_ipvs
+install_hosts
+common_os_setting
+
+# for es/opensearch
+sysctl_set_keyvalue "vm.max_map_count" "262144"
+# for kubeblocks
+sysctl_set_keyvalue "fs.inotify.max_user_watches" "524288"
+sysctl_set_keyvalue "fs.inotify.max_user_instances" "524288"
+sysctl -p
+
+# Make sure the iptables utility doesn't use the nftables backend.
+update-alternatives --set iptables /usr/sbin/iptables-legacy >/dev/null 2>&1 || true
+update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy >/dev/null 2>&1 || true
+update-alternatives --set arptables /usr/sbin/arptables-legacy >/dev/null 2>&1 || true
+update-alternatives --set ebtables /usr/sbin/ebtables-legacy >/dev/null 2>&1 || true
+
+ulimit -u 65535
+ulimit -n 65535