From 0b303556f08d0c23fc0c3015ad66b39746f5d92d Mon Sep 17 00:00:00 2001 From: Aleksei Sizov Date: Mon, 9 Sep 2024 16:26:53 -0500 Subject: [PATCH] Add vSphere hosted CP template --- .../cluster/vsphere-hosted-cp/.helmignore | 23 +++ .../cluster/vsphere-hosted-cp/Chart.yaml | 20 +++ .../vsphere-hosted-cp/templates/_helpers.tpl | 19 +++ .../vsphere-hosted-cp/templates/cluster.yaml | 17 ++ .../templates/k0smotroncontrolplane.yaml | 86 ++++++++++ .../templates/k0sworkerconfigtemplate.yaml | 14 ++ .../templates/machinedeployment.yaml | 26 +++ .../templates/vspherecluster.yaml | 13 ++ .../templates/vspheremachinetemplate.yaml | 25 +++ .../vsphere-hosted-cp/values.schema.json | 161 ++++++++++++++++++ .../cluster/vsphere-hosted-cp/values.yaml | 50 ++++++ .../files/templates/vsphere-hosted-cp.yaml | 8 + 12 files changed, 462 insertions(+) create mode 100644 templates/cluster/vsphere-hosted-cp/.helmignore create mode 100644 templates/cluster/vsphere-hosted-cp/Chart.yaml create mode 100644 templates/cluster/vsphere-hosted-cp/templates/_helpers.tpl create mode 100644 templates/cluster/vsphere-hosted-cp/templates/cluster.yaml create mode 100644 templates/cluster/vsphere-hosted-cp/templates/k0smotroncontrolplane.yaml create mode 100644 templates/cluster/vsphere-hosted-cp/templates/k0sworkerconfigtemplate.yaml create mode 100644 templates/cluster/vsphere-hosted-cp/templates/machinedeployment.yaml create mode 100644 templates/cluster/vsphere-hosted-cp/templates/vspherecluster.yaml create mode 100644 templates/cluster/vsphere-hosted-cp/templates/vspheremachinetemplate.yaml create mode 100644 templates/cluster/vsphere-hosted-cp/values.schema.json create mode 100644 templates/cluster/vsphere-hosted-cp/values.yaml create mode 100644 templates/provider/hmc-templates/files/templates/vsphere-hosted-cp.yaml diff --git a/templates/cluster/vsphere-hosted-cp/.helmignore b/templates/cluster/vsphere-hosted-cp/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/templates/cluster/vsphere-hosted-cp/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/templates/cluster/vsphere-hosted-cp/Chart.yaml b/templates/cluster/vsphere-hosted-cp/Chart.yaml new file mode 100644 index 000000000..613e74a63 --- /dev/null +++ b/templates/cluster/vsphere-hosted-cp/Chart.yaml @@ -0,0 +1,20 @@ +apiVersion: v2 +name: vsphere-hosted-cp +description: | + An HMC template to deploy a k8s cluster on vSphere with control plane components + within the management cluster. +type: application +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.1 +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.30.4+k0s.0" +annotations: + hmc.mirantis.com/type: deployment + hmc.mirantis.com/infrastructure-providers: vsphere + hmc.mirantis.com/controlplane-providers: k0s + hmc.mirantis.com/bootstrap-providers: k0s diff --git a/templates/cluster/vsphere-hosted-cp/templates/_helpers.tpl b/templates/cluster/vsphere-hosted-cp/templates/_helpers.tpl new file mode 100644 index 000000000..2de298ddb --- /dev/null +++ b/templates/cluster/vsphere-hosted-cp/templates/_helpers.tpl @@ -0,0 +1,19 @@ +{{- define "cluster.name" -}} + {{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "vspheremachinetemplate.name" -}} + {{- include "cluster.name" . }}-mt +{{- end }} + +{{- define "k0smotroncontrolplane.name" -}} + {{- include "cluster.name" . }}-cp +{{- end }} + +{{- define "k0sworkerconfigtemplate.name" -}} + {{- include "cluster.name" . }}-machine-config +{{- end }} + +{{- define "machinedeployment.name" -}} + {{- include "cluster.name" . }}-md +{{- end }} diff --git a/templates/cluster/vsphere-hosted-cp/templates/cluster.yaml b/templates/cluster/vsphere-hosted-cp/templates/cluster.yaml new file mode 100644 index 000000000..87c042604 --- /dev/null +++ b/templates/cluster/vsphere-hosted-cp/templates/cluster.yaml @@ -0,0 +1,17 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: {{ include "cluster.name" . }} +spec: + {{- with .Values.clusterNetwork }} + clusterNetwork: + {{- toYaml . | nindent 4 }} + {{- end }} + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: K0smotronControlPlane + name: {{ include "k0smotroncontrolplane.name" . }} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: VSphereCluster + name: {{ include "cluster.name" . }} diff --git a/templates/cluster/vsphere-hosted-cp/templates/k0smotroncontrolplane.yaml b/templates/cluster/vsphere-hosted-cp/templates/k0smotroncontrolplane.yaml new file mode 100644 index 000000000..c4badda51 --- /dev/null +++ b/templates/cluster/vsphere-hosted-cp/templates/k0smotroncontrolplane.yaml @@ -0,0 +1,86 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: K0smotronControlPlane +metadata: + name: {{ include "k0smotroncontrolplane.name" . }} +spec: + replicas: {{ .Values.controlPlaneNumber }} + version: {{ .Values.k0s.version | replace "+" "-" }} + {{- with .Values.k0smotron.service }} + service: + {{- toYaml . | nindent 4 }} + {{- end }} + controllerPlaneFlags: + - "--enable-cloud-provider=true" + - "--debug=true" + k0sConfig: + apiVersion: k0s.k0sproject.io/v1beta1 + kind: ClusterConfig + metadata: + name: k0s + spec: + network: + provider: calico + calico: + mode: vxlan + extensions: + helm: + repositories: + - name: vsphere-cpi + url: https://kubernetes.github.io/cloud-provider-vsphere + - name: mirantis + url: https://charts.mirantis.com + charts: + - name: vsphere-cpi + chartname: vsphere-cpi/vsphere-cpi + version: 1.31.0 + order: 1 + namespace: kube-system + values: | + config: + enabled: true + vcenter: {{ .Values.vsphere.server }} + datacenter: {{ .Values.vsphere.datacenter }} + username: "{{ .Values.vsphere.username }}" + password: "{{ .Values.vsphere.password }}" + daemonset: + affinity: null + tolerations: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + - key: CriticalAddonsOnly + effect: NoExecute + operator: Exists + - name: vsphere-csi + chartname: mirantis/vsphere-csi-driver + version: 0.0.1 + order: 2 + namespace: kube-system + values: | + vcenterConfig: + enabled: true + clusterID: {{ include "cluster.name" . }} + vcenter: {{ .Values.vsphere.server }} + user: "{{ .Values.vsphere.username }}" + password: "{{ .Values.vsphere.password }}" + datacenters: {{ .Values.vsphere.datacenter }} + controller: + nodeAffinity: null + node: + kubeletPath: /var/lib/k0s/kubelet + defaultStorageClass: + enabled: true + images: + driver: + tag: v3.1.2 + syncer: + tag: v3.1.2 diff --git a/templates/cluster/vsphere-hosted-cp/templates/k0sworkerconfigtemplate.yaml b/templates/cluster/vsphere-hosted-cp/templates/k0sworkerconfigtemplate.yaml new file mode 100644 index 000000000..37e09dd2c --- /dev/null +++ b/templates/cluster/vsphere-hosted-cp/templates/k0sworkerconfigtemplate.yaml @@ -0,0 +1,14 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: K0sWorkerConfigTemplate +metadata: + name: {{ include "k0sworkerconfigtemplate.name" . }} +spec: + template: + spec: + version: {{ .Values.k0s.version }} + files: + - path: /home/{{ .Values.ssh.user }}/.ssh/authorized_keys + permissions: "0600" + content: "{{ trim .Values.ssh.publicKey }}" + preStartCommands: + - chown {{ .Values.ssh.user }} /home/{{ .Values.ssh.user }}/.ssh/authorized_keys diff --git a/templates/cluster/vsphere-hosted-cp/templates/machinedeployment.yaml b/templates/cluster/vsphere-hosted-cp/templates/machinedeployment.yaml new file mode 100644 index 000000000..83599c25e --- /dev/null +++ b/templates/cluster/vsphere-hosted-cp/templates/machinedeployment.yaml @@ -0,0 +1,26 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: {{ include "machinedeployment.name" . }} +spec: + clusterName: {{ include "cluster.name" . }} + replicas: {{ .Values.workersNumber }} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: {{ include "cluster.name" . }} + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: {{ include "cluster.name" . }} + spec: + version: {{ regexReplaceAll "\\+k0s.+$" .Values.k0s.version "" }} + clusterName: {{ include "cluster.name" . }} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: K0sWorkerConfigTemplate + name: {{ include "k0sworkerconfigtemplate.name" . }} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: VSphereMachineTemplate + name: {{ include "vspheremachinetemplate.name" . }} diff --git a/templates/cluster/vsphere-hosted-cp/templates/vspherecluster.yaml b/templates/cluster/vsphere-hosted-cp/templates/vspherecluster.yaml new file mode 100644 index 000000000..4e151a560 --- /dev/null +++ b/templates/cluster/vsphere-hosted-cp/templates/vspherecluster.yaml @@ -0,0 +1,13 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereCluster +metadata: + name: {{ include "cluster.name" . }} +spec: + identityRef: + kind: VSphereClusterIdentity + name: {{ .Values.clusterIdentity.name }} + controlPlaneEndpoint: + host: {{ .Values.controlPlaneEndpointIP }} + port: 6443 + server: {{ .Values.vsphere.server }} + thumbprint: {{ .Values.vsphere.thumbprint }} diff --git a/templates/cluster/vsphere-hosted-cp/templates/vspheremachinetemplate.yaml b/templates/cluster/vsphere-hosted-cp/templates/vspheremachinetemplate.yaml new file mode 100644 index 000000000..14c6f419e --- /dev/null +++ b/templates/cluster/vsphere-hosted-cp/templates/vspheremachinetemplate.yaml @@ -0,0 +1,25 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereMachineTemplate +metadata: + name: {{ include "vspheremachinetemplate.name" . }} +spec: + template: + spec: + cloneMode: linkedClone + datacenter: {{ .Values.vsphere.datacenter }} + datastore: {{ .Values.vsphere.datastore }} + diskGiB: {{ .Values.rootVolumeSize }} + folder: {{ .Values.vsphere.folder }} + memoryMiB: {{ .Values.memory }} + network: + devices: + - dhcp4: true + networkName: {{ .Values.network }} + numCPUs: {{ .Values.cpus }} + os: Linux + powerOffMode: hard + resourcePool: {{ .Values.vsphere.resourcePool }} + server: {{ .Values.vsphere.server }} + storagePolicyName: "" + template: {{ .Values.vmTemplate }} + thumbprint: {{ .Values.vsphere.thumbprint }} diff --git a/templates/cluster/vsphere-hosted-cp/values.schema.json b/templates/cluster/vsphere-hosted-cp/values.schema.json new file mode 100644 index 000000000..81a5c0953 --- /dev/null +++ b/templates/cluster/vsphere-hosted-cp/values.schema.json @@ -0,0 +1,161 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "An HMC template to deploy a k0s cluster on VSphere with bootstrapped control plane nodes.", + "type": "object", + "required": [ + "controlPlaneNumber", + "workersNumber", + "vsphere", + "controlPlaneEndpointIP", + "clusterIdentity", + "ssh", + "rootVolumeSize", + "cpus", + "memory", + "vmTemplate", + "network" + ], + "properties": { + "controlPlaneNumber": { + "description": "The number of the control plane machines", + "type": "number", + "minimum": 1 + }, + "workersNumber": { + "description": "The number of the worker machines", + "type": "number", + "minimum": 1 + }, + "clusterNetwork": { + "type": "object", + "properties": { + "pods": { + "type": "object", + "properties": { + "cidrBlocks": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + } + } + }, + "services": { + "type": "object", + "properties": { + "cidrBlocks": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + } + } + } + } + }, + "clusterIdentity": { + "type": "object", + "description": "VSphereClusterIdentity object reference", + "required": [ + "name" + ], + "properties": { + "name": { + "description": "VSphereClusterIdentity object name", + "type": "string" + } + } + }, + "vsphere": { + "type": "object", + "description": "Data about vSphere instance where cluster will be deployed.", + "required": [ + "server", + "thumbprint", + "datacenter", + "datastore", + "resourcePool", + "folder", + "username", + "password" + ], + "properties": { + "server": { + "type": "string" + }, + "thumbprint": { + "type": "string" + }, + "datacenter": { + "type": "string" + }, + "datastore": { + "type": "string" + }, + "resourcePool": { + "type": "string" + }, + "folder": { + "type": "string" + }, + "username": { + "type": "string" + }, + "password": { + "type": "string" + } + } + }, + "controlPlaneEndpointIP": { + "description": "Virtual IP address which will be used for K8s API endpoint", + "type": "string" + }, + "ssh": { + "type": "object", + "required": [ + "user", + "publicKey" + ], + "properties": { + "user": { + "type": "string" + }, + "publicKey": { + "type": "string" + } + } + }, + "rootVolumeSize": { + "type": "integer" + }, + "cpus": { + "type": "integer" + }, + "memory": { + "type": "integer" + }, + "vmTemplate": { + "type": "string" + }, + "network": { + "type": "string" + }, + "k0s": { + "description": "K0s parameters", + "type": "object", + "required": [ + "version" + ], + "properties": { + "version":{ + "description": "K0s version to use", + "type": "string" + } + } + } + } +} diff --git a/templates/cluster/vsphere-hosted-cp/values.yaml b/templates/cluster/vsphere-hosted-cp/values.yaml new file mode 100644 index 000000000..ca9334205 --- /dev/null +++ b/templates/cluster/vsphere-hosted-cp/values.yaml @@ -0,0 +1,50 @@ +# Cluster parameters +controlPlaneNumber: 3 +workersNumber: 2 + +clusterNetwork: + pods: + cidrBlocks: + - "10.244.0.0/16" + services: + cidrBlocks: + - "10.96.0.0/12" + +# vSphere cluster parameters +clusterIdentity: + name: "" +vsphere: + server: "" + thumbprint: "" + datacenter: "" + datastore: "" + resourcePool: "" + folder: "" + username: "" + password: "" +controlPlaneEndpointIP: "" + +# vSphere machines parameters +ssh: + user: "" + publicKey: "" +rootVolumeSize: 30 +cpus: 2 +memory: 4096 +vmTemplate: "" +network: "" + +# K0s parameters +k0s: + version: v1.30.4+k0s.0 + +# K0smotron parameters +k0smotron: + service: + type: LoadBalancer + apiPort: 6443 + konnectivityPort: 8132 + +# K0s parameters +k0s: + version: v1.30.4+k0s.0 diff --git a/templates/provider/hmc-templates/files/templates/vsphere-hosted-cp.yaml b/templates/provider/hmc-templates/files/templates/vsphere-hosted-cp.yaml new file mode 100644 index 000000000..7efcbd5b1 --- /dev/null +++ b/templates/provider/hmc-templates/files/templates/vsphere-hosted-cp.yaml @@ -0,0 +1,8 @@ +apiVersion: hmc.mirantis.com/v1alpha1 +kind: ClusterTemplate +metadata: + name: vsphere-hosted-cp +spec: + helm: + chartName: vsphere-hosted-cp + chartVersion: 0.0.1