From 9b92854126613f5e82732e39804c74ab4087a02d Mon Sep 17 00:00:00 2001 From: Faiq Date: Tue, 9 Apr 2024 15:27:58 -0400 Subject: [PATCH] feat: get helm config from a well known configmap (#27) * ci: adds tooling to create configmap * feat: use a configmap to get helmchart info * fix: precommit issues * fix: typo in cilium * fix: remove workspace files * build: template name for configmap * refactor: names for helm chart info getter * refactor: use nutanix-storage name instead of nutnaix-csi * refactor: move to globaloptions * fix: adds snapshot to helm config * fix: comments after review * fix: adds a warning and removes ebs csi * fix: typo * fix: adds missing script file * fix: precommit --- .pre-commit-config.yaml | 8 +- .../README.md | 1 + ...aml => nutanix-storage-csi-configmap.yaml} | 6 +- .../templates/deployment.yaml | 1 + .../templates/helm-config.yaml | 37 ++++ .../values.yaml | 2 + go.mod | 2 +- hack/addons/add-warning-helm-configmap.sh | 23 +++ .../kustomization.yaml.tmpl | 19 ++ .../helm-values.yaml | 0 .../kustomization.yaml.tmpl | 2 +- hack/addons/update-nutanix-csi.sh | 26 +-- hack/tools/helm-cm/main.go | 167 ++++++++++++++++++ make/addons.mk | 20 ++- .../lifecycle/clusterautoscaler/handler.go | 39 +++- .../clusterautoscaler/strategy_helmaddon.go | 13 +- .../generic/lifecycle/cni/calico/handler.go | 50 ++++-- .../cni/calico/strategy_helmaddon.go | 16 +- .../generic/lifecycle/cni/cilium/handler.go | 47 +++-- .../cni/cilium/strategy_helmaddon.go | 20 +-- pkg/handlers/generic/lifecycle/config/cm.go | 94 ++++++++++ .../lifecycle/csi/nutanix-csi/handler.go | 66 +++---- pkg/handlers/generic/lifecycle/handlers.go | 33 +++- pkg/handlers/generic/lifecycle/nfd/handler.go | 35 +++- .../lifecycle/nfd/strategy_helmaddon.go | 15 +- pkg/handlers/options/global.go | 13 +- 26 files changed, 610 insertions(+), 145 deletions(-) rename charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/{nutanix-csi-configmap.yaml => nutanix-storage-csi-configmap.yaml} (99%) create mode 100644 charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml create mode 100755 hack/addons/add-warning-helm-configmap.sh create mode 100644 hack/addons/kustomize/nutanix-snapshot-csi/kustomization.yaml.tmpl rename hack/addons/kustomize/{nutanix-csi => nutanix-storage-csi}/helm-values.yaml (100%) rename hack/addons/kustomize/{nutanix-csi => nutanix-storage-csi}/kustomization.yaml.tmpl (89%) create mode 100644 hack/tools/helm-cm/main.go create mode 100644 pkg/handlers/generic/lifecycle/config/cm.go diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e9fc7bedb..643fdf161 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,6 +34,12 @@ repos: language: system files: "^(hack/addons/|charts/cluster-api-runtime-extensions-nutanix/templates/.+/manifests/|make/addons.mk$)" pass_filenames: false + - id: addons-configmap + name: addons-configmap + entry: make generate-helm-configmap + language: system + files: "^(hack/addons/|charts/cluster-api-runtime-extensions-nutanix/templates/.+|make/addons.mk$)" + pass_filenames: false - repo: https://github.com/tekwizely/pre-commit-golang rev: v1.0.0-rc.1 hooks: @@ -117,7 +123,7 @@ repos: name: License headers - YAML and Makefiles stages: [commit] files: (^Makefile|\.(ya?ml|mk))$ - exclude: ^(pkg/handlers/.+/embedded|examples|charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses)/.+\.ya?ml|docs/static/helm/index\.yaml$ + exclude: ^(pkg/handlers/.+/embedded|examples|charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses)/.+\.ya?ml|docs/static/helm/index\.yaml|charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml$ args: - --license-filepath - hack/license-header.txt diff --git a/charts/cluster-api-runtime-extensions-nutanix/README.md b/charts/cluster-api-runtime-extensions-nutanix/README.md index 4baba852b..46601fdf4 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/README.md +++ b/charts/cluster-api-runtime-extensions-nutanix/README.md @@ -31,6 +31,7 @@ A Helm chart for cluster-api-runtime-extensions-nutanix | deployDefaultClusterClasses | bool | `true` | | | deployment.replicas | int | `1` | | | env | object | `{}` | | +| helmAddonsConfigMap | string | `"default-helm-addons-config"` | | | hooks.clusterAutoscaler.crsStrategy.defaultInstallationConfigMap.name | string | `"cluster-autoscaler"` | | | hooks.clusterAutoscaler.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | | hooks.clusterAutoscaler.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cluster-autoscaler-helm-values-template"` | | diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/nutanix-storage-csi-configmap.yaml similarity index 99% rename from charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml rename to charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/nutanix-storage-csi-configmap.yaml index 03c2539b2..8353d3f9a 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/nutanix-storage-csi-configmap.yaml @@ -3,11 +3,11 @@ #================================================================= # DO NOT EDIT THIS FILE -# IT HAS BEEN GENERATED BY /hack/addons/update-nutanix-csi.sh +# IT HAS BEEN GENERATED BY /hack/addons/update-nutanix-storage-csi.sh #================================================================= apiVersion: v1 data: - nutanix-csi.yaml: | + nutanix-storage-csi.yaml: | apiVersion: v1 kind: ServiceAccount metadata: @@ -540,4 +540,4 @@ data: kind: ConfigMap metadata: creationTimestamp: null - name: nutanix-csi + name: nutanix-storage-csi diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml index 1ce472756..b54b1d72d 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml @@ -30,6 +30,7 @@ spec: args: - --webhook-cert-dir=/runtimehooks-certs/ - --defaults-namespace=$(POD_NAMESPACE) + - --helm-addons-configmap={{ .Values.helmAddonsConfigMap }} - --cni.cilium.helm-addon.default-values-template-configmap-name={{ .Values.hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name }} - --nfd.helm-addon.default-values-template-configmap-name={{ .Values.hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.name }} {{- range $key, $value := .Values.extraArgs }} diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml new file mode 100644 index 000000000..255e8d69f --- /dev/null +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml @@ -0,0 +1,37 @@ +# Copyright 2023 D2iQ, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +#================================================================= +# DO NOT EDIT THIS FILE +# IT HAS BEEN GENERATED BY /hack/tools/helm-cm/main.go +#================================================================= +apiVersion: v1 +data: + cilium: | + ChartName: cilium + ChartVersion: 1.15.0 + RepositoryURL: https://helm.cilium.io/ + cluster-autoscaler: | + ChartName: cluster-autoscaler + ChartVersion: 9.35.0 + RepositoryURL: https://kubernetes.github.io/autoscaler + nfd: | + ChartName: node-feature-discovery + ChartVersion: 0.15.2 + RepositoryURL: https://kubernetes-sigs.github.io/node-feature-discovery/charts + nutanix-snapshot-csi: | + ChartName: nutanix-csi-snapshot + ChartVersion: v6.3.2 + RepositoryURL: https://nutanix.github.io/helm/ + nutanix-storage-csi: | + ChartName: nutanix-csi-storage + ChartVersion: v2.6.6 + RepositoryURL: https://nutanix.github.io/helm/ + tigera-operator: | + ChartName: tigera-operator + ChartVersion: v3.26.4 + RepositoryURL: https://docs.tigera.io/calico/charts +kind: ConfigMap +metadata: + creationTimestamp: null + name: {{ .Values.helmAddonsConfigMap }} diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.yaml b/charts/cluster-api-runtime-extensions-nutanix/values.yaml index 93b1df268..88ac607ea 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/values.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/values.yaml @@ -66,6 +66,8 @@ hooks: create: true name: default-cluster-autoscaler-helm-values-template +helmAddonsConfigMap: default-helm-addons-config + deployDefaultClusterClasses: true deployment: diff --git a/go.mod b/go.mod index 8459609da..d2777daae 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/onsi/gomega v1.31.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 + gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.29.3 k8s.io/apiextensions-apiserver v0.29.3 k8s.io/apimachinery v0.29.3 @@ -135,7 +136,6 @@ require ( google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/cluster-bootstrap v0.29.3 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect diff --git a/hack/addons/add-warning-helm-configmap.sh b/hack/addons/add-warning-helm-configmap.sh new file mode 100755 index 000000000..484c229fc --- /dev/null +++ b/hack/addons/add-warning-helm-configmap.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR + +# shellcheck source=hack/common.sh +source "${SCRIPT_DIR}/../common.sh" +ASSETS_DIR="$(mktemp -d -p "${TMPDIR:-/tmp}")" +mv "${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml" "${ASSETS_DIR}/helm-config.yaml" +# add warning not to edit file directly +cat <"${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml" +$(cat "${GIT_REPO_ROOT}/hack/license-header.yaml.txt") + +#================================================================= +# DO NOT EDIT THIS FILE +# IT HAS BEEN GENERATED BY /hack/tools/helm-cm/main.go +#================================================================= +$(cat "${ASSETS_DIR}/helm-config.yaml") +EOF + +sed -i s/placeholder/"{{ .Values.helmAddonsConfigMap }}"/g "${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml" diff --git a/hack/addons/kustomize/nutanix-snapshot-csi/kustomization.yaml.tmpl b/hack/addons/kustomize/nutanix-snapshot-csi/kustomization.yaml.tmpl new file mode 100644 index 000000000..7c1d21cdf --- /dev/null +++ b/hack/addons/kustomize/nutanix-snapshot-csi/kustomization.yaml.tmpl @@ -0,0 +1,19 @@ +# Copyright 2023 D2iQ, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +metadata: + name: nutanix-csi-kustomize + +namespace: kube-system + +helmCharts: +- name: nutanix-csi-snapshot + repo: https://nutanix.github.io/helm/ + releaseName: nutanix-csi-storage + version: ${NUTANIX_SNAPSHOT_CSI_CHART_VERSION} + includeCRDs: true + skipTests: true + namespace: nutanix-system diff --git a/hack/addons/kustomize/nutanix-csi/helm-values.yaml b/hack/addons/kustomize/nutanix-storage-csi/helm-values.yaml similarity index 100% rename from hack/addons/kustomize/nutanix-csi/helm-values.yaml rename to hack/addons/kustomize/nutanix-storage-csi/helm-values.yaml diff --git a/hack/addons/kustomize/nutanix-csi/kustomization.yaml.tmpl b/hack/addons/kustomize/nutanix-storage-csi/kustomization.yaml.tmpl similarity index 89% rename from hack/addons/kustomize/nutanix-csi/kustomization.yaml.tmpl rename to hack/addons/kustomize/nutanix-storage-csi/kustomization.yaml.tmpl index bc838e438..2e67558f4 100644 --- a/hack/addons/kustomize/nutanix-csi/kustomization.yaml.tmpl +++ b/hack/addons/kustomize/nutanix-storage-csi/kustomization.yaml.tmpl @@ -13,7 +13,7 @@ helmCharts: - name: nutanix-csi-storage repo: https://nutanix.github.io/helm/ releaseName: nutanix-csi-storage - version: ${NUTANIX_CSI_CHART_VERSION} + version: ${NUTANIX_STORAGE_CSI_CHART_VERSION} valuesFile: helm-values.yaml includeCRDs: true skipTests: true diff --git a/hack/addons/update-nutanix-csi.sh b/hack/addons/update-nutanix-csi.sh index e18414a86..c6a488d7e 100755 --- a/hack/addons/update-nutanix-csi.sh +++ b/hack/addons/update-nutanix-csi.sh @@ -8,8 +8,8 @@ readonly SCRIPT_DIR # shellcheck source=hack/common.sh source "${SCRIPT_DIR}/../common.sh" -if [ -z "${NUTANIX_CSI_CHART_VERSION:-}" ]; then - echo "Missing environment variable: NUTANIX_CSI_CHART_VERSION" +if [ -z "${NUTANIX_STORAGE_CSI_CHART_VERSION:-}" ]; then + echo "Missing environment variable: NUTANIX_STORAGE_CSI_CHART_VERSION" exit 1 fi @@ -17,26 +17,26 @@ ASSETS_DIR="$(mktemp -d -p "${TMPDIR:-/tmp}")" readonly ASSETS_DIR trap_add "rm -rf ${ASSETS_DIR}" EXIT -readonly FILE_NAME="nutanix-csi.yaml" +readonly FILE_NAME="nutanix-storage-csi.yaml" -readonly KUSTOMIZE_BASE_DIR="${SCRIPT_DIR}/kustomize/nutanix-csi" -mkdir -p "${ASSETS_DIR}/nutanix-csi" -envsubst -no-unset <"${KUSTOMIZE_BASE_DIR}/kustomization.yaml.tmpl" >"${ASSETS_DIR}/nutanix-csi/kustomization.yaml" -cp -r "${KUSTOMIZE_BASE_DIR}"/*.yaml "${ASSETS_DIR}/nutanix-csi/" +readonly KUSTOMIZE_BASE_DIR="${SCRIPT_DIR}/kustomize/nutanix-storage-csi" +mkdir -p "${ASSETS_DIR}/nutanix-storage-csi" +envsubst -no-unset <"${KUSTOMIZE_BASE_DIR}/kustomization.yaml.tmpl" >"${ASSETS_DIR}/nutanix-storage-csi/kustomization.yaml" +cp -r "${KUSTOMIZE_BASE_DIR}"/*.yaml "${ASSETS_DIR}/nutanix-storage-csi/" -kustomize build --enable-helm "${ASSETS_DIR}/nutanix-csi/" >"${ASSETS_DIR}/${FILE_NAME}" +kustomize build --enable-helm "${ASSETS_DIR}/nutanix-storage-csi/" >"${ASSETS_DIR}/${FILE_NAME}" -kubectl create configmap nutanix-csi --dry-run=client --output yaml \ +kubectl create configmap nutanix-storage-csi --dry-run=client --output yaml \ --from-file "${ASSETS_DIR}/${FILE_NAME}" \ - >"${ASSETS_DIR}/nutanix-csi-configmap.yaml" + >"${ASSETS_DIR}/nutanix-storage-csi-configmap.yaml" # add warning not to edit file directly -cat <"${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml" +cat <"${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/nutanix-storage-csi-configmap.yaml" $(cat "${GIT_REPO_ROOT}/hack/license-header.yaml.txt") #================================================================= # DO NOT EDIT THIS FILE -# IT HAS BEEN GENERATED BY /hack/addons/update-nutanix-csi.sh +# IT HAS BEEN GENERATED BY /hack/addons/update-nutanix-storage-csi.sh #================================================================= -$(cat "${ASSETS_DIR}/nutanix-csi-configmap.yaml") +$(cat "${ASSETS_DIR}/nutanix-storage-csi-configmap.yaml") EOF diff --git a/hack/tools/helm-cm/main.go b/hack/tools/helm-cm/main.go new file mode 100644 index 000000000..54f919e50 --- /dev/null +++ b/hack/tools/helm-cm/main.go @@ -0,0 +1,167 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "flag" + "fmt" + "io/fs" + "os" + "path" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/yaml" + ctrl "sigs.k8s.io/controller-runtime" + yamlMarshal "sigs.k8s.io/yaml" +) + +const ( + createHelmAddonsConfigMap = "helm-addons" +) + +var log = ctrl.LoggerFrom(context.Background()) + +func main() { + args := os.Args + var ( + kustomizeDirectory string + outputFile string + ) + flagSet := flag.NewFlagSet(createHelmAddonsConfigMap, flag.ExitOnError) + flagSet.StringVar(&kustomizeDirectory, "kustomize-directory", "", + "Kustomize base directory for all addons") + flagSet.StringVar(&outputFile, "output-file", "", + "output file name to write config map to.") + err := flagSet.Parse(args[1:]) + if err != nil { + log.Error(err, "failed to parse args") + } + cm, err := createConfigMapFromDir(kustomizeDirectory) + if err != nil { + log.Error(err, "failed to create configMap") + return + } + b, err := yamlMarshal.Marshal(*cm) + if err != nil { + log.Error(err, "failed ") + } + fullOutputfilePath := outputFile + if !path.IsAbs(outputFile) { + wd, err := os.Getwd() + if err != nil { + log.Error(err, "failed") + } + fullOutputfilePath = path.Join(wd, outputFile) + } + f, err := os.OpenFile(fullOutputfilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o666) + if err != nil { + log.Error(err, "failed to create file") + } + defer f.Close() + _, err = bytes.NewBuffer(b).WriteTo(f) + if err != nil { + log.Error(err, "failed to write to file") + } +} + +type configMapInfo struct { + configMapFieldName string + RepositoryURL string `json:"RepositoryURL"` + ChartVersion string `json:"ChartVersion"` + ChartName string `json:"ChartName"` +} + +func createConfigMapFromDir(kustomizeDir string) (*corev1.ConfigMap, error) { + fullPath := kustomizeDir + if !path.IsAbs(fullPath) { + wd, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("failed to get wd %w", err) + } + fullPath = path.Join(wd, kustomizeDir) + } + configDirFS := os.DirFS(fullPath) + results := []configMapInfo{} + err := fs.WalkDir(configDirFS, ".", func(filepath string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if strings.Contains(filepath, "kustomization.yaml.tmpl") && !isIgnored(filepath) { + f, err := os.Open(path.Join(fullPath, filepath)) + if err != nil { + return fmt.Errorf("failed to open file: %w", err) + } + defer f.Close() + obj := make(map[string]interface{}) + err = yaml.NewYAMLOrJSONDecoder(f, 1024).Decode(&obj) + if err != nil { + return err + } + charts, ok := obj["helmCharts"] + if !ok { + log.Info("obj %v does not have field helmCharts. skipping \n", obj) + return nil + } + parsedCharts, ok := charts.([]interface{}) + if !ok { + return fmt.Errorf("charts obj %v is not of type []interface", charts) + } + info, ok := parsedCharts[0].(map[string]interface{}) + if !ok { + return fmt.Errorf("info obj %v is not of type map[string]interface", parsedCharts) + } + repo := info["repo"].(string) + name := info["name"].(string) + dirName := strings.Split(filepath, "/")[0] + i := configMapInfo{ + configMapFieldName: dirName, + RepositoryURL: repo, + ChartName: name, + } + versionEnvVar := info["version"].(string) + version := os.ExpandEnv(versionEnvVar) + i.ChartVersion = version + results = append(results, i) + return nil + } + return nil + }) + + finalCM := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "placeholder", + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + Data: make(map[string]string), + } + for _, res := range results { + d, err := yamlMarshal.Marshal(res) + if err != nil { + return &finalCM, err + } + finalCM.Data[res.configMapFieldName] = string(d) + } + return &finalCM, err +} + +var ignored = []string{ + "aws-ccm", + "aws-ebs-csi", +} + +func isIgnored(filepath string) bool { + for _, i := range ignored { + if strings.Contains(filepath, i) { + return true + } + } + return false +} diff --git a/make/addons.mk b/make/addons.mk index c09890922..648fc47d9 100644 --- a/make/addons.mk +++ b/make/addons.mk @@ -1,13 +1,14 @@ # Copyright 2023 D2iQ, Inc. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -export CALICO_VERSION := $(shell goprintconst -file pkg/handlers/generic/lifecycle/cni/calico/strategy_helmaddon.go -name defaultCalicoHelmChartVersion) -export CILIUM_VERSION := $(shell goprintconst -file pkg/handlers/generic/lifecycle/cni/cilium/strategy_helmaddon.go -name defaultCiliumHelmChartVersion) -export NODE_FEATURE_DISCOVERY_VERSION := $(shell goprintconst -file pkg/handlers/generic/lifecycle/nfd/strategy_helmaddon.go -name defaultHelmChartVersion) +export CALICO_VERSION := v3.26.4 +export CILIUM_VERSION := 1.15.0 +export NODE_FEATURE_DISCOVERY_VERSION := 0.15.2 export CLUSTER_AUTOSCALER_VERSION := 9.35.0 export AWS_CSI_SNAPSHOT_CONTROLLER_VERSION := v6.3.3 export AWS_EBS_CSI_CHART_VERSION := v2.28.1 -export NUTANIX_CSI_CHART_VERSION := v2.6.6 +export NUTANIX_STORAGE_CSI_CHART_VERSION := v2.6.6 +export NUTANIX_SNAPSHOT_CSI_CHART_VERSION := v6.3.2 # a map of AWS CCM versions export AWS_CCM_VERSION_127 := v1.27.1 export AWS_CCM_CHART_VERSION_127 := 0.0.8 @@ -15,7 +16,7 @@ export AWS_CCM_VERSION_128 := v1.28.1 export AWS_CCM_CHART_VERSION_128 := 0.0.8 .PHONY: addons.sync -addons.sync: $(addprefix update-addon.,calico cilium nfd cluster-autoscaler aws-ebs-csi aws-ccm.127 aws-ccm.128) +addons.sync: $(addprefix update-addon.,calico cilium nfd cluster-autoscaler aws-ebs-csi aws-ccm.127 nutanix-storage-csi aws-ccm.128) .PHONY: update-addon.calico update-addon.calico: ; $(info $(M) updating calico manifests) @@ -41,6 +42,11 @@ update-addon.aws-ebs-csi: ; $(info $(M) updating aws ebs csi manifests) update-addon.aws-ccm.%: ; $(info $(M) updating aws ccm $* manifests) ./hack/addons/update-aws-ccm.sh $(AWS_CCM_VERSION_$*) $(AWS_CCM_CHART_VERSION_$*) -.PHONY: update-addon.nutanix-csi -update-addon.nutanix-csi: ; $(info $(M) updating nutanix csi manifests) +.PHONY: update-addon.nutanix-storage-csi +update-addon.nutanix-storage-csi: ; $(info $(M) updating nutanix-storage csi manifests) ./hack/addons/update-nutanix-csi.sh + +.PHONY: generate-helm-configmap +generate-helm-configmap: + go run hack/tools/helm-cm/main.go -kustomize-directory="./hack/addons/kustomize" -output-file="./charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml" + ./hack/addons/add-warning-helm-configmap.sh diff --git a/pkg/handlers/generic/lifecycle/clusterautoscaler/handler.go b/pkg/handlers/generic/lifecycle/clusterautoscaler/handler.go index 6ad7c5925..a7110d40e 100644 --- a/pkg/handlers/generic/lifecycle/clusterautoscaler/handler.go +++ b/pkg/handlers/generic/lifecycle/clusterautoscaler/handler.go @@ -18,6 +18,7 @@ import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/lifecycle" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" ) @@ -43,8 +44,9 @@ func (c *Config) AddFlags(prefix string, flags *pflag.FlagSet) { } type DefaultClusterAutoscaler struct { - client ctrlclient.Client - config *Config + client ctrlclient.Client + config *Config + helmChartInfoGetter *config.HelmChartGetter variableName string // points to the global config variable variablePath []string // path of this variable on the global config variable @@ -58,12 +60,14 @@ var ( func New( c ctrlclient.Client, cfg *Config, + helmChartInfoGetter *config.HelmChartGetter, ) *DefaultClusterAutoscaler { return &DefaultClusterAutoscaler{ - client: c, - config: cfg, - variableName: clusterconfig.MetaVariableName, - variablePath: []string{"addons", v1alpha1.ClusterAutoscalerVariableName}, + client: c, + config: cfg, + helmChartInfoGetter: helmChartInfoGetter, + variableName: clusterconfig.MetaVariableName, + variablePath: []string{"addons", v1alpha1.ClusterAutoscalerVariableName}, } } @@ -117,9 +121,28 @@ func (n *DefaultClusterAutoscaler) AfterControlPlaneInitialized( client: n.client, } case v1alpha1.AddonStrategyHelmAddon: + helmChart, err := n.helmChartInfoGetter.For( + ctx, + log, + config.Autoscaler, + ) + if err != nil { + log.Error( + err, + "failed to get configmap with helm settings", + ) + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage( + fmt.Sprintf("failed to get config to create helm addon: %v", + err, + ), + ) + return + } strategy = helmAddonStrategy{ - config: n.config.helmAddonConfig, - client: n.client, + config: n.config.helmAddonConfig, + client: n.client, + helmChart: helmChart, } default: resp.SetStatus(runtimehooksv1.ResponseStatusFailure) diff --git a/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_helmaddon.go b/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_helmaddon.go index a22e69d2f..7f1ba5469 100644 --- a/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_helmaddon.go +++ b/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_helmaddon.go @@ -17,13 +17,11 @@ import ( caaphv1 "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" ) const ( - defaultHelmRepositoryURL = "https://kubernetes.github.io/autoscaler" - defaultHelmChartVersion = "9.35.0" - defaultHelmChartName = "cluster-autoscaler" defaultHelmReleaseNameTemplate = "cluster-autoscaler-%s" ) @@ -43,7 +41,8 @@ func (c *helmAddonConfig) AddFlags(prefix string, flags *pflag.FlagSet) { type helmAddonStrategy struct { config helmAddonConfig - client ctrlclient.Client + client ctrlclient.Client + helmChart *config.HelmChart } func (s helmAddonStrategy) apply( @@ -87,14 +86,14 @@ func (s helmAddonStrategy) apply( Name: "cluster-autoscaler-" + req.Cluster.Name, }, Spec: caaphv1.HelmChartProxySpec{ - RepoURL: defaultHelmRepositoryURL, - ChartName: defaultHelmChartName, + RepoURL: s.helmChart.Repository, + ChartName: s.helmChart.Name, ClusterSelector: metav1.LabelSelector{ MatchLabels: map[string]string{capiv1.ClusterNameLabel: targetCluster.Name}, }, ReleaseNamespace: req.Cluster.Namespace, ReleaseName: fmt.Sprintf(defaultHelmReleaseNameTemplate, req.Cluster.Name), - Version: defaultHelmChartVersion, + Version: s.helmChart.Version, ValuesTemplate: values, }, } diff --git a/pkg/handlers/generic/lifecycle/cni/calico/handler.go b/pkg/handlers/generic/lifecycle/cni/calico/handler.go index ed3b82259..3ff526d02 100644 --- a/pkg/handlers/generic/lifecycle/cni/calico/handler.go +++ b/pkg/handlers/generic/lifecycle/cni/calico/handler.go @@ -18,6 +18,7 @@ import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/lifecycle" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" ) @@ -43,8 +44,9 @@ func (c *CNIConfig) AddFlags(prefix string, flags *pflag.FlagSet) { } type CalicoCNI struct { - client ctrlclient.Client - config *CNIConfig + client ctrlclient.Client + config *CNIConfig + helmChartInfoGetter *config.HelmChartGetter variableName string variablePath []string @@ -58,20 +60,22 @@ var ( func New( c ctrlclient.Client, cfg *CNIConfig, + helmChartInfoGetter *config.HelmChartGetter, ) *CalicoCNI { return &CalicoCNI{ - client: c, - config: cfg, - variableName: clusterconfig.MetaVariableName, - variablePath: []string{"addons", v1alpha1.CNIVariableName}, + client: c, + config: cfg, + helmChartInfoGetter: helmChartInfoGetter, + variableName: clusterconfig.MetaVariableName, + variablePath: []string{"addons", v1alpha1.CNIVariableName}, } } -func (s *CalicoCNI) Name() string { +func (c *CalicoCNI) Name() string { return "CalicoCNI" } -func (s *CalicoCNI) AfterControlPlaneInitialized( +func (c *CalicoCNI) AfterControlPlaneInitialized( ctx context.Context, req *runtimehooksv1.AfterControlPlaneInitializedRequest, resp *runtimehooksv1.AfterControlPlaneInitializedResponse, @@ -85,7 +89,7 @@ func (s *CalicoCNI) AfterControlPlaneInitialized( varMap := variables.ClusterVariablesToVariablesMap(req.Cluster.Spec.Topology.Variables) - cniVar, found, err := variables.Get[v1alpha1.CNI](varMap, s.variableName, s.variablePath...) + cniVar, found, err := variables.Get[v1alpha1.CNI](varMap, c.variableName, c.variablePath...) if err != nil { log.Error( err, @@ -120,13 +124,31 @@ func (s *CalicoCNI) AfterControlPlaneInitialized( switch cniVar.Strategy { case v1alpha1.AddonStrategyClusterResourceSet: strategy = crsStrategy{ - config: s.config.crsConfig, - client: s.client, + config: c.config.crsConfig, + client: c.client, } case v1alpha1.AddonStrategyHelmAddon: + // this is tigera and not calico because we deploy calico via operataor + log.Info("fetching settings for tigera-operator-config") + helmChart, err := c.helmChartInfoGetter.For(ctx, log, config.Tigera) + if err != nil { + log.Error( + err, + "failed to get configmap with helm settings", + ) + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage( + fmt.Sprintf("failed to get configration to create helm addon: %v", + err, + ), + ) + return + } + log.Info(fmt.Sprintf("using settings %v to install helm chart config", helmChart)) strategy = helmAddonStrategy{ - config: s.config.helmAddonConfig, - client: s.client, + config: c.config.helmAddonConfig, + client: c.client, + helmChart: helmChart, } default: resp.SetStatus(runtimehooksv1.ResponseStatusFailure) @@ -134,7 +156,7 @@ func (s *CalicoCNI) AfterControlPlaneInitialized( return } - if err := strategy.apply(ctx, req, s.config.DefaultsNamespace(), log); err != nil { + if err := strategy.apply(ctx, req, c.config.DefaultsNamespace(), log); err != nil { resp.SetStatus(runtimehooksv1.ResponseStatusFailure) resp.SetMessage(err.Error()) return diff --git a/pkg/handlers/generic/lifecycle/cni/calico/strategy_helmaddon.go b/pkg/handlers/generic/lifecycle/cni/calico/strategy_helmaddon.go index 0495afee1..545f2fcef 100644 --- a/pkg/handlers/generic/lifecycle/cni/calico/strategy_helmaddon.go +++ b/pkg/handlers/generic/lifecycle/cni/calico/strategy_helmaddon.go @@ -18,12 +18,10 @@ import ( caaphv1 "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" ) const ( - defaultCalicoHelmRepositoryURL = "https://docs.tigera.io/calico/charts" - defaultCalicoHelmChartVersion = "v3.26.4" - defaultTigeraOperatorChartName = "tigera-operator" defaultTigeraOperatorReleaseName = "tigera-operator" defaultTigerOperatorNamespace = "tigera-operator" ) @@ -46,9 +44,9 @@ func (c *helmAddonConfig) AddFlags(prefix string, flags *pflag.FlagSet) { } type helmAddonStrategy struct { - config helmAddonConfig - - client ctrlclient.Client + config helmAddonConfig + helmChart *config.HelmChart + client ctrlclient.Client } func (s helmAddonStrategy) apply( @@ -92,14 +90,14 @@ func (s helmAddonStrategy) apply( Name: "calico-cni-installation-" + req.Cluster.Name, }, Spec: caaphv1.HelmChartProxySpec{ - RepoURL: defaultCalicoHelmRepositoryURL, - ChartName: defaultTigeraOperatorChartName, + RepoURL: s.helmChart.Repository, + ChartName: s.helmChart.Name, ClusterSelector: metav1.LabelSelector{ MatchLabels: map[string]string{capiv1.ClusterNameLabel: req.Cluster.Name}, }, ReleaseNamespace: defaultTigerOperatorNamespace, ReleaseName: defaultTigeraOperatorReleaseName, - Version: defaultCalicoHelmChartVersion, + Version: s.helmChart.Version, ValuesTemplate: valuesTemplateConfigMap.Data["values.yaml"], }, } diff --git a/pkg/handlers/generic/lifecycle/cni/cilium/handler.go b/pkg/handlers/generic/lifecycle/cni/cilium/handler.go index 1e7c7db3c..aef9b93a2 100644 --- a/pkg/handlers/generic/lifecycle/cni/cilium/handler.go +++ b/pkg/handlers/generic/lifecycle/cni/cilium/handler.go @@ -18,6 +18,7 @@ import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/lifecycle" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" ) @@ -43,8 +44,9 @@ func (c *CNIConfig) AddFlags(prefix string, flags *pflag.FlagSet) { } type CiliumCNI struct { - client ctrlclient.Client - config *CNIConfig + client ctrlclient.Client + config *CNIConfig + helmChartInfoGetter *config.HelmChartGetter variableName string variablePath []string @@ -58,20 +60,22 @@ var ( func New( c ctrlclient.Client, cfg *CNIConfig, + helmChartInfoGetter *config.HelmChartGetter, ) *CiliumCNI { return &CiliumCNI{ - client: c, - config: cfg, - variableName: clusterconfig.MetaVariableName, - variablePath: []string{"addons", v1alpha1.CNIVariableName}, + client: c, + config: cfg, + helmChartInfoGetter: helmChartInfoGetter, + variableName: clusterconfig.MetaVariableName, + variablePath: []string{"addons", v1alpha1.CNIVariableName}, } } -func (s *CiliumCNI) Name() string { +func (c *CiliumCNI) Name() string { return "CiliumCNI" } -func (s *CiliumCNI) AfterControlPlaneInitialized( +func (c *CiliumCNI) AfterControlPlaneInitialized( ctx context.Context, req *runtimehooksv1.AfterControlPlaneInitializedRequest, resp *runtimehooksv1.AfterControlPlaneInitializedResponse, @@ -85,7 +89,7 @@ func (s *CiliumCNI) AfterControlPlaneInitialized( varMap := variables.ClusterVariablesToVariablesMap(req.Cluster.Spec.Topology.Variables) - cniVar, found, err := variables.Get[v1alpha1.CNI](varMap, s.variableName, s.variablePath...) + cniVar, found, err := variables.Get[v1alpha1.CNI](varMap, c.variableName, c.variablePath...) if err != nil { log.Error( err, @@ -120,13 +124,28 @@ func (s *CiliumCNI) AfterControlPlaneInitialized( switch cniVar.Strategy { case v1alpha1.AddonStrategyClusterResourceSet: strategy = crsStrategy{ - config: s.config.crsConfig, - client: s.client, + config: c.config.crsConfig, + client: c.client, } case v1alpha1.AddonStrategyHelmAddon: + helmChart, err := c.helmChartInfoGetter.For(ctx, log, config.Cilium) + if err != nil { + log.Error( + err, + "failed to get configmap with helm settings", + ) + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage( + fmt.Sprintf("failed to get configration to create helm addon: %v", + err, + ), + ) + return + } strategy = helmAddonStrategy{ - config: s.config.helmAddonConfig, - client: s.client, + config: c.config.helmAddonConfig, + client: c.client, + helmChart: helmChart, } default: resp.SetStatus(runtimehooksv1.ResponseStatusFailure) @@ -134,7 +153,7 @@ func (s *CiliumCNI) AfterControlPlaneInitialized( return } - if err := strategy.apply(ctx, req, s.config.DefaultsNamespace(), log); err != nil { + if err := strategy.apply(ctx, req, c.config.DefaultsNamespace(), log); err != nil { resp.SetStatus(runtimehooksv1.ResponseStatusFailure) resp.SetMessage(err.Error()) return diff --git a/pkg/handlers/generic/lifecycle/cni/cilium/strategy_helmaddon.go b/pkg/handlers/generic/lifecycle/cni/cilium/strategy_helmaddon.go index ecf8fe3ab..b4ab4b29d 100644 --- a/pkg/handlers/generic/lifecycle/cni/cilium/strategy_helmaddon.go +++ b/pkg/handlers/generic/lifecycle/cni/cilium/strategy_helmaddon.go @@ -18,14 +18,12 @@ import ( caaphv1 "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" ) const ( - defaultCiliumHelmRepositoryURL = "https://helm.cilium.io/" - defaultCiliumHelmChartVersion = "1.15.0" - defaultCiliumChartName = "cilium" - defaultCiliumReleaseName = "cilium" - defaultCiliumNamespace = "kube-system" + defaultCiliumReleaseName = "cilium" + defaultCiliumNamespace = "kube-system" ) type helmAddonConfig struct { @@ -42,9 +40,9 @@ func (c *helmAddonConfig) AddFlags(prefix string, flags *pflag.FlagSet) { } type helmAddonStrategy struct { - config helmAddonConfig - - client ctrlclient.Client + config helmAddonConfig + client ctrlclient.Client + helmChart *config.HelmChart } func (s helmAddonStrategy) apply( @@ -72,14 +70,14 @@ func (s helmAddonStrategy) apply( Name: "cilium-cni-installation-" + req.Cluster.Name, }, Spec: caaphv1.HelmChartProxySpec{ - RepoURL: defaultCiliumHelmRepositoryURL, - ChartName: defaultCiliumChartName, + RepoURL: s.helmChart.Repository, + ChartName: s.helmChart.Name, ClusterSelector: metav1.LabelSelector{ MatchLabels: map[string]string{capiv1.ClusterNameLabel: req.Cluster.Name}, }, ReleaseNamespace: defaultCiliumNamespace, ReleaseName: defaultCiliumReleaseName, - Version: defaultCiliumHelmChartVersion, + Version: s.helmChart.Version, ValuesTemplate: valuesTemplateConfigMap.Data["values.yaml"], }, } diff --git a/pkg/handlers/generic/lifecycle/config/cm.go b/pkg/handlers/generic/lifecycle/config/cm.go new file mode 100644 index 000000000..5bb58caa8 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/config/cm.go @@ -0,0 +1,94 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Component string + +const ( + Autoscaler Component = "cluster-autoscaler" + Tigera Component = "tigera-operator" + Cilium Component = "cilium" + NFD Component = "nfd" + NutanixStorageCSI Component = "nutanix-storage-csi" + NutanixSnapshotCSI Component = "nutanix-snapshot-csi" +) + +type HelmChartGetter struct { + cl ctrlclient.Reader + cmName string + cmNamespace string +} + +type HelmChart struct { + Name string `yaml:"ChartName"` + Version string `yaml:"ChartVersion"` + Repository string `yaml:"RepositoryURL"` +} + +func NewHelmChartGetterFromConfigMap( + cmName, cmNamespace string, + cl ctrlclient.Reader, +) *HelmChartGetter { + return &HelmChartGetter{ + cl, + cmName, + cmNamespace, + } +} + +func (h *HelmChartGetter) get( + ctx context.Context, +) (*corev1.ConfigMap, error) { + cm := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: h.cmNamespace, + Name: h.cmName, + }, + } + err := h.cl.Get( + ctx, + ctrlclient.ObjectKeyFromObject(cm), + cm, + ) + return cm, err +} + +func (h *HelmChartGetter) For( + ctx context.Context, + log logr.Logger, + name Component, +) (*HelmChart, error) { + log.Info( + fmt.Sprintf("Fetching HelmChart info for %s from configmap %s/%s", + string(name), + h.cmName, + h.cmNamespace), + ) + cm, err := h.get(ctx) + if err != nil { + return nil, err + } + d, ok := cm.Data[string(name)] + if !ok { + return nil, fmt.Errorf("did not find key %s in %v", name, cm.Data) + } + var settings HelmChart + err = yaml.Unmarshal([]byte(d), &settings) + return &settings, err +} diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go index ff1119ac8..6042e3f78 100644 --- a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -12,27 +12,24 @@ import ( "k8s.io/apimachinery/pkg/runtime" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" caaphv1 "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" lifecycleutils "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" ) const ( - defaultHelmRepositoryURL = "https://nutanix.github.io/helm/" - defaultStorageHelmChartVersion = "v2.6.6" - defaultStorageHelmChartName = "nutanix-csi-storage" - defaultStorageHelmReleaseName = "nutanix-csi-storage" - defaultStorageHelmReleaseNamespace = "ntnx-system" + defaultStorageHelmReleaseNameTemplate = "nutanix-csi-storage-%s" + defaultStorageHelmReleaseNamespace = "ntnx-system" - defaultSnapshotHelmChartVersion = "v6.3.2" - defaultSnapshotHelmChartName = "nutanix-csi-snapshot" - defaultSnapshotHelmReleaseName = "nutanix-csi-snapshot" - defaultSnapshotHelmReleaseNamespace = "ntnx-system" + defaultSnapshotHelmReleaseNameTemplate = "nutanix-csi-snapshot-%s" + defaultSnapshotHelmReleaseNamespace = "ntnx-system" //nolint:gosec // Does not contain hard coded credentials. defaultCredentialsSecretName = "nutanix-csi-credentials" @@ -64,17 +61,20 @@ func (n *NutanixCSIConfig) AddFlags(prefix string, flags *pflag.FlagSet) { } type NutanixCSI struct { - client ctrlclient.Client - config *NutanixCSIConfig + client ctrlclient.Client + config *NutanixCSIConfig + helmChartInfoGetter *config.HelmChartGetter } func New( c ctrlclient.Client, cfg *NutanixCSIConfig, + helmChartInfoGetter *config.HelmChartGetter, ) *NutanixCSI { return &NutanixCSI{ - client: c, - config: cfg, + client: c, + config: cfg, + helmChartInfoGetter: helmChartInfoGetter, } } @@ -144,6 +144,14 @@ func (n *NutanixCSI) handleHelmAddonApply( ) } values := valuesTemplateConfigMap.Data["values.yaml"] + log := ctrl.LoggerFrom(ctx).WithValues( + "cluster", + ctrlclient.ObjectKeyFromObject(&req.Cluster), + ) + helmChart, err := n.helmChartInfoGetter.For(ctx, log, config.NutanixStorageCSI) + if err != nil { + return fmt.Errorf("failed to get values for nutanix-csi-config %w", err) + } hcp := &caaphv1.HelmChartProxy{ TypeMeta: metav1.TypeMeta{ @@ -155,14 +163,14 @@ func (n *NutanixCSI) handleHelmAddonApply( Name: "nutanix-csi-" + req.Cluster.Name, }, Spec: caaphv1.HelmChartProxySpec{ - RepoURL: defaultHelmRepositoryURL, - ChartName: defaultStorageHelmChartName, + RepoURL: helmChart.Repository, + ChartName: helmChart.Name, ClusterSelector: metav1.LabelSelector{ MatchLabels: map[string]string{clusterv1.ClusterNameLabel: req.Cluster.Name}, }, - ReleaseNamespace: defaultStorageHelmReleaseNamespace, - ReleaseName: defaultStorageHelmReleaseName, - Version: defaultStorageHelmChartVersion, + ReleaseNamespace: req.Cluster.Namespace, + ReleaseName: fmt.Sprintf(defaultStorageHelmReleaseNameTemplate, req.Cluster.Name), + Version: helmChart.Version, ValuesTemplate: values, }, } @@ -178,6 +186,11 @@ func (n *NutanixCSI) handleHelmAddonApply( return fmt.Errorf("failed to apply nutanix-csi installation HelmChartProxy: %w", err) } + snapshotHelmChart, err := n.helmChartInfoGetter.For(ctx, log, config.NutanixSnapshotCSI) + if err != nil { + return fmt.Errorf("failed to get values for nutanix-csi-config %w", err) + } + snapshotChart := &caaphv1.HelmChartProxy{ TypeMeta: metav1.TypeMeta{ APIVersion: caaphv1.GroupVersion.String(), @@ -188,24 +201,17 @@ func (n *NutanixCSI) handleHelmAddonApply( Name: "nutanix-csi-snapshot-" + req.Cluster.Name, }, Spec: caaphv1.HelmChartProxySpec{ - RepoURL: defaultHelmRepositoryURL, - ChartName: defaultSnapshotHelmChartName, + RepoURL: snapshotHelmChart.Repository, + ChartName: snapshotHelmChart.Name, ClusterSelector: metav1.LabelSelector{ MatchLabels: map[string]string{clusterv1.ClusterNameLabel: req.Cluster.Name}, }, - ReleaseNamespace: defaultSnapshotHelmReleaseNamespace, - ReleaseName: defaultSnapshotHelmReleaseName, - Version: defaultSnapshotHelmChartVersion, + ReleaseNamespace: req.Cluster.Namespace, + ReleaseName: fmt.Sprintf(defaultSnapshotHelmReleaseNameTemplate, req.Cluster.Name), + Version: snapshotHelmChart.Version, }, } - if err = controllerutil.SetOwnerReference(&req.Cluster, snapshotChart, n.client.Scheme()); err != nil { - return fmt.Errorf( - "failed to set owner reference on nutanix-csi installation HelmChartProxy: %w", - err, - ) - } - if err = client.ServerSideApply(ctx, n.client, snapshotChart); err != nil { return fmt.Errorf( "failed to apply nutanix-csi-snapshot installation HelmChartProxy: %w", diff --git a/pkg/handlers/generic/lifecycle/handlers.go b/pkg/handlers/generic/lifecycle/handlers.go index b3b0d22fb..77a2e7a6e 100644 --- a/pkg/handlers/generic/lifecycle/handlers.go +++ b/pkg/handlers/generic/lifecycle/handlers.go @@ -14,6 +14,7 @@ import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/clusterautoscaler" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/cni/calico" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/cni/cilium" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi" awsebs "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/aws-ebs" nutanixcsi "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/nutanix-csi" @@ -23,6 +24,7 @@ import ( ) type Handlers struct { + globalOptions *options.GlobalOptions calicoCNIConfig *calico.CNIConfig ciliumCNIConfig *cilium.CNIConfig nfdConfig *nfd.Config @@ -32,9 +34,14 @@ type Handlers struct { awsccmConfig *awsccm.AWSCCMConfig } -func New(globalOptions *options.GlobalOptions) *Handlers { +func New( + globalOptions *options.GlobalOptions, +) *Handlers { return &Handlers{ - calicoCNIConfig: &calico.CNIConfig{GlobalOptions: globalOptions}, + globalOptions: globalOptions, + calicoCNIConfig: &calico.CNIConfig{ + GlobalOptions: globalOptions, + }, ciliumCNIConfig: &cilium.CNIConfig{GlobalOptions: globalOptions}, nfdConfig: &nfd.Config{GlobalOptions: globalOptions}, clusterAutoscalerConfig: &clusterautoscaler.Config{GlobalOptions: globalOptions}, @@ -45,19 +52,27 @@ func New(globalOptions *options.GlobalOptions) *Handlers { } func (h *Handlers) AllHandlers(mgr manager.Manager) []handlers.Named { + helmChartInfoGetter := config.NewHelmChartGetterFromConfigMap( + h.globalOptions.HelmAddonsConfigMapName(), + h.globalOptions.DefaultsNamespace(), + mgr.GetClient(), + ) csiHandlers := map[string]csi.CSIProvider{ - v1alpha1.CSIProviderAWSEBS: awsebs.New(mgr.GetClient(), h.ebsConfig), - v1alpha1.CSIProviderNutanix: nutanixcsi.New(mgr.GetClient(), h.nutnaixCSIConfig), + v1alpha1.CSIProviderAWSEBS: awsebs.New(mgr.GetClient(), h.ebsConfig), + v1alpha1.CSIProviderNutanix: nutanixcsi.New( + mgr.GetClient(), + h.nutnaixCSIConfig, + helmChartInfoGetter, + ), } ccmHandlers := map[string]ccm.CCMProvider{ v1alpha1.CCMProviderAWS: awsccm.New(mgr.GetClient(), h.awsccmConfig), } - return []handlers.Named{ - calico.New(mgr.GetClient(), h.calicoCNIConfig), - cilium.New(mgr.GetClient(), h.ciliumCNIConfig), - nfd.New(mgr.GetClient(), h.nfdConfig), - clusterautoscaler.New(mgr.GetClient(), h.clusterAutoscalerConfig), + calico.New(mgr.GetClient(), h.calicoCNIConfig, helmChartInfoGetter), + cilium.New(mgr.GetClient(), h.ciliumCNIConfig, helmChartInfoGetter), + nfd.New(mgr.GetClient(), h.nfdConfig, helmChartInfoGetter), + clusterautoscaler.New(mgr.GetClient(), h.clusterAutoscalerConfig, helmChartInfoGetter), servicelbgc.New(mgr.GetClient()), csi.New(mgr.GetClient(), csiHandlers), ccm.New(mgr.GetClient(), ccmHandlers), diff --git a/pkg/handlers/generic/lifecycle/nfd/handler.go b/pkg/handlers/generic/lifecycle/nfd/handler.go index baa3903d7..8eab1edc6 100644 --- a/pkg/handlers/generic/lifecycle/nfd/handler.go +++ b/pkg/handlers/generic/lifecycle/nfd/handler.go @@ -18,6 +18,7 @@ import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/lifecycle" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" ) @@ -43,8 +44,9 @@ func (c *Config) AddFlags(prefix string, flags *pflag.FlagSet) { } type DefaultNFD struct { - client ctrlclient.Client - config *Config + client ctrlclient.Client + config *Config + helmChartInfoGetter *config.HelmChartGetter variableName string // points to the global config variable variablePath []string // path of this variable on the global config variable @@ -58,12 +60,14 @@ var ( func New( c ctrlclient.Client, cfg *Config, + helmChartInfoGetter *config.HelmChartGetter, ) *DefaultNFD { return &DefaultNFD{ - client: c, - config: cfg, - variableName: clusterconfig.MetaVariableName, - variablePath: []string{"addons", v1alpha1.NFDVariableName}, + client: c, + config: cfg, + helmChartInfoGetter: helmChartInfoGetter, + variableName: clusterconfig.MetaVariableName, + variablePath: []string{"addons", v1alpha1.NFDVariableName}, } } @@ -112,9 +116,24 @@ func (n *DefaultNFD) AfterControlPlaneInitialized( client: n.client, } case v1alpha1.AddonStrategyHelmAddon: + helmChart, err := n.helmChartInfoGetter.For(ctx, log, config.NFD) + if err != nil { + log.Error( + err, + "failed to get configmap with helm settings", + ) + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage( + fmt.Sprintf("failed to get configration to create helm addon: %v", + err, + ), + ) + return + } strategy = helmAddonStrategy{ - config: n.config.helmAddonConfig, - client: n.client, + config: n.config.helmAddonConfig, + client: n.client, + helmChart: helmChart, } default: resp.SetStatus(runtimehooksv1.ResponseStatusFailure) diff --git a/pkg/handlers/generic/lifecycle/nfd/strategy_helmaddon.go b/pkg/handlers/generic/lifecycle/nfd/strategy_helmaddon.go index 602400017..4ef0834b3 100644 --- a/pkg/handlers/generic/lifecycle/nfd/strategy_helmaddon.go +++ b/pkg/handlers/generic/lifecycle/nfd/strategy_helmaddon.go @@ -18,12 +18,10 @@ import ( caaphv1 "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" ) const ( - defaultHelmRepositoryURL = "https://kubernetes-sigs.github.io/node-feature-discovery/charts" - defaultHelmChartVersion = "0.15.2" - defaultHelmChartName = "node-feature-discovery" defaultHelmReleaseName = "node-feature-discovery" defaultHelmReleaseNamespace = "node-feature-discovery" ) @@ -44,7 +42,8 @@ func (c *helmAddonConfig) AddFlags(prefix string, flags *pflag.FlagSet) { type helmAddonStrategy struct { config helmAddonConfig - client ctrlclient.Client + client ctrlclient.Client + helmChart *config.HelmChart } func (s helmAddonStrategy) apply( @@ -66,7 +65,7 @@ func (s helmAddonStrategy) apply( values += fmt.Sprintf(` image: tag: v%s-minimal -`, defaultHelmChartVersion) +`, s.helmChart.Version) hcp := &caaphv1.HelmChartProxy{ TypeMeta: metav1.TypeMeta{ @@ -78,14 +77,14 @@ image: Name: "node-feature-discovery-" + req.Cluster.Name, }, Spec: caaphv1.HelmChartProxySpec{ - RepoURL: defaultHelmRepositoryURL, - ChartName: defaultHelmChartName, + RepoURL: s.helmChart.Repository, + ChartName: s.helmChart.Name, ClusterSelector: metav1.LabelSelector{ MatchLabels: map[string]string{capiv1.ClusterNameLabel: req.Cluster.Name}, }, ReleaseNamespace: defaultHelmReleaseNamespace, ReleaseName: defaultHelmReleaseName, - Version: defaultHelmChartVersion, + Version: s.helmChart.Version, ValuesTemplate: values, }, } diff --git a/pkg/handlers/options/global.go b/pkg/handlers/options/global.go index e355a4366..6f1de11a9 100644 --- a/pkg/handlers/options/global.go +++ b/pkg/handlers/options/global.go @@ -13,7 +13,8 @@ func NewGlobalOptions() *GlobalOptions { } type GlobalOptions struct { - defaultsNamespace string + defaultsNamespace string + helmAddonsConfigMapName string } func (o *GlobalOptions) AddFlags(flags *pflag.FlagSet) { @@ -23,8 +24,18 @@ func (o *GlobalOptions) AddFlags(flags *pflag.FlagSet) { corev1.NamespaceDefault, "namespace for default configurations", ) + flags.StringVar( + &o.helmAddonsConfigMapName, + "helm-addons-configmap", + "default-helm-addons-config", + "Name of helm addons configmap", + ) } func (o *GlobalOptions) DefaultsNamespace() string { return o.defaultsNamespace } + +func (o *GlobalOptions) HelmAddonsConfigMapName() string { + return o.helmAddonsConfigMapName +}