diff --git a/Tiltfile b/Tiltfile index 463f8fbcf82a..166f8f0befaa 100644 --- a/Tiltfile +++ b/Tiltfile @@ -1,4 +1,4 @@ -# -*- mode: Python -*- +Tiltfile# -*- mode: Python -*- envsubst_cmd = "./hack/tools/bin/envsubst" clusterctl_cmd = "./bin/clusterctl" @@ -482,6 +482,19 @@ def deploy_observability(): objects = ["capi-visualizer:serviceaccount"], ) +def deploy_kustomizations(): + for name in settings.get("deploy_kustomizations", []): + yaml = read_file("./.tiltbuild/yaml/{}.kustomization.yaml".format(name)) + k8s_yaml(yaml) + objs = decode_yaml_stream(yaml) + print("objects") + print(find_all_objects_names(objs)) + k8s_resource( + new_name = name, + objects = find_all_objects_names(objs), + labels = ["kustomization"], + ) + def prepare_all(): tools_arg = "--tools kustomize,envsubst,clusterctl " tilt_settings_file_arg = "--tilt-settings-file " + tilt_file @@ -640,6 +653,8 @@ deploy_provider_crds() deploy_observability() +deploy_kustomizations() + enable_providers() cluster_templates() diff --git a/hack/observability/kube-state-metrics/chart/cm-crd-sidecar.yaml b/hack/observability/kube-state-metrics/chart/cm-crd-sidecar.yaml new file mode 100644 index 000000000000..98c7eb16dfd3 --- /dev/null +++ b/hack/observability/kube-state-metrics/chart/cm-crd-sidecar.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-state-metrics-crd-sidecar-script +data: + compile.sh: | + #!/bin/sh + + set -x + + SOURCE_DIR="/tmp" + TARGET_FILE="/etc/config/crd-config.yaml" + # This script will use a temporary file to only overwrite the target file once. + TARGET_FILE_TMP="${TARGET_FILE}.tmp" + + # Create header + cat << EOF > "${TARGET_FILE_TMP}" + kind: CustomResourceStateMetrics + spec: + resources: + EOF + + # Append custom resource config of all files but remove headers + for f in $(ls -1 ${SOURCE_DIR}/*.yaml); do + cat $f | grep -v -E -e '^(-|kind: CustomResourceStateMetrics|spec:| +resources:)' \ + >> "${TARGET_FILE_TMP}" + done + + # Overwrite target file + mv "${TARGET_FILE_TMP}" "${TARGET_FILE}" diff --git a/hack/observability/kube-state-metrics/chart/kustomization.yaml b/hack/observability/kube-state-metrics/chart/kustomization.yaml index dcc7ae500fd4..517ac6be0d3d 100644 --- a/hack/observability/kube-state-metrics/chart/kustomization.yaml +++ b/hack/observability/kube-state-metrics/chart/kustomization.yaml @@ -11,3 +11,12 @@ helmGlobals: # Otherwise "go mod tidy" picks up dependencies of go files contained in the Helm Chart. # "go mod tidy" ignores folders that begin with ".": https://pkg.go.dev/cmd/go#hdr-Package_lists_and_patterns. chartHome: .charts + +resources: +- rbac-crd-aggregation.yaml +- cm-crd-sidecar.yaml + +patches: +- path: patch-crd-sidecar.yaml + target: + kind: Deployment \ No newline at end of file diff --git a/hack/observability/kube-state-metrics/chart/patch-crd-sidecar.yaml b/hack/observability/kube-state-metrics/chart/patch-crd-sidecar.yaml new file mode 100644 index 000000000000..d75d9db31ebe --- /dev/null +++ b/hack/observability/kube-state-metrics/chart/patch-crd-sidecar.yaml @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-state-metrics +spec: + template: + spec: + containers: + - env: + - name: LABEL + value: kube-state-metrics/custom-resource + - name: FOLDER + value: /tmp + - name: RESOURCE + value: configmap + - name: NAMESPACE + value: observability + - name: SCRIPT + value: /script/compile.sh + image: kiwigrid/k8s-sidecar:latest + name: crd-sidecar + volumeMounts: + - mountPath: /etc/config + name: config-volume + - mountPath: /script + name: compile-script + initContainers: + - command: + - /bin/sh + - -c + - | + cat << EOF > "/etc/config/crd-config.yaml" + kind: CustomResourceStateMetrics + spec: + resources: [] + EOF + image: kiwigrid/k8s-sidecar:latest + name: init-crd-config + volumeMounts: + - mountPath: /etc/config + name: config-volume + volumes: + - configMap: + defaultMode: 511 + name: kube-state-metrics-crd-sidecar-script + name: compile-script \ No newline at end of file diff --git a/hack/observability/kube-state-metrics/chart/rbac-crd-aggregation.yaml b/hack/observability/kube-state-metrics/chart/rbac-crd-aggregation.yaml new file mode 100644 index 000000000000..e6bd331ad60c --- /dev/null +++ b/hack/observability/kube-state-metrics/chart/rbac-crd-aggregation.yaml @@ -0,0 +1,24 @@ +--- +# ClusterRole to aggregate other ClusterRoles for different Custom Resource Configurations +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-state-metrics-aggregated-role +aggregationRule: + clusterRoleSelectors: + - matchLabels: + kube-state-metrics/aggregate-to-manager: "true" +--- +# ClusterRoleBinding for the aggregation role +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-state-metrics-custom-resource-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-state-metrics-aggregated-role +subjects: +- kind: ServiceAccount + name: kube-state-metrics + namespace: observability \ No newline at end of file diff --git a/hack/observability/kube-state-metrics/chart/values.yaml b/hack/observability/kube-state-metrics/chart/values.yaml index 7413a56439bd..faa58753e331 100644 --- a/hack/observability/kube-state-metrics/chart/values.yaml +++ b/hack/observability/kube-state-metrics/chart/values.yaml @@ -4,9 +4,8 @@ volumeMounts: name: config-volume volumes: - - configMap: - name: kube-state-metrics-crd-config - name: config-volume + - emptyDir: {} + name: config-volume extraArgs: - "--custom-resource-state-config-file=/etc/config/crd-config.yaml" @@ -21,33 +20,3 @@ rbac: - get - list - watch - - apiGroups: - - cluster.x-k8s.io - resources: - - clusterclasses - - clusters - - machinedeployments - - machinepools - - machinesets - - machines - - machinehealthchecks - verbs: - - get - - list - - watch - - apiGroups: - - controlplane.cluster.x-k8s.io - resources: - - kubeadmcontrolplanes - verbs: - - get - - list - - watch - - apiGroups: - - bootstrap.cluster.x-k8s.io - resources: - - kubeadmconfigs - verbs: - - get - - list - - watch diff --git a/hack/observability/kube-state-metrics/crd-clusterrole.yaml b/hack/observability/kube-state-metrics/crd-clusterrole.yaml new file mode 100644 index 000000000000..4e482ffbcb24 --- /dev/null +++ b/hack/observability/kube-state-metrics/crd-clusterrole.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-state-metrics-custom-resource-capi + labels: + kube-state-metrics/aggregate-to-manager: "true" +rules: +- apiGroups: + - cluster.x-k8s.io + resources: + - clusterclasses + - clusters + - machinedeployments + - machinepools + - machinesets + - machines + - machinehealthchecks + verbs: + - get + - list + - watch +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - kubeadmcontrolplanes + verbs: + - get + - list + - watch +- apiGroups: + - bootstrap.cluster.x-k8s.io + resources: + - kubeadmconfigs + verbs: + - get + - list + - watch \ No newline at end of file diff --git a/hack/observability/kube-state-metrics/kustomization.yaml b/hack/observability/kube-state-metrics/kustomization.yaml index cdcfcee5dc5d..489ba25e795f 100644 --- a/hack/observability/kube-state-metrics/kustomization.yaml +++ b/hack/observability/kube-state-metrics/kustomization.yaml @@ -1,15 +1,15 @@ resources: - ../namespace.yaml - # The kube-state-metrics helm chart will reference a configmap with name `kube-state-metrics-crd-config`. - # The configMapGenerator below will create the configmap and append a hash suffix calculated from its - # content to the name. Kustomize will append the suffix hash to all references in the helm chart, but - # only when the helm chart content is referenced in "resources". - # This would not work if the helm chart is configured in this file via the "helmCharts" option. - ./chart + - ./crd-clusterrole.yaml namespace: observability configMapGenerator: -- name: kube-state-metrics-crd-config +- name: kube-state-metrics-crd-config-capi files: - - crd-config.yaml + - capi.yaml=crd-config.yaml + options: + disableNameSuffixHash: true + labels: + kube-state-metrics/custom-resource: "true" diff --git a/hack/tools/internal/tilt-prepare/main.go b/hack/tools/internal/tilt-prepare/main.go index 5e7f4f7e6ba9..ddd41ee9531a 100644 --- a/hack/tools/internal/tilt-prepare/main.go +++ b/hack/tools/internal/tilt-prepare/main.go @@ -104,13 +104,14 @@ var ( // Types used to de-serialize the tilt-settings.yaml/json file from the Cluster API repository. type tiltSettings struct { - Debug map[string]tiltSettingsDebugConfig `json:"debug,omitempty"` - ExtraArgs map[string]tiltSettingsExtraArgs `json:"extra_args,omitempty"` - DeployCertManager *bool `json:"deploy_cert_manager,omitempty"` - DeployObservability []string `json:"deploy_observability,omitempty"` - EnableProviders []string `json:"enable_providers,omitempty"` - AllowedContexts []string `json:"allowed_contexts,omitempty"` - ProviderRepos []string `json:"provider_repos,omitempty"` + Debug map[string]tiltSettingsDebugConfig `json:"debug,omitempty"` + ExtraArgs map[string]tiltSettingsExtraArgs `json:"extra_args,omitempty"` + DeployCertManager *bool `json:"deploy_cert_manager,omitempty"` + DeployObservability []string `json:"deploy_observability,omitempty"` + DeployKustomizations map[string]string `json:"deploy_kustomizations,omitempty"` + EnableProviders []string `json:"enable_providers,omitempty"` + AllowedContexts []string `json:"allowed_contexts,omitempty"` + ProviderRepos []string `json:"provider_repos,omitempty"` } type tiltSettingsDebugConfig struct { @@ -308,6 +309,13 @@ func tiltResources(ctx context.Context, ts *tiltSettings) error { ) } + for name, path := range ts.DeployKustomizations { + name := fmt.Sprintf("%s.kustomization", name) + tasks[name] = sequential( + kustomizeTask(path, fmt.Sprintf("%s.yaml", name)), + ) + } + // Add read configurations from provider repos for _, p := range ts.ProviderRepos { tiltProviderConfigs, err := loadTiltProvider(p)