diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ec4e5a2..51b7f17 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -10,6 +10,8 @@ rules: - configmaps verbs: - get + - list + - watch - apiGroups: - "" resources: diff --git a/go.mod b/go.mod index 6cde7f8..bf00221 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,12 @@ toolchain go1.22.4 require ( github.com/Mellanox/maintenance-operator/api v0.0.0-20240916123230-810ab7bb25f4 github.com/Mellanox/rdmamap v1.1.0 + github.com/go-task/slim-sprig/v3 v3.0.0 github.com/jaypipes/ghw v0.12.0 github.com/jaypipes/pcidb v1.0.1 github.com/onsi/ginkgo/v2 v2.20.0 github.com/onsi/gomega v1.34.1 + github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.9.0 github.com/vishvananda/netlink v1.3.0 go.uber.org/zap v1.26.0 @@ -38,7 +40,6 @@ require ( github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect - github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect @@ -56,7 +57,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.20.2 // indirect github.com/prometheus/client_model v0.6.1 // indirect diff --git a/internal/controller/nicconfigurationtemplate_controller.go b/internal/controller/nicconfigurationtemplate_controller.go index 9cec4df..3dc0f4b 100644 --- a/internal/controller/nicconfigurationtemplate_controller.go +++ b/internal/controller/nicconfigurationtemplate_controller.go @@ -19,6 +19,7 @@ package controller import ( "context" "fmt" + "os" "reflect" "slices" "strings" @@ -36,6 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" v1alpha1 "github.com/Mellanox/nic-configuration-operator/api/v1alpha1" + "github.com/Mellanox/nic-configuration-operator/pkg/syncdaemon" ) const nicConfigurationTemplateSyncEventName = "nic-configuration-template-sync-event" @@ -54,6 +56,7 @@ type NicConfigurationTemplateReconciler struct { //+kubebuilder:rbac:groups=configuration.net.nvidia.com,resources=nicdevices,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=configuration.net.nvidia.com,resources=nicdevices/finalizers,verbs=update //+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;update;patch +//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get //+kubebuilder:rbac:groups="",resources=pods,verbs=list //+kubebuilder:rbac:groups="",resources=pods/eviction,verbs=create;delete;get;list;patch;update;watch @@ -172,6 +175,12 @@ func (r *NicConfigurationTemplateReconciler) Reconcile(ctx context.Context, req } } + err = syncdaemon.SyncConfigDaemonObjs(ctx, r.Client, r.Scheme, os.Getenv("NAMESPACE")) + if err != nil { + log.Log.Error(err, "failed to sync ds") + return ctrl.Result{}, err + } + return ctrl.Result{}, nil } diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index fac5f0a..89a0eb4 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -82,4 +82,8 @@ const ( Mlx5ModuleVersionPath = "/sys/bus/pci/drivers/mlx5_core/module/version" FwConfigNotAppliedAfterRebootErrorMsg = "firmware configuration failed to apply after reboot" + + ConfigDaemonManifestsPath = "./bindata/manifests/daemon" + + OperatorConfigMapName = "nic-configuration-operator-config" ) diff --git a/pkg/render/render.go b/pkg/render/render.go new file mode 100644 index 0000000..0d47994 --- /dev/null +++ b/pkg/render/render.go @@ -0,0 +1,134 @@ +/* +2024 NVIDIA CORPORATION & AFFILIATES +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package render + +import ( + "bytes" + "io" + "os" + "path/filepath" + "strings" + "text/template" + + sprig "github.com/go-task/slim-sprig/v3" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/yaml" +) + +type RenderData struct { + Funcs template.FuncMap + Data map[string]interface{} +} + +func MakeRenderData() RenderData { + return RenderData{ + Funcs: template.FuncMap{}, + Data: map[string]interface{}{}, + } +} + +// RenderDir will render all manifests in a directory, descending in to subdirectories +// It will perform template substitutions based on the data supplied by the RenderData +func RenderDir(manifestDir string, d *RenderData) ([]*unstructured.Unstructured, error) { + out := []*unstructured.Unstructured{} + + if err := filepath.Walk(manifestDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + // Skip non-manifest files + if !(strings.HasSuffix(path, ".yml") || strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".json")) { + return nil + } + + objs, err := RenderTemplate(path, d) + if err != nil { + return err + } + out = append(out, objs...) + return nil + }); err != nil { + return nil, errors.Wrap(err, "error rendering manifests") + } + + return out, nil +} + +// RenderTemplate reads, renders, and attempts to parse a yaml or +// json file representing one or more k8s api objects +func RenderTemplate(path string, d *RenderData) ([]*unstructured.Unstructured, error) { + rendered, err := renderTemplate(path, d) + if err != nil { + return nil, err + } + + out := []*unstructured.Unstructured{} + + // special case - if the entire file is whitespace, skip + if len(strings.TrimSpace(rendered.String())) == 0 { + return out, nil + } + + decoder := yaml.NewYAMLOrJSONDecoder(rendered, 4096) + for { + u := unstructured.Unstructured{} + if err := decoder.Decode(&u); err != nil { + if err == io.EOF { + break + } + return nil, errors.Wrapf(err, "failed to unmarshal manifest %s", path) + } + + if u.Object == nil { + continue + } + + out = append(out, &u) + } + + return out, nil +} + +func renderTemplate(path string, d *RenderData) (*bytes.Buffer, error) { + tmpl := template.New(path).Option("missingkey=error") + if d.Funcs != nil { + tmpl.Funcs(d.Funcs) + } + + // Add universal functions + tmpl.Funcs(sprig.TxtFuncMap()) + + source, err := os.ReadFile(path) + if err != nil { + return nil, errors.Wrapf(err, "failed to read manifest %s", path) + } + + if _, err := tmpl.Parse(string(source)); err != nil { + return nil, errors.Wrapf(err, "failed to parse manifest %s as template", path) + } + + rendered := bytes.Buffer{} + if err := tmpl.Execute(&rendered, d.Data); err != nil { + return nil, errors.Wrapf(err, "failed to render manifest %s", path) + } + + return &rendered, nil +} diff --git a/pkg/render/render_test.go b/pkg/render/render_test.go new file mode 100644 index 0000000..9682300 --- /dev/null +++ b/pkg/render/render_test.go @@ -0,0 +1,132 @@ +/* +2024 NVIDIA CORPORATION & AFFILIATES +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package render + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +// TestRenderSimple tests rendering a single object with no templates +func TestRenderSimple(t *testing.T) { + g := NewGomegaWithT(t) + + d := MakeRenderData() + + o1, err := RenderTemplate("testdata/manifests/simple.yaml", &d) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(o1).To(HaveLen(1)) + expected := ` +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "busybox1", + "namespace": "ns" + }, + "spec": { + "containers": [ + { + "image": "busybox" + } + ] + } +} +` + g.Expect(o1[0].MarshalJSON()).To(MatchJSON(expected)) + + // test that json parses the same + o2, err := RenderTemplate("testdata/manifests/simple.json", &d) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(o2).To(Equal(o1)) +} + +func TestRenderMultiple(t *testing.T) { + g := NewGomegaWithT(t) + + p := "testdata/manifests/multiple.yaml" + d := MakeRenderData() + + o, err := RenderTemplate(p, &d) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(o).To(HaveLen(3)) + + g.Expect(o[0].GetObjectKind().GroupVersionKind().String()).To(Equal("/v1, Kind=Pod")) + g.Expect(o[1].GetObjectKind().GroupVersionKind().String()).To(Equal("rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding")) + g.Expect(o[2].GetObjectKind().GroupVersionKind().String()).To(Equal("/v1, Kind=ConfigMap")) +} + +func TestTemplate(t *testing.T) { + g := NewGomegaWithT(t) + + p := "testdata/manifests/template.yaml" + + // Test that missing variables are detected + d := MakeRenderData() + _, err := RenderTemplate(p, &d) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(HaveSuffix(`function "fname" not defined`)) + + // Set expected function (but not variable) + d.Funcs["fname"] = func(s string) string { return "test-" + s } + _, err = RenderTemplate(p, &d) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(HaveSuffix(`has no entry for key "Namespace"`)) + + // now we can render + d.Data["Namespace"] = "myns" + o, err := RenderTemplate(p, &d) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(o[0].GetName()).To(Equal("test-podname")) + g.Expect(o[0].GetNamespace()).To(Equal("myns")) +} + +// TestTemplateWithEmptyObject tests the case where a file generates additional nil objects when rendered. An empty +// object can also occur in the particular case shown in the testfile below when minus is missing at the end of the +// first expression (i.e. {{- if .Enable }}). +func TestTemplateWithEmptyObject(t *testing.T) { + g := NewGomegaWithT(t) + + p := "testdata/manifests/template_with_empty_object.yaml" + + d := MakeRenderData() + d.Data["Enable"] = true + o, err := RenderTemplate(p, &d) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(len(o)).To(Equal(2)) + g.Expect(o[0].GetName()).To(Equal("pod1")) + g.Expect(o[0].GetNamespace()).To(Equal("namespace1")) + g.Expect(o[1].GetName()).To(Equal("pod2")) + g.Expect(o[1].GetNamespace()).To(Equal("namespace2")) +} + +func TestRenderDir(t *testing.T) { + g := NewGomegaWithT(t) + + d := MakeRenderData() + d.Funcs["fname"] = func(s string) string { return s } + d.Data["Namespace"] = "myns" + d.Data["Enable"] = true + + o, err := RenderDir("testdata/manifests", &d) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(o).To(HaveLen(8)) +} diff --git a/pkg/render/testdata/doc.txt b/pkg/render/testdata/doc.txt new file mode 100644 index 0000000..3203f10 --- /dev/null +++ b/pkg/render/testdata/doc.txt @@ -0,0 +1 @@ +Testing that documentation files are ignored. diff --git a/pkg/render/testdata/manifests/multiple.yaml b/pkg/render/testdata/manifests/multiple.yaml new file mode 100644 index 0000000..f76f01a --- /dev/null +++ b/pkg/render/testdata/manifests/multiple.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Pod +metadata: + name: busybox1 + namespace: ns +spec: + containers: + - image: busybox + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: crb +roleRef: + apiGroup: rbac.authorization.k8s.io/v1 + kind: ClusterRole + name: cr +subjects: + - kind: ServiceAccount + name: sa + namespace: ns + +--- + +apiVersion: v1 +kind: ConfigMap +metadata: + name: cm + namespace: ns +data: + key: val diff --git a/pkg/render/testdata/manifests/simple.json b/pkg/render/testdata/manifests/simple.json new file mode 100644 index 0000000..ed1a1b3 --- /dev/null +++ b/pkg/render/testdata/manifests/simple.json @@ -0,0 +1,15 @@ +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "busybox1", + "namespace": "ns" + }, + "spec": { + "containers": [ + { + "image": "busybox" + } + ] + } +} diff --git a/pkg/render/testdata/manifests/simple.yaml b/pkg/render/testdata/manifests/simple.yaml new file mode 100644 index 0000000..637e355 --- /dev/null +++ b/pkg/render/testdata/manifests/simple.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Pod +metadata: + name: busybox1 + namespace: ns +spec: + containers: + - image: busybox diff --git a/pkg/render/testdata/manifests/template.yaml b/pkg/render/testdata/manifests/template.yaml new file mode 100644 index 0000000..2e9a2f7 --- /dev/null +++ b/pkg/render/testdata/manifests/template.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Pod +metadata: + namespace: {{.Namespace}} + name: {{fname "podname"}} +spec: + containers: + - image: "busybox" diff --git a/pkg/render/testdata/manifests/template_with_empty_object.yaml b/pkg/render/testdata/manifests/template_with_empty_object.yaml new file mode 100644 index 0000000..a3b5a5d --- /dev/null +++ b/pkg/render/testdata/manifests/template_with_empty_object.yaml @@ -0,0 +1,20 @@ +{{- if .Enable }} +--- +apiVersion: v1 +kind: Pod +metadata: + namespace: namespace1 + name: pod1 +spec: + containers: + - image: "busybox" +--- +apiVersion: v1 +kind: Pod +metadata: + namespace: namespace2 + name: pod2 +spec: + containers: + - image: "busybox" +{{- end }} diff --git a/pkg/syncdaemon/syncdaemon.go b/pkg/syncdaemon/syncdaemon.go new file mode 100644 index 0000000..893cfda --- /dev/null +++ b/pkg/syncdaemon/syncdaemon.go @@ -0,0 +1,130 @@ +/* +2024 NVIDIA CORPORATION & AFFILIATES +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package syncdaemon + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/Mellanox/nic-configuration-operator/pkg/consts" + "github.com/Mellanox/nic-configuration-operator/pkg/render" +) + +type ConfigDaemonParameters struct { + Image string + Namespace string + ReleaseVersion string + ServiceAccountName string + NodeSelector string + ImagePullSecrets string + Resources string + LogLevel string +} + +func SyncConfigDaemonObjs(ctx context.Context, + client client.Client, + scheme *runtime.Scheme, + namespace string) error { + log.Log.V(2).Info("Synchronizing configuration daemon objects") + + cfg := &v1.ConfigMap{} + err := client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: consts.OperatorConfigMapName}, cfg) + if err != nil { + if apierrors.IsNotFound(err) { + log.Log.Info("daemon config map not found, daemon set should not be created") + } else { + log.Log.Error(err, "failed to get daemon config map") + return err + } + } + + data := render.MakeRenderData() + data.Data["Image"] = cfg.Data["configDaemonImage"] + data.Data["Namespace"] = namespace + data.Data["ReleaseVersion"] = cfg.Data["releaseVersion"] + data.Data["ServiceAccountName"] = cfg.Data["serviceAccountName"] + data.Data["NodeSelector"] = cfg.Data["nodeSelector"] + data.Data["ImagePullSecrets"] = cfg.Data["imagePullSecrets"] + data.Data["Resources"] = cfg.Data["resources"] + data.Data["LogLevel"] = cfg.Data["logLevel"] + + log.Log.Info("data", "data", data.Data) + + objs, err := render.RenderDir(consts.ConfigDaemonManifestsPath, &data) + if err != nil { + log.Log.Error(err, "Fail to render config daemon manifests") + return err + } + // Sync DaemonSets + for _, obj := range objs { + if err := controllerutil.SetControllerReference(cfg, obj, scheme, controllerutil.WithBlockOwnerDeletion(false)); err != nil { + return err + } + + if err := applyObject(ctx, client, obj); err != nil { + return fmt.Errorf("failed to apply object %v with err: %v", obj, err) + } + } + return nil +} + +// applyObject applies the desired object against the apiserver, +// merging it with any existing objects if already present. +func applyObject(ctx context.Context, client client.Client, obj *unstructured.Unstructured) error { + name := obj.GetName() + namespace := obj.GetNamespace() + if name == "" { + return errors.Errorf("Object %s has no name", obj.GroupVersionKind().String()) + } + gvk := obj.GroupVersionKind() + // used for logging and errors + objDesc := fmt.Sprintf("(%s) %s/%s", gvk.String(), namespace, name) + + // Get existing + existing := &unstructured.Unstructured{} + existing.SetGroupVersionKind(gvk) + err := client.Get(ctx, types.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()}, existing) + + if err != nil && apierrors.IsNotFound(err) { + err := client.Create(ctx, obj) + if err != nil { + return errors.Wrapf(err, "could not create %s", objDesc) + } + return nil + } + if err != nil { + return errors.Wrapf(err, "could not retrieve existing %s", objDesc) + } + + if !equality.Semantic.DeepEqual(existing, obj) { + if err := client.Update(ctx, obj); err != nil { + return errors.Wrapf(err, "could not update object %s", objDesc) + } + } + + return nil +}