From 4e79207dd182e2ab368e44fa1b4c6aa24695b3b3 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Tue, 2 Jul 2024 12:07:15 +0300 Subject: [PATCH] device-injector: add support for CDI injection. Add support for injecting annotated CDI devices using the new native NRI CDI injection API. Signed-off-by: Krisztian Litkey --- plugins/device-injector/README.md | 42 ++- plugins/device-injector/device-injector.go | 198 ++++++++----- .../device-injector/device-injector_test.go | 267 ++++++++++++++++++ plugins/device-injector/go.mod | 4 + plugins/device-injector/go.sum | 4 + 5 files changed, 438 insertions(+), 77 deletions(-) create mode 100644 plugins/device-injector/device-injector_test.go diff --git a/plugins/device-injector/README.md b/plugins/device-injector/README.md index b5cb89a3..670f09ca 100644 --- a/plugins/device-injector/README.md +++ b/plugins/device-injector/README.md @@ -1,16 +1,19 @@ ## Device Injector Plugin -This sample plugin can inject devices and mounts into containers using -pod annotations. +This sample plugin can inject Linux device nodes, CDI devices, and mounts into +containers using pod annotations. ### Device Annotations Devices are annotated using the `devices.nri.io` annotation key prefix. The key `devices.nri.io/container.$CONTAINER_NAME` annotates devices to be injected into `$CONTAINER_NAME`. The keys `devices.nri.io` and -`devices.nri.io/pod` annotate devices to be injected into all containers. +`devices.nri.io/pod` annotate devices to be injected into containers +without any other, container-specific device annotations. Only one of +these latter two annotations will be ever taken into account. If both are +present, it is unspecified which one is used. -The annotation syntax for device injection is +The annotation value syntax for device injection is ``` - path: /dev/dev0 @@ -26,10 +29,39 @@ The annotation syntax for device injection is `file_mode`, `uid` and `gid` can be omitted, the rest are mandatory. +### CDI Device Annotations + +CDI devices are annotated in a similar manner to devices, but using the +`cdi-devices.nri.io` annotation key prefix. The annotation value for CDI +devices is the list of CDI device names to inject. + +For instance, the following annotation + +``` +metadata: + name: bash + annotations: + cdi-devices.nri.io/container.c0: | + - vendor0.com/device=null + cdi-devices.nri.io/container.c1: | + - vendor0.com/device=zero + cdi-devices.nri.io/container.c2: | + - vendor1.com/device=dev0 + - vendor1.com/device=dev1 + cdi-devices.nri.io/container.mgmt: | + - vendor0.com/device=all +``` + +requests the injection of the CDI device vendor0.com/device=null to container +c0, the injection of the CDI device vendor0.com/device=zero to container c1, +the injection of the CDI devices vendor1.com/device=dev0 and vendor1.com/device=dev1 +to container c2, and the injection of the CDI device vendor0.com/device=all to +container mgmt. + ### Mount Annotations Mounts are annotated in a similar manner to devices, but using the -`mounts.nri.io` annotation key prefix. The annotation syntax for mount +`mounts.nri.io` annotation key prefix. The annotation value syntax for mount injection is ``` diff --git a/plugins/device-injector/device-injector.go b/plugins/device-injector/device-injector.go index 3c19cd2b..c355e965 100644 --- a/plugins/device-injector/device-injector.go +++ b/plugins/device-injector/device-injector.go @@ -35,6 +35,8 @@ const ( deviceKey = "devices.nri.io" // Prefix of the key used for mount annotations. mountKey = "mounts.nri.io" + // Prefix of the key used for CDI device annotations. + cdiDeviceKey = "cdi-devices.nri.io" ) var ( @@ -67,88 +69,67 @@ type plugin struct { } // CreateContainer handles container creation requests. -func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, container *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) { - var ( - ctrName string - devices []device - mounts []mount - err error - ) - - ctrName = containerName(pod, container) - +func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, ctr *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) { if verbose { - dump("CreateContainer", "pod", pod, "container", container) + dump("CreateContainer", "pod", pod, "container", ctr) } adjust := &api.ContainerAdjustment{} - // inject devices to container - devices, err = parseDevices(container.Name, pod.Annotations) - if err != nil { + if err := injectDevices(pod, ctr, adjust); err != nil { return nil, nil, err } - if len(devices) == 0 { - log.Infof("%s: no devices annotated...", ctrName) - } else { - if verbose { - dump(ctrName, "annotated devices", devices) - } - - for _, d := range devices { - adjust.AddDevice(d.toNRI()) - if !verbose { - log.Infof("%s: injected device %q...", ctrName, d.Path) - } - } + if err := injectCDIDevices(pod, ctr, adjust); err != nil { + return nil, nil, err } - // inject mounts to container - mounts, err = parseMounts(container.Name, pod.Annotations) - if err != nil { + if err := injectMounts(pod, ctr, adjust); err != nil { return nil, nil, err } - if len(mounts) == 0 { - log.Infof("%s: no mounts annotated...", ctrName) - } else { - if verbose { - dump(ctrName, "annotated mounts", mounts) - } + return adjust, nil, nil - for _, m := range mounts { - adjust.AddMount(m.toNRI()) - if !verbose { - log.Infof("%s: injected mount %q -> %q...", ctrName, m.Source, m.Destination) - } - } + if verbose { + dump(containerName(pod, ctr), "ContainerAdjustment", adjust) + } + + return adjust, nil, nil +} + +func injectDevices(pod *api.PodSandbox, ctr *api.Container, a *api.ContainerAdjustment) error { + devices, err := parseDevices(ctr.Name, pod.Annotations) + if err != nil { + return err + } + + if len(devices) == 0 { + log.Debugf("%s: no devices annotated...", containerName(pod, ctr)) + return nil } if verbose { - dump(ctrName, "ContainerAdjustment", adjust) + dump(containerName(pod, ctr), "annotated devices", devices) } - return adjust, nil, nil + for _, d := range devices { + a.AddDevice(d.toNRI()) + if !verbose { + log.Infof("%s: injected device %q...", containerName(pod, ctr), d.Path) + } + } + + return nil } func parseDevices(ctr string, annotations map[string]string) ([]device, error) { var ( - key string - annotation []byte - devices []device + devices []device ) - // look up effective device annotation and unmarshal devices - for _, key = range []string{ - deviceKey + "/container." + ctr, - deviceKey + "/pod", - deviceKey, - } { - if value, ok := annotations[key]; ok { - annotation = []byte(value) - break - } + annotation := getAnnotation(annotations, deviceKey, ctr) + if annotation == nil { + return nil, nil } if annotation == nil { @@ -156,42 +137,115 @@ func parseDevices(ctr string, annotations map[string]string) ([]device, error) { } if err := yaml.Unmarshal(annotation, &devices); err != nil { - return nil, fmt.Errorf("invalid device annotation %q: %w", key, err) + return nil, fmt.Errorf("invalid device annotation %q: %w", string(annotation), err) } return devices, nil } -func parseMounts(ctr string, annotations map[string]string) ([]mount, error) { +func injectCDIDevices(pod *api.PodSandbox, ctr *api.Container, a *api.ContainerAdjustment) error { + devices, err := parseCDIDevices(ctr.Name, pod.Annotations) + if err != nil { + return err + } + + if len(devices) == 0 { + log.Debugf("%s: no CDI devices annotated...", containerName(pod, ctr)) + return nil + } + + if verbose { + dump(containerName(pod, ctr), "annotated CDI devices", devices) + } + + for _, name := range devices { + a.AddCDIDevice( + &api.CDIDevice{ + Name: name, + }, + ) + if !verbose { + log.Infof("%s: injected CDI device %q...", containerName(pod, ctr), name) + } + } + + return nil +} + +func parseCDIDevices(ctr string, annotations map[string]string) ([]string, error) { var ( - key string - annotation []byte - mounts []mount + cdiDevices []string ) - // look up effective device annotation and unmarshal devices - for _, key = range []string{ - mountKey + "/container." + ctr, - mountKey + "/pod", - mountKey, - } { - if value, ok := annotations[key]; ok { - annotation = []byte(value) - break + annotation := getAnnotation(annotations, cdiDeviceKey, ctr) + if annotation == nil { + return nil, nil + } + + if err := yaml.Unmarshal(annotation, &cdiDevices); err != nil { + return nil, fmt.Errorf("invalid CDI device annotation %q: %w", string(annotation), err) + } + + return cdiDevices, nil +} + +func injectMounts(pod *api.PodSandbox, ctr *api.Container, a *api.ContainerAdjustment) error { + mounts, err := parseMounts(ctr.Name, pod.Annotations) + if err != nil { + return err + } + + if len(mounts) == 0 { + log.Debugf("%s: no mounts annotated...", containerName(pod, ctr)) + return nil + } + + if verbose { + dump(containerName(pod, ctr), "annotated mounts", mounts) + } + + for _, m := range mounts { + a.AddMount(m.toNRI()) + if !verbose { + log.Infof("%s: injected mount %q -> %q...", containerName(pod, ctr), + m.Source, m.Destination) } } + return nil +} + +func parseMounts(ctr string, annotations map[string]string) ([]mount, error) { + var ( + mounts []mount + ) + + annotation := getAnnotation(annotations, mountKey, ctr) if annotation == nil { return nil, nil } if err := yaml.Unmarshal(annotation, &mounts); err != nil { - return nil, fmt.Errorf("invalid mount annotation %q: %w", key, err) + return nil, fmt.Errorf("invalid mount annotation %q: %w", string(annotation), err) } return mounts, nil } +func getAnnotation(annotations map[string]string, mainKey, ctr string) []byte { + for _, key := range []string{ + mainKey + "/container." + ctr, + mainKey + "/pod", + mainKey, + } { + if value, ok := annotations[key]; ok { + return []byte(value) + } + } + + return nil +} + // Convert a device to the NRI API representation. func (d *device) toNRI() *api.LinuxDevice { apiDev := &api.LinuxDevice{ diff --git a/plugins/device-injector/device-injector_test.go b/plugins/device-injector/device-injector_test.go new file mode 100644 index 00000000..350f83a2 --- /dev/null +++ b/plugins/device-injector/device-injector_test.go @@ -0,0 +1,267 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseDevices(t *testing.T) { + type testCase struct { + name string + annotations map[string]string + result []device + } + + for _, tc := range []*testCase{ + { + name: "no annotated devices", + annotations: map[string]string{ + "foo": "bar", + }, + }, + { + name: "a single annotated device", + annotations: map[string]string{ + "devices.nri.io/container.ctr0": ` +- path: /dev/test-null + type: c + major: 1 + minor: 3 +`, + }, + result: []device{ + { + Path: "/dev/test-null", + Type: "c", + Major: 1, + Minor: 3, + }, + }, + }, + { + name: "multiple annotated devices", + annotations: map[string]string{ + "devices.nri.io/container.ctr0": ` +- path: /dev/test-null + type: c + major: 1 + minor: 3 +- path: /dev/test-zero + type: c + major: 1 + minor: 5 +`, + }, + result: []device{ + { + Path: "/dev/test-null", + Type: "c", + Major: 1, + Minor: 3, + }, + { + Path: "/dev/test-zero", + Type: "c", + Major: 1, + Minor: 5, + }, + }, + }, + { + name: "annotated devices for non-matching container name", + annotations: map[string]string{ + "devices.nri.io/container.ctr1": ` +- path: /dev/test-null + type: c + major: 1 + minor: 3 +- path: /dev/test-zero + type: c + major: 1 + minor: 5 +`, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + devices, err := parseDevices("ctr0", tc.annotations) + require.Nil(t, err, "device parsing error") + require.Equal(t, tc.result, devices, "parsed devices") + }) + } +} + +func TestParseCDIDevices(t *testing.T) { + type testCase struct { + name string + annotations map[string]string + result []string + } + + for _, tc := range []*testCase{ + { + name: "no annotated CDI devices", + annotations: map[string]string{ + "foo": "bar", + }, + }, + { + name: "a single annotated CDI device", + annotations: map[string]string{ + "cdi-devices.nri.io/container.ctr0": ` +- vendor0.com/device=null +`, + }, + result: []string{ + "vendor0.com/device=null", + }, + }, + { + name: "multiple annotated CDI devices", + annotations: map[string]string{ + "cdi-devices.nri.io/container.ctr0": ` +- vendor0.com/device=null +- vendor0.com/device=zero +`, + }, + result: []string{ + "vendor0.com/device=null", + "vendor0.com/device=zero", + }, + }, + { + name: "annotated CDI devices for non-matching container name", + annotations: map[string]string{ + "cdi-devices.nri.io/container.ctr1": ` +- vendor0.com/device=null +- vendor0.com/device=zero +`, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + devices, err := parseCDIDevices("ctr0", tc.annotations) + require.Nil(t, err, "CDI device parsing error") + require.Equal(t, tc.result, devices, "parsed CDI devices") + }) + } +} + +func TestParseMounts(t *testing.T) { + type testCase struct { + name string + annotations map[string]string + result []mount + } + + for _, tc := range []*testCase{ + { + name: "no annotated mounts", + annotations: map[string]string{ + "foo": "bar", + }, + }, + { + name: "a single annotated mount", + annotations: map[string]string{ + "mounts.nri.io/container.ctr0": ` +- source: /foo + destination: /host/foo + type: bind + options: + - bind + - ro +`, + }, + result: []mount{ + { + Source: "/foo", + Destination: "/host/foo", + Type: "bind", + Options: []string{ + "bind", + "ro", + }, + }, + }, + }, + { + name: "multiple annotated mounts", + annotations: map[string]string{ + "mounts.nri.io/container.ctr0": ` +- source: /foo + destination: /host/foo + type: bind + options: + - bind +- source: /bar + destination: /host/bar + type: bind + options: + - bind + - ro +`, + }, + result: []mount{ + { + Source: "/foo", + Destination: "/host/foo", + Type: "bind", + Options: []string{ + "bind", + }, + }, + { + Source: "/bar", + Destination: "/host/bar", + Type: "bind", + Options: []string{ + "bind", + "ro", + }, + }, + }, + }, + { + name: "annotated mounts for non-matching container name", + annotations: map[string]string{ + "mounts.nri.io/container.ctr1": ` +- source: /foo + destination: /host/foo + type: bind + options: + - bind +- source: /bar + destination: /host/bar + type: bind + options: + - bind + - ro +`, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + devices, err := parseMounts("ctr0", tc.annotations) + require.Nil(t, err, "mount parsing error") + require.Equal(t, tc.result, devices, "parsed mounts") + }) + } +} diff --git a/plugins/device-injector/go.mod b/plugins/device-injector/go.mod index e651c9d4..4952b4aa 100644 --- a/plugins/device-injector/go.mod +++ b/plugins/device-injector/go.mod @@ -5,14 +5,17 @@ go 1.20 require ( github.com/containerd/nri v0.2.0 github.com/sirupsen/logrus v1.9.0 + github.com/stretchr/testify v1.8.0 sigs.k8s.io/yaml v1.3.0 ) require ( github.com/containerd/ttrpc v1.2.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect @@ -20,6 +23,7 @@ require ( google.golang.org/grpc v1.57.1 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/cri-api v0.25.3 // indirect ) diff --git a/plugins/device-injector/go.sum b/plugins/device-injector/go.sum index 238d00b7..01491189 100644 --- a/plugins/device-injector/go.sum +++ b/plugins/device-injector/go.sum @@ -25,8 +25,11 @@ github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3x github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -75,6 +78,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/cri-api v0.25.3 h1:YaiQ05CM4+5L2DAz0KoSa4sv4/VlQvLbf3WHKICPSXs= k8s.io/cri-api v0.25.3/go.mod h1:riC/P0yOGUf2K1735wW+CXs1aY2ctBgePtnnoFLd0dU= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=