Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

api: add support for NRI-native CDI injection #98

Merged
merged 7 commits into from
Aug 13, 2024
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ $(BIN_PATH)/template: $(wildcard plugins/template/*.go)

test-gopkgs: ginkgo-tests test-ulimits

SKIPPED_PKGS="ulimit-adjuster"
SKIPPED_PKGS="ulimit-adjuster,device-injector"

ginkgo-tests:
$(Q)$(GINKGO) run \
Expand All @@ -143,6 +143,9 @@ ginkgo-tests:
test-ulimits:
$(Q)cd ./plugins/ulimit-adjuster && $(GO_TEST) -v

test-device-injector:
$(Q)cd ./plugins/device-injector && $(GO_TEST) -v

codecov: SHELL := $(shell which bash)
codecov:
bash <(curl -s https://codecov.io/bash) -f $(COVERAGE_PATH)/coverprofile
Expand Down
23 changes: 23 additions & 0 deletions pkg/adaptation/adaptation_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -488,6 +488,13 @@ var _ = Describe("Plugin container creation adjustments", func() {
case "rlimit":
a.AddRlimit("nofile", 456, 123)

case "CDI-device":
a.AddCDIDevice(
&api.CDIDevice{
Name: "vendor0.com/dev=dev0",
},
)

case "resources/cpu":
a.SetLinuxCPUShares(123)
a.SetLinuxCPUQuota(456)
Expand Down Expand Up @@ -632,6 +639,15 @@ var _ = Describe("Plugin container creation adjustments", func() {
Rlimits: []*api.POSIXRlimit{{Type: "nofile", Soft: 123, Hard: 456}},
},
),
Entry("adjust CDI Devices", "CDI-device",
&api.ContainerAdjustment{
CDIDevices: []*api.CDIDevice{
{
Name: "vendor0.com/dev=dev0",
},
},
},
),
Entry("adjust CPU resources", "resources/cpu",
&api.ContainerAdjustment{
Linux: &api.LinuxContainerAdjustment{
Expand Down Expand Up @@ -1858,6 +1874,7 @@ func stripAdjustment(a *api.ContainerAdjustment) *api.ContainerAdjustment {
stripHooks(a)
stripRlimits(a)
stripLinuxAdjustment(a)
stripCDIDevices(a)
return a
}

Expand Down Expand Up @@ -1918,6 +1935,12 @@ func stripLinuxDevices(a *api.ContainerAdjustment) {
}
}

func stripCDIDevices(a *api.ContainerAdjustment) {
if len(a.CDIDevices) == 0 {
a.CDIDevices = nil
}
}

func stripLinuxResources(r *api.LinuxResources) *api.LinuxResources {
if r == nil {
return nil
Expand Down
1 change: 1 addition & 0 deletions pkg/adaptation/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ type (
LinuxMemory = api.LinuxMemory
LinuxDevice = api.LinuxDevice
LinuxDeviceCgroup = api.LinuxDeviceCgroup
CDIDevice = api.CDIDevice
klihub marked this conversation as resolved.
Show resolved Hide resolved
HugepageLimit = api.HugepageLimit
Hooks = api.Hooks
Hook = api.Hook
Expand Down
50 changes: 50 additions & 0 deletions pkg/adaptation/result.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ func collectCreateContainerResult(request *CreateContainerRequest) *result {
Env: []*KeyValue{},
Hooks: &Hooks{},
Rlimits: []*POSIXRlimit{},
CDIDevices: []*CDIDevice{},
Linux: &LinuxContainerAdjustment{
Devices: []*LinuxDevice{},
Resources: &LinuxResources{
Expand Down Expand Up @@ -219,6 +220,9 @@ func (r *result) adjust(rpl *ContainerAdjustment, plugin string) error {
if err := r.adjustRlimits(rpl.Rlimits, plugin); err != nil {
return err
}
if err := r.adjustCDIDevices(rpl.CDIDevices, plugin); err != nil {
return err
}

return nil
}
Expand Down Expand Up @@ -388,6 +392,36 @@ func (r *result) adjustDevices(devices []*LinuxDevice, plugin string) error {
return nil
}

func (r *result) adjustCDIDevices(devices []*CDIDevice, plugin string) error {
if len(devices) == 0 {
return nil
}

// Notes:
// CDI devices are opaque references, typically to vendor specific
// devices. They get resolved to actual devices and potential related
// mounts, environment variables, etc. in the runtime. Unlike with
// devices, we only allow CDI device references to be added to a
// container, not removed. We pass them unresolved to the runtime and
// have them resolved there. Also unlike with devices, we don't include
// CDI device references in creation requests. However, since there
// is typically a strong ownership and a single related management entity
// per vendor/class for these devices we do treat multiple injection of
// the same CDI device reference as an error here.

id := r.request.create.Container.Id

// apply additions to collected adjustments
for _, d := range devices {
if err := r.owners.claimCDIDevice(id, d.Name, plugin); err != nil {
return err
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

// TODO: consider support for a higher level management plugin having the cap to adjust, esp. outside k8s scenarios

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you mean to verbatim include that comment here... as a reminder to consider that in the future ?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sort of.. suggesting the consideration of a TODO here, not that explicit text, or we can just open an issue though the way issues get removed when the bots are turned on :-)

}
r.reply.adjust.CDIDevices = append(r.reply.adjust.CDIDevices, d)
}

return nil
}

func (r *result) adjustEnv(env []*KeyValue, plugin string) error {
if len(env) == 0 {
return nil
Expand Down Expand Up @@ -891,6 +925,7 @@ type owners struct {
annotations map[string]string
mounts map[string]string
devices map[string]string
cdiDevices map[string]string
env map[string]string
memLimit string
memReservation string
Expand Down Expand Up @@ -937,6 +972,10 @@ func (ro resultOwners) claimDevice(id, path, plugin string) error {
return ro.ownersFor(id).claimDevice(path, plugin)
}

func (ro resultOwners) claimCDIDevice(id, path, plugin string) error {
return ro.ownersFor(id).claimCDIDevice(path, plugin)
}

func (ro resultOwners) claimEnv(id, name, plugin string) error {
return ro.ownersFor(id).claimEnv(name, plugin)
}
Expand Down Expand Up @@ -1062,6 +1101,17 @@ func (o *owners) claimDevice(path, plugin string) error {
return nil
}

func (o *owners) claimCDIDevice(name, plugin string) error {
if o.cdiDevices == nil {
o.cdiDevices = make(map[string]string)
}
if other, taken := o.cdiDevices[name]; taken {
return conflict(plugin, other, "CDI device", name)
}
o.cdiDevices[name] = plugin
return nil
}

func (o *owners) claimEnv(name, plugin string) error {
if o.env == nil {
o.env = make(map[string]string)
Expand Down
5 changes: 5 additions & 0 deletions pkg/api/adjustment.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,11 @@ func (a *ContainerAdjustment) RemoveDevice(path string) {
})
}

// AddCDIDevice records the addition of the given CDI device to a container.
func (a *ContainerAdjustment) AddCDIDevice(d *CDIDevice) {
a.CDIDevices = append(a.CDIDevices, d) // TODO: should we dup d here ?
}

// SetLinuxMemoryLimit records setting the memory limit for a container.
func (a *ContainerAdjustment) SetLinuxMemoryLimit(value int64) {
a.initLinuxResourcesMemory()
Expand Down
Loading
Loading