From 88fc3f674ffd4ec4f8a77729ac0836e0f6a234a8 Mon Sep 17 00:00:00 2001 From: Sebastian Sch Date: Mon, 28 Oct 2024 15:28:41 +0200 Subject: [PATCH] Add kernel args for rdma mode to complement the modprobe file Signed-off-by: Sebastian Sch --- Makefile | 2 +- bindata/scripts/enable-kargs.sh | 33 -- bindata/scripts/kargs.sh | 55 ++++ .../sriovnetworknodepolicy_controller.go | 13 +- go.mod | 2 +- pkg/consts/constants.go | 8 +- pkg/daemon/daemon.go | 17 +- pkg/daemon/plugin_test.go | 7 + pkg/daemon/writer.go | 1 + .../internal/lib/netlink/mock/mock_netlink.go | 30 +- pkg/host/internal/lib/netlink/netlink.go | 8 +- pkg/host/internal/network/network.go | 24 +- pkg/host/internal/network/network_test.go | 10 +- pkg/plugins/generic/generic_plugin.go | 187 ++++++----- pkg/plugins/generic/generic_plugin_test.go | 13 +- test/conformance/tests/test_networkpool.go | 307 ++++++++++++++++++ .../{enable-kargs_test.sh => kargs_test.sh} | 29 +- test/scripts/rpm-ostree_mock | 6 + 18 files changed, 574 insertions(+), 178 deletions(-) delete mode 100755 bindata/scripts/enable-kargs.sh create mode 100755 bindata/scripts/kargs.sh create mode 100644 test/conformance/tests/test_networkpool.go rename test/scripts/{enable-kargs_test.sh => kargs_test.sh} (61%) diff --git a/Makefile b/Makefile index 310f1dc52..f5ca7edc8 100644 --- a/Makefile +++ b/Makefile @@ -226,7 +226,7 @@ test-e2e-k8s: export NAMESPACE=sriov-network-operator test-e2e-k8s: test-e2e test-bindata-scripts: fakechroot - fakechroot ./test/scripts/enable-kargs_test.sh + fakechroot ./test/scripts/kargs_test.sh test-%: generate manifests envtest KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir=/tmp -p path)" HOME="$(shell pwd)" go test ./$*/... -coverprofile cover-$*.out -coverpkg ./... -v diff --git a/bindata/scripts/enable-kargs.sh b/bindata/scripts/enable-kargs.sh deleted file mode 100755 index 0dc18c784..000000000 --- a/bindata/scripts/enable-kargs.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -set -x - -declare -a kargs=( "$@" ) -ret=0 -args=$(chroot /host/ cat /proc/cmdline) - -if chroot /host/ test -f /run/ostree-booted ; then - for t in "${kargs[@]}";do - if [[ $args != *${t}* ]];then - if chroot /host/ rpm-ostree kargs | grep -vq ${t}; then - chroot /host/ rpm-ostree kargs --append ${t} > /dev/null 2>&1 - fi - let ret++ - fi - done -else - chroot /host/ which grubby > /dev/null 2>&1 - # if grubby is not there, let's tell it - if [ $? -ne 0 ]; then - exit 127 - fi - for t in "${kargs[@]}";do - if [[ $args != *${t}* ]];then - if chroot /host/ grubby --info=DEFAULT | grep args | grep -vq ${t}; then - chroot /host/ grubby --update-kernel=DEFAULT --args=${t} > /dev/null 2>&1 - fi - let ret++ - fi - done -fi - -echo $ret diff --git a/bindata/scripts/kargs.sh b/bindata/scripts/kargs.sh new file mode 100755 index 000000000..8d118456e --- /dev/null +++ b/bindata/scripts/kargs.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -x + +command=$1 +shift +declare -a kargs=( "$@" ) +ret=0 +args=$(chroot /host/ cat /proc/cmdline) + +if chroot /host/ test -f /run/ostree-booted ; then + for t in "${kargs[@]}";do + if [[ $command == "add" ]];then + if [[ $args != *${t}* ]];then + if chroot /host/ rpm-ostree kargs | grep -vq ${t}; then + chroot /host/ rpm-ostree kargs --append ${t} > /dev/null 2>&1 + fi + let ret++ + fi + fi + if [[ $command == "remove" ]];then + if [[ $args == *${t}* ]];then + if chroot /host/ rpm-ostree kargs | grep -q ${t}; then + chroot /host/ rpm-ostree kargs --delete ${t} > /dev/null 2>&1 + fi + let ret++ + fi + fi + done +else + chroot /host/ which grubby > /dev/null 2>&1 + # if grubby is not there, let's tell it + if [ $? -ne 0 ]; then + exit 127 + fi + for t in "${kargs[@]}";do + if [[ $command == "add" ]];then + if [[ $args != *${t}* ]];then + if chroot /host/ grubby --info=DEFAULT | grep args | grep -vq ${t}; then + chroot /host/ grubby --update-kernel=DEFAULT --args=${t} > /dev/null 2>&1 + fi + let ret++ + fi + fi + if [[ $command == "remove" ]];then + if [[ $args == *${t}* ]];then + if chroot /host/ grubby --info=DEFAULT | grep args | grep -q ${t}; then + chroot /host/ grubby --update-kernel=DEFAULT --remove-args=${t} > /dev/null 2>&1 + fi + let ret++ + fi + fi + done +fi + +echo $ret diff --git a/controllers/sriovnetworknodepolicy_controller.go b/controllers/sriovnetworknodepolicy_controller.go index 1d2811fac..62218436f 100644 --- a/controllers/sriovnetworknodepolicy_controller.go +++ b/controllers/sriovnetworknodepolicy_controller.go @@ -155,22 +155,22 @@ func (r *SriovNetworkNodePolicyReconciler) SetupWithManager(mgr ctrl.Manager) er delayedEventHandler := handler.Funcs{ CreateFunc: func(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { log.Log.WithName("SriovNetworkNodePolicy"). - Info("Enqueuing sync for create event", "resource", e.Object.GetName()) + Info("Enqueuing sync for create event", "resource", e.Object.GetName(), "type", e.Object.GetObjectKind().GroupVersionKind().String()) qHandler(q) }, UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { log.Log.WithName("SriovNetworkNodePolicy"). - Info("Enqueuing sync for update event", "resource", e.ObjectNew.GetName()) + Info("Enqueuing sync for update event", "resource", e.ObjectNew.GetName(), "type", e.ObjectNew.GetObjectKind().GroupVersionKind().String()) qHandler(q) }, DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { log.Log.WithName("SriovNetworkNodePolicy"). - Info("Enqueuing sync for delete event", "resource", e.Object.GetName()) + Info("Enqueuing sync for delete event", "resource", e.Object.GetName(), "type", e.Object.GetObjectKind().GroupVersionKind().String()) qHandler(q) }, GenericFunc: func(ctx context.Context, e event.GenericEvent, q workqueue.RateLimitingInterface) { log.Log.WithName("SriovNetworkNodePolicy"). - Info("Enqueuing sync for generic event", "resource", e.Object.GetName()) + Info("Enqueuing sync for generic event", "resource", e.Object.GetName(), "type", e.Object.GetObjectKind().GroupVersionKind().String()) qHandler(q) }, } @@ -199,6 +199,7 @@ func (r *SriovNetworkNodePolicyReconciler) SetupWithManager(mgr ctrl.Manager) er For(&sriovnetworkv1.SriovNetworkNodePolicy{}). Watches(&corev1.Node{}, nodeEvenHandler). Watches(&sriovnetworkv1.SriovNetworkNodePolicy{}, delayedEventHandler). + Watches(&sriovnetworkv1.SriovNetworkPoolConfig{}, delayedEventHandler). WatchesRawSource(&source.Channel{Source: eventChan}, delayedEventHandler). Complete(r) } @@ -271,14 +272,14 @@ func (r *SriovNetworkNodePolicyReconciler) syncAllSriovNetworkNodeStates(ctx con ns := &sriovnetworkv1.SriovNetworkNodeState{} ns.Name = node.Name ns.Namespace = vars.Namespace - j, _ := json.Marshal(ns) netPoolConfig, _, err := findNodePoolConfig(ctx, &node, r.Client) if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): failed to get SriovNetworkPoolConfig for the current node") + logger.Error(err, "failed to get SriovNetworkPoolConfig for the current node") } if netPoolConfig != nil { ns.Spec.System.RdmaMode = netPoolConfig.Spec.RdmaMode } + j, _ := json.Marshal(ns) logger.V(2).Info("SriovNetworkNodeState CR", "content", j) if err := r.syncSriovNetworkNodeState(ctx, dc, npl, ns, &node); err != nil { logger.Error(err, "Fail to sync", "SriovNetworkNodeState", ns.Name) diff --git a/go.mod b/go.mod index 0353c7ec1..6e7171c53 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/vishvananda/netlink v1.2.1-beta.2.0.20240221172127-ec7bcb248e94 github.com/vishvananda/netns v0.0.4 go.uber.org/zap v1.25.0 + golang.org/x/net v0.23.0 golang.org/x/time v0.3.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/yaml.v3 v3.0.1 @@ -144,7 +145,6 @@ require ( golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/mod v0.13.0 // indirect - golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.13.0 // indirect golang.org/x/sync v0.4.0 // indirect golang.org/x/sys v0.20.0 // indirect diff --git a/pkg/consts/constants.go b/pkg/consts/constants.go index f62cd3195..890a05544 100644 --- a/pkg/consts/constants.go +++ b/pkg/consts/constants.go @@ -124,9 +124,11 @@ const ( `IMPORT{program}="/etc/udev/switchdev-vf-link-name.sh $attr{phys_port_name}", ` + `NAME="%s_$env{NUMBER}"` - KernelArgPciRealloc = "pci=realloc" - KernelArgIntelIommu = "intel_iommu=on" - KernelArgIommuPt = "iommu=pt" + KernelArgPciRealloc = "pci=realloc" + KernelArgIntelIommu = "intel_iommu=on" + KernelArgIommuPt = "iommu=pt" + KernelArgRdmaShared = "ib_core.netns_mode=1" + KernelArgRdmaExclusive = "ib_core.netns_mode=0" // Feature gates // ParallelNicConfigFeatureGate: allow to configure nics in parallel diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 0867685dc..53fe82b8b 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "math/rand" - "os/exec" "reflect" "sync" "time" @@ -429,16 +428,6 @@ func (dn *Daemon) nodeStateSyncHandler() error { reqReboot = reqReboot || r } - if dn.currentNodeState.Status.System.RdmaMode != dn.desiredNodeState.Spec.System.RdmaMode { - err = dn.HostHelpers.SetRDMASubsystem(dn.desiredNodeState.Spec.System.RdmaMode) - if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): failed to set RDMA subsystem") - return err - } - reqReboot = true - reqDrain = true - } - // When running using systemd check if the applied configuration is the latest one // or there is a new config we need to apply // When using systemd configuration we write the file @@ -761,11 +750,11 @@ func (dn *Daemon) rebootNode() { // However note we use `;` instead of `&&` so we keep rebooting even // if kubelet failed to shutdown - that way the machine will still eventually reboot // as systemd will time out the stop invocation. - cmd := exec.Command("systemd-run", "--unit", "sriov-network-config-daemon-reboot", + stdOut, StdErr, err := dn.HostHelpers.RunCommand("systemd-run", "--unit", "sriov-network-config-daemon-reboot", "--description", "sriov-network-config-daemon reboot node", "/bin/sh", "-c", "systemctl stop kubelet.service; reboot") - if err := cmd.Run(); err != nil { - log.Log.Error(err, "failed to reboot node") + if err != nil { + log.Log.Error(err, "failed to reboot node", "stdOut", stdOut, "StdErr", StdErr) } } diff --git a/pkg/daemon/plugin_test.go b/pkg/daemon/plugin_test.go index a13fc1f8b..2ad3fbb97 100644 --- a/pkg/daemon/plugin_test.go +++ b/pkg/daemon/plugin_test.go @@ -41,6 +41,13 @@ var _ = Describe("config daemon plugin loading tests", func() { vars.ClusterType = consts.ClusterTypeKubernetes gmockController = gomock.NewController(GinkgoT()) helperMock = helperMocks.NewMockHostHelpersInterface(gmockController) + helperMock.EXPECT().GetCurrentKernelArgs().Return("", nil).AnyTimes() + helperMock.EXPECT().IsKernelArgsSet("", consts.KernelArgIntelIommu).Return(false) + helperMock.EXPECT().IsKernelArgsSet("", consts.KernelArgIommuPt).Return(false) + helperMock.EXPECT().IsKernelArgsSet("", consts.KernelArgPciRealloc).Return(false) + helperMock.EXPECT().IsKernelArgsSet("", consts.KernelArgRdmaExclusive).Return(false) + helperMock.EXPECT().IsKernelArgsSet("", consts.KernelArgRdmaShared).Return(false) + // k8s plugin is ATM the only plugin which require mocking/faking, as its New method performs additional logic // other than simple plugin struct initialization K8sPlugin = func(_ helper.HostHelpersInterface) (plugin.VendorPlugin, error) { diff --git a/pkg/daemon/writer.go b/pkg/daemon/writer.go index 60d4e8d91..42eeb2928 100644 --- a/pkg/daemon/writer.go +++ b/pkg/daemon/writer.go @@ -189,6 +189,7 @@ func (w *NodeStateStatusWriter) setNodeStateStatus(msg Message) (*sriovnetworkv1 nodeState, err := w.updateNodeStateStatusRetry(func(nodeState *sriovnetworkv1.SriovNetworkNodeState) { nodeState.Status.Interfaces = w.status.Interfaces nodeState.Status.Bridges = w.status.Bridges + nodeState.Status.System = w.status.System if msg.lastSyncError != "" || msg.syncStatus == consts.SyncStatusSucceeded { // clear lastSyncError when sync Succeeded nodeState.Status.LastSyncError = msg.lastSyncError diff --git a/pkg/host/internal/lib/netlink/mock/mock_netlink.go b/pkg/host/internal/lib/netlink/mock/mock_netlink.go index 758346a3f..ec136bf29 100644 --- a/pkg/host/internal/lib/netlink/mock/mock_netlink.go +++ b/pkg/host/internal/lib/netlink/mock/mock_netlink.go @@ -145,21 +145,6 @@ func (mr *MockNetlinkLibMockRecorder) DevlinkSetDeviceParam(bus, device, param, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DevlinkSetDeviceParam", reflect.TypeOf((*MockNetlinkLib)(nil).DevlinkSetDeviceParam), bus, device, param, cmode, value) } -// DiscoverRDMASubsystem mocks base method. -func (m *MockNetlinkLib) DiscoverRDMASubsystem() (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DiscoverRDMASubsystem") - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// DiscoverRDMASubsystem indicates an expected call of DiscoverRDMASubsystem. -func (mr *MockNetlinkLibMockRecorder) DiscoverRDMASubsystem() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverRDMASubsystem", reflect.TypeOf((*MockNetlinkLib)(nil).DiscoverRDMASubsystem)) -} - // IsLinkAdminStateUp mocks base method. func (m *MockNetlinkLib) IsLinkAdminStateUp(link netlink.Link) bool { m.ctrl.T.Helper() @@ -304,6 +289,21 @@ func (mr *MockNetlinkLibMockRecorder) RdmaLinkByName(name interface{}) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RdmaLinkByName", reflect.TypeOf((*MockNetlinkLib)(nil).RdmaLinkByName), name) } +// RdmaSystemGetNetnsMode mocks base method. +func (m *MockNetlinkLib) RdmaSystemGetNetnsMode() (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RdmaSystemGetNetnsMode") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RdmaSystemGetNetnsMode indicates an expected call of RdmaSystemGetNetnsMode. +func (mr *MockNetlinkLibMockRecorder) RdmaSystemGetNetnsMode() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RdmaSystemGetNetnsMode", reflect.TypeOf((*MockNetlinkLib)(nil).RdmaSystemGetNetnsMode)) +} + // VDPADelDev mocks base method. func (m *MockNetlinkLib) VDPADelDev(name string) error { m.ctrl.T.Helper() diff --git a/pkg/host/internal/lib/netlink/netlink.go b/pkg/host/internal/lib/netlink/netlink.go index 7d857921d..ad6056710 100644 --- a/pkg/host/internal/lib/netlink/netlink.go +++ b/pkg/host/internal/lib/netlink/netlink.go @@ -68,8 +68,8 @@ type NetlinkLib interface { RdmaLinkByName(name string) (*netlink.RdmaLink, error) // IsLinkAdminStateUp checks if the admin state of a link is up IsLinkAdminStateUp(link Link) bool - // DiscoverRDMASubsystem returns RDMA subsystem mode - DiscoverRDMASubsystem() (string, error) + // RdmaSystemGetNetnsMode returns RDMA subsystem mode + RdmaSystemGetNetnsMode() (string, error) } type libWrapper struct{} @@ -188,7 +188,7 @@ func (w *libWrapper) IsLinkAdminStateUp(link Link) bool { return link.Attrs().Flags&net.FlagUp == 1 } -// DiscoverRDMASubsystem returns RDMA subsystem mode -func (w *libWrapper) DiscoverRDMASubsystem() (string, error) { +// RdmaSystemGetNetnsMode returns RDMA subsystem mode +func (w *libWrapper) RdmaSystemGetNetnsMode() (string, error) { return netlink.RdmaSystemGetNetnsMode() } diff --git a/pkg/host/internal/network/network.go b/pkg/host/internal/network/network.go index 940c4b248..f8846169a 100644 --- a/pkg/host/internal/network/network.go +++ b/pkg/host/internal/network/network.go @@ -432,30 +432,40 @@ func (n *network) GetPciAddressFromInterfaceName(interfaceName string) (string, func (n *network) DiscoverRDMASubsystem() (string, error) { log.Log.Info("DiscoverRDMASubsystem(): retrieving RDMA subsystem mode") - subsystem, err := n.netlinkLib.DiscoverRDMASubsystem() + subsystem, err := n.netlinkLib.RdmaSystemGetNetnsMode() if err != nil { log.Log.Error(err, "DiscoverRDMASubsystem(): failed to get RDMA subsystem mode") return "", err } + log.Log.V(2).Info("DiscoverRDMASubsystem(): RDMA subsystem mode", "mode", subsystem) return subsystem, nil } func (n *network) SetRDMASubsystem(mode string) error { - log.Log.Info("SetRDMASubsystem(): Updating RDMA subsystem mode") + log.Log.Info("SetRDMASubsystem(): Updating RDMA subsystem mode", "mode", mode) + path := filepath.Join(vars.FilesystemRoot, consts.Host, "etc", "modprobe.d", "sriov_network_operator_modules_config.conf") + + if mode == "" { + err := os.Remove(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + log.Log.Error(err, "failed to remove ib_core config file") + return err + } + return nil + } modeValue := 1 if mode == "exclusive" { modeValue = 0 } - config := fmt.Sprintf("options ib_core netns_mode=%d\n", modeValue) - path := filepath.Join(vars.FilesystemRoot, consts.Host, "etc", "modprobe.d", "ib_core.conf") - err := os.WriteFile(path, []byte(config), 0644) + config := fmt.Sprintf("# This file is managed by sriov-network-operator do not edit.\noptions ib_core netns_mode=%d\n", modeValue) + err := os.WriteFile(path, []byte(config), 0644) if err != nil { - log.Log.Error(err, "SetRDMASubsystem(): failed to write ib_core config") - return fmt.Errorf("failed to write ib_core config: %v", err) + log.Log.Error(err, "SetRDMASubsystem(): failed to write sriov_network_operator_modules_config.conf") + return fmt.Errorf("failed to write sriov_network_operator_modules_config.conf: %v", err) } return nil diff --git a/pkg/host/internal/network/network_test.go b/pkg/host/internal/network/network_test.go index 51c56b875..3e197c3f8 100644 --- a/pkg/host/internal/network/network_test.go +++ b/pkg/host/internal/network/network_test.go @@ -285,7 +285,7 @@ var _ = Describe("Network", func() { }) Context("DiscoverRDMASubsystem", func() { It("Should get RDMA Subsystem using netlink", func() { - netlinkLibMock.EXPECT().DiscoverRDMASubsystem().Return("shared", nil) + netlinkLibMock.EXPECT().RdmaSystemGetNetnsMode().Return("shared", nil) pci, err := n.DiscoverRDMASubsystem() Expect(err).NotTo(HaveOccurred()) @@ -297,21 +297,21 @@ var _ = Describe("Network", func() { helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ Dirs: []string{"/host/etc/modprobe.d"}, Files: map[string][]byte{ - "/host/etc/modprobe.d/ib_core.conf": {}, + "/host/etc/modprobe.d/sriov_network_operator_modules_config.conf": {}, }, }) Expect(n.SetRDMASubsystem("shared")).NotTo(HaveOccurred()) - helpers.GinkgoAssertFileContentsEquals("/host/etc/modprobe.d/ib_core.conf", "options ib_core netns_mode=1\n") + helpers.GinkgoAssertFileContentsEquals("/host/etc/modprobe.d/sriov_network_operator_modules_config.conf", "# This file is managed by sriov-network-operator do not edit.\noptions ib_core netns_mode=1\n") }) It("Should set RDMA Subsystem exclusive mode", func() { helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ Dirs: []string{"/host/etc/modprobe.d"}, Files: map[string][]byte{ - "/host/etc/modprobe.d/ib_core.conf": {}, + "/host/etc/modprobe.d/sriov_network_operator_modules_config.conf": {}, }, }) Expect(n.SetRDMASubsystem("exclusive")).NotTo(HaveOccurred()) - helpers.GinkgoAssertFileContentsEquals("/host/etc/modprobe.d/ib_core.conf", "options ib_core netns_mode=0\n") + helpers.GinkgoAssertFileContentsEquals("/host/etc/modprobe.d/sriov_network_operator_modules_config.conf", "# This file is managed by sriov-network-operator do not edit.\noptions ib_core netns_mode=0\n") }) }) }) diff --git a/pkg/plugins/generic/generic_plugin.go b/pkg/plugins/generic/generic_plugin.go index 14b1903e5..68e09e0ea 100644 --- a/pkg/plugins/generic/generic_plugin.go +++ b/pkg/plugins/generic/generic_plugin.go @@ -1,11 +1,8 @@ package generic import ( - "bytes" "errors" - "os/exec" - "strconv" - "strings" + "fmt" "syscall" "sigs.k8s.io/controller-runtime/pkg/log" @@ -47,12 +44,14 @@ type DriverState struct { type DriverStateMapType map[uint]*DriverState +type KargStateMapType map[string]bool + type GenericPlugin struct { PluginName string SpecVersion string DesireState *sriovnetworkv1.SriovNetworkNodeState DriverStateMap DriverStateMapType - DesiredKernelArgs map[string]bool + DesiredKernelArgs KargStateMapType helpers helper.HostHelpersInterface skipVFConfiguration bool skipBridgeConfiguration bool @@ -81,7 +80,7 @@ type genericPluginOptions struct { skipBridgeConfiguration bool } -const scriptsPath = "bindata/scripts/enable-kargs.sh" +const scriptsPath = "bindata/scripts/kargs.sh" // Initialize our plugin and set up initial values func NewGenericPlugin(helpers helper.HostHelpersInterface, options ...Option) (plugin.VendorPlugin, error) { @@ -111,11 +110,26 @@ func NewGenericPlugin(helpers helper.HostHelpersInterface, options ...Option) (p NeedDriverFunc: needDriverCheckVdpaType, DriverLoaded: false, } + + // To maintain backward compatibility we don't remove the intel_iommu, iommu and pcirealloc + // kernel args if they are configured + kargs, err := helpers.GetCurrentKernelArgs() + if err != nil { + return nil, err + } + desiredKernelArgs := KargStateMapType{ + consts.KernelArgPciRealloc: helpers.IsKernelArgsSet(kargs, consts.KernelArgPciRealloc), + consts.KernelArgIntelIommu: helpers.IsKernelArgsSet(kargs, consts.KernelArgIntelIommu), + consts.KernelArgIommuPt: helpers.IsKernelArgsSet(kargs, consts.KernelArgIommuPt), + consts.KernelArgRdmaShared: false, + consts.KernelArgRdmaExclusive: false, + } + return &GenericPlugin{ PluginName: PluginName, SpecVersion: "1.0", DriverStateMap: driverStateMap, - DesiredKernelArgs: make(map[string]bool), + DesiredKernelArgs: desiredKernelArgs, helpers: helpers, skipVFConfiguration: cfg.skipVFConfiguration, skipBridgeConfiguration: cfg.skipBridgeConfiguration, @@ -178,18 +192,13 @@ func (p *GenericPlugin) CheckStatusChanges(current *sriovnetworkv1.SriovNetworkN } } - missingKernelArgs, err := p.getMissingKernelArgs() + shouldUpdate, err := p.shouldUpdateKernelArgs() if err != nil { log.Log.Error(err, "generic-plugin CheckStatusChanges(): failed to verify missing kernel arguments") return false, err } - if len(missingKernelArgs) != 0 { - log.Log.V(0).Info("generic-plugin CheckStatusChanges(): kernel args missing", - "kernelArgs", missingKernelArgs) - } - - return len(missingKernelArgs) != 0, nil + return shouldUpdate, nil } func (p *GenericPlugin) syncDriverState() error { @@ -227,7 +236,7 @@ func (p *GenericPlugin) Apply() error { p.DesireState.Status.Interfaces, p.skipVFConfiguration); err != nil { // Catch the "cannot allocate memory" error and try to use PCI realloc if errors.Is(err, syscall.ENOMEM) { - p.addToDesiredKernelArgs(consts.KernelArgPciRealloc) + p.enableDesiredKernelArgs(consts.KernelArgPciRealloc) } return err } @@ -264,84 +273,83 @@ func needDriverCheckVdpaType(state *sriovnetworkv1.SriovNetworkNodeState, driver } // setKernelArg Tries to add the kernel args via ostree or grubby. -func setKernelArg(karg string) (bool, error) { +func setKernelArg(helper helper.HostHelpersInterface, mode, karg string) error { log.Log.Info("generic plugin setKernelArg()") - var stdout, stderr bytes.Buffer - cmd := exec.Command("/bin/sh", scriptsPath, karg) - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { + _, _, err := helper.RunCommand("/bin/sh", scriptsPath, mode, karg) + if err != nil { // if grubby is not there log and assume kernel args are set correctly. if utils.IsCommandNotFound(err) { log.Log.Error(err, "generic plugin setKernelArg(): grubby or ostree command not found. Please ensure that kernel arg are set", "kargs", karg) - return false, nil + return nil } log.Log.Error(err, "generic plugin setKernelArg(): fail to enable kernel arg", "karg", karg) - return false, err - } - - i, err := strconv.Atoi(strings.TrimSpace(stdout.String())) - if err == nil { - if i > 0 { - log.Log.Info("generic plugin setKernelArg(): need to reboot node for kernel arg", "karg", karg) - return true, nil - } + return err } - return false, err + return nil } -// addToDesiredKernelArgs Should be called to queue a kernel arg to be added to the node. -func (p *GenericPlugin) addToDesiredKernelArgs(karg string) { - if _, ok := p.DesiredKernelArgs[karg]; !ok { - log.Log.Info("generic plugin addToDesiredKernelArgs(): Adding to desired kernel arg", "karg", karg) - p.DesiredKernelArgs[karg] = false - } +// enableDesiredKernelArgs Should be called to mark a kernel arg as enabled. +func (p *GenericPlugin) enableDesiredKernelArgs(karg string) { + log.Log.Info("generic plugin enableDesiredKernelArgs(): enable kernel arg", "karg", karg) + p.DesiredKernelArgs[karg] = true } -// getMissingKernelArgs gets Kernel arguments that have not been set. -func (p *GenericPlugin) getMissingKernelArgs() ([]string, error) { - missingArgs := make([]string, 0, len(p.DesiredKernelArgs)) - if len(p.DesiredKernelArgs) == 0 { - return nil, nil - } +// disableDesiredKernelArgs Should be called to mark a kernel arg as disabled. +func (p *GenericPlugin) disableDesiredKernelArgs(karg string) { + log.Log.Info("generic plugin disableDesiredKernelArgs(): disable kernel arg", "karg", karg) + p.DesiredKernelArgs[karg] = false +} +// getKernelArgs gets Kernel arguments on the running sy. +func (p *GenericPlugin) shouldUpdateKernelArgs() (bool, error) { kargs, err := p.helpers.GetCurrentKernelArgs() if err != nil { - return nil, err + return false, err } - for desiredKarg := range p.DesiredKernelArgs { - if !p.helpers.IsKernelArgsSet(kargs, desiredKarg) { - missingArgs = append(missingArgs, desiredKarg) + for karg, kargState := range p.DesiredKernelArgs { + if kargState && !p.helpers.IsKernelArgsSet(kargs, karg) { + return true, nil + } + + if !kargState && p.helpers.IsKernelArgsSet(kargs, karg) { + return true, nil } } - return missingArgs, nil + return false, nil } // syncDesiredKernelArgs should be called to set all the kernel arguments. Returns bool if node update is needed. -func (p *GenericPlugin) syncDesiredKernelArgs(kargs []string) (bool, error) { +func (p *GenericPlugin) syncDesiredKernelArgs() (bool, error) { + kargs, err := p.helpers.GetCurrentKernelArgs() + if err != nil { + return false, err + } + needReboot := false + for karg, kargState := range p.DesiredKernelArgs { + if kargState { + err = setKernelArg(p.helpers, "add", karg) + if err != nil { + log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to set kernel arg", "karg", karg) + return false, err + } - for _, karg := range kargs { - if p.DesiredKernelArgs[karg] { - log.Log.V(2).Info("generic-plugin syncDesiredKernelArgs(): previously attempted to set kernel arg", - "karg", karg) - } - // There is a case when we try to set the kernel argument here, the daemon could decide to not reboot because - // the daemon encountered a potentially one-time error. However we always want to make sure that the kernel - // argument is set once the daemon goes through node state sync again. - update, err := setKernelArg(karg) - if err != nil { - log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to set kernel arg", "karg", karg) - return false, err - } - if update { - needReboot = true - log.Log.V(2).Info("generic-plugin syncDesiredKernelArgs(): need reboot for setting kernel arg", "karg", karg) + if !p.helpers.IsKernelArgsSet(kargs, karg) { + needReboot = true + } + } else { + err = setKernelArg(p.helpers, "remove", karg) + if err != nil { + log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to remove kernel arg", "karg", karg) + return false, err + } + + if p.helpers.IsKernelArgsSet(kargs, karg) { + needReboot = true + } } - p.DesiredKernelArgs[karg] = true } return needReboot, nil } @@ -420,31 +428,46 @@ func (p *GenericPlugin) shouldConfigureBridges() bool { func (p *GenericPlugin) addVfioDesiredKernelArg(state *sriovnetworkv1.SriovNetworkNodeState) { driverState := p.DriverStateMap[Vfio] if !driverState.DriverLoaded && driverState.NeedDriverFunc(state, driverState) { - p.addToDesiredKernelArgs(consts.KernelArgIntelIommu) - p.addToDesiredKernelArgs(consts.KernelArgIommuPt) + p.enableDesiredKernelArgs(consts.KernelArgIntelIommu) + p.enableDesiredKernelArgs(consts.KernelArgIommuPt) } } +func (p *GenericPlugin) configRdmaKernelArg(state *sriovnetworkv1.SriovNetworkNodeState) error { + if state.Spec.System.RdmaMode == "" { + p.disableDesiredKernelArgs(consts.KernelArgRdmaExclusive) + p.disableDesiredKernelArgs(consts.KernelArgRdmaShared) + } else if state.Spec.System.RdmaMode == "shared" { + p.enableDesiredKernelArgs(consts.KernelArgRdmaShared) + p.disableDesiredKernelArgs(consts.KernelArgRdmaExclusive) + } else if state.Spec.System.RdmaMode == "exclusive" { + p.enableDesiredKernelArgs(consts.KernelArgRdmaExclusive) + p.disableDesiredKernelArgs(consts.KernelArgRdmaShared) + } else { + err := fmt.Errorf("unexpected rdma mode: %s", state.Spec.System.RdmaMode) + log.Log.Error(err, "generic-plugin configRdmaKernelArg(): failed to configure kernel arguments for rdma") + return err + } + + return p.helpers.SetRDMASubsystem(state.Spec.System.RdmaMode) +} + func (p *GenericPlugin) needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState) (bool, error) { needReboot := false p.addVfioDesiredKernelArg(state) - - missingKernelArgs, err := p.getMissingKernelArgs() + err := p.configRdmaKernelArg(state) if err != nil { - log.Log.Error(err, "generic-plugin needRebootNode(): failed to verify missing kernel arguments") return false, err } - if len(missingKernelArgs) != 0 { - needReboot, err = p.syncDesiredKernelArgs(missingKernelArgs) - if err != nil { - log.Log.Error(err, "generic-plugin needRebootNode(): failed to set the desired kernel arguments") - return false, err - } - if needReboot { - log.Log.V(2).Info("generic-plugin needRebootNode(): need reboot for updating kernel arguments") - } + needReboot, err = p.syncDesiredKernelArgs() + if err != nil { + log.Log.Error(err, "generic-plugin needRebootNode(): failed to set the desired kernel arguments") + return false, err + } + if needReboot { + log.Log.V(2).Info("generic-plugin needRebootNode(): need reboot for updating kernel arguments") } return needReboot, nil diff --git a/pkg/plugins/generic/generic_plugin_test.go b/pkg/plugins/generic/generic_plugin_test.go index 0d6701a64..d71418830 100644 --- a/pkg/plugins/generic/generic_plugin_test.go +++ b/pkg/plugins/generic/generic_plugin_test.go @@ -33,6 +33,15 @@ var _ = Describe("Generic plugin", func() { ctrl = gomock.NewController(t) hostHelper = mock_helper.NewMockHostHelpersInterface(ctrl) + hostHelper.EXPECT().SetRDMASubsystem("").Return(nil).AnyTimes() + hostHelper.EXPECT().GetCurrentKernelArgs().Return("", nil).AnyTimes() + hostHelper.EXPECT().IsKernelArgsSet("", consts.KernelArgIntelIommu).Return(false).AnyTimes() + hostHelper.EXPECT().IsKernelArgsSet("", consts.KernelArgIommuPt).Return(false).AnyTimes() + hostHelper.EXPECT().IsKernelArgsSet("", consts.KernelArgPciRealloc).Return(false).AnyTimes() + hostHelper.EXPECT().IsKernelArgsSet("", consts.KernelArgRdmaExclusive).Return(false).AnyTimes() + hostHelper.EXPECT().IsKernelArgsSet("", consts.KernelArgRdmaShared).Return(false).AnyTimes() + + hostHelper.EXPECT().RunCommand(gomock.Any(), gomock.Any()).Return("", "", nil).AnyTimes() genericPlugin, err = NewGenericPlugin(hostHelper) Expect(err).ToNot(HaveOccurred()) @@ -899,10 +908,6 @@ var _ = Describe("Generic plugin", func() { // Load required kernel args. genericPlugin.(*GenericPlugin).addVfioDesiredKernelArg(networkNodeState) - hostHelper.EXPECT().GetCurrentKernelArgs().Return("", nil) - hostHelper.EXPECT().IsKernelArgsSet("", consts.KernelArgIntelIommu).Return(false) - hostHelper.EXPECT().IsKernelArgsSet("", consts.KernelArgIommuPt).Return(false) - changed, err := genericPlugin.CheckStatusChanges(networkNodeState) Expect(err).ToNot(HaveOccurred()) Expect(changed).To(BeTrue()) diff --git a/test/conformance/tests/test_networkpool.go b/test/conformance/tests/test_networkpool.go new file mode 100644 index 000000000..98edb9c6d --- /dev/null +++ b/test/conformance/tests/test_networkpool.go @@ -0,0 +1,307 @@ +package tests + +import ( + "fmt" + "strconv" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "golang.org/x/net/context" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/cluster" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/discovery" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/namespaces" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/network" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/pod" +) + +var _ = Describe("[sriov] NetworkPool", Ordered, func() { + var testNode string + var interfaces []*sriovv1.InterfaceExt + + BeforeAll(func() { + err := namespaces.Create(namespaces.Test, clients) + Expect(err).ToNot(HaveOccurred()) + err = namespaces.Clean(operatorNamespace, namespaces.Test, clients, discovery.Enabled()) + Expect(err).ToNot(HaveOccurred()) + + sriovInfos, err := cluster.DiscoverSriov(clients, operatorNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(len(sriovInfos.Nodes)).ToNot(BeZero()) + + testNode, interfaces, err = sriovInfos.FindSriovDevicesAndNode() + Expect(err).ToNot(HaveOccurred()) + + By(fmt.Sprintf("Testing on node %s, %d devices found", testNode, len(interfaces))) + WaitForSRIOVStable() + }) + + AfterEach(func() { + err := namespaces.Clean(operatorNamespace, namespaces.Test, clients, discovery.Enabled()) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterAll(func() { + err := clients.DeleteAllOf(context.Background(), &sriovv1.SriovNetworkPoolConfig{}, client.InNamespace(operatorNamespace)) + Expect(err).ToNot(HaveOccurred()) + WaitForSRIOVStable() + }) + + Context("Configure rdma namespace mode", func() { + It("should switch rdma mode", func() { + By("create a pool with only that node") + networkPool := &sriovv1.SriovNetworkPoolConfig{ + ObjectMeta: metav1.ObjectMeta{Name: testNode, Namespace: operatorNamespace}, + Spec: sriovv1.SriovNetworkPoolConfigSpec{RdmaMode: consts.RdmaSubsystemModeExclusive, + NodeSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/hostname": testNode}}}} + + By("configure rdma mode to exclusive") + err := clients.Create(context.Background(), networkPool) + Expect(err).ToNot(HaveOccurred()) + By("waiting for operator to finish the configuration") + WaitForSRIOVStable() + nodeState := &sriovv1.SriovNetworkNodeState{} + err = clients.Get(context.Background(), client.ObjectKey{Name: testNode, Namespace: operatorNamespace}, nodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(nodeState.Spec.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeExclusive)) + Expect(nodeState.Status.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeExclusive)) + + By("Checking rdma mode and kernel args") + output, _, err := runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline | grep ib_core.netns_mode=0 | wc -l") + Expect(err).ToNot(HaveOccurred()) + Expect(strings.HasPrefix(output, "1")).To(BeTrue()) + + output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline | grep ib_core.netns_mode=1 | wc -l") + Expect(err).ToNot(HaveOccurred()) + Expect(strings.HasPrefix(output, "0")).To(BeTrue()) + + output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/etc/modprobe.d/sriov_network_operator_modules_config.conf | grep mode=0 | wc -l") + Expect(err).ToNot(HaveOccurred()) + Expect(strings.HasPrefix(output, "1")).To(BeTrue()) + + By("configure rdma mode to shared") + networkPool.Spec.RdmaMode = consts.RdmaSubsystemModeShared + err = clients.Update(context.Background(), networkPool) + Expect(err).ToNot(HaveOccurred()) + WaitForSRIOVStable() + err = clients.Get(context.Background(), client.ObjectKey{Name: testNode, Namespace: operatorNamespace}, nodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(nodeState.Spec.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeShared)) + Expect(nodeState.Status.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeShared)) + + By("Checking rdma mode and kernel args") + output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline | grep ib_core.netns_mode=0 | wc -l") + Expect(err).ToNot(HaveOccurred()) + Expect(strings.HasPrefix(output, "0")).To(BeTrue()) + + output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline | grep ib_core.netns_mode=1 | wc -l") + Expect(err).ToNot(HaveOccurred()) + Expect(strings.HasPrefix(output, "1")).To(BeTrue()) + + output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/etc/modprobe.d/sriov_network_operator_modules_config.conf | grep mode=1 | wc -l") + Expect(err).ToNot(HaveOccurred()) + Expect(strings.HasPrefix(output, "1")).To(BeTrue()) + + By("removing rdma mode configuration") + err = clients.Delete(context.Background(), networkPool) + Expect(err).ToNot(HaveOccurred()) + WaitForSRIOVStable() + + err = clients.Get(context.Background(), client.ObjectKey{Name: testNode, Namespace: operatorNamespace}, nodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(nodeState.Spec.System.RdmaMode).To(Equal("")) + Expect(nodeState.Status.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeShared)) + + By("Checking rdma mode and kernel args") + output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline | grep ib_core.netns_mode=0 | wc -l") + Expect(err).ToNot(HaveOccurred()) + Expect(strings.HasPrefix(output, "0")).To(BeTrue()) + + output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline | grep ib_core.netns_mode=1 | wc -l") + Expect(err).ToNot(HaveOccurred()) + Expect(strings.HasPrefix(output, "0")).To(BeTrue()) + + output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "ls /host/etc/modprobe.d | grep sriov_network_operator_modules_config.conf | wc -l") + Expect(err).ToNot(HaveOccurred()) + Expect(strings.HasPrefix(output, "0")).To(BeTrue()) + }) + }) + + Context("Check rdma metrics inside a pod in exclusive mode", func() { + var iface *sriovv1.InterfaceExt + + BeforeAll(func() { + sriovInfos, err := cluster.DiscoverSriov(clients, operatorNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(len(sriovInfos.Nodes)).ToNot(BeZero()) + + for _, node := range sriovInfos.Nodes { + iface, err = sriovInfos.FindOneMellanoxSriovDevice(node) + if err == nil { + testNode = node + break + } + } + + if iface == nil { + Skip("no mellanox card available to test rdma") + } + + networkPool := &sriovv1.SriovNetworkPoolConfig{ + ObjectMeta: metav1.ObjectMeta{Name: testNode, Namespace: operatorNamespace}, + Spec: sriovv1.SriovNetworkPoolConfigSpec{RdmaMode: consts.RdmaSubsystemModeExclusive, + NodeSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/hostname": testNode}}}} + + err = clients.Create(context.Background(), networkPool) + Expect(err).ToNot(HaveOccurred()) + By("waiting for operator to finish the configuration") + WaitForSRIOVStable() + }) + + It("should run pod with RDMA cni and expose nic metrics and another one without rdma info", func() { + By("creating a policy") + resourceName := "testrdma" + _, err := network.CreateSriovPolicy(clients, "test-policy-", operatorNamespace, iface.Name, testNode, 5, resourceName, "netdevice", + func(policy *sriovv1.SriovNetworkNodePolicy) { policy.Spec.IsRdma = true }) + Expect(err).ToNot(HaveOccurred()) + WaitForSRIOVStable() + + By("Creating sriov network to use the rdma device") + sriovNetwork := &sriovv1.SriovNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-rdmanetwork", + Namespace: operatorNamespace, + }, + Spec: sriovv1.SriovNetworkSpec{ + ResourceName: resourceName, + IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181","routes":[{"dst":"0.0.0.0/0"}],"gateway":"10.10.10.1"}`, + NetworkNamespace: namespaces.Test, + MetaPluginsConfig: `{"type": "rdma"}`, + }} + + err = clients.Create(context.Background(), sriovNetwork) + Expect(err).ToNot(HaveOccurred()) + waitForNetAttachDef("test-rdmanetwork", namespaces.Test) + + sriovNetwork = &sriovv1.SriovNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-nordmanetwork", + Namespace: operatorNamespace, + }, + Spec: sriovv1.SriovNetworkSpec{ + ResourceName: resourceName, + IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181","routes":[{"dst":"0.0.0.0/0"}],"gateway":"10.10.10.1"}`, + NetworkNamespace: namespaces.Test, + }} + + err = clients.Create(context.Background(), sriovNetwork) + Expect(err).ToNot(HaveOccurred()) + waitForNetAttachDef("test-nordmanetwork", namespaces.Test) + + podDefinition := pod.DefineWithNetworks([]string{"test-rdmanetwork"}) + firstPod, err := clients.Pods(namespaces.Test).Create(context.Background(), podDefinition, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + podDefinition = pod.DefineWithNetworks([]string{"test-nordmanetwork"}) + secondPod, err := clients.Pods(namespaces.Test).Create(context.Background(), podDefinition, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + firstPod = waitForPodRunning(firstPod) + secondPod = waitForPodRunning(secondPod) + + strOut, _, err := pod.ExecCommand(clients, firstPod, "/bin/bash", "-c", "ip link show net1 | grep net1 | wc -l") + Expect(err).ToNot(HaveOccurred()) + Expect(strings.HasPrefix(strOut, "1")).To(BeTrue()) + strOut, _, err = pod.ExecCommand(clients, firstPod, "/bin/bash", "-c", "ls /sys/bus/pci/devices/${PCIDEVICE_OPENSHIFT_IO_TESTRDMA}/infiniband/*/ports/*/hw_counters | wc -l") + strOut = strings.TrimSpace(strOut) + Expect(err).ToNot(HaveOccurred()) + num, err := strconv.Atoi(strOut) + Expect(err).ToNot(HaveOccurred()) + Expect(num).To(BeNumerically(">", 0)) + + strOut, _, err = pod.ExecCommand(clients, secondPod, "/bin/bash", "-c", "ls /sys/bus/pci/devices/${PCIDEVICE_OPENSHIFT_IO_TESTRDMA}/infiniband/ | wc -l") + Expect(err).ToNot(HaveOccurred()) + strOut = strings.TrimSpace(strOut) + num, err = strconv.Atoi(strOut) + Expect(err).ToNot(HaveOccurred()) + Expect(num).To(BeNumerically("==", 0)) + }) + }) + + Context("Check rdma metrics inside a pod in shared mode not exist", func() { + var iface *sriovv1.InterfaceExt + BeforeAll(func() { + sriovInfos, err := cluster.DiscoverSriov(clients, operatorNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(len(sriovInfos.Nodes)).ToNot(BeZero()) + + for _, node := range sriovInfos.Nodes { + iface, err = sriovInfos.FindOneMellanoxSriovDevice(node) + if err == nil { + testNode = node + break + } + } + + if iface == nil { + Skip("no mellanox card available to test rdma") + } + + networkPool := &sriovv1.SriovNetworkPoolConfig{ + ObjectMeta: metav1.ObjectMeta{Name: testNode, Namespace: operatorNamespace}, + Spec: sriovv1.SriovNetworkPoolConfigSpec{RdmaMode: consts.RdmaSubsystemModeShared, + NodeSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/hostname": testNode}}}} + + err = clients.Create(context.Background(), networkPool) + Expect(err).ToNot(HaveOccurred()) + By("waiting for operator to finish the configuration") + WaitForSRIOVStable() + }) + + It("should run pod with RDMA cni and not expose nic metrics", func() { + By("creating a policy") + resourceName := "testrdma" + _, err := network.CreateSriovPolicy(clients, "test-policy-", operatorNamespace, iface.Name, testNode, 5, resourceName, "netdevice", + func(policy *sriovv1.SriovNetworkNodePolicy) { policy.Spec.IsRdma = true }) + Expect(err).ToNot(HaveOccurred()) + WaitForSRIOVStable() + + By("Creating sriov network to use the rdma device") + sriovNetwork := &sriovv1.SriovNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-rdmanetwork", + Namespace: operatorNamespace, + }, + Spec: sriovv1.SriovNetworkSpec{ + ResourceName: resourceName, + IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181","routes":[{"dst":"0.0.0.0/0"}],"gateway":"10.10.10.1"}`, + NetworkNamespace: namespaces.Test, + }} + + err = clients.Create(context.Background(), sriovNetwork) + Expect(err).ToNot(HaveOccurred()) + waitForNetAttachDef("test-rdmanetwork", namespaces.Test) + + podDefinition := pod.DefineWithNetworks([]string{"test-rdmanetwork"}) + firstPod, err := clients.Pods(namespaces.Test).Create(context.Background(), podDefinition, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + firstPod = waitForPodRunning(firstPod) + + strOut, _, err := pod.ExecCommand(clients, firstPod, "/bin/bash", "-c", "ip link show net1 | grep net1 | wc -l") + Expect(err).ToNot(HaveOccurred()) + Expect(strings.HasPrefix(strOut, "1")).To(BeTrue()) + strOut, _, err = pod.ExecCommand(clients, firstPod, "/bin/bash", "-c", "ls /sys/bus/pci/devices/${PCIDEVICE_OPENSHIFT_IO_TESTRDMA}/infiniband/*/ports/* | grep hw_counters | wc -l") + strOut = strings.TrimSpace(strOut) + Expect(err).ToNot(HaveOccurred()) + num, err := strconv.Atoi(strOut) + Expect(err).ToNot(HaveOccurred()) + Expect(num).To(BeNumerically("==", 0)) + }) + }) +}) diff --git a/test/scripts/enable-kargs_test.sh b/test/scripts/kargs_test.sh similarity index 61% rename from test/scripts/enable-kargs_test.sh rename to test/scripts/kargs_test.sh index 93a985700..053bd5200 100755 --- a/test/scripts/enable-kargs_test.sh +++ b/test/scripts/kargs_test.sh @@ -2,14 +2,14 @@ SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -SUT_SCRIPT="${SCRIPTPATH}/../../bindata/scripts/enable-kargs.sh" +SUT_SCRIPT="${SCRIPTPATH}/../../bindata/scripts/kargs.sh" test_RpmOstree_Add_All_Arguments() { echo "a b c=d eee=fff" > ${FAKE_HOST}/proc/cmdline touch ${FAKE_HOST}/run/ostree-booted - output=`$SUT_SCRIPT X=Y W=Z` + output=`$SUT_SCRIPT add X=Y W=Z` assertEquals 0 $? assertEquals "2" $output @@ -22,7 +22,7 @@ test_RpmOstree_Add_Only_Missing_Arguments() { echo "a b c=d eee=fff K=L" > ${FAKE_HOST}/proc/cmdline touch ${FAKE_HOST}/run/ostree-booted - output=`$SUT_SCRIPT K=L X=Y` + output=`$SUT_SCRIPT add K=L X=Y` assertEquals 0 $? assertEquals "1" $output @@ -30,6 +30,29 @@ test_RpmOstree_Add_Only_Missing_Arguments() { assertNotContains "`cat ${FAKE_HOST}/rpm-ostree_calls`" "--append K=L" } +test_RpmOstree_Delete_All_Arguments() { + echo "a b c=d eee=fff X=Y W=Z" > ${FAKE_HOST}/proc/cmdline + touch ${FAKE_HOST}/run/ostree-booted + + output=`$SUT_SCRIPT remove X=Y W=Z` + assertEquals 0 $? + assertEquals "2" $output + + assertContains "`cat ${FAKE_HOST}/rpm-ostree_calls`" "--delete X=Y" + assertContains "`cat ${FAKE_HOST}/rpm-ostree_calls`" "--delete W=Z" +} + +test_RpmOstree_Delete_Only_Exist_Arguments() { + echo "a b c=d eee=fff X=Y" > ${FAKE_HOST}/proc/cmdline + touch ${FAKE_HOST}/run/ostree-booted + + output=`$SUT_SCRIPT remove X=Y W=Z` + assertEquals 0 $? + assertEquals "1" $output + + assertContains "`cat ${FAKE_HOST}/rpm-ostree_calls`" "--delete X=Y" + assertContains "`cat ${FAKE_HOST}/rpm-ostree_calls`" "--delete W=Z" +} ###### Mock /host directory ###### export FAKE_HOST="$(mktemp -d)" diff --git a/test/scripts/rpm-ostree_mock b/test/scripts/rpm-ostree_mock index db6f66040..37845483d 100755 --- a/test/scripts/rpm-ostree_mock +++ b/test/scripts/rpm-ostree_mock @@ -10,3 +10,9 @@ then # Caller is trying to read kernel arguments. cat /proc/cmdline fi + +if ! echo "$*" | grep -q "\--delete" +then + # Caller is trying to read kernel arguments. + cat /proc/cmdline +fi \ No newline at end of file