Implement RDMA subsystem mode change

api/v1/sriovnetworknodestate_types.go

@@ -27,6 +27,7 @@ import (
 type SriovNetworkNodeStateSpec struct {
 	Interfaces Interfaces `json:"interfaces,omitempty"`
 	Bridges    Bridges    `json:"bridges,omitempty"`
+	System     System     `json:"system,omitempty"`
 }
 
 type Interfaces []Interface
@@ -114,10 +115,16 @@ type OVSUplinkConfigExt struct {
 	Interface OVSInterfaceConfig `json:"interface,omitempty"`
 }
 
+type System struct {
+	// +kubebuilder:validation:Enum=shared;exclusive
+	RdmaMode string `json:"rdmaMode,omitempty"`
+}
+
 // SriovNetworkNodeStateStatus defines the observed state of SriovNetworkNodeState
 type SriovNetworkNodeStateStatus struct {
 	Interfaces    InterfaceExts `json:"interfaces,omitempty"`
 	Bridges       Bridges       `json:"bridges,omitempty"`
+	System        System        `json:"system,omitempty"`
 	SyncStatus    string        `json:"syncStatus,omitempty"`
 	LastSyncError string        `json:"lastSyncError,omitempty"`
 }

api/v1/sriovnetworkpoolconfig_types.go

@@ -21,6 +21,10 @@ type SriovNetworkPoolConfigSpec struct {
 	// Drain will respect Pod Disruption Budgets (PDBs) such as etcd quorum guards,
 	// even if maxUnavailable is greater than one.
 	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
+
+	// +kubebuilder:validation:Enum=shared;exclusive
+	// RDMA subsystem. Allowed value "shared", "exclusive".
+	RdmaMode string `json:"rdmaMode,omitempty"`
 }
 
 type OvsHardwareOffloadConfig struct {

config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml

@@ -174,6 +174,14 @@ spec:
                   - pciAddress
                   type: object
                 type: array
+              system:
+                properties:
+                  rdmaMode:
+                    enum:
+                    - shared
+                    - exclusive
+                    type: string
+                type: object
             type: object
           status:
             description: SriovNetworkNodeStateStatus defines the observed state of
@@ -335,6 +343,14 @@ spec:
                 type: string
               syncStatus:
                 type: string
+              system:
+                properties:
+                  rdmaMode:
+                    enum:
+                    - shared
+                    - exclusive
+                    type: string
+                type: object
             type: object
         type: object
     served: true

config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml

@@ -111,6 +111,12 @@ spec:
                       Name is the name of MachineConfigPool to be enabled with OVS hardware offload
                     type: string
                 type: object
+              rdmaMode:
+                description: RDMA subsystem. Allowed value "shared", "exclusive".
+                enum:
+                - shared
+                - exclusive
+                type: string
             type: object
           status:
             description: SriovNetworkPoolConfigStatus defines the observed state of

controllers/drain_controller.go

@@ -24,11 +24,8 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -48,13 +45,6 @@ import (
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
 )
 
-var (
-	oneNode     = intstr.FromInt32(1)
-	defaultNpcl = &sriovnetworkv1.SriovNetworkPoolConfig{Spec: sriovnetworkv1.SriovNetworkPoolConfigSpec{
-		MaxUnavailable: &oneNode,
-		NodeSelector:   &metav1.LabelSelector{}}}
-)
-
 type DrainReconcile struct {
 	client.Client
 	Scheme   *runtime.Scheme
@@ -346,94 +336,7 @@ func (dr *DrainReconcile) tryDrainNode(ctx context.Context, node *corev1.Node) (
 }
 
 func (dr *DrainReconcile) findNodePoolConfig(ctx context.Context, node *corev1.Node) (*sriovnetworkv1.SriovNetworkPoolConfig, []corev1.Node, error) {
-	logger := log.FromContext(ctx)
-	logger.Info("findNodePoolConfig():")
-	// get all the sriov network pool configs
-	npcl := &sriovnetworkv1.SriovNetworkPoolConfigList{}
-	err := dr.List(ctx, npcl)
-	if err != nil {
-		logger.Error(err, "failed to list sriovNetworkPoolConfig")
-		return nil, nil, err
-	}
-
-	selectedNpcl := []*sriovnetworkv1.SriovNetworkPoolConfig{}
-	nodesInPools := map[string]interface{}{}
-
-	for _, npc := range npcl.Items {
-		// we skip hw offload objects
-		if npc.Spec.OvsHardwareOffloadConfig.Name != "" {
-			continue
-		}
-
-		if npc.Spec.NodeSelector == nil {
-			npc.Spec.NodeSelector = &metav1.LabelSelector{}
-		}
-
-		selector, err := metav1.LabelSelectorAsSelector(npc.Spec.NodeSelector)
-		if err != nil {
-			logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", npc.Spec.NodeSelector)
-			return nil, nil, err
-		}
-
-		if selector.Matches(labels.Set(node.Labels)) {
-			selectedNpcl = append(selectedNpcl, npc.DeepCopy())
-		}
-
-		nodeList := &corev1.NodeList{}
-		err = dr.List(ctx, nodeList, &client.ListOptions{LabelSelector: selector})
-		if err != nil {
-			logger.Error(err, "failed to list all the nodes matching the pool with label selector from nodeSelector",
-				"machineConfigPoolName", npc,
-				"nodeSelector", npc.Spec.NodeSelector)
-			return nil, nil, err
-		}
-
-		for _, nodeName := range nodeList.Items {
-			nodesInPools[nodeName.Name] = nil
-		}
-	}
-
-	if len(selectedNpcl) > 1 {
-		// don't allow the node to be part of multiple pools
-		err = fmt.Errorf("node is part of more then one pool")
-		logger.Error(err, "multiple pools founded for a specific node", "numberOfPools", len(selectedNpcl), "pools", selectedNpcl)
-		return nil, nil, err
-	} else if len(selectedNpcl) == 1 {
-		// found one pool for our node
-		logger.V(2).Info("found sriovNetworkPool", "pool", *selectedNpcl[0])
-		selector, err := metav1.LabelSelectorAsSelector(selectedNpcl[0].Spec.NodeSelector)
-		if err != nil {
-			logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", selectedNpcl[0].Spec.NodeSelector)
-			return nil, nil, err
-		}
-
-		// list all the nodes that are also part of this pool and return them
-		nodeList := &corev1.NodeList{}
-		err = dr.List(ctx, nodeList, &client.ListOptions{LabelSelector: selector})
-		if err != nil {
-			logger.Error(err, "failed to list nodes using with label selector", "labelSelector", selector)
-			return nil, nil, err
-		}
-
-		return selectedNpcl[0], nodeList.Items, nil
-	} else {
-		// in this case we get all the nodes and remove the ones that already part of any pool
-		logger.V(1).Info("node doesn't belong to any pool, using default drain configuration with MaxUnavailable of one", "pool", *defaultNpcl)
-		nodeList := &corev1.NodeList{}
-		err = dr.List(ctx, nodeList)
-		if err != nil {
-			logger.Error(err, "failed to list all the nodes")
-			return nil, nil, err
-		}
-
-		defaultNodeLists := []corev1.Node{}
-		for _, nodeObj := range nodeList.Items {
-			if _, exist := nodesInPools[nodeObj.Name]; !exist {
-				defaultNodeLists = append(defaultNodeLists, nodeObj)
-			}
-		}
-		return defaultNpcl, defaultNodeLists, nil
-	}
+	return findNodePoolConfig(ctx, node, dr.Client)
 }
 
 // SetupWithManager sets up the controller with the Manager.

controllers/helper.go

@@ -30,9 +30,12 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	uns "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	kscheme "k8s.io/client-go/kubernetes/scheme"
 	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -47,10 +50,17 @@ import (
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
 )
 
-var webhooks = map[string](string){
-	constants.InjectorWebHookName: constants.InjectorWebHookPath,
-	constants.OperatorWebHookName: constants.OperatorWebHookPath,
-}
+var (
+	webhooks = map[string](string){
+		constants.InjectorWebHookName: constants.InjectorWebHookPath,
+		constants.OperatorWebHookName: constants.OperatorWebHookPath,
+	}
+	oneNode           = intstr.FromInt32(1)
+	defaultPoolConfig = &sriovnetworkv1.SriovNetworkPoolConfig{Spec: sriovnetworkv1.SriovNetworkPoolConfigSpec{
+		MaxUnavailable: &oneNode,
+		NodeSelector:   &metav1.LabelSelector{},
+		RdmaMode:       ""}}
+)
 
 const (
 	clusterRoleResourceName               = "ClusterRole"
@@ -397,3 +407,94 @@ func updateDaemonsetNodeSelector(obj *uns.Unstructured, nodeSelector map[string]
 	}
 	return nil
 }
+
+func findNodePoolConfig(ctx context.Context, node *corev1.Node, c k8sclient.Client) (*sriovnetworkv1.SriovNetworkPoolConfig, []corev1.Node, error) {
+	logger := log.FromContext(ctx)
+	logger.Info("FindNodePoolConfig():")
+	// get all the sriov network pool configs
+	npcl := &sriovnetworkv1.SriovNetworkPoolConfigList{}
+	err := c.List(ctx, npcl)
+	if err != nil {
+		logger.Error(err, "failed to list sriovNetworkPoolConfig")
+		return nil, nil, err
+	}
+
+	selectedNpcl := []*sriovnetworkv1.SriovNetworkPoolConfig{}
+	nodesInPools := map[string]interface{}{}
+
+	for _, npc := range npcl.Items {
+		// we skip hw offload objects
+		if npc.Spec.OvsHardwareOffloadConfig.Name != "" {
+			continue
+		}
+
+		if npc.Spec.NodeSelector == nil {
+			npc.Spec.NodeSelector = &metav1.LabelSelector{}
+		}
+
+		selector, err := metav1.LabelSelectorAsSelector(npc.Spec.NodeSelector)
+		if err != nil {
+			logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", npc.Spec.NodeSelector)
+			return nil, nil, err
+		}
+
+		if selector.Matches(labels.Set(node.Labels)) {
+			selectedNpcl = append(selectedNpcl, npc.DeepCopy())
+		}
+
+		nodeList := &corev1.NodeList{}
+		err = c.List(ctx, nodeList, &k8sclient.ListOptions{LabelSelector: selector})
+		if err != nil {
+			logger.Error(err, "failed to list all the nodes matching the pool with label selector from nodeSelector",
+				"machineConfigPoolName", npc,
+				"nodeSelector", npc.Spec.NodeSelector)
+			return nil, nil, err
+		}
+
+		for _, nodeName := range nodeList.Items {
+			nodesInPools[nodeName.Name] = nil
+		}
+	}
+
+	if len(selectedNpcl) > 1 {
+		// don't allow the node to be part of multiple pools
+		err = fmt.Errorf("node is part of more then one pool")
+		logger.Error(err, "multiple pools founded for a specific node", "numberOfPools", len(selectedNpcl), "pools", selectedNpcl)
+		return nil, nil, err
+	} else if len(selectedNpcl) == 1 {
+		// found one pool for our node
+		logger.V(2).Info("found sriovNetworkPool", "pool", *selectedNpcl[0])
+		selector, err := metav1.LabelSelectorAsSelector(selectedNpcl[0].Spec.NodeSelector)
+		if err != nil {
+			logger.Error(err, "failed to create label selector from nodeSelector", "nodeSelector", selectedNpcl[0].Spec.NodeSelector)
+			return nil, nil, err
+		}
+
+		// list all the nodes that are also part of this pool and return them
+		nodeList := &corev1.NodeList{}
+		err = c.List(ctx, nodeList, &k8sclient.ListOptions{LabelSelector: selector})
+		if err != nil {
+			logger.Error(err, "failed to list nodes using with label selector", "labelSelector", selector)
+			return nil, nil, err
+		}
+
+		return selectedNpcl[0], nodeList.Items, nil
+	} else {
+		// in this case we get all the nodes and remove the ones that already part of any pool
+		logger.V(1).Info("node doesn't belong to any pool, using default drain configuration with MaxUnavailable of one", "pool", *defaultPoolConfig)
+		nodeList := &corev1.NodeList{}
+		err = c.List(ctx, nodeList)
+		if err != nil {
+			logger.Error(err, "failed to list all the nodes")
+			return nil, nil, err
+		}
+
+		defaultNodeLists := []corev1.Node{}
+		for _, nodeObj := range nodeList.Items {
+			if _, exist := nodesInPools[nodeObj.Name]; !exist {
+				defaultNodeLists = append(defaultNodeLists, nodeObj)
+			}
+		}
+		return defaultPoolConfig, defaultNodeLists, nil
+	}
+}

controllers/sriovnetworknodepolicy_controller.go

@@ -272,6 +272,13 @@ func (r *SriovNetworkNodePolicyReconciler) syncAllSriovNetworkNodeStates(ctx con
 		ns.Name = node.Name
 		ns.Namespace = vars.Namespace
 		j, _ := json.Marshal(ns)
+		netPoolConfig, _, err := findNodePoolConfig(ctx, &node, r.Client)
+		if err != nil {
+			log.Log.Error(err, "nodeStateSyncHandler(): failed to get SriovNetworkPoolConfig for the current node")
+		}
+		if netPoolConfig != nil {
+			ns.Spec.System.RdmaMode = netPoolConfig.Spec.RdmaMode
+		}
 		logger.V(2).Info("SriovNetworkNodeState CR", "content", j)
 		if err := r.syncSriovNetworkNodeState(ctx, dc, npl, ns, &node); err != nil {
 			logger.Error(err, "Fail to sync", "SriovNetworkNodeState", ns.Name)