Implement RDMA subsystem mode change

Now it's possible to configure RDMA subsystem mode using SR-IOV Network Operator in systemd mode We can't configure RDMA subsystem in a daemon mode because it should be done on host before any network namespace is created.
k8snetworkplumbingwg · Mar 24, 2024 · c986251 · c986251
1 parent bbdfd9d
commit c986251
Show file tree

Hide file tree

Showing 22 changed files with 432 additions and 282 deletions.
diff --git a/api/v1/sriovnetworkpoolconfig_types.go b/api/v1/sriovnetworkpoolconfig_types.go
@@ -21,6 +21,10 @@ type SriovNetworkPoolConfigSpec struct {
 	// Drain will respect Pod Disruption Budgets (PDBs) such as etcd quorum guards,
 	// even if maxUnavailable is greater than one.
 	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
+
+	// +kubebuilder:validation:Enum=shared;exclusive
+	// RDMA subsystem. Allowed value "shared", "exclusive".
+	RdmaMode string `json:"rdmaMode,omitempty"`
 }
 
 type OvsHardwareOffloadConfig struct {

diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
diff --git a/cmd/sriov-network-config-daemon/service.go b/cmd/sriov-network-config-daemon/service.go
@@ -152,6 +152,17 @@ func phasePre(setupLog logr.Logger, conf *systemd.SriovConfig, hostHelpers helpe
 	hostHelpers.TryEnableTun()
 	hostHelpers.TryEnableVhostNet()
 
+	rdmaSubsystem, err := hostHelpers.GetRDMASubsystem()
+	if err != nil {
+		setupLog.Error(err, "failed to get RDMA subsystem mode")
+	}
+	if rdmaSubsystem != conf.RdmaMode {
+		err = hostHelpers.SetRDMASubsystem(conf.RdmaMode)
+		if err != nil {
+			setupLog.Error(err, "failed to set RDMA subsystem mode")
+		}
+	}
+
 	return callPlugin(setupLog, PhasePre, conf, hostHelpers)
 }
 

diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovibnetworks.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovibnetworks.yaml
@@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.0
-  creationTimestamp: null
+    controller-gen.kubebuilder.io/version: v0.14.0
   name: sriovibnetworks.sriovnetwork.openshift.io
 spec:
   group: sriovnetwork.openshift.io
@@ -21,23 +20,29 @@ spec:
         description: SriovIBNetwork is the Schema for the sriovibnetworks API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
           spec:
             description: SriovIBNetworkSpec defines the desired state of SriovIBNetwork
             properties:
               capabilities:
-                description: 'Capabilities to be configured for this network. Capabilities
-                  supported: (infinibandGUID), e.g. ''{"infinibandGUID": true}'''
+                description: |-
+                  Capabilities to be configured for this network.
+                  Capabilities supported: (infinibandGUID), e.g. '{"infinibandGUID": true}'
                 type: string
               ipam:
                 description: IPAM configuration to be used for this network.
@@ -50,8 +55,9 @@ spec:
                 - disable
                 type: string
               metaPlugins:
-                description: MetaPluginsConfig configuration to be used in order to
-                  chain metaplugins to the sriov interface returned by the operator.
+                description: |-
+                  MetaPluginsConfig configuration to be used in order to chain metaplugins to the sriov interface returned
+                  by the operator.
                 type: string
               networkNamespace:
                 description: Namespace of the NetworkAttachmentDefinition custom resource

diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml
@@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.0
-  creationTimestamp: null
+    controller-gen.kubebuilder.io/version: v0.14.0
   name: sriovnetworknodepolicies.sriovnetwork.openshift.io
 spec:
   group: sriovnetwork.openshift.io
@@ -22,14 +21,19 @@ spec:
           API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object

diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml
@@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.0
-  creationTimestamp: null
+    controller-gen.kubebuilder.io/version: v0.14.0
   name: sriovnetworknodestates.sriovnetwork.openshift.io
 spec:
   group: sriovnetwork.openshift.io
@@ -35,14 +34,19 @@ spec:
           API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object

diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml
@@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.0
-  creationTimestamp: null
+    controller-gen.kubebuilder.io/version: v0.14.0
   name: sriovnetworkpoolconfigs.sriovnetwork.openshift.io
 spec:
   group: sriovnetwork.openshift.io
@@ -22,14 +21,19 @@ spec:
           API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -40,13 +44,15 @@ spec:
                 anyOf:
                 - type: integer
                 - type: string
-                description: "maxUnavailable defines either an integer number or percentage
-                  of nodes in the pool that can go Unavailable during an update. \n
-                  A value larger than 1 will mean multiple nodes going unavailable
-                  during the update, which may affect your workload stress on the
-                  remaining nodes. Drain will respect Pod Disruption Budgets (PDBs)
-                  such as etcd quorum guards, even if maxUnavailable is greater than
-                  one."
+                description: |-
+                  maxUnavailable defines either an integer number or percentage
+                  of nodes in the pool that can go Unavailable during an update.
+
+
+                  A value larger than 1 will mean multiple nodes going unavailable during
+                  the update, which may affect your workload stress on the remaining nodes.
+                  Drain will respect Pod Disruption Budgets (PDBs) such as etcd quorum guards,
+                  even if maxUnavailable is greater than one.
                 x-kubernetes-int-or-string: true
               nodeSelector:
                 description: nodeSelector specifies a label selector for Nodes
@@ -55,24 +61,24 @@ spec:
                     description: matchExpressions is a list of label selector requirements.
                       The requirements are ANDed.
                     items:
-                      description: A label selector requirement is a selector that
-                        contains values, a key, and an operator that relates the key
-                        and values.
+                      description: |-
+                        A label selector requirement is a selector that contains values, a key, and an operator that
+                        relates the key and values.
                       properties:
                         key:
                           description: key is the label key that the selector applies
                             to.
                           type: string
                         operator:
-                          description: operator represents a key's relationship to
-                            a set of values. Valid operators are In, NotIn, Exists
-                            and DoesNotExist.
+                          description: |-
+                            operator represents a key's relationship to a set of values.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
                           type: string
                         values:
-                          description: values is an array of string values. If the
-                            operator is In or NotIn, the values array must be non-empty.
-                            If the operator is Exists or DoesNotExist, the values
-                            array must be empty. This array is replaced during a strategic
+                          description: |-
+                            values is an array of string values. If the operator is In or NotIn,
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                            the values array must be empty. This array is replaced during a strategic
                             merge patch.
                           items:
                             type: string
@@ -85,24 +91,32 @@ spec:
                   matchLabels:
                     additionalProperties:
                       type: string
-                    description: matchLabels is a map of {key,value} pairs. A single
-                      {key,value} in the matchLabels map is equivalent to an element
-                      of matchExpressions, whose key field is "key", the operator
-                      is "In", and the values array contains only "value". The requirements
-                      are ANDed.
+                    description: |-
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
                     type: object
                 type: object
+                x-kubernetes-map-type: atomic
               ovsHardwareOffloadConfig:
                 description: OvsHardwareOffloadConfig describes the OVS HWOL configuration
                   for selected Nodes
                 properties:
                   name:
-                    description: 'Name is mandatory and must be unique. On Kubernetes:
-                      Name is the name of OvsHardwareOffloadConfig On OpenShift: Name
-                      is the name of MachineConfigPool to be enabled with OVS hardware
-                      offload'
+                    description: |-
+                      Name is mandatory and must be unique.
+                      On Kubernetes:
+                      Name is the name of OvsHardwareOffloadConfig
+                      On OpenShift:
+                      Name is the name of MachineConfigPool to be enabled with OVS hardware offload
                     type: string
                 type: object
+              rdmaMode:
+                description: RDMA subsystem. Allowed value "shared", "exclusive".
+                enum:
+                - shared
+                - exclusive
+                type: string
             type: object
           status:
             description: SriovNetworkPoolConfigStatus defines the observed state of