Skip to content

Commit

Permalink
Merge pull request ovn-org#4473 from qinqon/user-defined-network-prim…
Browse files Browse the repository at this point in the history
…ary-lsp

UDN: Use synthetic network selection element
  • Loading branch information
trozet authored Jul 12, 2024
2 parents 741cba6 + 0731759 commit 61b0a6b
Show file tree
Hide file tree
Showing 18 changed files with 567 additions and 73 deletions.
20 changes: 10 additions & 10 deletions go-controller/pkg/clustermanager/pod/allocator.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ func (a *PodAllocator) Init() error {

// getActiveNetworkForNamespace returns the active network for the given namespace
// and is a wrapper around util.GetActiveNetworkForNamespace
func (a *PodAllocator) getActiveNetworkForNamespace(namespace string) (string, error) {
func (a *PodAllocator) getActiveNetworkForNamespace(namespace string) (util.NetInfo, error) {
return util.GetActiveNetworkForNamespace(namespace, a.nadLister)
}

Expand Down Expand Up @@ -139,16 +139,11 @@ func (a *PodAllocator) GetNetworkRole(pod *corev1.Pod) (string, error) {
}
activeNetwork, err := a.getActiveNetworkForNamespace(pod.Namespace)
if err != nil {
return "", err
}
if activeNetwork == types.UnknownNetworkName {
// FIXME(tssurya) emit event here; add support for
// FIXME(tssurya) emit event here if util.IsUnknownActiveNetworkError; add support for
// recorder in the NCM controller
return "", fmt.Errorf("unable to determine what is the"+
"primary network for this pod %s; please remove multiple primary network"+
"NADs from namespace %s", pod.Name, pod.Namespace)
return "", err
}
if activeNetwork == a.netInfo.GetNetworkName() {
if activeNetwork.GetNetworkName() == a.netInfo.GetNetworkName() {
return types.NetworkRolePrimary, nil
}
if a.netInfo.IsDefault() {
Expand Down Expand Up @@ -206,7 +201,12 @@ func (a *PodAllocator) reconcile(old, new *corev1.Pod, releaseFromAllocator bool
return nil
}

onNetwork, networkMap, err := util.GetPodNADToNetworkMapping(pod, a.netInfo)
activeNetwork, err := a.getActiveNetworkForNamespace(pod.Namespace)
if err != nil {
return fmt.Errorf("failed looking for an active network: %w", err)
}

onNetwork, networkMap, err := util.GetPodNADToNetworkMappingWithActiveNetwork(pod, a.netInfo, activeNetwork)
if err != nil {
return fmt.Errorf("failed to get NAD to network mapping: %w", err)
}
Expand Down
65 changes: 61 additions & 4 deletions go-controller/pkg/cni/cni.go
Original file line number Diff line number Diff line change
@@ -1,15 +1,21 @@
package cni

import (
"context"
"fmt"
"net"
"time"

v1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"

kapi "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/klog/v2"
utilnet "k8s.io/utils/net"

current "github.com/containernetworking/cni/pkg/types/100"
ovncnitypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/types"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/cni/udn"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kubevirt"
Expand Down Expand Up @@ -131,17 +137,17 @@ func (pr *PodRequest) cmdAdd(kubeAuth *KubeAPIAuth, clientset *ClientSet) (*Resp
}
// Get the IP address and MAC address of the pod
// for DPU, ensure connection-details is present
pod, annotations, podNADAnnotation, err := GetPodWithAnnotations(pr.ctx, clientset, namespace, podName,
pr.nadName, annotCondFn)

primaryUDN := udn.NewPrimaryNetwork(clientset.nadLister)
pod, annotations, podNADAnnotation, err := GetPodWithAnnotations(pr.ctx, clientset, namespace, podName, pr.nadName, primaryUDN.WaitForPrimaryAnnotationFn(namespace, annotCondFn))
if err != nil {
return nil, fmt.Errorf("failed to get pod annotation: %v", err)
}
if err = pr.checkOrUpdatePodUID(pod); err != nil {
return nil, err
}

podInterfaceInfo, err := PodAnnotation2PodInfo(annotations, podNADAnnotation, pr.PodUID, netdevName,
pr.nadName, pr.netName, pr.CNIConf.MTU)
podInterfaceInfo, err := pr.buildPodInterfaceInfo(annotations, podNADAnnotation, netdevName)
if err != nil {
return nil, err
}
Expand All @@ -150,10 +156,22 @@ func (pr *PodRequest) cmdAdd(kubeAuth *KubeAPIAuth, clientset *ClientSet) (*Resp

response := &Response{KubeAuth: kubeAuth}
if !config.UnprivilegedMode {
//TODO: There is nothing technical to run this at unprivileged mode but
// we will tackle that later on.
response.Result, err = pr.getCNIResult(clientset, podInterfaceInfo)
if err != nil {
return nil, err
}
if primaryUDN.Found() {
primaryUDNPodRequest := pr.buildPrimaryUDNPodRequest(pod, primaryUDN)
primaryUDNPodInfo, err := primaryUDNPodRequest.buildPodInterfaceInfo(annotations, primaryUDN.Annotation(), primaryUDN.NetworkDevice())
if err != nil {
return nil, err
}
if _, err := primaryUDNPodRequest.getCNIResult(clientset, primaryUDNPodInfo); err != nil {
return nil, err
}
}
} else {
response.PodIFInfo = podInterfaceInfo
}
Expand Down Expand Up @@ -329,3 +347,42 @@ func (pr *PodRequest) getCNIResult(getter PodInfoGetter, podInterfaceInfo *PodIn
IPs: ips,
}, nil
}

func (pr *PodRequest) buildPrimaryUDNPodRequest(
pod *kapi.Pod,
primaryUDN *udn.UserDefinedPrimaryNetwork,
) *PodRequest {
req := &PodRequest{
Command: pr.Command,
PodNamespace: pod.Namespace,
PodName: pod.Name,
PodUID: string(pod.UID),
SandboxID: pr.SandboxID,
Netns: pr.Netns,
IfName: primaryUDN.InterfaceName(),
CNIConf: &ovncnitypes.NetConf{
// primary UDN MTU will be taken from config.Default.MTU
// if not specified at the NAD
MTU: primaryUDN.MTU(),
},
timestamp: time.Now(),
IsVFIO: pr.IsVFIO,
netName: primaryUDN.NetworkName(),
nadName: primaryUDN.NADName(),
deviceInfo: v1.DeviceInfo{},
}
req.ctx, req.cancel = context.WithTimeout(context.Background(), 2*time.Minute)
return req
}

func (pr *PodRequest) buildPodInterfaceInfo(annotations map[string]string, podAnnotation *util.PodAnnotation, netDevice string) (*PodInterfaceInfo, error) {
return PodAnnotation2PodInfo(
annotations,
podAnnotation,
pr.PodUID,
netDevice,
pr.nadName,
pr.netName,
pr.CNIConf.MTU,
)
}
4 changes: 4 additions & 0 deletions go-controller/pkg/cni/cniserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,10 @@ func NewCNIServer(factory factory.NodeWatchFactory, kclient kubernetes.Interface
handlePodRequestFunc: HandlePodRequest,
}

if util.IsNetworkSegmentationSupportEnabled() {
s.clientSet.nadLister = factory.NADInformer().Lister()
}

if len(config.Kubernetes.CAData) > 0 {
s.kubeAuth.KubeCAData = base64.StdEncoding.EncodeToString(config.Kubernetes.CAData)
}
Expand Down
6 changes: 3 additions & 3 deletions go-controller/pkg/cni/helper_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ func setupNetwork(link netlink.Link, ifInfo *PodInterfaceInfo) error {
}
for _, gw := range ifInfo.Gateways {
if err := cniPluginLibOps.AddRoute(nil, gw, link, ifInfo.RoutableMTU); err != nil {
return fmt.Errorf("failed to add gateway route: %v", err)
return fmt.Errorf("failed to add gateway route to link '%s': %v", link.Attrs().Name, err)
}
}
for _, route := range ifInfo.Routes {
Expand Down Expand Up @@ -438,8 +438,8 @@ func ConfigureOVS(ctx context.Context, namespace, podName, hostIfaceName string,
return fmt.Errorf("failed to get datapath type for bridge br-int : %v", err)
}

klog.Infof("ConfigureOVS: namespace: %s, podName: %s, network: %s, NAD %s, SandboxID: %q, PCI device ID: %s, UID: %q, MAC: %s, IPs: %v",
namespace, podName, ifInfo.NetName, ifInfo.NADName, sandboxID, deviceID, initialPodUID, ifInfo.MAC, ipStrs)
klog.Infof("ConfigureOVS: namespace: %s, podName: %s, hostIfaceName: %s, network: %s, NAD %s, SandboxID: %q, PCI device ID: %s, UID: %q, MAC: %s, IPs: %v",
namespace, podName, hostIfaceName, ifInfo.NetName, ifInfo.NADName, sandboxID, deviceID, initialPodUID, ifInfo.MAC, ipStrs)

// Find and remove any existing OVS port with this iface-id. Pods can
// have multiple sandboxes if some are waiting for garbage collection,
Expand Down
4 changes: 2 additions & 2 deletions go-controller/pkg/cni/helper_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,7 @@ func TestSetupNetwork(t *testing.T) {
{OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int"}, RetArgList: []interface{}{fmt.Errorf("mock error")}},
},
linkMockHelper: []ovntest.TestifyMockHelper{
{OnCallMethodName: "Attrs", OnCallMethodArgType: []string{}, RetArgList: []interface{}{&netlink.LinkAttrs{Name: "testIfaceName"}}},
{OnCallMethodName: "Attrs", OnCallMethodArgType: []string{}, RetArgList: []interface{}{&netlink.LinkAttrs{Name: "testIfaceName"}}, CallTimes: 2},
},
},
{
Expand Down Expand Up @@ -1128,7 +1128,7 @@ func TestSetupSriovInterface(t *testing.T) {
{OnCallMethodName: "AddRoute", OnCallMethodArgType: []string{"*net.IPNet", "net.IP", "*mocks.Link", "int"}, RetArgList: []interface{}{fmt.Errorf("mock error")}},
},
linkMockHelper: []ovntest.TestifyMockHelper{
{OnCallMethodName: "Attrs", OnCallMethodArgType: []string{}, RetArgList: []interface{}{&netlink.LinkAttrs{Flags: net.FlagUp}}},
{OnCallMethodName: "Attrs", OnCallMethodArgType: []string{}, RetArgList: []interface{}{&netlink.LinkAttrs{Flags: net.FlagUp}}, CallTimes: 2},
},
nsMockHelper: []ovntest.TestifyMockHelper{},
},
Expand Down
3 changes: 3 additions & 0 deletions go-controller/pkg/cni/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ import (
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"

nadapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
nadlister "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1"

kapi "k8s.io/api/core/v1"
)

Expand Down Expand Up @@ -177,6 +179,7 @@ type ClientSet struct {
PodInfoGetter
kclient kubernetes.Interface
podLister corev1listers.PodLister
nadLister nadlister.NetworkAttachmentDefinitionLister
}

func NewClientSet(kclient kubernetes.Interface, podLister corev1listers.PodLister) *ClientSet {
Expand Down
132 changes: 132 additions & 0 deletions go-controller/pkg/cni/udn/primary_network.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
package udn

import (
"fmt"

"k8s.io/klog/v2"

nadlister "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/listers/k8s.cni.cncf.io/v1"

"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
)

// wait on a certain pod annotation related condition
type podAnnotWaitCond = func(map[string]string, string) (*util.PodAnnotation, bool)

type UserDefinedPrimaryNetwork struct {
nadLister nadlister.NetworkAttachmentDefinitionLister
annotation *util.PodAnnotation
activeNetwork util.NetInfo
}

func NewPrimaryNetwork(nadLister nadlister.NetworkAttachmentDefinitionLister) *UserDefinedPrimaryNetwork {
return &UserDefinedPrimaryNetwork{
nadLister: nadLister,
}
}

func (p *UserDefinedPrimaryNetwork) InterfaceName() string {
return "ovn-udn1"
}

func (p *UserDefinedPrimaryNetwork) NetworkDevice() string {
// TODO: Support for non VFIO devices like SRIOV have to be implemented
return ""
}

func (p *UserDefinedPrimaryNetwork) Annotation() *util.PodAnnotation {
return p.annotation
}

func (p *UserDefinedPrimaryNetwork) NetworkName() string {
if p.activeNetwork == nil {
return ""
}
return p.activeNetwork.GetNetworkName()
}

func (p *UserDefinedPrimaryNetwork) NADName() string {
if p.activeNetwork == nil || p.activeNetwork.IsDefault() {
return ""
}
nads := p.activeNetwork.GetNADs()
if len(nads) < 1 {
return ""
}
return nads[0]
}

func (p *UserDefinedPrimaryNetwork) MTU() int {
if p.activeNetwork == nil {
return 0
}
return p.activeNetwork.MTU()
}

func (p *UserDefinedPrimaryNetwork) Found() bool {
return p.annotation != nil && p.activeNetwork != nil
}

func (p *UserDefinedPrimaryNetwork) WaitForPrimaryAnnotationFn(namespace string, annotCondFn podAnnotWaitCond) podAnnotWaitCond {
return func(annotations map[string]string, nadName string) (*util.PodAnnotation, bool) {
annotation, isReady := annotCondFn(annotations, nadName)
if annotation == nil {
return nil, false
}
if nadName != types.DefaultNetworkName || annotation.Role == types.NetworkRolePrimary {
return annotation, isReady
}

if err := p.ensureAnnotation(annotations); err != nil {
//TODO: Event ?
klog.Warningf("Failed looking for primary network annotation: %v", err)
return nil, false
}
if err := p.ensureActiveNetwork(namespace); err != nil {
//TODO: Event ?
klog.Warningf("Failed looking for primary network name: %v", err)
return nil, false
}
return annotation, isReady
}
}

func (p *UserDefinedPrimaryNetwork) ensureActiveNetwork(namespace string) error {
if p.activeNetwork != nil {
return nil
}
activeNetwork, err := util.GetActiveNetworkForNamespace(namespace, p.nadLister)
if err != nil {
return err
}
if activeNetwork.IsDefault() {
return fmt.Errorf("missing primary user defined network NAD")
}
p.activeNetwork = activeNetwork
return nil
}

func (p *UserDefinedPrimaryNetwork) ensureAnnotation(annotations map[string]string) error {
if p.annotation != nil {
return nil
}
podNetworks, err := util.UnmarshalPodAnnotationAllNetworks(annotations)
if err != nil {
return err
}
for nadName, podNetwork := range podNetworks {
if podNetwork.Role != types.NetworkRolePrimary {
continue
}
p.annotation, err = util.UnmarshalPodAnnotation(annotations, nadName)
if err != nil {
return err
}
break
}
if p.annotation == nil {
return fmt.Errorf("missing network annotation with primary role '%+v'", annotations)
}
return nil
}
Loading

0 comments on commit 61b0a6b

Please sign in to comment.