From 4e6cc2ff1e6eac131221cd03db062a324da85370 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Fri, 10 Nov 2023 16:07:40 -0800 Subject: [PATCH] fix: wait for vnet ns to create and ensure veths are inside namespace instead of assuming (#2341) * Make initial changes to enable forwarding Reuse existing code to enable ip forwarding Add forwarding message Add transparent vlan config to dropgz Add enable ip forward to each network namespace for redundancy * Add general run with retries function * Migrate existing code to use retry function * Add retries for checking of vnet and container veth creation * Add preliminary repair item fixes * Clarify log message * Fix unit tests, move ensure clean populate vm to add endpoints, remove ip forwarding fix as it is in other branch * Move setting link netns and confirmation to function * Add tests for setLinkNetNSAndConfirm * Address linter issues * Add vm ns to vnet namespace log statement * Fix after rebasing in swift refactoring changes from master * Address feedback and check error message * Address feedback * Address linter issue * Address log feedback * Address feedback by asserting any error is an interface not found error --- network/endpoint_snatroute_linux.go | 2 +- network/networkutils/networkutils_linux.go | 4 +- .../transparent_vlan_endpointclient_linux.go | 177 ++++++++--- ...nsparent_vlan_endpointclient_linux_test.go | 282 +++++++++++++++++- 4 files changed, 410 insertions(+), 55 deletions(-) diff --git a/network/endpoint_snatroute_linux.go b/network/endpoint_snatroute_linux.go index 6b659327c3..3f45a7f16e 100644 --- a/network/endpoint_snatroute_linux.go +++ b/network/endpoint_snatroute_linux.go @@ -37,7 +37,7 @@ func AddSnatEndpointRules(snatClient *snat.Client, hostToNC, ncToHost bool, nl n return errors.Wrap(err, "failed to block ip addresses on snat bridge") } nuc := networkutils.NewNetworkUtils(nl, plc) - if err := nuc.EnableIPForwarding(snat.SnatBridgeName); err != nil { + if err := nuc.EnableIPForwarding(); err != nil { return errors.Wrap(err, "failed to enable ip forwarding") } diff --git a/network/networkutils/networkutils_linux.go b/network/networkutils/networkutils_linux.go index c6b94ddb2e..4d7473b0b5 100644 --- a/network/networkutils/networkutils_linux.go +++ b/network/networkutils/networkutils_linux.go @@ -200,8 +200,8 @@ func BlockIPAddresses(bridgeName, action string) error { return nil } -// This fucntion enables ip forwarding in VM and allow forwarding packets from the interface -func (nu NetworkUtils) EnableIPForwarding(ifName string) error { +// This function enables ip forwarding in VM and allow forwarding packets from the interface +func (nu NetworkUtils) EnableIPForwarding() error { // Enable ip forwading on linux vm. // sysctl -w net.ipv4.ip_forward=1 cmd := fmt.Sprint(enableIPForwardCmd) diff --git a/network/transparent_vlan_endpointclient_linux.go b/network/transparent_vlan_endpointclient_linux.go index 5532db20d1..56ae8517c3 100644 --- a/network/transparent_vlan_endpointclient_linux.go +++ b/network/transparent_vlan_endpointclient_linux.go @@ -26,14 +26,11 @@ const ( tunnelingTable = 2 // Packets not entering on the vlan interface go to this routing table tunnelingMark = 333 // The packets that are to tunnel will be marked with this number DisableRPFilterCmd = "sysctl -w net.ipv4.conf.all.rp_filter=0" // Command to disable the rp filter for tunneling + numRetries = 5 + sleepInMs = 100 ) -var errNamespaceCreation = fmt.Errorf("Network namespace creation error") - -var ( - numRetries = 5 - sleepInMs = 100 -) +var errNamespaceCreation = fmt.Errorf("network namespace creation error") type netnsClient interface { Get() (fileDescriptor int, err error) @@ -113,8 +110,10 @@ func NewTransparentVlanEndpointClient( // Adds interfaces to the vnet (created if not existing) and vm namespace func (client *TransparentVlanEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { // VM Namespace - err := client.PopulateVM(epInfo) - if err != nil { + if err := client.ensureCleanPopulateVM(); err != nil { + return errors.Wrap(err, "failed to ensure both network namespace and vlan veth were present or both absent") + } + if err := client.PopulateVM(epInfo); err != nil { return err } if err := client.AddSnatEndpoint(); err != nil { @@ -126,32 +125,80 @@ func (client *TransparentVlanEndpointClient) AddEndpoints(epInfo *EndpointInfo) }) } -func (client *TransparentVlanEndpointClient) createNetworkNamespace(vmNS, numRetries int) error { - var isNamespaceUnique bool - - for i := 0; i < numRetries; i++ { - vnetNS, err := client.netnsClient.NewNamed(client.vnetNSName) - if err != nil { - return errors.Wrap(err, "failed to create vnet ns") - } - logger.Info("Vnet Namespace created", zap.String("vnetNS", client.netnsClient.NamespaceUniqueID(vnetNS))) - if !client.netnsClient.IsNamespaceEqual(vnetNS, vmNS) { - client.vnetNSFileDescriptor = vnetNS - isNamespaceUnique = true - break +// Called from AddEndpoints, Namespace: VM and Vnet +func (client *TransparentVlanEndpointClient) ensureCleanPopulateVM() error { + // Clean up vlan interface in the VM namespace and ensure the network namespace (if it exists) has a vlan interface + logger.Info("Checking if NS and vlan veth exists...") + var existingErr error + client.vnetNSFileDescriptor, existingErr = client.netnsClient.GetFromName(client.vnetNSName) + if existingErr == nil { + // The namespace exists + vlanIfErr := ExecuteInNS(client.nsClient, client.vnetNSName, func() error { + // Ensure the vlan interface exists in the namespace + _, err := client.netioshim.GetNetworkInterfaceByName(client.vlanIfName) + return errors.Wrap(err, "failed to get vlan interface in namespace") + }) + if vlanIfErr != nil { + // Assume any error is the vlan interface not found + logger.Info("vlan interface doesn't exist even though network namespace exists, deleting network namespace...", zap.String("message", vlanIfErr.Error())) + delErr := client.netnsClient.DeleteNamed(client.vnetNSName) + if delErr != nil { + return errors.Wrap(delErr, "failed to cleanup/delete ns after noticing vlan veth does not exist") + } } - logger.Info("Vnet Namespace is the same as VM namespace. Deleting and retrying...") - delErr := client.netnsClient.DeleteNamed(client.vnetNSName) - if delErr != nil { - logger.Error("failed to cleanup/delete ns after failing to create vlan veth", zap.Any("error:", delErr.Error())) + } + // Delete the vlan interface in the VM namespace if it exists + _, vlanIfInVMErr := client.netioshim.GetNetworkInterfaceByName(client.vlanIfName) + if vlanIfInVMErr == nil { + // The vlan interface exists in the VM ns because it failed to move into the network ns previously and needs to be cleaned up + logger.Info("vlan interface exists on the VM namespace, deleting", zap.String("vlanIfName", client.vlanIfName)) + if delErr := client.netlink.DeleteLink(client.vlanIfName); delErr != nil { + return errors.Wrap(delErr, "failed to clean up vlan interface") } - time.Sleep(time.Duration(sleepInMs) * time.Millisecond) } + return nil +} + +// Called from PopulateVM, Namespace: VM +func (client *TransparentVlanEndpointClient) createNetworkNamespace(vmNS int) error { + // If this call succeeds, the namespace is set to the new namespace + vnetNS, err := client.netnsClient.NewNamed(client.vnetNSName) + if err != nil { + return errors.Wrap(err, "failed to create vnet ns") + } + logger.Info("Vnet Namespace created", zap.String("vnetNS", client.netnsClient.NamespaceUniqueID(vnetNS)), zap.String("vmNS", client.netnsClient.NamespaceUniqueID(vmNS))) + if !client.netnsClient.IsNamespaceEqual(vnetNS, vmNS) { + client.vnetNSFileDescriptor = vnetNS + return nil + } + // the vnet and vm namespace are the same by this point + logger.Info("Vnet Namespace is the same as VM namespace. Deleting...") + delErr := client.netnsClient.DeleteNamed(client.vnetNSName) + if delErr != nil { + logger.Error("failed to cleanup/delete ns after noticing vnet ns is the same as vm ns", zap.Any("error:", delErr.Error())) + } + return errors.Wrap(errNamespaceCreation, "vnet and vm namespace are the same") +} - if !isNamespaceUnique { - return errors.Wrap(errNamespaceCreation, "vnet and vm namespace are same") +// Called from PopulateVM, Namespace: VM and namespace represented by fd +func (client *TransparentVlanEndpointClient) setLinkNetNSAndConfirm(name string, fd uintptr) error { + logger.Info("Move link to NS", zap.String("ifName", name), zap.Any("NSFileDescriptor", fd)) + err := client.netlink.SetLinkNetNs(name, fd) + if err != nil { + return errors.Wrapf(err, "failed to set %v inside namespace %v", name, fd) } + // confirm veth was moved successfully + err = RunWithRetries(func() error { + // retry checking in the namespace if the interface is not detected + return ExecuteInNS(client.nsClient, client.vnetNSName, func() error { + _, ifDetectedErr := client.netioshim.GetNetworkInterfaceByName(client.vlanIfName) + return errors.Wrap(ifDetectedErr, "failed to get vlan veth in namespace") + }) + }, numRetries, sleepInMs) + if err != nil { + return errors.Wrapf(err, "failed to detect %v inside namespace %v", name, fd) + } return nil } @@ -169,10 +216,13 @@ func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) er // This will also (we assume) mean the vlan veth does not exist if existingErr != nil { // We assume the only possible error is that the namespace doesn't exist - logger.Info("No existing NS detected. Creating the vnet namespace and switching to it") + logger.Info("No existing NS detected. Creating the vnet namespace and switching to it", zap.String("message", existingErr.Error())) - if err = client.createNetworkNamespace(vmNS, numRetries); err != nil { - return errors.Wrap(err, "") + err = RunWithRetries(func() error { + return client.createNetworkNamespace(vmNS) + }, numRetries, sleepInMs) + if err != nil { + return errors.Wrap(err, "failed to create network namespace") } deleteNSIfNotNilErr := client.netnsClient.Set(vmNS) @@ -214,6 +264,8 @@ func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) er // Any failure to add the link should error (auto delete NS) return errors.Wrap(deleteNSIfNotNilErr, "failed to create vlan vnet link after making new ns") } + // Prevent accidentally deleting NS and vlan interface since we ignore this error + deleteNSIfNotNilErr = nil } defer func() { if deleteNSIfNotNilErr != nil { @@ -225,12 +277,11 @@ func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) er }() // sometimes there is slight delay in interface creation. check if it exists - for i := 0; i < numRetries; i++ { - if _, err = client.netioshim.GetNetworkInterfaceByName(client.vlanIfName); err == nil { - break - } - time.Sleep(time.Duration(sleepInMs) * time.Millisecond) - } + err = RunWithRetries(func() error { + _, err = client.netioshim.GetNetworkInterfaceByName(client.vlanIfName) + return errors.Wrap(err, "failed to get vlan veth") + }, numRetries, sleepInMs) + if err != nil { deleteNSIfNotNilErr = errors.Wrapf(err, "failed to get vlan veth interface:%s", client.vlanIfName) return deleteNSIfNotNilErr @@ -242,12 +293,13 @@ func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) er } // vlan veth was created successfully, so move the vlan veth you created logger.Info("Move vlan link to vnet NS", zap.String("vlanIfName", client.vlanIfName), zap.Any("vnetNSFileDescriptor", uintptr(client.vnetNSFileDescriptor))) - deleteNSIfNotNilErr = client.netlink.SetLinkNetNs(client.vlanIfName, uintptr(client.vnetNSFileDescriptor)) + deleteNSIfNotNilErr = client.setLinkNetNSAndConfirm(client.vlanIfName, uintptr(client.vnetNSFileDescriptor)) if deleteNSIfNotNilErr != nil { - return errors.Wrap(deleteNSIfNotNilErr, "deleting vlan veth in vm ns due to addendpoint failure") + return errors.Wrap(deleteNSIfNotNilErr, "failed to move or detect vlan veth inside vnet namespace") } + logger.Info("Moving vlan veth into namespace confirmed") } else { - logger.Info("Existing NS detected. Assuming exists too", zap.String("vnetNSName", client.vnetNSName), zap.String("vlanIfName", client.vlanIfName)) + logger.Info("Existing NS detected. vlan interface should exist or namespace would've been deleted.", zap.String("vnetNSName", client.vnetNSName), zap.String("vlanIfName", client.vlanIfName)) } // Get the default constant host veth mac @@ -260,6 +312,27 @@ func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) er if err = client.netUtilsClient.CreateEndpoint(client.vnetVethName, client.containerVethName, mac); err != nil { return errors.Wrap(err, "failed to create veth pair") } + + // Ensure vnet veth is created, as there may be a slight delay + err = RunWithRetries(func() error { + _, getErr := client.netioshim.GetNetworkInterfaceByName(client.vnetVethName) + return errors.Wrap(getErr, "failed to get vnet veth") + }, numRetries, sleepInMs) + if err != nil { + return errors.Wrap(err, "vnet veth does not exist") + } + + // Ensure container veth is created, as there may be a slight delay + var containerIf *net.Interface + err = RunWithRetries(func() error { + var getErr error + containerIf, getErr = client.netioshim.GetNetworkInterfaceByName(client.containerVethName) + return errors.Wrap(getErr, "failed to get container veth") + }, numRetries, sleepInMs) + if err != nil { + return errors.Wrap(err, "container veth does not exist") + } + // Disable RA for veth pair, and delete if any failure if err = client.netUtilsClient.DisableRAForInterface(client.vnetVethName); err != nil { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { @@ -274,16 +347,11 @@ func (client *TransparentVlanEndpointClient) PopulateVM(epInfo *EndpointInfo) er return errors.Wrap(err, "failed to disable RA on container veth, deleting") } - if err = client.netlink.SetLinkNetNs(client.vnetVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { + if err = client.setLinkNetNSAndConfirm(client.vnetVethName, uintptr(client.vnetNSFileDescriptor)); err != nil { if delErr := client.netlink.DeleteLink(client.vnetVethName); delErr != nil { logger.Error("Deleting vnet veth failed on addendpoint failure with", zap.Error(delErr)) } - return errors.Wrap(err, "failed to move vnetVethName into vnet ns, deleting") - } - - containerIf, err := client.netioshim.GetNetworkInterfaceByName(client.containerVethName) - if err != nil { - return errors.Wrap(err, "container veth does not exist") + return errors.Wrap(err, "failed to move or detect vnetVethName in vnet ns, deleting") } client.containerMac = containerIf.HardwareAddr return nil @@ -492,7 +560,7 @@ func (client *TransparentVlanEndpointClient) GetVnetRoutes(ipAddresses []net.IPN // Helper that creates routing rules for the current NS which direct packets // to the virtual gateway ip on linkToName device interface -// Route 1: 169.254.1.1 dev +// Route 1: 169.254.2.1 dev // Route 2: default via 169.254.2.1 dev func (client *TransparentVlanEndpointClient) addDefaultRoutes(linkToName string, table int) error { // Add route for virtualgwip (ip route add 169.254.2.1/32 dev eth0) @@ -637,3 +705,16 @@ func ExecuteInNS(nsc NamespaceClientInterface, nsName string, f func() error) er }() return f() } + +func RunWithRetries(f func() error, maxRuns, sleepMs int) error { + var err error + for i := 0; i < maxRuns; i++ { + err = f() + if err == nil { + break + } + logger.Info("Retrying after delay...", zap.String("error", err.Error()), zap.Int("retry", i), zap.Int("sleepMs", sleepMs)) + time.Sleep(time.Duration(sleepMs) * time.Millisecond) + } + return err +} diff --git a/network/transparent_vlan_endpointclient_linux_test.go b/network/transparent_vlan_endpointclient_linux_test.go index 9600236324..be64142bc5 100644 --- a/network/transparent_vlan_endpointclient_linux_test.go +++ b/network/transparent_vlan_endpointclient_linux_test.go @@ -16,6 +16,8 @@ import ( ) var errNetnsMock = errors.New("mock netns error") +var errMockNetIOFail = errors.New("netio fail") +var errMockNetIONoIfFail = &net.OpError{Op: "route", Net: "ip+net", Source: nil, Addr: nil, Err: errors.New("no such network interface")} func newNetnsErrorMock(errStr string) error { return errors.Wrap(errNetnsMock, errStr) @@ -77,6 +79,42 @@ func defaultDeleteNamed(name string) error { return nil } +// This mock netioshim provides more flexbility in when it errors compared to the one in the netio package +type mockNetIO struct { + existingInterfaces map[string]bool + err error +} + +func (ns *mockNetIO) GetNetworkInterfaceByName(name string) (*net.Interface, error) { + if ns.existingInterfaces[name] { + hwAddr, _ := net.ParseMAC("ab:cd:ef:12:34:56") + return &net.Interface{ + //nolint:gomnd // Dummy MTU + MTU: 1000, + Name: name, + HardwareAddr: hwAddr, + //nolint:gomnd // Dummy interface index + Index: 2, + }, nil + } + return nil, errors.Wrap(ns.err, name) +} + +func (ns *mockNetIO) GetNetworkInterfaceAddrs(_ *net.Interface) ([]net.Addr, error) { + return []net.Addr{}, nil +} + +func (ns *mockNetIO) GetNetworkInterfaceByMac(mac net.HardwareAddr) (*net.Interface, error) { + return &net.Interface{ + //nolint:gomnd // Dummy MTU + MTU: 1000, + Name: "eth1", + HardwareAddr: mac, + //nolint:gomnd // Dummy interface index + Index: 2, + }, nil +} + func TestTransparentVlanAddEndpoints(t *testing.T) { nl := netlink.NewMockNetlink(false, "") plc := platform.NewMockExecClient(false) @@ -87,6 +125,180 @@ func TestTransparentVlanAddEndpoints(t *testing.T) { epInfo *EndpointInfo wantErr bool wantErrMsg string + }{ + // Set the link network namespace and confirm that it was moved inside + { + name: "Set link netns good path", + client: &TransparentVlanEndpointClient{ + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + nsClient: NewMockNamespaceClient(), + }, + epInfo: &EndpointInfo{}, + wantErr: false, + }, + { + name: "Set link netns fail to set", + client: &TransparentVlanEndpointClient{ + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(true, "netlink fail"), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + nsClient: NewMockNamespaceClient(), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "failed to set eth0.1", + }, + { + name: "Set link netns fail to detect", + client: &TransparentVlanEndpointClient{ + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: &mockNetIO{ + existingInterfaces: map[string]bool{}, + err: errMockNetIOFail, + }, + nsClient: NewMockNamespaceClient(), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "failed to detect eth0.1", + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.setLinkNetNSAndConfirm(tt.client.vlanIfName, 1) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } + + tests = []struct { + name string + client *TransparentVlanEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string + }{ + // Ensuring vnet namespace and vlan both exist or are both absent before populating the vm + { + name: "Ensure clean populate VM neither vnet ns nor vlan if exists", + client: &TransparentVlanEndpointClient{ + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: func(name string) (fileDescriptor int, err error) { + return 0, newNetnsErrorMock("netns failure") + }, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + nsClient: NewMockNamespaceClient(), + }, + epInfo: &EndpointInfo{}, + wantErr: false, + }, + { + name: "Ensure clean populate VM vnet ns exists vlan does not exist", + client: &TransparentVlanEndpointClient{ + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: defaultGetFromName, + deleteNamed: func(name string) (err error) { + return newNetnsErrorMock("netns failure") + }, + }, + netlink: netlink.NewMockNetlink(false, ""), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: &mockNetIO{ + existingInterfaces: map[string]bool{}, + err: errMockNetIONoIfFail, + }, + nsClient: NewMockNamespaceClient(), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "failed to cleanup/delete ns after noticing vlan veth does not exist: netns failure: " + errNetnsMock.Error(), + }, + { + name: "Ensure clean populate VM cleanup straggling vlan if in vm ns", + client: &TransparentVlanEndpointClient{ + primaryHostIfName: "eth0", + vlanIfName: "eth0.1", + vnetVethName: "A1veth0", + containerVethName: "B1veth0", + vnetNSName: "az_ns_1", + netnsClient: &mockNetns{ + get: defaultGet, + getFromName: func(name string) (fileDescriptor int, err error) { + return 0, newNetnsErrorMock("netns failure") + }, + }, + netlink: netlink.NewMockNetlink(true, "netlink fail"), + plClient: platform.NewMockExecClient(false), + netUtilsClient: networkutils.NewNetworkUtils(nl, plc), + netioshim: netio.NewMockNetIO(false, 0), + nsClient: NewMockNamespaceClient(), + }, + epInfo: &EndpointInfo{}, + wantErr: true, + wantErrMsg: "failed to clean up vlan interface", + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := tt.client.ensureCleanPopulateVM() + if tt.wantErr { + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErrMsg, "Expected:%v actual:%v", tt.wantErrMsg, err.Error()) + } else { + require.NoError(t, err) + } + }) + } + tests = []struct { + name string + client *TransparentVlanEndpointClient + epInfo *EndpointInfo + wantErr bool + wantErrMsg string }{ // Populating VM with data and creating interfaces/links { @@ -134,6 +346,7 @@ func TestTransparentVlanAddEndpoints(t *testing.T) { plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), netioshim: netio.NewMockNetIO(false, 0), + nsClient: NewMockNamespaceClient(), }, epInfo: &EndpointInfo{}, wantErr: false, @@ -160,7 +373,7 @@ func TestTransparentVlanAddEndpoints(t *testing.T) { }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "failed to move vnetVethName into vnet ns, deleting: " + netlink.ErrorMockNetlink.Error() + " : netlink fail", + wantErrMsg: "failed to move or detect vnetVethName in vnet ns, deleting: failed to set A1veth0 inside namespace 1: " + netlink.ErrorMockNetlink.Error() + " : netlink fail", }, { name: "Add endpoints get interface fail for primary interface (eth0)", @@ -206,11 +419,17 @@ func TestTransparentVlanAddEndpoints(t *testing.T) { netlink: netlink.NewMockNetlink(false, ""), plClient: platform.NewMockExecClient(false), netUtilsClient: networkutils.NewNetworkUtils(nl, plc), - netioshim: netio.NewMockNetIO(true, 1), + netioshim: &mockNetIO{ + existingInterfaces: map[string]bool{ + "A1veth0": true, + }, + err: errMockNetIOFail, + }, + nsClient: NewMockNamespaceClient(), }, epInfo: &EndpointInfo{}, wantErr: true, - wantErrMsg: "container veth does not exist: " + netio.ErrMockNetIOFail.Error() + ":B1veth0", + wantErrMsg: "container veth does not exist: failed to get container veth: B1veth0: " + errMockNetIOFail.Error() + "", }, { name: "Add endpoints NetNS Get fail", @@ -235,7 +454,7 @@ func TestTransparentVlanAddEndpoints(t *testing.T) { wantErrMsg: "failed to get vm ns handle: netns failure: " + errNetnsMock.Error(), }, { - name: "Add endpoints NetNS Set fail", + name: "Add endpoints no vnet ns NetNS Set fail", client: &TransparentVlanEndpointClient{ primaryHostIfName: "eth0", vlanIfName: "eth0.1", @@ -689,3 +908,58 @@ func TestTransparentVlanConfigureContainerInterfacesAndRoutes(t *testing.T) { }) } } + +func createFunctionWithFailurePattern(errorPattern []error) func() error { + s := 0 + return func() error { + if s >= len(errorPattern) { + return nil + } + result := errorPattern[s] + s++ + return result + } +} + +func TestRunWithRetries(t *testing.T) { + errMock := errors.New("mock error") + runs := 4 + + tests := []struct { + name string + wantErr bool + f func() error + }{ + { + name: "Succeed on first try", + f: createFunctionWithFailurePattern([]error{}), + wantErr: false, + }, + { + name: "Succeed on first try do not check again", + f: createFunctionWithFailurePattern([]error{nil, errMock, errMock, errMock}), + wantErr: false, + }, + { + name: "Succeed on last try", + f: createFunctionWithFailurePattern([]error{errMock, errMock, errMock, nil, errMock}), + wantErr: false, + }, + { + name: "Fail after too many attempts", + f: createFunctionWithFailurePattern([]error{errMock, errMock, errMock, errMock, nil, nil}), + wantErr: true, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := RunWithRetries(tt.f, runs, 100) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +}