L1VH singluarity CNI

cni/azure-windows-swiftv2.conflist

@@ -0,0 +1,42 @@
+{
+    "cniVersion": "0.3.0",
+    "name": "azure",
+    "adapterName" : "",
+    "plugins": [
+        {
+            "type": "azure-vnet",
+            "executionMode": "v4swift",
+            "mode": "transparent",
+            "capabilities": {
+                "portMappings": true,
+                "dns": true
+            },
+            "ipam": {
+                "type": "azure-cns"
+            },
+            "dns": {
+                "Nameservers": [
+                    "168.63.129.16"
+                ],
+                "Search": [
+                    "svc.cluster.local"
+                ]
+            },
+            "AdditionalArgs": [
+                {
+                    "Name": "EndpointPolicy",
+                    "Value": {
+                        "Type": "OutBoundNAT",
+                        "ExceptionList": [
+                            "10.240.0.0/16",
+                            "10.0.0.0/8"
+                        ]
+                    }
+                }
+            ],
+             "windowsSettings": {
+                 "hnsTimeoutDurationInSeconds" : 120
+             }
+        }
+    ]
+}

cni/network/invoker_cns.go

@@ -450,6 +450,13 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p
 		return errors.Wrap(err, "Invalid mac address")
 	}
 
+	_, hostIPNet, err := net.ParseCIDR(info.hostSubnet)
+	if err != nil {
+		return fmt.Errorf("unable to parse hostSubnet: %w", err)
+	}
+
+	addResult.hostSubnetPrefix = *hostIPNet
+
 	routes, err := getRoutes(info.routes, info.skipDefaultRoutes)
 	if err != nil {
 		return err
@@ -462,6 +469,7 @@ func configureSecondaryAddResult(info *IPResultInfo, addResult *IPAMAddResult, p
 					IP:   ip,
 					Mask: ipnet.Mask,
 				},
+				Gateway: net.ParseIP(info.ncGatewayIPAddress),
 			},
 		},
 		Routes:            routes,

cni/network/network.go

@@ -76,6 +76,7 @@ type NetPlugin struct {
 	tb                 *telemetry.TelemetryBuffer
 	nnsClient          NnsClient
 	multitenancyClient MultitenancyClient
+	interfaceUsage     map[string]string // save master interfaces that are being used; key is ipnet, value is ifName
 }
 
 type PolicyArgs struct {
@@ -313,6 +314,15 @@ func addNatIPV6SubnetInfo(nwCfg *cni.NetworkConfig,
 	}
 }
 
+func hasSecondaryInterface(ipamAddResult IPAMAddResult, nicType cns.NICType) bool {
+	if ipamAddResult.secondaryInterfacesInfo != nil {
+		if ipamAddResult.secondaryInterfacesInfo[0].NICType == nicType {
+			return true
+		}
+	}
+	return false
+}
+
 // CNI implementation
 // https://github.com/containernetworking/cni/blob/master/SPEC.md
 
@@ -362,12 +372,17 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error {
 		telemetry.SendCNIMetric(&cniMetric, plugin.tb)
 
 		// Add Interfaces to result.
-		defaultCniResult := convertInterfaceInfoToCniResult(ipamAddResult.defaultInterfaceInfo, args.IfName)
+		cniResult := cniTypesCurr.Result{}
+		if hasSecondaryInterface(ipamAddResult, cns.DelegatedVMNIC) {
+			cniResult = *convertInterfaceInfoToCniResult(ipamAddResult.secondaryInterfacesInfo[0], args.IfName)
+		} else {
+			cniResult = *convertInterfaceInfoToCniResult(ipamAddResult.defaultInterfaceInfo, args.IfName)
+		}
 
-		addSnatInterface(nwCfg, defaultCniResult)
+		addSnatInterface(nwCfg, &cniResult)
 
 		// Convert result to the requested CNI version.
-		res, vererr := defaultCniResult.GetAsVersion(nwCfg.CNIVersion)
+		res, vererr := cniResult.GetAsVersion(nwCfg.CNIVersion)
 		if vererr != nil {
 			logger.Error("GetAsVersion failed", zap.Error(vererr))
 			plugin.Error(vererr)
@@ -380,7 +395,8 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error {
 
 		logger.Info("ADD command completed for",
 			zap.String("pod", k8sPodName),
-			zap.Any("IPs", defaultCniResult.IPs),
+			zap.Any("IPs", cniResult.IPs),
+			zap.Any("Interfaces", cniResult.Interfaces),
 			zap.Error(err))
 	}()
 
@@ -434,6 +450,9 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error {
 		return fmt.Errorf("failed to create cns client with error: %w", err)
 	}
 
+	options := make(map[string]any)
+	ipamAddConfig := IPAMAddConfig{nwCfg: nwCfg, args: args, options: options}
+
 	if nwCfg.MultiTenancy {
 		plugin.report.Context = "AzureCNIMultitenancy"
 		plugin.multitenancyClient.Init(cnsClient, AzureNetIOShim{})
@@ -460,6 +479,13 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error {
 				zap.Any("results", ipamAddResult))
 			return plugin.Errorf(errMsg)
 		}
+	} else if !nwCfg.MultiTenancy && nwCfg.IPAM.Type == network.AzureCNS {
+		plugin.ipamInvoker = NewCNSInvoker(k8sPodName, k8sNamespace, cnsClient, util.ExecutionMode(nwCfg.ExecutionMode), util.IpamMode(nwCfg.IPAM.Mode))
+		ipamAddResult, err = plugin.ipamInvoker.Add(ipamAddConfig)
+		if err != nil {
+			return fmt.Errorf("IPAM Invoker Add failed with error: %w", err)
+		}
+		sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.defaultInterfaceInfo, ipamAddResult.secondaryInterfacesInfo))
 	} else {
 		// TODO: refactor this code for simplification
 		// Add dummy ipamAddResult nil object for single tenancy mode
@@ -472,7 +498,6 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error {
 		var networkID string
 		ipamAddResult = ipamAddResults[i]
 
-		options := make(map[string]any)
 		networkID, err = plugin.getNetworkName(args.Netns, &ipamAddResult, nwCfg)
 
 		endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName)
@@ -507,22 +532,14 @@ func (plugin *NetPlugin) Add(args *cniSkel.CmdArgs) error {
 
 		// Initialize azureipam/cns ipam
 		if plugin.ipamInvoker == nil {
-			switch nwCfg.IPAM.Type {
-			case network.AzureCNS:
-				plugin.ipamInvoker = NewCNSInvoker(k8sPodName, k8sNamespace, cnsClient, util.ExecutionMode(nwCfg.ExecutionMode), util.IpamMode(nwCfg.IPAM.Mode))
-
-			default:
-				plugin.ipamInvoker = NewAzureIpamInvoker(plugin, &nwInfo)
-			}
-		}
-
-		ipamAddConfig := IPAMAddConfig{nwCfg: nwCfg, args: args, options: options}
-		if !nwCfg.MultiTenancy {
-			ipamAddResult, err = plugin.ipamInvoker.Add(ipamAddConfig)
-			if err != nil {
-				return fmt.Errorf("IPAM Invoker Add failed with error: %w", err)
+			plugin.ipamInvoker = NewAzureIpamInvoker(plugin, &nwInfo)
+			if !nwCfg.MultiTenancy {
+				ipamAddResult, err = plugin.ipamInvoker.Add(ipamAddConfig)
+				if err != nil {
+					return fmt.Errorf("IPAM Invoker Add failed with error: %w", err)
+				}
+				sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.defaultInterfaceInfo, ipamAddResult.secondaryInterfacesInfo))
 			}
-			sendEvent(plugin, fmt.Sprintf("Allocated IPAddress from ipam DefaultInterface: %+v, SecondaryInterfaces: %+v", ipamAddResult.defaultInterfaceInfo, ipamAddResult.secondaryInterfacesInfo))
 		}
 
 		defer func() { //nolint:gocritic
@@ -605,15 +622,40 @@ func (plugin *NetPlugin) createNetworkInternal(
 	ipamAddResult IPAMAddResult,
 ) (network.NetworkInfo, error) {
 	nwInfo := network.NetworkInfo{}
+
 	ipamAddResult.hostSubnetPrefix.IP = ipamAddResult.hostSubnetPrefix.IP.Mask(ipamAddResult.hostSubnetPrefix.Mask)
 	ipamAddConfig.nwCfg.IPAM.Subnet = ipamAddResult.hostSubnetPrefix.String()
+
+	ipnet := net.IPNet{}
+	interfaceInfo := network.InterfaceInfo{}
+	if hasSecondaryInterface(ipamAddResult, cns.DelegatedVMNIC) {
+		interfaceInfo = ipamAddResult.secondaryInterfacesInfo[0]
+		_, net, err := net.ParseCIDR(ipamAddResult.secondaryInterfacesInfo[0].IPConfigs[0].Address.String())
+		if err != nil {
+			logger.Error("Failed to parse secondary interface ipnet", zap.Error(err))
+		}
+		ipnet = *net
+	} else {
+		interfaceInfo = ipamAddResult.defaultInterfaceInfo
+		ipnet = ipamAddResult.hostSubnetPrefix
+	}
+
 	// Find the master interface.
-	masterIfName := plugin.findMasterInterface(ipamAddConfig.nwCfg, &ipamAddResult.hostSubnetPrefix)
+	masterIfName := plugin.findMasterInterface(ipamAddConfig.nwCfg, &ipnet)
 	if masterIfName == "" {
-		err := plugin.Errorf("Failed to find the master interface")
-		return nwInfo, err
+		// check if masterInterface is in masterInterfaces map
+		// if not, then no master interface is found
+		if ifName, ok := plugin.interfaceUsage[ipnet.String()]; ok {
+			masterIfName = ifName
+
+		} else {
+			err := plugin.Errorf("Failed to find the master interface")
+			return nwInfo, err
+		}
 	}
+
 	logger.Info("Found master interface", zap.String("ifname", masterIfName))
+	plugin.interfaceUsage[ipnet.String()] = masterIfName
 
 	// Add the master as an external interface.
 	err := plugin.nm.AddExternalInterface(masterIfName, ipamAddResult.hostSubnetPrefix.String())
@@ -622,7 +664,7 @@ func (plugin *NetPlugin) createNetworkInternal(
 		return nwInfo, err
 	}
 
-	nwDNSInfo, err := getNetworkDNSSettings(ipamAddConfig.nwCfg, ipamAddResult.defaultInterfaceInfo.DNS)
+	nwDNSInfo, err := getNetworkDNSSettings(ipamAddConfig.nwCfg, interfaceInfo.DNS)
 	if err != nil {
 		err = plugin.Errorf("Failed to getDNSSettings: %v", err)
 		return nwInfo, err
@@ -666,7 +708,14 @@ func (plugin *NetPlugin) createNetworkInternal(
 
 // construct network info with ipv4/ipv6 subnets
 func addSubnetToNetworkInfo(ipamAddResult IPAMAddResult, nwInfo *network.NetworkInfo) error {
-	for _, ipConfig := range ipamAddResult.defaultInterfaceInfo.IPConfigs {
+	interfaceInfo := network.InterfaceInfo{}
+	if hasSecondaryInterface(ipamAddResult, cns.DelegatedVMNIC) {
+		interfaceInfo = ipamAddResult.secondaryInterfacesInfo[0]
+	} else {
+		interfaceInfo = ipamAddResult.defaultInterfaceInfo
+	}
+
+	for _, ipConfig := range interfaceInfo.IPConfigs {
 		ip, podSubnetPrefix, err := net.ParseCIDR(ipConfig.Address.String())
 		if err != nil {
 			return fmt.Errorf("Failed to ParseCIDR for pod subnet prefix: %w", err)
@@ -1081,6 +1130,15 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error {
 		sendEvent(plugin, fmt.Sprintf("Deleting endpoint:%v", endpointID))
 		// Delete the endpoint.
 		if err = plugin.nm.DeleteEndpoint(networkID, endpointID, epInfo); err != nil {
+			// cleanup interfaces usage map
+			delete(plugin.interfaceUsage, nwInfo.MasterIfName)
+			// delete hnsNetwork in delegatedVMNIC scenario
+			if epInfo.NICType == cns.DelegatedVMNIC {
+				err = plugin.nm.DeleteNetwork(networkID)
+				if err != nil {
+					logger.Error("Failed to delete hnsNetwork", zap.Error(err))
+				}
+			}
 			// return a retriable error so the container runtime will retry this DEL later
 			// the implementation of this function returns nil if the endpoint doens't exist, so
 			// we don't have to check that here

cni/network/network_windows.go

@@ -144,8 +144,15 @@ func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) {
 
 func (plugin *NetPlugin) getNetworkName(netNs string, ipamAddResult *IPAMAddResult, nwCfg *cni.NetworkConfig) (string, error) {
 	determineWinVer()
+
+	// check if it's swiftv2 mode
+	hasDelegatedNic := false
+	if hasSecondaryInterface(ipamAddResult, cns.DelegatedVMNIC) {
+		hasDelegatedNic = true
+	}
+
 	// For singletenancy, the network name is simply the nwCfg.Name
-	if !nwCfg.MultiTenancy {
+	if !nwCfg.MultiTenancy && !hasDelegatedNic {
 		return nwCfg.Name, nil
 	}
 
@@ -154,6 +161,14 @@ func (plugin *NetPlugin) getNetworkName(netNs string, ipamAddResult *IPAMAddResu
 		return "", fmt.Errorf("NetNs cannot be empty")
 	}
 
+	// if it's swiftv2 delegatedVMNIC, then use "azure-macAddres" format networkName
+	// networkName will look like ~ azure-01:23:ab:f4:ac:95
+	if ipamAddResult != nil and hasDelegatedNic {
+		swiftv2NetworkName := "azure-" + ipamAddResult.secondaryInterfacesInfo[0].MacAddress.String()
+		logger.Info("swiftv2 network name is", zap.String("swiftv2NetworkName", swiftv2NetworkName))
+		return swiftv2NetworkName, nil
+	}
+
 	// First try to build the network name from the cnsResponse if present
 	// This will happen during ADD call
 	if ipamAddResult != nil && ipamAddResult.ncResponse != nil {

network/endpoint_windows.go

@@ -9,6 +9,7 @@ import (
 	"net"
 	"strings"
 
+	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/netio"
 	"github.com/Azure/azure-container-networking/netlink"
 	"github.com/Azure/azure-container-networking/network/policy"
@@ -74,16 +75,23 @@ func (nw *network) newEndpointImpl(
 	_ ipTablesClient,
 	epInfo []*EndpointInfo,
 ) (*endpoint, error) {
-	// there is only 1 epInfo for windows, multiple interfaces will be added in the future
-	if useHnsV2, err := UseHnsV2(epInfo[0].NetNsPath); useHnsV2 {
+
+	endpointInfo := epInfo[0]
+	for _, ep := range epInfo {
+		if ep.NICType == cns.DelegatedVMNIC {
+			endpointInfo = epInfo[1]
+		}
+	}
+
+	if useHnsV2, err := UseHnsV2(endpointInfo.NetNsPath); useHnsV2 {
 		if err != nil {
 			return nil, err
 		}
 
-		return nw.newEndpointImplHnsV2(cli, epInfo[0])
+		return nw.newEndpointImplHnsV2(cli, endpointInfo)
 	}
 
-	return nw.newEndpointImplHnsV1(epInfo[0], plc)
+	return nw.newEndpointImplHnsV1(endpointInfo, plc)
 }
 
 // newEndpointImplHnsV1 creates a new endpoint in the network using HnsV1
@@ -219,9 +227,14 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE
 			Major: hcnSchemaVersionMajor,
 			Minor: hcnSchemaVersionMinor,
 		},
-		MacAddress: epInfo.MacAddress.String(),
 	}
 
+	macAddress := epInfo.MacAddress.String()
+	if epInfo.NICType == cns.DelegatedVMNIC {
+		macAddress = strings.Join(strings.Split(macAddress, ":"), "-")
+	}
+	hcnEndpoint.MacAddress = macAddress
+
 	if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data, epInfo.EnableSnatForDns, epInfo.EnableMultiTenancy, epInfo.NATInfo); err == nil {
 		for _, epPolicy := range endpointPolicies {
 			hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicy)

network/network_windows.go

@@ -290,6 +290,9 @@ func (nm *networkManager) configureHcnNetwork(nwInfo *NetworkInfo, extIf *extern
 		hcnNetwork.Type = hcn.L2Bridge
 	case opModeTunnel:
 		hcnNetwork.Type = hcn.L2Tunnel
+	case opModeTransparent:
+		hcnNetwork.Type = hcn.Transparent
+		hcnNetwork.Flags = hcn.DisableHostPort
 	default:
 		return nil, errNetworkModeInvalid
 	}