feature: add sfswiftv2 middleware support for standalone service fabr…

…ic swiftv2 windows path

cns/NetworkContainerContract.go

@@ -465,6 +465,7 @@ type PodIpInfo struct {
 	PodIPConfig                     IPSubnet
 	NetworkContainerPrimaryIPConfig IPConfiguration
 	HostPrimaryIPInfo               HostIPInfo
+	HostSecondaryIPInfo             HostIPInfo
 	// NICType defines whether NIC is InfraNIC or DelegatedVMNIC or BackendNIC
 	NICType       NICType
 	InterfaceName string
@@ -477,9 +478,10 @@ type PodIpInfo struct {
 }
 
 type HostIPInfo struct {
-	Gateway   string
-	PrimaryIP string
-	Subnet    string
+	Gateway     string
+	PrimaryIP   string
+	SecondaryIP string
+	Subnet      string
 }
 
 type IPConfigRequest struct {

cns/api.go

@@ -9,6 +9,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/Azure/azure-container-networking/cns/configuration"
+
 	"github.com/Azure/azure-container-networking/cns/common"
 	"github.com/Azure/azure-container-networking/cns/types"
 	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
@@ -59,6 +61,7 @@ type IPConfigsHandlerFunc func(context.Context, IPConfigsRequest) (*IPConfigsRes
 // IPConfigsHandlerMiddleware
 type IPConfigsHandlerMiddleware interface {
 	IPConfigsRequestHandlerWrapper(defaultHandler IPConfigsHandlerFunc, failureHandler IPConfigsHandlerFunc) IPConfigsHandlerFunc
+	GetMiddlewareType() configuration.SWIFTV2Mode
 }
 
 // This is used for KubernetesCRD orchestrator Type where NC has multiple ips.

cns/configuration/configuration.go

@@ -37,7 +37,6 @@ type CNSConfig struct {
 	EnablePprof                 bool
 	EnableStateMigration        bool
 	EnableSubnetScarcity        bool
-	EnableSwiftV2               bool
 	InitializeFromCNI           bool
 	KeyVaultSettings            KeyVaultSettings
 	MSISettings                 MSISettings
@@ -220,5 +219,5 @@ func SetCNSConfigDefaults(config *CNSConfig) {
 	if config.AsyncPodDeletePath == "" {
 		config.AsyncPodDeletePath = "/var/run/azure-vnet/deleteIDs"
 	}
-	config.WatchPods = config.EnableIPAMv2 || config.EnableSwiftV2
+	config.WatchPods = config.EnableIPAMv2 || config.SWIFTV2Mode == K8sSWIFTV2
 }

cns/hnsclient/hnsclient_windows.go

@@ -27,8 +27,9 @@ const (
 	ExtHnsNetworkGwAddress = "192.168.255.1"
 
 	// HNS network types
-	hnsL2Bridge = "l2bridge"
-	hnsL2Tunnel = "l2tunnel"
+	hnsL2Bridge    = "l2bridge"
+	hnsL2Tunnel    = "l2tunnel"
+	hnsTransparent = "transparent"
 
 	// hcnSchemaVersionMajor indicates major version number for hcn schema
 	hcnSchemaVersionMajor = 2
@@ -137,7 +138,7 @@ func CreateDefaultExtNetwork(networkType string) error {
 		return nil
 	}
 
-	if networkType != hnsL2Bridge && networkType != hnsL2Tunnel {
+	if networkType != hnsL2Bridge && networkType != hnsL2Tunnel && networkType != hnsTransparent {
 		return fmt.Errorf("Invalid hns network type %s", networkType)
 	}
 

cns/middlewares/SFSwiftV2.go

@@ -0,0 +1,33 @@
+package middlewares
+
+import (
+	"context"
+
+	"github.com/Azure/azure-container-networking/cns/configuration"
+
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/types"
+	"github.com/pkg/errors"
+)
+
+type SFSWIFTv2Middleware struct{}
+
+// IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP config requests for SF standalone scenario. This function wraps the default SWIFT request
+// and release IP configs handlers.
+func (m *SFSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(ipRequestHandler, _ cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc {
+	return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) {
+		ipConfigsResp, err := ipRequestHandler(ctx, req)
+		if err != nil {
+			ipConfigsResp.Response.ReturnCode = types.UnexpectedError
+			return ipConfigsResp, errors.Wrapf(err, "Failed to requestIPConfigs for SF from IPConfigsRequest %v", req)
+		}
+
+		// SwiftV2-SF will always request for secondaryInterfaces for a pod
+		req.SecondaryInterfacesExist = true
+		return ipConfigsResp, nil
+	}
+}
+
+func (m *SFSWIFTv2Middleware) GetMiddlewareType() configuration.SWIFTV2Mode {
+	return configuration.SFSWIFTV2
+}

cns/middlewares/k8sSwiftV2.go

@@ -282,3 +282,7 @@ func (m *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
 	}
 	return nil
 }
+
+func (m *K8sSWIFTv2Middleware) GetMiddlewareType() configuration.SWIFTV2Mode {
+	return configuration.SFSWIFTV2
+}

cns/restserver/ipam.go

@@ -11,6 +11,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/Azure/azure-container-networking/cns/configuration"
+
 	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/cns/filter"
 	"github.com/Azure/azure-container-networking/cns/logger"
@@ -91,6 +93,103 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context
 	}, nil
 }
 
+// requestIPConfigHandlerHelperSF validates the request, assign IPs and return the IPConfigs
+func (service *HTTPRestService) requestIPConfigHandlerHelperSF(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) {
+	// For SWIFT v2 scenario, the validator function will also modify the ipconfigsRequest.
+	podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ctx, ipconfigsRequest)
+	if returnCode != types.Success {
+		return &cns.IPConfigsResponse{
+			Response: cns.Response{
+				ReturnCode: returnCode,
+				Message:    returnMessage,
+			},
+		}, errors.New("failed to validate ip config request")
+	}
+
+	// record a pod requesting an IP
+	service.podsPendingIPAssignment.Push(podInfo.Key())
+	// unmarshal & retrieve podInfo from OrchestratorContext
+	podInfo, err := cns.NewPodInfoFromIPConfigsRequest(ipconfigsRequest)
+	orchestratorContext, err := podInfo.OrchestratorContext()
+	if err != nil {
+		return &cns.IPConfigsResponse{}, fmt.Errorf("error getting orchestrator context from PodInfo %w", err)
+	}
+	cnsRequest := cns.GetNetworkContainerRequest{OrchestratorContext: orchestratorContext}
+	resp := service.getAllNetworkContainerResponses(cnsRequest)
+	// return err if nil - error should be failed due to no nc response above
+	if resp == nil {
+		return &cns.IPConfigsResponse{
+			Response: cns.Response{
+				ReturnCode: types.FailedToAllocateIPConfig,
+				Message:    fmt.Sprintf("AllocateIPConfig failed due to not getting NC Response: %v, IP config request is %v", err, ipconfigsRequest),
+			},
+		}, err
+	}
+	podIPInfo := cns.PodIpInfo{
+		PodIPConfig:                     resp[0].IPConfiguration.IPSubnet,
+		MacAddress:                      resp[0].NetworkInterfaceInfo.MACAddress,
+		NICType:                         resp[0].NetworkInterfaceInfo.NICType,
+		SkipDefaultRoutes:               false,
+		NetworkContainerPrimaryIPConfig: resp[0].IPConfiguration,
+	}
+	ipConfigsResp := &cns.IPConfigsResponse{
+		Response: cns.Response{
+			ReturnCode: types.Success,
+		},
+		PodIPInfo: []cns.PodIpInfo{},
+	}
+
+	ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, podIPInfo)
+	ipConfigsResp, err = service.updatePodInfoWithInterfaces(ctx, ipConfigsResp)
+	if err != nil {
+		return &cns.IPConfigsResponse{
+			Response: cns.Response{
+				ReturnCode: types.FailedToAllocateIPConfig,
+				Message:    fmt.Sprintf("AllocateIPConfig failed while updating pod with interfaces: %v, IP config request is %v", err, ipconfigsRequest),
+			},
+		}, err
+	}
+	return &cns.IPConfigsResponse{
+		Response: cns.Response{
+			ReturnCode: types.Success,
+		},
+		PodIPInfo: ipConfigsResp.PodIPInfo,
+	}, nil
+}
+
+func (service *HTTPRestService) updatePodInfoWithInterfaces(ctx context.Context, ipconfigResponse *cns.IPConfigsResponse) (*cns.IPConfigsResponse, error) {
+	podIPInfoList := make([]cns.PodIpInfo, 0, len(ipconfigResponse.PodIPInfo))
+	for i := range ipconfigResponse.PodIPInfo {
+		// populating podIpInfo with primary & secondary interface info & updating IpConfigsResponse
+		hostPrimaryInterface, err := service.getPrimaryHostInterface(ctx)
+		if err != nil {
+			return &cns.IPConfigsResponse{}, err
+		}
+
+		hostSecondaryInterface, err := service.getSecondaryHostInterface(ctx, ipconfigResponse.PodIPInfo[i].MacAddress)
+		if err != nil {
+			return &cns.IPConfigsResponse{}, err
+		}
+
+		ipconfigResponse.PodIPInfo[i].HostPrimaryIPInfo = cns.HostIPInfo{
+			Gateway:   hostPrimaryInterface.Gateway,
+			PrimaryIP: hostPrimaryInterface.PrimaryIP,
+			Subnet:    hostPrimaryInterface.Subnet,
+		}
+
+		ipconfigResponse.PodIPInfo[i].HostSecondaryIPInfo = cns.HostIPInfo{
+			Gateway:     hostSecondaryInterface.Gateway,
+			SecondaryIP: hostSecondaryInterface.SecondaryIPs[0],
+			Subnet:      hostSecondaryInterface.Subnet,
+		}
+
+		podIPInfoList = append(podIPInfoList, ipconfigResponse.PodIPInfo[i])
+
+	}
+	ipconfigResponse.PodIPInfo = podIPInfoList
+	return ipconfigResponse, nil
+}
+
 // requestIPConfigHandler requests an IPConfig from the CNS state
 func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r *http.Request) {
 	var ipconfigRequest cns.IPConfigRequest
@@ -179,8 +278,13 @@ func (service *HTTPRestService) requestIPConfigsHandler(w http.ResponseWriter, r
 
 	// Check if IPConfigsHandlerMiddleware is set
 	if service.IPConfigsHandlerMiddleware != nil {
-		// Wrap the default datapath handlers with the middleware
-		wrappedHandler := service.IPConfigsHandlerMiddleware.IPConfigsRequestHandlerWrapper(service.requestIPConfigHandlerHelper, service.releaseIPConfigHandlerHelper)
+		// Wrap the default datapath handlers with the middleware depending on middleware type
+		var wrappedHandler cns.IPConfigsHandlerFunc
+		if service.IPConfigsHandlerMiddleware.GetMiddlewareType() == configuration.K8sSWIFTV2 {
+			wrappedHandler = service.IPConfigsHandlerMiddleware.IPConfigsRequestHandlerWrapper(service.requestIPConfigHandlerHelper, service.releaseIPConfigHandlerHelper)
+		} else if service.IPConfigsHandlerMiddleware.GetMiddlewareType() == configuration.SFSWIFTV2 {
+			wrappedHandler = service.IPConfigsHandlerMiddleware.IPConfigsRequestHandlerWrapper(service.requestIPConfigHandlerHelperSF, service.releaseIPConfigHandlerHelper)
+		}
 		ipConfigsResp, err = wrappedHandler(r.Context(), ipconfigsRequest)
 	} else {
 		ipConfigsResp, err = service.requestIPConfigHandlerHelper(r.Context(), ipconfigsRequest) // nolint:contextcheck // appease linter

cns/restserver/restserver.go

@@ -147,6 +147,7 @@ type httpRestServiceState struct {
 	TimeStamp                        time.Time
 	joinedNetworks                   map[string]struct{}
 	primaryInterface                 *wireserver.InterfaceInfo
+	secondaryInterface               *wireserver.InterfaceInfo
 }
 
 type networkInfo struct {

cns/restserver/util.go

@@ -756,12 +756,12 @@ func (service *HTTPRestService) SendNCSnapShotPeriodically(ctx context.Context,
 }
 
 func (service *HTTPRestService) validateIPConfigsRequest(ctx context.Context, ipConfigsRequest cns.IPConfigsRequest) (cns.PodInfo, types.ResponseCode, string) {
-	if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes {
+	if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes && service.state.OrchestratorType != cns.ServiceFabric {
 		return nil, types.UnsupportedOrchestratorType, "ReleaseIPConfig API supported only for kubernetes orchestrator"
 	}
 
 	if ipConfigsRequest.OrchestratorContext == nil {
-		return nil, types.EmptyOrchestratorContext, fmt.Sprintf("OrchastratorContext is not set in the req: %+v", ipConfigsRequest)
+		return nil, types.EmptyOrchestratorContext, fmt.Sprintf("OrchestratorContext is not set in the req: %+v", ipConfigsRequest)
 	}
 
 	// retrieve podinfo from orchestrator context
@@ -790,6 +790,24 @@ func (service *HTTPRestService) getPrimaryHostInterface(ctx context.Context) (*w
 	return service.state.primaryInterface, nil
 }
 
+// getSecondaryHostInterface returns the cached InterfaceInfo, if available, otherwise
+// queries the IMDS to get the secondary interface info and caches it in the server-state before returning the result.
+func (service *HTTPRestService) getSecondaryHostInterface(ctx context.Context, macAddress string) (*wireserver.InterfaceInfo, error) {
+	if service.state.secondaryInterface == nil {
+		res, err := service.wscli.GetInterfaces(ctx)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get interfaces from wireserver client")
+		}
+		secondary, err := wireserver.GetSecondaryInterfaceFromResult(res, macAddress)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get secondary interface from wireserver client")
+		}
+
+		service.state.secondaryInterface = secondary
+	}
+	return service.state.secondaryInterface, nil
+}
+
 //nolint:gocritic // ignore hugeParam pls
 func (service *HTTPRestService) populateIPConfigInfoUntransacted(ipConfigStatus cns.IPConfigurationStatus, podIPInfo *cns.PodIpInfo) error {
 	ncStatus, exists := service.state.ContainerStatus[ipConfigStatus.NCID]

cns/service/main.go

@@ -801,6 +801,12 @@ func main() {
 		if platform.HasMellanoxAdapter() {
 			go platform.MonitorAndSetMellanoxRegKeyPriorityVLANTag(rootCtx, cnsconfig.MellanoxMonitorIntervalSecs)
 		}
+
+		// if swiftv2 scenario is enabled, we need to initialize the Service Fabric (standalone) swiftv2 middleware to process IP configs requests
+		if cnsconfig.SWIFTV2Mode == configuration.SFSWIFTV2 {
+			swiftV2Middleware := &middlewares.SFSWIFTv2Middleware{}
+			httpRestService.AttachIPConfigsHandlerMiddleware(swiftV2Middleware)
+		}
 	}
 
 	// Initialze state in if CNS is running in CRD mode
@@ -1220,7 +1226,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 
 	// check the Node labels for Swift V2
 	if _, ok := node.Labels[configuration.LabelNodeSwiftV2]; ok {
-		cnsconfig.EnableSwiftV2 = true
+		cnsconfig.SWIFTV2Mode = configuration.K8sSWIFTV2
 		cnsconfig.WatchPods = true
 		if nodeInfoErr := createOrUpdateNodeInfoCRD(ctx, kubeConfig, node); nodeInfoErr != nil {
 			return errors.Wrap(nodeInfoErr, "error creating or updating nodeinfo crd")
@@ -1404,22 +1410,13 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 		}
 	}
 
-	if cnsconfig.EnableSwiftV2 {
+	if cnsconfig.SWIFTV2Mode == configuration.K8sSWIFTV2 {
 		if err := mtpncctrl.SetupWithManager(manager); err != nil {
 			return errors.Wrapf(err, "failed to setup mtpnc reconciler with manager")
 		}
 		// if SWIFT v2 is enabled on CNS, attach multitenant middleware to rest service
-		// switch here for different type of swift v2 middleware (k8s or SF)
-		var swiftV2Middleware cns.IPConfigsHandlerMiddleware
-		switch cnsconfig.SWIFTV2Mode {
-		case configuration.K8sSWIFTV2:
-			swiftV2Middleware = &middlewares.K8sSWIFTv2Middleware{Cli: manager.GetClient()}
-		case configuration.SFSWIFTV2:
-		default:
-			// default to K8s middleware for now, in a later changes we where start to pass in
-			// SWIFT v2 mode in CNS config, this should throw an error if the mode is not set.
-			swiftV2Middleware = &middlewares.K8sSWIFTv2Middleware{Cli: manager.GetClient()}
-		}
+		// switch here for AKS(K8s) swiftv2 middleware to process IP configs requests
+		swiftV2Middleware := &middlewares.K8sSWIFTv2Middleware{Cli: manager.GetClient()}
 		httpRestService.AttachIPConfigsHandlerMiddleware(swiftV2Middleware)
 	}