From bb41ff342297c70774247c4dda48c843c3683cfb Mon Sep 17 00:00:00 2001 From: Kshitija Murudi Date: Wed, 6 Mar 2024 02:11:26 -0800 Subject: [PATCH] feature: add sfswiftv2 middleware support for standalone service fabric swiftv2 windows path --- cns/NetworkContainerContract.go | 8 ++- cns/api.go | 3 + cns/configuration/configuration.go | 3 +- cns/hnsclient/hnsclient_windows.go | 7 +- cns/middlewares/SFSwiftV2.go | 33 +++++++++ cns/middlewares/k8sSwiftV2.go | 4 ++ cns/restserver/ipam.go | 108 ++++++++++++++++++++++++++++- cns/restserver/restserver.go | 1 + cns/restserver/util.go | 22 +++++- cns/service/main.go | 23 +++--- cns/wireserver/net.go | 53 +++++++++++++- 11 files changed, 239 insertions(+), 26 deletions(-) create mode 100644 cns/middlewares/SFSwiftV2.go diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 11a4cbf25a..4aaf931ba2 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -465,6 +465,7 @@ type PodIpInfo struct { PodIPConfig IPSubnet NetworkContainerPrimaryIPConfig IPConfiguration HostPrimaryIPInfo HostIPInfo + HostSecondaryIPInfo HostIPInfo // NICType defines whether NIC is InfraNIC or DelegatedVMNIC or BackendNIC NICType NICType InterfaceName string @@ -477,9 +478,10 @@ type PodIpInfo struct { } type HostIPInfo struct { - Gateway string - PrimaryIP string - Subnet string + Gateway string + PrimaryIP string + SecondaryIP string + Subnet string } type IPConfigRequest struct { diff --git a/cns/api.go b/cns/api.go index 3894e9aff6..94f7fdeec9 100644 --- a/cns/api.go +++ b/cns/api.go @@ -9,6 +9,8 @@ import ( "fmt" "time" + "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" @@ -59,6 +61,7 @@ type IPConfigsHandlerFunc func(context.Context, IPConfigsRequest) (*IPConfigsRes // IPConfigsHandlerMiddleware type IPConfigsHandlerMiddleware interface { IPConfigsRequestHandlerWrapper(defaultHandler IPConfigsHandlerFunc, failureHandler IPConfigsHandlerFunc) IPConfigsHandlerFunc + GetMiddlewareType() configuration.SWIFTV2Mode } // This is used for KubernetesCRD orchestrator Type where NC has multiple ips. diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index 44da9c3e87..b31ae0c7f0 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -37,7 +37,6 @@ type CNSConfig struct { EnablePprof bool EnableStateMigration bool EnableSubnetScarcity bool - EnableSwiftV2 bool InitializeFromCNI bool KeyVaultSettings KeyVaultSettings MSISettings MSISettings @@ -220,5 +219,5 @@ func SetCNSConfigDefaults(config *CNSConfig) { if config.AsyncPodDeletePath == "" { config.AsyncPodDeletePath = "/var/run/azure-vnet/deleteIDs" } - config.WatchPods = config.EnableIPAMv2 || config.EnableSwiftV2 + config.WatchPods = config.EnableIPAMv2 || config.SWIFTV2Mode == K8sSWIFTV2 } diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index cc24b7917a..64c958dd36 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -27,8 +27,9 @@ const ( ExtHnsNetworkGwAddress = "192.168.255.1" // HNS network types - hnsL2Bridge = "l2bridge" - hnsL2Tunnel = "l2tunnel" + hnsL2Bridge = "l2bridge" + hnsL2Tunnel = "l2tunnel" + hnsTransparent = "transparent" // hcnSchemaVersionMajor indicates major version number for hcn schema hcnSchemaVersionMajor = 2 @@ -137,7 +138,7 @@ func CreateDefaultExtNetwork(networkType string) error { return nil } - if networkType != hnsL2Bridge && networkType != hnsL2Tunnel { + if networkType != hnsL2Bridge && networkType != hnsL2Tunnel && networkType != hnsTransparent { return fmt.Errorf("Invalid hns network type %s", networkType) } diff --git a/cns/middlewares/SFSwiftV2.go b/cns/middlewares/SFSwiftV2.go new file mode 100644 index 0000000000..ef313f61bb --- /dev/null +++ b/cns/middlewares/SFSwiftV2.go @@ -0,0 +1,33 @@ +package middlewares + +import ( + "context" + + "github.com/Azure/azure-container-networking/cns/configuration" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/types" + "github.com/pkg/errors" +) + +type SFSWIFTv2Middleware struct{} + +// IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP config requests for SF standalone scenario. This function wraps the default SWIFT request +// and release IP configs handlers. +func (m *SFSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(ipRequestHandler, _ cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc { + return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { + ipConfigsResp, err := ipRequestHandler(ctx, req) + if err != nil { + ipConfigsResp.Response.ReturnCode = types.UnexpectedError + return ipConfigsResp, errors.Wrapf(err, "Failed to requestIPConfigs for SF from IPConfigsRequest %v", req) + } + + // SwiftV2-SF will always request for secondaryInterfaces for a pod + req.SecondaryInterfacesExist = true + return ipConfigsResp, nil + } +} + +func (m *SFSWIFTv2Middleware) GetMiddlewareType() configuration.SWIFTV2Mode { + return configuration.SFSWIFTV2 +} diff --git a/cns/middlewares/k8sSwiftV2.go b/cns/middlewares/k8sSwiftV2.go index a9721c3995..5e6e1775be 100644 --- a/cns/middlewares/k8sSwiftV2.go +++ b/cns/middlewares/k8sSwiftV2.go @@ -282,3 +282,7 @@ func (m *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { } return nil } + +func (m *K8sSWIFTv2Middleware) GetMiddlewareType() configuration.SWIFTV2Mode { + return configuration.SFSWIFTV2 +} diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 0362765076..56bf324eea 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -11,6 +11,8 @@ import ( "strconv" "strings" + "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/filter" "github.com/Azure/azure-container-networking/cns/logger" @@ -91,6 +93,103 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context }, nil } +// requestIPConfigHandlerHelperSF validates the request, assign IPs and return the IPConfigs +func (service *HTTPRestService) requestIPConfigHandlerHelperSF(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { + // For SWIFT v2 scenario, the validator function will also modify the ipconfigsRequest. + podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ctx, ipconfigsRequest) + if returnCode != types.Success { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: returnCode, + Message: returnMessage, + }, + }, errors.New("failed to validate ip config request") + } + + // record a pod requesting an IP + service.podsPendingIPAssignment.Push(podInfo.Key()) + // unmarshal & retrieve podInfo from OrchestratorContext + podInfo, err := cns.NewPodInfoFromIPConfigsRequest(ipconfigsRequest) + orchestratorContext, err := podInfo.OrchestratorContext() + if err != nil { + return &cns.IPConfigsResponse{}, fmt.Errorf("error getting orchestrator context from PodInfo %w", err) + } + cnsRequest := cns.GetNetworkContainerRequest{OrchestratorContext: orchestratorContext} + resp := service.getAllNetworkContainerResponses(cnsRequest) + // return err if nil - error should be failed due to no nc response above + if resp == nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed due to not getting NC Response: %v, IP config request is %v", err, ipconfigsRequest), + }, + }, err + } + podIPInfo := cns.PodIpInfo{ + PodIPConfig: resp[0].IPConfiguration.IPSubnet, + MacAddress: resp[0].NetworkInterfaceInfo.MACAddress, + NICType: resp[0].NetworkInterfaceInfo.NICType, + SkipDefaultRoutes: false, + NetworkContainerPrimaryIPConfig: resp[0].IPConfiguration, + } + ipConfigsResp := &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.Success, + }, + PodIPInfo: []cns.PodIpInfo{}, + } + + ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, podIPInfo) + ipConfigsResp, err = service.updatePodInfoWithInterfaces(ctx, ipConfigsResp) + if err != nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed while updating pod with interfaces: %v, IP config request is %v", err, ipconfigsRequest), + }, + }, err + } + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.Success, + }, + PodIPInfo: ipConfigsResp.PodIPInfo, + }, nil +} + +func (service *HTTPRestService) updatePodInfoWithInterfaces(ctx context.Context, ipconfigResponse *cns.IPConfigsResponse) (*cns.IPConfigsResponse, error) { + podIPInfoList := make([]cns.PodIpInfo, 0, len(ipconfigResponse.PodIPInfo)) + for i := range ipconfigResponse.PodIPInfo { + // populating podIpInfo with primary & secondary interface info & updating IpConfigsResponse + hostPrimaryInterface, err := service.getPrimaryHostInterface(ctx) + if err != nil { + return &cns.IPConfigsResponse{}, err + } + + hostSecondaryInterface, err := service.getSecondaryHostInterface(ctx, ipconfigResponse.PodIPInfo[i].MacAddress) + if err != nil { + return &cns.IPConfigsResponse{}, err + } + + ipconfigResponse.PodIPInfo[i].HostPrimaryIPInfo = cns.HostIPInfo{ + Gateway: hostPrimaryInterface.Gateway, + PrimaryIP: hostPrimaryInterface.PrimaryIP, + Subnet: hostPrimaryInterface.Subnet, + } + + ipconfigResponse.PodIPInfo[i].HostSecondaryIPInfo = cns.HostIPInfo{ + Gateway: hostSecondaryInterface.Gateway, + SecondaryIP: hostSecondaryInterface.SecondaryIPs[0], + Subnet: hostSecondaryInterface.Subnet, + } + + podIPInfoList = append(podIPInfoList, ipconfigResponse.PodIPInfo[i]) + + } + ipconfigResponse.PodIPInfo = podIPInfoList + return ipconfigResponse, nil +} + // requestIPConfigHandler requests an IPConfig from the CNS state func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r *http.Request) { var ipconfigRequest cns.IPConfigRequest @@ -179,8 +278,13 @@ func (service *HTTPRestService) requestIPConfigsHandler(w http.ResponseWriter, r // Check if IPConfigsHandlerMiddleware is set if service.IPConfigsHandlerMiddleware != nil { - // Wrap the default datapath handlers with the middleware - wrappedHandler := service.IPConfigsHandlerMiddleware.IPConfigsRequestHandlerWrapper(service.requestIPConfigHandlerHelper, service.releaseIPConfigHandlerHelper) + // Wrap the default datapath handlers with the middleware depending on middleware type + var wrappedHandler cns.IPConfigsHandlerFunc + if service.IPConfigsHandlerMiddleware.GetMiddlewareType() == configuration.K8sSWIFTV2 { + wrappedHandler = service.IPConfigsHandlerMiddleware.IPConfigsRequestHandlerWrapper(service.requestIPConfigHandlerHelper, service.releaseIPConfigHandlerHelper) + } else if service.IPConfigsHandlerMiddleware.GetMiddlewareType() == configuration.SFSWIFTV2 { + wrappedHandler = service.IPConfigsHandlerMiddleware.IPConfigsRequestHandlerWrapper(service.requestIPConfigHandlerHelperSF, service.releaseIPConfigHandlerHelper) + } ipConfigsResp, err = wrappedHandler(r.Context(), ipconfigsRequest) } else { ipConfigsResp, err = service.requestIPConfigHandlerHelper(r.Context(), ipconfigsRequest) // nolint:contextcheck // appease linter diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index cc71d33e0c..5fbdc60488 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -147,6 +147,7 @@ type httpRestServiceState struct { TimeStamp time.Time joinedNetworks map[string]struct{} primaryInterface *wireserver.InterfaceInfo + secondaryInterface *wireserver.InterfaceInfo } type networkInfo struct { diff --git a/cns/restserver/util.go b/cns/restserver/util.go index b65e440a54..6c90000587 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -756,12 +756,12 @@ func (service *HTTPRestService) SendNCSnapShotPeriodically(ctx context.Context, } func (service *HTTPRestService) validateIPConfigsRequest(ctx context.Context, ipConfigsRequest cns.IPConfigsRequest) (cns.PodInfo, types.ResponseCode, string) { - if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes { + if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes && service.state.OrchestratorType != cns.ServiceFabric { return nil, types.UnsupportedOrchestratorType, "ReleaseIPConfig API supported only for kubernetes orchestrator" } if ipConfigsRequest.OrchestratorContext == nil { - return nil, types.EmptyOrchestratorContext, fmt.Sprintf("OrchastratorContext is not set in the req: %+v", ipConfigsRequest) + return nil, types.EmptyOrchestratorContext, fmt.Sprintf("OrchestratorContext is not set in the req: %+v", ipConfigsRequest) } // retrieve podinfo from orchestrator context @@ -790,6 +790,24 @@ func (service *HTTPRestService) getPrimaryHostInterface(ctx context.Context) (*w return service.state.primaryInterface, nil } +// getSecondaryHostInterface returns the cached InterfaceInfo, if available, otherwise +// queries the IMDS to get the secondary interface info and caches it in the server-state before returning the result. +func (service *HTTPRestService) getSecondaryHostInterface(ctx context.Context, macAddress string) (*wireserver.InterfaceInfo, error) { + if service.state.secondaryInterface == nil { + res, err := service.wscli.GetInterfaces(ctx) + if err != nil { + return nil, errors.Wrap(err, "failed to get interfaces from wireserver client") + } + secondary, err := wireserver.GetSecondaryInterfaceFromResult(res, macAddress) + if err != nil { + return nil, errors.Wrap(err, "failed to get secondary interface from wireserver client") + } + + service.state.secondaryInterface = secondary + } + return service.state.secondaryInterface, nil +} + //nolint:gocritic // ignore hugeParam pls func (service *HTTPRestService) populateIPConfigInfoUntransacted(ipConfigStatus cns.IPConfigurationStatus, podIPInfo *cns.PodIpInfo) error { ncStatus, exists := service.state.ContainerStatus[ipConfigStatus.NCID] diff --git a/cns/service/main.go b/cns/service/main.go index 5a0bac7dc0..9a07caa303 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -801,6 +801,12 @@ func main() { if platform.HasMellanoxAdapter() { go platform.MonitorAndSetMellanoxRegKeyPriorityVLANTag(rootCtx, cnsconfig.MellanoxMonitorIntervalSecs) } + + // if swiftv2 scenario is enabled, we need to initialize the Service Fabric (standalone) swiftv2 middleware to process IP configs requests + if cnsconfig.SWIFTV2Mode == configuration.SFSWIFTV2 { + swiftV2Middleware := &middlewares.SFSWIFTv2Middleware{} + httpRestService.AttachIPConfigsHandlerMiddleware(swiftV2Middleware) + } } // Initialze state in if CNS is running in CRD mode @@ -1220,7 +1226,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn // check the Node labels for Swift V2 if _, ok := node.Labels[configuration.LabelNodeSwiftV2]; ok { - cnsconfig.EnableSwiftV2 = true + cnsconfig.SWIFTV2Mode = configuration.K8sSWIFTV2 cnsconfig.WatchPods = true if nodeInfoErr := createOrUpdateNodeInfoCRD(ctx, kubeConfig, node); nodeInfoErr != nil { return errors.Wrap(nodeInfoErr, "error creating or updating nodeinfo crd") @@ -1404,22 +1410,13 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn } } - if cnsconfig.EnableSwiftV2 { + if cnsconfig.SWIFTV2Mode == configuration.K8sSWIFTV2 { if err := mtpncctrl.SetupWithManager(manager); err != nil { return errors.Wrapf(err, "failed to setup mtpnc reconciler with manager") } // if SWIFT v2 is enabled on CNS, attach multitenant middleware to rest service - // switch here for different type of swift v2 middleware (k8s or SF) - var swiftV2Middleware cns.IPConfigsHandlerMiddleware - switch cnsconfig.SWIFTV2Mode { - case configuration.K8sSWIFTV2: - swiftV2Middleware = &middlewares.K8sSWIFTv2Middleware{Cli: manager.GetClient()} - case configuration.SFSWIFTV2: - default: - // default to K8s middleware for now, in a later changes we where start to pass in - // SWIFT v2 mode in CNS config, this should throw an error if the mode is not set. - swiftV2Middleware = &middlewares.K8sSWIFTv2Middleware{Cli: manager.GetClient()} - } + // switch here for AKS(K8s) swiftv2 middleware to process IP configs requests + swiftV2Middleware := &middlewares.K8sSWIFTv2Middleware{Cli: manager.GetClient()} httpRestService.AttachIPConfigsHandlerMiddleware(swiftV2Middleware) } diff --git a/cns/wireserver/net.go b/cns/wireserver/net.go index ed0791d0af..e51d922b7f 100644 --- a/cns/wireserver/net.go +++ b/cns/wireserver/net.go @@ -2,13 +2,16 @@ package wireserver import ( "net" + "strings" "github.com/pkg/errors" ) var ( - // ErrNoPrimaryInterface indicates the wireserver respnose does not have a primary interface indicated. + // ErrNoPrimaryInterface indicates the wireserver response does not have a primary interface indicated. ErrNoPrimaryInterface = errors.New("no primary interface found") + // ErrNoSecondaryInterface indicates the wireserver response does not have secondary interface on the node + ErrNoSecondaryInterface = errors.New("no secondary interface found") // ErrInsufficientAddressSpace indicates that the CIDR space is too small to include a gateway IP; it is 1 IP. ErrInsufficientAddressSpace = errors.New("insufficient address space to generate gateway IP") ) @@ -49,6 +52,48 @@ func GetPrimaryInterfaceFromResult(res *GetInterfacesResult) (*InterfaceInfo, er return nil, ErrNoPrimaryInterface } +// Gets secondary interface details for swiftv2 secondary nics scenario +func GetSecondaryInterfaceFromResult(res *GetInterfacesResult, macAddress string) (*InterfaceInfo, error) { + for _, i := range res.Interface { + // skip if primary + if i.IsPrimary { + continue + } + + // skip if no subnets + if len(i.IPSubnet) == 0 { + continue + } + + if macAddressesEqual(i.MacAddress, macAddress) { + // get the second subnet + s := i.IPSubnet[0] + gw, err := calculateGatewayIP(s.Prefix) + if err != nil { + return nil, err + } + + secondaryIP := "" + for _, ip := range s.IPAddress { + if !ip.IsPrimary { + secondaryIP = ip.Address + break + } + } + var secondaryIPs []string + secondaryIPs = append(secondaryIPs, secondaryIP) + + return &InterfaceInfo{ + Subnet: s.Prefix, + IsPrimary: false, + Gateway: gw.String(), + SecondaryIPs: secondaryIPs, + }, nil + } + } + return nil, ErrNoSecondaryInterface +} + // calculateGatewayIP parses the passed CIDR string and returns the first IP in the range. func calculateGatewayIP(cidr string) (net.IP, error) { _, subnet, err := net.ParseCIDR(cidr) @@ -79,3 +124,9 @@ func calculateGatewayIP(cidr string) (net.IP, error) { } return gw, nil } + +func macAddressesEqual(macAddress1, macAddress2 string) bool { + macAddress1 = strings.ToLower(strings.ReplaceAll(macAddress1, ":", "")) + macAddress2 = strings.ToLower(strings.ReplaceAll(macAddress2, ":", "")) + return macAddress1 == macAddress2 +}