From d8afe37be6f6a681178700458e18b4c2762891e4 Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Thu, 25 Jan 2024 20:19:53 -0800 Subject: [PATCH] Addressing the comments. --- cni/linux.Dockerfile | 3 + cni/network/network.go | 4 +- cni/network/network_windows.go | 2 +- cnm/network/network.go | 2 +- cns/cnireconciler/podinfoprovider.go | 5 +- cns/cnireconciler/podinfoprovider_test.go | 27 ++++++++- cns/configuration/configuration.go | 1 + cns/service/main.go | 52 ++++++++++------- network/endpoint.go | 2 +- network/endpoint_windows.go | 14 ++--- network/manager.go | 69 +++++++++++++---------- network/manager_mock.go | 2 +- 12 files changed, 114 insertions(+), 69 deletions(-) diff --git a/cni/linux.Dockerfile b/cni/linux.Dockerfile index 1a83b3a895..18b93be7d5 100644 --- a/cni/linux.Dockerfile +++ b/cni/linux.Dockerfile @@ -13,6 +13,9 @@ RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azure-vnet-telemetry -trimpath RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azure-vnet-ipam -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/ipam/plugin/main.go RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azurecni-stateless -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/network/stateless/main.go +FROM scratch as bins +COPY --from=azure-vnet /go/bin/* / + FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 AS compressor ARG OS WORKDIR /payload diff --git a/cni/network/network.go b/cni/network/network.go index 2381b8ffc4..65d6e0c17b 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -889,7 +889,7 @@ func (plugin *NetPlugin) Get(args *cniSkel.CmdArgs) error { } // Query the endpoint. - if epInfo, err = plugin.nm.GetEndpointInfo(networkID, endpointID); err != nil { + if epInfo, err = plugin.nm.GetEndpointInfo(networkID, endpointID, args.IfName); err != nil { logger.Error("Failed to query endpoint", zap.Error(err)) return err } @@ -1051,7 +1051,7 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName) // Query the endpoint. - if epInfo, err = plugin.nm.GetEndpointInfo(networkID, endpointID); err != nil { + if epInfo, err = plugin.nm.GetEndpointInfo(networkID, endpointID, args.IfName); err != nil { logger.Info("GetEndpoint", zap.String("endpoint", endpointID), zap.Error(err)) diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index ba0fdfaf4a..3fe382bf18 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -40,7 +40,7 @@ var ( func (plugin *NetPlugin) handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, networkId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig, ) (*cniTypesCurr.Result, error) { - epInfo, _ := plugin.nm.GetEndpointInfo(networkId, endpointId) + epInfo, _ := plugin.nm.GetEndpointInfo(networkId, endpointId, "") if epInfo == nil { return nil, nil } diff --git a/cnm/network/network.go b/cnm/network/network.go index 4358d1a485..ec0957d6b2 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -357,7 +357,7 @@ func (plugin *netPlugin) endpointOperInfo(w http.ResponseWriter, r *http.Request } // Process request. - epInfo, err := plugin.nm.GetEndpointInfo(req.NetworkID, req.EndpointID) + epInfo, err := plugin.nm.GetEndpointInfo(req.NetworkID, req.EndpointID, "") if err != nil { plugin.SendErrorResponse(w, err) return diff --git a/cns/cnireconciler/podinfoprovider.go b/cns/cnireconciler/podinfoprovider.go index 9e3b318005..3330b8e0d3 100644 --- a/cns/cnireconciler/podinfoprovider.go +++ b/cns/cnireconciler/podinfoprovider.go @@ -69,7 +69,6 @@ func cniStateToPodInfoByIP(state *api.AzureCNIState) (map[string]cns.PodInfo, er for _, endpoint := range state.ContainerInterfaces { for _, epIP := range endpoint.IPAddresses { podInfo := cns.NewPodInfo(endpoint.ContainerID, endpoint.PodEndpointId, endpoint.PodName, endpoint.PodNamespace) - logger.Printf("podInfoByIp [%+v]", podInfoByIP) ipKey := epIP.IP.String() if prevPodInfo, ok := podInfoByIP[ipKey]; ok { return nil, errors.Wrapf(cns.ErrDuplicateIP, "duplicate ip %s found for different pods: pod: %+v, pod: %+v", ipKey, podInfo, prevPodInfo) @@ -78,7 +77,6 @@ func cniStateToPodInfoByIP(state *api.AzureCNIState) (map[string]cns.PodInfo, er podInfoByIP[ipKey] = podInfo } } - logger.Printf("podInfoByIP [%+v]", podInfoByIP) return podInfoByIP, nil } @@ -117,7 +115,7 @@ func endpointStateToPodInfoByIP(state map[string]*restserver.EndpointInfo) (map[ // into a EndpointInfo map, using the containerID as keys in the map. // The map then will be saved on CNS endpoint state func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserver.EndpointInfo, error) { - logger.Printf("Generating CNS ENdpoint State") + logger.Printf("Generating CNS Endpoint State") endpointState := map[string]*restserver.EndpointInfo{} for epID, endpoint := range state.ContainerInterfaces { endpointInfo := &restserver.EndpointInfo{PodName: endpoint.PodName, PodNamespace: endpoint.PodNamespace, IfnameToIPMap: make(map[string]*restserver.IPInfo)} @@ -147,6 +145,7 @@ func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserve endpointID, Ifname := extractEndpointInfo(epID, endpoint.ContainerID) endpointInfo.IfnameToIPMap[Ifname] = ipInfo endpointState[endpointID] = endpointInfo + logger.Printf("CNS endpoint state extracted from CNI: [%+v]", *endpointInfo) } return endpointState, nil } diff --git a/cns/cnireconciler/podinfoprovider_test.go b/cns/cnireconciler/podinfoprovider_test.go index 8a6cb85723..da51b840e8 100644 --- a/cns/cnireconciler/podinfoprovider_test.go +++ b/cns/cnireconciler/podinfoprovider_test.go @@ -31,7 +31,13 @@ func TestNewCNIPodInfoProvider(t *testing.T) { { name: "good", exec: newCNIStateFakeExec( - `{"ContainerInterfaces":{"3f813b02-eth0":{"PodName":"metrics-server-77c8679d7d-6ksdh","IfName":"eth0","PodNamespace":"kube-system","PodEndpointID":"3f813b02-eth0","ContainerID":"3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46","IPAddresses":[{"IP":"10.241.0.17","Mask":"//8AAA=="}]},"6e688597-eth0":{"PodName":"tunnelfront-5d96f9b987-65xbn","IfName":"eth0","PodNamespace":"kube-system","PodEndpointID":"6e688597-eth0","ContainerID":"6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed","IPAddresses":[{"IP":"10.241.0.13","Mask":"//8AAA=="}]}}}`, + `{"ContainerInterfaces":{"3f813b02-eth0":{"PodName":"metrics-server-77c8679d7d-6ksdh","IfName":"eth0", + "PodNamespace":"kube-system","PodEndpointID":"3f813b02-eth0", + "ContainerID":"3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46", + "IPAddresses":[{"IP":"10.241.0.17","Mask":"//8AAA=="}]}, + "6e688597-eth0":{"PodName":"tunnelfront-5d96f9b987-65xbn","IfName":"eth0","PodNamespace":"kube-system", + "PodEndpointID":"6e688597-eth0","ContainerID":"6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", + "IPAddresses":[{"IP":"10.241.0.13","Mask":"//8AAA=="}]}}}`, ), want: map[string]cns.PodInfo{ "10.241.0.13": cns.NewPodInfo("6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", "6e688597-eth0", "tunnelfront-5d96f9b987-65xbn", "kube-system"), @@ -39,6 +45,25 @@ func TestNewCNIPodInfoProvider(t *testing.T) { }, wantErr: false, }, + { + name: "dualstack", + exec: newCNIStateFakeExec( + `{"ContainerInterfaces":{"3f813b02-eth0":{"PodName":"metrics-server-77c8679d7d-6ksdh","IfName":"eth0", + "PodNamespace":"kube-system","PodEndpointID":"3f813b02-eth0", + "ContainerID":"3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46", + "IPAddresses":[{"IP":"10.241.0.17","Mask":"//8AAA=="},{"IP":"2001:0db8:abcd:0015::0","Mask":"//8AAA=="}]}, + "6e688597-eth0":{"PodName":"tunnelfront-5d96f9b987-65xbn","IfName":"eth0","PodNamespace":"kube-system", + "PodEndpointID":"6e688597-eth0","ContainerID":"6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", + "IPAddresses":[{"IP":"10.241.0.13","Mask":"//8AAA=="},{"IP":"2001:0db8:abcd:0014::0","Mask":"//8AAA=="}]}}}`, + ), + want: map[string]cns.PodInfo{ + "2001:db8:abcd:15::": cns.NewPodInfo("3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46", "3f813b02-eth0", "metrics-server-77c8679d7d-6ksdh", "kube-system"), + "2001:db8:abcd:14::": cns.NewPodInfo("6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", "6e688597-eth0", "tunnelfront-5d96f9b987-65xbn", "kube-system"), + "10.241.0.17": cns.NewPodInfo("3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46", "3f813b02-eth0", "metrics-server-77c8679d7d-6ksdh", "kube-system"), + "10.241.0.13": cns.NewPodInfo("6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", "6e688597-eth0", "tunnelfront-5d96f9b987-65xbn", "kube-system"), + }, + wantErr: false, + }, { name: "empty CNI response", exec: newCNIStateFakeExec( diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index fd49613131..69735de7b7 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -56,6 +56,7 @@ type CNSConfig struct { UseHTTPS bool WatchPods bool `json:"-"` WireserverIP string + StatelessCNIMigration bool } type TelemetrySettings struct { diff --git a/cns/service/main.go b/cns/service/main.go index b0e1901e49..2c155a1ec2 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -839,7 +839,7 @@ func main() { logger.Printf("Set GlobalPodInfoScheme %v (InitializeFromCNI=%t)", cns.GlobalPodInfoScheme, cnsconfig.InitializeFromCNI) - err = InitializeCRDState(rootCtx, httpRestService, cnsconfig, endpointStateStore) + err = InitializeCRDState(rootCtx, httpRestService, cnsconfig) if err != nil { logger.Errorf("Failed to start CRD Controller, err:%v.\n", err) return @@ -1182,7 +1182,7 @@ func reconcileInitialCNSState(ctx context.Context, cli nodeNetworkConfigGetter, } // InitializeCRDState builds and starts the CRD controllers. -func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cnsconfig *configuration.CNSConfig, endpointStateStore store.KeyValueStore) error { +func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cnsconfig *configuration.CNSConfig) error { // convert interface type to implementation type httpRestServiceImplementation, ok := httpRestService.(*restserver.HTTPRestService) if !ok { @@ -1234,26 +1234,9 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn switch { case cnsconfig.ManageEndpointState: logger.Printf("Initializing from self managed endpoint store") - podInfoByIPProvider, err = cnireconciler.NewCNSPodInfoProvider(httpRestServiceImplementation.EndpointStateStore) // get reference to endpoint state store from rest server + podInfoByIPProvider, err = InitializeStateFromCNS(cnsconfig, httpRestServiceImplementation.EndpointStateStore) if err != nil { - if errors.Is(err, store.ErrKeyNotFound) { - logger.Printf("[Azure CNS] No endpoint state found, skipping initializing CNS state") - if cnsconfig.StatelessCNIMigration { - logger.Printf("StatelessCNI Migration is enabled") - logger.Printf("initializing from Statefull CNI") - var endpointState map[string]*restserver.EndpointInfo - podInfoByIPProvider, endpointState, err = cnireconciler.NewCNIPodInfoProvider() - if err != nil { - return errors.Wrap(err, "failed to create CNI PodInfoProvider") - } - err = endpointStateStore.Write(restserver.EndpointStoreKey, endpointState) - if err != nil { - return fmt.Errorf("failed to write endpoint state to store: %w", err) - } - } - } else { - return errors.Wrap(err, "failed to create CNS PodInfoProvider") - } + return errors.Wrap(err, "failed to create CNI PodInfoProvider") } case cnsconfig.InitializeFromCNI: @@ -1530,3 +1513,30 @@ func createOrUpdateNodeInfoCRD(ctx context.Context, restConfig *rest.Config, nod return nil } + +// InitializeStateFromCNS initilizes CNS Endpoint State from CNS or perform the Migration of state from statefull CNI. +func InitializeStateFromCNS(cnsconfig *configuration.CNSConfig, endpointStateStore store.KeyValueStore) (cns.PodInfoByIPProvider, error) { + logger.Printf("Initializing from self managed endpoint store") + podInfoByIPProvider, err := cnireconciler.NewCNSPodInfoProvider(endpointStateStore) // get reference to endpoint state store from rest server + if err != nil { + if errors.Is(err, store.ErrKeyNotFound) { + logger.Printf("[Azure CNS] No endpoint state found, skipping initializing CNS state") + if cnsconfig.StatelessCNIMigration { + logger.Printf("StatelessCNI Migration is enabled") + logger.Printf("initializing from Statefull CNI") + var endpointState map[string]*restserver.EndpointInfo + podInfoByIPProvider, endpointState, err = cnireconciler.NewCNIPodInfoProvider() + if err != nil { + return nil, errors.Wrap(err, "failed to create CNI PodInfoProvider") + } + err = endpointStateStore.Write(restserver.EndpointStoreKey, endpointState) + if err != nil { + return nil, fmt.Errorf("failed to write endpoint state to store: %w", err) + } + } + } else { + return nil, errors.Wrap(err, "failed to create CNS PodInfoProvider") + } + } + return podInfoByIPProvider, nil +} diff --git a/network/endpoint.go b/network/endpoint.go index 79b4be9903..c8b4cce10b 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -364,7 +364,7 @@ func (epInfo *EndpointInfo) GetEndpointInfoByIP(ipAddresses []net.IPNet, network return endpointInfo, nil } -// IsEndpointStateComplete returns true if both HNSEndpointID and HostVethName are missing. +// IsEndpointStateInComplete returns true if both HNSEndpointID and HostVethName are missing. func (epInfo *EndpointInfo) IsEndpointStateIncomplete() bool { if epInfo.HNSEndpointID == "" && epInfo.IfName == "" { return true diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index fff6428403..c0ddf486bf 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -5,7 +5,6 @@ package network import ( "context" - "errors" "fmt" "net" "strings" @@ -16,6 +15,7 @@ import ( "github.com/Azure/azure-container-networking/platform" "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" + "github.com/pkg/errors" "go.uber.org/zap" ) @@ -502,22 +502,22 @@ func (epInfo *EndpointInfo) GetEndpointInfoByIPImpl(ipAddresses []net.IPNet, net // check if network exists, only create the network does not exist hnsResponse, err := Hnsv2.GetNetworkByName(networkID) if err != nil { - return epInfo, errors.Wrap(err, "HNS Network not found") + return epInfo, errors.Wrapf(err, "HNS Network not found") } hcnEndpoints, err := Hnsv2.ListEndpointsOfNetwork(hnsResponse.Id) if err != nil { - return epInfo, errors.Wrap(err, "failed to fetch HNS endpoints for the given network") + return epInfo, errors.Wrapf(err, "failed to fetch HNS endpoints for the given network") } - for _, hcnEndpoint := range hcnEndpoints { - for _, ipConfiguration := range hcnEndpoint.IpConfigurations { + for i := range hcnEndpoints { + for _, ipConfiguration := range hcnEndpoints[i].IpConfigurations { for _, ipAddress := range ipAddresses { prefixLength, _ := ipAddress.Mask.Size() if ipConfiguration.IpAddress == ipAddress.IP.String() && ipConfiguration.PrefixLength == uint8(prefixLength) { - epInfo.HNSEndpointID = hcnEndpoint.Id + epInfo.HNSEndpointID = hcnEndpoints[i].Id return epInfo, nil } } } } - return epInfo, errors.wrap(err, "No HNSEndpointID matches the IPAddress: "+ipAddresses[0].IP.String()) + return epInfo, errors.Wrapf(err, "No HNSEndpointID matches the IPAddress: "+ipAddresses[0].IP.String()) } diff --git a/network/manager.go b/network/manager.go index 064a3274b5..bf95467811 100644 --- a/network/manager.go +++ b/network/manager.go @@ -101,7 +101,7 @@ type NetworkManager interface { CreateEndpoint(client apipaClient, networkID string, epInfo []*EndpointInfo) error DeleteEndpoint(networkID string, endpointID string, epInfo *EndpointInfo) error - GetEndpointInfo(networkID string, endpointID string) (*EndpointInfo, error) + GetEndpointInfo(networkID string, endpointID string, ifName string) (*EndpointInfo, error) GetAllEndpoints(networkID string) (map[string]*EndpointInfo, error) GetEndpointInfoBasedOnPODDetails(networkID string, podName string, podNameSpace string, doExactMatchForPodName bool) (*EndpointInfo, error) AttachEndpoint(networkID string, endpointID string, sandboxKey string) (*endpoint, error) @@ -420,6 +420,38 @@ func (nm *networkManager) UpdateEndpointState(ep *endpoint) error { return nil } +// GetEndpointState will make a call to CNS GetEndpointState API in the stateless CNI mode to fetch the endpointInfo +func (nm *networkManager) GetEndpointState(networkID, endpointID, ifName string) (*EndpointInfo, error) { + endpointResponse, err := nm.CnsClient.GetEndpoint(context.TODO(), endpointID, ifName) + if err != nil { + return nil, errors.Wrapf(err, "Get endpoint API returend with error") + } + epInfo := &EndpointInfo{ + Id: endpointID, + IfIndex: EndpointIfIndex, // Azure CNI supports only one interface + IfName: endpointResponse.EndpointInfo.HostVethName, + ContainerID: endpointID, + PODName: endpointResponse.EndpointInfo.PodName, + PODNameSpace: endpointResponse.EndpointInfo.PodNamespace, + NetworkContainerID: endpointID, + HNSEndpointID: endpointResponse.EndpointInfo.HnsEndpointID, + } + + for _, ip := range endpointResponse.EndpointInfo.IfnameToIPMap { + epInfo.IPAddresses = ip.IPv4 + epInfo.IPAddresses = append(epInfo.IPAddresses, ip.IPv6...) + + } + if epInfo.IsEndpointStateIncomplete() { + epInfo, err = epInfo.GetEndpointInfoByIP(epInfo.IPAddresses, networkID) + if err != nil { + return nil, errors.Wrapf(err, "Get endpoint API returend with error") + } + } + logger.Info("returning getEndpoint API with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", epInfo.HNSEndpointID)) + return epInfo, nil +} + // DeleteEndpoint deletes an existing container endpoint. func (nm *networkManager) DeleteEndpoint(networkID, endpointID string, epInfo *EndpointInfo) error { nm.Lock() @@ -475,48 +507,23 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint } // GetEndpointInfo returns information about the given endpoint. -func (nm *networkManager) GetEndpointInfo(networkId string, endpointId string) (*EndpointInfo, error) { +func (nm *networkManager) GetEndpointInfo(networkID, endpointID, ifName string) (*EndpointInfo, error) { nm.Lock() defer nm.Unlock() if nm.IsStatelessCNIMode() { logger.Info("calling cns getEndpoint API") - endpointResponse, err := nm.CnsClient.GetEndpoint(context.TODO(), endpointId, "eth0") - if err != nil { - return nil, errors.Wrapf(err, "Get endpoint API returend with error") - } - epInfo := &EndpointInfo{ - Id: endpointId, - IfIndex: EndpointIfIndex, // Azure CNI supports only one interface - IfName: endpointResponse.EndpointInfo.HostVethName, - ContainerID: endpointId, - PODName: endpointResponse.EndpointInfo.PodName, - PODNameSpace: endpointResponse.EndpointInfo.PodNamespace, - NetworkContainerID: endpointId, - HNSEndpointID: endpointResponse.EndpointInfo.HnsEndpointID, - } - - for _, ip := range endpointResponse.EndpointInfo.IfnameToIPMap { - epInfo.IPAddresses = ip.IPv4 - epInfo.IPAddresses = append(epInfo.IPAddresses, ip.IPv6...) + epInfo, err := nm.GetEndpointState(networkID, endpointID, ifName) - } - if epInfo.IsEndpointStateIncomplete() { - epInfo, err = epInfo.GetEndpointInfoByIP(epInfo.IPAddresses, networkId) - if err != nil { - return nil, errors.Wrapf(err, "Get endpoint API returend with error") - } - } - logger.Info("returning getEndpoint API with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", epInfo.HNSEndpointID)) - return epInfo, nil + return epInfo, err } - nw, err := nm.getNetwork(networkId) + nw, err := nm.getNetwork(networkID) if err != nil { return nil, err } - ep, err := nw.getEndpoint(endpointId) + ep, err := nw.getEndpoint(endpointID) if err != nil { return nil, err } diff --git a/network/manager_mock.go b/network/manager_mock.go index 0f69315839..2f397c3aca 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -99,7 +99,7 @@ func (nm *MockNetworkManager) GetAllEndpoints(networkID string) (map[string]*End } // GetEndpointInfo mock -func (nm *MockNetworkManager) GetEndpointInfo(networkID string, endpointID string) (*EndpointInfo, error) { +func (nm *MockNetworkManager) GetEndpointInfo(_, endpointID, _ string) (*EndpointInfo, error) { if info, exists := nm.TestEndpointInfoMap[endpointID]; exists { return info, nil }