From d51a8b3ff988e86c44062e5a7e07f381db62c99c Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Fri, 8 Dec 2023 10:13:44 -0800 Subject: [PATCH] Make changes to statless CNI migration branch. --- Makefile | 2 +- cni/api/api.go | 3 ++ cni/network/network.go | 3 ++ cns/cnireconciler/podinfoprovider.go | 45 +++++++++++++++++++++-- cns/cnireconciler/podinfoprovider_test.go | 2 +- cns/configuration/configuration.go | 1 + cns/service/main.go | 15 +++++--- network/endpoint.go | 3 ++ 8 files changed, 63 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 8c583b98f5c..d74f74cb3da 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ ACN_VERSION ?= $(shell git describe --exclude "azure-ipam*" --exclude "dropgz AZURE_IPAM_VERSION ?= $(notdir $(shell git describe --match "azure-ipam*" --tags --always)) CNI_VERSION ?= $(ACN_VERSION) CNI_DROPGZ_VERSION ?= $(notdir $(shell git describe --match "dropgz*" --tags --always)) -CNS_VERSION ?= $(ACN_VERSION) +CNS_VERSION ?= "v1.5.15-22-g3d1fa46d" NPM_VERSION ?= $(ACN_VERSION) ZAPAI_VERSION ?= $(notdir $(shell git describe --match "zapai*" --tags --always)) diff --git a/cni/api/api.go b/cni/api/api.go index aed0f5ac0bd..0b2c4f9a4f6 100644 --- a/cni/api/api.go +++ b/cni/api/api.go @@ -17,6 +17,9 @@ type PodNetworkInterfaceInfo struct { PodEndpointId string ContainerID string IPAddresses []net.IPNet + HostIfName string + HNSEndpointID string + IfName string } type AzureCNIState struct { diff --git a/cni/network/network.go b/cni/network/network.go index 2381b8ffc4e..728bc9a485c 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -190,6 +190,9 @@ func (plugin *NetPlugin) GetAllEndpointState(networkid string) (*api.AzureCNISta PodEndpointId: ep.Id, ContainerID: ep.ContainerID, IPAddresses: ep.IPAddresses, + HostIfName: ep.HostIfName, + HNSEndpointID: ep.HNSEndpointID, + IfName: ep.IfName, } st.ContainerInterfaces[id] = info diff --git a/cns/cnireconciler/podinfoprovider.go b/cns/cnireconciler/podinfoprovider.go index fd237ebb66e..11eee2133a5 100644 --- a/cns/cnireconciler/podinfoprovider.go +++ b/cns/cnireconciler/podinfoprovider.go @@ -2,10 +2,12 @@ package cnireconciler import ( "fmt" + "net" "github.com/Azure/azure-container-networking/cni/api" "github.com/Azure/azure-container-networking/cni/client" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/restserver" "github.com/Azure/azure-container-networking/store" "github.com/pkg/errors" @@ -14,7 +16,7 @@ import ( // NewCNIPodInfoProvider returns an implementation of cns.PodInfoByIPProvider // that execs out to the CNI and uses the response to build the PodInfo map. -func NewCNIPodInfoProvider() (cns.PodInfoByIPProvider, error) { +func NewCNIPodInfoProvider() (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) { return newCNIPodInfoProvider(exec.New()) } @@ -39,15 +41,16 @@ func newCNSPodInfoProvider(endpointStore store.KeyValueStore) (cns.PodInfoByIPPr }), nil } -func newCNIPodInfoProvider(exec exec.Interface) (cns.PodInfoByIPProvider, error) { +func newCNIPodInfoProvider(exec exec.Interface) (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) { cli := client.New(exec) state, err := cli.GetEndpointState() if err != nil { - return nil, fmt.Errorf("failed to invoke CNI client.GetEndpointState(): %w", err) + return nil, nil, fmt.Errorf("failed to invoke CNI client.GetEndpointState(): %w", err) } + endpointState, err := cniStateToCnsEndpointState(state) return cns.PodInfoByIPProviderFunc(func() (map[string]cns.PodInfo, error) { return cniStateToPodInfoByIP(state) - }), nil + }), endpointState, nil } // cniStateToPodInfoByIP converts an AzureCNIState dumped from a CNI exec @@ -101,3 +104,37 @@ func endpointStateToPodInfoByIP(state map[string]*restserver.EndpointInfo) (map[ } return podInfoByIP, nil } + +// cniStateToCnsEndpointState converts an AzureCNIState dumped from a CNI exec +// into a EndpointInfo map, using the containerID as keys in the map. +// The map then will be saved on CNS endpoint state +func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserver.EndpointInfo, error) { + logger.Printf("Generating CNS ENdpoint State") + endpointState := map[string]*restserver.EndpointInfo{} + for _, endpoint := range state.ContainerInterfaces { + endpointInfo := &restserver.EndpointInfo{PodName: endpoint.PodName, PodNamespace: endpoint.PodNamespace, HnsEndpointID: endpoint.HNSEndpointID, HostVethName: endpoint.HostIfName, IfnameToIPMap: make(map[string]*restserver.IPInfo)} + for _, epIP := range endpoint.IPAddresses { + if epIP.IP.To4() == nil { // is an ipv6 address + ipconfig := net.IPNet{IP: epIP.IP, Mask: epIP.Mask} + for _, ipconf := range endpointInfo.IfnameToIPMap[endpoint.IfName].IPv6 { + if ipconf.IP.Equal(ipconfig.IP) { + logger.Printf("Found existing ipv6 ipconfig for infra container %s", endpoint.ContainerID) + return nil, nil + } + } + endpointInfo.IfnameToIPMap[endpoint.IfName].IPv6 = append(endpointInfo.IfnameToIPMap[endpoint.IfName].IPv6, ipconfig) + } else { + ipconfig := net.IPNet{IP: epIP.IP, Mask: epIP.Mask} + for _, ipconf := range endpointInfo.IfnameToIPMap[endpoint.IfName].IPv4 { + if ipconf.IP.Equal(ipconfig.IP) { + logger.Printf("Found existing ipv4 ipconfig for infra container %s", endpoint.ContainerID) + return nil, nil + } + } + endpointInfo.IfnameToIPMap[endpoint.IfName].IPv4 = append(endpointInfo.IfnameToIPMap[endpoint.IfName].IPv4, ipconfig) + } + } + endpointState[endpoint.ContainerID] = endpointInfo + } + return endpointState, nil +} diff --git a/cns/cnireconciler/podinfoprovider_test.go b/cns/cnireconciler/podinfoprovider_test.go index a9db49490fa..d4e253a7c83 100644 --- a/cns/cnireconciler/podinfoprovider_test.go +++ b/cns/cnireconciler/podinfoprovider_test.go @@ -51,7 +51,7 @@ func TestNewCNIPodInfoProvider(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - got, err := newCNIPodInfoProvider(tt.exec) + got, _, err := newCNIPodInfoProvider(tt.exec) if tt.wantErr { assert.Error(t, err) return diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index 1f21c477ac3..faf2d2fd965 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -48,6 +48,7 @@ type CNSConfig struct { WatchPods bool EnableAsyncPodDelete bool AsyncPodDeletePath string + StatelessCNIMigration bool } type TelemetrySettings struct { diff --git a/cns/service/main.go b/cns/service/main.go index 3d1f77cbfab..24a278934d1 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -829,7 +829,7 @@ func main() { logger.Printf("Set GlobalPodInfoScheme %v (InitializeFromCNI=%t)", cns.GlobalPodInfoScheme, cnsconfig.InitializeFromCNI) - err = InitializeCRDState(rootCtx, httpRestService, cnsconfig) + err = InitializeCRDState(rootCtx, httpRestService, cnsconfig, endpointStateStore) if err != nil { logger.Errorf("Failed to start CRD Controller, err:%v.\n", err) return @@ -1172,7 +1172,7 @@ func reconcileInitialCNSState(ctx context.Context, cli nodeNetworkConfigGetter, } // InitializeCRDState builds and starts the CRD controllers. -func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cnsconfig *configuration.CNSConfig) error { +func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cnsconfig *configuration.CNSConfig, endpointStateStore store.KeyValueStore) error { // convert interface type to implementation type httpRestServiceImplementation, ok := httpRestService.(*restserver.HTTPRestService) if !ok { @@ -1227,13 +1227,18 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn if err != nil { if errors.Is(err, store.ErrKeyNotFound) { logger.Printf("[Azure CNS] No endpoint state found, skipping initializing CNS state") - if os.Getenv("StatelessCNIMigration") == "true" { + if cnsconfig.StatelessCNIMigration { logger.Printf("StatelessCNI Migration is enabled") logger.Printf("Initializing from Statefull CNI") - podInfoByIPProvider, err = cnireconciler.NewCNIPodInfoProvider() + var endpointState map[string]*restserver.EndpointInfo + podInfoByIPProvider, endpointState, err = cnireconciler.NewCNIPodInfoProvider() if err != nil { return errors.Wrap(err, "failed to create CNI PodInfoProvider") } + err := endpointStateStore.Write(restserver.EndpointStoreKey, endpointState) + if err != nil { + return fmt.Errorf("failed to write endpoint state to store: %w", err) + } } } else { return errors.Wrap(err, "failed to create CNS PodInfoProvider") @@ -1242,7 +1247,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn case cnsconfig.InitializeFromCNI: logger.Printf("Initializing from CNI") - podInfoByIPProvider, err = cnireconciler.NewCNIPodInfoProvider() + podInfoByIPProvider, _, err = cnireconciler.NewCNIPodInfoProvider() if err != nil { return errors.Wrap(err, "failed to create CNI PodInfoProvider") } diff --git a/network/endpoint.go b/network/endpoint.go index 518aa804f3b..79b4be99031 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -92,6 +92,7 @@ type EndpointInfo struct { NICType cns.NICType SkipDefaultRoutes bool HNSEndpointID string + HostIfName string } // RouteInfo contains information about an IP route. @@ -267,6 +268,8 @@ func (ep *endpoint) getInfo() *EndpointInfo { PODName: ep.PODName, PODNameSpace: ep.PODNameSpace, NetworkContainerID: ep.NetworkContainerID, + HNSEndpointID: ep.HnsId, + HostIfName: ep.HostIfName, } info.Routes = append(info.Routes, ep.Routes...)