feat: Migration stateless cni

cni/linux.Dockerfile

@@ -13,6 +13,9 @@ RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azure-vnet-telemetry -trimpath
 RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azure-vnet-ipam -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/ipam/plugin/main.go
 RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azurecni-stateless -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/network/stateless/main.go
 
+FROM scratch as bins
+COPY --from=azure-vnet /go/bin/* /
+
 FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 AS compressor
 ARG OS
 WORKDIR /payload

cns/cnireconciler/podinfoprovider.go

@@ -2,10 +2,13 @@ package cnireconciler
 
 import (
 	"fmt"
+	"net"
+	"strings"
 
 	"github.com/Azure/azure-container-networking/cni/api"
 	"github.com/Azure/azure-container-networking/cni/client"
 	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/cns/logger"
 	"github.com/Azure/azure-container-networking/cns/restserver"
 	"github.com/Azure/azure-container-networking/store"
 	"github.com/pkg/errors"
@@ -14,8 +17,9 @@ import (
 
 // NewCNIPodInfoProvider returns an implementation of cns.PodInfoByIPProvider
 // that execs out to the CNI and uses the response to build the PodInfo map.
-func NewCNIPodInfoProvider() (cns.PodInfoByIPProvider, error) {
-	return newCNIPodInfoProvider(exec.New())
+// if stateMigration flag is set to true it will also returns a map of containerID->EndpointInfo
+func NewCNIPodInfoProvider(stateMigration bool) (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) {
+	return newCNIPodInfoProvider(exec.New(), stateMigration)
 }
 
 func NewCNSPodInfoProvider(endpointStore store.KeyValueStore) (cns.PodInfoByIPProvider, error) {
@@ -39,15 +43,24 @@ func newCNSPodInfoProvider(endpointStore store.KeyValueStore) (cns.PodInfoByIPPr
 	}), nil
 }
 
-func newCNIPodInfoProvider(exec exec.Interface) (cns.PodInfoByIPProvider, error) {
-	cli := client.New(exec)
+func newCNIPodInfoProvider(exc exec.Interface, stateMigration bool) (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) {
+	cli := client.New(exc)
 	state, err := cli.GetEndpointState()
 	if err != nil {
-		return nil, fmt.Errorf("failed to invoke CNI client.GetEndpointState(): %w", err)
+		return nil, nil, fmt.Errorf("failed to invoke CNI client.GetEndpointState(): %w", err)
+	}
+	for containerID, endpointInfo := range state.ContainerInterfaces {
+		logger.Printf("state dump from CNI: [%+v], [%+v]", containerID, endpointInfo)
+	}
+	var endpointState map[string]*restserver.EndpointInfo
+	if stateMigration {
+		endpointState = cniStateToCnsEndpointState(state)
+	} else {
+		endpointState = nil
 	}
 	return cns.PodInfoByIPProviderFunc(func() (map[string]cns.PodInfo, error) {
 		return cniStateToPodInfoByIP(state)
-	}), nil
+	}), endpointState, err
 }
 
 // cniStateToPodInfoByIP converts an AzureCNIState dumped from a CNI exec
@@ -59,7 +72,6 @@ func cniStateToPodInfoByIP(state *api.AzureCNIState) (map[string]cns.PodInfo, er
 	for _, endpoint := range state.ContainerInterfaces {
 		for _, epIP := range endpoint.IPAddresses {
 			podInfo := cns.NewPodInfo(endpoint.ContainerID, endpoint.PodEndpointId, endpoint.PodName, endpoint.PodNamespace)
-
 			ipKey := epIP.IP.String()
 			if prevPodInfo, ok := podInfoByIP[ipKey]; ok {
 				return nil, errors.Wrapf(cns.ErrDuplicateIP, "duplicate ip %s found for different pods: pod: %+v, pod: %+v", ipKey, podInfo, prevPodInfo)
@@ -101,3 +113,42 @@ func endpointStateToPodInfoByIP(state map[string]*restserver.EndpointInfo) (map[
 	}
 	return podInfoByIP, nil
 }
+
+// cniStateToCnsEndpointState converts an AzureCNIState dumped from a CNI exec
+// into a EndpointInfo map, using the containerID as keys in the map.
+// The map then will be saved on CNS endpoint state
+func cniStateToCnsEndpointState(state *api.AzureCNIState) map[string]*restserver.EndpointInfo {
+	logger.Printf("Generating CNS Endpoint State")
+	endpointState := map[string]*restserver.EndpointInfo{}
+	for epID, endpoint := range state.ContainerInterfaces {
+		endpointInfo := &restserver.EndpointInfo{PodName: endpoint.PodName, PodNamespace: endpoint.PodNamespace, IfnameToIPMap: make(map[string]*restserver.IPInfo)}
+		ipInfo := &restserver.IPInfo{}
+		for _, epIP := range endpoint.IPAddresses {
+			if epIP.IP.To4() == nil { // is an ipv6 address
+				ipconfig := net.IPNet{IP: epIP.IP, Mask: epIP.Mask}
+				ipInfo.IPv6 = append(ipInfo.IPv6, ipconfig)
+
+			} else {
+				ipconfig := net.IPNet{IP: epIP.IP, Mask: epIP.Mask}
+				ipInfo.IPv4 = append(ipInfo.IPv4, ipconfig)
+			}
+		}
+		endpointID, Ifname := extractEndpointInfo(epID, endpoint.ContainerID)
+		endpointInfo.IfnameToIPMap[Ifname] = ipInfo
+		endpointState[endpointID] = endpointInfo
+		logger.Printf("CNS endpoint state extracted from CNI: [%+v]", *endpointInfo)
+	}
+	return endpointState
+}
+
+// extractEndpointInfo extract Interface Name and endpointID for each endpoint based the CNI state
+func extractEndpointInfo(epID, containerID string) (endpointID, interfaceName string) {
+	ifName := restserver.InterfaceName
+	if strings.Contains(epID, "-eth") {
+		ifName = epID[len(epID)-4:]
+	}
+	if containerID == "" {
+		return epID, ifName
+	}
+	return containerID, ifName
+}

cns/cnireconciler/podinfoprovider_test.go

@@ -31,14 +31,39 @@ func TestNewCNIPodInfoProvider(t *testing.T) {
 		{
 			name: "good",
 			exec: newCNIStateFakeExec(
-				`{"ContainerInterfaces":{"3f813b02-eth0":{"PodName":"metrics-server-77c8679d7d-6ksdh","PodNamespace":"kube-system","PodEndpointID":"3f813b02-eth0","ContainerID":"3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46","IPAddresses":[{"IP":"10.241.0.17","Mask":"//8AAA=="}]},"6e688597-eth0":{"PodName":"tunnelfront-5d96f9b987-65xbn","PodNamespace":"kube-system","PodEndpointID":"6e688597-eth0","ContainerID":"6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed","IPAddresses":[{"IP":"10.241.0.13","Mask":"//8AAA=="}]}}}`,
+				`{"ContainerInterfaces":{"3f813b02-eth0":{"PodName":"metrics-server-77c8679d7d-6ksdh","IfName":"eth0",
+				"PodNamespace":"kube-system","PodEndpointID":"3f813b02-eth0",
+				"ContainerID":"3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46",
+				"IPAddresses":[{"IP":"10.241.0.17","Mask":"//8AAA=="}]},
+				"6e688597-eth0":{"PodName":"tunnelfront-5d96f9b987-65xbn","IfName":"eth0","PodNamespace":"kube-system",
+				"PodEndpointID":"6e688597-eth0","ContainerID":"6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed",
+				"IPAddresses":[{"IP":"10.241.0.13","Mask":"//8AAA=="}]}}}`,
 			),
 			want: map[string]cns.PodInfo{
 				"10.241.0.13": cns.NewPodInfo("6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", "6e688597-eth0", "tunnelfront-5d96f9b987-65xbn", "kube-system"),
 				"10.241.0.17": cns.NewPodInfo("3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46", "3f813b02-eth0", "metrics-server-77c8679d7d-6ksdh", "kube-system"),
 			},
 			wantErr: false,
 		},
+		{
+			name: "dualstack",
+			exec: newCNIStateFakeExec(
+				`{"ContainerInterfaces":{"3f813b02-eth0":{"PodName":"metrics-server-77c8679d7d-6ksdh","IfName":"eth0",
+				"PodNamespace":"kube-system","PodEndpointID":"3f813b02-eth0",
+				"ContainerID":"3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46",
+				"IPAddresses":[{"IP":"10.241.0.17","Mask":"//8AAA=="},{"IP":"2001:0db8:abcd:0015::0","Mask":"//8AAA=="}]},
+				"6e688597-eth0":{"PodName":"tunnelfront-5d96f9b987-65xbn","IfName":"eth0","PodNamespace":"kube-system",
+				"PodEndpointID":"6e688597-eth0","ContainerID":"6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed",
+				"IPAddresses":[{"IP":"10.241.0.13","Mask":"//8AAA=="},{"IP":"2001:0db8:abcd:0014::0","Mask":"//8AAA=="}]}}}`,
+			),
+			want: map[string]cns.PodInfo{
+				"2001:db8:abcd:15::": cns.NewPodInfo("3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46", "3f813b02-eth0", "metrics-server-77c8679d7d-6ksdh", "kube-system"),
+				"2001:db8:abcd:14::": cns.NewPodInfo("6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", "6e688597-eth0", "tunnelfront-5d96f9b987-65xbn", "kube-system"),
+				"10.241.0.17":        cns.NewPodInfo("3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46", "3f813b02-eth0", "metrics-server-77c8679d7d-6ksdh", "kube-system"),
+				"10.241.0.13":        cns.NewPodInfo("6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", "6e688597-eth0", "tunnelfront-5d96f9b987-65xbn", "kube-system"),
+			},
+			wantErr: false,
+		},
 		{
 			name: "empty CNI response",
 			exec: newCNIStateFakeExec(
@@ -51,14 +76,14 @@ func TestNewCNIPodInfoProvider(t *testing.T) {
 	for _, tt := range tests {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := newCNIPodInfoProvider(tt.exec)
+			got, endpointState, err := newCNIPodInfoProvider(tt.exec, true)
 			if tt.wantErr {
 				assert.Error(t, err)
 				return
 			}
 			assert.NoError(t, err)
 			podInfoByIP, _ := got.PodInfoByIP()
-			assert.Equal(t, tt.want, podInfoByIP)
+			assert.Equal(t, tt.want, podInfoByIP, endpointState)
 		})
 	}
 }

cns/configuration/configuration.go

@@ -56,6 +56,7 @@ type CNSConfig struct {
 	UseHTTPS                    bool
 	WatchPods                   bool `json:"-"`
 	WireserverIP                string
+	StateMigration              bool
 }
 
 type TelemetrySettings struct {

cns/restserver/ipam.go

@@ -29,6 +29,11 @@ var (
 	ErrEndpointStateNotFound  = errors.New("endpoint state could not be found in the statefile")
 )
 
+const (
+	ContainerIDLength = 8
+	InterfaceName     = "eth0"
+)
+
 // requestIPConfigHandlerHelper validates the request, assign IPs and return the IPConfigs
 func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) {
 	// For SWIFT v2 scenario, the validator function will also modify the ipconfigsRequest.
@@ -1008,9 +1013,9 @@ func (service *HTTPRestService) EndpointHandlerAPI(w http.ResponseWriter, r *htt
 // GetEndpointHandler handles the incoming GetEndpoint requests with http Get method
 func (service *HTTPRestService) GetEndpointHandler(w http.ResponseWriter, r *http.Request) {
 	logger.Printf("[GetEndpointState] GetEndpoint for %s", r.URL.Path)
-
 	endpointID := strings.TrimPrefix(r.URL.Path, cns.EndpointPath)
 	endpointInfo, err := service.GetEndpointHelper(endpointID)
+	// Check if the request is valid
 	if err != nil {
 		response := GetEndpointResponse{
 			Response: Response{
@@ -1068,6 +1073,14 @@ func (service *HTTPRestService) GetEndpointHelper(endpointID string) (*EndpointI
 		logger.Warnf("[GetEndpointState] Found existing endpoint state for container %s", endpointID)
 		return endpointInfo, nil
 	}
+	// This part is a temprory fix if we have endpoint states belong to CNI version 1.4.X on Windows since the states don't have the containerID
+	// In case there was no endpoint founded with ContainerID as the key,
+	// then [First 8 character of containerid]-eth0 will be tried
+	legacyEndpointID := endpointID[:ContainerIDLength] + "-" + InterfaceName
+	if endpointInfo, ok := service.EndpointState[legacyEndpointID]; ok {
+		logger.Warnf("[GetEndpointState] Found existing endpoint state for container %s", legacyEndpointID)
+		return endpointInfo, nil
+	}
 	return nil, ErrEndpointStateNotFound
 }
 

cns/service/main.go

@@ -1234,17 +1234,14 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn
 	switch {
 	case cnsconfig.ManageEndpointState:
 		logger.Printf("Initializing from self managed endpoint store")
-		podInfoByIPProvider, err = cnireconciler.NewCNSPodInfoProvider(httpRestServiceImplementation.EndpointStateStore) // get reference to endpoint state store from rest server
+		podInfoByIPProvider, err = InitializeStateFromCNS(cnsconfig, httpRestServiceImplementation.EndpointStateStore)
 		if err != nil {
-			if errors.Is(err, store.ErrKeyNotFound) {
-				logger.Printf("[Azure CNS] No endpoint state found, skipping initializing CNS state")
-			} else {
-				return errors.Wrap(err, "failed to create CNS PodInfoProvider")
-			}
+			return errors.Wrap(err, "failed to create CNI PodInfoProvider")
 		}
+
 	case cnsconfig.InitializeFromCNI:
 		logger.Printf("Initializing from CNI")
-		podInfoByIPProvider, err = cnireconciler.NewCNIPodInfoProvider()
+		podInfoByIPProvider, _, err = cnireconciler.NewCNIPodInfoProvider(false)
 		if err != nil {
 			return errors.Wrap(err, "failed to create CNI PodInfoProvider")
 		}
@@ -1516,3 +1513,30 @@ func createOrUpdateNodeInfoCRD(ctx context.Context, restConfig *rest.Config, nod
 
 	return nil
 }
+
+// InitializeStateFromCNS initilizes CNS Endpoint State from CNS or perform the Migration of state from statefull CNI.
+func InitializeStateFromCNS(cnsconfig *configuration.CNSConfig, endpointStateStore store.KeyValueStore) (cns.PodInfoByIPProvider, error) {
+	logger.Printf("Initializing from self managed endpoint store")
+	podInfoByIPProvider, err := cnireconciler.NewCNSPodInfoProvider(endpointStateStore) // get reference to endpoint state store from rest server
+	if err != nil {
+		if errors.Is(err, store.ErrKeyNotFound) {
+			logger.Printf("[Azure CNS] No endpoint state found, skipping initializing CNS state")
+			if cnsconfig.StateMigration {
+				logger.Printf("StatelessCNI Migration is enabled")
+				logger.Printf("initializing from Statefull CNI")
+				var endpointState map[string]*restserver.EndpointInfo
+				podInfoByIPProvider, endpointState, err = cnireconciler.NewCNIPodInfoProvider(cnsconfig.StateMigration)
+				if err != nil {
+					return nil, errors.Wrap(err, "failed to create CNI PodInfoProvider")
+				}
+				err = endpointStateStore.Write(restserver.EndpointStoreKey, endpointState)
+				if err != nil {
+					return nil, fmt.Errorf("failed to write endpoint state to store: %w", err)
+				}
+			}
+		} else {
+			return nil, errors.Wrap(err, "failed to create CNS PodInfoProvider")
+		}
+	}
+	return podInfoByIPProvider, nil
+}

network/endpoint.go

@@ -92,6 +92,7 @@ type EndpointInfo struct {
 	NICType                  cns.NICType
 	SkipDefaultRoutes        bool
 	HNSEndpointID            string
+	HostIfName               string
 }
 
 // RouteInfo contains information about an IP route.
@@ -267,6 +268,8 @@ func (ep *endpoint) getInfo() *EndpointInfo {
 		PODName:                  ep.PODName,
 		PODNameSpace:             ep.PODNameSpace,
 		NetworkContainerID:       ep.NetworkContainerID,
+		HNSEndpointID:            ep.HnsId,
+		HostIfName:               ep.HostIfName,
 	}
 
 	info.Routes = append(info.Routes, ep.Routes...)
@@ -350,3 +353,21 @@ func GetPodNameWithoutSuffix(podName string) string {
 	logger.Info("Pod name after splitting based on", zap.Any("nameSplit", nameSplit))
 	return strings.Join(nameSplit, "-")
 }
+
+// GetEndpointInfoByIP returns an EndpointInfo with complete state when HNS Endpoint ID or HostVeth Name are missing on the CNS state.
+func (epInfo *EndpointInfo) GetEndpointInfoByIP(ipAddresses []net.IPNet, networkID string) (*EndpointInfo, error) {
+	// Call the platform implementation.
+	endpointInfo, err := epInfo.GetEndpointInfoByIPImpl(ipAddresses, networkID)
+	if err != nil {
+		return nil, err
+	}
+	return endpointInfo, nil
+}
+
+// IsEndpointStateInComplete returns true if both HNSEndpointID and HostVethName are missing.
+func (epInfo *EndpointInfo) IsEndpointStateIncomplete() bool {
+	if epInfo.HNSEndpointID == "" && epInfo.IfName == "" {
+		return true
+	}
+	return false
+}

network/endpoint_linux.go

@@ -531,3 +531,9 @@ func getDefaultGateway(routes []RouteInfo) net.IP {
 
 	return nil
 }
+
+// GetEndpointInfoByIPImpl returns an endpointInfo that contains corresponding HostVethName.
+// TODO: It needs to be tested to see if HostVethName is required for SingleTenancy, WorkItem: 26606939
+func (epInfo *EndpointInfo) GetEndpointInfoByIPImpl(_ []net.IPNet, _ string) (*EndpointInfo, error) {
+	return epInfo, nil
+}

network/endpoint_windows.go

@@ -15,6 +15,7 @@ import (
 	"github.com/Azure/azure-container-networking/platform"
 	"github.com/Microsoft/hcsshim"
 	"github.com/Microsoft/hcsshim/hcn"
+	"github.com/pkg/errors"
 	"go.uber.org/zap"
 )
 
@@ -495,3 +496,28 @@ func (ep *endpoint) getInfoImpl(epInfo *EndpointInfo) {
 func (nm *networkManager) updateEndpointImpl(nw *network, existingEpInfo *EndpointInfo, targetEpInfo *EndpointInfo) (*endpoint, error) {
 	return nil, nil
 }
+
+// GetEndpointInfoByIPImpl returns an endpointInfo with the corrsponding HNS Endpoint ID that matches an specific IP Address.
+func (epInfo *EndpointInfo) GetEndpointInfoByIPImpl(ipAddresses []net.IPNet, networkID string) (*EndpointInfo, error) {
+	// check if network exists, only create the network does not exist
+	hnsResponse, err := Hnsv2.GetNetworkByName(networkID)
+	if err != nil {
+		return epInfo, errors.Wrapf(err, "HNS Network not found")
+	}
+	hcnEndpoints, err := Hnsv2.ListEndpointsOfNetwork(hnsResponse.Id)
+	if err != nil {
+		return epInfo, errors.Wrapf(err, "failed to fetch HNS endpoints for the given network")
+	}
+	for i := range hcnEndpoints {
+		for _, ipConfiguration := range hcnEndpoints[i].IpConfigurations {
+			for _, ipAddress := range ipAddresses {
+				prefixLength, _ := ipAddress.Mask.Size()
+				if ipConfiguration.IpAddress == ipAddress.IP.String() && ipConfiguration.PrefixLength == uint8(prefixLength) {
+					epInfo.HNSEndpointID = hcnEndpoints[i].Id
+					return epInfo, nil
+				}
+			}
+		}
+	}
+	return epInfo, errors.Wrapf(err, "No HNSEndpointID matches the IPAddress: "+ipAddresses[0].IP.String())
+}