From 763ff05b1560038a7d39cc6fdb94d0f4f8a93ab6 Mon Sep 17 00:00:00 2001 From: Vipul Singh Date: Tue, 3 Oct 2023 09:20:57 -0700 Subject: [PATCH 01/17] =?UTF-8?q?=F0=9F=8C=88=20feat:=20adding=20flags=20f?= =?UTF-8?q?or=20stateless=20cni=20(#2103)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feat: stateless cni --- Makefile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index dc85d2b2a2..97a9739128 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,7 @@ ZAPAI_VERSION ?= $(notdir $(shell git describe --match "zapai*" --tags --alway AZURE_IPAM_DIR = $(REPO_ROOT)/azure-ipam CNM_DIR = $(REPO_ROOT)/cnm/plugin CNI_NET_DIR = $(REPO_ROOT)/cni/network/plugin +STATELESS_CNI_NET_DIR = $(REPO_ROOT)/cni/network/stateless CNI_IPAM_DIR = $(REPO_ROOT)/cni/ipam/plugin CNI_IPAMV6_DIR = $(REPO_ROOT)/cni/ipam/pluginv6 CNI_TELEMETRY_DIR = $(REPO_ROOT)/cni/telemetry/service @@ -58,11 +59,13 @@ AZURE_IPAM_BUILD_DIR = $(BUILD_DIR)/azure-ipam IMAGE_DIR = $(OUTPUT_DIR)/images CNM_BUILD_DIR = $(BUILD_DIR)/cnm CNI_BUILD_DIR = $(BUILD_DIR)/cni +STATELESS_CNI_BUILD_DIR = $(CNI_BUILD_DIR)/stateless ACNCLI_BUILD_DIR = $(BUILD_DIR)/acncli CNI_MULTITENANCY_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy-transparent-vlan CNI_SWIFT_BUILD_DIR = $(BUILD_DIR)/cni-swift CNI_OVERLAY_BUILD_DIR = $(BUILD_DIR)/cni-overlay +STATELESS_CNI_OVERLAY_BUILD_DIR = $(CNI_OVERLAY_BUILD_DIR)/stateless CNI_BAREMETAL_BUILD_DIR = $(BUILD_DIR)/cni-baremetal CNI_DUALSTACK_BUILD_DIR = $(BUILD_DIR)/cni-dualstack CNS_BUILD_DIR = $(BUILD_DIR)/cns @@ -130,7 +133,7 @@ endif # Shorthand target names for convenience. azure-cnm-plugin: cnm-binary cnm-archive -azure-cni-plugin: azure-vnet-binary azure-vnet-ipam-binary azure-vnet-ipamv6-binary azure-vnet-telemetry-binary cni-archive +azure-cni-plugin: azure-vnet-binary azure-vnet-stateless-binary azure-vnet-ipam-binary azure-vnet-ipamv6-binary azure-vnet-telemetry-binary cni-archive azure-cns: azure-cns-binary cns-archive acncli: acncli-binary acncli-archive azure-cnms: azure-cnms-binary cnms-archive @@ -180,6 +183,10 @@ cnm-binary: azure-vnet-binary: cd $(CNI_NET_DIR) && CGO_ENABLED=0 go build -v -o $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) -ldflags "-X main.version=$(CNI_VERSION)" -gcflags="-dwarflocationlists=true" +# Build the Azure CNI stateless network binary +azure-vnet-stateless-binary: + cd $(STATELESS_CNI_NET_DIR) && CGO_ENABLED=0 go build -v -o $(STATELESS_CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) -ldflags "-X main.version=$(CNI_VERSION)" -gcflags="-dwarflocationlists=true" + # Build the Azure CNI IPAM binary. azure-vnet-ipam-binary: cd $(CNI_IPAM_DIR) && CGO_ENABLED=0 go build -v -o $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) -ldflags "-X main.version=$(CNI_VERSION)" -gcflags="-dwarflocationlists=true" @@ -681,6 +688,8 @@ endif cp cni/azure-$(GOOS)-swift-overlay.conflist $(CNI_OVERLAY_BUILD_DIR)/10-azure.conflist cp telemetry/azure-vnet-telemetry.config $(CNI_OVERLAY_BUILD_DIR)/azure-vnet-telemetry.config cp $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) $(CNI_OVERLAY_BUILD_DIR) + $(MKDIR) $(STATELESS_CNI_OVERLAY_BUILD_DIR) + cp $(STATELESS_CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(STATELESS_CNI_OVERLAY_BUILD_DIR) cd $(CNI_OVERLAY_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_OVERLAY_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config $(MKDIR) $(CNI_DUALSTACK_BUILD_DIR) From 16f149c99929aa99058e14b7faebed4b73d0fa77 Mon Sep 17 00:00:00 2001 From: Vipul Singh Date: Thu, 5 Oct 2023 16:57:23 -0700 Subject: [PATCH 02/17] feat: create stateless cni binary for swift (#2275) --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 97a9739128..2cb38265c4 100644 --- a/Makefile +++ b/Makefile @@ -64,6 +64,7 @@ ACNCLI_BUILD_DIR = $(BUILD_DIR)/acncli CNI_MULTITENANCY_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy-transparent-vlan CNI_SWIFT_BUILD_DIR = $(BUILD_DIR)/cni-swift +STATELESS_CNI_SWIFT_BUILD_DIR = $(CNI_SWIFT_BUILD_DIR)/stateless CNI_OVERLAY_BUILD_DIR = $(BUILD_DIR)/cni-overlay STATELESS_CNI_OVERLAY_BUILD_DIR = $(CNI_OVERLAY_BUILD_DIR)/stateless CNI_BAREMETAL_BUILD_DIR = $(BUILD_DIR)/cni-baremetal @@ -682,6 +683,8 @@ endif cp cni/azure-$(GOOS)-swift.conflist $(CNI_SWIFT_BUILD_DIR)/10-azure.conflist cp telemetry/azure-vnet-telemetry.config $(CNI_SWIFT_BUILD_DIR)/azure-vnet-telemetry.config cp $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) $(CNI_SWIFT_BUILD_DIR) + $(MKDIR) $(STATELESS_CNI_SWIFT_BUILD_DIR) + cp $(STATELESS_CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(STATELESS_CNI_SWIFT_BUILD_DIR) cd $(CNI_SWIFT_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_SWIFT_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config $(MKDIR) $(CNI_OVERLAY_BUILD_DIR) From 0f8b1ca8ed1b6f814f08fe274f67e5ecbe6a45fe Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Wed, 18 Oct 2023 12:03:18 -0700 Subject: [PATCH 03/17] enabling CNS telemetry --- cns/service/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cns/service/main.go b/cns/service/main.go index 405ff9f16d..3ec2ea204d 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -317,7 +317,7 @@ var args = acn.ArgumentList{ Shorthand: acn.OptTelemetryServiceAlias, Description: "Flag to start telemetry service to receive telemetry events from CNI. Default, disabled.", Type: "bool", - DefaultValue: false, + DefaultValue: true, }, { Name: acn.OptCNIConflistFilepath, From c45fe63cee7bbc3cf64f2d439d8dcc64bb7e581e Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Wed, 18 Oct 2023 16:32:27 -0700 Subject: [PATCH 04/17] CNI Telemetry enabled on CNS --- cns/service/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cns/service/main.go b/cns/service/main.go index 3ec2ea204d..405ff9f16d 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -317,7 +317,7 @@ var args = acn.ArgumentList{ Shorthand: acn.OptTelemetryServiceAlias, Description: "Flag to start telemetry service to receive telemetry events from CNI. Default, disabled.", Type: "bool", - DefaultValue: true, + DefaultValue: false, }, { Name: acn.OptCNIConflistFilepath, From 2be34d0ce7d78c0a070953350125aad93289b1c4 Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Tue, 7 Nov 2023 14:14:39 -0800 Subject: [PATCH 05/17] Code changes for Statefull CNI Migration --- cni/client/client.go | 4 ++-- cni/client/client_integration_test.go | 5 +++-- cni/client/client_unit_test.go | 3 ++- cns/cnireconciler/podinfoprovider.go | 11 ++++++++--- cns/service/main.go | 9 +++++++++ network/endpoint_linux.go | 6 ++++++ network/endpoint_windows.go | 25 +++++++++++++++++++++++++ network/manager.go | 27 ++++++++++++++++++++++++--- platform/os_linux.go | 2 ++ platform/os_windows.go | 3 +++ 10 files changed, 84 insertions(+), 11 deletions(-) diff --git a/cni/client/client.go b/cni/client/client.go index 27ba4b744b..4c5bf5155f 100644 --- a/cni/client/client.go +++ b/cni/client/client.go @@ -28,8 +28,8 @@ func New(exec utilexec.Interface) *client { return &client{exec: exec} } -func (c *client) GetEndpointState() (*api.AzureCNIState, error) { - cmd := c.exec.Command(platform.CNIBinaryPath) +func (c *client) GetEndpointState(cniBinaryPath string) (*api.AzureCNIState, error) { + cmd := c.exec.Command(cniBinaryPath) cmd.SetDir(CNIExecDir) envs := os.Environ() cmdenv := fmt.Sprintf("%s=%s", cni.Cmd, cni.CmdGetEndpointsState) diff --git a/cni/client/client_integration_test.go b/cni/client/client_integration_test.go index c204eb8e43..88c83fcbf1 100644 --- a/cni/client/client_integration_test.go +++ b/cni/client/client_integration_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/Azure/azure-container-networking/cni/api" + "github.com/Azure/azure-container-networking/platform" testutils "github.com/Azure/azure-container-networking/test/utils" ver "github.com/hashicorp/go-version" "github.com/stretchr/testify/require" @@ -62,7 +63,7 @@ func TestMain(m *testing.M) { // todo: enable this test in CI, requires built azure vnet func TestGetStateFromAzureCNI(t *testing.T) { c := New(exec.New()) - state, err := c.GetEndpointState() + state, err := c.GetEndpointState(platform.CNIBinaryPath) require.NoError(t, err) res := &api.AzureCNIState{ @@ -101,7 +102,7 @@ func TestGetStateWithEmptyStateFile(t *testing.T) { require.NoError(t, err) out.Close() - state, err := c.GetEndpointState() + state, err := c.GetEndpointState(platform.CNIBinaryPath) require.NoError(t, err) require.Exactly(t, 0, len(state.ContainerInterfaces)) } diff --git a/cni/client/client_unit_test.go b/cni/client/client_unit_test.go index 94d0603066..8a49758432 100644 --- a/cni/client/client_unit_test.go +++ b/cni/client/client_unit_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/Azure/azure-container-networking/cni/api" + "github.com/Azure/azure-container-networking/platform" testutils "github.com/Azure/azure-container-networking/test/utils" ver "github.com/hashicorp/go-version" "github.com/stretchr/testify/require" @@ -20,7 +21,7 @@ func TestGetState(t *testing.T) { fakeexec := testutils.GetFakeExecWithScripts(calls) c := New(fakeexec) - state, err := c.GetEndpointState() + state, err := c.GetEndpointState(platform.CNIBinaryPath) require.NoError(t, err) res := &api.AzureCNIState{ diff --git a/cns/cnireconciler/podinfoprovider.go b/cns/cnireconciler/podinfoprovider.go index fd237ebb66..cd39c99973 100644 --- a/cns/cnireconciler/podinfoprovider.go +++ b/cns/cnireconciler/podinfoprovider.go @@ -7,6 +7,7 @@ import ( "github.com/Azure/azure-container-networking/cni/client" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/restserver" + "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/store" "github.com/pkg/errors" "k8s.io/utils/exec" @@ -15,7 +16,11 @@ import ( // NewCNIPodInfoProvider returns an implementation of cns.PodInfoByIPProvider // that execs out to the CNI and uses the response to build the PodInfo map. func NewCNIPodInfoProvider() (cns.PodInfoByIPProvider, error) { - return newCNIPodInfoProvider(exec.New()) + return newCNIPodInfoProvider(exec.New(), platform.CNIBinaryPath) +} + +func NewStatefullCNIPodInfoProvider() (cns.PodInfoByIPProvider, error) { + return newCNIPodInfoProvider(exec.New(), platform.StatefullCNIBinaryPath) } func NewCNSPodInfoProvider(endpointStore store.KeyValueStore) (cns.PodInfoByIPProvider, error) { @@ -39,9 +44,9 @@ func newCNSPodInfoProvider(endpointStore store.KeyValueStore) (cns.PodInfoByIPPr }), nil } -func newCNIPodInfoProvider(exec exec.Interface) (cns.PodInfoByIPProvider, error) { +func newCNIPodInfoProvider(exec exec.Interface, cniBinaryPath string) (cns.PodInfoByIPProvider, error) { cli := client.New(exec) - state, err := cli.GetEndpointState() + state, err := cli.GetEndpointState(cniBinaryPath) if err != nil { return nil, fmt.Errorf("failed to invoke CNI client.GetEndpointState(): %w", err) } diff --git a/cns/service/main.go b/cns/service/main.go index 405ff9f16d..a6d50483d7 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1238,10 +1238,19 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn if err != nil { if errors.Is(err, store.ErrKeyNotFound) { logger.Printf("[Azure CNS] No endpoint state found, skipping initializing CNS state") + if os.Getenv("StatelessCNIMigration") == "true" { + logger.Printf("StatelessCNI Migration is enabled") + logger.Printf("Initializing from Statefull CNI") + podInfoByIPProvider, err = cnireconciler.NewStatefullCNIPodInfoProvider() + if err != nil { + return errors.Wrap(err, "failed to create CNI PodInfoProvider") + } + } } else { return errors.Wrap(err, "failed to create CNS PodInfoProvider") } } + case cnsconfig.InitializeFromCNI: logger.Printf("Initializing from CNI") podInfoByIPProvider, err = cnireconciler.NewCNIPodInfoProvider() diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index f603f389fb..b8880d423b 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -6,6 +6,7 @@ package network import ( "crypto/sha1" "encoding/hex" + "errors" "fmt" "net" "strings" @@ -531,3 +532,8 @@ func getDefaultGateway(routes []RouteInfo) net.IP { return nil } + +// getHNSEndpointIdByIP returns an HNS Endpoint IP that matches an specific IPAddress. +func (nm *networkManager) GetEndpointInfoByIPImpl(ipAddresses []net.IPNet, _ string) (string, error) { + return "", errors.New("No HostVethName matches the IPAddress: " + ipAddresses[0].IP.String()) +} diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 244ab6f234..a45e324780 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -5,6 +5,7 @@ package network import ( "context" + "errors" "fmt" "net" "strings" @@ -495,3 +496,27 @@ func (ep *endpoint) getInfoImpl(epInfo *EndpointInfo) { func (nm *networkManager) updateEndpointImpl(nw *network, existingEpInfo *EndpointInfo, targetEpInfo *EndpointInfo) (*endpoint, error) { return nil, nil } + +// getHNSEndpointIdByIP returns an HNS Endpoint IP that matches an specific IPAddress. +func (nm *networkManager) GetEndpointInfoByIPImpl(ipAddresses []net.IPNet, networkId string) (string, error) { + // check if network exists, only create the network does not exist + hnsResponse, err := Hnsv2.GetNetworkByName(networkId) + if err != nil { + return "", err + } + hcnEndpoints, err := Hnsv2.ListEndpointsOfNetwork(hnsResponse.Id) + if err != nil { + return "", err + } + for _, hcnEndpoint := range hcnEndpoints { + for _, ipConfiguration := range hcnEndpoint.IpConfigurations { + for _, ipAddress := range ipAddresses { + prefixLength, _ := ipAddress.Mask.Size() + if ipConfiguration.IpAddress == ipAddress.IP.String() && ipConfiguration.PrefixLength == uint8(prefixLength) { + return hcnEndpoint.Id, nil + } + } + } + } + return "", errors.New("No HNSEndpointID matches the IPAddress: " + ipAddresses[0].IP.String()) +} diff --git a/network/manager.go b/network/manager.go index 65e45c2440..3835ae2739 100644 --- a/network/manager.go +++ b/network/manager.go @@ -6,6 +6,7 @@ package network import ( "context" "net" + "runtime" "sync" "time" @@ -102,6 +103,7 @@ type NetworkManager interface { CreateEndpoint(client apipaClient, networkID string, epInfo []*EndpointInfo) error DeleteEndpoint(networkID string, endpointID string, epInfo *EndpointInfo) error GetEndpointInfo(networkID string, endpointID string) (*EndpointInfo, error) + GetEndpointInfoByIP(ipAddresses []net.IPNet, networkId string) (string, error) GetAllEndpoints(networkID string) (map[string]*EndpointInfo, error) GetEndpointInfoBasedOnPODDetails(networkID string, podName string, podNameSpace string, doExactMatchForPodName bool) (*EndpointInfo, error) AttachEndpoint(networkID string, endpointID string, sandboxKey string) (*endpoint, error) @@ -485,9 +487,6 @@ func (nm *networkManager) GetEndpointInfo(networkId string, endpointId string) ( if err != nil { return nil, errors.Wrapf(err, "Get endpoint API returend with error") } - if endpointResponse.EndpointInfo.HnsEndpointID == "" && endpointResponse.EndpointInfo.HostVethName == "" { - return nil, errors.New("Get endpoint API returend with empty HNSEndpointID and HostVethName") - } epInfo := &EndpointInfo{ Id: endpointId, IfIndex: EndpointIfIndex, // Azure CNI supports only one interface @@ -504,6 +503,17 @@ func (nm *networkManager) GetEndpointInfo(networkId string, endpointId string) ( epInfo.IPAddresses = append(epInfo.IPAddresses, ip.IPv6...) } + if epInfo.HNSEndpointID == "" && epInfo.IfName == "" { + endpointInfoData, err := nm.GetEndpointInfoByIP(epInfo.IPAddresses, networkId) + if err != nil { + return nil, errors.Wrapf(err, "Get endpoint API returend with error") + } + if runtime.GOOS == "windows" { + epInfo.HNSEndpointID = endpointInfoData + } else { + epInfo.IfName = endpointInfoData + } + } logger.Info("returning getEndpoint API with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", epInfo.HNSEndpointID)) return epInfo, nil } @@ -680,4 +690,15 @@ func (nm *networkManager) GetEndpointID(containerID, ifName string) string { return "" } return containerID + "-" + ifName + +} + +// GetEndpointID returns a unique endpoint ID based on the CNI mode. +func (nm *networkManager) GetEndpointInfoByIP(ipAddresses []net.IPNet, networkId string) (string, error) { + // Call the platform implementation. + endpointData, err := nm.GetEndpointInfoByIPImpl(ipAddresses, networkId) + if err != nil { + return "", err + } + return endpointData, nil } diff --git a/platform/os_linux.go b/platform/os_linux.go index 0b29bb2868..9a8c8eb2a9 100644 --- a/platform/os_linux.go +++ b/platform/os_linux.go @@ -29,6 +29,8 @@ const ( CNIIpamStatePath = "/var/run/azure-vnet-ipam.json" // CNIBinaryPath is the path to the CNI binary CNIBinaryPath = "/opt/cni/bin/azure-vnet" + // StatefullCNIBinaryPath is the path to the Statefull CNI binary + StatefullCNIBinaryPath = "/opt/cni/bin/azure-vnet-statefull" // CNSRuntimePath is the path where CNS state files are stored. CNSRuntimePath = "/var/run/" // CNI runtime path on a Kubernetes cluster diff --git a/platform/os_windows.go b/platform/os_windows.go index e4de7b8dcd..8dd4cdcfb7 100644 --- a/platform/os_windows.go +++ b/platform/os_windows.go @@ -40,6 +40,9 @@ const ( // CNIBinaryPath is the path to the CNI binary CNIBinaryPath = "C:\\k\\azurecni\\bin\\azure-vnet.exe" + // StatefullCNIBinaryPath is the path to the Statefull CNI binary + StatefullCNIBinaryPath = "C:\\k\\azurecni\\bin\\azure-vnet-statefull.exe" + // CNI runtime path on a Kubernetes cluster K8SCNIRuntimePath = "C:\\k\\azurecni\\bin" From 153b5873bda5a20b36673f508cc64a491fadbfa2 Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Tue, 28 Nov 2023 15:29:52 -0800 Subject: [PATCH 06/17] Making changes to the CNI state migration code. --- cni/client/client.go | 4 ++-- cni/client/client_integration_test.go | 5 ++--- cni/client/client_unit_test.go | 3 +-- cns/cnireconciler/podinfoprovider.go | 11 +++-------- cns/service/main.go | 2 +- network/endpoint.go | 18 ++++++++++++++++++ network/endpoint_linux.go | 8 ++++---- network/endpoint_windows.go | 15 ++++++++------- network/manager.go | 22 ++-------------------- platform/os_linux.go | 2 -- platform/os_windows.go | 3 --- 11 files changed, 41 insertions(+), 52 deletions(-) diff --git a/cni/client/client.go b/cni/client/client.go index 4c5bf5155f..27ba4b744b 100644 --- a/cni/client/client.go +++ b/cni/client/client.go @@ -28,8 +28,8 @@ func New(exec utilexec.Interface) *client { return &client{exec: exec} } -func (c *client) GetEndpointState(cniBinaryPath string) (*api.AzureCNIState, error) { - cmd := c.exec.Command(cniBinaryPath) +func (c *client) GetEndpointState() (*api.AzureCNIState, error) { + cmd := c.exec.Command(platform.CNIBinaryPath) cmd.SetDir(CNIExecDir) envs := os.Environ() cmdenv := fmt.Sprintf("%s=%s", cni.Cmd, cni.CmdGetEndpointsState) diff --git a/cni/client/client_integration_test.go b/cni/client/client_integration_test.go index 88c83fcbf1..c204eb8e43 100644 --- a/cni/client/client_integration_test.go +++ b/cni/client/client_integration_test.go @@ -10,7 +10,6 @@ import ( "testing" "github.com/Azure/azure-container-networking/cni/api" - "github.com/Azure/azure-container-networking/platform" testutils "github.com/Azure/azure-container-networking/test/utils" ver "github.com/hashicorp/go-version" "github.com/stretchr/testify/require" @@ -63,7 +62,7 @@ func TestMain(m *testing.M) { // todo: enable this test in CI, requires built azure vnet func TestGetStateFromAzureCNI(t *testing.T) { c := New(exec.New()) - state, err := c.GetEndpointState(platform.CNIBinaryPath) + state, err := c.GetEndpointState() require.NoError(t, err) res := &api.AzureCNIState{ @@ -102,7 +101,7 @@ func TestGetStateWithEmptyStateFile(t *testing.T) { require.NoError(t, err) out.Close() - state, err := c.GetEndpointState(platform.CNIBinaryPath) + state, err := c.GetEndpointState() require.NoError(t, err) require.Exactly(t, 0, len(state.ContainerInterfaces)) } diff --git a/cni/client/client_unit_test.go b/cni/client/client_unit_test.go index 8a49758432..94d0603066 100644 --- a/cni/client/client_unit_test.go +++ b/cni/client/client_unit_test.go @@ -7,7 +7,6 @@ import ( "testing" "github.com/Azure/azure-container-networking/cni/api" - "github.com/Azure/azure-container-networking/platform" testutils "github.com/Azure/azure-container-networking/test/utils" ver "github.com/hashicorp/go-version" "github.com/stretchr/testify/require" @@ -21,7 +20,7 @@ func TestGetState(t *testing.T) { fakeexec := testutils.GetFakeExecWithScripts(calls) c := New(fakeexec) - state, err := c.GetEndpointState(platform.CNIBinaryPath) + state, err := c.GetEndpointState() require.NoError(t, err) res := &api.AzureCNIState{ diff --git a/cns/cnireconciler/podinfoprovider.go b/cns/cnireconciler/podinfoprovider.go index cd39c99973..fd237ebb66 100644 --- a/cns/cnireconciler/podinfoprovider.go +++ b/cns/cnireconciler/podinfoprovider.go @@ -7,7 +7,6 @@ import ( "github.com/Azure/azure-container-networking/cni/client" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/restserver" - "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/store" "github.com/pkg/errors" "k8s.io/utils/exec" @@ -16,11 +15,7 @@ import ( // NewCNIPodInfoProvider returns an implementation of cns.PodInfoByIPProvider // that execs out to the CNI and uses the response to build the PodInfo map. func NewCNIPodInfoProvider() (cns.PodInfoByIPProvider, error) { - return newCNIPodInfoProvider(exec.New(), platform.CNIBinaryPath) -} - -func NewStatefullCNIPodInfoProvider() (cns.PodInfoByIPProvider, error) { - return newCNIPodInfoProvider(exec.New(), platform.StatefullCNIBinaryPath) + return newCNIPodInfoProvider(exec.New()) } func NewCNSPodInfoProvider(endpointStore store.KeyValueStore) (cns.PodInfoByIPProvider, error) { @@ -44,9 +39,9 @@ func newCNSPodInfoProvider(endpointStore store.KeyValueStore) (cns.PodInfoByIPPr }), nil } -func newCNIPodInfoProvider(exec exec.Interface, cniBinaryPath string) (cns.PodInfoByIPProvider, error) { +func newCNIPodInfoProvider(exec exec.Interface) (cns.PodInfoByIPProvider, error) { cli := client.New(exec) - state, err := cli.GetEndpointState(cniBinaryPath) + state, err := cli.GetEndpointState() if err != nil { return nil, fmt.Errorf("failed to invoke CNI client.GetEndpointState(): %w", err) } diff --git a/cns/service/main.go b/cns/service/main.go index a6d50483d7..9044d19e86 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1241,7 +1241,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn if os.Getenv("StatelessCNIMigration") == "true" { logger.Printf("StatelessCNI Migration is enabled") logger.Printf("Initializing from Statefull CNI") - podInfoByIPProvider, err = cnireconciler.NewStatefullCNIPodInfoProvider() + podInfoByIPProvider, err = cnireconciler.NewCNIPodInfoProvider() if err != nil { return errors.Wrap(err, "failed to create CNI PodInfoProvider") } diff --git a/network/endpoint.go b/network/endpoint.go index 0adf857ea9..518aa804f3 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -350,3 +350,21 @@ func GetPodNameWithoutSuffix(podName string) string { logger.Info("Pod name after splitting based on", zap.Any("nameSplit", nameSplit)) return strings.Join(nameSplit, "-") } + +// GetEndpointInfoByIP returns an EndpointInfo with complete state when HNS Endpoint ID or HostVeth Name are missing on the CNS state. +func (epInfo *EndpointInfo) GetEndpointInfoByIP(ipAddresses []net.IPNet, networkID string) (*EndpointInfo, error) { + // Call the platform implementation. + endpointInfo, err := epInfo.GetEndpointInfoByIPImpl(ipAddresses, networkID) + if err != nil { + return nil, err + } + return endpointInfo, nil +} + +// IsEndpointStateComplete returns true if both HNSEndpointID and HostVethName are missing. +func (epInfo *EndpointInfo) IsEndpointStateIncomplete() bool { + if epInfo.HNSEndpointID == "" && epInfo.IfName == "" { + return true + } + return false +} diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index b8880d423b..766d432603 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -6,7 +6,6 @@ package network import ( "crypto/sha1" "encoding/hex" - "errors" "fmt" "net" "strings" @@ -533,7 +532,8 @@ func getDefaultGateway(routes []RouteInfo) net.IP { return nil } -// getHNSEndpointIdByIP returns an HNS Endpoint IP that matches an specific IPAddress. -func (nm *networkManager) GetEndpointInfoByIPImpl(ipAddresses []net.IPNet, _ string) (string, error) { - return "", errors.New("No HostVethName matches the IPAddress: " + ipAddresses[0].IP.String()) +// GetEndpointInfoByIPImpl returns an endpointInfo that contains corresponding HostVethName. +// TODO: It needs to be tested to see if HostVethName is required for SingleTenancy +func (epInfo *EndpointInfo) GetEndpointInfoByIPImpl(_ []net.IPNet, _ string) (*EndpointInfo, error) { + return epInfo, nil } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index a45e324780..b859328880 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -497,26 +497,27 @@ func (nm *networkManager) updateEndpointImpl(nw *network, existingEpInfo *Endpoi return nil, nil } -// getHNSEndpointIdByIP returns an HNS Endpoint IP that matches an specific IPAddress. -func (nm *networkManager) GetEndpointInfoByIPImpl(ipAddresses []net.IPNet, networkId string) (string, error) { +// GetEndpointInfoByIPImpl returns an endpointInfo with the corrsponding HNS Endpoint ID that matches an specific IPAddress. +func (epInfo *EndpointInfo) GetEndpointInfoByIPImpl(ipAddresses []net.IPNet, networkID string) (*EndpointInfo, error) { // check if network exists, only create the network does not exist - hnsResponse, err := Hnsv2.GetNetworkByName(networkId) + hnsResponse, err := Hnsv2.GetNetworkByName(networkID) if err != nil { - return "", err + return epInfo, err } hcnEndpoints, err := Hnsv2.ListEndpointsOfNetwork(hnsResponse.Id) if err != nil { - return "", err + return epInfo, err } for _, hcnEndpoint := range hcnEndpoints { for _, ipConfiguration := range hcnEndpoint.IpConfigurations { for _, ipAddress := range ipAddresses { prefixLength, _ := ipAddress.Mask.Size() if ipConfiguration.IpAddress == ipAddress.IP.String() && ipConfiguration.PrefixLength == uint8(prefixLength) { - return hcnEndpoint.Id, nil + epInfo.HNSEndpointID = hcnEndpoint.Id + return epInfo, nil } } } } - return "", errors.New("No HNSEndpointID matches the IPAddress: " + ipAddresses[0].IP.String()) + return epInfo, errors.New("No HNSEndpointID matches the IPAddress: " + ipAddresses[0].IP.String()) } diff --git a/network/manager.go b/network/manager.go index 3835ae2739..d233506fb7 100644 --- a/network/manager.go +++ b/network/manager.go @@ -6,7 +6,6 @@ package network import ( "context" "net" - "runtime" "sync" "time" @@ -103,7 +102,6 @@ type NetworkManager interface { CreateEndpoint(client apipaClient, networkID string, epInfo []*EndpointInfo) error DeleteEndpoint(networkID string, endpointID string, epInfo *EndpointInfo) error GetEndpointInfo(networkID string, endpointID string) (*EndpointInfo, error) - GetEndpointInfoByIP(ipAddresses []net.IPNet, networkId string) (string, error) GetAllEndpoints(networkID string) (map[string]*EndpointInfo, error) GetEndpointInfoBasedOnPODDetails(networkID string, podName string, podNameSpace string, doExactMatchForPodName bool) (*EndpointInfo, error) AttachEndpoint(networkID string, endpointID string, sandboxKey string) (*endpoint, error) @@ -503,16 +501,11 @@ func (nm *networkManager) GetEndpointInfo(networkId string, endpointId string) ( epInfo.IPAddresses = append(epInfo.IPAddresses, ip.IPv6...) } - if epInfo.HNSEndpointID == "" && epInfo.IfName == "" { - endpointInfoData, err := nm.GetEndpointInfoByIP(epInfo.IPAddresses, networkId) + if epInfo.IsEndpointStateIncomplete() { + epInfo, err = epInfo.GetEndpointInfoByIP(epInfo.IPAddresses, networkId) if err != nil { return nil, errors.Wrapf(err, "Get endpoint API returend with error") } - if runtime.GOOS == "windows" { - epInfo.HNSEndpointID = endpointInfoData - } else { - epInfo.IfName = endpointInfoData - } } logger.Info("returning getEndpoint API with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", epInfo.HNSEndpointID)) return epInfo, nil @@ -690,15 +683,4 @@ func (nm *networkManager) GetEndpointID(containerID, ifName string) string { return "" } return containerID + "-" + ifName - -} - -// GetEndpointID returns a unique endpoint ID based on the CNI mode. -func (nm *networkManager) GetEndpointInfoByIP(ipAddresses []net.IPNet, networkId string) (string, error) { - // Call the platform implementation. - endpointData, err := nm.GetEndpointInfoByIPImpl(ipAddresses, networkId) - if err != nil { - return "", err - } - return endpointData, nil } diff --git a/platform/os_linux.go b/platform/os_linux.go index 9a8c8eb2a9..0b29bb2868 100644 --- a/platform/os_linux.go +++ b/platform/os_linux.go @@ -29,8 +29,6 @@ const ( CNIIpamStatePath = "/var/run/azure-vnet-ipam.json" // CNIBinaryPath is the path to the CNI binary CNIBinaryPath = "/opt/cni/bin/azure-vnet" - // StatefullCNIBinaryPath is the path to the Statefull CNI binary - StatefullCNIBinaryPath = "/opt/cni/bin/azure-vnet-statefull" // CNSRuntimePath is the path where CNS state files are stored. CNSRuntimePath = "/var/run/" // CNI runtime path on a Kubernetes cluster diff --git a/platform/os_windows.go b/platform/os_windows.go index 8dd4cdcfb7..e4de7b8dcd 100644 --- a/platform/os_windows.go +++ b/platform/os_windows.go @@ -40,9 +40,6 @@ const ( // CNIBinaryPath is the path to the CNI binary CNIBinaryPath = "C:\\k\\azurecni\\bin\\azure-vnet.exe" - // StatefullCNIBinaryPath is the path to the Statefull CNI binary - StatefullCNIBinaryPath = "C:\\k\\azurecni\\bin\\azure-vnet-statefull.exe" - // CNI runtime path on a Kubernetes cluster K8SCNIRuntimePath = "C:\\k\\azurecni\\bin" From cbcf729b3b288ba1f78093f0247961fca9f86a54 Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Wed, 29 Nov 2023 05:27:29 -0800 Subject: [PATCH 07/17] Make code changes for Stateless CNI migrations. --- network/endpoint_windows.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index b859328880..ba2517de38 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -497,7 +497,7 @@ func (nm *networkManager) updateEndpointImpl(nw *network, existingEpInfo *Endpoi return nil, nil } -// GetEndpointInfoByIPImpl returns an endpointInfo with the corrsponding HNS Endpoint ID that matches an specific IPAddress. +// GetEndpointInfoByIPImpl returns an endpointInfo with the corrsponding HNS Endpoint ID that matches an specific IP Address. func (epInfo *EndpointInfo) GetEndpointInfoByIPImpl(ipAddresses []net.IPNet, networkID string) (*EndpointInfo, error) { // check if network exists, only create the network does not exist hnsResponse, err := Hnsv2.GetNetworkByName(networkID) From 58d892e3f917ff397a49032508d7ec93d834efaf Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Fri, 8 Dec 2023 10:13:44 -0800 Subject: [PATCH 08/17] Make changes to statless CNI migration branch. --- Makefile | 2 +- cni/api/api.go | 3 ++ cni/network/network.go | 3 ++ cns/cnireconciler/podinfoprovider.go | 45 +++++++++++++++++++++-- cns/cnireconciler/podinfoprovider_test.go | 2 +- cns/service/main.go | 15 +++++--- network/endpoint.go | 3 ++ 7 files changed, 62 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 2cb38265c4..e41036c574 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ ACN_VERSION ?= $(shell git describe --exclude "azure-ipam*" --exclude "dropgz AZURE_IPAM_VERSION ?= $(notdir $(shell git describe --match "azure-ipam*" --tags --always)) CNI_VERSION ?= $(ACN_VERSION) CNI_DROPGZ_VERSION ?= $(notdir $(shell git describe --match "dropgz*" --tags --always)) -CNS_VERSION ?= $(ACN_VERSION) +CNS_VERSION ?= "v1.5.15-22-g3d1fa46d" NPM_VERSION ?= $(ACN_VERSION) ZAPAI_VERSION ?= $(notdir $(shell git describe --match "zapai*" --tags --always)) diff --git a/cni/api/api.go b/cni/api/api.go index aed0f5ac0b..0b2c4f9a4f 100644 --- a/cni/api/api.go +++ b/cni/api/api.go @@ -17,6 +17,9 @@ type PodNetworkInterfaceInfo struct { PodEndpointId string ContainerID string IPAddresses []net.IPNet + HostIfName string + HNSEndpointID string + IfName string } type AzureCNIState struct { diff --git a/cni/network/network.go b/cni/network/network.go index 2381b8ffc4..728bc9a485 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -190,6 +190,9 @@ func (plugin *NetPlugin) GetAllEndpointState(networkid string) (*api.AzureCNISta PodEndpointId: ep.Id, ContainerID: ep.ContainerID, IPAddresses: ep.IPAddresses, + HostIfName: ep.HostIfName, + HNSEndpointID: ep.HNSEndpointID, + IfName: ep.IfName, } st.ContainerInterfaces[id] = info diff --git a/cns/cnireconciler/podinfoprovider.go b/cns/cnireconciler/podinfoprovider.go index fd237ebb66..11eee2133a 100644 --- a/cns/cnireconciler/podinfoprovider.go +++ b/cns/cnireconciler/podinfoprovider.go @@ -2,10 +2,12 @@ package cnireconciler import ( "fmt" + "net" "github.com/Azure/azure-container-networking/cni/api" "github.com/Azure/azure-container-networking/cni/client" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/restserver" "github.com/Azure/azure-container-networking/store" "github.com/pkg/errors" @@ -14,7 +16,7 @@ import ( // NewCNIPodInfoProvider returns an implementation of cns.PodInfoByIPProvider // that execs out to the CNI and uses the response to build the PodInfo map. -func NewCNIPodInfoProvider() (cns.PodInfoByIPProvider, error) { +func NewCNIPodInfoProvider() (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) { return newCNIPodInfoProvider(exec.New()) } @@ -39,15 +41,16 @@ func newCNSPodInfoProvider(endpointStore store.KeyValueStore) (cns.PodInfoByIPPr }), nil } -func newCNIPodInfoProvider(exec exec.Interface) (cns.PodInfoByIPProvider, error) { +func newCNIPodInfoProvider(exec exec.Interface) (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) { cli := client.New(exec) state, err := cli.GetEndpointState() if err != nil { - return nil, fmt.Errorf("failed to invoke CNI client.GetEndpointState(): %w", err) + return nil, nil, fmt.Errorf("failed to invoke CNI client.GetEndpointState(): %w", err) } + endpointState, err := cniStateToCnsEndpointState(state) return cns.PodInfoByIPProviderFunc(func() (map[string]cns.PodInfo, error) { return cniStateToPodInfoByIP(state) - }), nil + }), endpointState, nil } // cniStateToPodInfoByIP converts an AzureCNIState dumped from a CNI exec @@ -101,3 +104,37 @@ func endpointStateToPodInfoByIP(state map[string]*restserver.EndpointInfo) (map[ } return podInfoByIP, nil } + +// cniStateToCnsEndpointState converts an AzureCNIState dumped from a CNI exec +// into a EndpointInfo map, using the containerID as keys in the map. +// The map then will be saved on CNS endpoint state +func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserver.EndpointInfo, error) { + logger.Printf("Generating CNS ENdpoint State") + endpointState := map[string]*restserver.EndpointInfo{} + for _, endpoint := range state.ContainerInterfaces { + endpointInfo := &restserver.EndpointInfo{PodName: endpoint.PodName, PodNamespace: endpoint.PodNamespace, HnsEndpointID: endpoint.HNSEndpointID, HostVethName: endpoint.HostIfName, IfnameToIPMap: make(map[string]*restserver.IPInfo)} + for _, epIP := range endpoint.IPAddresses { + if epIP.IP.To4() == nil { // is an ipv6 address + ipconfig := net.IPNet{IP: epIP.IP, Mask: epIP.Mask} + for _, ipconf := range endpointInfo.IfnameToIPMap[endpoint.IfName].IPv6 { + if ipconf.IP.Equal(ipconfig.IP) { + logger.Printf("Found existing ipv6 ipconfig for infra container %s", endpoint.ContainerID) + return nil, nil + } + } + endpointInfo.IfnameToIPMap[endpoint.IfName].IPv6 = append(endpointInfo.IfnameToIPMap[endpoint.IfName].IPv6, ipconfig) + } else { + ipconfig := net.IPNet{IP: epIP.IP, Mask: epIP.Mask} + for _, ipconf := range endpointInfo.IfnameToIPMap[endpoint.IfName].IPv4 { + if ipconf.IP.Equal(ipconfig.IP) { + logger.Printf("Found existing ipv4 ipconfig for infra container %s", endpoint.ContainerID) + return nil, nil + } + } + endpointInfo.IfnameToIPMap[endpoint.IfName].IPv4 = append(endpointInfo.IfnameToIPMap[endpoint.IfName].IPv4, ipconfig) + } + } + endpointState[endpoint.ContainerID] = endpointInfo + } + return endpointState, nil +} diff --git a/cns/cnireconciler/podinfoprovider_test.go b/cns/cnireconciler/podinfoprovider_test.go index a9db49490f..d4e253a7c8 100644 --- a/cns/cnireconciler/podinfoprovider_test.go +++ b/cns/cnireconciler/podinfoprovider_test.go @@ -51,7 +51,7 @@ func TestNewCNIPodInfoProvider(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - got, err := newCNIPodInfoProvider(tt.exec) + got, _, err := newCNIPodInfoProvider(tt.exec) if tt.wantErr { assert.Error(t, err) return diff --git a/cns/service/main.go b/cns/service/main.go index 9044d19e86..b0820df594 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -839,7 +839,7 @@ func main() { logger.Printf("Set GlobalPodInfoScheme %v (InitializeFromCNI=%t)", cns.GlobalPodInfoScheme, cnsconfig.InitializeFromCNI) - err = InitializeCRDState(rootCtx, httpRestService, cnsconfig) + err = InitializeCRDState(rootCtx, httpRestService, cnsconfig, endpointStateStore) if err != nil { logger.Errorf("Failed to start CRD Controller, err:%v.\n", err) return @@ -1182,7 +1182,7 @@ func reconcileInitialCNSState(ctx context.Context, cli nodeNetworkConfigGetter, } // InitializeCRDState builds and starts the CRD controllers. -func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cnsconfig *configuration.CNSConfig) error { +func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cnsconfig *configuration.CNSConfig, endpointStateStore store.KeyValueStore) error { // convert interface type to implementation type httpRestServiceImplementation, ok := httpRestService.(*restserver.HTTPRestService) if !ok { @@ -1238,13 +1238,18 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn if err != nil { if errors.Is(err, store.ErrKeyNotFound) { logger.Printf("[Azure CNS] No endpoint state found, skipping initializing CNS state") - if os.Getenv("StatelessCNIMigration") == "true" { + if cnsconfig.StatelessCNIMigration { logger.Printf("StatelessCNI Migration is enabled") logger.Printf("Initializing from Statefull CNI") - podInfoByIPProvider, err = cnireconciler.NewCNIPodInfoProvider() + var endpointState map[string]*restserver.EndpointInfo + podInfoByIPProvider, endpointState, err = cnireconciler.NewCNIPodInfoProvider() if err != nil { return errors.Wrap(err, "failed to create CNI PodInfoProvider") } + err := endpointStateStore.Write(restserver.EndpointStoreKey, endpointState) + if err != nil { + return fmt.Errorf("failed to write endpoint state to store: %w", err) + } } } else { return errors.Wrap(err, "failed to create CNS PodInfoProvider") @@ -1253,7 +1258,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn case cnsconfig.InitializeFromCNI: logger.Printf("Initializing from CNI") - podInfoByIPProvider, err = cnireconciler.NewCNIPodInfoProvider() + podInfoByIPProvider, _, err = cnireconciler.NewCNIPodInfoProvider() if err != nil { return errors.Wrap(err, "failed to create CNI PodInfoProvider") } diff --git a/network/endpoint.go b/network/endpoint.go index 518aa804f3..79b4be9903 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -92,6 +92,7 @@ type EndpointInfo struct { NICType cns.NICType SkipDefaultRoutes bool HNSEndpointID string + HostIfName string } // RouteInfo contains information about an IP route. @@ -267,6 +268,8 @@ func (ep *endpoint) getInfo() *EndpointInfo { PODName: ep.PODName, PODNameSpace: ep.PODNameSpace, NetworkContainerID: ep.NetworkContainerID, + HNSEndpointID: ep.HnsId, + HostIfName: ep.HostIfName, } info.Routes = append(info.Routes, ep.Routes...) From e4fb4ad41c4dc811ffc8929eec5be88f34a85881 Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Wed, 3 Jan 2024 16:00:29 -0800 Subject: [PATCH 09/17] Stateless CNI migration code changes --- Makefile | 16 ++-------------- cns/cnireconciler/podinfoprovider.go | 11 +++++++---- cns/cnireconciler/podinfoprovider_test.go | 6 +++--- cns/service/main.go | 2 +- 4 files changed, 13 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index e41036c574..dc85d2b2a2 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ ACN_VERSION ?= $(shell git describe --exclude "azure-ipam*" --exclude "dropgz AZURE_IPAM_VERSION ?= $(notdir $(shell git describe --match "azure-ipam*" --tags --always)) CNI_VERSION ?= $(ACN_VERSION) CNI_DROPGZ_VERSION ?= $(notdir $(shell git describe --match "dropgz*" --tags --always)) -CNS_VERSION ?= "v1.5.15-22-g3d1fa46d" +CNS_VERSION ?= $(ACN_VERSION) NPM_VERSION ?= $(ACN_VERSION) ZAPAI_VERSION ?= $(notdir $(shell git describe --match "zapai*" --tags --always)) @@ -46,7 +46,6 @@ ZAPAI_VERSION ?= $(notdir $(shell git describe --match "zapai*" --tags --alway AZURE_IPAM_DIR = $(REPO_ROOT)/azure-ipam CNM_DIR = $(REPO_ROOT)/cnm/plugin CNI_NET_DIR = $(REPO_ROOT)/cni/network/plugin -STATELESS_CNI_NET_DIR = $(REPO_ROOT)/cni/network/stateless CNI_IPAM_DIR = $(REPO_ROOT)/cni/ipam/plugin CNI_IPAMV6_DIR = $(REPO_ROOT)/cni/ipam/pluginv6 CNI_TELEMETRY_DIR = $(REPO_ROOT)/cni/telemetry/service @@ -59,14 +58,11 @@ AZURE_IPAM_BUILD_DIR = $(BUILD_DIR)/azure-ipam IMAGE_DIR = $(OUTPUT_DIR)/images CNM_BUILD_DIR = $(BUILD_DIR)/cnm CNI_BUILD_DIR = $(BUILD_DIR)/cni -STATELESS_CNI_BUILD_DIR = $(CNI_BUILD_DIR)/stateless ACNCLI_BUILD_DIR = $(BUILD_DIR)/acncli CNI_MULTITENANCY_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy-transparent-vlan CNI_SWIFT_BUILD_DIR = $(BUILD_DIR)/cni-swift -STATELESS_CNI_SWIFT_BUILD_DIR = $(CNI_SWIFT_BUILD_DIR)/stateless CNI_OVERLAY_BUILD_DIR = $(BUILD_DIR)/cni-overlay -STATELESS_CNI_OVERLAY_BUILD_DIR = $(CNI_OVERLAY_BUILD_DIR)/stateless CNI_BAREMETAL_BUILD_DIR = $(BUILD_DIR)/cni-baremetal CNI_DUALSTACK_BUILD_DIR = $(BUILD_DIR)/cni-dualstack CNS_BUILD_DIR = $(BUILD_DIR)/cns @@ -134,7 +130,7 @@ endif # Shorthand target names for convenience. azure-cnm-plugin: cnm-binary cnm-archive -azure-cni-plugin: azure-vnet-binary azure-vnet-stateless-binary azure-vnet-ipam-binary azure-vnet-ipamv6-binary azure-vnet-telemetry-binary cni-archive +azure-cni-plugin: azure-vnet-binary azure-vnet-ipam-binary azure-vnet-ipamv6-binary azure-vnet-telemetry-binary cni-archive azure-cns: azure-cns-binary cns-archive acncli: acncli-binary acncli-archive azure-cnms: azure-cnms-binary cnms-archive @@ -184,10 +180,6 @@ cnm-binary: azure-vnet-binary: cd $(CNI_NET_DIR) && CGO_ENABLED=0 go build -v -o $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) -ldflags "-X main.version=$(CNI_VERSION)" -gcflags="-dwarflocationlists=true" -# Build the Azure CNI stateless network binary -azure-vnet-stateless-binary: - cd $(STATELESS_CNI_NET_DIR) && CGO_ENABLED=0 go build -v -o $(STATELESS_CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) -ldflags "-X main.version=$(CNI_VERSION)" -gcflags="-dwarflocationlists=true" - # Build the Azure CNI IPAM binary. azure-vnet-ipam-binary: cd $(CNI_IPAM_DIR) && CGO_ENABLED=0 go build -v -o $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) -ldflags "-X main.version=$(CNI_VERSION)" -gcflags="-dwarflocationlists=true" @@ -683,16 +675,12 @@ endif cp cni/azure-$(GOOS)-swift.conflist $(CNI_SWIFT_BUILD_DIR)/10-azure.conflist cp telemetry/azure-vnet-telemetry.config $(CNI_SWIFT_BUILD_DIR)/azure-vnet-telemetry.config cp $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) $(CNI_SWIFT_BUILD_DIR) - $(MKDIR) $(STATELESS_CNI_SWIFT_BUILD_DIR) - cp $(STATELESS_CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(STATELESS_CNI_SWIFT_BUILD_DIR) cd $(CNI_SWIFT_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_SWIFT_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config $(MKDIR) $(CNI_OVERLAY_BUILD_DIR) cp cni/azure-$(GOOS)-swift-overlay.conflist $(CNI_OVERLAY_BUILD_DIR)/10-azure.conflist cp telemetry/azure-vnet-telemetry.config $(CNI_OVERLAY_BUILD_DIR)/azure-vnet-telemetry.config cp $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) $(CNI_OVERLAY_BUILD_DIR) - $(MKDIR) $(STATELESS_CNI_OVERLAY_BUILD_DIR) - cp $(STATELESS_CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(STATELESS_CNI_OVERLAY_BUILD_DIR) cd $(CNI_OVERLAY_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_OVERLAY_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config $(MKDIR) $(CNI_DUALSTACK_BUILD_DIR) diff --git a/cns/cnireconciler/podinfoprovider.go b/cns/cnireconciler/podinfoprovider.go index 11eee2133a..8de4fe1eb0 100644 --- a/cns/cnireconciler/podinfoprovider.go +++ b/cns/cnireconciler/podinfoprovider.go @@ -113,27 +113,30 @@ func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserve endpointState := map[string]*restserver.EndpointInfo{} for _, endpoint := range state.ContainerInterfaces { endpointInfo := &restserver.EndpointInfo{PodName: endpoint.PodName, PodNamespace: endpoint.PodNamespace, HnsEndpointID: endpoint.HNSEndpointID, HostVethName: endpoint.HostIfName, IfnameToIPMap: make(map[string]*restserver.IPInfo)} + ipInfo := &restserver.IPInfo{} for _, epIP := range endpoint.IPAddresses { if epIP.IP.To4() == nil { // is an ipv6 address ipconfig := net.IPNet{IP: epIP.IP, Mask: epIP.Mask} - for _, ipconf := range endpointInfo.IfnameToIPMap[endpoint.IfName].IPv6 { + for _, ipconf := range ipInfo.IPv6 { if ipconf.IP.Equal(ipconfig.IP) { logger.Printf("Found existing ipv6 ipconfig for infra container %s", endpoint.ContainerID) return nil, nil } } - endpointInfo.IfnameToIPMap[endpoint.IfName].IPv6 = append(endpointInfo.IfnameToIPMap[endpoint.IfName].IPv6, ipconfig) + ipInfo.IPv6 = append(ipInfo.IPv6, ipconfig) + } else { ipconfig := net.IPNet{IP: epIP.IP, Mask: epIP.Mask} - for _, ipconf := range endpointInfo.IfnameToIPMap[endpoint.IfName].IPv4 { + for _, ipconf := range ipInfo.IPv4 { if ipconf.IP.Equal(ipconfig.IP) { logger.Printf("Found existing ipv4 ipconfig for infra container %s", endpoint.ContainerID) return nil, nil } } - endpointInfo.IfnameToIPMap[endpoint.IfName].IPv4 = append(endpointInfo.IfnameToIPMap[endpoint.IfName].IPv4, ipconfig) + ipInfo.IPv4 = append(ipInfo.IPv4, ipconfig) } } + endpointInfo.IfnameToIPMap[endpoint.IfName] = ipInfo endpointState[endpoint.ContainerID] = endpointInfo } return endpointState, nil diff --git a/cns/cnireconciler/podinfoprovider_test.go b/cns/cnireconciler/podinfoprovider_test.go index d4e253a7c8..8a6cb85723 100644 --- a/cns/cnireconciler/podinfoprovider_test.go +++ b/cns/cnireconciler/podinfoprovider_test.go @@ -31,7 +31,7 @@ func TestNewCNIPodInfoProvider(t *testing.T) { { name: "good", exec: newCNIStateFakeExec( - `{"ContainerInterfaces":{"3f813b02-eth0":{"PodName":"metrics-server-77c8679d7d-6ksdh","PodNamespace":"kube-system","PodEndpointID":"3f813b02-eth0","ContainerID":"3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46","IPAddresses":[{"IP":"10.241.0.17","Mask":"//8AAA=="}]},"6e688597-eth0":{"PodName":"tunnelfront-5d96f9b987-65xbn","PodNamespace":"kube-system","PodEndpointID":"6e688597-eth0","ContainerID":"6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed","IPAddresses":[{"IP":"10.241.0.13","Mask":"//8AAA=="}]}}}`, + `{"ContainerInterfaces":{"3f813b02-eth0":{"PodName":"metrics-server-77c8679d7d-6ksdh","IfName":"eth0","PodNamespace":"kube-system","PodEndpointID":"3f813b02-eth0","ContainerID":"3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46","IPAddresses":[{"IP":"10.241.0.17","Mask":"//8AAA=="}]},"6e688597-eth0":{"PodName":"tunnelfront-5d96f9b987-65xbn","IfName":"eth0","PodNamespace":"kube-system","PodEndpointID":"6e688597-eth0","ContainerID":"6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed","IPAddresses":[{"IP":"10.241.0.13","Mask":"//8AAA=="}]}}}`, ), want: map[string]cns.PodInfo{ "10.241.0.13": cns.NewPodInfo("6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", "6e688597-eth0", "tunnelfront-5d96f9b987-65xbn", "kube-system"), @@ -51,14 +51,14 @@ func TestNewCNIPodInfoProvider(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - got, _, err := newCNIPodInfoProvider(tt.exec) + got, endpointState, err := newCNIPodInfoProvider(tt.exec) if tt.wantErr { assert.Error(t, err) return } assert.NoError(t, err) podInfoByIP, _ := got.PodInfoByIP() - assert.Equal(t, tt.want, podInfoByIP) + assert.Equal(t, tt.want, podInfoByIP, endpointState) }) } } diff --git a/cns/service/main.go b/cns/service/main.go index b0820df594..98e305f678 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1240,7 +1240,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn logger.Printf("[Azure CNS] No endpoint state found, skipping initializing CNS state") if cnsconfig.StatelessCNIMigration { logger.Printf("StatelessCNI Migration is enabled") - logger.Printf("Initializing from Statefull CNI") + logger.Printf("initializing from Statefull CNI") var endpointState map[string]*restserver.EndpointInfo podInfoByIPProvider, endpointState, err = cnireconciler.NewCNIPodInfoProvider() if err != nil { From 7068714b675d9cf03cb3aa5b10072f0145bbf153 Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Wed, 17 Jan 2024 09:19:15 -0800 Subject: [PATCH 10/17] resolving migration issue --- Makefile | 2 +- cns/cnireconciler/podinfoprovider.go | 5 +++++ cns/service/main.go | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index dc85d2b2a2..34d8234d54 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ ACN_VERSION ?= $(shell git describe --exclude "azure-ipam*" --exclude "dropgz AZURE_IPAM_VERSION ?= $(notdir $(shell git describe --match "azure-ipam*" --tags --always)) CNI_VERSION ?= $(ACN_VERSION) CNI_DROPGZ_VERSION ?= $(notdir $(shell git describe --match "dropgz*" --tags --always)) -CNS_VERSION ?= $(ACN_VERSION) +CNS_VERSION ?= $(ACN_VERSION)-k NPM_VERSION ?= $(ACN_VERSION) ZAPAI_VERSION ?= $(notdir $(shell git describe --match "zapai*" --tags --always)) diff --git a/cns/cnireconciler/podinfoprovider.go b/cns/cnireconciler/podinfoprovider.go index 8de4fe1eb0..3c0e518e27 100644 --- a/cns/cnireconciler/podinfoprovider.go +++ b/cns/cnireconciler/podinfoprovider.go @@ -137,7 +137,12 @@ func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserve } } endpointInfo.IfnameToIPMap[endpoint.IfName] = ipInfo + logger.Printf("writing endpoint podName from stateful CNI %v", endpoint.PodName) + logger.Printf("writing endpoint info from stateful CNI [%+v]", *endpointInfo) endpointState[endpoint.ContainerID] = endpointInfo } + for containerID, endpointInfo := range endpointState { + logger.Printf("writing endpoint state from stateful CNI [%+v]:[%+v]", containerID, *endpointInfo) + } return endpointState, nil } diff --git a/cns/service/main.go b/cns/service/main.go index 98e305f678..02d45c2391 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1246,6 +1246,10 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn if err != nil { return errors.Wrap(err, "failed to create CNI PodInfoProvider") } + logger.Printf("writing endpoint info from stateful CNI [%+v]", endpointState) + for containerID, endpointInfo := range endpointState { + logger.Printf("writing endpoint state from stateful CNI [%+v]:[%+v]", containerID, *endpointInfo) + } err := endpointStateStore.Write(restserver.EndpointStoreKey, endpointState) if err != nil { return fmt.Errorf("failed to write endpoint state to store: %w", err) From 832f1e3232675a8f5d460a81ba9042dd97bc93ce Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Thu, 18 Jan 2024 17:01:17 -0800 Subject: [PATCH 11/17] remove cni changes --- Makefile | 2 +- cni/api/api.go | 6 +++--- cni/client/client.go | 4 +++- cni/network/network.go | 6 +++--- cns/cnireconciler/podinfoprovider.go | 7 ++++--- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 34d8234d54..47a15e356e 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ ACN_VERSION ?= $(shell git describe --exclude "azure-ipam*" --exclude "dropgz AZURE_IPAM_VERSION ?= $(notdir $(shell git describe --match "azure-ipam*" --tags --always)) CNI_VERSION ?= $(ACN_VERSION) CNI_DROPGZ_VERSION ?= $(notdir $(shell git describe --match "dropgz*" --tags --always)) -CNS_VERSION ?= $(ACN_VERSION)-k +CNS_VERSION ?= $(ACN_VERSION)-m NPM_VERSION ?= $(ACN_VERSION) ZAPAI_VERSION ?= $(notdir $(shell git describe --match "zapai*" --tags --always)) diff --git a/cni/api/api.go b/cni/api/api.go index 0b2c4f9a4f..a4750e3ec5 100644 --- a/cni/api/api.go +++ b/cni/api/api.go @@ -17,9 +17,9 @@ type PodNetworkInterfaceInfo struct { PodEndpointId string ContainerID string IPAddresses []net.IPNet - HostIfName string - HNSEndpointID string - IfName string + //HostIfName string + //HNSEndpointID string + //IfName string } type AzureCNIState struct { diff --git a/cni/client/client.go b/cni/client/client.go index 27ba4b744b..53b420438d 100644 --- a/cni/client/client.go +++ b/cni/client/client.go @@ -46,7 +46,9 @@ func (c *client) GetEndpointState() (*api.AzureCNIState, error) { if err := json.Unmarshal(output, state); err != nil { return nil, fmt.Errorf("failed to decode response from Azure CNI when retrieving state: [%w], response from CNI: [%s]", err, string(output)) } - + for containerID, endpointInfo := range state.ContainerInterfaces { + logger.Info("writing endpoint state from stateful CNI ", zap.String("podname:", containerID), zap.String("podname:", endpointInfo.PodName), zap.Any("endpoint:", endpointInfo)) + } return state, nil } diff --git a/cni/network/network.go b/cni/network/network.go index 728bc9a485..29e971dd92 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -190,9 +190,9 @@ func (plugin *NetPlugin) GetAllEndpointState(networkid string) (*api.AzureCNISta PodEndpointId: ep.Id, ContainerID: ep.ContainerID, IPAddresses: ep.IPAddresses, - HostIfName: ep.HostIfName, - HNSEndpointID: ep.HNSEndpointID, - IfName: ep.IfName, + //HostIfName: ep.HostIfName, + //HNSEndpointID: ep.HNSEndpointID, + //IfName: ep.IfName, } st.ContainerInterfaces[id] = info diff --git a/cns/cnireconciler/podinfoprovider.go b/cns/cnireconciler/podinfoprovider.go index 3c0e518e27..4f45bfc940 100644 --- a/cns/cnireconciler/podinfoprovider.go +++ b/cns/cnireconciler/podinfoprovider.go @@ -62,7 +62,7 @@ func cniStateToPodInfoByIP(state *api.AzureCNIState) (map[string]cns.PodInfo, er for _, endpoint := range state.ContainerInterfaces { for _, epIP := range endpoint.IPAddresses { podInfo := cns.NewPodInfo(endpoint.ContainerID, endpoint.PodEndpointId, endpoint.PodName, endpoint.PodNamespace) - + logger.Printf("podInfoByIp [%+v]", podInfoByIP) ipKey := epIP.IP.String() if prevPodInfo, ok := podInfoByIP[ipKey]; ok { return nil, errors.Wrapf(cns.ErrDuplicateIP, "duplicate ip %s found for different pods: pod: %+v, pod: %+v", ipKey, podInfo, prevPodInfo) @@ -71,6 +71,7 @@ func cniStateToPodInfoByIP(state *api.AzureCNIState) (map[string]cns.PodInfo, er podInfoByIP[ipKey] = podInfo } } + logger.Printf("podInfoByIP [%+v]", podInfoByIP) return podInfoByIP, nil } @@ -112,7 +113,7 @@ func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserve logger.Printf("Generating CNS ENdpoint State") endpointState := map[string]*restserver.EndpointInfo{} for _, endpoint := range state.ContainerInterfaces { - endpointInfo := &restserver.EndpointInfo{PodName: endpoint.PodName, PodNamespace: endpoint.PodNamespace, HnsEndpointID: endpoint.HNSEndpointID, HostVethName: endpoint.HostIfName, IfnameToIPMap: make(map[string]*restserver.IPInfo)} + endpointInfo := &restserver.EndpointInfo{PodName: endpoint.PodName, PodNamespace: endpoint.PodNamespace, IfnameToIPMap: make(map[string]*restserver.IPInfo)} ipInfo := &restserver.IPInfo{} for _, epIP := range endpoint.IPAddresses { if epIP.IP.To4() == nil { // is an ipv6 address @@ -136,7 +137,7 @@ func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserve ipInfo.IPv4 = append(ipInfo.IPv4, ipconfig) } } - endpointInfo.IfnameToIPMap[endpoint.IfName] = ipInfo + endpointInfo.IfnameToIPMap["eth0"] = ipInfo logger.Printf("writing endpoint podName from stateful CNI %v", endpoint.PodName) logger.Printf("writing endpoint info from stateful CNI [%+v]", *endpointInfo) endpointState[endpoint.ContainerID] = endpointInfo From 5208dcc96418b8de5c69d57edca997bf6ee36824 Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Mon, 22 Jan 2024 13:29:59 -0800 Subject: [PATCH 12/17] Applying changes to CNIReonciler --- Makefile | 2 +- cni/api/api.go | 3 -- cni/client/client.go | 4 +-- cni/network/network.go | 3 -- cns/api.go | 1 + cns/client/client.go | 16 ++++++++-- cns/client/client_test.go | 15 +++++++-- cns/cnireconciler/podinfoprovider.go | 47 ++++++++++++++++++---------- cns/restserver/ipam.go | 27 ++++++++++++++-- cns/service/main.go | 6 +--- network/endpoint_windows.go | 6 ++-- network/manager.go | 4 +-- 12 files changed, 90 insertions(+), 44 deletions(-) diff --git a/Makefile b/Makefile index 47a15e356e..dc85d2b2a2 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ ACN_VERSION ?= $(shell git describe --exclude "azure-ipam*" --exclude "dropgz AZURE_IPAM_VERSION ?= $(notdir $(shell git describe --match "azure-ipam*" --tags --always)) CNI_VERSION ?= $(ACN_VERSION) CNI_DROPGZ_VERSION ?= $(notdir $(shell git describe --match "dropgz*" --tags --always)) -CNS_VERSION ?= $(ACN_VERSION)-m +CNS_VERSION ?= $(ACN_VERSION) NPM_VERSION ?= $(ACN_VERSION) ZAPAI_VERSION ?= $(notdir $(shell git describe --match "zapai*" --tags --always)) diff --git a/cni/api/api.go b/cni/api/api.go index a4750e3ec5..aed0f5ac0b 100644 --- a/cni/api/api.go +++ b/cni/api/api.go @@ -17,9 +17,6 @@ type PodNetworkInterfaceInfo struct { PodEndpointId string ContainerID string IPAddresses []net.IPNet - //HostIfName string - //HNSEndpointID string - //IfName string } type AzureCNIState struct { diff --git a/cni/client/client.go b/cni/client/client.go index 53b420438d..27ba4b744b 100644 --- a/cni/client/client.go +++ b/cni/client/client.go @@ -46,9 +46,7 @@ func (c *client) GetEndpointState() (*api.AzureCNIState, error) { if err := json.Unmarshal(output, state); err != nil { return nil, fmt.Errorf("failed to decode response from Azure CNI when retrieving state: [%w], response from CNI: [%s]", err, string(output)) } - for containerID, endpointInfo := range state.ContainerInterfaces { - logger.Info("writing endpoint state from stateful CNI ", zap.String("podname:", containerID), zap.String("podname:", endpointInfo.PodName), zap.Any("endpoint:", endpointInfo)) - } + return state, nil } diff --git a/cni/network/network.go b/cni/network/network.go index 29e971dd92..2381b8ffc4 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -190,9 +190,6 @@ func (plugin *NetPlugin) GetAllEndpointState(networkid string) (*api.AzureCNISta PodEndpointId: ep.Id, ContainerID: ep.ContainerID, IPAddresses: ep.IPAddresses, - //HostIfName: ep.HostIfName, - //HNSEndpointID: ep.HNSEndpointID, - //IfName: ep.IfName, } st.ContainerInterfaces[id] = info diff --git a/cns/api.go b/cns/api.go index 3894e9aff6..969e437753 100644 --- a/cns/api.go +++ b/cns/api.go @@ -363,4 +363,5 @@ type GetHomeAzResponse struct { type EndpointRequest struct { HnsEndpointID string `json:"hnsEndpointID"` HostVethName string `json:"hostVethName"` + IFName string `json:"IFName"` } diff --git a/cns/client/client.go b/cns/client/client.go index 2ee524cfb9..125a135079 100644 --- a/cns/client/client.go +++ b/cns/client/client.go @@ -1024,11 +1024,20 @@ func (c *Client) GetHomeAz(ctx context.Context) (*cns.GetHomeAzResponse, error) } // GetEndpoint calls the EndpointHandlerAPI in CNS to retrieve the state of a given EndpointID -func (c *Client) GetEndpoint(ctx context.Context, endpointID string) (*restserver.GetEndpointResponse, error) { +func (c *Client) GetEndpoint(ctx context.Context, endpointID, ifName string) (*restserver.GetEndpointResponse, error) { // build the request + getEndpoint := cns.EndpointRequest{ + IFName: ifName, + } + var body bytes.Buffer + + if err := json.NewEncoder(&body).Encode(getEndpoint); err != nil { + return nil, errors.Wrap(err, "failed to encode getEndpoint") + } + u := c.routes[cns.EndpointAPI] uString := u.String() + endpointID - req, err := http.NewRequestWithContext(ctx, http.MethodGet, uString, http.NoBody) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, uString, &body) if err != nil { return nil, errors.Wrap(err, "failed to build request") } @@ -1059,11 +1068,12 @@ func (c *Client) GetEndpoint(ctx context.Context, endpointID string) (*restserve // UpdateEndpoint calls the EndpointHandlerAPI in CNS // to update the state of a given EndpointID with either HNSEndpointID or HostVethName -func (c *Client) UpdateEndpoint(ctx context.Context, endpointID, hnsID, vethName string) (*cns.Response, error) { +func (c *Client) UpdateEndpoint(ctx context.Context, endpointID, hnsID, vethName, ifName string) (*cns.Response, error) { // build the request updateEndpoint := cns.EndpointRequest{ HnsEndpointID: hnsID, HostVethName: vethName, + IFName: ifName, } var body bytes.Buffer diff --git a/cns/client/client_test.go b/cns/client/client_test.go index 871773eded..1f53829796 100644 --- a/cns/client/client_test.go +++ b/cns/client/client_test.go @@ -2777,6 +2777,7 @@ func TestUpdateEndpoint(t *testing.T) { containerID string hnsID string vethName string + ifName string response *RequestCapture expReq *cns.EndpointRequest shouldErr bool @@ -2786,6 +2787,7 @@ func TestUpdateEndpoint(t *testing.T) { "", "", "", + "", &RequestCapture{ Next: &mockdo{}, }, @@ -2797,6 +2799,7 @@ func TestUpdateEndpoint(t *testing.T) { "foo", "bar", "", + "too", &RequestCapture{ Next: &mockdo{ httpStatusCodeToReturn: http.StatusOK, @@ -2804,6 +2807,7 @@ func TestUpdateEndpoint(t *testing.T) { }, &cns.EndpointRequest{ HnsEndpointID: "bar", + IFName: "too", }, false, }, @@ -2812,6 +2816,7 @@ func TestUpdateEndpoint(t *testing.T) { "foo", "", "bar", + "too", &RequestCapture{ Next: &mockdo{ httpStatusCodeToReturn: http.StatusOK, @@ -2819,6 +2824,7 @@ func TestUpdateEndpoint(t *testing.T) { }, &cns.EndpointRequest{ HostVethName: "bar", + IFName: "too", }, false, }, @@ -2827,6 +2833,7 @@ func TestUpdateEndpoint(t *testing.T) { "foo", "", "bar", + "", &RequestCapture{ Next: &mockdo{ httpStatusCodeToReturn: http.StatusBadRequest, @@ -2851,7 +2858,7 @@ func TestUpdateEndpoint(t *testing.T) { } // execute the method under test - res, err := client.UpdateEndpoint(context.TODO(), test.containerID, test.hnsID, test.vethName) + res, err := client.UpdateEndpoint(context.TODO(), test.containerID, test.hnsID, test.vethName, test.ifName) if err != nil && !test.shouldErr { t.Fatal("unexpected error: err: ", err, res.Message) } @@ -2897,12 +2904,14 @@ func TestGetEndpoint(t *testing.T) { getEndpointTests := []struct { name string containerID string + ifName string response *RequestCapture shouldErr bool }{ { "empty", "", + "", &RequestCapture{ Next: &mockdo{}, }, @@ -2911,6 +2920,7 @@ func TestGetEndpoint(t *testing.T) { { "with EndpointID", "foo", + "foo", &RequestCapture{ Next: &mockdo{ httpStatusCodeToReturn: http.StatusOK, @@ -2921,6 +2931,7 @@ func TestGetEndpoint(t *testing.T) { { "Bad Request", "foo", + "foo", &RequestCapture{ Next: &mockdo{ httpStatusCodeToReturn: http.StatusBadRequest, @@ -2942,7 +2953,7 @@ func TestGetEndpoint(t *testing.T) { } // execute the method under test - res, err := client.GetEndpoint(context.TODO(), test.containerID) + res, err := client.GetEndpoint(context.TODO(), test.containerID, test.ifName) if err != nil && !test.shouldErr { t.Fatal("unexpected error: err: ", err, res.Response.Message) } diff --git a/cns/cnireconciler/podinfoprovider.go b/cns/cnireconciler/podinfoprovider.go index 4f45bfc940..9e3b318005 100644 --- a/cns/cnireconciler/podinfoprovider.go +++ b/cns/cnireconciler/podinfoprovider.go @@ -3,6 +3,7 @@ package cnireconciler import ( "fmt" "net" + "strings" "github.com/Azure/azure-container-networking/cni/api" "github.com/Azure/azure-container-networking/cni/client" @@ -14,6 +15,8 @@ import ( "k8s.io/utils/exec" ) +const InterfaceName = "eth0" + // NewCNIPodInfoProvider returns an implementation of cns.PodInfoByIPProvider // that execs out to the CNI and uses the response to build the PodInfo map. func NewCNIPodInfoProvider() (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) { @@ -41,16 +44,20 @@ func newCNSPodInfoProvider(endpointStore store.KeyValueStore) (cns.PodInfoByIPPr }), nil } -func newCNIPodInfoProvider(exec exec.Interface) (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) { - cli := client.New(exec) +func newCNIPodInfoProvider(exc exec.Interface) (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) { + cli := client.New(exc) state, err := cli.GetEndpointState() if err != nil { return nil, nil, fmt.Errorf("failed to invoke CNI client.GetEndpointState(): %w", err) } - endpointState, err := cniStateToCnsEndpointState(state) + for containerID, endpointInfo := range state.ContainerInterfaces { + logger.Printf("state dump from CNI: [%+v], [%+v]", containerID, endpointInfo) + } + var endpointState map[string]*restserver.EndpointInfo + endpointState, err = cniStateToCnsEndpointState(state) return cns.PodInfoByIPProviderFunc(func() (map[string]cns.PodInfo, error) { return cniStateToPodInfoByIP(state) - }), endpointState, nil + }), endpointState, err } // cniStateToPodInfoByIP converts an AzureCNIState dumped from a CNI exec @@ -112,7 +119,7 @@ func endpointStateToPodInfoByIP(state map[string]*restserver.EndpointInfo) (map[ func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserver.EndpointInfo, error) { logger.Printf("Generating CNS ENdpoint State") endpointState := map[string]*restserver.EndpointInfo{} - for _, endpoint := range state.ContainerInterfaces { + for epID, endpoint := range state.ContainerInterfaces { endpointInfo := &restserver.EndpointInfo{PodName: endpoint.PodName, PodNamespace: endpoint.PodNamespace, IfnameToIPMap: make(map[string]*restserver.IPInfo)} ipInfo := &restserver.IPInfo{} for _, epIP := range endpoint.IPAddresses { @@ -120,8 +127,8 @@ func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserve ipconfig := net.IPNet{IP: epIP.IP, Mask: epIP.Mask} for _, ipconf := range ipInfo.IPv6 { if ipconf.IP.Equal(ipconfig.IP) { - logger.Printf("Found existing ipv6 ipconfig for infra container %s", endpoint.ContainerID) - return nil, nil + logger.Errorf("Found existing ipv6 ipconfig for infra container %s", endpoint.ContainerID) + return nil, restserver.ErrExistingIpconfigFound } } ipInfo.IPv6 = append(ipInfo.IPv6, ipconfig) @@ -130,20 +137,28 @@ func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserve ipconfig := net.IPNet{IP: epIP.IP, Mask: epIP.Mask} for _, ipconf := range ipInfo.IPv4 { if ipconf.IP.Equal(ipconfig.IP) { - logger.Printf("Found existing ipv4 ipconfig for infra container %s", endpoint.ContainerID) - return nil, nil + logger.Errorf("Found existing ipv4 ipconfig for infra container %s", endpoint.ContainerID) + return nil, restserver.ErrExistingIpconfigFound } } ipInfo.IPv4 = append(ipInfo.IPv4, ipconfig) } } - endpointInfo.IfnameToIPMap["eth0"] = ipInfo - logger.Printf("writing endpoint podName from stateful CNI %v", endpoint.PodName) - logger.Printf("writing endpoint info from stateful CNI [%+v]", *endpointInfo) - endpointState[endpoint.ContainerID] = endpointInfo - } - for containerID, endpointInfo := range endpointState { - logger.Printf("writing endpoint state from stateful CNI [%+v]:[%+v]", containerID, *endpointInfo) + endpointID, Ifname := extractEndpointInfo(epID, endpoint.ContainerID) + endpointInfo.IfnameToIPMap[Ifname] = ipInfo + endpointState[endpointID] = endpointInfo } return endpointState, nil } + +// extractEndpointInfo extract Interface Name and endpointID for each endpoint based the CNI state +func extractEndpointInfo(epID, containerID string) (endpointID, interfaceName string) { + ifName := InterfaceName + if strings.Contains(epID, "-eth") { + ifName = epID[len(epID)-4:] + } + if containerID == "" { + return epID, ifName + } + return containerID, ifName +} diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index d9cf742848..dbb5bbffca 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -27,8 +27,11 @@ var ( ErrNoNCs = errors.New("no NCs found in the CNS internal state") ErrOptManageEndpointState = errors.New("CNS is not set to manage the endpoint state") ErrEndpointStateNotFound = errors.New("endpoint state could not be found in the statefile") + ErrExistingIpconfigFound = errors.New("Found existing ipconfig for infra container") ) +const ContainerIDLength = 8 + // requestIPConfigHandlerHelper validates the request, assign IPs and return the IPConfigs func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { // For SWIFT v2 scenario, the validator function will also modify the ipconfigsRequest. @@ -1008,9 +1011,22 @@ func (service *HTTPRestService) EndpointHandlerAPI(w http.ResponseWriter, r *htt // GetEndpointHandler handles the incoming GetEndpoint requests with http Get method func (service *HTTPRestService) GetEndpointHandler(w http.ResponseWriter, r *http.Request) { logger.Printf("[GetEndpointState] GetEndpoint for %s", r.URL.Path) - + var req cns.EndpointRequest + err := service.Listener.Decode(w, r, &req) endpointID := strings.TrimPrefix(r.URL.Path, cns.EndpointPath) - endpointInfo, err := service.GetEndpointHelper(endpointID) + logger.Request(service.Name, &req, err) + // Check if the request is valid + if err != nil || req.IFName == "" { + response := cns.Response{ + ReturnCode: types.InvalidRequest, + Message: fmt.Sprintf("[getEndpoint] getEndpoint failed with error: %s", err.Error()), + } + w.Header().Set(cnsReturnCode, response.ReturnCode.String()) + err = service.Listener.Encode(w, &response) + logger.Response(service.Name, response, response.ReturnCode, err) + return + } + endpointInfo, err := service.GetEndpointHelper(endpointID, req) if err != nil { response := GetEndpointResponse{ Response: Response{ @@ -1044,7 +1060,7 @@ func (service *HTTPRestService) GetEndpointHandler(w http.ResponseWriter, r *htt } // GetEndpointHelper returns the state of the given endpointId -func (service *HTTPRestService) GetEndpointHelper(endpointID string) (*EndpointInfo, error) { +func (service *HTTPRestService) GetEndpointHelper(endpointID string, req cns.EndpointRequest) (*EndpointInfo, error) { logger.Printf("[GetEndpointState] Get endpoint state for infra container %s", endpointID) // Skip if a store is not provided. @@ -1068,6 +1084,11 @@ func (service *HTTPRestService) GetEndpointHelper(endpointID string) (*EndpointI logger.Warnf("[GetEndpointState] Found existing endpoint state for container %s", endpointID) return endpointInfo, nil } + legacyEndpointID := endpointID[:ContainerIDLength] + "-" + req.IFName + if endpointInfo, ok := service.EndpointState[legacyEndpointID]; ok { + logger.Warnf("[GetEndpointState] Found existing endpoint state for container %s", legacyEndpointID) + return endpointInfo, nil + } return nil, ErrEndpointStateNotFound } diff --git a/cns/service/main.go b/cns/service/main.go index 02d45c2391..b0e1901e49 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1246,11 +1246,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn if err != nil { return errors.Wrap(err, "failed to create CNI PodInfoProvider") } - logger.Printf("writing endpoint info from stateful CNI [%+v]", endpointState) - for containerID, endpointInfo := range endpointState { - logger.Printf("writing endpoint state from stateful CNI [%+v]:[%+v]", containerID, *endpointInfo) - } - err := endpointStateStore.Write(restserver.EndpointStoreKey, endpointState) + err = endpointStateStore.Write(restserver.EndpointStoreKey, endpointState) if err != nil { return fmt.Errorf("failed to write endpoint state to store: %w", err) } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index ba2517de38..fff6428403 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -502,11 +502,11 @@ func (epInfo *EndpointInfo) GetEndpointInfoByIPImpl(ipAddresses []net.IPNet, net // check if network exists, only create the network does not exist hnsResponse, err := Hnsv2.GetNetworkByName(networkID) if err != nil { - return epInfo, err + return epInfo, errors.Wrap(err, "HNS Network not found") } hcnEndpoints, err := Hnsv2.ListEndpointsOfNetwork(hnsResponse.Id) if err != nil { - return epInfo, err + return epInfo, errors.Wrap(err, "failed to fetch HNS endpoints for the given network") } for _, hcnEndpoint := range hcnEndpoints { for _, ipConfiguration := range hcnEndpoint.IpConfigurations { @@ -519,5 +519,5 @@ func (epInfo *EndpointInfo) GetEndpointInfoByIPImpl(ipAddresses []net.IPNet, net } } } - return epInfo, errors.New("No HNSEndpointID matches the IPAddress: " + ipAddresses[0].IP.String()) + return epInfo, errors.wrap(err, "No HNSEndpointID matches the IPAddress: "+ipAddresses[0].IP.String()) } diff --git a/network/manager.go b/network/manager.go index d233506fb7..064a3274b5 100644 --- a/network/manager.go +++ b/network/manager.go @@ -412,7 +412,7 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn // It will add HNSEndpointID or HostVeth name to the endpoint state func (nm *networkManager) UpdateEndpointState(ep *endpoint) error { logger.Info("Calling cns updateEndpoint API with ", zap.String("containerID: ", ep.ContainerID), zap.String("HnsId: ", ep.HnsId), zap.String("HostIfName: ", ep.HostIfName)) - response, err := nm.CnsClient.UpdateEndpoint(context.TODO(), ep.ContainerID, ep.HnsId, ep.HostIfName) + response, err := nm.CnsClient.UpdateEndpoint(context.TODO(), ep.ContainerID, ep.HnsId, ep.HostIfName, ep.IfName) if err != nil { return errors.Wrapf(err, "Update endpoint API returend with error") } @@ -481,7 +481,7 @@ func (nm *networkManager) GetEndpointInfo(networkId string, endpointId string) ( if nm.IsStatelessCNIMode() { logger.Info("calling cns getEndpoint API") - endpointResponse, err := nm.CnsClient.GetEndpoint(context.TODO(), endpointId) + endpointResponse, err := nm.CnsClient.GetEndpoint(context.TODO(), endpointId, "eth0") if err != nil { return nil, errors.Wrapf(err, "Get endpoint API returend with error") } From d8afe37be6f6a681178700458e18b4c2762891e4 Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Thu, 25 Jan 2024 20:19:53 -0800 Subject: [PATCH 13/17] Addressing the comments. --- cni/linux.Dockerfile | 3 + cni/network/network.go | 4 +- cni/network/network_windows.go | 2 +- cnm/network/network.go | 2 +- cns/cnireconciler/podinfoprovider.go | 5 +- cns/cnireconciler/podinfoprovider_test.go | 27 ++++++++- cns/configuration/configuration.go | 1 + cns/service/main.go | 52 ++++++++++------- network/endpoint.go | 2 +- network/endpoint_windows.go | 14 ++--- network/manager.go | 69 +++++++++++++---------- network/manager_mock.go | 2 +- 12 files changed, 114 insertions(+), 69 deletions(-) diff --git a/cni/linux.Dockerfile b/cni/linux.Dockerfile index 1a83b3a895..18b93be7d5 100644 --- a/cni/linux.Dockerfile +++ b/cni/linux.Dockerfile @@ -13,6 +13,9 @@ RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azure-vnet-telemetry -trimpath RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azure-vnet-ipam -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/ipam/plugin/main.go RUN GOOS=$OS CGO_ENABLED=0 go build -a -o /go/bin/azurecni-stateless -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/network/stateless/main.go +FROM scratch as bins +COPY --from=azure-vnet /go/bin/* / + FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 AS compressor ARG OS WORKDIR /payload diff --git a/cni/network/network.go b/cni/network/network.go index 2381b8ffc4..65d6e0c17b 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -889,7 +889,7 @@ func (plugin *NetPlugin) Get(args *cniSkel.CmdArgs) error { } // Query the endpoint. - if epInfo, err = plugin.nm.GetEndpointInfo(networkID, endpointID); err != nil { + if epInfo, err = plugin.nm.GetEndpointInfo(networkID, endpointID, args.IfName); err != nil { logger.Error("Failed to query endpoint", zap.Error(err)) return err } @@ -1051,7 +1051,7 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName) // Query the endpoint. - if epInfo, err = plugin.nm.GetEndpointInfo(networkID, endpointID); err != nil { + if epInfo, err = plugin.nm.GetEndpointInfo(networkID, endpointID, args.IfName); err != nil { logger.Info("GetEndpoint", zap.String("endpoint", endpointID), zap.Error(err)) diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index ba0fdfaf4a..3fe382bf18 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -40,7 +40,7 @@ var ( func (plugin *NetPlugin) handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, networkId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig, ) (*cniTypesCurr.Result, error) { - epInfo, _ := plugin.nm.GetEndpointInfo(networkId, endpointId) + epInfo, _ := plugin.nm.GetEndpointInfo(networkId, endpointId, "") if epInfo == nil { return nil, nil } diff --git a/cnm/network/network.go b/cnm/network/network.go index 4358d1a485..ec0957d6b2 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -357,7 +357,7 @@ func (plugin *netPlugin) endpointOperInfo(w http.ResponseWriter, r *http.Request } // Process request. - epInfo, err := plugin.nm.GetEndpointInfo(req.NetworkID, req.EndpointID) + epInfo, err := plugin.nm.GetEndpointInfo(req.NetworkID, req.EndpointID, "") if err != nil { plugin.SendErrorResponse(w, err) return diff --git a/cns/cnireconciler/podinfoprovider.go b/cns/cnireconciler/podinfoprovider.go index 9e3b318005..3330b8e0d3 100644 --- a/cns/cnireconciler/podinfoprovider.go +++ b/cns/cnireconciler/podinfoprovider.go @@ -69,7 +69,6 @@ func cniStateToPodInfoByIP(state *api.AzureCNIState) (map[string]cns.PodInfo, er for _, endpoint := range state.ContainerInterfaces { for _, epIP := range endpoint.IPAddresses { podInfo := cns.NewPodInfo(endpoint.ContainerID, endpoint.PodEndpointId, endpoint.PodName, endpoint.PodNamespace) - logger.Printf("podInfoByIp [%+v]", podInfoByIP) ipKey := epIP.IP.String() if prevPodInfo, ok := podInfoByIP[ipKey]; ok { return nil, errors.Wrapf(cns.ErrDuplicateIP, "duplicate ip %s found for different pods: pod: %+v, pod: %+v", ipKey, podInfo, prevPodInfo) @@ -78,7 +77,6 @@ func cniStateToPodInfoByIP(state *api.AzureCNIState) (map[string]cns.PodInfo, er podInfoByIP[ipKey] = podInfo } } - logger.Printf("podInfoByIP [%+v]", podInfoByIP) return podInfoByIP, nil } @@ -117,7 +115,7 @@ func endpointStateToPodInfoByIP(state map[string]*restserver.EndpointInfo) (map[ // into a EndpointInfo map, using the containerID as keys in the map. // The map then will be saved on CNS endpoint state func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserver.EndpointInfo, error) { - logger.Printf("Generating CNS ENdpoint State") + logger.Printf("Generating CNS Endpoint State") endpointState := map[string]*restserver.EndpointInfo{} for epID, endpoint := range state.ContainerInterfaces { endpointInfo := &restserver.EndpointInfo{PodName: endpoint.PodName, PodNamespace: endpoint.PodNamespace, IfnameToIPMap: make(map[string]*restserver.IPInfo)} @@ -147,6 +145,7 @@ func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserve endpointID, Ifname := extractEndpointInfo(epID, endpoint.ContainerID) endpointInfo.IfnameToIPMap[Ifname] = ipInfo endpointState[endpointID] = endpointInfo + logger.Printf("CNS endpoint state extracted from CNI: [%+v]", *endpointInfo) } return endpointState, nil } diff --git a/cns/cnireconciler/podinfoprovider_test.go b/cns/cnireconciler/podinfoprovider_test.go index 8a6cb85723..da51b840e8 100644 --- a/cns/cnireconciler/podinfoprovider_test.go +++ b/cns/cnireconciler/podinfoprovider_test.go @@ -31,7 +31,13 @@ func TestNewCNIPodInfoProvider(t *testing.T) { { name: "good", exec: newCNIStateFakeExec( - `{"ContainerInterfaces":{"3f813b02-eth0":{"PodName":"metrics-server-77c8679d7d-6ksdh","IfName":"eth0","PodNamespace":"kube-system","PodEndpointID":"3f813b02-eth0","ContainerID":"3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46","IPAddresses":[{"IP":"10.241.0.17","Mask":"//8AAA=="}]},"6e688597-eth0":{"PodName":"tunnelfront-5d96f9b987-65xbn","IfName":"eth0","PodNamespace":"kube-system","PodEndpointID":"6e688597-eth0","ContainerID":"6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed","IPAddresses":[{"IP":"10.241.0.13","Mask":"//8AAA=="}]}}}`, + `{"ContainerInterfaces":{"3f813b02-eth0":{"PodName":"metrics-server-77c8679d7d-6ksdh","IfName":"eth0", + "PodNamespace":"kube-system","PodEndpointID":"3f813b02-eth0", + "ContainerID":"3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46", + "IPAddresses":[{"IP":"10.241.0.17","Mask":"//8AAA=="}]}, + "6e688597-eth0":{"PodName":"tunnelfront-5d96f9b987-65xbn","IfName":"eth0","PodNamespace":"kube-system", + "PodEndpointID":"6e688597-eth0","ContainerID":"6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", + "IPAddresses":[{"IP":"10.241.0.13","Mask":"//8AAA=="}]}}}`, ), want: map[string]cns.PodInfo{ "10.241.0.13": cns.NewPodInfo("6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", "6e688597-eth0", "tunnelfront-5d96f9b987-65xbn", "kube-system"), @@ -39,6 +45,25 @@ func TestNewCNIPodInfoProvider(t *testing.T) { }, wantErr: false, }, + { + name: "dualstack", + exec: newCNIStateFakeExec( + `{"ContainerInterfaces":{"3f813b02-eth0":{"PodName":"metrics-server-77c8679d7d-6ksdh","IfName":"eth0", + "PodNamespace":"kube-system","PodEndpointID":"3f813b02-eth0", + "ContainerID":"3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46", + "IPAddresses":[{"IP":"10.241.0.17","Mask":"//8AAA=="},{"IP":"2001:0db8:abcd:0015::0","Mask":"//8AAA=="}]}, + "6e688597-eth0":{"PodName":"tunnelfront-5d96f9b987-65xbn","IfName":"eth0","PodNamespace":"kube-system", + "PodEndpointID":"6e688597-eth0","ContainerID":"6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", + "IPAddresses":[{"IP":"10.241.0.13","Mask":"//8AAA=="},{"IP":"2001:0db8:abcd:0014::0","Mask":"//8AAA=="}]}}}`, + ), + want: map[string]cns.PodInfo{ + "2001:db8:abcd:15::": cns.NewPodInfo("3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46", "3f813b02-eth0", "metrics-server-77c8679d7d-6ksdh", "kube-system"), + "2001:db8:abcd:14::": cns.NewPodInfo("6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", "6e688597-eth0", "tunnelfront-5d96f9b987-65xbn", "kube-system"), + "10.241.0.17": cns.NewPodInfo("3f813b029429b4e41a09ab33b6f6d365d2ed704017524c78d1d0dece33cdaf46", "3f813b02-eth0", "metrics-server-77c8679d7d-6ksdh", "kube-system"), + "10.241.0.13": cns.NewPodInfo("6e688597eafb97c83c84e402cc72b299bfb8aeb02021e4c99307a037352c0bed", "6e688597-eth0", "tunnelfront-5d96f9b987-65xbn", "kube-system"), + }, + wantErr: false, + }, { name: "empty CNI response", exec: newCNIStateFakeExec( diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index fd49613131..69735de7b7 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -56,6 +56,7 @@ type CNSConfig struct { UseHTTPS bool WatchPods bool `json:"-"` WireserverIP string + StatelessCNIMigration bool } type TelemetrySettings struct { diff --git a/cns/service/main.go b/cns/service/main.go index b0e1901e49..2c155a1ec2 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -839,7 +839,7 @@ func main() { logger.Printf("Set GlobalPodInfoScheme %v (InitializeFromCNI=%t)", cns.GlobalPodInfoScheme, cnsconfig.InitializeFromCNI) - err = InitializeCRDState(rootCtx, httpRestService, cnsconfig, endpointStateStore) + err = InitializeCRDState(rootCtx, httpRestService, cnsconfig) if err != nil { logger.Errorf("Failed to start CRD Controller, err:%v.\n", err) return @@ -1182,7 +1182,7 @@ func reconcileInitialCNSState(ctx context.Context, cli nodeNetworkConfigGetter, } // InitializeCRDState builds and starts the CRD controllers. -func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cnsconfig *configuration.CNSConfig, endpointStateStore store.KeyValueStore) error { +func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cnsconfig *configuration.CNSConfig) error { // convert interface type to implementation type httpRestServiceImplementation, ok := httpRestService.(*restserver.HTTPRestService) if !ok { @@ -1234,26 +1234,9 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn switch { case cnsconfig.ManageEndpointState: logger.Printf("Initializing from self managed endpoint store") - podInfoByIPProvider, err = cnireconciler.NewCNSPodInfoProvider(httpRestServiceImplementation.EndpointStateStore) // get reference to endpoint state store from rest server + podInfoByIPProvider, err = InitializeStateFromCNS(cnsconfig, httpRestServiceImplementation.EndpointStateStore) if err != nil { - if errors.Is(err, store.ErrKeyNotFound) { - logger.Printf("[Azure CNS] No endpoint state found, skipping initializing CNS state") - if cnsconfig.StatelessCNIMigration { - logger.Printf("StatelessCNI Migration is enabled") - logger.Printf("initializing from Statefull CNI") - var endpointState map[string]*restserver.EndpointInfo - podInfoByIPProvider, endpointState, err = cnireconciler.NewCNIPodInfoProvider() - if err != nil { - return errors.Wrap(err, "failed to create CNI PodInfoProvider") - } - err = endpointStateStore.Write(restserver.EndpointStoreKey, endpointState) - if err != nil { - return fmt.Errorf("failed to write endpoint state to store: %w", err) - } - } - } else { - return errors.Wrap(err, "failed to create CNS PodInfoProvider") - } + return errors.Wrap(err, "failed to create CNI PodInfoProvider") } case cnsconfig.InitializeFromCNI: @@ -1530,3 +1513,30 @@ func createOrUpdateNodeInfoCRD(ctx context.Context, restConfig *rest.Config, nod return nil } + +// InitializeStateFromCNS initilizes CNS Endpoint State from CNS or perform the Migration of state from statefull CNI. +func InitializeStateFromCNS(cnsconfig *configuration.CNSConfig, endpointStateStore store.KeyValueStore) (cns.PodInfoByIPProvider, error) { + logger.Printf("Initializing from self managed endpoint store") + podInfoByIPProvider, err := cnireconciler.NewCNSPodInfoProvider(endpointStateStore) // get reference to endpoint state store from rest server + if err != nil { + if errors.Is(err, store.ErrKeyNotFound) { + logger.Printf("[Azure CNS] No endpoint state found, skipping initializing CNS state") + if cnsconfig.StatelessCNIMigration { + logger.Printf("StatelessCNI Migration is enabled") + logger.Printf("initializing from Statefull CNI") + var endpointState map[string]*restserver.EndpointInfo + podInfoByIPProvider, endpointState, err = cnireconciler.NewCNIPodInfoProvider() + if err != nil { + return nil, errors.Wrap(err, "failed to create CNI PodInfoProvider") + } + err = endpointStateStore.Write(restserver.EndpointStoreKey, endpointState) + if err != nil { + return nil, fmt.Errorf("failed to write endpoint state to store: %w", err) + } + } + } else { + return nil, errors.Wrap(err, "failed to create CNS PodInfoProvider") + } + } + return podInfoByIPProvider, nil +} diff --git a/network/endpoint.go b/network/endpoint.go index 79b4be9903..c8b4cce10b 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -364,7 +364,7 @@ func (epInfo *EndpointInfo) GetEndpointInfoByIP(ipAddresses []net.IPNet, network return endpointInfo, nil } -// IsEndpointStateComplete returns true if both HNSEndpointID and HostVethName are missing. +// IsEndpointStateInComplete returns true if both HNSEndpointID and HostVethName are missing. func (epInfo *EndpointInfo) IsEndpointStateIncomplete() bool { if epInfo.HNSEndpointID == "" && epInfo.IfName == "" { return true diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index fff6428403..c0ddf486bf 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -5,7 +5,6 @@ package network import ( "context" - "errors" "fmt" "net" "strings" @@ -16,6 +15,7 @@ import ( "github.com/Azure/azure-container-networking/platform" "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" + "github.com/pkg/errors" "go.uber.org/zap" ) @@ -502,22 +502,22 @@ func (epInfo *EndpointInfo) GetEndpointInfoByIPImpl(ipAddresses []net.IPNet, net // check if network exists, only create the network does not exist hnsResponse, err := Hnsv2.GetNetworkByName(networkID) if err != nil { - return epInfo, errors.Wrap(err, "HNS Network not found") + return epInfo, errors.Wrapf(err, "HNS Network not found") } hcnEndpoints, err := Hnsv2.ListEndpointsOfNetwork(hnsResponse.Id) if err != nil { - return epInfo, errors.Wrap(err, "failed to fetch HNS endpoints for the given network") + return epInfo, errors.Wrapf(err, "failed to fetch HNS endpoints for the given network") } - for _, hcnEndpoint := range hcnEndpoints { - for _, ipConfiguration := range hcnEndpoint.IpConfigurations { + for i := range hcnEndpoints { + for _, ipConfiguration := range hcnEndpoints[i].IpConfigurations { for _, ipAddress := range ipAddresses { prefixLength, _ := ipAddress.Mask.Size() if ipConfiguration.IpAddress == ipAddress.IP.String() && ipConfiguration.PrefixLength == uint8(prefixLength) { - epInfo.HNSEndpointID = hcnEndpoint.Id + epInfo.HNSEndpointID = hcnEndpoints[i].Id return epInfo, nil } } } } - return epInfo, errors.wrap(err, "No HNSEndpointID matches the IPAddress: "+ipAddresses[0].IP.String()) + return epInfo, errors.Wrapf(err, "No HNSEndpointID matches the IPAddress: "+ipAddresses[0].IP.String()) } diff --git a/network/manager.go b/network/manager.go index 064a3274b5..bf95467811 100644 --- a/network/manager.go +++ b/network/manager.go @@ -101,7 +101,7 @@ type NetworkManager interface { CreateEndpoint(client apipaClient, networkID string, epInfo []*EndpointInfo) error DeleteEndpoint(networkID string, endpointID string, epInfo *EndpointInfo) error - GetEndpointInfo(networkID string, endpointID string) (*EndpointInfo, error) + GetEndpointInfo(networkID string, endpointID string, ifName string) (*EndpointInfo, error) GetAllEndpoints(networkID string) (map[string]*EndpointInfo, error) GetEndpointInfoBasedOnPODDetails(networkID string, podName string, podNameSpace string, doExactMatchForPodName bool) (*EndpointInfo, error) AttachEndpoint(networkID string, endpointID string, sandboxKey string) (*endpoint, error) @@ -420,6 +420,38 @@ func (nm *networkManager) UpdateEndpointState(ep *endpoint) error { return nil } +// GetEndpointState will make a call to CNS GetEndpointState API in the stateless CNI mode to fetch the endpointInfo +func (nm *networkManager) GetEndpointState(networkID, endpointID, ifName string) (*EndpointInfo, error) { + endpointResponse, err := nm.CnsClient.GetEndpoint(context.TODO(), endpointID, ifName) + if err != nil { + return nil, errors.Wrapf(err, "Get endpoint API returend with error") + } + epInfo := &EndpointInfo{ + Id: endpointID, + IfIndex: EndpointIfIndex, // Azure CNI supports only one interface + IfName: endpointResponse.EndpointInfo.HostVethName, + ContainerID: endpointID, + PODName: endpointResponse.EndpointInfo.PodName, + PODNameSpace: endpointResponse.EndpointInfo.PodNamespace, + NetworkContainerID: endpointID, + HNSEndpointID: endpointResponse.EndpointInfo.HnsEndpointID, + } + + for _, ip := range endpointResponse.EndpointInfo.IfnameToIPMap { + epInfo.IPAddresses = ip.IPv4 + epInfo.IPAddresses = append(epInfo.IPAddresses, ip.IPv6...) + + } + if epInfo.IsEndpointStateIncomplete() { + epInfo, err = epInfo.GetEndpointInfoByIP(epInfo.IPAddresses, networkID) + if err != nil { + return nil, errors.Wrapf(err, "Get endpoint API returend with error") + } + } + logger.Info("returning getEndpoint API with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", epInfo.HNSEndpointID)) + return epInfo, nil +} + // DeleteEndpoint deletes an existing container endpoint. func (nm *networkManager) DeleteEndpoint(networkID, endpointID string, epInfo *EndpointInfo) error { nm.Lock() @@ -475,48 +507,23 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint } // GetEndpointInfo returns information about the given endpoint. -func (nm *networkManager) GetEndpointInfo(networkId string, endpointId string) (*EndpointInfo, error) { +func (nm *networkManager) GetEndpointInfo(networkID, endpointID, ifName string) (*EndpointInfo, error) { nm.Lock() defer nm.Unlock() if nm.IsStatelessCNIMode() { logger.Info("calling cns getEndpoint API") - endpointResponse, err := nm.CnsClient.GetEndpoint(context.TODO(), endpointId, "eth0") - if err != nil { - return nil, errors.Wrapf(err, "Get endpoint API returend with error") - } - epInfo := &EndpointInfo{ - Id: endpointId, - IfIndex: EndpointIfIndex, // Azure CNI supports only one interface - IfName: endpointResponse.EndpointInfo.HostVethName, - ContainerID: endpointId, - PODName: endpointResponse.EndpointInfo.PodName, - PODNameSpace: endpointResponse.EndpointInfo.PodNamespace, - NetworkContainerID: endpointId, - HNSEndpointID: endpointResponse.EndpointInfo.HnsEndpointID, - } - - for _, ip := range endpointResponse.EndpointInfo.IfnameToIPMap { - epInfo.IPAddresses = ip.IPv4 - epInfo.IPAddresses = append(epInfo.IPAddresses, ip.IPv6...) + epInfo, err := nm.GetEndpointState(networkID, endpointID, ifName) - } - if epInfo.IsEndpointStateIncomplete() { - epInfo, err = epInfo.GetEndpointInfoByIP(epInfo.IPAddresses, networkId) - if err != nil { - return nil, errors.Wrapf(err, "Get endpoint API returend with error") - } - } - logger.Info("returning getEndpoint API with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", epInfo.HNSEndpointID)) - return epInfo, nil + return epInfo, err } - nw, err := nm.getNetwork(networkId) + nw, err := nm.getNetwork(networkID) if err != nil { return nil, err } - ep, err := nw.getEndpoint(endpointId) + ep, err := nw.getEndpoint(endpointID) if err != nil { return nil, err } diff --git a/network/manager_mock.go b/network/manager_mock.go index 0f69315839..2f397c3aca 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -99,7 +99,7 @@ func (nm *MockNetworkManager) GetAllEndpoints(networkID string) (map[string]*End } // GetEndpointInfo mock -func (nm *MockNetworkManager) GetEndpointInfo(networkID string, endpointID string) (*EndpointInfo, error) { +func (nm *MockNetworkManager) GetEndpointInfo(_, endpointID, _ string) (*EndpointInfo, error) { if info, exists := nm.TestEndpointInfoMap[endpointID]; exists { return info, nil } From 5dca27d29b6d77ed09f2744998768899ec57561b Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Thu, 1 Feb 2024 14:02:40 -0800 Subject: [PATCH 14/17] Addressing the comments --- cni/network/network.go | 4 ++-- cni/network/network_windows.go | 2 +- cnm/network/network.go | 2 +- cns/api.go | 1 - cns/client/client.go | 16 +++------------ cns/client/client_test.go | 15 ++------------ cns/cnireconciler/podinfoprovider.go | 17 +++++++++------- cns/cnireconciler/podinfoprovider_test.go | 2 +- cns/restserver/ipam.go | 24 +++++++---------------- cns/service/main.go | 4 ++-- network/endpoint_linux.go | 2 +- network/manager.go | 13 ++++++------ network/manager_mock.go | 2 +- 13 files changed, 38 insertions(+), 66 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 65d6e0c17b..2381b8ffc4 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -889,7 +889,7 @@ func (plugin *NetPlugin) Get(args *cniSkel.CmdArgs) error { } // Query the endpoint. - if epInfo, err = plugin.nm.GetEndpointInfo(networkID, endpointID, args.IfName); err != nil { + if epInfo, err = plugin.nm.GetEndpointInfo(networkID, endpointID); err != nil { logger.Error("Failed to query endpoint", zap.Error(err)) return err } @@ -1051,7 +1051,7 @@ func (plugin *NetPlugin) Delete(args *cniSkel.CmdArgs) error { endpointID := plugin.nm.GetEndpointID(args.ContainerID, args.IfName) // Query the endpoint. - if epInfo, err = plugin.nm.GetEndpointInfo(networkID, endpointID, args.IfName); err != nil { + if epInfo, err = plugin.nm.GetEndpointInfo(networkID, endpointID); err != nil { logger.Info("GetEndpoint", zap.String("endpoint", endpointID), zap.Error(err)) diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 3fe382bf18..ba0fdfaf4a 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -40,7 +40,7 @@ var ( func (plugin *NetPlugin) handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, networkId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig, ) (*cniTypesCurr.Result, error) { - epInfo, _ := plugin.nm.GetEndpointInfo(networkId, endpointId, "") + epInfo, _ := plugin.nm.GetEndpointInfo(networkId, endpointId) if epInfo == nil { return nil, nil } diff --git a/cnm/network/network.go b/cnm/network/network.go index ec0957d6b2..4358d1a485 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -357,7 +357,7 @@ func (plugin *netPlugin) endpointOperInfo(w http.ResponseWriter, r *http.Request } // Process request. - epInfo, err := plugin.nm.GetEndpointInfo(req.NetworkID, req.EndpointID, "") + epInfo, err := plugin.nm.GetEndpointInfo(req.NetworkID, req.EndpointID) if err != nil { plugin.SendErrorResponse(w, err) return diff --git a/cns/api.go b/cns/api.go index 969e437753..3894e9aff6 100644 --- a/cns/api.go +++ b/cns/api.go @@ -363,5 +363,4 @@ type GetHomeAzResponse struct { type EndpointRequest struct { HnsEndpointID string `json:"hnsEndpointID"` HostVethName string `json:"hostVethName"` - IFName string `json:"IFName"` } diff --git a/cns/client/client.go b/cns/client/client.go index 125a135079..2ee524cfb9 100644 --- a/cns/client/client.go +++ b/cns/client/client.go @@ -1024,20 +1024,11 @@ func (c *Client) GetHomeAz(ctx context.Context) (*cns.GetHomeAzResponse, error) } // GetEndpoint calls the EndpointHandlerAPI in CNS to retrieve the state of a given EndpointID -func (c *Client) GetEndpoint(ctx context.Context, endpointID, ifName string) (*restserver.GetEndpointResponse, error) { +func (c *Client) GetEndpoint(ctx context.Context, endpointID string) (*restserver.GetEndpointResponse, error) { // build the request - getEndpoint := cns.EndpointRequest{ - IFName: ifName, - } - var body bytes.Buffer - - if err := json.NewEncoder(&body).Encode(getEndpoint); err != nil { - return nil, errors.Wrap(err, "failed to encode getEndpoint") - } - u := c.routes[cns.EndpointAPI] uString := u.String() + endpointID - req, err := http.NewRequestWithContext(ctx, http.MethodGet, uString, &body) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, uString, http.NoBody) if err != nil { return nil, errors.Wrap(err, "failed to build request") } @@ -1068,12 +1059,11 @@ func (c *Client) GetEndpoint(ctx context.Context, endpointID, ifName string) (*r // UpdateEndpoint calls the EndpointHandlerAPI in CNS // to update the state of a given EndpointID with either HNSEndpointID or HostVethName -func (c *Client) UpdateEndpoint(ctx context.Context, endpointID, hnsID, vethName, ifName string) (*cns.Response, error) { +func (c *Client) UpdateEndpoint(ctx context.Context, endpointID, hnsID, vethName string) (*cns.Response, error) { // build the request updateEndpoint := cns.EndpointRequest{ HnsEndpointID: hnsID, HostVethName: vethName, - IFName: ifName, } var body bytes.Buffer diff --git a/cns/client/client_test.go b/cns/client/client_test.go index 1f53829796..871773eded 100644 --- a/cns/client/client_test.go +++ b/cns/client/client_test.go @@ -2777,7 +2777,6 @@ func TestUpdateEndpoint(t *testing.T) { containerID string hnsID string vethName string - ifName string response *RequestCapture expReq *cns.EndpointRequest shouldErr bool @@ -2787,7 +2786,6 @@ func TestUpdateEndpoint(t *testing.T) { "", "", "", - "", &RequestCapture{ Next: &mockdo{}, }, @@ -2799,7 +2797,6 @@ func TestUpdateEndpoint(t *testing.T) { "foo", "bar", "", - "too", &RequestCapture{ Next: &mockdo{ httpStatusCodeToReturn: http.StatusOK, @@ -2807,7 +2804,6 @@ func TestUpdateEndpoint(t *testing.T) { }, &cns.EndpointRequest{ HnsEndpointID: "bar", - IFName: "too", }, false, }, @@ -2816,7 +2812,6 @@ func TestUpdateEndpoint(t *testing.T) { "foo", "", "bar", - "too", &RequestCapture{ Next: &mockdo{ httpStatusCodeToReturn: http.StatusOK, @@ -2824,7 +2819,6 @@ func TestUpdateEndpoint(t *testing.T) { }, &cns.EndpointRequest{ HostVethName: "bar", - IFName: "too", }, false, }, @@ -2833,7 +2827,6 @@ func TestUpdateEndpoint(t *testing.T) { "foo", "", "bar", - "", &RequestCapture{ Next: &mockdo{ httpStatusCodeToReturn: http.StatusBadRequest, @@ -2858,7 +2851,7 @@ func TestUpdateEndpoint(t *testing.T) { } // execute the method under test - res, err := client.UpdateEndpoint(context.TODO(), test.containerID, test.hnsID, test.vethName, test.ifName) + res, err := client.UpdateEndpoint(context.TODO(), test.containerID, test.hnsID, test.vethName) if err != nil && !test.shouldErr { t.Fatal("unexpected error: err: ", err, res.Message) } @@ -2904,14 +2897,12 @@ func TestGetEndpoint(t *testing.T) { getEndpointTests := []struct { name string containerID string - ifName string response *RequestCapture shouldErr bool }{ { "empty", "", - "", &RequestCapture{ Next: &mockdo{}, }, @@ -2920,7 +2911,6 @@ func TestGetEndpoint(t *testing.T) { { "with EndpointID", "foo", - "foo", &RequestCapture{ Next: &mockdo{ httpStatusCodeToReturn: http.StatusOK, @@ -2931,7 +2921,6 @@ func TestGetEndpoint(t *testing.T) { { "Bad Request", "foo", - "foo", &RequestCapture{ Next: &mockdo{ httpStatusCodeToReturn: http.StatusBadRequest, @@ -2953,7 +2942,7 @@ func TestGetEndpoint(t *testing.T) { } // execute the method under test - res, err := client.GetEndpoint(context.TODO(), test.containerID, test.ifName) + res, err := client.GetEndpoint(context.TODO(), test.containerID) if err != nil && !test.shouldErr { t.Fatal("unexpected error: err: ", err, res.Response.Message) } diff --git a/cns/cnireconciler/podinfoprovider.go b/cns/cnireconciler/podinfoprovider.go index 3330b8e0d3..47fbeb269f 100644 --- a/cns/cnireconciler/podinfoprovider.go +++ b/cns/cnireconciler/podinfoprovider.go @@ -15,12 +15,11 @@ import ( "k8s.io/utils/exec" ) -const InterfaceName = "eth0" - // NewCNIPodInfoProvider returns an implementation of cns.PodInfoByIPProvider // that execs out to the CNI and uses the response to build the PodInfo map. -func NewCNIPodInfoProvider() (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) { - return newCNIPodInfoProvider(exec.New()) +// if stateMigration flag is set to true it will also returns a map of containerID->EndpointInfo +func NewCNIPodInfoProvider(stateMigration bool) (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) { + return newCNIPodInfoProvider(exec.New(), stateMigration) } func NewCNSPodInfoProvider(endpointStore store.KeyValueStore) (cns.PodInfoByIPProvider, error) { @@ -44,7 +43,7 @@ func newCNSPodInfoProvider(endpointStore store.KeyValueStore) (cns.PodInfoByIPPr }), nil } -func newCNIPodInfoProvider(exc exec.Interface) (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) { +func newCNIPodInfoProvider(exc exec.Interface, stateMigration bool) (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) { cli := client.New(exc) state, err := cli.GetEndpointState() if err != nil { @@ -54,7 +53,11 @@ func newCNIPodInfoProvider(exc exec.Interface) (cns.PodInfoByIPProvider, map[str logger.Printf("state dump from CNI: [%+v], [%+v]", containerID, endpointInfo) } var endpointState map[string]*restserver.EndpointInfo - endpointState, err = cniStateToCnsEndpointState(state) + if stateMigration { + endpointState, err = cniStateToCnsEndpointState(state) + } else { + endpointState = nil + } return cns.PodInfoByIPProviderFunc(func() (map[string]cns.PodInfo, error) { return cniStateToPodInfoByIP(state) }), endpointState, err @@ -152,7 +155,7 @@ func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserve // extractEndpointInfo extract Interface Name and endpointID for each endpoint based the CNI state func extractEndpointInfo(epID, containerID string) (endpointID, interfaceName string) { - ifName := InterfaceName + ifName := restserver.InterfaceName if strings.Contains(epID, "-eth") { ifName = epID[len(epID)-4:] } diff --git a/cns/cnireconciler/podinfoprovider_test.go b/cns/cnireconciler/podinfoprovider_test.go index da51b840e8..022c805251 100644 --- a/cns/cnireconciler/podinfoprovider_test.go +++ b/cns/cnireconciler/podinfoprovider_test.go @@ -76,7 +76,7 @@ func TestNewCNIPodInfoProvider(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - got, endpointState, err := newCNIPodInfoProvider(tt.exec) + got, endpointState, err := newCNIPodInfoProvider(tt.exec, true) if tt.wantErr { assert.Error(t, err) return diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index dbb5bbffca..66459c5a68 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -30,7 +30,10 @@ var ( ErrExistingIpconfigFound = errors.New("Found existing ipconfig for infra container") ) -const ContainerIDLength = 8 +const ( + ContainerIDLength = 8 + InterfaceName = "eth0" +) // requestIPConfigHandlerHelper validates the request, assign IPs and return the IPConfigs func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { @@ -1011,22 +1014,9 @@ func (service *HTTPRestService) EndpointHandlerAPI(w http.ResponseWriter, r *htt // GetEndpointHandler handles the incoming GetEndpoint requests with http Get method func (service *HTTPRestService) GetEndpointHandler(w http.ResponseWriter, r *http.Request) { logger.Printf("[GetEndpointState] GetEndpoint for %s", r.URL.Path) - var req cns.EndpointRequest - err := service.Listener.Decode(w, r, &req) endpointID := strings.TrimPrefix(r.URL.Path, cns.EndpointPath) - logger.Request(service.Name, &req, err) + endpointInfo, err := service.GetEndpointHelper(endpointID) // Check if the request is valid - if err != nil || req.IFName == "" { - response := cns.Response{ - ReturnCode: types.InvalidRequest, - Message: fmt.Sprintf("[getEndpoint] getEndpoint failed with error: %s", err.Error()), - } - w.Header().Set(cnsReturnCode, response.ReturnCode.String()) - err = service.Listener.Encode(w, &response) - logger.Response(service.Name, response, response.ReturnCode, err) - return - } - endpointInfo, err := service.GetEndpointHelper(endpointID, req) if err != nil { response := GetEndpointResponse{ Response: Response{ @@ -1060,7 +1050,7 @@ func (service *HTTPRestService) GetEndpointHandler(w http.ResponseWriter, r *htt } // GetEndpointHelper returns the state of the given endpointId -func (service *HTTPRestService) GetEndpointHelper(endpointID string, req cns.EndpointRequest) (*EndpointInfo, error) { +func (service *HTTPRestService) GetEndpointHelper(endpointID string) (*EndpointInfo, error) { logger.Printf("[GetEndpointState] Get endpoint state for infra container %s", endpointID) // Skip if a store is not provided. @@ -1084,7 +1074,7 @@ func (service *HTTPRestService) GetEndpointHelper(endpointID string, req cns.End logger.Warnf("[GetEndpointState] Found existing endpoint state for container %s", endpointID) return endpointInfo, nil } - legacyEndpointID := endpointID[:ContainerIDLength] + "-" + req.IFName + legacyEndpointID := endpointID[:ContainerIDLength] + "-" + InterfaceName if endpointInfo, ok := service.EndpointState[legacyEndpointID]; ok { logger.Warnf("[GetEndpointState] Found existing endpoint state for container %s", legacyEndpointID) return endpointInfo, nil diff --git a/cns/service/main.go b/cns/service/main.go index 2c155a1ec2..e274c847f5 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1241,7 +1241,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn case cnsconfig.InitializeFromCNI: logger.Printf("Initializing from CNI") - podInfoByIPProvider, _, err = cnireconciler.NewCNIPodInfoProvider() + podInfoByIPProvider, _, err = cnireconciler.NewCNIPodInfoProvider(false) if err != nil { return errors.Wrap(err, "failed to create CNI PodInfoProvider") } @@ -1525,7 +1525,7 @@ func InitializeStateFromCNS(cnsconfig *configuration.CNSConfig, endpointStateSto logger.Printf("StatelessCNI Migration is enabled") logger.Printf("initializing from Statefull CNI") var endpointState map[string]*restserver.EndpointInfo - podInfoByIPProvider, endpointState, err = cnireconciler.NewCNIPodInfoProvider() + podInfoByIPProvider, endpointState, err = cnireconciler.NewCNIPodInfoProvider(cnsconfig.StatelessCNIMigration) if err != nil { return nil, errors.Wrap(err, "failed to create CNI PodInfoProvider") } diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 766d432603..179d28cc8c 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -533,7 +533,7 @@ func getDefaultGateway(routes []RouteInfo) net.IP { } // GetEndpointInfoByIPImpl returns an endpointInfo that contains corresponding HostVethName. -// TODO: It needs to be tested to see if HostVethName is required for SingleTenancy +// TODO: It needs to be tested to see if HostVethName is required for SingleTenancy, WorkItem: 26606939 func (epInfo *EndpointInfo) GetEndpointInfoByIPImpl(_ []net.IPNet, _ string) (*EndpointInfo, error) { return epInfo, nil } diff --git a/network/manager.go b/network/manager.go index bf95467811..19e46498f0 100644 --- a/network/manager.go +++ b/network/manager.go @@ -101,7 +101,7 @@ type NetworkManager interface { CreateEndpoint(client apipaClient, networkID string, epInfo []*EndpointInfo) error DeleteEndpoint(networkID string, endpointID string, epInfo *EndpointInfo) error - GetEndpointInfo(networkID string, endpointID string, ifName string) (*EndpointInfo, error) + GetEndpointInfo(networkID string, endpointID string) (*EndpointInfo, error) GetAllEndpoints(networkID string) (map[string]*EndpointInfo, error) GetEndpointInfoBasedOnPODDetails(networkID string, podName string, podNameSpace string, doExactMatchForPodName bool) (*EndpointInfo, error) AttachEndpoint(networkID string, endpointID string, sandboxKey string) (*endpoint, error) @@ -412,7 +412,7 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn // It will add HNSEndpointID or HostVeth name to the endpoint state func (nm *networkManager) UpdateEndpointState(ep *endpoint) error { logger.Info("Calling cns updateEndpoint API with ", zap.String("containerID: ", ep.ContainerID), zap.String("HnsId: ", ep.HnsId), zap.String("HostIfName: ", ep.HostIfName)) - response, err := nm.CnsClient.UpdateEndpoint(context.TODO(), ep.ContainerID, ep.HnsId, ep.HostIfName, ep.IfName) + response, err := nm.CnsClient.UpdateEndpoint(context.TODO(), ep.ContainerID, ep.HnsId, ep.HostIfName) if err != nil { return errors.Wrapf(err, "Update endpoint API returend with error") } @@ -421,8 +421,9 @@ func (nm *networkManager) UpdateEndpointState(ep *endpoint) error { } // GetEndpointState will make a call to CNS GetEndpointState API in the stateless CNI mode to fetch the endpointInfo -func (nm *networkManager) GetEndpointState(networkID, endpointID, ifName string) (*EndpointInfo, error) { - endpointResponse, err := nm.CnsClient.GetEndpoint(context.TODO(), endpointID, ifName) +// TODO unit tests need to be added, WorkItem: 26606939 +func (nm *networkManager) GetEndpointState(networkID, endpointID string) (*EndpointInfo, error) { + endpointResponse, err := nm.CnsClient.GetEndpoint(context.TODO(), endpointID) if err != nil { return nil, errors.Wrapf(err, "Get endpoint API returend with error") } @@ -507,13 +508,13 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint } // GetEndpointInfo returns information about the given endpoint. -func (nm *networkManager) GetEndpointInfo(networkID, endpointID, ifName string) (*EndpointInfo, error) { +func (nm *networkManager) GetEndpointInfo(networkID, endpointID string) (*EndpointInfo, error) { nm.Lock() defer nm.Unlock() if nm.IsStatelessCNIMode() { logger.Info("calling cns getEndpoint API") - epInfo, err := nm.GetEndpointState(networkID, endpointID, ifName) + epInfo, err := nm.GetEndpointState(networkID, endpointID) return epInfo, err } diff --git a/network/manager_mock.go b/network/manager_mock.go index 2f397c3aca..188d2bb2ad 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -99,7 +99,7 @@ func (nm *MockNetworkManager) GetAllEndpoints(networkID string) (map[string]*End } // GetEndpointInfo mock -func (nm *MockNetworkManager) GetEndpointInfo(_, endpointID, _ string) (*EndpointInfo, error) { +func (nm *MockNetworkManager) GetEndpointInfo(_, endpointID string) (*EndpointInfo, error) { if info, exists := nm.TestEndpointInfoMap[endpointID]; exists { return info, nil } From 78f692244a9b77cb572730a7dfcd959c36a56b26 Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Wed, 7 Feb 2024 15:30:50 -0800 Subject: [PATCH 15/17] addressing the latest comments --- cns/cnireconciler/podinfoprovider.go | 18 +++--------------- cns/configuration/configuration.go | 2 +- cns/restserver/ipam.go | 4 +++- cns/service/main.go | 4 ++-- 4 files changed, 9 insertions(+), 19 deletions(-) diff --git a/cns/cnireconciler/podinfoprovider.go b/cns/cnireconciler/podinfoprovider.go index 47fbeb269f..68a7d1d864 100644 --- a/cns/cnireconciler/podinfoprovider.go +++ b/cns/cnireconciler/podinfoprovider.go @@ -54,7 +54,7 @@ func newCNIPodInfoProvider(exc exec.Interface, stateMigration bool) (cns.PodInfo } var endpointState map[string]*restserver.EndpointInfo if stateMigration { - endpointState, err = cniStateToCnsEndpointState(state) + endpointState = cniStateToCnsEndpointState(state) } else { endpointState = nil } @@ -117,7 +117,7 @@ func endpointStateToPodInfoByIP(state map[string]*restserver.EndpointInfo) (map[ // cniStateToCnsEndpointState converts an AzureCNIState dumped from a CNI exec // into a EndpointInfo map, using the containerID as keys in the map. // The map then will be saved on CNS endpoint state -func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserver.EndpointInfo, error) { +func cniStateToCnsEndpointState(state *api.AzureCNIState) map[string]*restserver.EndpointInfo { logger.Printf("Generating CNS Endpoint State") endpointState := map[string]*restserver.EndpointInfo{} for epID, endpoint := range state.ContainerInterfaces { @@ -126,22 +126,10 @@ func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserve for _, epIP := range endpoint.IPAddresses { if epIP.IP.To4() == nil { // is an ipv6 address ipconfig := net.IPNet{IP: epIP.IP, Mask: epIP.Mask} - for _, ipconf := range ipInfo.IPv6 { - if ipconf.IP.Equal(ipconfig.IP) { - logger.Errorf("Found existing ipv6 ipconfig for infra container %s", endpoint.ContainerID) - return nil, restserver.ErrExistingIpconfigFound - } - } ipInfo.IPv6 = append(ipInfo.IPv6, ipconfig) } else { ipconfig := net.IPNet{IP: epIP.IP, Mask: epIP.Mask} - for _, ipconf := range ipInfo.IPv4 { - if ipconf.IP.Equal(ipconfig.IP) { - logger.Errorf("Found existing ipv4 ipconfig for infra container %s", endpoint.ContainerID) - return nil, restserver.ErrExistingIpconfigFound - } - } ipInfo.IPv4 = append(ipInfo.IPv4, ipconfig) } } @@ -150,7 +138,7 @@ func cniStateToCnsEndpointState(state *api.AzureCNIState) (map[string]*restserve endpointState[endpointID] = endpointInfo logger.Printf("CNS endpoint state extracted from CNI: [%+v]", *endpointInfo) } - return endpointState, nil + return endpointState } // extractEndpointInfo extract Interface Name and endpointID for each endpoint based the CNI state diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index 69735de7b7..c6e636b276 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -56,7 +56,7 @@ type CNSConfig struct { UseHTTPS bool WatchPods bool `json:"-"` WireserverIP string - StatelessCNIMigration bool + StateMigration bool } type TelemetrySettings struct { diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 66459c5a68..522e1a38d5 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -27,7 +27,6 @@ var ( ErrNoNCs = errors.New("no NCs found in the CNS internal state") ErrOptManageEndpointState = errors.New("CNS is not set to manage the endpoint state") ErrEndpointStateNotFound = errors.New("endpoint state could not be found in the statefile") - ErrExistingIpconfigFound = errors.New("Found existing ipconfig for infra container") ) const ( @@ -1074,6 +1073,9 @@ func (service *HTTPRestService) GetEndpointHelper(endpointID string) (*EndpointI logger.Warnf("[GetEndpointState] Found existing endpoint state for container %s", endpointID) return endpointInfo, nil } + // This part is a temprory fix if we have endpoint states belong to CNI version 1.4.X on Windows since the states don't have the containerID + // In case there was no endpoint founded with ContainerID as the key, + // then [First 8 character of containerid]-eth0 will be tried legacyEndpointID := endpointID[:ContainerIDLength] + "-" + InterfaceName if endpointInfo, ok := service.EndpointState[legacyEndpointID]; ok { logger.Warnf("[GetEndpointState] Found existing endpoint state for container %s", legacyEndpointID) diff --git a/cns/service/main.go b/cns/service/main.go index e274c847f5..cf55559595 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1521,11 +1521,11 @@ func InitializeStateFromCNS(cnsconfig *configuration.CNSConfig, endpointStateSto if err != nil { if errors.Is(err, store.ErrKeyNotFound) { logger.Printf("[Azure CNS] No endpoint state found, skipping initializing CNS state") - if cnsconfig.StatelessCNIMigration { + if cnsconfig.StateMigration { logger.Printf("StatelessCNI Migration is enabled") logger.Printf("initializing from Statefull CNI") var endpointState map[string]*restserver.EndpointInfo - podInfoByIPProvider, endpointState, err = cnireconciler.NewCNIPodInfoProvider(cnsconfig.StatelessCNIMigration) + podInfoByIPProvider, endpointState, err = cnireconciler.NewCNIPodInfoProvider(cnsconfig.StateMigration) if err != nil { return nil, errors.Wrap(err, "failed to create CNI PodInfoProvider") } From ac43bb28b6c73e3634e96c517be839c9e6870acf Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Thu, 8 Feb 2024 12:06:26 -0800 Subject: [PATCH 16/17] Addressing Evan's comments --- cns/cnireconciler/podinfoprovider.go | 21 +++++++------- cns/cnireconciler/podinfoprovider_test.go | 2 +- cns/configuration/configuration.go | 2 +- cns/service/main.go | 34 +++++++++++------------ network/endpoint.go | 10 ------- network/manager.go | 2 +- 6 files changed, 29 insertions(+), 42 deletions(-) diff --git a/cns/cnireconciler/podinfoprovider.go b/cns/cnireconciler/podinfoprovider.go index 68a7d1d864..3b81bf7f73 100644 --- a/cns/cnireconciler/podinfoprovider.go +++ b/cns/cnireconciler/podinfoprovider.go @@ -17,9 +17,14 @@ import ( // NewCNIPodInfoProvider returns an implementation of cns.PodInfoByIPProvider // that execs out to the CNI and uses the response to build the PodInfo map. -// if stateMigration flag is set to true it will also returns a map of containerID->EndpointInfo -func NewCNIPodInfoProvider(stateMigration bool) (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) { - return newCNIPodInfoProvider(exec.New(), stateMigration) +// if EnableStateMigration flag is set to true it will also returns a map of containerID->EndpointInfo +func NewCNIPodInfoProvider(enableStateMigration bool) (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) { + podInfoByIPProvider, cniState, err := newCNIPodInfoProvider(exec.New()) + if enableStateMigration { + endpointState := cniStateToCnsEndpointState(cniState) + return podInfoByIPProvider, endpointState, err + } + return podInfoByIPProvider, nil, err } func NewCNSPodInfoProvider(endpointStore store.KeyValueStore) (cns.PodInfoByIPProvider, error) { @@ -43,7 +48,7 @@ func newCNSPodInfoProvider(endpointStore store.KeyValueStore) (cns.PodInfoByIPPr }), nil } -func newCNIPodInfoProvider(exc exec.Interface, stateMigration bool) (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) { +func newCNIPodInfoProvider(exc exec.Interface) (cns.PodInfoByIPProvider, *api.AzureCNIState, error) { cli := client.New(exc) state, err := cli.GetEndpointState() if err != nil { @@ -52,15 +57,9 @@ func newCNIPodInfoProvider(exc exec.Interface, stateMigration bool) (cns.PodInfo for containerID, endpointInfo := range state.ContainerInterfaces { logger.Printf("state dump from CNI: [%+v], [%+v]", containerID, endpointInfo) } - var endpointState map[string]*restserver.EndpointInfo - if stateMigration { - endpointState = cniStateToCnsEndpointState(state) - } else { - endpointState = nil - } return cns.PodInfoByIPProviderFunc(func() (map[string]cns.PodInfo, error) { return cniStateToPodInfoByIP(state) - }), endpointState, err + }), state, err } // cniStateToPodInfoByIP converts an AzureCNIState dumped from a CNI exec diff --git a/cns/cnireconciler/podinfoprovider_test.go b/cns/cnireconciler/podinfoprovider_test.go index 022c805251..da51b840e8 100644 --- a/cns/cnireconciler/podinfoprovider_test.go +++ b/cns/cnireconciler/podinfoprovider_test.go @@ -76,7 +76,7 @@ func TestNewCNIPodInfoProvider(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - got, endpointState, err := newCNIPodInfoProvider(tt.exec, true) + got, endpointState, err := newCNIPodInfoProvider(tt.exec) if tt.wantErr { assert.Error(t, err) return diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index c6e636b276..2e56a9841f 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -35,6 +35,7 @@ type CNSConfig struct { EnableCNIConflistGeneration bool EnableIPAMv2 bool EnablePprof bool + EnableStateMigration bool EnableSubnetScarcity bool EnableSwiftV2 bool InitializeFromCNI bool @@ -56,7 +57,6 @@ type CNSConfig struct { UseHTTPS bool WatchPods bool `json:"-"` WireserverIP string - StateMigration bool } type TelemetrySettings struct { diff --git a/cns/service/main.go b/cns/service/main.go index cf55559595..f0b2cade4c 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1516,27 +1516,25 @@ func createOrUpdateNodeInfoCRD(ctx context.Context, restConfig *rest.Config, nod // InitializeStateFromCNS initilizes CNS Endpoint State from CNS or perform the Migration of state from statefull CNI. func InitializeStateFromCNS(cnsconfig *configuration.CNSConfig, endpointStateStore store.KeyValueStore) (cns.PodInfoByIPProvider, error) { + if cnsconfig.EnableStateMigration && !endpointStateStore.Exists() { // initilize form CNI and perform state migration + logger.Printf("StatelessCNI Migration is enabled") + logger.Printf("initializing from Statefull CNI") + var endpointState map[string]*restserver.EndpointInfo + podInfoByIPProvider, endpointState, err := cnireconciler.NewCNIPodInfoProvider(cnsconfig.EnableStateMigration) + if err != nil { + return nil, errors.Wrap(err, "failed to create CNI PodInfoProvider") + } + err = endpointStateStore.Write(restserver.EndpointStoreKey, endpointState) + if err != nil { + return nil, fmt.Errorf("failed to write endpoint state to store: %w", err) + } + return podInfoByIPProvider, nil + } + // initilize form CNS and avoid state migration logger.Printf("Initializing from self managed endpoint store") podInfoByIPProvider, err := cnireconciler.NewCNSPodInfoProvider(endpointStateStore) // get reference to endpoint state store from rest server if err != nil { - if errors.Is(err, store.ErrKeyNotFound) { - logger.Printf("[Azure CNS] No endpoint state found, skipping initializing CNS state") - if cnsconfig.StateMigration { - logger.Printf("StatelessCNI Migration is enabled") - logger.Printf("initializing from Statefull CNI") - var endpointState map[string]*restserver.EndpointInfo - podInfoByIPProvider, endpointState, err = cnireconciler.NewCNIPodInfoProvider(cnsconfig.StateMigration) - if err != nil { - return nil, errors.Wrap(err, "failed to create CNI PodInfoProvider") - } - err = endpointStateStore.Write(restserver.EndpointStoreKey, endpointState) - if err != nil { - return nil, fmt.Errorf("failed to write endpoint state to store: %w", err) - } - } - } else { - return nil, errors.Wrap(err, "failed to create CNS PodInfoProvider") - } + return nil, errors.Wrap(err, "failed to create CNS PodInfoProvider") } return podInfoByIPProvider, nil } diff --git a/network/endpoint.go b/network/endpoint.go index c8b4cce10b..19202c8be7 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -354,16 +354,6 @@ func GetPodNameWithoutSuffix(podName string) string { return strings.Join(nameSplit, "-") } -// GetEndpointInfoByIP returns an EndpointInfo with complete state when HNS Endpoint ID or HostVeth Name are missing on the CNS state. -func (epInfo *EndpointInfo) GetEndpointInfoByIP(ipAddresses []net.IPNet, networkID string) (*EndpointInfo, error) { - // Call the platform implementation. - endpointInfo, err := epInfo.GetEndpointInfoByIPImpl(ipAddresses, networkID) - if err != nil { - return nil, err - } - return endpointInfo, nil -} - // IsEndpointStateInComplete returns true if both HNSEndpointID and HostVethName are missing. func (epInfo *EndpointInfo) IsEndpointStateIncomplete() bool { if epInfo.HNSEndpointID == "" && epInfo.IfName == "" { diff --git a/network/manager.go b/network/manager.go index 19e46498f0..5064a7265a 100644 --- a/network/manager.go +++ b/network/manager.go @@ -444,7 +444,7 @@ func (nm *networkManager) GetEndpointState(networkID, endpointID string) (*Endpo } if epInfo.IsEndpointStateIncomplete() { - epInfo, err = epInfo.GetEndpointInfoByIP(epInfo.IPAddresses, networkID) + epInfo, err = epInfo.GetEndpointInfoByIPImpl(epInfo.IPAddresses, networkID) if err != nil { return nil, errors.Wrapf(err, "Get endpoint API returend with error") } From bf4a00e18e6378a8affc849038acfcaebcb21846 Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Thu, 8 Feb 2024 13:50:29 -0800 Subject: [PATCH 17/17] Adding a MigrateSate() function to the cnireconciler --- cns/cnireconciler/podinfoprovider.go | 37 ++++++++++-------- cns/cnireconciler/podinfoprovider_test.go | 4 +- cns/service/main.go | 47 +++++++++++------------ 3 files changed, 47 insertions(+), 41 deletions(-) diff --git a/cns/cnireconciler/podinfoprovider.go b/cns/cnireconciler/podinfoprovider.go index 3b81bf7f73..69bb7db519 100644 --- a/cns/cnireconciler/podinfoprovider.go +++ b/cns/cnireconciler/podinfoprovider.go @@ -17,14 +17,8 @@ import ( // NewCNIPodInfoProvider returns an implementation of cns.PodInfoByIPProvider // that execs out to the CNI and uses the response to build the PodInfo map. -// if EnableStateMigration flag is set to true it will also returns a map of containerID->EndpointInfo -func NewCNIPodInfoProvider(enableStateMigration bool) (cns.PodInfoByIPProvider, map[string]*restserver.EndpointInfo, error) { - podInfoByIPProvider, cniState, err := newCNIPodInfoProvider(exec.New()) - if enableStateMigration { - endpointState := cniStateToCnsEndpointState(cniState) - return podInfoByIPProvider, endpointState, err - } - return podInfoByIPProvider, nil, err +func NewCNIPodInfoProvider() (cns.PodInfoByIPProvider, error) { + return newCNIPodInfoProvider(exec.New()) } func NewCNSPodInfoProvider(endpointStore store.KeyValueStore) (cns.PodInfoByIPProvider, error) { @@ -48,18 +42,15 @@ func newCNSPodInfoProvider(endpointStore store.KeyValueStore) (cns.PodInfoByIPPr }), nil } -func newCNIPodInfoProvider(exc exec.Interface) (cns.PodInfoByIPProvider, *api.AzureCNIState, error) { - cli := client.New(exc) +func newCNIPodInfoProvider(exec exec.Interface) (cns.PodInfoByIPProvider, error) { + cli := client.New(exec) state, err := cli.GetEndpointState() if err != nil { - return nil, nil, fmt.Errorf("failed to invoke CNI client.GetEndpointState(): %w", err) - } - for containerID, endpointInfo := range state.ContainerInterfaces { - logger.Printf("state dump from CNI: [%+v], [%+v]", containerID, endpointInfo) + return nil, fmt.Errorf("failed to invoke CNI client.GetEndpointState(): %w", err) } return cns.PodInfoByIPProviderFunc(func() (map[string]cns.PodInfo, error) { return cniStateToPodInfoByIP(state) - }), state, err + }), nil } // cniStateToPodInfoByIP converts an AzureCNIState dumped from a CNI exec @@ -71,6 +62,7 @@ func cniStateToPodInfoByIP(state *api.AzureCNIState) (map[string]cns.PodInfo, er for _, endpoint := range state.ContainerInterfaces { for _, epIP := range endpoint.IPAddresses { podInfo := cns.NewPodInfo(endpoint.ContainerID, endpoint.PodEndpointId, endpoint.PodName, endpoint.PodNamespace) + ipKey := epIP.IP.String() if prevPodInfo, ok := podInfoByIP[ipKey]; ok { return nil, errors.Wrapf(cns.ErrDuplicateIP, "duplicate ip %s found for different pods: pod: %+v, pod: %+v", ipKey, podInfo, prevPodInfo) @@ -113,6 +105,21 @@ func endpointStateToPodInfoByIP(state map[string]*restserver.EndpointInfo) (map[ return podInfoByIP, nil } +// MigrateCNISate returns an endpoint state of CNS by reading the CNI state file +func MigrateCNISate() (map[string]*restserver.EndpointInfo, error) { + return migrateCNISate(exec.New()) +} + +func migrateCNISate(exc exec.Interface) (map[string]*restserver.EndpointInfo, error) { + cli := client.New(exc) + state, err := cli.GetEndpointState() + if err != nil { + return nil, fmt.Errorf("failed to invoke CNI client.GetEndpointState(): %w", err) + } + endpointState := cniStateToCnsEndpointState(state) + return endpointState, nil +} + // cniStateToCnsEndpointState converts an AzureCNIState dumped from a CNI exec // into a EndpointInfo map, using the containerID as keys in the map. // The map then will be saved on CNS endpoint state diff --git a/cns/cnireconciler/podinfoprovider_test.go b/cns/cnireconciler/podinfoprovider_test.go index da51b840e8..8d10b1c586 100644 --- a/cns/cnireconciler/podinfoprovider_test.go +++ b/cns/cnireconciler/podinfoprovider_test.go @@ -76,14 +76,14 @@ func TestNewCNIPodInfoProvider(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - got, endpointState, err := newCNIPodInfoProvider(tt.exec) + got, err := newCNIPodInfoProvider(tt.exec) if tt.wantErr { assert.Error(t, err) return } assert.NoError(t, err) podInfoByIP, _ := got.PodInfoByIP() - assert.Equal(t, tt.want, podInfoByIP, endpointState) + assert.Equal(t, tt.want, podInfoByIP) }) } } diff --git a/cns/service/main.go b/cns/service/main.go index f0b2cade4c..0343773637 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1230,18 +1230,28 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn } } + // perform state migration from CNI in case CNS is set to manage the endpoint state and has emty state + if cnsconfig.EnableStateMigration && !httpRestServiceImplementation.EndpointStateStore.Exists() { + if err = PopulateCNSEndpointState(httpRestServiceImplementation.EndpointStateStore); err != nil { + return errors.Wrap(err, "failed to create CNS EndpointState From CNI") + } + } + var podInfoByIPProvider cns.PodInfoByIPProvider switch { case cnsconfig.ManageEndpointState: logger.Printf("Initializing from self managed endpoint store") - podInfoByIPProvider, err = InitializeStateFromCNS(cnsconfig, httpRestServiceImplementation.EndpointStateStore) + podInfoByIPProvider, err = cnireconciler.NewCNSPodInfoProvider(httpRestServiceImplementation.EndpointStateStore) // get reference to endpoint state store from rest server if err != nil { - return errors.Wrap(err, "failed to create CNI PodInfoProvider") + if errors.Is(err, store.ErrKeyNotFound) { + logger.Printf("[Azure CNS] No endpoint state found, skipping initializing CNS state") + } else { + return errors.Wrap(err, "failed to create CNS PodInfoProvider") + } } - case cnsconfig.InitializeFromCNI: logger.Printf("Initializing from CNI") - podInfoByIPProvider, _, err = cnireconciler.NewCNIPodInfoProvider(false) + podInfoByIPProvider, err = cnireconciler.NewCNIPodInfoProvider() if err != nil { return errors.Wrap(err, "failed to create CNI PodInfoProvider") } @@ -1514,27 +1524,16 @@ func createOrUpdateNodeInfoCRD(ctx context.Context, restConfig *rest.Config, nod return nil } -// InitializeStateFromCNS initilizes CNS Endpoint State from CNS or perform the Migration of state from statefull CNI. -func InitializeStateFromCNS(cnsconfig *configuration.CNSConfig, endpointStateStore store.KeyValueStore) (cns.PodInfoByIPProvider, error) { - if cnsconfig.EnableStateMigration && !endpointStateStore.Exists() { // initilize form CNI and perform state migration - logger.Printf("StatelessCNI Migration is enabled") - logger.Printf("initializing from Statefull CNI") - var endpointState map[string]*restserver.EndpointInfo - podInfoByIPProvider, endpointState, err := cnireconciler.NewCNIPodInfoProvider(cnsconfig.EnableStateMigration) - if err != nil { - return nil, errors.Wrap(err, "failed to create CNI PodInfoProvider") - } - err = endpointStateStore.Write(restserver.EndpointStoreKey, endpointState) - if err != nil { - return nil, fmt.Errorf("failed to write endpoint state to store: %w", err) - } - return podInfoByIPProvider, nil +// PopulateCNSEndpointState initilizes CNS Endpoint State by Migrating the CNI state. +func PopulateCNSEndpointState(endpointStateStore store.KeyValueStore) error { + logger.Printf("State Migration is enabled") + endpointState, err := cnireconciler.MigrateCNISate() + if err != nil { + return errors.Wrap(err, "failed to create CNS Endpoint state from CNI") } - // initilize form CNS and avoid state migration - logger.Printf("Initializing from self managed endpoint store") - podInfoByIPProvider, err := cnireconciler.NewCNSPodInfoProvider(endpointStateStore) // get reference to endpoint state store from rest server + err = endpointStateStore.Write(restserver.EndpointStoreKey, endpointState) if err != nil { - return nil, errors.Wrap(err, "failed to create CNS PodInfoProvider") + return fmt.Errorf("failed to write endpoint state to store: %w", err) } - return podInfoByIPProvider, nil + return nil }