From 792751317567ae9ecf801e5432bb38ee2d825238 Mon Sep 17 00:00:00 2001 From: Hongliang Liu Date: Fri, 12 Jul 2024 16:36:31 +0800 Subject: [PATCH] Add e2e tests for BGPPolicy Signed-off-by: Hongliang Liu --- test/e2e/bgppolicy_test.go | 319 +++++++++++++++++++++++++++++++++++++ test/e2e/fixtures.go | 10 +- test/e2e/framework.go | 71 ++++++++- 3 files changed, 394 insertions(+), 6 deletions(-) create mode 100644 test/e2e/bgppolicy_test.go diff --git a/test/e2e/bgppolicy_test.go b/test/e2e/bgppolicy_test.go new file mode 100644 index 00000000000..3a00f177100 --- /dev/null +++ b/test/e2e/bgppolicy_test.go @@ -0,0 +1,319 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "fmt" + "log" + "regexp" + "sort" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" + "antrea.io/antrea/pkg/features" + "antrea.io/antrea/test/e2e/providers/exec" +) + +type BGPPolicySpecBuilder struct { + Spec crdv1alpha1.BGPPolicySpec + Name string +} + +var ( + remoteASN = int32(65000) + + localASN = int32(64512) + updatedLocalASN = int32(64513) + + password = "password" + defaultBGPPolicySecretName = "antrea-bgp-passwords" // #nosec G101 + + bpName = "test-bp" +) + +func skipIfBGPPolicyDisabled(tb testing.TB) { + skipIfFeatureDisabled(tb, features.BGPPolicy, true, false) +} + +func getAllNodeIPs() []string { + ips := make([]string, 0, clusterInfo.numNodes) + for _, node := range clusterInfo.nodes { + ips = append(ips, node.ipv4Addr) + } + return ips +} + +type FRRRoute struct { + Prefix string + Nexthops []string +} + +func (f *FRRRoute) String() string { + sort.Strings(f.Nexthops) + return fmt.Sprintf("%s via %s", f.Prefix, strings.Join(f.Nexthops, ",")) +} + +func routesToStrings(routes []FRRRoute) []string { + s := make([]string, 0, len(routes)) + for _, route := range routes { + s = append(s, route.String()) + } + return s +} + +func TestBGPPolicy(t *testing.T) { + skipIfBGPPolicyDisabled(t) + skipIfProviderIsNot(t, "kind", "This test is only supported in KinD") + data, err := setupTest(t) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + defer teardownTest(t, data) + + t.Log("Configure the remote FRR router with BGP") + configureFRRRouterBGP(t, remoteASN, localASN) + defer cleanupFRRRouterBGP(t, remoteASN) + + t.Log("Update the specific Secret storing the passwords of BGP peers") + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: kubeNamespace, + Name: defaultBGPPolicySecretName, + }, + Data: map[string][]byte{ + fmt.Sprintf("%s-%d", externalInfo.externalFRRIPv4, remoteASN): []byte(password), + }, + } + _, err = data.clientset.CoreV1().Secrets(kubeNamespace).Create(context.TODO(), secret, metav1.CreateOptions{}) + require.NoError(t, err) + defer func() { + data.clientset.CoreV1().Secrets(kubeNamespace).Delete(context.TODO(), defaultBGPPolicySecretName, metav1.DeleteOptions{}) + }() + + t.Log("Create a test agnhost Pod") + _, podIPs, cleanupFunc := createAndWaitForPod(t, data, data.createAgnhostPodWithHTTPOnNode, "agnhost-", nodeName(0), data.testNamespace, false) + defer cleanupFunc() + podIP := podIPs.IPv4.String() + + t.Log("Create a test Service") + svcClusterIP, err := data.createAgnhostClusterIPService("agnhost-svc", false, ptr.To[corev1.IPFamily](corev1.IPv4Protocol)) + defer data.deleteService(svcClusterIP.Namespace, svcClusterIP.Name) + require.NoError(t, err) + require.NotEqual(t, "", svcClusterIP.Spec.ClusterIP, "ClusterIP should not be empty") + clusterIP := svcClusterIP.Spec.ClusterIP + + t.Log("Create a test BGPPolicy selecting all Nodes as well as advertising ClusterIPs and Pod CIDRs") + bpBuilder := &BGPPolicySpecBuilder{} + bgpPolicy := bpBuilder.SetName(bpName). + SetListenPort(179). + SetLocalASN(localASN). + SetNodeSelector(map[string]string{}). + SetAdvertiseServiceIPs([]crdv1alpha1.ServiceIPType{crdv1alpha1.ServiceIPTypeClusterIP}). + SetAdvertisePodCIDRs(). + SetBGPPeers([]crdv1alpha1.BGPPeer{{Address: externalInfo.externalFRRIPv4, ASN: remoteASN}}). + Get() + bgpPolicy, err = data.crdClient.CrdV1alpha1().BGPPolicies().Create(context.TODO(), bgpPolicy, metav1.CreateOptions{}) + defer data.crdClient.CrdV1alpha1().BGPPolicies().Delete(context.TODO(), bpName, metav1.DeleteOptions{}) + require.NoError(t, err) + + t.Log("Get the routes installed on remote FRR router and verify them") + expectedRoutes := make([]FRRRoute, 0) + for _, node := range clusterInfo.nodes { + expectedRoutes = append(expectedRoutes, FRRRoute{Prefix: node.podV4NetworkCIDR, Nexthops: []string{node.ipv4Addr}}) + } + expectedRoutes = append(expectedRoutes, FRRRoute{Prefix: clusterIP + "/32", Nexthops: getAllNodeIPs()}) + expectedRouteStrings := routesToStrings(expectedRoutes) + assert.EventuallyWithT(t, func(tc *assert.CollectT) { + gotRoutes := dumpFRRRouterBGPRoutes() + gotRouteStrings := routesToStrings(gotRoutes) + for _, expectedRouteString := range expectedRouteStrings { + assert.Contains(tc, gotRouteStrings, expectedRouteString) + } + }, 30*time.Second, time.Second) + + t.Log("Verify the connectivity of the installed routes on remote FRR route") + ipsToConnect := []string{podIP, clusterIP} + for _, ip := range ipsToConnect { + cmd := fmt.Sprintf("/usr/bin/wget -O - http://%s:8080/hostname -T 5", ip) + rc, stdout, _, err := exec.RunDockerExecCommand(externalInfo.externalFRRCID, cmd, "/", nil, "") + require.NoError(t, err) + require.Equal(t, 0, rc) + require.Contains(t, stdout, "agnhost-") + } + + t.Log("Update the BGP configuration on the remote FRR router") + configureFRRRouterBGP(t, remoteASN, updatedLocalASN) + + _, err = data.updateServiceInternalTrafficPolicy("agnhost-svc", true) + require.NoError(t, err) + + t.Log("Update the test BGPPolicy") + updatedBGPPolicy := bgpPolicy.DeepCopy() + updatedBGPPolicy.Spec.LocalASN = updatedLocalASN + updatedBGPPolicy.Spec.Advertisements.Pod = nil + _, err = data.crdClient.CrdV1alpha1().BGPPolicies().Update(context.TODO(), updatedBGPPolicy, metav1.UpdateOptions{}) + require.NoError(t, err) + + t.Log("Get routes installed on remote FRR router and verify them") + expectedRoutes = []FRRRoute{{Prefix: clusterIP + "/32", Nexthops: []string{nodeIPv4(0)}}} + expectedRouteStrings = routesToStrings(expectedRoutes) + assert.EventuallyWithT(t, func(tc *assert.CollectT) { + gotRoutes := dumpFRRRouterBGPRoutes() + gotRouteStrings := routesToStrings(gotRoutes) + for _, expectedRouteString := range expectedRouteStrings { + assert.Contains(tc, gotRouteStrings, expectedRouteString) + } + }, 30*time.Second, time.Second) + + t.Log("verify the connectivity of the installed routes on remote FRR route") + ipsToConnect = []string{clusterIP} + for _, ip := range ipsToConnect { + cmd := fmt.Sprintf("/usr/bin/wget -O - http://%s:8080/hostname -T 5", ip) + rc, stdout, _, err := exec.RunDockerExecCommand(externalInfo.externalFRRCID, cmd, "/", nil, "") + require.NoError(t, err) + require.Equal(t, 0, rc) + require.Contains(t, stdout, "agnhost-") + } +} + +func configureFRRRouterBGP(t *testing.T, localASN, remoteASN int32) { + frrCommands := []string{ + "configure terminal", + fmt.Sprintf("router bgp %d", localASN), + "no bgp ebgp-requires-policy", + "no bgp network import-check", + } + for _, node := range clusterInfo.nodes { + frrCommands = append(frrCommands, fmt.Sprintf("neighbor %s remote-as %d", node.ipv4Addr, remoteASN)) + frrCommands = append(frrCommands, fmt.Sprintf("neighbor %s password %s", node.ipv4Addr, password)) + } + frrCommands = append(frrCommands, + "exit", + "exit", + "write memory") + + rc, stdout, stderr, err := exec.RunDockerExecCommand(externalInfo.externalFRRCID, "/usr/bin/vtysh", "/", nil, strings.Join(frrCommands, "\n")) + t.Log(stdout) + t.Log(stderr) + require.NoError(t, err, fmt.Sprintf("error when running FRR commands '%v'", frrCommands)) + require.Equal(t, 0, rc) +} + +func cleanupFRRRouterBGP(t *testing.T, asn int32) { + frrCommands := []string{ + "configure terminal", + fmt.Sprintf("no router bgp %d", asn), + "exit", + "write memory", + } + + rc, stdout, stderr, err := exec.RunDockerExecCommand(externalInfo.externalFRRCID, "/usr/bin/vtysh", "/", nil, strings.Join(frrCommands, "\n")) + t.Log(stdout) + t.Log(stderr) + require.NoError(t, err, fmt.Sprintf("error when running FRR commands '%v'", frrCommands)) + require.Equal(t, 0, rc) +} + +func dumpFRRRouterBGPRoutes() []FRRRoute { + frrCommands := []string{"show ip route bgp"} + rc, stdout, _, err := exec.RunDockerExecCommand(externalInfo.externalFRRCID, "/usr/bin/vtysh", "/", nil, strings.Join(frrCommands, "\n")) + if err != nil || rc != 0 { + log.Println(fmt.Sprintf("Error when running FRR command '%v': %v", frrCommands, err)) + return nil + } + + routePattern := regexp.MustCompile(`B>\* ([\d\.\/]+) \[.*?\] via ([\d\.]+),`) + nexthopPattern := regexp.MustCompile(`\* +via ([\d\.]+),`) + var routes []FRRRoute + lines := strings.Split(stdout, "\n") + for _, line := range lines { + routeMatches := routePattern.FindStringSubmatch(line) + if routeMatches != nil { + route := FRRRoute{ + Prefix: routeMatches[1], + Nexthops: []string{routeMatches[2]}, + } + routes = append(routes, route) + continue + } + + nexthopMatches := nexthopPattern.FindStringSubmatch(line) + if nexthopMatches != nil && len(routes) > 0 { + last := len(routes) - 1 + routes[last].Nexthops = append(routes[last].Nexthops, nexthopMatches[1]) + } + } + return routes +} + +func (b *BGPPolicySpecBuilder) SetName(name string) *BGPPolicySpecBuilder { + b.Name = name + return b +} + +func (b *BGPPolicySpecBuilder) SetListenPort(port int32) *BGPPolicySpecBuilder { + b.Spec.ListenPort = ptr.To[int32](port) + return b +} + +func (b *BGPPolicySpecBuilder) SetLocalASN(asn int32) *BGPPolicySpecBuilder { + b.Spec.LocalASN = asn + return b +} + +func (b *BGPPolicySpecBuilder) SetNodeSelector(nodeSelector map[string]string) *BGPPolicySpecBuilder { + b.Spec.NodeSelector = metav1.LabelSelector{ + MatchLabels: nodeSelector, + } + return b +} + +func (b *BGPPolicySpecBuilder) SetAdvertiseServiceIPs(serviceIPTypes []crdv1alpha1.ServiceIPType) *BGPPolicySpecBuilder { + b.Spec.Advertisements.Service = &crdv1alpha1.ServiceAdvertisement{IPTypes: serviceIPTypes} + return b +} + +func (b *BGPPolicySpecBuilder) SetAdvertiseEgressIPs() *BGPPolicySpecBuilder { + b.Spec.Advertisements.Egress = &crdv1alpha1.EgressAdvertisement{} + return b +} + +func (b *BGPPolicySpecBuilder) SetAdvertisePodCIDRs() *BGPPolicySpecBuilder { + b.Spec.Advertisements.Pod = &crdv1alpha1.PodAdvertisement{} + return b +} + +func (b *BGPPolicySpecBuilder) SetBGPPeers(peers []crdv1alpha1.BGPPeer) *BGPPolicySpecBuilder { + b.Spec.BGPPeers = peers + return b +} + +func (b *BGPPolicySpecBuilder) Get() *crdv1alpha1.BGPPolicy { + return &crdv1alpha1.BGPPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: b.Name, + }, + Spec: b.Spec, + } +} diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index 55297af4d5d..3930cc8311d 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -68,6 +68,12 @@ func skipIfProviderIs(tb testing.TB, name string, reason string) { } } +func skipIfProviderIsNot(tb testing.TB, name string, reason string) { + if testOptions.providerName != name { + tb.Skipf("Skipping test for the '%s' provider: %s", name, reason) + } +} + func skipIfNotRequired(tb testing.TB, keys ...string) { for _, v := range keys { if strings.Contains(testOptions.skipCases, v) { @@ -685,7 +691,9 @@ func testMain(m *testing.M) int { flag.StringVar(&testOptions.skipCases, "skip-cases", "", "Key words to skip cases") flag.StringVar(&testOptions.linuxVMs, "linuxVMs", "", "hostname of Linux VMs") flag.StringVar(&testOptions.windowsVMs, "windowsVMs", "", "hostname of Windows VMs") - flag.StringVar(&testOptions.externalServerIPs, "external-server-ips", "", "IP addresses of external server, at most one IP per IP family") + flag.StringVar(&testOptions.externalAgnhostIPs, "external-agnhost-ips", "", "IP addresses of external agnhost, at most one IP per IP family") + flag.StringVar(&testOptions.externalFRRIPs, "external-frr-ips", "", "IP addresses of external FRR, at most one IP per IP family") + flag.StringVar(&testOptions.externalFRRCID, "external-frr-cid", "", "Container ID of external FRR") flag.StringVar(&testOptions.vlanSubnets, "vlan-subnets", "", "IP subnets of the VLAN network the Nodes reside in, at most one subnet per IP family") flag.IntVar(&testOptions.vlanID, "vlan-id", 0, "ID of the VLAN network the Nodes reside in") flag.Parse() diff --git a/test/e2e/framework.go b/test/e2e/framework.go index e752434be2c..ad072c67771 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -190,6 +190,10 @@ type ExternalInfo struct { vlanSubnetIPv6 string vlanGatewayIPv6 string vlanID int + + externalFRRIPv4 string + externalFRRIPv6 string + externalFRRCID string } var clusterInfo ClusterInfo @@ -213,9 +217,16 @@ type TestOptions struct { // the home directory of the control-plane Node. Note it doesn't affect the tests that redeploy Antrea themselves. deployAntrea bool - externalServerIPs string - vlanSubnets string - vlanID int + externalAgnhostIPs string + vlanSubnets string + vlanID int + + externalFRRIPs string + // FRR cannot currently be configured remotely over networking. As a result, the e2e tests for BGPPolicy can only + // be run in a Kind cluster, where the FRR container can be configured using Docker exec with the container ID. + // TODO: Introduce a BGP router implementation that can be configured remotely over networking to replace FRR. + // This would allow the e2e tests for BGPPolicy to be run in environments other than just a Kind cluster. + externalFRRCID string } type flowVisibilityTestOptions struct { @@ -498,14 +509,14 @@ func (data *TestData) RunCommandOnNodeExt(nodeName, cmd string, envs map[string] } func (data *TestData) collectExternalInfo() error { - ips := strings.Split(testOptions.externalServerIPs, ",") + ips := strings.Split(testOptions.externalAgnhostIPs, ",") for _, ip := range ips { if ip == "" { continue } parsedIP := net.ParseIP(ip) if parsedIP == nil { - return fmt.Errorf("invalid external server IP %s", ip) + return fmt.Errorf("invalid external agnhost IP %s", ip) } if parsedIP.To4() != nil { externalInfo.externalServerIPv4 = ip @@ -532,6 +543,25 @@ func (data *TestData) collectExternalInfo() error { } } externalInfo.vlanID = testOptions.vlanID + + frrIPs := strings.Split(testOptions.externalFRRIPs, ",") + for _, ip := range frrIPs { + if ip == "" { + continue + } + parsedIP := net.ParseIP(ip) + if parsedIP == nil { + return fmt.Errorf("invalid external FRR IP %s", ip) + } + if parsedIP.To4() != nil { + externalInfo.externalFRRIPv4 = ip + } else { + externalInfo.externalFRRIPv6 = ip + } + } + + externalInfo.externalFRRCID = testOptions.externalFRRCID + return nil } @@ -2038,6 +2068,20 @@ func (data *TestData) updateServiceExternalTrafficPolicy(serviceName string, nod return data.clientset.CoreV1().Services(data.testNamespace).Update(context.TODO(), svc, metav1.UpdateOptions{}) } +func (data *TestData) updateServiceInternalTrafficPolicy(serviceName string, nodeLocalInternal bool) (*corev1.Service, error) { + svc, err := data.clientset.CoreV1().Services(data.testNamespace).Get(context.TODO(), serviceName, metav1.GetOptions{}) + if err != nil { + return svc, err + } + if nodeLocalInternal { + svc.Spec.InternalTrafficPolicy = ptr.To[corev1.ServiceInternalTrafficPolicyType](corev1.ServiceInternalTrafficPolicyLocal) + } else { + svc.Spec.InternalTrafficPolicy = ptr.To[corev1.ServiceInternalTrafficPolicyType](corev1.ServiceInternalTrafficPolicyCluster) + } + + return data.clientset.CoreV1().Services(data.testNamespace).Update(context.TODO(), svc, metav1.UpdateOptions{}) +} + func (data *TestData) updateService(serviceName string, mutateFunc func(service *corev1.Service)) (*corev1.Service, error) { svc, err := data.clientset.CoreV1().Services(data.testNamespace).Get(context.TODO(), serviceName, metav1.GetOptions{}) if err != nil { @@ -2876,6 +2920,23 @@ func (data *TestData) createAgnhostPodOnNode(name string, ns string, nodeName st return NewPodBuilder(name, ns, agnhostImage).OnNode(nodeName).WithHostNetwork(hostNetwork).Create(data) } +func (data *TestData) createAgnhostPodWithHTTPOnNode(name string, ns string, nodeName string, hostNetwork bool) error { + args := []string{"netexec", "--http-port=8080"} + ports := []corev1.ContainerPort{ + { + Name: "http", + ContainerPort: 8080, + Protocol: corev1.ProtocolTCP, + }, + } + return NewPodBuilder(name, ns, agnhostImage). + OnNode(nodeName). + WithArgs(args). + WithPorts(ports). + WithHostNetwork(hostNetwork). + Create(data) +} + // createAgnhostPodWithSAOnNode creates a Pod in the test namespace with a single // agnhost container and a specific ServiceAccount. The Pod will be scheduled on // the specified Node (if nodeName is not empty).