diff --git a/contrib/kind-helm.sh b/contrib/kind-helm.sh index 577afa79092..a868e8f8380 100755 --- a/contrib/kind-helm.sh +++ b/contrib/kind-helm.sh @@ -31,6 +31,10 @@ set_default_params() { # Setup KUBECONFIG patch based on cluster-name export KUBECONFIG=${KUBECONFIG:-${HOME}/${KIND_CLUSTER_NAME}.conf} + # Validated params that work + export MASQUERADE_SUBNET_IPV4=${MASQUERADE_SUBNET_IPV4:-169.254.0.0/16} + export MASQUERADE_SUBNET_IPV6=${MASQUERADE_SUBNET_IPV6:-fd69::/112} + # Input not currently validated. Modify outside script at your own risk. # These are the same values defaulted to in KIND code (kind/default.go). # NOTE: KIND NET_CIDR_IPV6 default use a /64 but OVN have a /64 per host @@ -43,8 +47,6 @@ set_default_params() { export SVC_CIDR_IPV6=${SVC_CIDR_IPV6:-fd00:10:96::/112} export JOIN_SUBNET_IPV4=${JOIN_SUBNET_IPV4:-100.64.0.0/16} export JOIN_SUBNET_IPV6=${JOIN_SUBNET_IPV6:-fd98::/64} - export MASQUERADE_SUBNET_IPV4=${MASQUERADE_SUBNET_IPV4:-169.254.169.0/29} - export MASQUERADE_SUBNET_IPV6=${MASQUERADE_SUBNET_IPV6:-fd69::/125} export TRANSIT_SWITCH_SUBNET_IPV4=${TRANSIT_SWITCH_SUBNET_IPV4:-100.88.0.0/16} export TRANSIT_SWITCH_SUBNET_IPV6=${TRANSIT_SWITCH_SUBNET_IPV6:-fd97::/64} export METALLB_CLIENT_NET_SUBNET_IPV4=${METALLB_CLIENT_NET_SUBNET_IPV4:-172.22.0.0/16} @@ -312,7 +314,9 @@ create_ovn_kubernetes() { --set global.enableHybridOverlay=$(if [ "${OVN_HYBRID_OVERLAY_ENABLE}" == "true" ]; then echo "true"; else echo "false"; fi) \ --set global.emptyLbEvents=$(if [ "${OVN_EMPTY_LB_EVENTS}" == "true" ]; then echo "true"; else echo "false"; fi) \ --set tags.ovnkube-db-raft=$(if [ "${OVN_HA}" == "true" ]; then echo "true"; else echo "false"; fi) \ - --set tags.ovnkube-db=$(if [ "${OVN_HA}" == "false" ]; then echo "true"; else echo "false"; fi) + --set tags.ovnkube-db=$(if [ "${OVN_HA}" == "false" ]; then echo "true"; else echo "false"; fi) \ + --set global.v4MasqueradeSubnet=${MASQUERADE_SUBNET_IPV4} \ + --set global.v6MasqueradeSubnet=${MASQUERADE_SUBNET_IPV6} } delete() { diff --git a/contrib/kind.sh b/contrib/kind.sh index 7bc71797a38..e862e518d9f 100755 --- a/contrib/kind.sh +++ b/contrib/kind.sh @@ -543,8 +543,8 @@ set_default_params() { SVC_CIDR_IPV6=${SVC_CIDR_IPV6:-fd00:10:96::/112} JOIN_SUBNET_IPV4=${JOIN_SUBNET_IPV4:-100.64.0.0/16} JOIN_SUBNET_IPV6=${JOIN_SUBNET_IPV6:-fd98::/64} - MASQUERADE_SUBNET_IPV4=${MASQUERADE_SUBNET_IPV4:-169.254.169.0/29} - MASQUERADE_SUBNET_IPV6=${MASQUERADE_SUBNET_IPV6:-fd69::/125} + MASQUERADE_SUBNET_IPV4=${MASQUERADE_SUBNET_IPV4:-169.254.0.0/16} + MASQUERADE_SUBNET_IPV6=${MASQUERADE_SUBNET_IPV6:-fd69::/112} TRANSIT_SWITCH_SUBNET_IPV4=${TRANSIT_SWITCH_SUBNET_IPV4:-100.88.0.0/16} TRANSIT_SWITCH_SUBNET_IPV6=${TRANSIT_SWITCH_SUBNET_IPV6:-fd97::/64} METALLB_CLIENT_NET_SUBNET_IPV4=${METALLB_CLIENT_NET_SUBNET_IPV4:-172.22.0.0/16} diff --git a/dist/images/daemonset.sh b/dist/images/daemonset.sh index a04794b3d5d..09450383c26 100755 --- a/dist/images/daemonset.sh +++ b/dist/images/daemonset.sh @@ -820,6 +820,8 @@ ovn_image=${ovnkube_image} \ ovn_disable_pkt_mtu_check=${ovn_disable_pkt_mtu_check} \ ovn_v4_join_subnet=${ovn_v4_join_subnet} \ ovn_v6_join_subnet=${ovn_v6_join_subnet} \ + ovn_v4_masquerade_subnet=${ovn_v4_masquerade_subnet} \ + ovn_v6_masquerade_subnet=${ovn_v6_masquerade_subnet} \ ovn_multicast_enable=${ovn_multicast_enable} \ ovn_admin_network_policy_enable=${ovn_admin_network_policy_enable} \ ovn_egress_ip_enable=${ovn_egress_ip_enable} \ @@ -880,6 +882,8 @@ ovn_image=${ovnkube_image} \ ovn_disable_pkt_mtu_check=${ovn_disable_pkt_mtu_check} \ ovn_v4_join_subnet=${ovn_v4_join_subnet} \ ovn_v6_join_subnet=${ovn_v6_join_subnet} \ + ovn_v4_masquerade_subnet=${ovn_v4_masquerade_subnet} \ + ovn_v6_masquerade_subnet=${ovn_v6_masquerade_subnet} \ ovn_multicast_enable=${ovn_multicast_enable} \ ovn_admin_network_policy_enable=${ovn_admin_network_policy_enable} \ ovn_egress_ip_enable=${ovn_egress_ip_enable} \ diff --git a/dist/images/ovnkube.sh b/dist/images/ovnkube.sh index 22b5fb47765..15043aa8bdb 100755 --- a/dist/images/ovnkube.sh +++ b/dist/images/ovnkube.sh @@ -2497,6 +2497,16 @@ ovn-node() { fi echo "ovn_conntrack_zone_flag=${ovn_conntrack_zone_flag}" + ovn_v4_masquerade_subnet_opt= + if [[ -n ${ovn_v4_masquerade_subnet} ]]; then + ovn_v4_masquerade_subnet_opt="--gateway-v4-masquerade-subnet=${ovn_v4_masquerade_subnet}" + fi + + ovn_v6_masquerade_subnet_opt= + if [[ -n ${ovn_v6_masquerade_subnet} ]]; then + ovn_v6_masquerade_subnet_opt="--gateway-v6-masquerade-subnet=${ovn_v6_masquerade_subnet}" + fi + echo "=============== ovn-node --init-node" /usr/bin/ovnkube --init-node ${K8S_NODE} \ ${anp_enabled_flag} \ @@ -2526,6 +2536,8 @@ ovn-node() { ${ovn_conntrack_zone_flag} \ ${ovnkube_enable_interconnect_flag} \ ${ovnkube_enable_multi_external_gateway_flag} \ + ${ovn_v4_masquerade_subnet_opt} \ + ${ovn_v6_masquerade_subnet_opt} \ ${ovnkube_metrics_tls_opts} \ ${ovnkube_node_certs_flags} \ ${ovnkube_node_mgmt_port_netdev_flag} \ diff --git a/dist/templates/ovnkube-single-node-zone.yaml.j2 b/dist/templates/ovnkube-single-node-zone.yaml.j2 index 3505cfce29d..0488a4d7bb2 100644 --- a/dist/templates/ovnkube-single-node-zone.yaml.j2 +++ b/dist/templates/ovnkube-single-node-zone.yaml.j2 @@ -399,6 +399,10 @@ spec: value: "{{ ovn_v4_join_subnet }}" - name: OVN_V6_JOIN_SUBNET value: "{{ ovn_v6_join_subnet }}" + - name: OVN_V4_MASQUERADE_SUBNET + value: "{{ ovn_v4_masquerade_subnet }}" + - name: OVN_V6_MASQUERADE_SUBNET + value: "{{ ovn_v6_masquerade_subnet }}" - name: OVN_MULTICAST_ENABLE value: "{{ ovn_multicast_enable }}" - name: OVN_UNPRIVILEGED_MODE diff --git a/dist/templates/ovnkube-zone-controller.yaml.j2 b/dist/templates/ovnkube-zone-controller.yaml.j2 index d0042a09c6c..24f21f677f2 100644 --- a/dist/templates/ovnkube-zone-controller.yaml.j2 +++ b/dist/templates/ovnkube-zone-controller.yaml.j2 @@ -357,6 +357,10 @@ spec: value: "{{ ovn_v4_join_subnet }}" - name: OVN_V6_JOIN_SUBNET value: "{{ ovn_v6_join_subnet }}" + - name: OVN_V4_MASQUERADE_SUBNET + value: "{{ ovn_v4_masquerade_subnet }}" + - name: OVN_V6_MASQUERADE_SUBNET + value: "{{ ovn_v6_masquerade_subnet }}" - name: OVN_SSL_ENABLE value: "{{ ovn_ssl_en }}" - name: OVN_GATEWAY_MODE diff --git a/go-controller/pkg/config/config.go b/go-controller/pkg/config/config.go index 872d8151a6b..f174f1c00e2 100644 --- a/go-controller/pkg/config/config.go +++ b/go-controller/pkg/config/config.go @@ -1877,7 +1877,7 @@ func completeGatewayConfig(allSubnets *configSubnets, masqueradeIPs *MasqueradeI if err != nil || utilnet.IsIPv6(v4MasqueradeCIDR.IP) { return fmt.Errorf("invalid gateway v4 masquerade subnet specified, subnet: %s: error: %v", Gateway.V4MasqueradeSubnet, err) } - if err = allocateV4MasqueradeIPs(v4MasqueradeIP, masqueradeIPs); err != nil { + if err = AllocateV4MasqueradeIPs(v4MasqueradeIP, masqueradeIPs); err != nil { return fmt.Errorf("unable to allocate V4MasqueradeIPs: %s", err) } @@ -1885,7 +1885,7 @@ func completeGatewayConfig(allSubnets *configSubnets, masqueradeIPs *MasqueradeI if err != nil || !utilnet.IsIPv6(v6MasqueradeCIDR.IP) { return fmt.Errorf("invalid gateway v6 masquerade subnet specified, subnet: %s: error: %v", Gateway.V6MasqueradeSubnet, err) } - if err = allocateV6MasqueradeIPs(v6MasqueradeIP, masqueradeIPs); err != nil { + if err = AllocateV6MasqueradeIPs(v6MasqueradeIP, masqueradeIPs); err != nil { return fmt.Errorf("unable to allocate V6MasqueradeIPs: %s", err) } diff --git a/go-controller/pkg/config/utils.go b/go-controller/pkg/config/utils.go index 3fd1ae6e7cb..eeb39ae9b3e 100644 --- a/go-controller/pkg/config/utils.go +++ b/go-controller/pkg/config/utils.go @@ -292,7 +292,7 @@ type MasqueradeIPsConfig struct { // allocateV4/6MasqueradeIPs allocates the masqueradeIPs based off of the passed in masqueradeSubnet (.0) // it does this by cascading down from the initial ip down to the .5 currently (more masqueradeIps may be added in the future) -func allocateV4MasqueradeIPs(masqueradeSubnetNetworkAddress net.IP, masqueradeIPs *MasqueradeIPsConfig) error { +func AllocateV4MasqueradeIPs(masqueradeSubnetNetworkAddress net.IP, masqueradeIPs *MasqueradeIPsConfig) error { masqueradeIPs.V4OVNMasqueradeIP = iputils.NextIP(masqueradeSubnetNetworkAddress) if masqueradeIPs.V4OVNMasqueradeIP == nil { return fmt.Errorf("error setting V4OVNMasqueradeIP: %s", masqueradeSubnetNetworkAddress) @@ -316,7 +316,7 @@ func allocateV4MasqueradeIPs(masqueradeSubnetNetworkAddress net.IP, masqueradeIP return nil } -func allocateV6MasqueradeIPs(masqueradeSubnetNetworkAddress net.IP, masqueradeIPs *MasqueradeIPsConfig) error { +func AllocateV6MasqueradeIPs(masqueradeSubnetNetworkAddress net.IP, masqueradeIPs *MasqueradeIPsConfig) error { masqueradeIPs.V6OVNMasqueradeIP = iputils.NextIP(masqueradeSubnetNetworkAddress) if masqueradeIPs.V6OVNMasqueradeIP == nil { return fmt.Errorf("error setting V6OVNMasqueradeIP: %s", masqueradeSubnetNetworkAddress) diff --git a/go-controller/pkg/libovsdb/ops/mac_binding.go b/go-controller/pkg/libovsdb/ops/mac_binding.go index ef27a5aa19a..1f7a76ba8b1 100644 --- a/go-controller/pkg/libovsdb/ops/mac_binding.go +++ b/go-controller/pkg/libovsdb/ops/mac_binding.go @@ -39,3 +39,23 @@ func DeleteStaticMacBindings(nbClient libovsdbclient.Client, smbs ...*nbdb.Stati m := newModelClient(nbClient) return m.Delete(opModels...) } + +type staticMACBindingPredicate func(*nbdb.StaticMACBinding) bool + +// DeleteStaticMACBindingWithPredicate deletes a Static MAC entry for a logical port from the cache +func DeleteStaticMACBindingWithPredicate(nbClient libovsdbclient.Client, p staticMACBindingPredicate) error { + found := []*nbdb.StaticMACBinding{} + opModel := operationModel{ + ModelPredicate: p, + ExistingResult: &found, + ErrNotFound: false, + BulkOp: false, + } + + m := newModelClient(nbClient) + err := m.Delete(opModel) + if err != nil { + return err + } + return nil +} diff --git a/go-controller/pkg/node/gateway_init.go b/go-controller/pkg/node/gateway_init.go index 3b7fee43cb4..ac32b5346c6 100644 --- a/go-controller/pkg/node/gateway_init.go +++ b/go-controller/pkg/node/gateway_init.go @@ -457,6 +457,13 @@ func (nc *DefaultNodeNetworkController) initGatewayDPUHost(kubeNodeIP net.IP) er return err } + // Delete stale masquerade resources if there are any. This is to make sure that there + // are no Linux resources with IP from old masquerade subnet when masquerade subnet + // gets changed as part of day2 operation. + if err := deleteStaleMasqueradeResources(gwIntf, nc.name, nc.watchFactory); err != nil { + return fmt.Errorf("failed to remove stale masquerade resources: %w", err) + } + if err := setNodeMasqueradeIPOnExtBridge(gwIntf); err != nil { return fmt.Errorf("failed to set the node masquerade IP on the ext bridge %s: %v", gwIntf, err) } @@ -465,6 +472,11 @@ func (nc *DefaultNodeNetworkController) initGatewayDPUHost(kubeNodeIP net.IP) er return fmt.Errorf("failed to set the node masquerade route to OVN: %v", err) } + // Masquerade config mostly done on node, update annotation + if err := updateMasqueradeAnnotation(nc.name, nc.Kube); err != nil { + return fmt.Errorf("failed to update masquerade subnet annotation on node: %s, error: %v", nc.name, err) + } + err = configureSvcRouteViaInterface(nc.routeManager, gatewayIntf, gatewayNextHops) if err != nil { return err diff --git a/go-controller/pkg/node/gateway_init_linux_test.go b/go-controller/pkg/node/gateway_init_linux_test.go index 0f7fbe1345c..a6406e3fbe8 100644 --- a/go-controller/pkg/node/gateway_init_linux_test.go +++ b/go-controller/pkg/node/gateway_init_linux_test.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + utilnet "k8s.io/utils/net" "net" "runtime" "strings" @@ -88,15 +89,6 @@ func shareGatewayInterfaceTest(app *cli.App, testNS ns.NetNS, Action: func() error { return testNS.Do(func(ns.NetNS) error { defer GinkgoRecover() - - // Create breth0 as a dummy link - err := netlink.LinkAdd(&netlink.Dummy{ - LinkAttrs: netlink.LinkAttrs{ - Name: "br" + eth0Name, - HardwareAddr: ovntest.MustParseMAC(eth0MAC), - }, - }) - Expect(err).NotTo(HaveOccurred()) _, err = netlink.LinkByName("br" + eth0Name) Expect(err).NotTo(HaveOccurred()) return nil @@ -192,6 +184,10 @@ func shareGatewayInterfaceTest(app *cli.App, testNS ns.NetNS, existingNode := v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: nodeName, + Annotations: map[string]string{ + // add some fake previous subnets to force OVNK to try to clean it + util.OvnNodeMasqCIDR: "{\"ipv4\":\"170.254.0.0/16\",\"ipv6\":\"fa69::/112\"}", + }, }, } if setNodeIP { @@ -260,6 +256,39 @@ func shareGatewayInterfaceTest(app *cli.App, testNS ns.NetNS, err = testNS.Do(func(ns.NetNS) error { defer GinkgoRecover() + // setup stale masquerade + // Create breth0 as a dummy link + err := netlink.LinkAdd(&netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: "br" + eth0Name, + HardwareAddr: ovntest.MustParseMAC(eth0MAC), + }, + }) + Expect(err).NotTo(HaveOccurred()) + link, err := netlink.LinkByName("br" + eth0Name) + Expect(err).NotTo(HaveOccurred()) + err = netlink.LinkSetUp(link) + Expect(err).NotTo(HaveOccurred()) + staleAddr, err := netlink.ParseAddr("170.254.0.2/32") + Expect(err).NotTo(HaveOccurred()) + err = netlink.AddrAdd(link, staleAddr) + Expect(err).NotTo(HaveOccurred()) + _, gw, err := net.ParseCIDR("170.254.0.1/32") + Expect(err).NotTo(HaveOccurred()) + staleRoute := &netlink.Route{ + LinkIndex: link.Attrs().Index, + Dst: gw, + } + err = netlink.RouteAdd(staleRoute) + Expect(err).NotTo(HaveOccurred()) + // ensure stale route is present + r, err := util.LinkRouteGetFilteredRoute( + staleRoute, + netlink.RT_FILTER_DST|netlink.RT_FILTER_OIF|netlink.RT_FILTER_SRC, + ) + Expect(err).NotTo(HaveOccurred()) + Expect(r).NotTo(BeNil()) + gatewayNextHops, gatewayIntf, err := getGatewayNextHops() Expect(err).NotTo(HaveOccurred()) ifAddrs := ovntest.MustParseIPNets(eth0CIDR) @@ -284,16 +313,21 @@ func shareGatewayInterfaceTest(app *cli.App, testNS ns.NetNS, Expect(err).NotTo(HaveOccurred()) addrs, err := netlink.AddrList(l, syscall.AF_INET) Expect(err).NotTo(HaveOccurred()) - var found bool + var found, staleFound bool expectedAddr, err := netlink.ParseAddr(eth0CIDR) Expect(err).NotTo(HaveOccurred()) for _, a := range addrs { + // ensure stale masquerade IP was removed from the bridge + if a.IP.Equal(staleAddr.IP) && bytes.Equal(a.Mask, staleAddr.Mask) { + staleFound = true + } + // ensure code moved correct IP to bridge if a.IP.Equal(expectedAddr.IP) && bytes.Equal(a.Mask, expectedAddr.Mask) { found = true - break } } Expect(found).To(BeTrue()) + Expect(staleFound).To(BeFalse()) Expect(l.Attrs().HardwareAddr.String()).To(Equal(eth0MAC)) @@ -316,6 +350,13 @@ func shareGatewayInterfaceTest(app *cli.App, testNS ns.NetNS, } return nil }, 1*time.Second).ShouldNot(HaveOccurred()) + // ensure stale masquerade route is no longer present + r, err = util.LinkRouteGetFilteredRoute( + staleRoute, + netlink.RT_FILTER_DST|netlink.RT_FILTER_OIF|netlink.RT_FILTER_SRC, + ) + Expect(err).NotTo(HaveOccurred()) + Expect(r).To(BeNil()) return nil }) Expect(err).NotTo(HaveOccurred()) @@ -384,6 +425,20 @@ func shareGatewayInterfaceTest(app *cli.App, testNS ns.NetNS, f6 := iptV6.(*util.FakeIPTables) err = f6.MatchState(expectedTables, nil) Expect(err).NotTo(HaveOccurred()) + + // check that masquerade subnet annotation got updated + node, err := wf.GetNode(nodeName) + Expect(err).NotTo(HaveOccurred()) + subnets, err := util.ParseNodeMasqueradeSubnet(node) + Expect(err).NotTo(HaveOccurred()) + for _, subnet := range subnets { + if utilnet.IsIPv4CIDR(subnet) { + Expect(subnet.String()).To(Equal(config.Gateway.V4MasqueradeSubnet)) + } else if utilnet.IsIPv6CIDR(subnet) { + Expect(subnet.String()).To(Equal(config.Gateway.V6MasqueradeSubnet)) + } + } + return nil } @@ -740,7 +795,7 @@ func shareGatewayInterfaceDPUHostTest(app *cli.App, testNS ns.NetNS, uplinkName, ip, ipnet, err := net.ParseCIDR(hostIP + "/24") Expect(err).NotTo(HaveOccurred()) ipnet.IP = ip - cnnci := NewCommonNodeNetworkControllerInfo(nil, fakeClient.AdminPolicyRouteClient, wf, nil, nodeName) + cnnci := NewCommonNodeNetworkControllerInfo(kubeFakeClient, fakeClient.AdminPolicyRouteClient, wf, nil, nodeName) nc := newDefaultNodeNetworkController(cnnci, stop, wg) // must run route manager manually which is usually started with nc.Start() wg.Add(1) diff --git a/go-controller/pkg/node/gateway_iptables.go b/go-controller/pkg/node/gateway_iptables.go index 1e41a28d66a..c172b770a33 100644 --- a/go-controller/pkg/node/gateway_iptables.go +++ b/go-controller/pkg/node/gateway_iptables.go @@ -341,29 +341,53 @@ func getGatewayForwardRules(cidrs []*net.IPNet) []nodeipt.Rule { if protocol == iptables.ProtocolIPv6 { masqueradeIP = config.Gateway.MasqueradeIPs.V6OVNMasqueradeIP } - returnRules = append(returnRules, []nodeipt.Rule{ - { - Table: "filter", - Chain: "FORWARD", - Args: []string{ - "-s", masqueradeIP.String(), - "-j", "ACCEPT", - }, - Protocol: protocol, + returnRules = append(returnRules, getMasqueradeIpTablesForwardRules(masqueradeIP, protocol)...) + } + + return returnRules +} + +// getStaleMasqueradeIptablesRules returns all iptables rules may get added for a given masquerade IP. +func getStaleMasqueradeIptablesRules(masqueradeIP net.IP) []nodeipt.Rule { + return append(getMasqueradeIpTablesForwardRules(masqueradeIP, getIPTablesProtocol(masqueradeIP.String())), + getMasqueradeIpTablesNATRules(masqueradeIP, getIPTablesProtocol(masqueradeIP.String()))...) +} + +func getMasqueradeIpTablesForwardRules(masqueradeIP net.IP, protocol iptables.Protocol) []nodeipt.Rule { + return []nodeipt.Rule{ + { + Table: "filter", + Chain: "FORWARD", + Args: []string{ + "-s", masqueradeIP.String(), + "-j", "ACCEPT", }, - { - Table: "filter", - Chain: "FORWARD", - Args: []string{ - "-d", masqueradeIP.String(), - "-j", "ACCEPT", - }, - Protocol: protocol, + Protocol: protocol, + }, + { + Table: "filter", + Chain: "FORWARD", + Args: []string{ + "-d", masqueradeIP.String(), + "-j", "ACCEPT", }, - }...) + Protocol: protocol, + }, } +} - return returnRules +func getMasqueradeIpTablesNATRules(masqueradeIP net.IP, protocol iptables.Protocol) []nodeipt.Rule { + return []nodeipt.Rule{ + { + Table: "nat", + Chain: "POSTROUTING", + Args: []string{ + "-s", masqueradeIP.String(), + "-j", "MASQUERADE", + }, + Protocol: protocol, + }, + } } // initExternalBridgeForwardingRules sets up iptables rules for br-* interface svc traffic forwarding diff --git a/go-controller/pkg/node/gateway_localnet.go b/go-controller/pkg/node/gateway_localnet.go index ef7c1a69bd6..158056da10a 100644 --- a/go-controller/pkg/node/gateway_localnet.go +++ b/go-controller/pkg/node/gateway_localnet.go @@ -84,6 +84,13 @@ func newLocalGateway(nodeName string, hostSubnets []*net.IPNet, gwNextHops []net gw.nodeIPManager = newAddressManager(nodeName, kube, cfg, watchFactory, gwBridge) + // Delete stale masquerade resources if there are any. This is to make sure that there + // are no Linux resouces with IP from old masquerade subnet when masquerade subnet + // gets changed as part of day2 operation. + if err := deleteStaleMasqueradeResources(gwBridge.bridgeName, nodeName, watchFactory); err != nil { + return fmt.Errorf("failed to remove stale masquerade resources: %w", err) + } + if err := setNodeMasqueradeIPOnExtBridge(gwBridge.bridgeName); err != nil { return fmt.Errorf("failed to set the node masquerade IP on the ext bridge %s: %v", gwBridge.bridgeName, err) } @@ -92,6 +99,11 @@ func newLocalGateway(nodeName string, hostSubnets []*net.IPNet, gwNextHops []net return fmt.Errorf("failed to set the node masquerade route to OVN: %v", err) } + // Masquerade config mostly done on node, update annotation + if err := updateMasqueradeAnnotation(nodeName, kube); err != nil { + return fmt.Errorf("failed to update masquerade subnet annotation on node: %s, error: %v", nodeName, err) + } + gw.openflowManager, err = newGatewayOpenFlowManager(gwBridge, exGwBridge, hostSubnets, gw.nodeIPManager.ListAddresses()) if err != nil { return err diff --git a/go-controller/pkg/node/gateway_shared_intf.go b/go-controller/pkg/node/gateway_shared_intf.go index 9b8ad37ad07..dfa36c2498b 100644 --- a/go-controller/pkg/node/gateway_shared_intf.go +++ b/go-controller/pkg/node/gateway_shared_intf.go @@ -1761,6 +1761,13 @@ func newSharedGateway(nodeName string, subnets []*net.IPNet, gwNextHops []net.IP nodeIPs := gw.nodeIPManager.ListAddresses() if config.OvnKubeNode.Mode == types.NodeModeFull { + // Delete stale masquerade resources if there are any. This is to make sure that there + // are no Linux resources with IP from old masquerade subnet when masquerade subnet + // gets changed as part of day2 operation. + if err := deleteStaleMasqueradeResources(gwBridge.bridgeName, nodeName, watchFactory); err != nil { + return fmt.Errorf("failed to remove stale masquerade resources: %w", err) + } + if err := setNodeMasqueradeIPOnExtBridge(gwBridge.bridgeName); err != nil { return fmt.Errorf("failed to set the node masquerade IP on the ext bridge %s: %v", gwBridge.bridgeName, err) } @@ -1768,6 +1775,11 @@ func newSharedGateway(nodeName string, subnets []*net.IPNet, gwNextHops []net.IP if err := addMasqueradeRoute(routeManager, gwBridge.bridgeName, nodeName, gwIPs, watchFactory); err != nil { return fmt.Errorf("failed to set the node masquerade route to OVN: %v", err) } + + // Masquerade config mostly done on node, update annotation + if err := updateMasqueradeAnnotation(nodeName, kube); err != nil { + return fmt.Errorf("failed to update masquerade subnet annotation on node: %s, error: %v", nodeName, err) + } } gw.openflowManager, err = newGatewayOpenFlowManager(gwBridge, exGwBridge, subnets, nodeIPs) @@ -1812,7 +1824,7 @@ func newSharedGateway(nodeName string, subnets []*net.IPNet, gwNextHops []net.IP } if err := addHostMACBindings(gwBridge.bridgeName); err != nil { - return fmt.Errorf("failed to add MAC bindings for service routing") + return fmt.Errorf("failed to add MAC bindings for service routing: %w", err) } return nil @@ -2129,6 +2141,19 @@ func addHostMACBindings(bridgeName string) error { return nil } +func updateMasqueradeAnnotation(nodeName string, kube kube.Interface) error { + _, v4MasqueradeCIDR, _ := net.ParseCIDR(config.Gateway.V4MasqueradeSubnet) + _, v6MasqueradeCIDR, _ := net.ParseCIDR(config.Gateway.V6MasqueradeSubnet) + nodeAnnotation, err := util.CreateNodeMasqueradeSubnetAnnotation(nil, v4MasqueradeCIDR, v6MasqueradeCIDR) + if err != nil { + return fmt.Errorf("unable to generate masquerade subnet annotation update: %w", err) + } + if err := kube.SetAnnotationsOnNode(nodeName, nodeAnnotation); err != nil { + return fmt.Errorf("unable to set node masquerade subnet annotation update: %w", err) + } + return nil +} + // generateIPFragmentReassemblyFlow adds flows in table 0 that send packets to a // specific conntrack zone for reassembly with the same priority as node port // flows that match on L4 fields. After reassembly packets are reinjected to @@ -2158,3 +2183,136 @@ func generateIPFragmentReassemblyFlow(ofPortPhys string) []string { return flows } + +// deleteStaleMasqueradeResources removes stale Linux resources when config.Gateway.V4MasqueradeSubnet +// or config.Gateway.V6MasqueradeSubnet gets changed at day 2. +func deleteStaleMasqueradeResources(bridgeName, nodeName string, wf factory.NodeWatchFactory) error { + var staleMasqueradeIPs config.MasqueradeIPsConfig + node, err := wf.GetNode(nodeName) + if err != nil { + return err + } + subnets, err := util.ParseNodeMasqueradeSubnet(node) + if err != nil { + if util.IsAnnotationNotSetError(err) { + // no annotation set, must be initial bring up, nothing to clean + return nil + } + return err + } + + var v4ConfiguredMasqueradeNet, v6ConfiguredMasqueradeNet *net.IPNet + + for _, subnet := range subnets { + if utilnet.IsIPv6CIDR(subnet) { + v6ConfiguredMasqueradeNet = subnet + } else if utilnet.IsIPv4CIDR(subnet) { + v4ConfiguredMasqueradeNet = subnet + } else { + return fmt.Errorf("invalid subnet for masquerade annotation: %s", subnet) + } + } + + if v4ConfiguredMasqueradeNet != nil && config.Gateway.V4MasqueradeSubnet != v4ConfiguredMasqueradeNet.String() { + if err := config.AllocateV4MasqueradeIPs(v4ConfiguredMasqueradeNet.IP, &staleMasqueradeIPs); err != nil { + return fmt.Errorf("unable to determine stale V4MasqueradeIPs: %s", err) + } + } + if v6ConfiguredMasqueradeNet != nil && config.Gateway.V6MasqueradeSubnet != v6ConfiguredMasqueradeNet.String() { + if err := config.AllocateV6MasqueradeIPs(v6ConfiguredMasqueradeNet.IP, &staleMasqueradeIPs); err != nil { + return fmt.Errorf("unable to determine stale V6MasqueradeIPs: %s", err) + } + } + link, err := util.LinkByName(bridgeName) + if err != nil { + return fmt.Errorf("unable to get link for %s, error: %v", bridgeName, err) + } + + if staleMasqueradeIPs.V4HostMasqueradeIP != nil || staleMasqueradeIPs.V6HostMasqueradeIP != nil { + if err = deleteMasqueradeResources(link, &staleMasqueradeIPs); err != nil { + klog.Errorf("Unable to delete masquerade resources! Some configuration for the masquerade subnet "+ + "may be left on the node and may cause issues! Errors: %v", err) + } + } + + return nil +} + +// deleteMasqueradeResources removes following Linux resources given a config.MasqueradeIPsConfig +// struct and netlink.Link: +// - neighbour object for IPv4 and IPv6 OVNMasqueradeIP and DummyNextHopMasqueradeIP. +// - masquerade route added by addMasqueradeRoute function while starting up the gateway. +// - iptables rules created for masquerade subnet based on ipForwarding and Gateway mode. +// - stale HostMasqueradeIP address from gateway bridge +func deleteMasqueradeResources(link netlink.Link, staleMasqueradeIPs *config.MasqueradeIPsConfig) error { + var subnets []*net.IPNet + var neighborIPs []net.IP + var aggregatedErrors []error + klog.Infof("Stale masquerade resources detected, cleaning IPs: %s, %s, %s, %s", + staleMasqueradeIPs.V4HostMasqueradeIP, + staleMasqueradeIPs.V6HostMasqueradeIP, + staleMasqueradeIPs.V4OVNMasqueradeIP, + staleMasqueradeIPs.V6OVNMasqueradeIP) + if config.IPv4Mode && staleMasqueradeIPs.V4HostMasqueradeIP != nil { + // Delete any stale masquerade IP from external bridge. + hostMasqIPNet, err := util.LinkAddrGetIPNet(link, staleMasqueradeIPs.V4HostMasqueradeIP) + if err != nil { + aggregatedErrors = append(aggregatedErrors, fmt.Errorf("unable to get IPNet from link %s: %w", link, err)) + } + if hostMasqIPNet != nil { + if err := util.LinkAddrDel(link, hostMasqIPNet); err != nil { + aggregatedErrors = append(aggregatedErrors, fmt.Errorf("failed to remove masquerade IP from bridge %s: %w", link, err)) + } + } + + _, masqIPNet, err := net.ParseCIDR(fmt.Sprintf("%s/32", staleMasqueradeIPs.V4OVNMasqueradeIP.String())) + if err != nil { + aggregatedErrors = append(aggregatedErrors, + fmt.Errorf("failed to parse V4OVNMasqueradeIP %s: %v", staleMasqueradeIPs.V4OVNMasqueradeIP.String(), err)) + } + subnets = append(subnets, masqIPNet) + neighborIPs = append(neighborIPs, staleMasqueradeIPs.V4OVNMasqueradeIP, staleMasqueradeIPs.V4DummyNextHopMasqueradeIP) + if err := nodeipt.DelRules(getStaleMasqueradeIptablesRules(staleMasqueradeIPs.V4OVNMasqueradeIP)); err != nil { + aggregatedErrors = append(aggregatedErrors, + fmt.Errorf("failed to delete forwarding iptables rules for stale masquerade subnet %s: ", err)) + } + } + + if config.IPv6Mode && staleMasqueradeIPs.V6HostMasqueradeIP != nil { + // Delete any stale masquerade IP from external bridge. + hostMasqIPNet, err := util.LinkAddrGetIPNet(link, staleMasqueradeIPs.V6HostMasqueradeIP) + if err != nil { + aggregatedErrors = append(aggregatedErrors, fmt.Errorf("unable to get IPNet from link %s: %w", link, err)) + } + if hostMasqIPNet != nil { + if err := util.LinkAddrDel(link, hostMasqIPNet); err != nil { + aggregatedErrors = append(aggregatedErrors, fmt.Errorf("failed to remove masquerade IP from bridge %s: %w", link, err)) + } + } + + _, masqIPNet, err := net.ParseCIDR(fmt.Sprintf("%s/128", staleMasqueradeIPs.V6OVNMasqueradeIP.String())) + if err != nil { + return fmt.Errorf("failed to parse V6OVNMasqueradeIP %s: %v", staleMasqueradeIPs.V6OVNMasqueradeIP.String(), err) + } + subnets = append(subnets, masqIPNet) + neighborIPs = append(neighborIPs, staleMasqueradeIPs.V6OVNMasqueradeIP, staleMasqueradeIPs.V6DummyNextHopMasqueradeIP) + if err := nodeipt.DelRules(getStaleMasqueradeIptablesRules(staleMasqueradeIPs.V6OVNMasqueradeIP)); err != nil { + return fmt.Errorf("failed to delete forwarding iptables rules for stale masquerade subnet %s: ", err) + } + } + + for _, ip := range neighborIPs { + if err := util.LinkNeighDel(link, ip); err != nil { + aggregatedErrors = append(aggregatedErrors, fmt.Errorf("failed to remove IP neighbour entry for ip %s, "+ + "on iface %s: %v", ip, link.Attrs().Name, err)) + } + } + + if len(subnets) != 0 { + if err := util.LinkRoutesDel(link, subnets); err != nil { + aggregatedErrors = append(aggregatedErrors, fmt.Errorf("failed to list addresses for the link %s: %v", link.Attrs().Name, err)) + } + } + + return utilerrors.Join(aggregatedErrors...) +} diff --git a/go-controller/pkg/ovn/gateway_init.go b/go-controller/pkg/ovn/gateway_init.go index e07b4e2e4e8..edaa985a438 100644 --- a/go-controller/pkg/ovn/gateway_init.go +++ b/go-controller/pkg/ovn/gateway_init.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" + kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/sets" @@ -16,6 +17,7 @@ import ( libovsdbclient "github.com/ovn-org/libovsdb/client" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" libovsdbops "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdb/ops" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" @@ -290,6 +292,11 @@ func (oc *DefaultNetworkController) gatewayInit(nodeName string, clusterIPSubnet nextHops := l3GatewayConfig.NextHops + // Remove stale OVN resources with any old masquerade IP + if err := deleteStaleMasqueradeResources(oc.nbClient, gatewayRouter, nodeName, oc.watchFactory); err != nil { + return fmt.Errorf("failed to remove stale masquerade resources from northbound database: %w", err) + } + if err := gateway.CreateDummyGWMacBindings(oc.nbClient, gatewayRouter); err != nil { return err } @@ -301,9 +308,10 @@ func (oc *DefaultNetworkController) gatewayInit(nodeName string, clusterIPSubnet prefix = config.Gateway.V6MasqueradeSubnet } lrsr := nbdb.LogicalRouterStaticRoute{ - IPPrefix: prefix, - Nexthop: nextHop.String(), - OutputPort: &externalRouterPort, + IPPrefix: prefix, + Nexthop: nextHop.String(), + OutputPort: &externalRouterPort, + ExternalIDs: map[string]string{util.OvnNodeMasqCIDR: ""}, } p := func(item *nbdb.LogicalRouterStaticRoute) bool { return item.OutputPort != nil && *item.OutputPort == *lrsr.OutputPort && item.IPPrefix == lrsr.IPPrefix && @@ -781,3 +789,142 @@ func (oc *DefaultNetworkController) deletePolicyBasedRoutes(policyID, priority s return nil } + +// cleanupStaleMasqueradeData removes following from northbound database +// - LogicalRouterStaticRoute for rtoe- OutputPort anf IPPrefix is same as v4 or v6 +// StaleMasqueradeSubnet +// - StaticMACBinding for rtoe- LogicalPort and referencing old DummyNextHopMasqueradeIP +func deleteStaleMasqueradeResources(nbClient libovsdbclient.Client, routerName, nodeName string, wf *factory.WatchFactory) error { + var staleMasqueradeIPs config.MasqueradeIPsConfig + var nextHops []net.IP + logicalport := types.GWRouterToExtSwitchPrefix + routerName + + // we first examine the kapi node to see if we can determine if there is a stale masquerade subnet + // if the masquerade subnet on the kapi node matches whats configured for this controller, it + // doesn't necessarily mean there is not still a stale masquerade config in nbdb because + // node could have already updated before this process runs. + // As a backup, if we don't find the positive match in kapi, we execute a negative lookup in NBDB + + node, err := wf.GetNode(nodeName) + if err != nil { + if kerrors.IsNotFound(err) { + // node doesn't exist for some reason, assume we should still try to clean up with auto-detection + if err := deleteStaleMasqueradeRouteAndMACBinding(nbClient, routerName, nextHops); err != nil { + return fmt.Errorf("failed to remove stale MAC binding and static route for logical port %s: %w", logicalport, err) + } + return nil + } + return err + } + + subnets, err := util.ParseNodeMasqueradeSubnet(node) + if err != nil { + if util.IsAnnotationNotSetError(err) { + // no annotation set, must be initial bring up, nothing to clean + return nil + } + return err + } + + var v4ConfiguredMasqueradeNet, v6ConfiguredMasqueradeNet *net.IPNet + + for _, subnet := range subnets { + if utilnet.IsIPv6CIDR(subnet) { + v6ConfiguredMasqueradeNet = subnet + } else if utilnet.IsIPv4CIDR(subnet) { + v4ConfiguredMasqueradeNet = subnet + } else { + return fmt.Errorf("invalid subnet for masquerade annotation: %s", subnet) + } + } + + // Check for KAPI telling us there is a stale masquerade + if v4ConfiguredMasqueradeNet != nil && config.Gateway.V4MasqueradeSubnet != v4ConfiguredMasqueradeNet.String() { + if err := config.AllocateV4MasqueradeIPs(v4ConfiguredMasqueradeNet.IP, &staleMasqueradeIPs); err != nil { + return fmt.Errorf("unable to determine stale V4MasqueradeIPs: %s", err) + } + nextHops = append(nextHops, staleMasqueradeIPs.V4DummyNextHopMasqueradeIP) + } + + if v6ConfiguredMasqueradeNet != nil && config.Gateway.V6MasqueradeSubnet != v6ConfiguredMasqueradeNet.String() { + if err := config.AllocateV6MasqueradeIPs(v6ConfiguredMasqueradeNet.IP, &staleMasqueradeIPs); err != nil { + return fmt.Errorf("unable to determine stale V6MasqueradeIPs: %s", err) + } + nextHops = append(nextHops, staleMasqueradeIPs.V6DummyNextHopMasqueradeIP) + } + + if err := deleteStaleMasqueradeRouteAndMACBinding(nbClient, routerName, nextHops); err != nil { + return fmt.Errorf("failed to remove stale MAC binding and static route for logical port %s: %w", logicalport, err) + } + + return nil +} + +// deleteStaleMasqueradeRouteAndMACBinding will attempt to remove the corresponding routes and MAC bindings given the +// list of nextHopIPs. If nextHopIPs is empty, then an attempt will be made to detect the stale route and MAC bindings +func deleteStaleMasqueradeRouteAndMACBinding(nbClient libovsdbclient.Client, routerName string, nextHopIPs []net.IP) error { + logicalport := types.GWRouterToExtSwitchPrefix + routerName + if len(nextHopIPs) == 0 { + // build valid values + validNextHops := []net.IP{config.Gateway.MasqueradeIPs.V4DummyNextHopMasqueradeIP, config.Gateway.MasqueradeIPs.V6DummyNextHopMasqueradeIP} + // lookup routes for external id that dont match currently configured masquerade subnets + for _, validNextHop := range validNextHops { + staticRoutePredicate := func(item *nbdb.LogicalRouterStaticRoute) bool { + if item.OutputPort != nil && *item.OutputPort == logicalport && + item.Nexthop != validNextHop.String() && utilnet.IPFamilyOfString(item.Nexthop) == utilnet.IPFamilyOf(validNextHop) { + if _, ok := item.ExternalIDs[util.OvnNodeMasqCIDR]; ok { + return true + } + } + return false + } + + staleRoutes, err := libovsdbops.FindLogicalRouterStaticRoutesWithPredicate(nbClient, staticRoutePredicate) + if err != nil { + return fmt.Errorf("failed to search for stale masquerade routes: %w", err) + } + + for _, staleRoute := range staleRoutes { + klog.Infof("Stale masquerade route found: %#v", *staleRoute) + // found stale routes, derive nexthop and flush the route and mac binding if it exists + staleNextHop := staleRoute.Nexthop + + macBindingPredicate := func(item *nbdb.StaticMACBinding) bool { + return item.LogicalPort == logicalport && item.IP == staleNextHop && + utilnet.IPFamilyOfString(item.IP) == utilnet.IPFamilyOfString(staleNextHop) + } + if err := libovsdbops.DeleteStaticMACBindingWithPredicate(nbClient, macBindingPredicate); err != nil { + return fmt.Errorf("failed to delete static MAC binding for logical port %s: %v", logicalport, err) + } + } + if err := libovsdbops.DeleteLogicalRouterStaticRoutes(nbClient, routerName, staleRoutes...); err != nil { + return err + } + } + return nil + } + + for _, nextHop := range nextHopIPs { + staticRoutePredicate := func(item *nbdb.LogicalRouterStaticRoute) bool { + if item.OutputPort != nil && *item.OutputPort == logicalport && + item.Nexthop == nextHop.String() && utilnet.IPFamilyOfString(item.Nexthop) == utilnet.IPFamilyOf(nextHop) { + if _, ok := item.ExternalIDs[util.OvnNodeMasqCIDR]; ok { + return true + } + } + return false + } + if err := libovsdbops.DeleteLogicalRouterStaticRoutesWithPredicate(nbClient, routerName, staticRoutePredicate); err != nil { + return fmt.Errorf("failed to delete static route from gateway router %s: %v", routerName, err) + } + + macBindingPredicate := func(item *nbdb.StaticMACBinding) bool { + return item.LogicalPort == logicalport && item.IP == nextHop.String() && + utilnet.IPFamilyOfString(item.IP) == utilnet.IPFamilyOf(nextHop) + } + if err := libovsdbops.DeleteStaticMACBindingWithPredicate(nbClient, macBindingPredicate); err != nil { + return fmt.Errorf("failed to delete static MAC binding for logical port %s: %v", logicalport, err) + } + } + return nil +} diff --git a/go-controller/pkg/ovn/gateway_test.go b/go-controller/pkg/ovn/gateway_test.go index bed3cb9a975..106f87590e2 100644 --- a/go-controller/pkg/ovn/gateway_test.go +++ b/go-controller/pkg/ovn/gateway_test.go @@ -9,6 +9,7 @@ import ( "github.com/onsi/ginkgo" "github.com/onsi/gomega" "github.com/onsi/gomega/format" + v1 "k8s.io/api/core/v1" utilnet "k8s.io/utils/net" @@ -116,10 +117,11 @@ func generateGatewayInitExpectedNB(testData []libovsdbtest.TestData, expectedOVN grStaticRoutes = append(grStaticRoutes, staticServiceRouteNamedUUID) testData = append(testData, &nbdb.LogicalRouterStaticRoute{ - UUID: staticServiceRouteNamedUUID, - IPPrefix: config.Gateway.V4MasqueradeSubnet, - Nexthop: config.Gateway.MasqueradeIPs.V4DummyNextHopMasqueradeIP.String(), - OutputPort: &externalRouterPort, + UUID: staticServiceRouteNamedUUID, + IPPrefix: config.Gateway.V4MasqueradeSubnet, + Nexthop: config.Gateway.MasqueradeIPs.V4DummyNextHopMasqueradeIP.String(), + OutputPort: &externalRouterPort, + ExternalIDs: map[string]string{util.OvnNodeMasqCIDR: ""}, }) testData = append(testData, &nbdb.StaticMACBinding{ UUID: "MAC-binding-UUID", @@ -134,10 +136,11 @@ func generateGatewayInitExpectedNB(testData []libovsdbtest.TestData, expectedOVN grStaticRoutes = append(grStaticRoutes, staticServiceRouteNamedUUID) testData = append(testData, &nbdb.LogicalRouterStaticRoute{ - UUID: staticServiceRouteNamedUUID, - IPPrefix: config.Gateway.V6MasqueradeSubnet, - Nexthop: config.Gateway.MasqueradeIPs.V6DummyNextHopMasqueradeIP.String(), - OutputPort: &externalRouterPort, + UUID: staticServiceRouteNamedUUID, + IPPrefix: config.Gateway.V6MasqueradeSubnet, + Nexthop: config.Gateway.MasqueradeIPs.V6DummyNextHopMasqueradeIP.String(), + OutputPort: &externalRouterPort, + ExternalIDs: map[string]string{util.OvnNodeMasqCIDR: ""}, }) testData = append(testData, &nbdb.StaticMACBinding{ UUID: "MAC-binding-2-UUID", @@ -411,8 +414,213 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { ChassisID: "SYSTEM-ID", InterfaceID: "INTERFACE-ID", MACAddress: ovntest.MustParseMAC("11:22:33:44:55:66"), - IPAddresses: ovntest.MustParseIPNets("169.254.33.2/24"), - NextHops: ovntest.MustParseIPs("169.254.33.1"), + IPAddresses: ovntest.MustParseIPNets("169.255.33.2/24"), + NextHops: ovntest.MustParseIPs("169.255.33.1"), + NodePortEnable: true, + } + sctpSupport := false + + var err error + fakeOvn.controller.defaultCOPPUUID, err = EnsureDefaultCOPP(fakeOvn.nbClient) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + err = fakeOvn.controller.gatewayInit( + nodeName, clusterIPSubnets, hostSubnets, l3GatewayConfig, sctpSupport, joinLRPIPs, defLRPIPs, true) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + testData := []libovsdbtest.TestData{} + skipSnat := false + expectedOVNClusterRouter.StaticRoutes = []string{} // the leftover LGW route should have got deleted + // We don't set up the Allow from mgmt port ACL here + mgmtPortIP := "" + expectedDatabaseState := generateGatewayInitExpectedNB(testData, expectedOVNClusterRouter, expectedNodeSwitch, + nodeName, clusterIPSubnets, hostSubnets, l3GatewayConfig, joinLRPIPs, defLRPIPs, skipSnat, mgmtPortIP, + "1400") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + }) + + ginkgo.It("removes stale MAC and route for old masquerade subnet using auto-detect", func() { + routeUUID := "route-UUID" + outputPort := types.GWRouterToExtSwitchPrefix + types.GWRouterPrefix + nodeName + leftoverMasqueradeIPRoute := &nbdb.LogicalRouterStaticRoute{ + Nexthop: "170.254.169.4", + IPPrefix: "170.254.169.0/16", + OutputPort: &outputPort, + UUID: routeUUID, + ExternalIDs: map[string]string{util.OvnNodeMasqCIDR: ""}, + } + leftoverStaticMAC := &nbdb.StaticMACBinding{ + IP: "170.254.169.4", + LogicalPort: types.GWRouterToExtSwitchPrefix + types.GWRouterPrefix + nodeName, + } + GRName := "GR_" + nodeName + expectedOVNGatewayRouter := &nbdb.LogicalRouter{ + UUID: GRName + "-UUID", + Name: GRName, + Options: map[string]string{ + "lb_force_snat_ip": "router_ip", + "snat-ct-zone": "0", + "always_learn_from_arp_request": "false", + "dynamic_neigh_routers": "true", + "mac_binding_age_threshold": types.GRMACBindingAgeThreshold, + }, + StaticRoutes: []string{routeUUID}, + } + expectedOVNClusterRouter := &nbdb.LogicalRouter{ + UUID: types.OVNClusterRouter + "-UUID", + Name: types.OVNClusterRouter, + } + expectedNodeSwitch := &nbdb.LogicalSwitch{ + UUID: nodeName + "-UUID", + Name: nodeName, + } + expectedClusterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterLBGroupName + "-UUID", + Name: types.ClusterLBGroupName, + } + expectedSwitchLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterSwitchLBGroupName + "-UUID", + Name: types.ClusterSwitchLBGroupName, + } + expectedRouterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterRouterLBGroupName + "-UUID", + Name: types.ClusterRouterLBGroupName, + } + fakeOvn.startWithDBSetup(libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + // tests migration from local to shared + leftoverMasqueradeIPRoute, + leftoverStaticMAC, + &nbdb.LogicalSwitch{ + UUID: types.OVNJoinSwitch + "-UUID", + Name: types.OVNJoinSwitch, + }, + expectedOVNClusterRouter, + expectedOVNGatewayRouter, + expectedNodeSwitch, + expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, + }, + }) + + clusterIPSubnets := ovntest.MustParseIPNets("10.128.0.0/14") + hostSubnets := ovntest.MustParseIPNets("10.130.0.0/23") + joinLRPIPs := ovntest.MustParseIPNets("100.64.0.3/16") + defLRPIPs := ovntest.MustParseIPNets("100.64.0.1/16") + l3GatewayConfig := &util.L3GatewayConfig{ + Mode: config.GatewayModeLocal, + ChassisID: "SYSTEM-ID", + InterfaceID: "INTERFACE-ID", + MACAddress: ovntest.MustParseMAC("11:22:33:44:55:66"), + IPAddresses: ovntest.MustParseIPNets("169.255.33.2/24"), + NextHops: ovntest.MustParseIPs("169.255.33.1"), + NodePortEnable: true, + } + sctpSupport := false + + var err error + fakeOvn.controller.defaultCOPPUUID, err = EnsureDefaultCOPP(fakeOvn.nbClient) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + err = fakeOvn.controller.gatewayInit( + nodeName, clusterIPSubnets, hostSubnets, l3GatewayConfig, sctpSupport, joinLRPIPs, defLRPIPs, true) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + testData := []libovsdbtest.TestData{} + skipSnat := false + expectedOVNClusterRouter.StaticRoutes = []string{} // the leftover LGW route should have got deleted + // We don't set up the Allow from mgmt port ACL here + mgmtPortIP := "" + expectedDatabaseState := generateGatewayInitExpectedNB(testData, expectedOVNClusterRouter, expectedNodeSwitch, + nodeName, clusterIPSubnets, hostSubnets, l3GatewayConfig, joinLRPIPs, defLRPIPs, skipSnat, mgmtPortIP, + "1400") + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(expectedDatabaseState)) + }) + + ginkgo.It("removes stale MAC and route for old masquerade subnet using stale annotation", func() { + routeUUID := "route-UUID" + outputPort := types.GWRouterToExtSwitchPrefix + types.GWRouterPrefix + nodeName + leftoverMasqueradeIPRoute := &nbdb.LogicalRouterStaticRoute{ + Nexthop: "170.254.169.4", + IPPrefix: "170.254.169.0/16", + OutputPort: &outputPort, + UUID: routeUUID, + ExternalIDs: map[string]string{util.OvnNodeMasqCIDR: ""}, + } + leftoverStaticMAC := &nbdb.StaticMACBinding{ + IP: "170.254.169.4", + LogicalPort: types.GWRouterToExtSwitchPrefix + types.GWRouterPrefix + nodeName, + } + GRName := "GR_" + nodeName + expectedOVNGatewayRouter := &nbdb.LogicalRouter{ + UUID: GRName + "-UUID", + Name: GRName, + Options: map[string]string{ + "lb_force_snat_ip": "router_ip", + "snat-ct-zone": "0", + "always_learn_from_arp_request": "false", + "dynamic_neigh_routers": "true", + "mac_binding_age_threshold": types.GRMACBindingAgeThreshold, + }, + StaticRoutes: []string{routeUUID}, + } + expectedOVNClusterRouter := &nbdb.LogicalRouter{ + UUID: types.OVNClusterRouter + "-UUID", + Name: types.OVNClusterRouter, + } + expectedNodeSwitch := &nbdb.LogicalSwitch{ + UUID: nodeName + "-UUID", + Name: nodeName, + } + expectedClusterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterLBGroupName + "-UUID", + Name: types.ClusterLBGroupName, + } + expectedSwitchLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterSwitchLBGroupName + "-UUID", + Name: types.ClusterSwitchLBGroupName, + } + expectedRouterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterRouterLBGroupName + "-UUID", + Name: types.ClusterRouterLBGroupName, + } + a := newObjectMeta(nodeName, "") + a.Annotations = map[string]string{ + util.OvnNodeMasqCIDR: "{\"ipv4\":\"170.254.169.0/16\",\"ipv6\":\"fd69::/112\"}", + } + node1 := v1.Node{ObjectMeta: a} + fakeOvn.startWithDBSetup(libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + // tests migration from local to shared + leftoverMasqueradeIPRoute, + leftoverStaticMAC, + &nbdb.LogicalSwitch{ + UUID: types.OVNJoinSwitch + "-UUID", + Name: types.OVNJoinSwitch, + }, + expectedOVNClusterRouter, + expectedOVNGatewayRouter, + expectedNodeSwitch, + expectedClusterLBGroup, + expectedSwitchLBGroup, + expectedRouterLBGroup, + }, + }, + &v1.NodeList{Items: []v1.Node{node1}}, + ) + + clusterIPSubnets := ovntest.MustParseIPNets("10.128.0.0/14") + hostSubnets := ovntest.MustParseIPNets("10.130.0.0/23") + joinLRPIPs := ovntest.MustParseIPNets("100.64.0.3/16") + defLRPIPs := ovntest.MustParseIPNets("100.64.0.1/16") + l3GatewayConfig := &util.L3GatewayConfig{ + Mode: config.GatewayModeLocal, + ChassisID: "SYSTEM-ID", + InterfaceID: "INTERFACE-ID", + MACAddress: ovntest.MustParseMAC("11:22:33:44:55:66"), + IPAddresses: ovntest.MustParseIPNets("169.255.33.2/24"), + NextHops: ovntest.MustParseIPs("169.255.33.1"), NodePortEnable: true, } sctpSupport := false @@ -488,7 +696,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { ChassisID: "SYSTEM-ID", InterfaceID: "INTERFACE-ID", MACAddress: ovntest.MustParseMAC("11:22:33:44:55:66"), - IPAddresses: ovntest.MustParseIPNets("169.254.33.2/24"), + IPAddresses: ovntest.MustParseIPNets("169.255.33.2/24"), NodePortEnable: true, } sctpSupport := false @@ -557,8 +765,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { ChassisID: "SYSTEM-ID", InterfaceID: "INTERFACE-ID", MACAddress: ovntest.MustParseMAC("11:22:33:44:55:66"), - IPAddresses: ovntest.MustParseIPNets("169.254.33.2/24"), - NextHops: ovntest.MustParseIPs("169.254.33.1"), + IPAddresses: ovntest.MustParseIPNets("169.255.33.2/24"), + NextHops: ovntest.MustParseIPs("169.255.33.1"), NodePortEnable: true, } sctpSupport := false @@ -639,8 +847,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { ChassisID: "SYSTEM-ID", InterfaceID: "INTERFACE-ID", MACAddress: ovntest.MustParseMAC("11:22:33:44:55:66"), - IPAddresses: ovntest.MustParseIPNets("169.254.33.2/24"), - NextHops: ovntest.MustParseIPs("169.254.33.1"), + IPAddresses: ovntest.MustParseIPNets("169.255.33.2/24"), + NextHops: ovntest.MustParseIPs("169.255.33.1"), NodePortEnable: true, } sctpSupport := false @@ -875,8 +1083,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { ChassisID: "SYSTEM-ID", InterfaceID: "INTERFACE-ID", MACAddress: ovntest.MustParseMAC("11:22:33:44:55:66"), - IPAddresses: ovntest.MustParseIPNets("169.254.33.2/24", "fd99::2/64"), - NextHops: ovntest.MustParseIPs("169.254.33.1", "fd99::1"), + IPAddresses: ovntest.MustParseIPNets("169.255.33.2/24", "fd99::2/64"), + NextHops: ovntest.MustParseIPs("169.255.33.1", "fd99::1"), NodePortEnable: true, } sctpSupport := false @@ -944,8 +1152,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { ChassisID: "SYSTEM-ID", InterfaceID: "INTERFACE-ID", MACAddress: ovntest.MustParseMAC("11:22:33:44:55:66"), - IPAddresses: ovntest.MustParseIPNets("169.254.33.2/24"), - NextHops: ovntest.MustParseIPs("169.254.33.1"), + IPAddresses: ovntest.MustParseIPNets("169.255.33.2/24"), + NextHops: ovntest.MustParseIPs("169.255.33.1"), NodePortEnable: true, } sctpSupport := false @@ -1046,8 +1254,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { ChassisID: "SYSTEM-ID", InterfaceID: "INTERFACE-ID", MACAddress: ovntest.MustParseMAC("11:22:33:44:55:66"), - IPAddresses: ovntest.MustParseIPNets("169.254.33.2/24"), - NextHops: ovntest.MustParseIPs("169.254.33.1"), + IPAddresses: ovntest.MustParseIPNets("169.255.33.2/24"), + NextHops: ovntest.MustParseIPs("169.255.33.1"), NodePortEnable: true, } sctpSupport := false @@ -1150,8 +1358,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { ChassisID: "SYSTEM-ID", InterfaceID: "INTERFACE-ID", MACAddress: ovntest.MustParseMAC("11:22:33:44:55:66"), - IPAddresses: ovntest.MustParseIPNets("169.254.33.2/24", "fd99::2/64"), - NextHops: ovntest.MustParseIPs("169.254.33.1", "fd99::1"), + IPAddresses: ovntest.MustParseIPNets("169.255.33.2/24", "fd99::2/64"), + NextHops: ovntest.MustParseIPs("169.255.33.1", "fd99::1"), NodePortEnable: true, } sctpSupport := false @@ -1228,8 +1436,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { ChassisID: "SYSTEM-ID", InterfaceID: "INTERFACE-ID", MACAddress: ovntest.MustParseMAC("11:22:33:44:55:66"), - IPAddresses: ovntest.MustParseIPNets("169.254.33.2/24"), - NextHops: ovntest.MustParseIPs("169.254.33.1"), + IPAddresses: ovntest.MustParseIPNets("169.255.33.2/24"), + NextHops: ovntest.MustParseIPs("169.255.33.1"), NodePortEnable: true, } sctpSupport := false @@ -1335,8 +1543,8 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { ChassisID: "SYSTEM-ID", InterfaceID: "INTERFACE-ID", MACAddress: ovntest.MustParseMAC("11:22:33:44:55:66"), - IPAddresses: ovntest.MustParseIPNets("169.254.33.2/24"), - NextHops: ovntest.MustParseIPs("169.254.33.1"), + IPAddresses: ovntest.MustParseIPNets("169.255.33.2/24"), + NextHops: ovntest.MustParseIPs("169.255.33.1"), NodePortEnable: true, } sctpSupport := false diff --git a/go-controller/pkg/ovnwebhook/nodeadmission.go b/go-controller/pkg/ovnwebhook/nodeadmission.go index 30839aa4890..9425708c568 100644 --- a/go-controller/pkg/ovnwebhook/nodeadmission.go +++ b/go-controller/pkg/ovnwebhook/nodeadmission.go @@ -30,6 +30,7 @@ var commonNodeAnnotationChecks = map[string]checkNodeAnnot{ util.OvnNodeL3GatewayConfig: nil, util.OvnNodeManagementPortMacAddresses: nil, util.OvnNodeIfAddr: nil, + util.OvnNodeMasqCIDR: nil, util.OvnNodeGatewayMtuSupport: nil, util.OvnNodeManagementPort: nil, util.OvnNodeChassisID: func(v annotationChange, nodeName string) error { diff --git a/go-controller/pkg/util/net_linux.go b/go-controller/pkg/util/net_linux.go index 76b1636a3c1..414d6ed0f94 100644 --- a/go-controller/pkg/util/net_linux.go +++ b/go-controller/pkg/util/net_linux.go @@ -180,6 +180,15 @@ func getFamily(ip net.IP) int { } } +// LinkByName returns the netlink device +func LinkByName(interfaceName string) (netlink.Link, error) { + link, err := netLinkOps.LinkByName(interfaceName) + if err != nil { + return nil, fmt.Errorf("failed to lookup link %s: %w", interfaceName, err) + } + return link, nil +} + // LinkSetUp returns the netlink device with its state marked up func LinkSetUp(interfaceName string) (netlink.Link, error) { link, err := netLinkOps.LinkByName(interfaceName) @@ -240,6 +249,21 @@ func LinkAddrExist(link netlink.Link, address *net.IPNet) (bool, error) { return false, nil } +// LinkAddrGetIPNet returns IPNet given the IP of an address present on given link +func LinkAddrGetIPNet(link netlink.Link, ip net.IP) (*net.IPNet, error) { + addrs, err := netLinkOps.AddrList(link, getFamily(ip)) + if err != nil { + return nil, fmt.Errorf("failed to list addresses for the link %s: %v", + link.Attrs().Name, err) + } + for _, addr := range addrs { + if addr.IPNet.IP.Equal(ip) { + return addr.IPNet, nil + } + } + return nil, nil +} + // LinkAddrAdd adds a new address. If both preferredLifetime & validLifetime, // are zero, then they are not applied, but if either parameters are not zero, both are applied. func LinkAddrAdd(link netlink.Link, address *net.IPNet, flags, preferredLifetime, validLifetime int) error { @@ -247,6 +271,7 @@ func LinkAddrAdd(link netlink.Link, address *net.IPNet, flags, preferredLifetime if err != nil { return fmt.Errorf("failed to add address %s on link %s: %v", address.String(), link.Attrs().Name, err) } + return nil } diff --git a/go-controller/pkg/util/net_linux_unit_test.go b/go-controller/pkg/util/net_linux_unit_test.go index 81398d3b84c..d6ca4764c88 100644 --- a/go-controller/pkg/util/net_linux_unit_test.go +++ b/go-controller/pkg/util/net_linux_unit_test.go @@ -41,6 +41,50 @@ func TestGetFamily(t *testing.T) { } } +func TestLinkByName(t *testing.T) { + mockNetLinkOps := new(mocks.NetLinkOps) + mockLink := new(netlink_mocks.Link) + // below is defined in net_linux.go + netLinkOps = mockNetLinkOps + + tests := []struct { + desc string + input string + errExp bool + onRetArgsNetLinkLibOpers []ovntest.TestifyMockHelper + }{ + { + desc: "fails to look up link", + input: "invalidIfaceName", + errExp: true, + onRetArgsNetLinkLibOpers: []ovntest.TestifyMockHelper{ + {OnCallMethodName: "LinkByName", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{nil, fmt.Errorf("mock error")}}, + }, + }, + { + desc: "sets up the link successfully", + input: "testIfaceName", + errExp: false, + onRetArgsNetLinkLibOpers: []ovntest.TestifyMockHelper{ + {OnCallMethodName: "LinkByName", OnCallMethodArgType: []string{"string"}, RetArgList: []interface{}{mockLink, nil}}, + }, + }, + } + for i, tc := range tests { + t.Run(fmt.Sprintf("%d:%s", i, tc.desc), func(t *testing.T) { + ovntest.ProcessMockFnList(&mockNetLinkOps.Mock, tc.onRetArgsNetLinkLibOpers) + res, err := LinkByName(tc.input) + t.Log(res, err) + if tc.errExp { + assert.Error(t, err) + } else { + assert.NotNil(t, res) + } + mockNetLinkOps.AssertExpectations(t) + }) + } +} + func TestLinkSetUp(t *testing.T) { mockNetLinkOps := new(mocks.NetLinkOps) mockLink := new(netlink_mocks.Link) diff --git a/go-controller/pkg/util/node_annotations.go b/go-controller/pkg/util/node_annotations.go index 7efe7da4650..fc833de0080 100644 --- a/go-controller/pkg/util/node_annotations.go +++ b/go-controller/pkg/util/node_annotations.go @@ -32,14 +32,14 @@ import ( // "mode": "local", // "interface-id": "br-local_ip-10-0-129-64.us-east-2.compute.internal", // "mac-address": "f2:20:a0:3c:26:4c", -// "ip-addresses": ["169.254.33.2/24"], -// "next-hops": ["169.254.33.1"], +// "ip-addresses": ["169.255.33.2/24"], +// "next-hops": ["169.255.33.1"], // "node-port-enable": "true", // "vlan-id": "0" // // # backward-compat -// "ip-address": "169.254.33.2/24", -// "next-hop": "169.254.33.1", +// "ip-address": "169.255.33.2/24", +// "next-hop": "169.255.33.1", // } // } // k8s.ovn.org/node-chassis-id: b1f96182-2bdd-42b6-88f9-9a1fc1c85ece @@ -91,6 +91,9 @@ const ( // }", OVNNodeGRLRPAddrs = "k8s.ovn.org/node-gateway-router-lrp-ifaddrs" + // OvnNodeMasqCIDR is the CIDR form representation of the masquerade subnet that is currently configured on this node (i.e. 169.254.169.0/29) + OvnNodeMasqCIDR = "k8s.ovn.org/node-masquerade-subnet" + // OvnNodeEgressLabel is a user assigned node label indicating to ovn-kubernetes that the node is to be used for egress IP assignment ovnNodeEgressLabel = "k8s.ovn.org/egress-assignable" @@ -682,6 +685,12 @@ func NodeTransitSwitchPortAddrAnnotationChanged(oldNode, newNode *corev1.Node) b return oldNode.Annotations[ovnTransitSwitchPortAddr] != newNode.Annotations[ovnTransitSwitchPortAddr] } +// CreateNodeMasqueradeSubnetAnnotation sets the IPv4 / IPv6 values of the node's Masquerade subnet. +func CreateNodeMasqueradeSubnetAnnotation(nodeAnnotation map[string]interface{}, nodeIPNetv4, + nodeIPNetv6 *net.IPNet) (map[string]interface{}, error) { + return createPrimaryIfAddrAnnotation(OvnNodeMasqCIDR, nodeAnnotation, nodeIPNetv4, nodeIPNetv6) +} + const UnlimitedNodeCapacity = math.MaxInt32 type ifAddr struct { @@ -867,6 +876,12 @@ func ParseNodeTransitSwitchPortAddrs(node *kapi.Node) ([]*net.IPNet, error) { return parsePrimaryIfAddrAnnotation(node, ovnTransitSwitchPortAddr) } +// ParseNodeMasqueradeSubnet returns the IPv4 and/or IPv6 networks for the node's gateway router port +// stored in the 'OvnNodeMasqCIDR' annotation +func ParseNodeMasqueradeSubnet(node *kapi.Node) ([]*net.IPNet, error) { + return parsePrimaryIfAddrAnnotation(node, OvnNodeMasqCIDR) +} + // GetNodeEIPConfig attempts to generate EIP configuration from a nodes annotations. // If the platform is running in the cloud, retrieve config info from node obj annotation added by Cloud Network Config // Controller (CNCC). If not on a cloud platform (i.e. baremetal), retrieve from the node obj primary interface annotation. diff --git a/go-controller/pkg/util/node_annotations_unit_test.go b/go-controller/pkg/util/node_annotations_unit_test.go index d06eb931ba2..ab008f0b3fb 100644 --- a/go-controller/pkg/util/node_annotations_unit_test.go +++ b/go-controller/pkg/util/node_annotations_unit_test.go @@ -302,7 +302,7 @@ func TestParseNodeL3GatewayAnnotation(t *testing.T) { desc: "error: annotation for network not found", inpNode: &v1.Node{ ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{"k8s.ovn.org/l3-gateway-config": `{"nondefault":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.254.33.2/24", "next-hop":"169.254.33.1"}}`}, + Annotations: map[string]string{"k8s.ovn.org/l3-gateway-config": `{"nondefault":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.255.33.2/24", "next-hop":"169.255.33.1"}}`}, }, }, errAssert: true, @@ -312,7 +312,7 @@ func TestParseNodeL3GatewayAnnotation(t *testing.T) { desc: "error: nod chassis ID annotation not found", inpNode: &v1.Node{ ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{"k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.254.33.2/24", "next-hop":"169.254.33.1"}}`}, + Annotations: map[string]string{"k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.255.33.2/24", "next-hop":"169.255.33.1"}}`}, }, }, errAssert: true, @@ -323,7 +323,7 @@ func TestParseNodeL3GatewayAnnotation(t *testing.T) { inpNode: &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.254.33.2/24", "next-hop":"169.254.33.1"}}`, + "k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.255.33.2/24", "next-hop":"169.255.33.1"}}`, "k8s.ovn.org/node-chassis-id": "79fdcfc4-6fe6-4cd3-8242-c0f85a4668ec", }, }, @@ -362,7 +362,7 @@ func TestNodeL3GatewayAnnotationChanged(t *testing.T) { newNode: &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.254.33.2/24", "next-hop":"169.254.33.1"}}`, + "k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.255.33.2/24", "next-hop":"169.255.33.1"}}`, }, }, }, @@ -373,14 +373,14 @@ func TestNodeL3GatewayAnnotationChanged(t *testing.T) { newNode: &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.254.33.3/24", "next-hop":"169.254.33.1"}}`, + "k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.254.33.3/24", "next-hop":"169.255.33.1"}}`, }, }, }, oldNode: &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.254.33.2/24", "next-hop":"169.254.33.1"}}`, + "k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.255.33.2/24", "next-hop":"169.255.33.1"}}`, }, }, }, @@ -391,14 +391,14 @@ func TestNodeL3GatewayAnnotationChanged(t *testing.T) { newNode: &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.254.33.2/24", "next-hop":"169.254.33.1"}}`, + "k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.255.33.2/24", "next-hop":"169.255.33.1"}}`, }, }, }, oldNode: &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - "k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.254.33.2/24", "next-hop":"169.254.33.1"}}`, + "k8s.ovn.org/l3-gateway-config": `{"default":{"mode":"local","mac-address":"7e:57:f8:f0:3c:49", "ip-address":"169.255.33.2/24", "next-hop":"169.255.33.1"}}`, }, }, }, diff --git a/test/e2e/service.go b/test/e2e/service.go index f4babd385fc..be6675eb387 100644 --- a/test/e2e/service.go +++ b/test/e2e/service.go @@ -518,7 +518,7 @@ var _ = ginkgo.Describe("Services", func() { const ( v6ExternAddr = "2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF" v4ExternAddr = "8.8.8.8" - hostMasqIPv4 = "169.254.169.2" + hostMasqIPv4 = "169.254.0.2" hostMasqIPv6 = "fd69::2" ) nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.TODO(), cs, e2eservice.MaxNodesForEndpointsTests) @@ -1066,7 +1066,7 @@ spec: etpClusterServiceName, len(endPoints), time.Second, wait.ForeverTestTimeout) framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", etpClusterServiceName, f.Namespace.Name) - + ginkgo.By("Checking connectivity to the external container from egressIP pod " + egressPod.Name + " and verify that the source IP is the secondary NIC egress IP") framework.Logf("Destination IPs for external container are ip=%v", targetSecondaryNode.nodeIP) err = wait.PollImmediate(retryInterval, retryTimeout, targetExternalContainerAndTest(targetSecondaryNode, egressPod.Name, @@ -1318,7 +1318,7 @@ func getServiceBackendsFromPod(execPod *v1.Pod, serviceIP string, servicePort in // This test ensures that - when a pod that's a backend for a service curls the // service ip; if the traffic was DNAT-ed to the same src pod (hairpin/loopback case) - -// the srcIP of reply traffic is SNATed to the special masqurade IP 169.254.169.5 +// the srcIP of reply traffic is SNATed to the special masqurade IP 169.254.0.5 // or "fd69::5" var _ = ginkgo.Describe("Service Hairpin SNAT", func() { const ( @@ -1326,7 +1326,7 @@ var _ = ginkgo.Describe("Service Hairpin SNAT", func() { backendName = "hairpin-backend-pod" endpointHTTPPort = "80" serviceHTTPPort = 6666 - V4LBHairpinMasqueradeIP = "169.254.169.5" + V4LBHairpinMasqueradeIP = "169.254.0.5" V6LBHairpinMasqueradeIP = "fd69::5" )