From cfb72f74689f42daa69885204f89737bf29ed5be Mon Sep 17 00:00:00 2001 From: Jussi Maki Date: Mon, 27 May 2024 17:03:39 +0200 Subject: [PATCH] datapath: Extend LocalNodeConfiguration with devices and addresses The loader and the config writer were accessing node IP address information and devices via either the StateDB tables or via pkg/node globals. This made these components hard to test and made it hard to reason about reinitialization when data changed. Clean this up by extending the LocalNodeConfiguration with additional fields that capture this dynamic data that was accessed out-of-band previously. The creation of LocalNodeConfiguration is moved from NodeDiscovery into the datapath orchestrator where it belongs. Signed-off-by: Jussi Maki --- daemon/cmd/daemon_test.go | 2 - daemon/cmd/datapath.go | 6 - daemon/cmd/status_test.go | 23 +- pkg/datapath/fake/types/datapath.go | 10 +- pkg/datapath/linux/config/cell.go | 5 - pkg/datapath/linux/config/config.go | 81 +-- pkg/datapath/linux/config/config_test.go | 77 +-- pkg/datapath/linux/fuzz_test.go | 4 +- pkg/datapath/linux/ipsec.go | 21 +- pkg/datapath/linux/node.go | 46 +- pkg/datapath/linux/node_linux_test.go | 497 ++++++------------- pkg/datapath/linux/node_test.go | 21 +- pkg/datapath/loader/base.go | 37 +- pkg/datapath/loader/cache.go | 10 +- pkg/datapath/loader/cache_test.go | 32 +- pkg/datapath/loader/hash.go | 8 +- pkg/datapath/loader/hash_test.go | 23 +- pkg/datapath/loader/loader.go | 61 +-- pkg/datapath/loader/loader_test.go | 58 +-- pkg/datapath/loader/template_test.go | 8 +- pkg/datapath/loader/util_test.go | 42 +- pkg/datapath/orchestrator/localnodeconfig.go | 89 ++++ pkg/datapath/orchestrator/orchestrator.go | 35 +- pkg/datapath/types/config.go | 4 +- pkg/datapath/types/loader.go | 4 +- pkg/datapath/types/node.go | 71 ++- pkg/endpoint/bpf.go | 2 +- pkg/endpoint/bpf_test.go | 4 +- pkg/nodediscovery/cell.go | 2 - pkg/nodediscovery/localnodeconfig.go | 53 -- pkg/nodediscovery/nodediscovery.go | 11 +- 31 files changed, 570 insertions(+), 777 deletions(-) create mode 100644 pkg/datapath/orchestrator/localnodeconfig.go delete mode 100644 pkg/nodediscovery/localnodeconfig.go diff --git a/daemon/cmd/daemon_test.go b/daemon/cmd/daemon_test.go index 0dd8e3c03a831..7369d49c625d2 100644 --- a/daemon/cmd/daemon_test.go +++ b/daemon/cmd/daemon_test.go @@ -22,7 +22,6 @@ import ( fakecni "github.com/cilium/cilium/daemon/cmd/cni/fake" "github.com/cilium/cilium/pkg/controller" fakeDatapath "github.com/cilium/cilium/pkg/datapath/fake" - "github.com/cilium/cilium/pkg/datapath/loader" "github.com/cilium/cilium/pkg/datapath/prefilter" datapath "github.com/cilium/cilium/pkg/datapath/types" "github.com/cilium/cilium/pkg/endpoint" @@ -130,7 +129,6 @@ func setupDaemonSuite(tb testing.TB) *DaemonSuite { ), fakeDatapath.Cell, prefilter.Cell, - loader.Cell, monitorAgent.Cell, ControlPlane, metrics.Cell, diff --git a/daemon/cmd/datapath.go b/daemon/cmd/datapath.go index 7a75a5c8d2382..f302418c3d223 100644 --- a/daemon/cmd/datapath.go +++ b/daemon/cmd/datapath.go @@ -36,12 +36,6 @@ import ( "github.com/cilium/cilium/pkg/option" ) -// LocalConfig returns the local configuration of the daemon's nodediscovery. -func (d *Daemon) LocalConfig() *datapath.LocalNodeConfiguration { - d.nodeDiscovery.WaitForLocalNodeInit() - return &d.nodeDiscovery.LocalConfig -} - // listFilterIfs returns a map of interfaces based on the given filter. // The filter should take a link and, if found, return the index of that // interface, if not found return -1. diff --git a/daemon/cmd/status_test.go b/daemon/cmd/status_test.go index 4f206a8fa4cc0..05d8f35df9838 100644 --- a/daemon/cmd/status_test.go +++ b/daemon/cmd/status_test.go @@ -15,7 +15,6 @@ import ( . "github.com/cilium/cilium/api/v1/server/restapi/daemon" "github.com/cilium/cilium/daemon/cmd/cni/fake" fakeTypes "github.com/cilium/cilium/pkg/datapath/fake/types" - "github.com/cilium/cilium/pkg/mtu" "github.com/cilium/cilium/pkg/node/manager" nodeTypes "github.com/cilium/cilium/pkg/node/types" "github.com/cilium/cilium/pkg/nodediscovery" @@ -27,7 +26,6 @@ type GetNodesSuite struct { } var ( - mtuConfig = mtu.NewConfiguration(0, false, false, false, false, 0, nil) fakeConfig = &option.DaemonConfig{ RoutingMode: option.RoutingModeTunnel, EnableIPSec: true, @@ -80,8 +78,7 @@ func Test_getNodesHandle(t *testing.T) { { name: "create a client ID and store it locally", setupArgs: func() args { - lnc, _ := nodediscovery.NewLocalNodeConfig(&mtuConfig, option.Config) - nodeDiscovery := nodediscovery.NewNodeDiscovery(g.nm, nil, nil, lnc, &fake.FakeCNIConfigManager{}) + nodeDiscovery := nodediscovery.NewNodeDiscovery(g.nm, nil, nil, &fake.FakeCNIConfigManager{}) return args{ params: GetClusterNodesParams{ ClientID: &zero, @@ -112,8 +109,7 @@ func Test_getNodesHandle(t *testing.T) { { name: "retrieve nodes diff from a client that was already present", setupArgs: func() args { - lnc, _ := nodediscovery.NewLocalNodeConfig(&mtuConfig, option.Config) - nodeDiscovery := nodediscovery.NewNodeDiscovery(g.nm, nil, nil, lnc, &fake.FakeCNIConfigManager{}) + nodeDiscovery := nodediscovery.NewNodeDiscovery(g.nm, nil, nil, &fake.FakeCNIConfigManager{}) return args{ params: GetClusterNodesParams{ ClientID: &clientIDs[0], @@ -166,8 +162,7 @@ func Test_getNodesHandle(t *testing.T) { { name: "retrieve nodes from an expired client, it should be ok because the clean up only happens when on insertion", setupArgs: func() args { - lnc, _ := nodediscovery.NewLocalNodeConfig(&mtuConfig, option.Config) - nodeDiscovery := nodediscovery.NewNodeDiscovery(g.nm, nil, nil, lnc, &fake.FakeCNIConfigManager{}) + nodeDiscovery := nodediscovery.NewNodeDiscovery(g.nm, nil, nil, &fake.FakeCNIConfigManager{}) return args{ params: GetClusterNodesParams{ ClientID: &clientIDs[0], @@ -221,8 +216,7 @@ func Test_getNodesHandle(t *testing.T) { { name: "retrieve nodes for a new client, the expired client should be deleted", setupArgs: func() args { - lnc, _ := nodediscovery.NewLocalNodeConfig(&mtuConfig, option.Config) - nodeDiscovery := nodediscovery.NewNodeDiscovery(g.nm, nil, nil, lnc, &fake.FakeCNIConfigManager{}) + nodeDiscovery := nodediscovery.NewNodeDiscovery(g.nm, nil, nil, &fake.FakeCNIConfigManager{}) return args{ params: GetClusterNodesParams{ ClientID: &zero, @@ -271,8 +265,7 @@ func Test_getNodesHandle(t *testing.T) { { name: "retrieve nodes for a new client, however the randomizer allocated an existing clientID, so we should return a empty clientID", setupArgs: func() args { - lnc, _ := nodediscovery.NewLocalNodeConfig(&mtuConfig, option.Config) - nodeDiscovery := nodediscovery.NewNodeDiscovery(g.nm, nil, nil, lnc, &fake.FakeCNIConfigManager{}) + nodeDiscovery := nodediscovery.NewNodeDiscovery(g.nm, nil, nil, &fake.FakeCNIConfigManager{}) return args{ params: GetClusterNodesParams{ ClientID: &zero, @@ -321,8 +314,7 @@ func Test_getNodesHandle(t *testing.T) { { name: "retrieve nodes for a client that does not want to have diffs, leave all other stored clients alone", setupArgs: func() args { - lnc, _ := nodediscovery.NewLocalNodeConfig(&mtuConfig, option.Config) - nodeDiscovery := nodediscovery.NewNodeDiscovery(g.nm, nil, nil, lnc, &fake.FakeCNIConfigManager{}) + nodeDiscovery := nodediscovery.NewNodeDiscovery(g.nm, nil, nil, &fake.FakeCNIConfigManager{}) return args{ params: GetClusterNodesParams{}, daemon: &Daemon{ @@ -428,10 +420,9 @@ func Test_cleanupClients(t *testing.T) { args := tt.setupArgs() want := tt.setupWanted() h := &getNodes{clients: args.clients} - lnc, _ := nodediscovery.NewLocalNodeConfig(&mtuConfig, option.Config) h.cleanupClients( &Daemon{ - nodeDiscovery: nodediscovery.NewNodeDiscovery(g.nm, nil, nil, lnc, &fake.FakeCNIConfigManager{}), + nodeDiscovery: nodediscovery.NewNodeDiscovery(g.nm, nil, nil, &fake.FakeCNIConfigManager{}), }) require.EqualValues(t, want.clients, h.clients) } diff --git a/pkg/datapath/fake/types/datapath.go b/pkg/datapath/fake/types/datapath.go index 63be677ceae7c..5fb084c24ac20 100644 --- a/pkg/datapath/fake/types/datapath.go +++ b/pkg/datapath/fake/types/datapath.go @@ -72,12 +72,12 @@ func (f *FakeDatapath) WriteNetdevConfig(io.Writer, *option.IntOptions) error { } // WriteTemplateConfig pretends to write the endpoint configuration to a writer. -func (f *FakeDatapath) WriteTemplateConfig(io.Writer, datapath.EndpointConfiguration) error { +func (f *FakeDatapath) WriteTemplateConfig(io.Writer, *datapath.LocalNodeConfiguration, datapath.EndpointConfiguration) error { return nil } // WriteEndpointConfig pretends to write the endpoint configuration to a writer. -func (f *FakeDatapath) WriteEndpointConfig(io.Writer, datapath.EndpointConfiguration) error { +func (f *FakeDatapath) WriteEndpointConfig(io.Writer, *datapath.LocalNodeConfiguration, datapath.EndpointConfiguration) error { return nil } @@ -154,7 +154,7 @@ func (f *FakeLoader) CustomCallsMapPath(id uint16) string { } // Reinitialize does nothing. -func (f *FakeLoader) Reinitialize(ctx context.Context, tunnelConfig tunnel.Config, deviceMTU int, iptMgr datapath.IptablesManager, p datapath.Proxy) error { +func (f *FakeLoader) Reinitialize(ctx context.Context, cfg datapath.LocalNodeConfiguration, tunnelConfig tunnel.Config, iptMgr datapath.IptablesManager, p datapath.Proxy) error { return nil } @@ -170,6 +170,10 @@ func (f *FakeLoader) DetachXDP(ifaceName string, bpffsBase, progName string) err return nil } +func (f *FakeLoader) WriteEndpointConfig(w io.Writer, e datapath.EndpointConfiguration) error { + return nil +} + type FakeOrchestrator struct{} func (f *FakeOrchestrator) Reinitialize(ctx context.Context) error { diff --git a/pkg/datapath/linux/config/cell.go b/pkg/datapath/linux/config/cell.go index a99388a7b5baf..14160fe47fae4 100644 --- a/pkg/datapath/linux/config/cell.go +++ b/pkg/datapath/linux/config/cell.go @@ -5,12 +5,10 @@ package config import ( "github.com/cilium/hive/cell" - "github.com/cilium/statedb" "github.com/sirupsen/logrus" dpdef "github.com/cilium/cilium/pkg/datapath/linux/config/defines" "github.com/cilium/cilium/pkg/datapath/linux/sysctl" - "github.com/cilium/cilium/pkg/datapath/tables" datapath "github.com/cilium/cilium/pkg/datapath/types" "github.com/cilium/cilium/pkg/maps/nodemap" ) @@ -24,9 +22,6 @@ type WriterParams struct { NodeExtraDefines []dpdef.Map `group:"header-node-defines"` NodeExtraDefineFns []dpdef.Fn `group:"header-node-define-fns"` Sysctl sysctl.Sysctl - DB *statedb.DB - Devices statedb.Table[*tables.Device] - NodeAddresses statedb.Table[tables.NodeAddress] } var Cell = cell.Module( diff --git a/pkg/datapath/linux/config/config.go b/pkg/datapath/linux/config/config.go index 97c7a4c3235c9..9102240689164 100644 --- a/pkg/datapath/linux/config/config.go +++ b/pkg/datapath/linux/config/config.go @@ -13,6 +13,7 @@ import ( "io" "net" "net/netip" + "slices" "sort" "strconv" "strings" @@ -21,8 +22,6 @@ import ( "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" - "github.com/cilium/statedb" - "github.com/cilium/cilium/pkg/bpf" "github.com/cilium/cilium/pkg/byteorder" "github.com/cilium/cilium/pkg/cidr" @@ -59,7 +58,6 @@ import ( "github.com/cilium/cilium/pkg/maps/vtep" "github.com/cilium/cilium/pkg/maps/worldcidrsmap" "github.com/cilium/cilium/pkg/netns" - "github.com/cilium/cilium/pkg/node" "github.com/cilium/cilium/pkg/option" wgtypes "github.com/cilium/cilium/pkg/wireguard/types" ) @@ -67,9 +65,6 @@ import ( // HeaderfileWriter is a wrapper type which implements datapath.ConfigWriter. // It manages writing of configuration of datapath program headerfiles. type HeaderfileWriter struct { - db *statedb.DB - devices statedb.Table[*tables.Device] - nodeAddrs statedb.Table[tables.NodeAddress] log logrus.FieldLogger nodeMap nodemap.MapV2 nodeAddressing datapath.NodeAddressing @@ -87,9 +82,6 @@ func NewHeaderfileWriter(p WriterParams) (datapath.ConfigWriter, error) { } return &HeaderfileWriter{ nodeMap: p.NodeMap, - db: p.DB, - devices: p.Devices, - nodeAddrs: p.NodeAddresses, nodeAddressing: p.NodeAddressing, nodeExtraDefines: merged, nodeExtraDefineFns: p.NodeExtraDefineFns, @@ -104,27 +96,20 @@ func writeIncludes(w io.Writer) (int, error) { // WriteNodeConfig writes the local node configuration to the specified writer. func (h *HeaderfileWriter) WriteNodeConfig(w io.Writer, cfg *datapath.LocalNodeConfiguration) error { - txn := h.db.ReadTxn() - extraMacrosMap := make(dpdef.Map) cDefinesMap := make(dpdef.Map) - var nativeDevices []*tables.Device - if h.db != nil && h.devices != nil { - txn := h.db.ReadTxn() - nativeDevices, _ = tables.SelectedDevices(h.devices, txn) - } + nativeDevices := cfg.Devices fw := bufio.NewWriter(w) writeIncludes(w) - routerIP := node.GetIPv6Router() - hostIP := node.GetIPv6() + routerIP := cfg.CiliumInternalIPv6 + hostIP := cfg.NodeIPv6 var ipv4NodePortAddrs, ipv6NodePortAddrs []netip.Addr - iter, _ := h.nodeAddrs.All(txn) - for addr, _, ok := iter.Next(); ok; addr, _, ok = iter.Next() { + for _, addr := range cfg.NodeAddresses { if !addr.NodePort { continue } @@ -132,25 +117,24 @@ func (h *HeaderfileWriter) WriteNodeConfig(w io.Writer, cfg *datapath.LocalNodeC ipv4NodePortAddrs = append(ipv4NodePortAddrs, addr.Addr) } else { ipv6NodePortAddrs = append(ipv6NodePortAddrs, addr.Addr) - } } fmt.Fprintf(fw, "/*\n") if option.Config.EnableIPv6 { - fmt.Fprintf(fw, " cilium.v6.external.str %s\n", node.GetIPv6().String()) - fmt.Fprintf(fw, " cilium.v6.internal.str %s\n", node.GetIPv6Router().String()) + fmt.Fprintf(fw, " cilium.v6.external.str %s\n", cfg.NodeIPv6.String()) + fmt.Fprintf(fw, " cilium.v6.internal.str %s\n", cfg.CiliumInternalIPv6.String()) fmt.Fprintf(fw, " cilium.v6.nodeport.str %v\n", ipv6NodePortAddrs) fmt.Fprintf(fw, "\n") } - fmt.Fprintf(fw, " cilium.v4.external.str %s\n", node.GetIPv4().String()) - fmt.Fprintf(fw, " cilium.v4.internal.str %s\n", node.GetInternalIPv4Router().String()) + fmt.Fprintf(fw, " cilium.v4.external.str %s\n", cfg.NodeIPv4.String()) + fmt.Fprintf(fw, " cilium.v4.internal.str %s\n", cfg.CiliumInternalIPv4.String()) fmt.Fprintf(fw, " cilium.v4.nodeport.str %v\n", ipv4NodePortAddrs) fmt.Fprintf(fw, "\n") if option.Config.EnableIPv6 { - fw.WriteString(dumpRaw(defaults.RestoreV6Addr, node.GetIPv6Router())) + fw.WriteString(dumpRaw(defaults.RestoreV6Addr, cfg.CiliumInternalIPv6)) } - fw.WriteString(dumpRaw(defaults.RestoreV4Addr, node.GetInternalIPv4Router())) + fw.WriteString(dumpRaw(defaults.RestoreV4Addr, cfg.CiliumInternalIPv4)) fmt.Fprintf(fw, " */\n\n") cDefinesMap["KERNEL_HZ"] = fmt.Sprintf("%d", option.Config.KernelHz) @@ -161,9 +145,9 @@ func (h *HeaderfileWriter) WriteNodeConfig(w io.Writer, cfg *datapath.LocalNodeC } if option.Config.EnableIPv4 { - ipv4GW := node.GetInternalIPv4Router() - loopbackIPv4 := node.GetIPv4Loopback() - ipv4Range := node.GetIPv4AllocRange() + ipv4GW := cfg.CiliumInternalIPv4 + loopbackIPv4 := cfg.LoopbackIPv4 + ipv4Range := cfg.AllocCIDRIPv4 cDefinesMap["IPV4_GATEWAY"] = fmt.Sprintf("%#x", byteorder.NetIPv4ToHost32(ipv4GW)) cDefinesMap["IPV4_LOOPBACK"] = fmt.Sprintf("%#x", byteorder.NetIPv4ToHost32(loopbackIPv4)) cDefinesMap["IPV4_MASK"] = fmt.Sprintf("%#x", byteorder.NetIPv4ToHost32(net.IP(ipv4Range.Mask))) @@ -285,7 +269,7 @@ func (h *HeaderfileWriter) WriteNodeConfig(w io.Writer, cfg *datapath.LocalNodeC } cDefinesMap["TRACE_PAYLOAD_LEN"] = fmt.Sprintf("%dULL", option.Config.TracePayloadlen) - cDefinesMap["MTU"] = fmt.Sprintf("%d", cfg.MtuConfig.GetDeviceMTU()) + cDefinesMap["MTU"] = fmt.Sprintf("%d", cfg.DeviceMTU) if option.Config.EnableIPv4 { cDefinesMap["ENABLE_IPV4"] = "1" @@ -344,11 +328,11 @@ func (h *HeaderfileWriter) WriteNodeConfig(w io.Writer, cfg *datapath.LocalNodeC cDefinesMap["STRICT_IPV4_NET"] = fmt.Sprintf("%#x", byteorder.NetIPAddrToHost32(option.Config.EncryptionStrictModeCIDR.Addr())) cDefinesMap["STRICT_IPV4_NET_SIZE"] = fmt.Sprintf("%d", option.Config.EncryptionStrictModeCIDR.Bits()) - cDefinesMap["IPV4_ENCRYPT_IFACE"] = fmt.Sprintf("%#x", byteorder.NetIPv4ToHost32(node.GetIPv4())) + cDefinesMap["IPV4_ENCRYPT_IFACE"] = fmt.Sprintf("%#x", byteorder.NetIPv4ToHost32(cfg.NodeIPv4)) - ipv4Interface, ok := netip.AddrFromSlice(node.GetIPv4().To4()) + ipv4Interface, ok := netip.AddrFromSlice(cfg.NodeIPv4.To4()) if !ok { - return fmt.Errorf("unable to parse node IPv4 address %s", node.GetIPv4()) + return fmt.Errorf("unable to parse node IPv4 address %s", cfg.NodeIPv4) } if option.Config.EncryptionStrictModeCIDR.Contains(ipv4Interface) { @@ -615,7 +599,7 @@ func (h *HeaderfileWriter) WriteNodeConfig(w io.Writer, cfg *datapath.LocalNodeC } if option.Config.EnableIPSec { - nodeAddress := node.GetIPv4() + nodeAddress := cfg.NodeIPv4 if nodeAddress == nil { return errors.New("external IPv4 node address is required when IPSec is enabled, but none found") } @@ -1053,30 +1037,23 @@ func (h *HeaderfileWriter) writeStaticData(devices []string, fw io.Writer, e dat } // WriteEndpointConfig writes the BPF configuration for the endpoint to a writer. -func (h *HeaderfileWriter) WriteEndpointConfig(w io.Writer, e datapath.EndpointConfiguration) error { +func (h *HeaderfileWriter) WriteEndpointConfig(w io.Writer, cfg *datapath.LocalNodeConfiguration, e datapath.EndpointConfiguration) error { fw := bufio.NewWriter(w) - var ( - nativeDevices []*tables.Device - deviceNames []string - ) - if h.db != nil && h.devices != nil { - nativeDevices, _ = tables.SelectedDevices(h.devices, h.db.ReadTxn()) - deviceNames = tables.DeviceNames(nativeDevices) - } + deviceNames := cfg.DeviceNames() // Add cilium_wg0 if necessary. if option.Config.NeedBPFHostOnWireGuardDevice() { - deviceNames = append(deviceNames, wgtypes.IfaceName) + deviceNames = append(slices.Clone(deviceNames), wgtypes.IfaceName) } writeIncludes(w) h.writeStaticData(deviceNames, fw, e) - return h.writeTemplateConfig(fw, nativeDevices, e) + return h.writeTemplateConfig(fw, deviceNames, cfg.HostEndpointID, e) } -func (h *HeaderfileWriter) writeTemplateConfig(fw *bufio.Writer, devices []*tables.Device, e datapath.EndpointConfiguration) error { +func (h *HeaderfileWriter) writeTemplateConfig(fw *bufio.Writer, devices []string, hostEndpointID uint64, e datapath.EndpointConfiguration) error { if e.RequireEgressProg() { fmt.Fprintf(fw, "#define USE_BPF_PROG_FOR_INGRESS_POLICY 1\n") } @@ -1107,7 +1084,7 @@ func (h *HeaderfileWriter) writeTemplateConfig(fw *bufio.Writer, devices []*tabl } } - fmt.Fprintf(fw, "#define HOST_EP_ID %d\n", uint32(node.GetEndpointID())) + fmt.Fprintf(fw, "#define HOST_EP_ID %d\n", uint32(hostEndpointID)) if e.RequireARPPassthrough() { fmt.Fprint(fw, "#define ENABLE_ARP_PASSTHROUGH 1\n") @@ -1130,11 +1107,7 @@ func (h *HeaderfileWriter) writeTemplateConfig(fw *bufio.Writer, devices []*tabl } // WriteTemplateConfig writes the BPF configuration for the template to a writer. -func (h *HeaderfileWriter) WriteTemplateConfig(w io.Writer, e datapath.EndpointConfiguration) error { +func (h *HeaderfileWriter) WriteTemplateConfig(w io.Writer, cfg *datapath.LocalNodeConfiguration, e datapath.EndpointConfiguration) error { fw := bufio.NewWriter(w) - var nativeDevices []*tables.Device - if h.db != nil && h.devices != nil { - nativeDevices, _ = tables.SelectedDevices(h.devices, h.db.ReadTxn()) - } - return h.writeTemplateConfig(fw, nativeDevices, e) + return h.writeTemplateConfig(fw, cfg.DeviceNames(), cfg.HostEndpointID, e) } diff --git a/pkg/datapath/linux/config/config_test.go b/pkg/datapath/linux/config/config_test.go index 37c0863ef10c6..a30418d30a9fc 100644 --- a/pkg/datapath/linux/config/config_test.go +++ b/pkg/datapath/linux/config/config_test.go @@ -16,11 +16,11 @@ import ( "github.com/cilium/ebpf/rlimit" "github.com/cilium/hive/cell" "github.com/cilium/hive/hivetest" - "github.com/cilium/statedb" "github.com/spf13/afero" "github.com/stretchr/testify/require" "github.com/vishvananda/netlink" + "github.com/cilium/cilium/pkg/cidr" fakeTypes "github.com/cilium/cilium/pkg/datapath/fake/types" dpdef "github.com/cilium/cilium/pkg/datapath/linux/config/defines" "github.com/cilium/cilium/pkg/datapath/linux/sysctl" @@ -30,14 +30,21 @@ import ( "github.com/cilium/cilium/pkg/hive" "github.com/cilium/cilium/pkg/maps/nodemap" "github.com/cilium/cilium/pkg/maps/nodemap/fake" - "github.com/cilium/cilium/pkg/node" "github.com/cilium/cilium/pkg/option" "github.com/cilium/cilium/pkg/testutils" ) var ( dummyNodeCfg = datapath.LocalNodeConfiguration{ - MtuConfig: &fakeTypes.MTU{}, + NodeIPv4: ipv4DummyAddr.AsSlice(), + NodeIPv6: ipv6DummyAddr.AsSlice(), + CiliumInternalIPv4: ipv4DummyAddr.AsSlice(), + CiliumInternalIPv6: ipv6DummyAddr.AsSlice(), + AllocCIDRIPv4: cidr.MustParseCIDR("10.147.0.0/16"), + LoopbackIPv4: ipv4DummyAddr.AsSlice(), + Devices: []*tables.Device{}, + NodeAddresses: []tables.NodeAddress{}, + HostEndpointID: 1, } dummyDevCfg = testutils.NewTestEndpoint() dummyEPCfg = testutils.NewTestEndpoint() @@ -53,12 +60,6 @@ func setupConfigSuite(tb testing.TB) { require.NoError(tb, rlimit.RemoveMemlock(), "Failed to remove memory limits") option.Config.EnableHostLegacyRouting = true // Disable obtaining direct routing device. - node.SetTestLocalNodeStore() - node.InitDefaultPrefix("") - node.SetInternalIPv4Router(ipv4DummyAddr.AsSlice()) - node.SetIPv4Loopback(ipv4DummyAddr.AsSlice()) - - tb.Cleanup(node.UnsetTestLocalNodeStore) } type badWriter struct{} @@ -92,18 +93,10 @@ func writeConfig(t *testing.T, header string, write writeFn) { h := hive.New( provideNodemap, cell.Provide( - tables.NewNodeAddressTable, - statedb.RWTable[tables.NodeAddress].ToTable, - tables.NewDeviceTable, - statedb.RWTable[*tables.Device].ToTable, fakeTypes.NewNodeAddressing, func() sysctl.Sysctl { return sysctl.NewDirectSysctl(afero.NewOsFs(), "/proc") }, NewHeaderfileWriter, ), - cell.Invoke( - statedb.RegisterTable[*tables.Device], - statedb.RegisterTable[tables.NodeAddress], - ), cell.Invoke(func(writer_ datapath.ConfigWriter) { writer = writer_ }), @@ -112,7 +105,8 @@ func writeConfig(t *testing.T, header string, write writeFn) { tlog := hivetest.Logger(t) require.NoError(t, h.Start(tlog, context.TODO())) t.Cleanup(func() { require.Nil(t, h.Stop(tlog, context.TODO())) }) - require.True(t, test.wantErr != (write(test.output, writer) == nil)) + err := write(test.output, writer) + require.True(t, test.wantErr == (err != nil), "wantErr=%v, err=%s", test.wantErr, err) } } @@ -131,7 +125,7 @@ func TestWriteNetdevConfig(t *testing.T) { func TestWriteEndpointConfig(t *testing.T) { writeConfig(t, "endpoint", func(w io.Writer, dp datapath.ConfigWriter) error { - return dp.WriteEndpointConfig(w, &dummyEPCfg) + return dp.WriteEndpointConfig(w, &dummyNodeCfg, &dummyEPCfg) }) // Create copy of config option so that it can be restored at the end of @@ -368,32 +362,15 @@ func TestWriteNodeConfigExtraDefines(t *testing.T) { setupConfigSuite(t) var ( - db *statedb.DB - devices statedb.Table[*tables.Device] - nodeAddrs statedb.Table[tables.NodeAddress] - na datapath.NodeAddressing + na datapath.NodeAddressing ) h := hive.New( cell.Provide( fakeTypes.NewNodeAddressing, - tables.NewDeviceTable, - tables.NewNodeAddressTable, - statedb.RWTable[*tables.Device].ToTable, - statedb.RWTable[tables.NodeAddress].ToTable, - ), - cell.Invoke( - statedb.RegisterTable[*tables.Device], - statedb.RegisterTable[tables.NodeAddress], ), cell.Invoke(func( - db_ *statedb.DB, - devices_ statedb.Table[*tables.Device], - nodeAddrs_ statedb.Table[tables.NodeAddress], nodeaddressing datapath.NodeAddressing, ) { - db = db_ - devices = devices_ - nodeAddrs = nodeAddrs_ na = nodeaddressing }), ) @@ -406,9 +383,6 @@ func TestWriteNodeConfigExtraDefines(t *testing.T) { // Assert that configurations are propagated when all generated extra defines are valid cfg, err := NewHeaderfileWriter(WriterParams{ - DB: db, - Devices: devices, - NodeAddresses: nodeAddrs, NodeAddressing: na, NodeExtraDefines: nil, NodeExtraDefineFns: []dpdef.Fn{ @@ -430,9 +404,6 @@ func TestWriteNodeConfigExtraDefines(t *testing.T) { // Assert that an error is returned when one extra define function returns an error cfg, err = NewHeaderfileWriter(WriterParams{ - DB: db, - Devices: devices, - NodeAddresses: nodeAddrs, NodeAddressing: fakeTypes.NewNodeAddressing(), NodeExtraDefines: nil, NodeExtraDefineFns: []dpdef.Fn{ @@ -448,9 +419,6 @@ func TestWriteNodeConfigExtraDefines(t *testing.T) { // Assert that an error is returned when one extra define would overwrite an already existing entry cfg, err = NewHeaderfileWriter(WriterParams{ - DB: db, - Devices: devices, - NodeAddresses: nodeAddrs, NodeAddressing: fakeTypes.NewNodeAddressing(), NodeExtraDefines: nil, NodeExtraDefineFns: []dpdef.Fn{ @@ -473,19 +441,7 @@ func TestNewHeaderfileWriter(t *testing.T) { a := dpdef.Map{"A": "1"} var buffer bytes.Buffer - nodeAddrs, err := tables.NewNodeAddressTable() - require.NoError(t, err) - - devices, err := tables.NewDeviceTable() - require.NoError(t, err) - - db := statedb.New() - require.NoError(t, db.RegisterTable(devices, nodeAddrs), "RegisterTable") - - _, err = NewHeaderfileWriter(WriterParams{ - DB: db, - Devices: devices, - NodeAddresses: nodeAddrs, + _, err := NewHeaderfileWriter(WriterParams{ NodeAddressing: fakeTypes.NewNodeAddressing(), NodeExtraDefines: []dpdef.Map{a, a}, NodeExtraDefineFns: nil, @@ -496,9 +452,6 @@ func TestNewHeaderfileWriter(t *testing.T) { require.Error(t, err, "duplicate keys should be rejected") cfg, err := NewHeaderfileWriter(WriterParams{ - DB: db, - Devices: devices, - NodeAddresses: nodeAddrs, NodeAddressing: fakeTypes.NewNodeAddressing(), NodeExtraDefines: []dpdef.Map{a}, NodeExtraDefineFns: nil, diff --git a/pkg/datapath/linux/fuzz_test.go b/pkg/datapath/linux/fuzz_test.go index a59413a85aec0..a175fa52fc268 100644 --- a/pkg/datapath/linux/fuzz_test.go +++ b/pkg/datapath/linux/fuzz_test.go @@ -9,7 +9,6 @@ import ( fuzz "github.com/AdaLogics/go-fuzz-headers" - fakeTypes "github.com/cilium/cilium/pkg/datapath/fake/types" nodeTypes "github.com/cilium/cilium/pkg/node/types" ) @@ -22,8 +21,7 @@ func FuzzNodeHandler(f *testing.F) { t.Skip() } dpConfig := DatapathConfiguration{HostDevice: "veth0"} - fakeNodeAddressing := fakeTypes.NewNodeAddressing() - linuxNodeHandler := newNodeHandler(dpConfig, fakeNodeAddressing, nil, &fakeTypes.MTU{}, new(mockEnqueuer), nil, nil) + linuxNodeHandler := newNodeHandler(dpConfig, nil, new(mockEnqueuer)) if linuxNodeHandler == nil { panic("Should not be nil") } diff --git a/pkg/datapath/linux/ipsec.go b/pkg/datapath/linux/ipsec.go index f3b2b44b00c0a..ddc141a3f1031 100644 --- a/pkg/datapath/linux/ipsec.go +++ b/pkg/datapath/linux/ipsec.go @@ -16,7 +16,6 @@ import ( "github.com/cilium/cilium/pkg/datapath/linux/ipsec" "github.com/cilium/cilium/pkg/datapath/linux/linux_defaults" "github.com/cilium/cilium/pkg/datapath/linux/route" - "github.com/cilium/cilium/pkg/datapath/tables" "github.com/cilium/cilium/pkg/logging/logfields" "github.com/cilium/cilium/pkg/metrics" nodeTypes "github.com/cilium/cilium/pkg/node/types" @@ -32,7 +31,7 @@ func (n *linuxNodeHandler) getDefaultEncryptionInterface() string { if option.Config.TunnelingEnabled() { return n.datapathConfig.TunnelDevice } - devices, _ := tables.SelectedDevices(n.devices, n.db.ReadTxn()) + devices := n.nodeConfig.Devices if len(devices) > 0 { return devices[0].Name } @@ -169,12 +168,12 @@ func (n *linuxNodeHandler) enableIPsecIPv4(newNode *nodeTypes.Node, nodeID uint1 if newNode.IsLocal() { if n.subnetEncryption() { // FIXME: Remove the following four lines in Cilium v1.16 - if localCIDR := n.nodeAddressing.IPv4().AllocationCIDR(); localCIDR != nil { + if localCIDR := n.nodeConfig.AllocCIDRIPv4; localCIDR != nil { // This removes a bogus route that Cilium installed prior to v1.15 _ = route.Delete(n.createNodeIPSecInRoute(localCIDR.IPNet)) } } else { - localCIDR := n.nodeAddressing.IPv4().AllocationCIDR().IPNet + localCIDR := n.nodeConfig.AllocCIDRIPv4.IPNet errs = errors.Join(errs, n.replaceNodeIPSecInRoute(localCIDR)) } } else { @@ -195,7 +194,7 @@ func (n *linuxNodeHandler) enableIPsecIPv4(newNode *nodeTypes.Node, nodeID uint1 } remoteIP := remoteCiliumInternalIP - localCiliumInternalIP := n.nodeAddressing.IPv4().Router() + localCiliumInternalIP := n.nodeConfig.CiliumInternalIPv4 localIP := localCiliumInternalIP if n.subnetEncryption() { @@ -238,7 +237,7 @@ func (n *linuxNodeHandler) enableIPsecIPv4(newNode *nodeTypes.Node, nodeID uint1 } } } else { - localCIDR := n.nodeAddressing.IPv4().AllocationCIDR().IPNet + localCIDR := n.nodeConfig.AllocCIDRIPv4.IPNet remoteCIDR := newNode.IPv4AllocCIDR.IPNet if err := n.replaceNodeIPSecOutRoute(remoteCIDR); err != nil { errs = errors.Join(errs, fmt.Errorf("failed to replace ipsec OUT (%q): %w", remoteCIDR.IP, err)) @@ -281,12 +280,12 @@ func (n *linuxNodeHandler) enableIPsecIPv6(newNode *nodeTypes.Node, nodeID uint1 if newNode.IsLocal() { if n.subnetEncryption() { // FIXME: Remove the following four lines in Cilium v1.16 - if localCIDR := n.nodeAddressing.IPv6().AllocationCIDR(); localCIDR != nil { + if localCIDR := n.nodeConfig.AllocCIDRIPv6; localCIDR != nil { // This removes a bogus route that Cilium installed prior to v1.15 _ = route.Delete(n.createNodeIPSecInRoute(localCIDR.IPNet)) } } else { - localCIDR := n.nodeAddressing.IPv6().AllocationCIDR().IPNet + localCIDR := n.nodeConfig.AllocCIDRIPv6.IPNet errs = errors.Join(errs, n.replaceNodeIPSecInRoute(localCIDR)) } } else { @@ -307,7 +306,7 @@ func (n *linuxNodeHandler) enableIPsecIPv6(newNode *nodeTypes.Node, nodeID uint1 } remoteIP := remoteCiliumInternalIP - localCiliumInternalIP := n.nodeAddressing.IPv6().Router() + localCiliumInternalIP := n.nodeConfig.CiliumInternalIPv6 localIP := localCiliumInternalIP if n.subnetEncryption() { @@ -345,7 +344,7 @@ func (n *linuxNodeHandler) enableIPsecIPv6(newNode *nodeTypes.Node, nodeID uint1 } } } else { - localCIDR := n.nodeAddressing.IPv6().AllocationCIDR().IPNet + localCIDR := n.nodeConfig.AllocCIDRIPv6.IPNet remoteCIDR := newNode.IPv6AllocCIDR.IPNet if err := n.replaceNodeIPSecOutRoute(remoteCIDR); err != nil { errs = errors.Join(errs, fmt.Errorf("failed to replace ipsec OUT (%q): %w", remoteCIDR.IP, err)) @@ -430,7 +429,7 @@ func (n *linuxNodeHandler) createNodeIPSecOutRoute(ip *net.IPNet) route.Route { Device: n.datapathConfig.HostDevice, Prefix: *ip, Table: linux_defaults.RouteTableIPSec, - MTU: n.nodeConfig.MtuConfig.GetRoutePostEncryptMTU(), + MTU: n.nodeConfig.RoutePostEncryptMTU, Proto: linux_defaults.RTProto, } } diff --git a/pkg/datapath/linux/node.go b/pkg/datapath/linux/node.go index f77fee5e130c8..98ff5d6b3f682 100644 --- a/pkg/datapath/linux/node.go +++ b/pkg/datapath/linux/node.go @@ -20,8 +20,6 @@ import ( "github.com/vishvananda/netlink" "golang.org/x/sys/unix" - "github.com/cilium/statedb" - "github.com/cilium/cilium/pkg/cidr" cmtypes "github.com/cilium/cilium/pkg/clustermesh/types" "github.com/cilium/cilium/pkg/counter" @@ -29,7 +27,6 @@ import ( "github.com/cilium/cilium/pkg/datapath/linux/ipsec" "github.com/cilium/cilium/pkg/datapath/linux/linux_defaults" "github.com/cilium/cilium/pkg/datapath/linux/route" - "github.com/cilium/cilium/pkg/datapath/tables" dpTunnel "github.com/cilium/cilium/pkg/datapath/tunnel" datapath "github.com/cilium/cilium/pkg/datapath/types" "github.com/cilium/cilium/pkg/defaults" @@ -39,7 +36,6 @@ import ( "github.com/cilium/cilium/pkg/logging/logfields" "github.com/cilium/cilium/pkg/maps/nodemap" "github.com/cilium/cilium/pkg/maps/tunnel" - "github.com/cilium/cilium/pkg/mtu" "github.com/cilium/cilium/pkg/node" "github.com/cilium/cilium/pkg/node/manager" nodeTypes "github.com/cilium/cilium/pkg/node/types" @@ -63,7 +59,6 @@ type linuxNodeHandler struct { mutex lock.RWMutex isInitialized bool nodeConfig datapath.LocalNodeConfiguration - nodeAddressing datapath.NodeAddressing datapathConfig DatapathConfiguration nodes map[nodeTypes.Identity]*nodeTypes.Node enableNeighDiscovery bool @@ -91,9 +86,6 @@ type linuxNodeHandler struct { prefixClusterMutatorFn func(node *nodeTypes.Node) []cmtypes.PrefixClusterOpts enableEncapsulation func(node *nodeTypes.Node) bool nodeNeighborQueue datapath.NodeNeighborEnqueuer - - db *statedb.DB - devices statedb.Table[*tables.Device] } var ( @@ -106,19 +98,15 @@ var ( // implement the implications in the Linux datapath func NewNodeHandler( tunnelConfig dpTunnel.Config, - nodeAddressing datapath.NodeAddressing, nodeMap nodemap.MapV2, - mtu mtu.MTU, nodeManager manager.NodeManager, - db *statedb.DB, - devices statedb.Table[*tables.Device], ) (datapath.NodeHandler, datapath.NodeIDHandler, datapath.NodeNeighbors) { datapathConfig := DatapathConfiguration{ HostDevice: defaults.HostDevice, TunnelDevice: tunnelConfig.DeviceName(), } - handler := newNodeHandler(datapathConfig, nodeAddressing, nodeMap, mtu, nodeManager, db, devices) + handler := newNodeHandler(datapathConfig, nodeMap, nodeManager) return handler, handler, handler } @@ -126,19 +114,12 @@ func NewNodeHandler( // implement the implications in the Linux datapath func newNodeHandler( datapathConfig DatapathConfiguration, - nodeAddressing datapath.NodeAddressing, nodeMap nodemap.MapV2, - mtu datapath.MTUConfiguration, nbq datapath.NodeNeighborEnqueuer, - db *statedb.DB, - devices statedb.Table[*tables.Device], ) *linuxNodeHandler { return &linuxNodeHandler{ - db: db, - devices: devices, - nodeAddressing: nodeAddressing, datapathConfig: datapathConfig, - nodeConfig: datapath.LocalNodeConfiguration{MtuConfig: mtu}, + nodeConfig: datapath.LocalNodeConfiguration{}, nodes: map[nodeTypes.Identity]*nodeTypes.Node{}, neighNextHopByNode4: map[nodeTypes.Identity]map[string]string{}, neighNextHopByNode6: map[nodeTypes.Identity]map[string]string{}, @@ -445,26 +426,18 @@ func (n *linuxNodeHandler) createNodeRouteSpec(prefix *cidr.CIDR, isLocalNode bo mtu int ) if prefix.IP.To4() != nil { - if n.nodeAddressing.IPv4() == nil { - return route.Route{}, fmt.Errorf("IPv4 addressing unavailable") - } - - if n.nodeAddressing.IPv4().Router() == nil { + if n.nodeConfig.CiliumInternalIPv4 == nil { return route.Route{}, fmt.Errorf("IPv4 router address unavailable") } - local = n.nodeAddressing.IPv4().Router() + local = n.nodeConfig.CiliumInternalIPv4 nexthop = &local } else { - if n.nodeAddressing.IPv6() == nil { - return route.Route{}, fmt.Errorf("IPv6 addressing unavailable") - } - - if n.nodeAddressing.IPv6().Router() == nil { + if n.nodeConfig.CiliumInternalIPv6 == nil { return route.Route{}, fmt.Errorf("IPv6 router address unavailable") } - if n.nodeAddressing.IPv6().PrimaryExternal() == nil { + if n.nodeConfig.NodeIPv6 == nil { return route.Route{}, fmt.Errorf("external IPv6 address unavailable") } @@ -472,11 +445,11 @@ func (n *linuxNodeHandler) createNodeRouteSpec(prefix *cidr.CIDR, isLocalNode bo // with "Error: Gateway can not be a local address". Instead, we have to remove "via" // as "ip r a $cidr dev cilium_host" to make it work. nexthop = nil - local = n.nodeAddressing.IPv6().Router() + local = n.nodeConfig.CiliumInternalIPv6 } if !isLocalNode { - mtu = n.nodeConfig.MtuConfig.GetRouteMTU() + mtu = n.nodeConfig.RouteMTU } // The default routing table accounts for encryption overhead for encrypt-node traffic @@ -1223,8 +1196,7 @@ func (n *linuxNodeHandler) NodeConfigurationChanged(newConfig datapath.LocalNode return fmt.Errorf("direct routing device is required, but not defined") } - nativeDevices, _ := tables.SelectedDevices(n.devices, n.db.ReadTxn()) - devices := tables.DeviceNames(nativeDevices) + devices := n.nodeConfig.DeviceNames() targetDevices := make([]string, 0, len(devices)+1) targetDevices = append(targetDevices, option.Config.DirectRoutingDevice) diff --git a/pkg/datapath/linux/node_linux_test.go b/pkg/datapath/linux/node_linux_test.go index 9b2e1887fb264..12a00b711dcd9 100644 --- a/pkg/datapath/linux/node_linux_test.go +++ b/pkg/datapath/linux/node_linux_test.go @@ -10,14 +10,12 @@ import ( "fmt" "net" "runtime" + "slices" "sync" "testing" "time" "github.com/cilium/ebpf/rlimit" - "github.com/cilium/hive/cell" - "github.com/cilium/hive/hivetest" - "github.com/cilium/statedb" "github.com/spf13/afero" "github.com/stretchr/testify/require" "github.com/vishvananda/netlink" @@ -30,7 +28,6 @@ import ( "github.com/cilium/cilium/pkg/datapath/linux/sysctl" "github.com/cilium/cilium/pkg/datapath/tables" datapath "github.com/cilium/cilium/pkg/datapath/types" - "github.com/cilium/cilium/pkg/hive" nodemapfake "github.com/cilium/cilium/pkg/maps/nodemap/fake" "github.com/cilium/cilium/pkg/maps/tunnel" "github.com/cilium/cilium/pkg/mtu" @@ -43,11 +40,14 @@ import ( ) type linuxPrivilegedBaseTestSuite struct { - sysctl sysctl.Sysctl - nodeAddressing datapath.NodeAddressing - mtuConfig mtu.Configuration - enableIPv4 bool - enableIPv6 bool + sysctl sysctl.Sysctl + mtuConfig mtu.Configuration + enableIPv4 bool + enableIPv6 bool + + // nodeConfigTemplate is the partially filled template for local node configuration. + // copy it, don't mutate it. + nodeConfigTemplate datapath.LocalNodeConfiguration } type linuxPrivilegedIPv6OnlyTestSuite struct { @@ -95,7 +95,6 @@ func setupLinuxPrivilegedBaseTestSuite(tb testing.TB, addressing datapath.NodeAd s.sysctl = sysctl.NewDirectSysctl(afero.NewOsFs(), "/proc") rlimit.RemoveMemlock() - s.nodeAddressing = addressing s.mtuConfig = mtu.NewConfiguration(0, false, false, false, false, 1500, nil) s.enableIPv6 = enableIPv6 s.enableIPv4 = enableIPv4 @@ -107,24 +106,39 @@ func setupLinuxPrivilegedBaseTestSuite(tb testing.TB, addressing datapath.NodeAd ips := make([]net.IP, 0) if enableIPv6 { - ips = append(ips, s.nodeAddressing.IPv6().PrimaryExternal()) + ips = append(ips, addressing.IPv6().PrimaryExternal()) } if enableIPv4 { - ips = append(ips, s.nodeAddressing.IPv4().PrimaryExternal()) + ips = append(ips, addressing.IPv4().PrimaryExternal()) } - err := setupDummyDevice(dummyExternalDeviceName, ips...) + devExt, err := setupDummyDevice(dummyExternalDeviceName, ips...) require.NoError(tb, err) ips = []net.IP{} if enableIPv4 { - ips = append(ips, s.nodeAddressing.IPv4().Router()) + ips = append(ips, addressing.IPv4().Router()) } if enableIPv6 { - ips = append(ips, s.nodeAddressing.IPv6().Router()) + ips = append(ips, addressing.IPv6().Router()) } - err = setupDummyDevice(dummyHostDeviceName, ips...) + devHost, err := setupDummyDevice(dummyHostDeviceName, ips...) require.NoError(tb, err) + s.nodeConfigTemplate = datapath.LocalNodeConfiguration{ + Devices: []*tables.Device{devExt, devHost}, + NodeIPv4: addressing.IPv4().PrimaryExternal(), + NodeIPv6: addressing.IPv6().PrimaryExternal(), + CiliumInternalIPv4: addressing.IPv4().Router(), + CiliumInternalIPv6: addressing.IPv6().Router(), + AllocCIDRIPv4: addressing.IPv4().AllocationCIDR(), + AllocCIDRIPv6: addressing.IPv6().AllocationCIDR(), + EnableIPv4: s.enableIPv4, + EnableIPv6: s.enableIPv6, + DeviceMTU: s.mtuConfig.GetDeviceMTU(), + RouteMTU: s.mtuConfig.GetRouteMTU(), + RoutePostEncryptMTU: s.mtuConfig.GetRoutePostEncryptMTU(), + } + tunnel.SetTunnelMap(tunnel.NewTunnelMap("test_cilium_tunnel_map")) err = tunnel.TunnelMap().OpenOrCreate() require.NoError(tb, err) @@ -185,19 +199,19 @@ func tearDownTest(tb testing.TB) { require.NoError(tb, err) } -func setupDummyDevice(name string, ips ...net.IP) error { +func setupDummyDevice(name string, ips ...net.IP) (*tables.Device, error) { dummy := &netlink.Dummy{ LinkAttrs: netlink.LinkAttrs{ Name: name, }, } if err := netlink.LinkAdd(dummy); err != nil { - return err + return nil, err } if err := netlink.LinkSetUp(dummy); err != nil { removeDevice(name) - return err + return nil, err } for _, ip := range ips { @@ -211,11 +225,22 @@ func setupDummyDevice(name string, ips ...net.IP) error { addr := &netlink.Addr{IPNet: ipnet} if err := netlink.AddrAdd(dummy, addr); err != nil { removeDevice(name) - return err + return nil, err } } - return nil + link, err := netlink.LinkByName(name) + if err != nil { + return nil, err + } + return &tables.Device{ + Index: link.Attrs().Index, + MTU: link.Attrs().MTU, + Name: name, + HardwareAddr: tables.HardwareAddr(link.Attrs().HardwareAddr), + Type: "dummy", + Selected: true, + }, nil } func removeDevice(name string) { @@ -277,23 +302,11 @@ func (s *linuxPrivilegedBaseTestSuite) TestUpdateNodeRoute(t *testing.T) { require.NotNil(t, ip6CIDR) var linuxNodeHandler *linuxNodeHandler - h := hive.New( - DevicesControllerCell, - cell.Invoke(func(db *statedb.DB, devices statedb.Table[*tables.Device]) { - dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} - linuxNodeHandler = newNodeHandler(dpConfig, s.nodeAddressing, nodemapfake.NewFakeNodeMapV2(), &s.mtuConfig, new(mockEnqueuer), db, devices) - }), - ) - tlog := hivetest.Logger(t) - require.Nil(t, h.Start(tlog, context.TODO())) - defer func() { require.Nil(t, h.Stop(tlog, context.TODO())) }() + dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} + linuxNodeHandler = newNodeHandler(dpConfig, nodemapfake.NewFakeNodeMapV2(), new(mockEnqueuer)) require.NotNil(t, linuxNodeHandler) - nodeConfig := datapath.LocalNodeConfiguration{ - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - MtuConfig: &s.mtuConfig, - } + nodeConfig := s.nodeConfigTemplate err := linuxNodeHandler.NodeConfigurationChanged(nodeConfig) require.NoError(t, err) @@ -337,26 +350,12 @@ func (s *linuxPrivilegedBaseTestSuite) TestAuxiliaryPrefixes(t *testing.T) { net1 := cidr.MustParseCIDR("30.30.0.0/24") net2 := cidr.MustParseCIDR("cafe:f00d::/112") - var linuxNodeHandler *linuxNodeHandler - h := hive.New( - DevicesControllerCell, - cell.Invoke(func(db *statedb.DB, devices statedb.Table[*tables.Device]) { - dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} - linuxNodeHandler = newNodeHandler(dpConfig, s.nodeAddressing, nodemapfake.NewFakeNodeMapV2(), &s.mtuConfig, new(mockEnqueuer), db, devices) - }), - ) - - tlog := hivetest.Logger(t) - require.Nil(t, h.Start(tlog, context.TODO())) - defer func() { require.Nil(t, h.Stop(tlog, context.TODO())) }() + dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} + linuxNodeHandler := newNodeHandler(dpConfig, nodemapfake.NewFakeNodeMapV2(), new(mockEnqueuer)) require.NotNil(t, linuxNodeHandler) - nodeConfig := datapath.LocalNodeConfiguration{ - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - AuxiliaryPrefixes: []*cidr.CIDR{net1, net2}, - MtuConfig: &s.mtuConfig, - } + nodeConfig := s.nodeConfigTemplate + nodeConfig.AuxiliaryPrefixes = []*cidr.CIDR{net1, net2} err := linuxNodeHandler.NodeConfigurationChanged(nodeConfig) require.NoError(t, err) @@ -374,12 +373,8 @@ func (s *linuxPrivilegedBaseTestSuite) TestAuxiliaryPrefixes(t *testing.T) { } // remove aux prefix net2 - err = linuxNodeHandler.NodeConfigurationChanged(datapath.LocalNodeConfiguration{ - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - AuxiliaryPrefixes: []*cidr.CIDR{net1}, - MtuConfig: &s.mtuConfig, - }) + nodeConfig.AuxiliaryPrefixes = []*cidr.CIDR{net1} + err = linuxNodeHandler.NodeConfigurationChanged(nodeConfig) require.NoError(t, err) if s.enableIPv4 { @@ -395,12 +390,8 @@ func (s *linuxPrivilegedBaseTestSuite) TestAuxiliaryPrefixes(t *testing.T) { } // remove aux prefix net1, re-add net2 - err = linuxNodeHandler.NodeConfigurationChanged(datapath.LocalNodeConfiguration{ - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - AuxiliaryPrefixes: []*cidr.CIDR{net2}, - MtuConfig: &s.mtuConfig, - }) + nodeConfig.AuxiliaryPrefixes = []*cidr.CIDR{net2} + err = linuxNodeHandler.NodeConfigurationChanged(nodeConfig) require.NoError(t, err) if s.enableIPv4 { @@ -433,28 +424,13 @@ func (s *linuxPrivilegedBaseTestSuite) commonNodeUpdateEncapsulation(t *testing. externalNodeIP1 := net.ParseIP("4.4.4.4") externalNodeIP2 := net.ParseIP("8.8.8.8") - var linuxNodeHandler *linuxNodeHandler - h := hive.New( - DevicesControllerCell, - cell.Invoke(func(db *statedb.DB, devices statedb.Table[*tables.Device]) { - dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} - linuxNodeHandler = newNodeHandler(dpConfig, s.nodeAddressing, nodemapfake.NewFakeNodeMapV2(), &s.mtuConfig, new(mockEnqueuer), db, devices) - }), - ) - - tlog := hivetest.Logger(t) - require.Nil(t, h.Start(tlog, context.TODO())) - defer func() { require.Nil(t, h.Stop(tlog, context.TODO())) }() + dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} + linuxNodeHandler := newNodeHandler(dpConfig, nodemapfake.NewFakeNodeMapV2(), new(mockEnqueuer)) require.NotNil(t, linuxNodeHandler) linuxNodeHandler.OverrideEnableEncapsulation(override) - nodeConfig := datapath.LocalNodeConfiguration{ - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - EnableEncapsulation: encap, - MtuConfig: &s.mtuConfig, - } - + nodeConfig := s.nodeConfigTemplate + nodeConfig.EnableEncapsulation = encap err := linuxNodeHandler.NodeConfigurationChanged(nodeConfig) require.NoError(t, err) @@ -714,25 +690,11 @@ func (s *linuxPrivilegedBaseTestSuite) TestNodeUpdateIDs(t *testing.T) { nodeMap := nodemapfake.NewFakeNodeMapV2() - var linuxNodeHandler *linuxNodeHandler - h := hive.New( - DevicesControllerCell, - cell.Invoke(func(db *statedb.DB, devices statedb.Table[*tables.Device]) { - dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} - linuxNodeHandler = newNodeHandler(dpConfig, s.nodeAddressing, nodeMap, &s.mtuConfig, new(mockEnqueuer), db, devices) - }), - ) - - tlog := hivetest.Logger(t) - require.Nil(t, h.Start(tlog, context.TODO())) - defer func() { require.Nil(t, h.Stop(tlog, context.TODO())) }() - require.NotNil(t, linuxNodeHandler) + dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} + linuxNodeHandler := newNodeHandler(dpConfig, nodeMap, new(mockEnqueuer)) - err := linuxNodeHandler.NodeConfigurationChanged(datapath.LocalNodeConfiguration{ - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - MtuConfig: &s.mtuConfig, - }) + nodeConfig := s.nodeConfigTemplate + err := linuxNodeHandler.NodeConfigurationChanged(nodeConfig) require.NoError(t, err) // New node receives a node ID. @@ -825,12 +787,8 @@ func (s *linuxPrivilegedBaseTestSuite) TestNodeUpdateIDs(t *testing.T) { func (s *linuxPrivilegedBaseTestSuite) TestNodeChurnXFRMLeaks(t *testing.T) { // Cover the XFRM configuration for IPAM modes cluster-pool, kubernetes, etc. - config := datapath.LocalNodeConfiguration{ - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - EnableIPSec: true, - MtuConfig: &s.mtuConfig, - } + config := s.nodeConfigTemplate + config.EnableIPSec = true s.testNodeChurnXFRMLeaksWithConfig(t, config) } @@ -842,18 +800,15 @@ func TestNodeChurnXFRMLeaks(t *testing.T) { externalNodeDevice := "ipsec_interface" // Cover the XFRM configuration for IPAM modes cluster-pool, kubernetes, etc. - config := datapath.LocalNodeConfiguration{ - EnableIPv4: s.enableIPv4, - EnableIPSec: true, - MtuConfig: &s.mtuConfig, - } + config := s.nodeConfigTemplate + config.EnableIPSec = true s.testNodeChurnXFRMLeaksWithConfig(t, config) // In the case of subnet encryption (tested below), the IPsec logic // retrieves the IP address of the encryption interface directly so we need // a dummy interface. removeDevice(externalNodeDevice) - err := setupDummyDevice(externalNodeDevice, net.ParseIP("1.1.1.1"), net.ParseIP("face::1")) + _, err := setupDummyDevice(externalNodeDevice, net.ParseIP("1.1.1.1"), net.ParseIP("face::1")) require.NoError(t, err) defer removeDevice(externalNodeDevice) option.Config.EncryptInterface = []string{externalNodeDevice} @@ -876,19 +831,8 @@ func (s *linuxPrivilegedBaseTestSuite) testNodeChurnXFRMLeaksWithConfig(t *testi _, _, err := ipsec.LoadIPSecKeys(keys) require.NoError(t, err) - var linuxNodeHandler *linuxNodeHandler - h := hive.New( - DevicesControllerCell, - cell.Invoke(func(db *statedb.DB, devices statedb.Table[*tables.Device]) { - dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} - linuxNodeHandler = newNodeHandler(dpConfig, s.nodeAddressing, nodemapfake.NewFakeNodeMapV2(), &s.mtuConfig, new(mockEnqueuer), db, devices) - }), - ) - - tlog := hivetest.Logger(t) - require.Nil(t, h.Start(tlog, context.TODO())) - defer func() { require.Nil(t, h.Stop(tlog, context.TODO())) }() - require.NotNil(t, linuxNodeHandler) + dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} + linuxNodeHandler := newNodeHandler(dpConfig, nodemapfake.NewFakeNodeMapV2(), new(mockEnqueuer)) err = linuxNodeHandler.NodeConfigurationChanged(config) require.NoError(t, err) @@ -967,35 +911,23 @@ func (s *linuxPrivilegedBaseTestSuite) TestNodeUpdateDirectRouting(t *testing.T) externalNode1Device := "dummy_node1" removeDevice(externalNode1Device) - err := setupDummyDevice(externalNode1Device, externalNode1IP4v1, net.ParseIP("face::1")) + dev1, err := setupDummyDevice(externalNode1Device, externalNode1IP4v1, net.ParseIP("face::1")) require.NoError(t, err) defer removeDevice(externalNode1Device) externalNode2Device := "dummy_node2" removeDevice(externalNode2Device) - err = setupDummyDevice(externalNode2Device, externalNode1IP4v2, net.ParseIP("face::2")) + dev2, err := setupDummyDevice(externalNode2Device, externalNode1IP4v2, net.ParseIP("face::2")) require.NoError(t, err) defer removeDevice(externalNode2Device) - var linuxNodeHandler *linuxNodeHandler - h := hive.New( - DevicesControllerCell, - cell.Invoke(func(db *statedb.DB, devices statedb.Table[*tables.Device]) { - dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} - linuxNodeHandler = newNodeHandler(dpConfig, s.nodeAddressing, nodemapfake.NewFakeNodeMapV2(), &s.mtuConfig, new(mockEnqueuer), db, devices) - }), - ) - tlog := hivetest.Logger(t) - require.Nil(t, h.Start(tlog, context.TODO())) - defer func() { require.Nil(t, h.Stop(tlog, context.TODO())) }() + dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} + linuxNodeHandler := newNodeHandler(dpConfig, nodemapfake.NewFakeNodeMapV2(), new(mockEnqueuer)) require.NotNil(t, linuxNodeHandler) - nodeConfig := datapath.LocalNodeConfiguration{ - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - EnableAutoDirectRouting: true, - MtuConfig: &s.mtuConfig, - } + nodeConfig := s.nodeConfigTemplate + nodeConfig.Devices = append(slices.Clone(nodeConfig.Devices), dev1, dev2) + nodeConfig.EnableAutoDirectRouting = true expectedIPv4Routes := 0 if s.enableIPv4 { @@ -1203,25 +1135,12 @@ func (s *linuxPrivilegedBaseTestSuite) TestAgentRestartOptionChanges(t *testing. ip6Alloc1 := cidr.MustParseCIDR("2001:aaaa::/96") underlayIP := net.ParseIP("4.4.4.4") - var linuxNodeHandler *linuxNodeHandler - h := hive.New( - DevicesControllerCell, - cell.Invoke(func(db *statedb.DB, devices statedb.Table[*tables.Device]) { - dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} - linuxNodeHandler = newNodeHandler(dpConfig, s.nodeAddressing, nodemapfake.NewFakeNodeMapV2(), &s.mtuConfig, new(mockEnqueuer), db, devices) - }), - ) - tlog := hivetest.Logger(t) - require.Nil(t, h.Start(tlog, context.TODO())) - defer func() { require.Nil(t, h.Stop(tlog, context.TODO())) }() + dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} + linuxNodeHandler := newNodeHandler(dpConfig, nodemapfake.NewFakeNodeMapV2(), new(mockEnqueuer)) require.NotNil(t, linuxNodeHandler) - nodeConfig := datapath.LocalNodeConfiguration{ - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - EnableEncapsulation: true, - MtuConfig: &s.mtuConfig, - } + nodeConfig := s.nodeConfigTemplate + nodeConfig.EnableEncapsulation = true err := linuxNodeHandler.NodeConfigurationChanged(nodeConfig) require.NoError(t, err) @@ -1255,12 +1174,9 @@ func (s *linuxPrivilegedBaseTestSuite) TestAgentRestartOptionChanges(t *testing. } // Simulate agent restart with address families disables - err = linuxNodeHandler.NodeConfigurationChanged(datapath.LocalNodeConfiguration{ - EnableIPv6: false, - EnableIPv4: false, - EnableEncapsulation: true, - MtuConfig: &s.mtuConfig, - }) + nodeConfig.EnableIPv4 = false + nodeConfig.EnableIPv6 = false + err = linuxNodeHandler.NodeConfigurationChanged(nodeConfig) require.NoError(t, err) // Simulate initial node addition @@ -1274,12 +1190,9 @@ func (s *linuxPrivilegedBaseTestSuite) TestAgentRestartOptionChanges(t *testing. require.Error(t, err) // Simulate agent restart with address families enabled again - err = linuxNodeHandler.NodeConfigurationChanged(datapath.LocalNodeConfiguration{ - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - EnableEncapsulation: true, - MtuConfig: &s.mtuConfig, - }) + nodeConfig.EnableIPv4 = true + nodeConfig.EnableIPv6 = true + err = linuxNodeHandler.NodeConfigurationChanged(nodeConfig) require.NoError(t, err) // Simulate initial node addition @@ -1324,18 +1237,8 @@ func (s *linuxPrivilegedBaseTestSuite) TestNodeValidationDirectRouting(t *testin ip4Alloc1 := cidr.MustParseCIDR("5.5.5.0/24") ip6Alloc1 := cidr.MustParseCIDR("2001:aaaa::/96") - var linuxNodeHandler *linuxNodeHandler - h := hive.New( - DevicesControllerCell, - cell.Invoke(func(db *statedb.DB, devices statedb.Table[*tables.Device]) { - dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} - linuxNodeHandler = newNodeHandler(dpConfig, s.nodeAddressing, nodemapfake.NewFakeNodeMapV2(), &s.mtuConfig, new(mockEnqueuer), db, devices) - }), - ) - tlog := hivetest.Logger(t) - require.Nil(t, h.Start(tlog, context.TODO())) - defer func() { require.Nil(t, h.Stop(tlog, context.TODO())) }() - require.NotNil(t, linuxNodeHandler) + dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} + linuxNodeHandler := newNodeHandler(dpConfig, nodemapfake.NewFakeNodeMapV2(), new(mockEnqueuer)) if s.enableIPv4 { insertFakeRoute(t, linuxNodeHandler, ip4Alloc1) @@ -1345,12 +1248,9 @@ func (s *linuxPrivilegedBaseTestSuite) TestNodeValidationDirectRouting(t *testin insertFakeRoute(t, linuxNodeHandler, ip6Alloc1) } - err := linuxNodeHandler.NodeConfigurationChanged(datapath.LocalNodeConfiguration{ - EnableEncapsulation: false, - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - MtuConfig: &s.mtuConfig, - }) + nodeConfig := s.nodeConfigTemplate + nodeConfig.EnableEncapsulation = false + err := linuxNodeHandler.NodeConfigurationChanged(nodeConfig) require.NoError(t, err) nodev1 := nodeTypes.Node{ @@ -1360,7 +1260,7 @@ func (s *linuxPrivilegedBaseTestSuite) TestNodeValidationDirectRouting(t *testin if s.enableIPv4 { nodev1.IPAddresses = append(nodev1.IPAddresses, nodeTypes.Address{ - IP: s.nodeAddressing.IPv4().PrimaryExternal(), + IP: nodeConfig.NodeIPv4, Type: nodeaddressing.NodeInternalIP, }) nodev1.IPv4AllocCIDR = ip4Alloc1 @@ -1368,7 +1268,7 @@ func (s *linuxPrivilegedBaseTestSuite) TestNodeValidationDirectRouting(t *testin if s.enableIPv6 { nodev1.IPAddresses = append(nodev1.IPAddresses, nodeTypes.Address{ - IP: s.nodeAddressing.IPv6().PrimaryExternal(), + IP: nodeConfig.NodeIPv6, Type: nodeaddressing.NodeInternalIP, }) nodev1.IPv6AllocCIDR = ip6Alloc1 @@ -1494,30 +1394,14 @@ func TestArpPingHandlingIPv6(t *testing.T) { defer func() { option.Config.ARPPingRefreshPeriod = prevARPPeriod }() option.Config.ARPPingRefreshPeriod = time.Duration(1 * time.Nanosecond) - var linuxNodeHandler *linuxNodeHandler mq := new(mockEnqueuer) - h := hive.New( - DevicesControllerCell, - cell.Invoke(func(db *statedb.DB, devices statedb.Table[*tables.Device]) { - dpConfig := DatapathConfiguration{HostDevice: "veth0"} - linuxNodeHandler = newNodeHandler(dpConfig, s.nodeAddressing, nodemapfake.NewFakeNodeMapV2(), &s.mtuConfig, mq, db, devices) - mq.nh = linuxNodeHandler - }), - ) - hive.AddConfigOverride(h, func(c *DevicesConfig) { - c.Devices = []string{"veth0"} - }) + dpConfig := DatapathConfiguration{HostDevice: "veth0"} + linuxNodeHandler := newNodeHandler(dpConfig, nodemapfake.NewFakeNodeMapV2(), mq) + mq.nh = linuxNodeHandler - tlog := hivetest.Logger(t) - require.Nil(t, h.Start(tlog, context.TODO())) - defer func() { require.Nil(t, h.Stop(tlog, context.TODO())) }() - require.NotNil(t, linuxNodeHandler) - - err = linuxNodeHandler.NodeConfigurationChanged(datapath.LocalNodeConfiguration{ - EnableEncapsulation: false, - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - }) + nodeConfig := s.nodeConfigTemplate + nodeConfig.EnableEncapsulation = false + err = linuxNodeHandler.NodeConfigurationChanged(nodeConfig) require.NoError(t, err) // wait waits for neigh entry update or waits for removal if waitForDelete=true @@ -2038,6 +1922,12 @@ func TestArpPingHandlingIPv6(t *testing.T) { linuxNodeHandler.NodeCleanNeighborsLink(veth0, false) } +func getDevice(tb testing.TB, name string) *tables.Device { + link, err := netlink.LinkByName(name) + require.NoError(tb, err, "LinkByName") + return &tables.Device{Index: link.Attrs().Index, Name: name, Selected: true} +} + func TestArpPingHandlingForMultiDeviceIPv6(t *testing.T) { s := setupLinuxPrivilegedIPv6OnlyTestSuite(t) runtime.LockOSThread() @@ -2256,29 +2146,18 @@ func TestArpPingHandlingForMultiDeviceIPv6(t *testing.T) { option.Config.ARPPingRefreshPeriod = 1 * time.Nanosecond mq := new(mockEnqueuer) - var linuxNodeHandler *linuxNodeHandler - h := hive.New( - DevicesControllerCell, - cell.Invoke(func(db *statedb.DB, devices statedb.Table[*tables.Device]) { - dpConfig := DatapathConfiguration{HostDevice: "veth0"} - linuxNodeHandler = newNodeHandler(dpConfig, s.nodeAddressing, nodemapfake.NewFakeNodeMapV2(), &s.mtuConfig, mq, db, devices) - mq.nh = linuxNodeHandler - }), - ) - hive.AddConfigOverride(h, func(c *DevicesConfig) { - c.Devices = []string{"veth0", "veth2", "veth4"} - }) + dpConfig := DatapathConfiguration{HostDevice: "veth0"} + linuxNodeHandler := newNodeHandler(dpConfig, nodemapfake.NewFakeNodeMapV2(), mq) + mq.nh = linuxNodeHandler - tlog := hivetest.Logger(t) - require.Nil(t, h.Start(tlog, context.TODO())) - defer func() { require.Nil(t, h.Stop(tlog, context.TODO())) }() - require.NotNil(t, linuxNodeHandler) + nodeConfig := s.nodeConfigTemplate + nodeConfig.EnableEncapsulation = false + nodeConfig.Devices = append(slices.Clone(nodeConfig.Devices), + getDevice(t, "veth0"), + getDevice(t, "veth2"), + getDevice(t, "veth4")) - err = linuxNodeHandler.NodeConfigurationChanged(datapath.LocalNodeConfiguration{ - EnableEncapsulation: false, - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - }) + err = linuxNodeHandler.NodeConfigurationChanged(nodeConfig) require.NoError(t, err) // wait waits for neigh entry update or waits for removal if waitForDelete=true @@ -2542,25 +2421,16 @@ func TestArpPingHandlingIPv4(t *testing.T) { option.Config.ARPPingRefreshPeriod = time.Duration(1 * time.Nanosecond) mq := new(mockEnqueuer) - var linuxNodeHandler *linuxNodeHandler - h := hive.New( - DevicesControllerCell, - cell.Invoke(func(db *statedb.DB, devices statedb.Table[*tables.Device]) { - dpConfig := DatapathConfiguration{HostDevice: "veth0"} - linuxNodeHandler = newNodeHandler(dpConfig, s.nodeAddressing, nodemapfake.NewFakeNodeMapV2(), &s.mtuConfig, mq, db, devices) - mq.nh = linuxNodeHandler - }), - ) - tlog := hivetest.Logger(t) - require.Nil(t, h.Start(tlog, context.TODO())) - defer func() { require.Nil(t, h.Stop(tlog, context.TODO())) }() - require.NotNil(t, linuxNodeHandler) + dpConfig := DatapathConfiguration{HostDevice: "veth0"} + linuxNodeHandler := newNodeHandler(dpConfig, nodemapfake.NewFakeNodeMapV2(), mq) + mq.nh = linuxNodeHandler - err = linuxNodeHandler.NodeConfigurationChanged(datapath.LocalNodeConfiguration{ - EnableEncapsulation: false, - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - }) + nodeConfig := s.nodeConfigTemplate + nodeConfig.Devices = []*tables.Device{ + {Index: veth0.Attrs().Index, Name: "veth0", Selected: true}, + } + nodeConfig.EnableEncapsulation = false + err = linuxNodeHandler.NodeConfigurationChanged(nodeConfig) require.NoError(t, err) // wait waits for neigh entry update or waits for removal if waitForDelete=true @@ -3300,29 +3170,17 @@ func TestArpPingHandlingForMultiDeviceIPv4(t *testing.T) { option.Config.ARPPingRefreshPeriod = 1 * time.Nanosecond mq := new(mockEnqueuer) - var linuxNodeHandler *linuxNodeHandler - h := hive.New( - DevicesControllerCell, - cell.Invoke(func(db *statedb.DB, devices statedb.Table[*tables.Device]) { - dpConfig := DatapathConfiguration{HostDevice: "veth0"} - linuxNodeHandler = newNodeHandler(dpConfig, s.nodeAddressing, nodemapfake.NewFakeNodeMapV2(), &s.mtuConfig, mq, db, devices) - mq.nh = linuxNodeHandler - }), - ) - hive.AddConfigOverride(h, func(c *DevicesConfig) { - c.Devices = []string{"veth0", "veth2", "veth4"} - }) - - tlog := hivetest.Logger(t) - require.Nil(t, h.Start(tlog, context.TODO())) - defer func() { require.Nil(t, h.Stop(tlog, context.TODO())) }() - require.NotNil(t, linuxNodeHandler) - - err = linuxNodeHandler.NodeConfigurationChanged(datapath.LocalNodeConfiguration{ - EnableEncapsulation: false, - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - }) + dpConfig := DatapathConfiguration{HostDevice: "veth0"} + linuxNodeHandler := newNodeHandler(dpConfig, nodemapfake.NewFakeNodeMapV2(), mq) + mq.nh = linuxNodeHandler + + nodeConfig := s.nodeConfigTemplate + nodeConfig.EnableEncapsulation = false + nodeConfig.Devices = append(slices.Clone(nodeConfig.Devices), + getDevice(t, "veth0"), + getDevice(t, "veth2"), + getDevice(t, "veth4")) + err = linuxNodeHandler.NodeConfigurationChanged(nodeConfig) require.NoError(t, err) // wait waits for neigh entry update or waits for removal if waitForDelete=true @@ -3543,18 +3401,8 @@ func (s *linuxPrivilegedBaseTestSuite) benchmarkNodeUpdate(b *testing.B, config ip6Alloc1 := cidr.MustParseCIDR("2001:aaaa::/96") ip6Alloc2 := cidr.MustParseCIDR("2001:bbbb::/96") - var linuxNodeHandler *linuxNodeHandler - h := hive.New( - DevicesControllerCell, - cell.Invoke(func(db *statedb.DB, devices statedb.Table[*tables.Device]) { - dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} - linuxNodeHandler = newNodeHandler(dpConfig, s.nodeAddressing, nodemapfake.NewFakeNodeMapV2(), &s.mtuConfig, new(mockEnqueuer), db, devices) - }), - ) - tlog := hivetest.Logger(b) - require.Nil(b, h.Start(tlog, context.TODO())) - defer func() { require.Nil(b, h.Stop(tlog, context.TODO())) }() - require.NotNil(b, linuxNodeHandler) + dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} + linuxNodeHandler := newNodeHandler(dpConfig, nodemapfake.NewFakeNodeMapV2(), new(mockEnqueuer)) err := linuxNodeHandler.NodeConfigurationChanged(config) require.NoError(b, err) @@ -3566,7 +3414,7 @@ func (s *linuxPrivilegedBaseTestSuite) benchmarkNodeUpdate(b *testing.B, config if s.enableIPv4 { nodev1.IPAddresses = append(nodev1.IPAddresses, nodeTypes.Address{ - IP: s.nodeAddressing.IPv4().PrimaryExternal(), + IP: config.NodeIPv4, Type: nodeaddressing.NodeInternalIP, }) nodev1.IPv4AllocCIDR = ip4Alloc1 @@ -3574,7 +3422,7 @@ func (s *linuxPrivilegedBaseTestSuite) benchmarkNodeUpdate(b *testing.B, config if s.enableIPv6 { nodev1.IPAddresses = append(nodev1.IPAddresses, nodeTypes.Address{ - IP: s.nodeAddressing.IPv6().PrimaryExternal(), + IP: config.NodeIPv6, Type: nodeaddressing.NodeInternalIP, }) nodev1.IPv6AllocCIDR = ip6Alloc1 @@ -3587,7 +3435,7 @@ func (s *linuxPrivilegedBaseTestSuite) benchmarkNodeUpdate(b *testing.B, config if s.enableIPv4 { nodev2.IPAddresses = append(nodev2.IPAddresses, nodeTypes.Address{ - IP: s.nodeAddressing.IPv4().PrimaryExternal(), + IP: config.NodeIPv4, Type: nodeaddressing.NodeInternalIP, }) nodev2.IPv4AllocCIDR = ip4Alloc2 @@ -3595,7 +3443,7 @@ func (s *linuxPrivilegedBaseTestSuite) benchmarkNodeUpdate(b *testing.B, config if s.enableIPv6 { nodev2.IPAddresses = append(nodev2.IPAddresses, nodeTypes.Address{ - IP: s.nodeAddressing.IPv6().PrimaryExternal(), + IP: config.NodeIPv6, Type: nodeaddressing.NodeInternalIP, }) nodev2.IPv6AllocCIDR = ip6Alloc2 @@ -3649,18 +3497,8 @@ func (s *linuxPrivilegedBaseTestSuite) benchmarkNodeUpdateNOP(b *testing.B, conf ip4Alloc1 := cidr.MustParseCIDR("5.5.5.0/24") ip6Alloc1 := cidr.MustParseCIDR("2001:aaaa::/96") - var linuxNodeHandler *linuxNodeHandler - h := hive.New( - DevicesControllerCell, - cell.Invoke(func(db *statedb.DB, devices statedb.Table[*tables.Device]) { - dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} - linuxNodeHandler = newNodeHandler(dpConfig, s.nodeAddressing, nodemapfake.NewFakeNodeMapV2(), &s.mtuConfig, new(mockEnqueuer), db, devices) - }), - ) - tlog := hivetest.Logger(b) - require.Nil(b, h.Start(tlog, context.TODO())) - defer func() { require.Nil(b, h.Stop(tlog, context.TODO())) }() - require.NotNil(b, linuxNodeHandler) + dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} + linuxNodeHandler := newNodeHandler(dpConfig, nodemapfake.NewFakeNodeMapV2(), new(mockEnqueuer)) err := linuxNodeHandler.NodeConfigurationChanged(config) require.NoError(b, err) @@ -3672,7 +3510,7 @@ func (s *linuxPrivilegedBaseTestSuite) benchmarkNodeUpdateNOP(b *testing.B, conf if s.enableIPv4 { nodev1.IPAddresses = append(nodev1.IPAddresses, nodeTypes.Address{ - IP: s.nodeAddressing.IPv4().PrimaryExternal(), + IP: config.NodeIPv4, Type: nodeaddressing.NodeInternalIP, }) nodev1.IPv4AllocCIDR = ip4Alloc1 @@ -3680,7 +3518,7 @@ func (s *linuxPrivilegedBaseTestSuite) benchmarkNodeUpdateNOP(b *testing.B, conf if s.enableIPv6 { nodev1.IPAddresses = append(nodev1.IPAddresses, nodeTypes.Address{ - IP: s.nodeAddressing.IPv6().PrimaryExternal(), + IP: config.NodeIPv6, Type: nodeaddressing.NodeInternalIP, }) nodev1.IPv6AllocCIDR = ip6Alloc1 @@ -3727,18 +3565,8 @@ func (s *linuxPrivilegedBaseTestSuite) benchmarkNodeValidateImplementation(b *te ip4Alloc1 := cidr.MustParseCIDR("5.5.5.0/24") ip6Alloc1 := cidr.MustParseCIDR("2001:aaaa::/96") - var linuxNodeHandler *linuxNodeHandler - h := hive.New( - DevicesControllerCell, - cell.Invoke(func(db *statedb.DB, devices statedb.Table[*tables.Device]) { - dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} - linuxNodeHandler = newNodeHandler(dpConfig, s.nodeAddressing, nodemapfake.NewFakeNodeMapV2(), &s.mtuConfig, new(mockEnqueuer), db, devices) - }), - ) - tlog := hivetest.Logger(b) - require.Nil(b, h.Start(tlog, context.TODO())) - defer func() { require.Nil(b, h.Stop(tlog, context.TODO())) }() - require.NotNil(b, linuxNodeHandler) + dpConfig := DatapathConfiguration{HostDevice: dummyHostDeviceName} + linuxNodeHandler := newNodeHandler(dpConfig, nodemapfake.NewFakeNodeMapV2(), new(mockEnqueuer)) err := linuxNodeHandler.NodeConfigurationChanged(config) require.NoError(b, err) @@ -3750,7 +3578,7 @@ func (s *linuxPrivilegedBaseTestSuite) benchmarkNodeValidateImplementation(b *te if s.enableIPv4 { nodev1.IPAddresses = append(nodev1.IPAddresses, nodeTypes.Address{ - IP: s.nodeAddressing.IPv4().PrimaryExternal(), + IP: config.NodeIPv4, Type: nodeaddressing.NodeInternalIP, }) nodev1.IPv4AllocCIDR = ip4Alloc1 @@ -3758,7 +3586,7 @@ func (s *linuxPrivilegedBaseTestSuite) benchmarkNodeValidateImplementation(b *te if s.enableIPv6 { nodev1.IPAddresses = append(nodev1.IPAddresses, nodeTypes.Address{ - IP: s.nodeAddressing.IPv6().PrimaryExternal(), + IP: config.NodeIPv6, Type: nodeaddressing.NodeInternalIP, }) nodev1.IPv6AllocCIDR = ip6Alloc1 @@ -3779,24 +3607,17 @@ func (s *linuxPrivilegedBaseTestSuite) benchmarkNodeValidateImplementation(b *te } func (s *linuxPrivilegedBaseTestSuite) BenchmarkNodeValidateImplementation(b *testing.B) { - s.benchmarkNodeValidateImplementation(b, datapath.LocalNodeConfiguration{ - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - }) + s.benchmarkNodeValidateImplementation(b, s.nodeConfigTemplate) } func (s *linuxPrivilegedBaseTestSuite) BenchmarkNodeValidateImplementationEncap(b *testing.B) { - s.benchmarkNodeValidateImplementation(b, datapath.LocalNodeConfiguration{ - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - EnableEncapsulation: true, - }) + config := s.nodeConfigTemplate + config.EnableEncapsulation = true + s.benchmarkNodeValidateImplementation(b, config) } func (s *linuxPrivilegedBaseTestSuite) BenchmarkNodeValidateImplementationDirectRoute(b *testing.B) { - s.benchmarkNodeValidateImplementation(b, datapath.LocalNodeConfiguration{ - EnableIPv4: s.enableIPv4, - EnableIPv6: s.enableIPv6, - EnableAutoDirectRouting: true, - }) + config := s.nodeConfigTemplate + config.EnableAutoDirectRouting = true + s.benchmarkNodeValidateImplementation(b, config) } diff --git a/pkg/datapath/linux/node_test.go b/pkg/datapath/linux/node_test.go index 1c74211c33ea0..1b44d87f83085 100644 --- a/pkg/datapath/linux/node_test.go +++ b/pkg/datapath/linux/node_test.go @@ -24,12 +24,20 @@ import ( ) var ( + fakeNodeAddressing = fakeTypes.NewNodeAddressing() + + nodeConfig = datapath.LocalNodeConfiguration{ + NodeIPv4: fakeNodeAddressing.IPv4().PrimaryExternal(), + NodeIPv6: fakeNodeAddressing.IPv6().PrimaryExternal(), + CiliumInternalIPv4: fakeNodeAddressing.IPv4().Router(), + CiliumInternalIPv6: fakeNodeAddressing.IPv6().Router(), + DeviceMTU: mtuConfig.GetDeviceMTU(), + RouteMTU: mtuConfig.GetRouteMTU(), + RoutePostEncryptMTU: mtuConfig.GetRoutePostEncryptMTU(), + } mtuConfig = mtu.NewConfiguration(0, false, false, false, false, 100, net.IP("1.1.1.1")) nh = linuxNodeHandler{ - nodeConfig: datapath.LocalNodeConfiguration{ - MtuConfig: &mtuConfig, - }, - nodeAddressing: fakeTypes.NewNodeAddressing(), + nodeConfig: nodeConfig, datapathConfig: DatapathConfiguration{ HostDevice: "host_device", }, @@ -73,9 +81,8 @@ func TestCreateNodeRoute(t *testing.T) { HostDevice: "host_device", } - fakeNodeAddressing := fakeTypes.NewNodeAddressing() - - nodeHandler := newNodeHandler(dpConfig, fakeNodeAddressing, nil, &mtuConfig, new(mockEnqueuer), nil, nil) + nodeHandler := newNodeHandler(dpConfig, nil, new(mockEnqueuer)) + nodeHandler.NodeConfigurationChanged(nodeConfig) c1 := cidr.MustParseCIDR("10.10.0.0/16") generatedRoute, err := nodeHandler.createNodeRouteSpec(c1, false) diff --git a/pkg/datapath/loader/base.go b/pkg/datapath/loader/base.go index 30f20455717f8..877076224a94b 100644 --- a/pkg/datapath/loader/base.go +++ b/pkg/datapath/loader/base.go @@ -28,7 +28,6 @@ import ( ipamOption "github.com/cilium/cilium/pkg/ipam/option" "github.com/cilium/cilium/pkg/logging/logfields" "github.com/cilium/cilium/pkg/mac" - "github.com/cilium/cilium/pkg/node" "github.com/cilium/cilium/pkg/option" "github.com/cilium/cilium/pkg/socketlb" wgTypes "github.com/cilium/cilium/pkg/wireguard/types" @@ -58,7 +57,7 @@ func (l *loader) writeNetdevHeader(dir string) error { return nil } -func (l *loader) writeNodeConfigHeader() error { +func (l *loader) writeNodeConfigHeader(cfg *datapath.LocalNodeConfiguration) error { nodeConfigPath := option.Config.GetNodeConfigPath() f, err := os.Create(nodeConfigPath) if err != nil { @@ -66,7 +65,7 @@ func (l *loader) writeNodeConfigHeader() error { } defer f.Close() - if err = l.templateCache.WriteNodeConfig(f, &l.localNodeConfig); err != nil { + if err = l.templateCache.WriteNodeConfig(f, cfg); err != nil { return fmt.Errorf("failed to write node configuration file at %s: %w", nodeConfigPath, err) } return nil @@ -306,12 +305,9 @@ func (l *loader) reinitializeXDPLocked(ctx context.Context, extraCArgs []string, // and reinsertion of the object into the kernel as well as an atomic program replacement // at the XDP hook. extraCArgs can be passed-in in order to alter BPF code defines. func (l *loader) ReinitializeXDP(ctx context.Context, extraCArgs []string) error { - // TODO: react to changes (using the currently ignored watch channel) - nativeDevices, _ := tables.SelectedDevices(l.devices, l.db.ReadTxn()) - devices := tables.DeviceNames(nativeDevices) - l.compilationLock.Lock() defer l.compilationLock.Unlock() + devices := l.nodeConfig.Load().DeviceNames() return l.reinitializeXDPLocked(ctx, extraCArgs, devices) } @@ -319,7 +315,7 @@ func (l *loader) ReinitializeXDP(ctx context.Context, extraCArgs []string) error // BPF programs, netfilter rule configuration and reserving routes in IPAM for // locally detected prefixes. It may be run upon initial Cilium startup, after // restore from a previous Cilium run, or during regular Cilium operation. -func (l *loader) Reinitialize(ctx context.Context, tunnelConfig tunnel.Config, deviceMTU int, iptMgr datapath.IptablesManager, p datapath.Proxy) error { +func (l *loader) Reinitialize(ctx context.Context, cfg datapath.LocalNodeConfiguration, tunnelConfig tunnel.Config, iptMgr datapath.IptablesManager, p datapath.Proxy) error { sysSettings := []tables.Sysctl{ {Name: "net.core.bpf_jit_enable", Val: "1", IgnoreErr: true, Warn: "Unable to ensure that BPF JIT compilation is enabled. This can be ignored when Cilium is running inside non-host network namespace (e.g. with kind or minikube)"}, {Name: "net.ipv4.conf.all.rp_filter", Val: "0", IgnoreErr: false}, @@ -332,14 +328,16 @@ func (l *loader) Reinitialize(ctx context.Context, tunnelConfig tunnel.Config, d l.compilationLock.Lock() defer l.compilationLock.Unlock() - l.init() + // Store the new LocalNodeConfiguration + l.nodeConfig.Store(&cfg) + l.initTemplateCache(&cfg) - var nodeIPv4, nodeIPv6 net.IP + var internalIPv4, internalIPv6 net.IP if option.Config.EnableIPv4 { - nodeIPv4 = node.GetInternalIPv4Router() + internalIPv4 = cfg.CiliumInternalIPv4 } if option.Config.EnableIPv6 { - nodeIPv6 = node.GetIPv6Router() + internalIPv6 = cfg.CiliumInternalIPv6 // Docker <17.05 has an issue which causes IPv6 to be disabled in the initns for all // interface (https://github.com/docker/libnetwork/issues/1720) // Enable IPv6 for now @@ -348,7 +346,7 @@ func (l *loader) Reinitialize(ctx context.Context, tunnelConfig tunnel.Config, d } // Datapath initialization - hostDev1, _, err := setupBaseDevice(l.sysctl, deviceMTU) + hostDev1, _, err := setupBaseDevice(l.sysctl, cfg.DeviceMTU) if err != nil { return fmt.Errorf("failed to setup base devices: %w", err) } @@ -365,7 +363,7 @@ func (l *loader) Reinitialize(ctx context.Context, tunnelConfig tunnel.Config, d } } - if err := setupTunnelDevice(l.sysctl, tunnelConfig.Protocol(), tunnelConfig.Port(), deviceMTU); err != nil { + if err := setupTunnelDevice(l.sysctl, tunnelConfig.Protocol(), tunnelConfig.Port(), cfg.DeviceMTU); err != nil { return fmt.Errorf("failed to setup %s tunnel device: %w", tunnelConfig.Protocol(), err) } @@ -382,19 +380,18 @@ func (l *loader) Reinitialize(ctx context.Context, tunnelConfig tunnel.Config, d } // add internal ipv4 and ipv6 addresses to cilium_host - if err := addHostDeviceAddr(hostDev1, nodeIPv4, nodeIPv6); err != nil { + if err := addHostDeviceAddr(hostDev1, internalIPv4, internalIPv6); err != nil { return fmt.Errorf("failed to add internal IP address to %s: %w", hostDev1.Attrs().Name, err) } - // TODO: react to changes (using the currently ignored watch channel) - nativeDevices, _ := tables.SelectedDevices(l.devices, l.db.ReadTxn()) - devices := tables.DeviceNames(nativeDevices) + devices := cfg.DeviceNames() + if err := cleanIngressQdisc(devices); err != nil { log.WithError(err).Warn("Unable to clean up ingress qdiscs") return err } - if err := l.writeNodeConfigHeader(); err != nil { + if err := l.writeNodeConfigHeader(&cfg); err != nil { log.WithError(err).Error("Unable to write node config header") return err } @@ -458,7 +455,7 @@ func (l *loader) Reinitialize(ctx context.Context, tunnelConfig tunnel.Config, d return err } - if err := l.nodeHandler.NodeConfigurationChanged(l.localNodeConfig); err != nil { + if err := l.nodeHandler.NodeConfigurationChanged(cfg); err != nil { return err } diff --git a/pkg/datapath/loader/cache.go b/pkg/datapath/loader/cache.go index f2eda7cf12cca..c5bded1c9450a 100644 --- a/pkg/datapath/loader/cache.go +++ b/pkg/datapath/loader/cache.go @@ -97,7 +97,7 @@ func (o *objectCache) serialize(key string) *cachedObject { // build attempts to compile and cache a datapath template object file // corresponding to the specified endpoint configuration. -func (o *objectCache) build(ctx context.Context, cfg datapath.EndpointConfiguration, stats *metrics.SpanStat, dir *directoryInfo, hash string) (string, error) { +func (o *objectCache) build(ctx context.Context, nodeCfg *datapath.LocalNodeConfiguration, cfg datapath.EndpointConfiguration, stats *metrics.SpanStat, dir *directoryInfo, hash string) (string, error) { isHost := cfg.IsHost() templatePath := filepath.Join(o.workingDirectory, defaults.TemplatesDir, hash) dir = &directoryInfo{ @@ -123,7 +123,7 @@ func (o *objectCache) build(ctx context.Context, cfg datapath.EndpointConfigurat return "", fmt.Errorf("failed to open template header for writing: %w", err) } defer f.Close() - if err = o.ConfigWriter.WriteEndpointConfig(f, cfg); err != nil { + if err = o.ConfigWriter.WriteEndpointConfig(f, nodeCfg, cfg); err != nil { return "", fmt.Errorf("failed to write template header: %w", err) } @@ -150,11 +150,11 @@ func (o *objectCache) build(ctx context.Context, cfg datapath.EndpointConfigurat // // Returns the path to the compiled template datapath object and whether the // object was compiled, or an error. -func (o *objectCache) fetchOrCompile(ctx context.Context, cfg datapath.EndpointConfiguration, dir *directoryInfo, stats *metrics.SpanStat) (file *os.File, compiled bool, err error) { +func (o *objectCache) fetchOrCompile(ctx context.Context, nodeCfg *datapath.LocalNodeConfiguration, cfg datapath.EndpointConfiguration, dir *directoryInfo, stats *metrics.SpanStat) (file *os.File, compiled bool, err error) { cfg = wrap(cfg) var hash string - hash, err = o.baseHash.sumEndpoint(o, cfg, false) + hash, err = o.baseHash.sumEndpoint(o, nodeCfg, cfg, false) if err != nil { return nil, false, err } @@ -191,7 +191,7 @@ func (o *objectCache) fetchOrCompile(ctx context.Context, cfg datapath.EndpointC stats = &metrics.SpanStat{} } - path, err := o.build(ctx, cfg, stats, dir, hash) + path, err := o.build(ctx, nodeCfg, cfg, stats, dir, hash) if err != nil { if !errors.Is(err, context.Canceled) { scopedLog.WithError(err).Error("BPF template object creation failed") diff --git a/pkg/datapath/loader/cache_test.go b/pkg/datapath/loader/cache_test.go index 5f908ba9fb984..0de5f3e94cfad 100644 --- a/pkg/datapath/loader/cache_test.go +++ b/pkg/datapath/loader/cache_test.go @@ -11,11 +11,10 @@ import ( "github.com/stretchr/testify/require" - "github.com/cilium/statedb" - + fakeTypes "github.com/cilium/cilium/pkg/datapath/fake/types" "github.com/cilium/cilium/pkg/datapath/linux/config" - "github.com/cilium/cilium/pkg/datapath/tables" "github.com/cilium/cilium/pkg/datapath/types" + fakeNodeMap "github.com/cilium/cilium/pkg/maps/nodemap/fake" "github.com/cilium/cilium/pkg/testutils" ) @@ -27,30 +26,30 @@ func TestObjectCache(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), contextTimeout) defer cancel() - cache := newObjectCache(configWriterForTest(t), nil, tmpDir) + cache := newObjectCache(configWriterForTest(t), &localNodeConfig, tmpDir) realEP := testutils.NewTestEndpoint() dir := getDirs(t) // First run should compile and generate the object. - _, isNew, err := cache.fetchOrCompile(ctx, &realEP, dir, nil) + _, isNew, err := cache.fetchOrCompile(ctx, &localNodeConfig, &realEP, dir, nil) require.NoError(t, err) require.Equal(t, isNew, true) // Same EP should not be compiled twice. - _, isNew, err = cache.fetchOrCompile(ctx, &realEP, dir, nil) + _, isNew, err = cache.fetchOrCompile(ctx, &localNodeConfig, &realEP, dir, nil) require.NoError(t, err) require.Equal(t, isNew, false) // Changing the ID should not generate a new object. realEP.Id++ - _, isNew, err = cache.fetchOrCompile(ctx, &realEP, dir, nil) + _, isNew, err = cache.fetchOrCompile(ctx, &localNodeConfig, &realEP, dir, nil) require.NoError(t, err) require.Equal(t, isNew, false) // Changing a setting on the EP should generate a new object. realEP.Opts.SetBool("foo", true) - _, isNew, err = cache.fetchOrCompile(ctx, &realEP, dir, nil) + _, isNew, err = cache.fetchOrCompile(ctx, &localNodeConfig, &realEP, dir, nil) require.NoError(t, err) require.Equal(t, isNew, true) } @@ -108,13 +107,13 @@ func TestObjectCacheParallel(t *testing.T) { t.Logf(" %s", test.description) results := make(chan buildResult, test.builds) - cache := newObjectCache(configWriterForTest(t), nil, tmpDir) + cache := newObjectCache(configWriterForTest(t), &localNodeConfig, tmpDir) for i := 0; i < test.builds; i++ { go func(i int) { ep := testutils.NewTestEndpoint() opt := fmt.Sprintf("OPT%d", i/test.divisor) ep.Opts.SetBool(opt, true) - file, isNew, err := cache.fetchOrCompile(ctx, &ep, getDirs(t), nil) + file, isNew, err := cache.fetchOrCompile(ctx, &localNodeConfig, &ep, getDirs(t), nil) path := "" if file != nil { path = file.Name() @@ -158,17 +157,10 @@ func TestObjectCacheParallel(t *testing.T) { func configWriterForTest(t testing.TB) types.ConfigWriter { t.Helper() - devices, err := tables.NewDeviceTable() - if err != nil { - t.Fatalf("failed to create device table: %v", err) - } - db := statedb.New() - if err := db.RegisterTable(devices); err != nil { - t.Fatalf("failed to register devices: %v", err) - } cfg, err := config.NewHeaderfileWriter(config.WriterParams{ - DB: db, - Devices: devices, + NodeMap: fakeNodeMap.NewFakeNodeMapV2(), + NodeAddressing: fakeTypes.NewNodeAddressing(), + Sysctl: nil, }) if err != nil { t.Fatalf("failed to create header file writer: %v", err) diff --git a/pkg/datapath/loader/hash.go b/pkg/datapath/loader/hash.go index 405d60bab6764..68bba4c828744 100644 --- a/pkg/datapath/loader/hash.go +++ b/pkg/datapath/loader/hash.go @@ -53,7 +53,7 @@ func hashDatapath(c datapath.ConfigWriter, nodeCfg *datapath.LocalNodeConfigurat _ = c.WriteNetdevConfig(d, option.Config.Opts) } if epCfg != nil { - _ = c.WriteTemplateConfig(d, epCfg) + _ = c.WriteTemplateConfig(d, nodeCfg, epCfg) } return d @@ -61,15 +61,15 @@ func hashDatapath(c datapath.ConfigWriter, nodeCfg *datapath.LocalNodeConfigurat // sumEndpoint returns the hash of the complete datapath for an endpoint. // It does not change the underlying hash state. -func (d *datapathHash) sumEndpoint(c datapath.ConfigWriter, epCfg datapath.EndpointConfiguration, staticData bool) (string, error) { +func (d *datapathHash) sumEndpoint(c datapath.ConfigWriter, nodeCfg *datapath.LocalNodeConfiguration, epCfg datapath.EndpointConfiguration, staticData bool) (string, error) { result, err := d.Copy() if err != nil { return "", err } if staticData { - c.WriteEndpointConfig(result, epCfg) + c.WriteEndpointConfig(result, nodeCfg, epCfg) } else { - c.WriteTemplateConfig(result, epCfg) + c.WriteTemplateConfig(result, nodeCfg, epCfg) } return result.String(), nil } diff --git a/pkg/datapath/loader/hash_test.go b/pkg/datapath/loader/hash_test.go index e85ae01d51ceb..6f4e4a82372d2 100644 --- a/pkg/datapath/loader/hash_test.go +++ b/pkg/datapath/loader/hash_test.go @@ -9,14 +9,12 @@ import ( "github.com/cilium/hive/cell" "github.com/cilium/hive/hivetest" - "github.com/cilium/statedb" "github.com/spf13/afero" "github.com/stretchr/testify/require" fakeTypes "github.com/cilium/cilium/pkg/datapath/fake/types" "github.com/cilium/cilium/pkg/datapath/linux/config" "github.com/cilium/cilium/pkg/datapath/linux/sysctl" - "github.com/cilium/cilium/pkg/datapath/tables" datapath "github.com/cilium/cilium/pkg/datapath/types" "github.com/cilium/cilium/pkg/hive" "github.com/cilium/cilium/pkg/maps/nodemap" @@ -25,9 +23,6 @@ import ( ) var ( - dummyNodeCfg = datapath.LocalNodeConfiguration{ - MtuConfig: &fakeTypes.MTU{}, - } dummyDevCfg = testutils.NewTestEndpoint() dummyEPCfg = testutils.NewTestEndpoint() ) @@ -35,25 +30,15 @@ var ( // TestHashDatapath is done in this package just for easy access to dummy // configuration objects. func TestHashDatapath(t *testing.T) { - setupLocalNodeStore(t) - var cfg datapath.ConfigWriter hv := hive.New( provideNodemap, cell.Provide( - tables.NewNodeAddressTable, - statedb.RWTable[tables.NodeAddress].ToTable, - tables.NewDeviceTable, - statedb.RWTable[*tables.Device].ToTable, func() datapath.BandwidthManager { return &fakeTypes.BandwidthManager{} }, func() sysctl.Sysctl { return sysctl.NewDirectSysctl(afero.NewOsFs(), "/proc") }, config.NewHeaderfileWriter, fakeTypes.NewNodeAddressing, ), - cell.Invoke( - statedb.RegisterTable[*tables.Device], - statedb.RegisterTable[tables.NodeAddress], - ), cell.Invoke(func(writer_ datapath.ConfigWriter) { cfg = writer_ }), @@ -67,7 +52,7 @@ func TestHashDatapath(t *testing.T) { baseHash := h.String() // Ensure we get different hashes when config is added - h = hashDatapath(cfg, &dummyNodeCfg, &dummyDevCfg, &dummyEPCfg) + h = hashDatapath(cfg, &localNodeConfig, &dummyDevCfg, &dummyEPCfg) dummyHash := h.String() require.NotEqual(t, dummyHash, baseHash) @@ -78,7 +63,7 @@ func TestHashDatapath(t *testing.T) { // Ensure that with a copy of the endpoint config we get the same hash newEPCfg := dummyEPCfg - h = hashDatapath(cfg, &dummyNodeCfg, &dummyDevCfg, &newEPCfg) + h = hashDatapath(cfg, &localNodeConfig, &dummyDevCfg, &newEPCfg) require.NotEqual(t, h.String(), baseHash) require.Equal(t, h.String(), dummyHash) @@ -87,14 +72,14 @@ func TestHashDatapath(t *testing.T) { // This is the key to avoiding recompilation per endpoint; static // data substitution is performed via pkg/elf instead. newEPCfg.Id++ - h = hashDatapath(cfg, &dummyNodeCfg, &dummyDevCfg, &newEPCfg) + h = hashDatapath(cfg, &localNodeConfig, &dummyDevCfg, &newEPCfg) require.NotEqual(t, h.String(), baseHash) require.Equal(t, h.String(), dummyHash) // But when we configure the endpoint differently, it's different newEPCfg = testutils.NewTestEndpoint() newEPCfg.Opts.SetBool("foo", true) - h = hashDatapath(cfg, &dummyNodeCfg, &dummyDevCfg, &newEPCfg) + h = hashDatapath(cfg, &localNodeConfig, &dummyDevCfg, &newEPCfg) require.NotEqual(t, h.String(), baseHash) require.NotEqual(t, h.String(), dummyHash) } diff --git a/pkg/datapath/loader/loader.go b/pkg/datapath/loader/loader.go index 6f1830e401d5e..ecb4e3a7e136b 100644 --- a/pkg/datapath/loader/loader.go +++ b/pkg/datapath/loader/loader.go @@ -6,14 +6,15 @@ package loader import ( "context" "fmt" + "io" "net" "net/netip" "strings" "sync" + "sync/atomic" "github.com/cilium/ebpf" "github.com/cilium/hive/cell" - "github.com/cilium/statedb" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" @@ -70,6 +71,8 @@ var log = logging.DefaultLogger.WithField(logfields.LogSubsys, subsystem) type loader struct { cfg Config + nodeConfig atomic.Pointer[datapath.LocalNodeConfiguration] + once sync.Once // templateCache is the cache of pre-compiled datapaths. @@ -81,13 +84,9 @@ type loader struct { hostDpInitialized chan struct{} sysctl sysctl.Sysctl - db *statedb.DB - nodeAddrs statedb.Table[tables.NodeAddress] - devices statedb.Table[*tables.Device] prefilter datapath.PreFilter compilationLock datapath.CompilationLock configWriter datapath.ConfigWriter - localNodeConfig datapath.LocalNodeConfiguration nodeHandler datapath.NodeHandler } @@ -95,14 +94,10 @@ type Params struct { cell.In Config Config - DB *statedb.DB - NodeAddrs statedb.Table[tables.NodeAddress] Sysctl sysctl.Sysctl - Devices statedb.Table[*tables.Device] Prefilter datapath.PreFilter CompilationLock datapath.CompilationLock ConfigWriter datapath.ConfigWriter - LocalNodeConfig datapath.LocalNodeConfiguration NodeHandler datapath.NodeHandler } @@ -110,26 +105,22 @@ type Params struct { func newLoader(p Params) *loader { return &loader{ cfg: p.Config, - db: p.DB, - nodeAddrs: p.NodeAddrs, sysctl: p.Sysctl, - devices: p.Devices, hostDpInitialized: make(chan struct{}), prefilter: p.Prefilter, compilationLock: p.CompilationLock, configWriter: p.ConfigWriter, - localNodeConfig: p.LocalNodeConfig, nodeHandler: p.NodeHandler, } } -// Init initializes the datapath cache with base program hashes derived from +// initTemplateCache initializes the datapath cache with base program hashes derived from // the LocalNodeConfiguration. -func (l *loader) init() { +func (l *loader) initTemplateCache(cfg *datapath.LocalNodeConfiguration) { l.once.Do(func() { - l.templateCache = newObjectCache(l.configWriter, &l.localNodeConfig, option.Config.StateDir) + l.templateCache = newObjectCache(l.configWriter, cfg, option.Config.StateDir) }) - l.templateCache.Update(&l.localNodeConfig) + l.templateCache.Update(cfg) } func upsertEndpointRoute(ep datapath.Endpoint, ip net.IPNet) error { @@ -156,8 +147,13 @@ func (l *loader) bpfMasqAddrs(ifName string) (masq4, masq6 netip.Addr) { ifName = l.cfg.DeriveMasqIPAddrFromDevice } - find := func(iter statedb.Iterator[tables.NodeAddress]) bool { - for addr, _, ok := iter.Next(); ok; addr, _, ok = iter.Next() { + addrs := l.nodeConfig.Load().NodeAddresses + + find := func(devName string) bool { + for _, addr := range addrs { + if addr.DeviceName != devName { + continue + } if !addr.Primary { continue } @@ -176,19 +172,10 @@ func (l *loader) bpfMasqAddrs(ifName string) (masq4, masq6 netip.Addr) { } // Try to find suitable masquerade address first from the given interface. - txn := l.db.ReadTxn() - iter := l.nodeAddrs.List( - txn, - tables.NodeAddressDeviceNameIndex.Query(ifName), - ) - if !find(iter) { + if !find(ifName) { // No suitable masquerade addresses were found for this device. Try the fallback // addresses. - iter = l.nodeAddrs.List( - txn, - tables.NodeAddressDeviceNameIndex.Query(tables.WildcardDeviceName), - ) - find(iter) + find(tables.WildcardDeviceName) } return @@ -479,9 +466,7 @@ func (l *loader) reloadDatapath(ep datapath.Endpoint, spec *ebpf.CollectionSpec) } if ep.IsHost() { - // TODO: react to changes (using the currently ignored watch channel) - nativeDevices, _ := tables.SelectedDevices(l.devices, l.db.ReadTxn()) - devices := tables.DeviceNames(nativeDevices) + devices := l.nodeConfig.Load().DeviceNames() if option.Config.NeedBPFHostOnWireGuardDevice() { devices = append(devices, wgTypes.IfaceName) @@ -598,7 +583,9 @@ func (l *loader) ReloadDatapath(ctx context.Context, ep datapath.Endpoint, stats Output: ep.StateDir(), } - templateFile, _, err := l.templateCache.fetchOrCompile(ctx, ep, &dirs, stats) + cfg := l.nodeConfig.Load() + + templateFile, _, err := l.templateCache.fetchOrCompile(ctx, cfg, ep, &dirs, stats) if err != nil { return err } @@ -659,7 +646,7 @@ func (l *loader) Unload(ep datapath.Endpoint) { // EndpointHash hashes the specified endpoint configuration with the current // datapath hash cache and returns the hash as string. func (l *loader) EndpointHash(cfg datapath.EndpointConfiguration) (string, error) { - return l.templateCache.baseHash.sumEndpoint(l.templateCache, cfg, true) + return l.templateCache.baseHash.sumEndpoint(l.templateCache, l.nodeConfig.Load(), cfg, true) } // CallsMapPath gets the BPF Calls Map for the endpoint with the specified ID. @@ -678,3 +665,7 @@ func (l *loader) CustomCallsMapPath(id uint16) string { func (l *loader) HostDatapathInitialized() <-chan struct{} { return l.hostDpInitialized } + +func (l *loader) WriteEndpointConfig(w io.Writer, e datapath.EndpointConfiguration) error { + return l.configWriter.WriteEndpointConfig(w, l.nodeConfig.Load(), e) +} diff --git a/pkg/datapath/loader/loader_test.go b/pkg/datapath/loader/loader_test.go index 7735426c45077..51b951bac43d1 100644 --- a/pkg/datapath/loader/loader_test.go +++ b/pkg/datapath/loader/loader_test.go @@ -13,7 +13,6 @@ import ( "time" "github.com/cilium/ebpf/rlimit" - "github.com/cilium/statedb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vishvananda/netlink" @@ -203,39 +202,40 @@ func TestBPFMasqAddrs(t *testing.T) { }) l := newTestLoader(t) - nodeAddrs := l.nodeAddrs.(statedb.RWTable[tables.NodeAddress]) - db := l.db masq4, masq6 := l.bpfMasqAddrs("test") require.Equal(t, masq4.IsValid(), false) require.Equal(t, masq6.IsValid(), false) - txn := db.WriteTxn(nodeAddrs) - nodeAddrs.Insert(txn, tables.NodeAddress{ - Addr: netip.MustParseAddr("1.0.0.1"), - NodePort: true, - Primary: true, - DeviceName: "test", - }) - nodeAddrs.Insert(txn, tables.NodeAddress{ - Addr: netip.MustParseAddr("1000::1"), - NodePort: true, - Primary: true, - DeviceName: "test", - }) - nodeAddrs.Insert(txn, tables.NodeAddress{ - Addr: netip.MustParseAddr("2.0.0.2"), - NodePort: false, - Primary: true, - DeviceName: tables.WildcardDeviceName, - }) - nodeAddrs.Insert(txn, tables.NodeAddress{ - Addr: netip.MustParseAddr("2000::2"), - NodePort: false, - Primary: true, - DeviceName: tables.WildcardDeviceName, - }) - txn.Commit() + newConfig := *l.nodeConfig.Load() + + newConfig.NodeAddresses = []tables.NodeAddress{ + { + Addr: netip.MustParseAddr("1.0.0.1"), + NodePort: true, + Primary: true, + DeviceName: "test", + }, + { + Addr: netip.MustParseAddr("1000::1"), + NodePort: true, + Primary: true, + DeviceName: "test", + }, + { + Addr: netip.MustParseAddr("2.0.0.2"), + NodePort: false, + Primary: true, + DeviceName: tables.WildcardDeviceName, + }, + { + Addr: netip.MustParseAddr("2000::2"), + NodePort: false, + Primary: true, + DeviceName: tables.WildcardDeviceName, + }, + } + l.nodeConfig.Store(&newConfig) masq4, masq6 = l.bpfMasqAddrs("test") require.Equal(t, masq4.String(), "1.0.0.1") diff --git a/pkg/datapath/loader/template_test.go b/pkg/datapath/loader/template_test.go index cb0334b0eb631..a8200712e08b0 100644 --- a/pkg/datapath/loader/template_test.go +++ b/pkg/datapath/loader/template_test.go @@ -23,9 +23,9 @@ func TestWrap(t *testing.T) { cfg := configWriterForTest(t) // Write the configuration that should be the same, and verify it is. - err := cfg.WriteTemplateConfig(&realEPBuffer, &realEP) + err := cfg.WriteTemplateConfig(&realEPBuffer, &localNodeConfig, &realEP) require.NoError(t, err) - err = cfg.WriteTemplateConfig(&templateBuffer, template) + err = cfg.WriteTemplateConfig(&templateBuffer, &localNodeConfig, template) require.NoError(t, err) require.Equal(t, realEPBuffer.String(), templateBuffer.String()) @@ -35,9 +35,9 @@ func TestWrap(t *testing.T) { // define every bit of static data differently in the templates. realEPBuffer.Reset() templateBuffer.Reset() - err = cfg.WriteEndpointConfig(&realEPBuffer, &realEP) + err = cfg.WriteEndpointConfig(&realEPBuffer, &localNodeConfig, &realEP) require.NoError(t, err) - err = cfg.WriteEndpointConfig(&templateBuffer, template) + err = cfg.WriteEndpointConfig(&templateBuffer, &localNodeConfig, template) require.NoError(t, err) require.NotEqual(t, realEPBuffer.String(), templateBuffer.String()) diff --git a/pkg/datapath/loader/util_test.go b/pkg/datapath/loader/util_test.go index 3a207b7f48866..498b46ceee80f 100644 --- a/pkg/datapath/loader/util_test.go +++ b/pkg/datapath/loader/util_test.go @@ -8,26 +8,26 @@ import ( "path/filepath" "testing" - "github.com/cilium/statedb" "github.com/spf13/afero" - "github.com/stretchr/testify/require" - "github.com/cilium/cilium/pkg/datapath/linux/config" + "github.com/cilium/cilium/pkg/cidr" "github.com/cilium/cilium/pkg/datapath/linux/sysctl" - "github.com/cilium/cilium/pkg/datapath/tables" + datapath "github.com/cilium/cilium/pkg/datapath/types" "github.com/cilium/cilium/pkg/maps/callsmap" "github.com/cilium/cilium/pkg/maps/policymap" - "github.com/cilium/cilium/pkg/node" "github.com/cilium/cilium/pkg/option" ) -func setupLocalNodeStore(tb testing.TB) { - node.SetTestLocalNodeStore() - node.InitDefaultPrefix("") - node.SetInternalIPv4Router(templateIPv4[:]) - node.SetIPv4Loopback(templateIPv4[:]) - tb.Cleanup(node.UnsetTestLocalNodeStore) -} +var ( + localNodeConfig = datapath.LocalNodeConfiguration{ + NodeIPv4: templateIPv4[:], + CiliumInternalIPv4: templateIPv4[:], + AllocCIDRIPv4: cidr.MustParseCIDR("10.147.0.0/16"), + LoopbackIPv4: templateIPv4[:], + HostEndpointID: 1, + EnableIPv4: true, + } +) func setupCompilationDirectories(tb testing.TB) { option.Config.DryMode = true @@ -58,19 +58,13 @@ func setupCompilationDirectories(tb testing.TB) { func newTestLoader(tb testing.TB) *loader { setupCompilationDirectories(tb) - nodeAddrs, err := tables.NewNodeAddressTable() - require.NoError(tb, err, "NewNodeAddressTable") - devices, err := tables.NewDeviceTable() - require.NoError(tb, err, "NewDeviceTable") - db := statedb.New() - require.NoError(tb, db.RegisterTable(nodeAddrs, devices), "RegisterTable") + l := newLoader(Params{ - Config: DefaultConfig, - DB: db, - NodeAddrs: nodeAddrs, - Sysctl: sysctl.NewDirectSysctl(afero.NewOsFs(), "/proc"), - Devices: devices, + Config: DefaultConfig, + Sysctl: sysctl.NewDirectSysctl(afero.NewOsFs(), "/proc"), }) - l.templateCache = newObjectCache(&config.HeaderfileWriter{}, nil, tb.TempDir()) + l.nodeConfig.Store(&localNodeConfig) + cw := configWriterForTest(tb) + l.templateCache = newObjectCache(cw, &localNodeConfig, tb.TempDir()) return l } diff --git a/pkg/datapath/orchestrator/localnodeconfig.go b/pkg/datapath/orchestrator/localnodeconfig.go new file mode 100644 index 0000000000000..32ad1da6994d2 --- /dev/null +++ b/pkg/datapath/orchestrator/localnodeconfig.go @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Cilium + +package orchestrator + +import ( + "fmt" + + "github.com/cilium/statedb" + + "github.com/cilium/cilium/pkg/cidr" + "github.com/cilium/cilium/pkg/datapath/tables" + datapath "github.com/cilium/cilium/pkg/datapath/types" + ipamOption "github.com/cilium/cilium/pkg/ipam/option" + "github.com/cilium/cilium/pkg/mtu" + "github.com/cilium/cilium/pkg/node" + "github.com/cilium/cilium/pkg/option" +) + +const ( + // AutoCIDR indicates that a CIDR should be allocated + AutoCIDR = "auto" +) + +// newLocalNodeConfig constructs LocalNodeConfiguration from the global agent +// data sources. + +// LocalNodeConfiguration encapsulates the datapath relevant part of dynamic +// state of the agent, which allows the datapath code to operate against a +// pure data struct rather than complex APIs. When this data changes a new +// LocalNodeConfiguration instance is generated. Previous LocalNodeConfiguration +// is never mutated in-place. +func newLocalNodeConfig( + config *option.DaemonConfig, + localNode node.LocalNode, + mtu mtu.MTU, + txn statedb.ReadTxn, + devices statedb.Table[*tables.Device], + nodeAddresses statedb.Table[tables.NodeAddress], +) (datapath.LocalNodeConfiguration, error) { + auxPrefixes := []*cidr.CIDR{} + + if config.IPv4ServiceRange != AutoCIDR { + serviceCIDR, err := cidr.ParseCIDR(config.IPv4ServiceRange) + if err != nil { + return datapath.LocalNodeConfiguration{}, fmt.Errorf("Invalid IPv4 service prefix %q: %w", config.IPv4ServiceRange, err) + } + + auxPrefixes = append(auxPrefixes, serviceCIDR) + } + + if config.IPv6ServiceRange != AutoCIDR { + serviceCIDR, err := cidr.ParseCIDR(config.IPv6ServiceRange) + if err != nil { + return datapath.LocalNodeConfiguration{}, fmt.Errorf("Invalid IPv6 service prefix %q: %w", config.IPv6ServiceRange, err) + } + + auxPrefixes = append(auxPrefixes, serviceCIDR) + } + + nativeDevices, _ := tables.SelectedDevices(devices, txn) + nodeAddrsIter, _ := nodeAddresses.All(txn) + + return datapath.LocalNodeConfiguration{ + NodeIPv4: localNode.GetNodeIP(false), + NodeIPv6: localNode.GetNodeIP(true), + CiliumInternalIPv4: localNode.GetCiliumInternalIP(false), + CiliumInternalIPv6: localNode.GetCiliumInternalIP(true), + AllocCIDRIPv4: localNode.IPv4AllocCIDR, + AllocCIDRIPv6: localNode.IPv6AllocCIDR, + LoopbackIPv4: node.GetIPv4Loopback(), + Devices: nativeDevices, + NodeAddresses: statedb.Collect(nodeAddrsIter), + HostEndpointID: node.GetEndpointID(), + DeviceMTU: mtu.GetDeviceMTU(), + RouteMTU: mtu.GetRouteMTU(), + RoutePostEncryptMTU: mtu.GetRoutePostEncryptMTU(), + AuxiliaryPrefixes: auxPrefixes, + EnableIPv4: config.EnableIPv4, + EnableIPv6: config.EnableIPv6, + EnableEncapsulation: config.TunnelingEnabled(), + EnableAutoDirectRouting: config.EnableAutoDirectRouting, + EnableLocalNodeRoute: config.EnableLocalNodeRoute && config.IPAM != ipamOption.IPAMENI && config.IPAM != ipamOption.IPAMAzure && config.IPAM != ipamOption.IPAMAlibabaCloud, + EnableIPSec: config.EnableIPSec, + EncryptNode: config.EncryptNode, + IPv4PodSubnets: config.IPv4PodSubnets, + IPv6PodSubnets: config.IPv6PodSubnets, + }, nil +} diff --git a/pkg/datapath/orchestrator/orchestrator.go b/pkg/datapath/orchestrator/orchestrator.go index 003ca058531b9..700ea6a141433 100644 --- a/pkg/datapath/orchestrator/orchestrator.go +++ b/pkg/datapath/orchestrator/orchestrator.go @@ -5,13 +5,19 @@ package orchestrator import ( "context" + "fmt" "github.com/cilium/hive/cell" + "github.com/cilium/statedb" "github.com/cilium/cilium/pkg/datapath/iptables" + "github.com/cilium/cilium/pkg/datapath/tables" "github.com/cilium/cilium/pkg/datapath/tunnel" "github.com/cilium/cilium/pkg/datapath/types" "github.com/cilium/cilium/pkg/mtu" + "github.com/cilium/cilium/pkg/node" + "github.com/cilium/cilium/pkg/nodediscovery" + "github.com/cilium/cilium/pkg/option" "github.com/cilium/cilium/pkg/proxy" ) @@ -27,6 +33,11 @@ type orchestratorParams struct { MTU mtu.MTU IPTablesManager *iptables.Manager Proxy *proxy.Proxy + DB *statedb.DB + Devices statedb.Table[*tables.Device] + NodeAddresses statedb.Table[tables.NodeAddress] + LocalNodeStore *node.LocalNodeStore + NodeDiscovery *nodediscovery.NodeDiscovery } func newOrchestrator(params orchestratorParams) *orchestrator { @@ -36,10 +47,32 @@ func newOrchestrator(params orchestratorParams) *orchestrator { } func (o *orchestrator) Reinitialize(ctx context.Context) error { + // Wait until the local node has been populated by NodeDiscovery. + o.params.NodeDiscovery.WaitForLocalNodeInit() + + localNode, err := o.params.LocalNodeStore.Get(ctx) + if err != nil { + return fmt.Errorf("get local node: %w", err) + } + + // Construct the LocalNodeConfiguration that encapsulates the + // local node's dynamic configuration. + localNodeConfig, err := newLocalNodeConfig( + option.Config, + localNode, + o.params.MTU, + o.params.DB.ReadTxn(), + o.params.Devices, + o.params.NodeAddresses, + ) + if err != nil { + return fmt.Errorf("build LocalNodeConfiguration: %w", err) + } + return o.params.Loader.Reinitialize( ctx, + localNodeConfig, o.params.TunnelConfig, - o.params.MTU.GetDeviceMTU(), o.params.IPTablesManager, o.params.Proxy, ) diff --git a/pkg/datapath/types/config.go b/pkg/datapath/types/config.go index ca6328fa62465..7109b03afc070 100644 --- a/pkg/datapath/types/config.go +++ b/pkg/datapath/types/config.go @@ -106,11 +106,11 @@ type ConfigWriter interface { // WriteTemplateConfig writes the implementation-specific configuration // of configurable options for BPF templates to the specified writer. - WriteTemplateConfig(w io.Writer, cfg EndpointConfiguration) error + WriteTemplateConfig(w io.Writer, nodeCfg *LocalNodeConfiguration, cfg EndpointConfiguration) error // WriteEndpointConfig writes the implementation-specific configuration // of configurable options for the endpoint to the specified writer. - WriteEndpointConfig(w io.Writer, cfg EndpointConfiguration) error + WriteEndpointConfig(w io.Writer, nodeCfg *LocalNodeConfiguration, cfg EndpointConfiguration) error } // RemoteSNATDstAddrExclusionCIDRv4 returns a CIDR for SNAT exclusion. Any diff --git a/pkg/datapath/types/loader.go b/pkg/datapath/types/loader.go index 5283c8f881fe1..a3429ea1ae4b8 100644 --- a/pkg/datapath/types/loader.go +++ b/pkg/datapath/types/loader.go @@ -21,10 +21,12 @@ type Loader interface { ReinitializeXDP(ctx context.Context, extraCArgs []string) error EndpointHash(cfg EndpointConfiguration) (string, error) Unload(ep Endpoint) - Reinitialize(ctx context.Context, tunnelConfig tunnel.Config, deviceMTU int, iptMgr IptablesManager, p Proxy) error + Reinitialize(ctx context.Context, cfg LocalNodeConfiguration, tunnelConfig tunnel.Config, iptMgr IptablesManager, p Proxy) error HostDatapathInitialized() <-chan struct{} RestoreTemplates(stateDir string) error DetachXDP(iface string, bpffsBase, progName string) error + + WriteEndpointConfig(w io.Writer, cfg EndpointConfiguration) error } // PreFilter an interface for an XDP pre-filter. diff --git a/pkg/datapath/types/node.go b/pkg/datapath/types/node.go index fbe81acb95e26..3a4d6c652337b 100644 --- a/pkg/datapath/types/node.go +++ b/pkg/datapath/types/node.go @@ -9,6 +9,7 @@ import ( "github.com/cilium/cilium/api/v1/models" "github.com/cilium/cilium/pkg/cidr" + "github.com/cilium/cilium/pkg/datapath/tables" nodeTypes "github.com/cilium/cilium/pkg/node/types" ) @@ -19,12 +20,72 @@ type MTUConfiguration interface { } // LocalNodeConfiguration represents the configuration of the local node +// +// This configuration struct is immutable even when passed by reference. +// When the configuration is changed at runtime a new instance is allocated +// and passed down. type LocalNodeConfiguration struct { - // MtuConfig is the MTU configuration of the node. - // + // NodeIPv4 is the primary IPv4 address of this node. + // Mutable at runtime. + NodeIPv4 net.IP + + // NodeIPv6 is the primary IPv6 address of this node. + // Mutable at runtime. + NodeIPv6 net.IP + + // CiliumInternalIPv4 is the internal IP address assigned to the cilium_host + // interface. + // Immutable at runtime. + CiliumInternalIPv4 net.IP + + // CiliumInternalIPv6 is the internal IP address assigned to the cilium_host + // interface. + // Immutable at runtime. + CiliumInternalIPv6 net.IP + + // AllocCIDRIPv4 is the IPv4 allocation CIDR from which IP addresses for + // endpoints are allocated from. + // Immutable at runtime. + AllocCIDRIPv4 *cidr.CIDR + + // AllocCIDRIPv6 is the IPv6 allocation CIDR from which IP addresses for + // endpoints are allocated from. + // Immutable at runtime. + AllocCIDRIPv6 *cidr.CIDR + + // LoopbackIPv4 is the IPv4 loopback address. + // Immutable at runtime. + LoopbackIPv4 net.IP + + // Devices is the native network devices selected for datapath use. + // Mutable at runtime. + Devices []*tables.Device + + // NodeAddresses are the IP addresses of the local node that are considered + // as this node's addresses. From this set we pick the addresses that are + // used as NodePort frontends and the addresses to use for BPF masquerading. + // Mutable at runtime. + NodeAddresses []tables.NodeAddress + + // HostEndpointID is the endpoint ID assigned to the host endpoint. + // Immutable at runtime. + HostEndpointID uint64 + + // DeviceMTU is the MTU used on workload facing devices. + // This field is immutable at runtime. The value will not change in + // subsequent calls to NodeConfigurationChanged(). + DeviceMTU int + + // RouteMTU is the MTU used on the network. // This field is immutable at runtime. The value will not change in // subsequent calls to NodeConfigurationChanged(). - MtuConfig MTUConfiguration + RouteMTU int + + // RoutePostEncryptMTU is the MTU without the encryption overhead + // included. + // This field is immutable at runtime. The value will not change in + // subsequent calls to NodeConfigurationChanged(). + RoutePostEncryptMTU int // AuxiliaryPrefixes is the list of auxiliary prefixes that should be // configured in addition to the node PodCIDR @@ -89,6 +150,10 @@ type LocalNodeConfiguration struct { IPv6PodSubnets []*net.IPNet } +func (cfg *LocalNodeConfiguration) DeviceNames() []string { + return tables.DeviceNames(cfg.Devices) +} + // NodeHandler handles node related events such as addition, update or deletion // of nodes or changes to the local node configuration. // diff --git a/pkg/endpoint/bpf.go b/pkg/endpoint/bpf.go index 1a3fb1941e158..49bfae6dcbabd 100644 --- a/pkg/endpoint/bpf.go +++ b/pkg/endpoint/bpf.go @@ -186,7 +186,7 @@ func (e *Endpoint) writeHeaderfile(prefix string) error { return err } - if err = e.owner.Datapath().WriteEndpointConfig(f, e); err != nil { + if err = e.owner.Datapath().Loader().WriteEndpointConfig(f, e); err != nil { return err } diff --git a/pkg/endpoint/bpf_test.go b/pkg/endpoint/bpf_test.go index 58e3a631d7fa4..e73e0f143a3f4 100644 --- a/pkg/endpoint/bpf_test.go +++ b/pkg/endpoint/bpf_test.go @@ -13,6 +13,7 @@ import ( "github.com/cilium/cilium/pkg/datapath/linux" "github.com/cilium/cilium/pkg/datapath/linux/config" + datapath "github.com/cilium/cilium/pkg/datapath/types" "github.com/cilium/cilium/pkg/testutils" testidentity "github.com/cilium/cilium/pkg/testutils/identity" testipcache "github.com/cilium/cilium/pkg/testutils/ipcache" @@ -42,12 +43,13 @@ func BenchmarkWriteHeaderfile(b *testing.B) { NodeMap: nil, ConfigWriter: &config.HeaderfileWriter{}, }) + cfg := datapath.LocalNodeConfiguration{} targetComments := func(w io.Writer) error { return e.writeInformationalComments(w) } targetConfig := func(w io.Writer) error { - return dp.WriteEndpointConfig(w, e) + return dp.WriteEndpointConfig(w, &cfg, e) } var buf bytes.Buffer diff --git a/pkg/nodediscovery/cell.go b/pkg/nodediscovery/cell.go index c4b299ba76a74..45fb5259140bb 100644 --- a/pkg/nodediscovery/cell.go +++ b/pkg/nodediscovery/cell.go @@ -13,6 +13,4 @@ var Cell = cell.Module( // Node discovery communicates changes in local node information to the API server or KVStore cell.Provide(NewNodeDiscovery), - // LocalNodeConfig provides a subset of the DaemonConfig with a little pre-processing - cell.Provide(NewLocalNodeConfig), ) diff --git a/pkg/nodediscovery/localnodeconfig.go b/pkg/nodediscovery/localnodeconfig.go deleted file mode 100644 index 61145d080e6af..0000000000000 --- a/pkg/nodediscovery/localnodeconfig.go +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Copyright Authors of Cilium - -package nodediscovery - -import ( - "fmt" - - "github.com/cilium/cilium/pkg/cidr" - datapath "github.com/cilium/cilium/pkg/datapath/types" - ipamOption "github.com/cilium/cilium/pkg/ipam/option" - "github.com/cilium/cilium/pkg/mtu" - "github.com/cilium/cilium/pkg/option" -) - -func NewLocalNodeConfig(mtu mtu.MTU, config *option.DaemonConfig) (datapath.LocalNodeConfiguration, error) { - auxPrefixes := []*cidr.CIDR{} - - if config.IPv4ServiceRange != AutoCIDR { - serviceCIDR, err := cidr.ParseCIDR(config.IPv4ServiceRange) - if err != nil { - return datapath.LocalNodeConfiguration{}, fmt.Errorf("Invalid IPv4 service prefix %q: %w", config.IPv4ServiceRange, err) - } - - auxPrefixes = append(auxPrefixes, serviceCIDR) - } - - if config.IPv6ServiceRange != AutoCIDR { - serviceCIDR, err := cidr.ParseCIDR(config.IPv6ServiceRange) - if err != nil { - return datapath.LocalNodeConfiguration{}, fmt.Errorf("Invalid IPv6 service prefix %q: %w", config.IPv6ServiceRange, err) - } - - auxPrefixes = append(auxPrefixes, serviceCIDR) - } - - return datapath.LocalNodeConfiguration{ - MtuConfig: mtu, - EnableIPv4: config.EnableIPv4, - EnableIPv6: config.EnableIPv6, - EnableEncapsulation: config.TunnelingEnabled(), - EnableAutoDirectRouting: config.EnableAutoDirectRouting, - EnableLocalNodeRoute: config.EnableLocalNodeRoute && - config.IPAM != ipamOption.IPAMENI && - config.IPAM != ipamOption.IPAMAzure && - config.IPAM != ipamOption.IPAMAlibabaCloud, - AuxiliaryPrefixes: auxPrefixes, - EnableIPSec: config.EnableIPSec, - EncryptNode: config.EncryptNode, - IPv4PodSubnets: config.IPv4PodSubnets, - IPv6PodSubnets: config.IPv6PodSubnets, - }, nil -} diff --git a/pkg/nodediscovery/nodediscovery.go b/pkg/nodediscovery/nodediscovery.go index 9047a0784bb1b..511c975054296 100644 --- a/pkg/nodediscovery/nodediscovery.go +++ b/pkg/nodediscovery/nodediscovery.go @@ -24,7 +24,6 @@ import ( "github.com/cilium/cilium/pkg/aws/metadata" azureTypes "github.com/cilium/cilium/pkg/azure/types" "github.com/cilium/cilium/pkg/controller" - datapath "github.com/cilium/cilium/pkg/datapath/types" "github.com/cilium/cilium/pkg/defaults" "github.com/cilium/cilium/pkg/identity" ipamOption "github.com/cilium/cilium/pkg/ipam/option" @@ -42,9 +41,6 @@ import ( ) const ( - // AutoCIDR indicates that a CIDR should be allocated - AutoCIDR = "auto" - nodeDiscoverySubsys = "nodediscovery" maxRetryCount = 10 ) @@ -66,7 +62,6 @@ type GetNodeAddresses interface { // NodeDiscovery represents a node discovery action type NodeDiscovery struct { Manager nodemanager.NodeManager - LocalConfig datapath.LocalNodeConfiguration Registrar nodestore.NodeRegistrar Registered chan struct{} localStateInitialized chan struct{} @@ -82,12 +77,10 @@ func NewNodeDiscovery( manager nodemanager.NodeManager, clientset client.Clientset, lns *node.LocalNodeStore, - localConfig datapath.LocalNodeConfiguration, cniConfigManager cni.CNIConfigManager, ) *NodeDiscovery { return &NodeDiscovery{ Manager: manager, - LocalConfig: localConfig, localNodeStore: lns, Registered: make(chan struct{}), localStateInitialized: make(chan struct{}), @@ -349,9 +342,9 @@ func (n *NodeDiscovery) mutateNodeResource(nodeResource *ciliumv2.CiliumNode, ln // b) the LocalNode store contains an IP address which we can use instead switch net.IPFamilyOfString(address.IP) { case net.IPv4: - return !n.LocalConfig.EnableIPv4 || ln.GetCiliumInternalIP(false) != nil + return !option.Config.EnableIPv4 || ln.GetCiliumInternalIP(false) != nil case net.IPv6: - return !n.LocalConfig.EnableIPv6 || ln.GetCiliumInternalIP(true) != nil + return !option.Config.EnableIPv6 || ln.GetCiliumInternalIP(true) != nil } }