From f71de74d03be79295b953e647baa71bd54abb206 Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Thu, 30 Nov 2023 13:22:32 +0100 Subject: [PATCH 1/2] fix: multiple xdp attach to one iface --- xdp/bandwidth/bandwidth.go | 29 ++---- xdp/{bandwidth => }/bpf_bpfeb.go | 11 +- xdp/{bandwidth => }/bpf_bpfel.go | 11 +- xdp/kerns/main.c | 17 ++++ xdp/{bandwidth => }/kerns/xdp_bandwidth.c | 32 +++--- xdp/kerns/xdp_packetloss.c | 38 +++++++ xdp/packetloss/bpf_bpfeb.go | 119 ---------------------- xdp/packetloss/bpf_bpfel.go | 119 ---------------------- xdp/packetloss/kerns/xdp_drop_percent.c | 38 ------- xdp/packetloss/packetloss.go | 31 +++--- xdp/types.go | 60 +++++++++++ 11 files changed, 170 insertions(+), 335 deletions(-) rename xdp/{bandwidth => }/bpf_bpfeb.go (91%) rename xdp/{bandwidth => }/bpf_bpfel.go (91%) create mode 100644 xdp/kerns/main.c rename xdp/{bandwidth => }/kerns/xdp_bandwidth.c (80%) create mode 100644 xdp/kerns/xdp_packetloss.c delete mode 100644 xdp/packetloss/bpf_bpfeb.go delete mode 100644 xdp/packetloss/bpf_bpfel.go delete mode 100644 xdp/packetloss/kerns/xdp_drop_percent.c diff --git a/xdp/bandwidth/bandwidth.go b/xdp/bandwidth/bandwidth.go index 05778ee..0c7970f 100644 --- a/xdp/bandwidth/bandwidth.go +++ b/xdp/bandwidth/bandwidth.go @@ -7,12 +7,9 @@ import ( "github.com/celestiaorg/bittwister/xdp" "github.com/cilium/ebpf" - "github.com/cilium/ebpf/link" "go.uber.org/zap" ) -//go:generate go run github.com/cilium/ebpf/cmd/bpf2go bpf kerns/xdp_bandwidth.c -- -I../headers - type Bandwidth struct { NetworkInterface *net.Interface Limit int64 // Bytes per second @@ -22,26 +19,15 @@ type Bandwidth struct { var _ xdp.XdpLoader = (*Bandwidth)(nil) func (b *Bandwidth) Start(ctx context.Context, logger *zap.Logger) { - // Load pre-compiled programs into the kernel. - objs := bpfObjects{} - if err := loadBpfObjects(&objs, nil); err != nil { - logger.Error(fmt.Sprintf("loading objects: %v", err)) - return - } - defer objs.Close() - - l, err := link.AttachXDP(link.XDPOptions{ - Program: objs.XdpBandwidthLimit, - Interface: b.NetworkInterface.Index, - }) + x, err := xdp.GetPreparedXdpObject(b.NetworkInterface.Index) if err != nil { - logger.Error(fmt.Sprintf("could not attach XDP program: %v", err)) + logger.Error(fmt.Sprintf("Preparing XDP objects: %v", err)) return } - defer l.Close() + defer x.Close() key := uint32(0) - err = objs.BandwidthLimitMap.Update(key, b.Limit, ebpf.UpdateAny) + err = x.BpfObjs.BandwidthLimitMap.Update(key, b.Limit, ebpf.UpdateAny) if err != nil { logger.Error(fmt.Sprintf("could not update bandwidth limit rate: %v", err)) return @@ -57,6 +43,13 @@ func (b *Bandwidth) Start(ctx context.Context, logger *zap.Logger) { b.ready = true <-ctx.Done() + // Update the map with a rate of 0 to disable the bandwidth limiter. + err = x.BpfObjs.BandwidthLimitMap.Update(key, 0, ebpf.UpdateAny) + if err != nil { + logger.Error(fmt.Sprintf("could not update bandwidth limit rate to zero: %v", err)) + return + } + b.ready = false logger.Info(fmt.Sprintf("Bandwidth limiter stopped on device %q", b.NetworkInterface.Name)) } diff --git a/xdp/bandwidth/bpf_bpfeb.go b/xdp/bpf_bpfeb.go similarity index 91% rename from xdp/bandwidth/bpf_bpfeb.go rename to xdp/bpf_bpfeb.go index 62d2922..e0f55b2 100644 --- a/xdp/bandwidth/bpf_bpfeb.go +++ b/xdp/bpf_bpfeb.go @@ -1,7 +1,7 @@ // Code generated by bpf2go; DO NOT EDIT. //go:build arm64be || armbe || mips || mips64 || mips64p32 || ppc64 || s390 || s390x || sparc || sparc64 -package bandwidth +package xdp import ( "bytes" @@ -53,7 +53,7 @@ type bpfSpecs struct { // // It can be passed ebpf.CollectionSpec.Assign. type bpfProgramSpecs struct { - XdpBandwidthLimit *ebpf.ProgramSpec `ebpf:"xdp_bandwidth_limit"` + XdpMain *ebpf.ProgramSpec `ebpf:"xdp_main"` } // bpfMapSpecs contains maps before they are loaded into the kernel. @@ -63,6 +63,7 @@ type bpfMapSpecs struct { BandwidthLimitMap *ebpf.MapSpec `ebpf:"bandwidth_limit_map"` ByteCounter *ebpf.MapSpec `ebpf:"byte_counter"` LastPacketTimestamp *ebpf.MapSpec `ebpf:"last_packet_timestamp"` + PacketlossRateMap *ebpf.MapSpec `ebpf:"packetloss_rate_map"` } // bpfObjects contains all objects after they have been loaded into the kernel. @@ -87,6 +88,7 @@ type bpfMaps struct { BandwidthLimitMap *ebpf.Map `ebpf:"bandwidth_limit_map"` ByteCounter *ebpf.Map `ebpf:"byte_counter"` LastPacketTimestamp *ebpf.Map `ebpf:"last_packet_timestamp"` + PacketlossRateMap *ebpf.Map `ebpf:"packetloss_rate_map"` } func (m *bpfMaps) Close() error { @@ -94,6 +96,7 @@ func (m *bpfMaps) Close() error { m.BandwidthLimitMap, m.ByteCounter, m.LastPacketTimestamp, + m.PacketlossRateMap, ) } @@ -101,12 +104,12 @@ func (m *bpfMaps) Close() error { // // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfPrograms struct { - XdpBandwidthLimit *ebpf.Program `ebpf:"xdp_bandwidth_limit"` + XdpMain *ebpf.Program `ebpf:"xdp_main"` } func (p *bpfPrograms) Close() error { return _BpfClose( - p.XdpBandwidthLimit, + p.XdpMain, ) } diff --git a/xdp/bandwidth/bpf_bpfel.go b/xdp/bpf_bpfel.go similarity index 91% rename from xdp/bandwidth/bpf_bpfel.go rename to xdp/bpf_bpfel.go index 880e0e9..fa12801 100644 --- a/xdp/bandwidth/bpf_bpfel.go +++ b/xdp/bpf_bpfel.go @@ -1,7 +1,7 @@ // Code generated by bpf2go; DO NOT EDIT. //go:build 386 || amd64 || amd64p32 || arm || arm64 || loong64 || mips64le || mips64p32le || mipsle || ppc64le || riscv64 -package bandwidth +package xdp import ( "bytes" @@ -53,7 +53,7 @@ type bpfSpecs struct { // // It can be passed ebpf.CollectionSpec.Assign. type bpfProgramSpecs struct { - XdpBandwidthLimit *ebpf.ProgramSpec `ebpf:"xdp_bandwidth_limit"` + XdpMain *ebpf.ProgramSpec `ebpf:"xdp_main"` } // bpfMapSpecs contains maps before they are loaded into the kernel. @@ -63,6 +63,7 @@ type bpfMapSpecs struct { BandwidthLimitMap *ebpf.MapSpec `ebpf:"bandwidth_limit_map"` ByteCounter *ebpf.MapSpec `ebpf:"byte_counter"` LastPacketTimestamp *ebpf.MapSpec `ebpf:"last_packet_timestamp"` + PacketlossRateMap *ebpf.MapSpec `ebpf:"packetloss_rate_map"` } // bpfObjects contains all objects after they have been loaded into the kernel. @@ -87,6 +88,7 @@ type bpfMaps struct { BandwidthLimitMap *ebpf.Map `ebpf:"bandwidth_limit_map"` ByteCounter *ebpf.Map `ebpf:"byte_counter"` LastPacketTimestamp *ebpf.Map `ebpf:"last_packet_timestamp"` + PacketlossRateMap *ebpf.Map `ebpf:"packetloss_rate_map"` } func (m *bpfMaps) Close() error { @@ -94,6 +96,7 @@ func (m *bpfMaps) Close() error { m.BandwidthLimitMap, m.ByteCounter, m.LastPacketTimestamp, + m.PacketlossRateMap, ) } @@ -101,12 +104,12 @@ func (m *bpfMaps) Close() error { // // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfPrograms struct { - XdpBandwidthLimit *ebpf.Program `ebpf:"xdp_bandwidth_limit"` + XdpMain *ebpf.Program `ebpf:"xdp_main"` } func (p *bpfPrograms) Close() error { return _BpfClose( - p.XdpBandwidthLimit, + p.XdpMain, ) } diff --git a/xdp/kerns/main.c b/xdp/kerns/main.c new file mode 100644 index 0000000..4378fbf --- /dev/null +++ b/xdp/kerns/main.c @@ -0,0 +1,17 @@ +// go:build ignore + +#include "xdp_bandwidth.c" +#include "xdp_packetloss.c" + +char _license[] SEC("license") = "GPL"; + +SEC("xdp") +int xdp_main(struct xdp_md *ctx) +{ + int action = xdp_packetloss(ctx); + if (action != XDP_PASS) + { + return action; + } + return xdp_bandwidth_limit(ctx); +} \ No newline at end of file diff --git a/xdp/bandwidth/kerns/xdp_bandwidth.c b/xdp/kerns/xdp_bandwidth.c similarity index 80% rename from xdp/bandwidth/kerns/xdp_bandwidth.c rename to xdp/kerns/xdp_bandwidth.c index 8317ffa..0e6065b 100644 --- a/xdp/bandwidth/kerns/xdp_bandwidth.c +++ b/xdp/kerns/xdp_bandwidth.c @@ -32,11 +32,24 @@ struct __uint(max_entries, MAX_MAP_ENTRIES); } bandwidth_limit_map SEC(".maps"); -SEC("xdp") int xdp_bandwidth_limit(struct xdp_md *ctx) { - __u64 current_timestamp = bpf_ktime_get_ns(); __u32 key = 0; + __u64 *bandwidth_limit_ptr = bpf_map_lookup_elem(&bandwidth_limit_map, &key); + if (!bandwidth_limit_ptr) + { + // if it has not set by the user space program, + // or the service is not started yet + return XDP_PASS; + } + + if (*bandwidth_limit_ptr == 0) + { + // If the service is stopped + return XDP_PASS; + } + + __u64 current_timestamp = bpf_ktime_get_ns(); __u64 *last_time_window_start = bpf_map_lookup_elem(&last_packet_timestamp, &key); if (!last_time_window_start) { @@ -65,16 +78,9 @@ int xdp_bandwidth_limit(struct xdp_md *ctx) bpf_map_update_elem(&last_packet_timestamp, &key, ¤t_timestamp, BPF_ANY); } - // Look it up only once for performance purposes - static __u64 allowed_bytes = 0; // number of bytes per window - if (allowed_bytes == 0) - { - __u64 *bandwidth_limit_ptr = bpf_map_lookup_elem(&bandwidth_limit_map, &key); - if (!bandwidth_limit_ptr) - return XDP_ABORTED; - - allowed_bytes = (*bandwidth_limit_ptr / 8 * TIME_WINDOW_SEC); // divide by 8 to convert from bits to bytes - } + // number of bytes per window + // divide by 8 to convert from bits to bytes + __u64 allowed_bytes = (*bandwidth_limit_ptr / 8 * TIME_WINDOW_SEC); __u64 *accumulated_bytes = bpf_map_lookup_elem(&byte_counter, &key); if (!accumulated_bytes) @@ -85,5 +91,3 @@ int xdp_bandwidth_limit(struct xdp_md *ctx) return XDP_PASS; } - -char _license[] SEC("license") = "GPL"; diff --git a/xdp/kerns/xdp_packetloss.c b/xdp/kerns/xdp_packetloss.c new file mode 100644 index 0000000..4e98981 --- /dev/null +++ b/xdp/kerns/xdp_packetloss.c @@ -0,0 +1,38 @@ +// go:build ignore +#include +#include +#include + +#define MAX_MAP_ENTRIES 1 +struct +{ + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __type(key, __u32); + __type(value, __s32); + __uint(max_entries, MAX_MAP_ENTRIES); +} packetloss_rate_map SEC(".maps"); + +int xdp_packetloss(struct xdp_md *ctx) +{ + __u32 key = 0; + __s32 *drop_rate_ptr = bpf_map_lookup_elem(&packetloss_rate_map, &key); + if (!drop_rate_ptr) + { + // if it has not set by the user space program, + // or the service is not started yet + return XDP_PASS; + } + + if (*drop_rate_ptr == 0) + { + // If the service is stopped + return XDP_PASS; + } + + if (bpf_get_prandom_u32() % 100 < *drop_rate_ptr) + { + return XDP_DROP; + } + + return XDP_PASS; +} \ No newline at end of file diff --git a/xdp/packetloss/bpf_bpfeb.go b/xdp/packetloss/bpf_bpfeb.go deleted file mode 100644 index 6324dfd..0000000 --- a/xdp/packetloss/bpf_bpfeb.go +++ /dev/null @@ -1,119 +0,0 @@ -// Code generated by bpf2go; DO NOT EDIT. -//go:build arm64be || armbe || mips || mips64 || mips64p32 || ppc64 || s390 || s390x || sparc || sparc64 - -package packetloss - -import ( - "bytes" - _ "embed" - "fmt" - "io" - - "github.com/cilium/ebpf" -) - -// loadBpf returns the embedded CollectionSpec for bpf. -func loadBpf() (*ebpf.CollectionSpec, error) { - reader := bytes.NewReader(_BpfBytes) - spec, err := ebpf.LoadCollectionSpecFromReader(reader) - if err != nil { - return nil, fmt.Errorf("can't load bpf: %w", err) - } - - return spec, err -} - -// loadBpfObjects loads bpf and converts it into a struct. -// -// The following types are suitable as obj argument: -// -// *bpfObjects -// *bpfPrograms -// *bpfMaps -// -// See ebpf.CollectionSpec.LoadAndAssign documentation for details. -func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error { - spec, err := loadBpf() - if err != nil { - return err - } - - return spec.LoadAndAssign(obj, opts) -} - -// bpfSpecs contains maps and programs before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type bpfSpecs struct { - bpfProgramSpecs - bpfMapSpecs -} - -// bpfSpecs contains programs before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type bpfProgramSpecs struct { - XdpDropPercent *ebpf.ProgramSpec `ebpf:"xdp_drop_percent"` -} - -// bpfMapSpecs contains maps before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type bpfMapSpecs struct { - DropRateMap *ebpf.MapSpec `ebpf:"drop_rate_map"` -} - -// bpfObjects contains all objects after they have been loaded into the kernel. -// -// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. -type bpfObjects struct { - bpfPrograms - bpfMaps -} - -func (o *bpfObjects) Close() error { - return _BpfClose( - &o.bpfPrograms, - &o.bpfMaps, - ) -} - -// bpfMaps contains all maps after they have been loaded into the kernel. -// -// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. -type bpfMaps struct { - DropRateMap *ebpf.Map `ebpf:"drop_rate_map"` -} - -func (m *bpfMaps) Close() error { - return _BpfClose( - m.DropRateMap, - ) -} - -// bpfPrograms contains all programs after they have been loaded into the kernel. -// -// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. -type bpfPrograms struct { - XdpDropPercent *ebpf.Program `ebpf:"xdp_drop_percent"` -} - -func (p *bpfPrograms) Close() error { - return _BpfClose( - p.XdpDropPercent, - ) -} - -func _BpfClose(closers ...io.Closer) error { - for _, closer := range closers { - if err := closer.Close(); err != nil { - return err - } - } - return nil -} - -// Do not access this directly. -// -//go:embed bpf_bpfeb.o -var _BpfBytes []byte diff --git a/xdp/packetloss/bpf_bpfel.go b/xdp/packetloss/bpf_bpfel.go deleted file mode 100644 index ea3f894..0000000 --- a/xdp/packetloss/bpf_bpfel.go +++ /dev/null @@ -1,119 +0,0 @@ -// Code generated by bpf2go; DO NOT EDIT. -//go:build 386 || amd64 || amd64p32 || arm || arm64 || loong64 || mips64le || mips64p32le || mipsle || ppc64le || riscv64 - -package packetloss - -import ( - "bytes" - _ "embed" - "fmt" - "io" - - "github.com/cilium/ebpf" -) - -// loadBpf returns the embedded CollectionSpec for bpf. -func loadBpf() (*ebpf.CollectionSpec, error) { - reader := bytes.NewReader(_BpfBytes) - spec, err := ebpf.LoadCollectionSpecFromReader(reader) - if err != nil { - return nil, fmt.Errorf("can't load bpf: %w", err) - } - - return spec, err -} - -// loadBpfObjects loads bpf and converts it into a struct. -// -// The following types are suitable as obj argument: -// -// *bpfObjects -// *bpfPrograms -// *bpfMaps -// -// See ebpf.CollectionSpec.LoadAndAssign documentation for details. -func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error { - spec, err := loadBpf() - if err != nil { - return err - } - - return spec.LoadAndAssign(obj, opts) -} - -// bpfSpecs contains maps and programs before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type bpfSpecs struct { - bpfProgramSpecs - bpfMapSpecs -} - -// bpfSpecs contains programs before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type bpfProgramSpecs struct { - XdpDropPercent *ebpf.ProgramSpec `ebpf:"xdp_drop_percent"` -} - -// bpfMapSpecs contains maps before they are loaded into the kernel. -// -// It can be passed ebpf.CollectionSpec.Assign. -type bpfMapSpecs struct { - DropRateMap *ebpf.MapSpec `ebpf:"drop_rate_map"` -} - -// bpfObjects contains all objects after they have been loaded into the kernel. -// -// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. -type bpfObjects struct { - bpfPrograms - bpfMaps -} - -func (o *bpfObjects) Close() error { - return _BpfClose( - &o.bpfPrograms, - &o.bpfMaps, - ) -} - -// bpfMaps contains all maps after they have been loaded into the kernel. -// -// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. -type bpfMaps struct { - DropRateMap *ebpf.Map `ebpf:"drop_rate_map"` -} - -func (m *bpfMaps) Close() error { - return _BpfClose( - m.DropRateMap, - ) -} - -// bpfPrograms contains all programs after they have been loaded into the kernel. -// -// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. -type bpfPrograms struct { - XdpDropPercent *ebpf.Program `ebpf:"xdp_drop_percent"` -} - -func (p *bpfPrograms) Close() error { - return _BpfClose( - p.XdpDropPercent, - ) -} - -func _BpfClose(closers ...io.Closer) error { - for _, closer := range closers { - if err := closer.Close(); err != nil { - return err - } - } - return nil -} - -// Do not access this directly. -// -//go:embed bpf_bpfel.o -var _BpfBytes []byte diff --git a/xdp/packetloss/kerns/xdp_drop_percent.c b/xdp/packetloss/kerns/xdp_drop_percent.c deleted file mode 100644 index 59c478c..0000000 --- a/xdp/packetloss/kerns/xdp_drop_percent.c +++ /dev/null @@ -1,38 +0,0 @@ -// go:build ignore -#include -#include -#include - -#define MAX_MAP_ENTRIES 1 -struct -{ - __uint(type, BPF_MAP_TYPE_LRU_HASH); - __type(key, __u32); - __type(value, __s32); - __uint(max_entries, MAX_MAP_ENTRIES); -} drop_rate_map SEC(".maps"); - -SEC("xdp") -int xdp_drop_percent(struct xdp_md *ctx) -{ - // the map is looked up only once for performance purposes - static __s32 drop_rate = -1; - if (drop_rate == -1) - { - __u32 key = 0; - __s32 *drop_rate_ptr = bpf_map_lookup_elem(&drop_rate_map, &key); - if (!drop_rate_ptr) - return XDP_ABORTED; - - drop_rate = *drop_rate_ptr; - } - - if (drop_rate > 0 && bpf_get_prandom_u32() % 100 < drop_rate) - { - return XDP_DROP; - } - - return XDP_PASS; -} - -char _license[] SEC("license") = "GPL"; \ No newline at end of file diff --git a/xdp/packetloss/packetloss.go b/xdp/packetloss/packetloss.go index ee44c83..8510e02 100644 --- a/xdp/packetloss/packetloss.go +++ b/xdp/packetloss/packetloss.go @@ -7,12 +7,9 @@ import ( "github.com/celestiaorg/bittwister/xdp" "github.com/cilium/ebpf" - "github.com/cilium/ebpf/link" "go.uber.org/zap" ) -//go:generate go run github.com/cilium/ebpf/cmd/bpf2go bpf kerns/xdp_drop_percent.c -- -I../headers - type PacketLoss struct { NetworkInterface *net.Interface PacketLossRate int32 @@ -22,28 +19,17 @@ type PacketLoss struct { var _ xdp.XdpLoader = (*PacketLoss)(nil) func (p *PacketLoss) Start(ctx context.Context, logger *zap.Logger) { - // Load pre-compiled programs into the kernel. - objs := bpfObjects{} - if err := loadBpfObjects(&objs, nil); err != nil { - logger.Error(fmt.Sprintf("loading objects: %v", err)) - return - } - defer objs.Close() - - l, err := link.AttachXDP(link.XDPOptions{ - Program: objs.XdpDropPercent, - Interface: p.NetworkInterface.Index, - }) + x, err := xdp.GetPreparedXdpObject(p.NetworkInterface.Index) if err != nil { - logger.Error(fmt.Sprintf("could not attach XDP program: %v", err)) + logger.Error(fmt.Sprintf("Preparing XDP objects: %v", err)) return } - defer l.Close() + defer x.Close() key := uint32(0) - err = objs.DropRateMap.Update(key, p.PacketLossRate, ebpf.UpdateAny) + err = x.BpfObjs.PacketlossRateMap.Update(key, p.PacketLossRate, ebpf.UpdateAny) if err != nil { - logger.Error(fmt.Sprintf("could not update drop rate: %v", err)) + logger.Error(fmt.Sprintf("could not update packetloss drop rate: %v", err)) return } @@ -57,6 +43,13 @@ func (p *PacketLoss) Start(ctx context.Context, logger *zap.Logger) { p.ready = true <-ctx.Done() + // Update the map with a rate of 0 to disable the packetloss. + err = x.BpfObjs.PacketlossRateMap.Update(key, 0, ebpf.UpdateAny) + if err != nil { + logger.Error(fmt.Sprintf("could not update packetloss drop rate to zero: %v", err)) + return + } + p.ready = false logger.Info(fmt.Sprintf("Packetloss stopped on device %q", p.NetworkInterface.Name)) } diff --git a/xdp/types.go b/xdp/types.go index 1eb168b..a1c7091 100644 --- a/xdp/types.go +++ b/xdp/types.go @@ -2,11 +2,71 @@ package xdp import ( "context" + "fmt" + "sync" + "github.com/cilium/ebpf/link" "go.uber.org/zap" ) +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go bpf kerns/main.c -- -I../headers + type XdpLoader interface { Start(ctx context.Context, logger *zap.Logger) Ready() bool } + +type XdpObject struct { + BpfObjs bpfObjects + Link link.Link + totalServices int32 + mu sync.Mutex +} + +var xdpObject XdpObject + +func GetPreparedXdpObject(netInterfaceIndex int) (*XdpObject, error) { + + xdpObject.mu.Lock() + defer xdpObject.mu.Unlock() + + // We add this once, so we know how many services are using this object. + xdpObject.totalServices++ + + if xdpObject.Link != nil { + return &xdpObject, nil + } + // Load pre-compiled programs into the kernel. + err := loadBpfObjects(&xdpObject.BpfObjs, nil) + if err != nil { + return nil, fmt.Errorf("could not load XDP program: %w", err) + } + + xdpObject.Link, err = link.AttachXDP(link.XDPOptions{ + Program: xdpObject.BpfObjs.XdpMain, + Interface: netInterfaceIndex, + }) + + if err != nil { + return nil, fmt.Errorf("could not attach XDP program: %w", err) + } + return &xdpObject, nil +} + +func (x *XdpObject) Close() error { + x.mu.Lock() + defer x.mu.Unlock() + + // The object is actually closed when all services using it are closed. + x.totalServices-- + if x.totalServices > 0 { + return nil + } + + if x.Link != nil { + if err := x.Link.Close(); err != nil { + return err + } + } + return x.BpfObjs.Close() +} From 46e3d671d04071be16be018574de735520599f0a Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Thu, 30 Nov 2023 13:47:26 +0100 Subject: [PATCH 2/2] fix: stopping service with API issue --- xdp/bandwidth/bandwidth.go | 3 ++- xdp/packetloss/packetloss.go | 3 ++- xdp/types.go | 13 ++++++++----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/xdp/bandwidth/bandwidth.go b/xdp/bandwidth/bandwidth.go index 0c7970f..4b97027 100644 --- a/xdp/bandwidth/bandwidth.go +++ b/xdp/bandwidth/bandwidth.go @@ -44,7 +44,8 @@ func (b *Bandwidth) Start(ctx context.Context, logger *zap.Logger) { <-ctx.Done() // Update the map with a rate of 0 to disable the bandwidth limiter. - err = x.BpfObjs.BandwidthLimitMap.Update(key, 0, ebpf.UpdateAny) + zero := int64(0) + err = x.BpfObjs.BandwidthLimitMap.Update(key, zero, ebpf.UpdateAny) if err != nil { logger.Error(fmt.Sprintf("could not update bandwidth limit rate to zero: %v", err)) return diff --git a/xdp/packetloss/packetloss.go b/xdp/packetloss/packetloss.go index 8510e02..0fe0f6f 100644 --- a/xdp/packetloss/packetloss.go +++ b/xdp/packetloss/packetloss.go @@ -44,7 +44,8 @@ func (p *PacketLoss) Start(ctx context.Context, logger *zap.Logger) { <-ctx.Done() // Update the map with a rate of 0 to disable the packetloss. - err = x.BpfObjs.PacketlossRateMap.Update(key, 0, ebpf.UpdateAny) + zero := int32(0) + err = x.BpfObjs.PacketlossRateMap.Update(key, zero, ebpf.UpdateAny) if err != nil { logger.Error(fmt.Sprintf("could not update packetloss drop rate to zero: %v", err)) return diff --git a/xdp/types.go b/xdp/types.go index a1c7091..a86843e 100644 --- a/xdp/types.go +++ b/xdp/types.go @@ -17,10 +17,11 @@ type XdpLoader interface { } type XdpObject struct { - BpfObjs bpfObjects - Link link.Link - totalServices int32 - mu sync.Mutex + BpfObjs bpfObjects + Link link.Link + totalServices int32 + mu sync.Mutex + netInterfaceIndex int } var xdpObject XdpObject @@ -33,9 +34,11 @@ func GetPreparedXdpObject(netInterfaceIndex int) (*XdpObject, error) { // We add this once, so we know how many services are using this object. xdpObject.totalServices++ - if xdpObject.Link != nil { + if xdpObject.Link != nil && xdpObject.netInterfaceIndex == netInterfaceIndex { return &xdpObject, nil } + xdpObject.netInterfaceIndex = netInterfaceIndex + // Load pre-compiled programs into the kernel. err := loadBpfObjects(&xdpObject.BpfObjs, nil) if err != nil {