From 93c654679cd01d09dc859a4c295715cef431346c Mon Sep 17 00:00:00 2001 From: Louis Royer Date: Thu, 28 Nov 2024 15:58:01 +0100 Subject: [PATCH] Uplink working --- config/config.yaml | 2 +- internal/app/gtp.go | 21 ++++-- internal/app/pdu_session.go | 2 +- internal/app/pdu_sessions_manager.go | 102 +++++++++++++++++++-------- internal/app/radio.go | 13 ++-- internal/app/radio_daemon.go | 99 ++++++++++++++++++++++++++ internal/app/setup.go | 14 ++-- internal/app/tun.go | 80 --------------------- 8 files changed, 203 insertions(+), 130 deletions(-) create mode 100644 internal/app/radio_daemon.go delete mode 100644 internal/app/tun.go diff --git a/config/config.yaml b/config/config.yaml index 17e207e..6020bd0 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -8,4 +8,4 @@ cp: gtp: "198.51.100.10" logger: - level: "debug" + level: "trace" diff --git a/internal/app/gtp.go b/internal/app/gtp.go index d884318..f710b25 100644 --- a/internal/app/gtp.go +++ b/internal/app/gtp.go @@ -11,7 +11,6 @@ import ( "net/netip" "github.com/sirupsen/logrus" - "github.com/songgao/water" "github.com/wmnsk/go-gtp/gtpv1" "github.com/wmnsk/go-gtp/gtpv1/message" ) @@ -31,7 +30,7 @@ func (s *Setup) StartGtpUProtocolEntity(ctx context.Context, ipAddress netip.Add defer uConn.Close() uConn.DisableErrorIndication() uConn.AddHandler(message.MsgTypeTPDU, func(c gtpv1.Conn, senderAddr net.Addr, msg message.Message) error { - return tpduHandler(c, senderAddr, msg, s.tunInterface, s.radio) + return tpduHandler(c, senderAddr, msg, s.psMan, s.rDaemon) }) go func(ctx context.Context) error { if err := uConn.ListenAndServe(ctx); err != nil { @@ -43,9 +42,17 @@ func (s *Setup) StartGtpUProtocolEntity(ctx context.Context, ipAddress netip.Add return nil } -// handle GTP PDU (Uplink) -func tpduHandler(c gtpv1.Conn, senderAddr net.Addr, msg message.Message, tuniface *water.Interface, radio *Radio) error { - // - logrus.Error("Not implemented") - return nil +// handle GTP PDU (Downlink) +func tpduHandler(c gtpv1.Conn, senderAddr net.Addr, msg message.Message, psMan *PduSessionsManager, rDaemon *RadioDaemon) error { + teid := msg.TEID() + ue, err := psMan.GetUECtrl(teid) + if err != nil { + return err + } + packet := make([]byte, msg.MarshalLen()) + err = msg.MarshalTo(packet) + if err != nil { + return err + } + return rDaemon.WriteDownlink(packet, ue) } diff --git a/internal/app/pdu_session.go b/internal/app/pdu_session.go index 7ac4326..d25e021 100644 --- a/internal/app/pdu_session.go +++ b/internal/app/pdu_session.go @@ -118,7 +118,7 @@ func (p *PduSessions) N2EstablishmentRequest(c *gin.Context) { "uplink-teid": ps.UplinkTeid, }).Info("New PDU Session establishment Request") // allocate downlink teid - downlinkTeid, err := p.manager.NewPduSession(c, ps.UeInfo.Addr, ps.Upf, ps.UplinkTeid) + downlinkTeid, err := p.manager.NewPduSession(c, ps.UeInfo.Addr, ps.UeInfo.Header.Ue, ps.Upf, ps.UplinkTeid) if err != nil { c.JSON(http.StatusInternalServerError, jsonapi.MessageWithError{Message: "could create PDU Session", Error: err}) return diff --git a/internal/app/pdu_sessions_manager.go b/internal/app/pdu_sessions_manager.go index 02ef4f0..b70900f 100644 --- a/internal/app/pdu_sessions_manager.go +++ b/internal/app/pdu_sessions_manager.go @@ -7,43 +7,94 @@ package app import ( "context" + "fmt" "math/rand" + "net" "net/netip" "sync" "time" + + "github.com/nextmn/json-api/jsonapi" + + "github.com/sirupsen/logrus" + "github.com/wmnsk/go-gtp/gtpv1" + "github.com/wmnsk/go-gtp/gtpv1/message" ) type PduSessionsManager struct { sync.Mutex - radio *Radio - Downlink map[uint32]netip.Addr // teid: ue 5G ip addr - Uplink map[netip.Addr]*Fteid // ue 5G ip address: uplink fteid - - isInit bool + Downlink map[uint32]jsonapi.ControlURI // teid: UE control uri + Uplink map[netip.Addr]*Fteid // ue 5G ip address: uplink fteid + GtpAddr netip.Addr + upfs map[netip.Addr]*gtpv1.UPlaneConn } -func NewPduSessionsManager(radio *Radio) *PduSessionsManager { +func NewPduSessionsManager(gtpAddr netip.Addr) *PduSessionsManager { return &PduSessionsManager{ - radio: radio, - Downlink: make(map[uint32]netip.Addr), + Downlink: make(map[uint32]jsonapi.ControlURI), Uplink: make(map[netip.Addr]*Fteid), - isInit: false, + GtpAddr: gtpAddr, + upfs: make(map[netip.Addr]*gtpv1.UPlaneConn), } } -func (p *PduSessionsManager) Init() error { - // TODO: - // 1. create ip rule iif TUN_NAME from => table nextmn-gnb-lite-ul - // 2. create ip route default dev TUN_NAME table nextmn-gnb-lite-ul proto nextmn-gnb-lite - // 3. create ip rule iif TUN_NAME to => table nextmn-gnb-lite-dl - // 4. create ip route via table nextmn-gnb-lite-dl proto nextmn-gnb-lite - p.isInit = true - return nil +func (p *PduSessionsManager) WriteUplink(ctx context.Context, pkt []byte) error { + if len(pkt) < 20 { + logrus.Trace("too small to be an ipv4 packet") + return fmt.Errorf("Too small to be an ipv4 packet") + } + if (pkt[0] >> 4) != 4 { + logrus.Trace("not an ipv4 packet") + return fmt.Errorf("Not an ipv4 packet") + } + src := netip.AddrFrom4([4]byte{pkt[12], pkt[13], pkt[14], pkt[15]}) + fteid, ok := p.Uplink[src] + if !ok { + logrus.WithFields(logrus.Fields{ + "ue": src, + }).Trace("unknown UE") + return fmt.Errorf("Unknown UE") + } + gpdu := message.NewHeaderWithExtensionHeaders(0x30, message.MsgTypeTPDU, fteid.Teid, 0, pkt, []*message.ExtensionHeader{}...) + b, err := gpdu.Marshal() + if err != nil { + return err + } + uConn, ok := p.upfs[fteid.IpAddr] + raddr := net.UDPAddrFromAddrPort(netip.AddrPortFrom(fteid.IpAddr, GTPU_PORT)) + if !ok { + laddr := net.UDPAddrFromAddrPort(netip.AddrPortFrom(p.GtpAddr, 0)) + uConn, err = gtpv1.DialUPlane(ctx, laddr, raddr) + if err != nil { + logrus.WithFields(logrus.Fields{ + "upf": raddr, + }).Error("Failure to dial UPF") + return err + } + p.upfs[fteid.IpAddr] = uConn + go func(ctx context.Context, uConn *gtpv1.UPlaneConn) error { + select { + case <-ctx.Done(): + uConn.Close() + return ctx.Err() + } + return nil + }(ctx, uConn) + } + logrus.WithFields(logrus.Fields{ + "fteid": fteid, + }).Trace("Forwarding packet to GTP") + _, err = uConn.WriteTo(b, raddr) + return err } -func (p *PduSessionsManager) Exit() error { - return nil +func (p *PduSessionsManager) GetUECtrl(teid uint32) (jsonapi.ControlURI, error) { + ueCtrl, ok := p.Downlink[teid] + if !ok { + return ueCtrl, fmt.Errorf("Unknown UE") + } + return ueCtrl, nil } type Fteid struct { @@ -51,16 +102,13 @@ type Fteid struct { Teid uint32 } -func (p *PduSessionsManager) NewPduSession(ctx context.Context, ueIpAddr netip.Addr, upf netip.Addr, uplinkTeid uint32) (uint32, error) { +func (p *PduSessionsManager) NewPduSession(ctx context.Context, ueIpAddr netip.Addr, ueControlURI jsonapi.ControlURI, upf netip.Addr, uplinkTeid uint32) (uint32, error) { p.Lock() defer p.Unlock() - if !p.isInit { - p.Init() - } ctxTimeout, cancel := context.WithTimeout(ctx, time.Duration(time.Millisecond*10)) // 10 ms should be more than enough… defer cancel() - dlTeid, err := p.newTeidDl(ctxTimeout, ueIpAddr) + dlTeid, err := p.newTeidDl(ctxTimeout, ueControlURI) if err != nil { return dlTeid, err } @@ -68,13 +116,11 @@ func (p *PduSessionsManager) NewPduSession(ctx context.Context, ueIpAddr netip.A IpAddr: upf, Teid: uplinkTeid, } - // TODO: add route to UE - return dlTeid, err } // Warning: not thread safe -func (p *PduSessionsManager) newTeidDl(ctx context.Context, ueIpAddr netip.Addr) (uint32, error) { +func (p *PduSessionsManager) newTeidDl(ctx context.Context, ueControlURI jsonapi.ControlURI) (uint32, error) { // teid are attributed randomly, and unique per pdu session for { select { @@ -86,7 +132,7 @@ func (p *PduSessionsManager) newTeidDl(ctx context.Context, ueIpAddr netip.Addr) continue // bad luck :( } if _, exists := p.Downlink[teid]; !exists { - p.Downlink[teid] = ueIpAddr + p.Downlink[teid] = ueControlURI return teid, nil } } diff --git a/internal/app/radio.go b/internal/app/radio.go index 80c7e97..6e5e25e 100644 --- a/internal/app/radio.go +++ b/internal/app/radio.go @@ -9,6 +9,7 @@ import ( "bytes" "encoding/json" "fmt" + "net" "net/http" "net/netip" "sync" @@ -37,12 +38,16 @@ func NewRadio(control jsonapi.ControlURI, data netip.AddrPort, userAgent string) } } -func (r *Radio) GetPeer(gnb jsonapi.ControlURI) (netip.Addr, error) { - p, ok := r.peerMap.Load(gnb) +func (r *Radio) Write(pkt []byte, srv *net.UDPConn, ue jsonapi.ControlURI) error { + ueRan, ok := r.peerMap.Load(ue) if !ok { - return netip.Addr{}, fmt.Errorf("not found") + logrus.Trace("Unknown UE") + return fmt.Errorf("Unknown UE") } - return p.(netip.Addr), nil + + _, err := srv.WriteToUDPAddrPort(pkt, ueRan.(netip.AddrPort)) + + return err } // TODO: move this to json-api diff --git a/internal/app/radio_daemon.go b/internal/app/radio_daemon.go new file mode 100644 index 0000000..5719c61 --- /dev/null +++ b/internal/app/radio_daemon.go @@ -0,0 +1,99 @@ +// Copyright 2024 Louis Royer and the NextMN contributors. All rights reserved. +// Use of this source code is governed by a MIT-style license that can be +// found in the LICENSE file. +// SPDX-License-Identifier: MIT + +package app + +import ( + "context" + "fmt" + "net" + "net/netip" + + "github.com/nextmn/json-api/jsonapi" + + "github.com/sirupsen/logrus" +) + +const ( + TUN_MTU = 1400 +) + +type RadioDaemon struct { + DlQueue chan DLPkt + radio *Radio + gnbRanAddr netip.AddrPort + PduSessionsManager *PduSessionsManager + srv *net.UDPConn +} + +func NewRadioDaemon(radio *Radio, psMan *PduSessionsManager, gnbRanAddr netip.AddrPort) *RadioDaemon { + return &RadioDaemon{ + DlQueue: make(chan DLPkt), + radio: radio, + PduSessionsManager: psMan, + gnbRanAddr: gnbRanAddr, + } +} + +func (r *RadioDaemon) runUplinkDaemon(ctx context.Context, srv *net.UDPConn) error { + if srv == nil { + logrus.Error("nil server") + return fmt.Errorf("nil srv") + } + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + buf := make([]byte, TUN_MTU) + n, err := srv.Read(buf) + if err != nil { + logrus.WithError(err).Trace("error reading udp packet") + return err + } + logrus.Trace("received new packet from ue") + r.PduSessionsManager.WriteUplink(ctx, buf[:n]) + } + } + return nil +} + +type DLPkt struct { + Ue jsonapi.ControlURI + Payload []byte +} + +func (r *RadioDaemon) WriteDownlink(payload []byte, ue jsonapi.ControlURI) error { + if r.srv == nil { + return fmt.Errorf("nil srv") + } + return r.radio.Write(payload, r.srv, ue) +} + +func (r *RadioDaemon) Start(ctx context.Context) error { + srv, err := net.ListenUDP("udp", net.UDPAddrFromAddrPort(r.gnbRanAddr)) + if err != nil { + return err + } + logrus.WithFields(logrus.Fields{ + "bind-addr": r.gnbRanAddr, + }).Info("Starting Radio Simulatior") + go func(ctx context.Context, srv *net.UDPConn) error { + if srv == nil { + return fmt.Errorf("nil srv") + } + select { + case <-ctx.Done(): + srv.Close() + return ctx.Err() + } + return nil + }(ctx, srv) + go func(ctx context.Context, srv *net.UDPConn) { + defer srv.Close() + r.runUplinkDaemon(ctx, srv) + }(ctx, srv) + return nil +} diff --git a/internal/app/setup.go b/internal/app/setup.go index 1fab2c5..44ff1fd 100644 --- a/internal/app/setup.go +++ b/internal/app/setup.go @@ -9,34 +9,31 @@ import ( "context" "github.com/nextmn/gnb-lite/internal/config" - - "github.com/songgao/water" ) type Setup struct { config *config.GNBConfig httpServerEntity *HttpServerEntity radio *Radio - tunInterface *water.Interface + rDaemon *RadioDaemon psMan *PduSessionsManager } func NewSetup(config *config.GNBConfig) *Setup { radio := NewRadio(config.Control.Uri, config.Ran.BindAddr, "go-github-nextmn-gnb-lite") - psMan := NewPduSessionsManager(radio) + psMan := NewPduSessionsManager(config.Gtp) + rDaemon := NewRadioDaemon(radio, psMan, config.Ran.BindAddr) ps := NewPduSessions(config.Control.Uri, config.Cp.Uri, psMan, "go-github-nextmn-gnb-lite", config.Gtp) return &Setup{ config: config, httpServerEntity: NewHttpServerEntity(config.Control.BindAddr, radio, ps), radio: radio, + rDaemon: rDaemon, psMan: psMan, } } func (s *Setup) Init(ctx context.Context) error { - if err := s.createTun(); err != nil { - return err - } - if err := s.psMan.Init(); err != nil { + if err := s.rDaemon.Start(ctx); err != nil { return err } if err := s.StartGtpUProtocolEntity(ctx, s.config.Gtp); err != nil { @@ -60,7 +57,6 @@ func (s *Setup) Run(ctx context.Context) error { } func (s *Setup) Exit() error { - s.psMan.Exit() s.httpServerEntity.Stop() return nil } diff --git a/internal/app/tun.go b/internal/app/tun.go deleted file mode 100644 index 6abdd2a..0000000 --- a/internal/app/tun.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2024 Louis Royer and the NextMN contributors. All rights reserved. -// Use of this source code is governed by a MIT-style license that can be -// found in the LICENSE file. -// SPDX-License-Identifier: MIT - -package app - -import ( - "fmt" - "os" - "os/exec" - "strconv" - - "github.com/sirupsen/logrus" - "github.com/songgao/water" -) - -const ( - TUN_NAME = "nextmn-gnb-lite" - MTU = 1400 -) - -func (s *Setup) createTun() error { - config := water.Config{ - DeviceType: water.TUN, - } - config.Name = TUN_NAME - iface, err := water.New(config) - if nil != err { - logrus.WithError(err).Error("Unable to allocate TUN interface") - return err - } - err = runIP("link", "set", "dev", iface.Name(), "mtu", strconv.Itoa(MTU)) - if nil != err { - logrus.WithError(err).WithFields(logrus.Fields{ - "mtu": MTU, - "interface": iface.Name(), - }).Error("Unable to set MTU") - return err - } - err = runIP("link", "set", "dev", iface.Name(), "up") - if nil != err { - logrus.WithError(err).WithFields(logrus.Fields{ - "interface": iface.Name(), - }).Error("Unable to set interface up") - return err - } - s.tunInterface = iface - - err = runIPTables("-A", "OUTPUT", "-o", iface.Name(), "-p", "icmp", "--icmp-type", "redirect", "-j", "DROP") - if err != nil { - logrus.WithError(err).WithFields(logrus.Fields{"interface": iface.Name()}).Error("Error while setting iptable rule to drop icmp redirects") - return err - } - return nil -} - -// Run ip command -func runIP(args ...string) error { - cmd := exec.Command("ip", args...) - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stdout - cmd.Stdin = os.Stdin - if err := cmd.Run(); err != nil { - return fmt.Errorf("Error running %s: %s", cmd.Args, err) - } - return nil -} - -// Run iptables command -func runIPTables(args ...string) error { - cmd := exec.Command("iptables", args...) - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stdout - cmd.Stdin = os.Stdin - if err := cmd.Run(); err != nil { - return fmt.Errorf("Error running %s: %s", cmd.Args, err) - } - return nil -}