smoke: make tests easier to debug

Signed-off-by: Robin Jarry <rjarry@redhat.com>
DPDK · May 31, 2024 · 2e45370 · 2e45370
1 parent f82cb3b
commit 2e45370
Show file tree

Hide file tree

Showing 7 changed files with 35 additions and 19 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,8 +13,9 @@ jobs:
   build:
     runs-on: ubuntu-24.04
     env:
-      SANITIZE: address
+      SANITIZE: none
       BUILDTYPE: debugoptimized
+      MESON_EXTRA_OPTS: -Dc_args=-DTRACE_PACKETS
     steps:
       - name: install system dependencies
         run: |
@@ -48,7 +49,7 @@ jobs:
           set -xe
           sudo apt-get update -qy
           sudo apt-get install -qy \
-            git socat tcpdump \
+            git socat tcpdump gdb binutils \
             iproute2 iputils-ping libasan8 libedit2 \
             libevent-2.1-7t64 libsmartcols1 libnuma1
       - uses: actions/checkout@v4
@@ -57,7 +58,7 @@ jobs:
           path: build
           merge-multiple: true
       - run: chmod -c a+x build/br* build/subprojects/ecoli/src/libecoli.so
-      - run: sudo smoke/run.sh build
+      - run: sudo smoke/run.sh --coredump build
   lint:
     runs-on: ubuntu-latest
     container: fedora:latest

diff --git a/GNUmakefile b/GNUmakefile
@@ -40,6 +40,7 @@ install: $(BUILDDIR)/build.ninja
 	$Q ninja -C $(BUILDDIR) install $(ninja_opts)
 
 meson_opts := --buildtype=$(BUILDTYPE) --werror --warnlevel=2 -Db_sanitize=$(SANITIZE)
+meson_opts += $(MESON_EXTRA_OPTS)
 
 $(BUILDDIR)/build.ninja:
 	meson setup $(BUILDDIR) $(meson_opts)

diff --git a/smoke/_init.sh b/smoke/_init.sh
@@ -5,12 +5,9 @@ set -e
 
 cleanup() {
 	set +e
-	br-cli show stats software
+	sh -x $tmp/cleanup
 	kill -INT %?br
 	wait
-	if [ -r $tmp/cleanup ]; then
-		sh -x $tmp/cleanup
-	fi
 	rm -rf -- "$tmp"
 }
 
@@ -22,6 +19,11 @@ run_id=$(echo $SRANDOM$SRANDOM | base32 -w6 | tr '[:upper:]' '[:lower:]' | head
 export BR_SOCK_PATH=$tmp/br.sock
 export PATH=$builddir:$PATH
 
+cat > $tmp/cleanup <<EOF
+br-cli show stats software
+br-cli show interface all
+EOF
+
 set -x
 
 br -tv &

diff --git a/smoke/ip_forward_test.sh b/smoke/ip_forward_test.sh
@@ -11,8 +11,7 @@ br-cli add interface port $p0 devargs net_tap0,iface=$p0
 br-cli add interface port $p1 devargs net_tap1,iface=$p1
 br-cli add ip address 172.16.0.1/24 iface $p0
 br-cli add ip address 172.16.1.1/24 iface $p1
-br-cli show interface all
-br-cli show ip address
+echo br-cli show ip nexthop >> $tmp/cleanup
 
 for n in 0 1; do
 	p=$run_id$n
@@ -22,6 +21,7 @@ for n in 0 1; do
 	ip -n $p link set $p up
 	ip -n $p addr add 172.16.$n.2/24 dev $p
 	ip -n $p route add default via 172.16.$n.1
+	ip -n $p addr show
 done
 
 ip netns exec $p0 ping -i0.01 -c3 172.16.1.2

diff --git a/smoke/run.sh b/smoke/run.sh
@@ -3,10 +3,16 @@
 # Copyright (c) 2024 Robin Jarry
 
 here=$(dirname $0)
+if [ "$1" = "--coredump" ]; then
+	coredump=true
+	ulimit -c unlimited
+	trap "sysctl -qw 'kernel.core_pattern=$(sysctl -n kernel.core_pattern)'" EXIT
+	sysctl -qw kernel.core_pattern=/tmp/br-core.%e.%p
+	shift
+fi
 builddir=${1?build dir}
 log=$(mktemp)
 result=0
-trap "if [ -s $log ]; then cat $log; fi; rm -f $log" EXIT
 
 run() {
 	local script="$1"
@@ -26,6 +32,16 @@ for script in $here/*_test.sh; do
 		if ! "$script" "$builddir"; then
 			res=FAILED
 		fi
+		for core in /tmp/br-core.*.*; do
+			[ -s "$core" ] || continue
+			binary=$(file -b "$core" | sed -En "s/.*, execfn: '([^']+)',.*/\\1/p")
+			[ -x "$binary" ] || continue
+			gdb -ex 'info threads' \
+				-ex 'thread apply all bt full' \
+				-ex 'quit' \
+				"$binary" -c "$core" || true
+			rm -f "$core"
+		done
 		end=$(date +%s)
 		duration=$(date -d "@$((end - start))" "+%Mm%Ss")
 		echo "-----------------------------------------------------"

diff --git a/smoke/vlan_forward_test.sh b/smoke/vlan_forward_test.sh
@@ -15,8 +15,7 @@ br-cli add interface vlan $v0 parent $p0 vlan_id 42
 br-cli add interface vlan $v1 parent $p1 vlan_id 43
 br-cli add ip address 172.16.0.1/24 iface $v0
 br-cli add ip address 172.16.1.1/24 iface $v1
-br-cli show interface all
-br-cli show ip address
+echo br-cli show ip nexthop >> $tmp/cleanup
 
 for n in 0 1; do
 	p=$run_id$n
@@ -29,6 +28,7 @@ for n in 0 1; do
 	ip -n $p link set $v up
 	ip -n $p addr add 172.16.$n.2/24 dev $v
 	ip -n $p route add default via 172.16.$n.1
+	ip -n $p addr show
 done
 
 ip netns exec $p0 ping -i0.01 -c3 172.16.1.2

diff --git a/smoke/vrf_forward_test.sh b/smoke/vrf_forward_test.sh
@@ -17,8 +17,8 @@ br-cli add ip address 172.16.0.1/24 iface $p0
 br-cli add ip address 172.16.1.1/24 iface $p1
 br-cli add ip address 172.16.0.1/24 iface $p2
 br-cli add ip address 172.16.1.1/24 iface $p3
-br-cli show ip address vrf 1
-br-cli show ip address vrf 2
+echo br-cli show ip nexthop vrf 1 >> $tmp/cleanup
+echo br-cli show ip nexthop vrf 2 >> $tmp/cleanup
 
 for n in 0 1 2 3; do
 	p=$run_id$n
@@ -28,6 +28,7 @@ for n in 0 1 2 3; do
 	ip -n $p link set $p up
 	ip -n $p addr add 172.16.$((n % 2)).2/16 dev $p
 	ip -n $p route add default via 172.16.$((n % 2)).1
+	ip -n $p addr show
 done
 
 tcpdump_opts="--immediate-mode --no-promiscuous-mode"
@@ -43,8 +44,3 @@ timeout 3 ip netns exec $p3 \
 sleep 1
 ip netns exec $p2 ping -i0.01 -c3 172.16.1.2
 wait -f %?tcpdump
-
-br-cli show ip route vrf 1
-br-cli show ip route vrf 2
-br-cli show ip nexthop vrf 1
-br-cli show ip nexthop vrf 2