feature: multi-threaded UDP listeners

This commit adds the ability for STUNner to run UDP listeners over multiple parallel readloops. The
idea is to create a configurable number of UDP server sockets using `SO_REUSEPORT` and spawn a
separate goroutine to run a parallel readloop for each. The kernel will load-balance allocations
across the sockets/readloops per the IP 5-tuple, so the same allocation will always stay at the
same CPU. This allows UDP listeners to scale to multiple CPUs.

Note that this is enabled only for UDP at the moment: TCP, TLS and DTLS listeners spawn a
per-client readloop anyway. Also note that `SO_REUSEPORT` is not portable, so currently we enable
this only for UNIX architectures.

The feature is exposed via the command line flag `--udp-thread-num=<THREAD_NUMBER>` in
`stunnerd`, and the `UDPListenerThreadNum` field of the `Options` struct in the lib.

The commit also adds tests plus a benchmark script that tests `stunnerd` and `turncat` using
iperf. Use `./benchmark.sh udp 16 8000000` to run a benchmark with a UDP listener running 16
readloops, and `./benchmark.sh tcp 0 8000000` for a simple TCP benchmark.

benchmark.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+trap 'kill $(jobs -p)' EXIT
+RATE=1600
+THREADS=0
+
+[ -z "$1" ] && echo "usage: test2.sh <proto> [udp-thread-num] [PACKET-RATE]" && exit 1
+[ -z "$1" ] || PROTO=$1
+[ -z "$2" ] || THREADS=$2
+[ -z "$3" ] || RATE=$3
+
+go run cmd/stunnerd/main.go -l all:ERROR --udp-thread-num=${THREADS} turn://user:pass@127.0.0.1:5000?transport=${PROTO} &
+iperf -s -u -e -i 5 &
+
+go run cmd/turncat/main.go -l all:ERROR udp://127.0.0.1:4999 "turn://user:pass@127.0.0.1:5000?transport=${PROTO}" udp://localhost:5001 &
+iperf -c 127.0.0.1 -u -p 4999 -t 0 -l 100 -b $RATE &
+
+go run cmd/turncat/main.go -l all:ERROR udp://127.0.0.1:5999 "turn://user:pass@127.0.0.1:5000?transport=${PROTO}" udp://localhost:5001 &
+iperf -c 127.0.0.1 -u -p 5999 -t 0 -l 100 -b $RATE &
+
+go run cmd/turncat/main.go -l all:ERROR udp://127.0.0.1:6999 "turn://user:pass@127.0.0.1:5000?transport=${PROTO}" udp://localhost:5001 &
+iperf -c 127.0.0.1 -u -p 6999 -t 0 -l 100 -b $RATE &
+
+go run cmd/turncat/main.go -l all:ERROR udp://127.0.0.1:7999 "turn://user:pass@127.0.0.1:5000?transport=${PROTO}" udp://localhost:5001 &
+iperf -c 127.0.0.1 -u -p 7999 -t 0 -l 100 -b $RATE &
+
+go run cmd/turncat/main.go -l all:ERROR udp://127.0.0.1:8999 "turn://user:pass@127.0.0.1:5000?transport=${PROTO}" udp://localhost:5001 &
+iperf -c 127.0.0.1 -u -p 8999 -t 0 -l 100 -b $RATE &
+
+go run cmd/turncat/main.go -l all:ERROR udp://127.0.0.1:9999 "turn://user:pass@127.0.0.1:5000?transport=${PROTO}" udp://localhost:5001 &
+iperf -c 127.0.0.1 -u -p 9999 -t 0 -l 100 -b $RATE &
+
+go run cmd/turncat/main.go -l all:ERROR udp://127.0.0.1:10999 "turn://user:pass@127.0.0.1:5000?transport=${PROTO}" udp://localhost:5001 &
+iperf -c 127.0.0.1 -u -p 10999 -t 0 -l 100 -b $RATE &
+
+go run cmd/turncat/main.go -l all:ERROR udp://127.0.0.1:11999 "turn://user:pass@127.0.0.1:5000?transport=${PROTO}" udp://localhost:5001 &
+iperf -c 127.0.0.1 -u -p 11999 -t 0 -l 100 -b $RATE 
+
+exit 0

cmd/stunnerd/main.go

@@ -26,6 +26,7 @@ func main() {
 	var config = flag.StringP("config", "c", "", "Config file.")
 	var level = flag.StringP("log", "l", "", "Log level (default: all:INFO).")
 	var watch = flag.BoolP("watch", "w", false, "Watch config file for updates (default: false).")
+	var udpThreadNum = flag.IntP("udp-thread-num", "u", 0, "Number of readloop threads (CPU cores) per UDP listener. Zero disables UDP multithreading (default: 0).")
 	var dryRun = flag.BoolP("dry-run", "d", false, "Suppress side-effects, intended for testing (default: false).")
 	var verbose = flag.BoolP("verbose", "v", false, "Verbose logging, identical to <-l all:DEBUG>.")
 	flag.Parse()
@@ -40,7 +41,11 @@ func main() {
 		logLevel = *level
 	}
 
-	st := stunner.NewStunner(stunner.Options{LogLevel: logLevel, DryRun: *dryRun})
+	st := stunner.NewStunner(stunner.Options{
+		LogLevel:             logLevel,
+		DryRun:               *dryRun,
+		UDPListenerThreadNum: *udpThreadNum,
+	})
 	defer st.Close()
 
 	log := st.GetLogger().NewLogger("stunnerd")

config.go

@@ -8,12 +8,43 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/pion/transport/v2"
 	"sigs.k8s.io/yaml"
 
+	"github.com/l7mp/stunner/internal/resolver"
 	"github.com/l7mp/stunner/internal/util"
 	"github.com/l7mp/stunner/pkg/apis/v1alpha1"
 )
 
+// Options defines various options for the STUNner server.
+type Options struct {
+	// DryRun suppresses sideeffects: STUNner will not initialize listener sockets and bring up
+	// the TURN server, and it will not fire up the health-check and the metrics
+	// servers. Intended for testing, default is false.
+	DryRun bool
+	// SuppressRollback controls whether to rollback to the last working configuration after a
+	// failed reconciliation request. Default is false, which means to always do a rollback.
+	SuppressRollback bool
+	// LogLevel specifies the required loglevel for STUNner and each of its sub-objects, e.g.,
+	// "all:TRACE" will force maximal loglevel throughout, "all:ERROR,auth:TRACE,turn:DEBUG"
+	// will suppress all logs except in the authentication subsystem and the TURN protocol
+	// logic.
+	LogLevel string
+	// Resolver swaps the internal DNS resolver with a custom implementation. Intended for
+	// testing.
+	Resolver resolver.DnsResolver
+	// UDPListenerThreadNum determines the number of readloop threads spawned per UDP listener
+	// (default is 4, must be >0 integer). TURN allocations will be automatically load-balanced
+	// by the kernel UDP stack based on the client 5-tuple. This setting controls the maximum
+	// number of CPU cores UDP listeners can scale to. Note that all other listener protocol
+	// types (TCP, TLS and DTLS) use per-client threads, so this setting affects only UDP
+	// listeners. For more info see /~https://github.com/pion/turn/pull/295.
+	UDPListenerThreadNum int
+	// VNet will switch on testing mode, using a vnet.Net instance to run STUNner over an
+	// emulated data-plane.
+	Net transport.Net
+}
+
 // NewDefaultStunnerConfig builds a default configuration from a TURN server URI. Example: the URI
 // `turn://user:pass@127.0.0.1:3478?transport=udp` will be parsed into a STUNner configuration with
 // a server running on the localhost at UDP port 3478, with plain-text authentication using the

go.mod

@@ -1,12 +1,12 @@
 module github.com/l7mp/stunner
 
-go 1.17
+go 1.19
 
 require (
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/pion/dtls/v2 v2.1.5
 	github.com/pion/logging v0.2.2
-	github.com/pion/transport/v2 v2.0.2 // indirect
+	github.com/pion/transport/v2 v2.0.2
 	// replace from l7mp/turn
 	github.com/pion/turn/v2 v2.1.0
 	github.com/prometheus/client_golang v1.14.0
@@ -22,6 +22,7 @@ require (
 	github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb
 	github.com/pion/randutil v0.1.0
 	github.com/pion/transport v0.13.0
+	golang.org/x/sys v0.6.0
 )
 
 require (
@@ -58,7 +59,6 @@ require (
 	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
 	golang.org/x/net v0.7.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
-	golang.org/x/sys v0.5.0 // indirect
 	golang.org/x/term v0.5.0 // indirect
 	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect