benchmarks/tcp: set a number of channels to GOMAXPROCS

Updates #231

PiperOrigin-RevId: 289502669

2 files changed, 51 insertions, 28 deletions

diff --git a/benchmarks/tcp/tcp_benchmark.sh b/benchmarks/tcp/tcp_benchmark.sh
index 69344c9c3..e65801a7b 100755
--- a/benchmarks/tcp/tcp_benchmark.sh
+++ b/benchmarks/tcp/tcp_benchmark.sh
@@ -41,6 +41,8 @@ duplicate=0.1           # 0.1% means duplicates are 1/10x as frequent as losses.
 duration=30             # 30s is enough time to consistent results (experimentally).
 helper_dir=$(dirname $0)
 netstack_opts=
+disable_linux_gso=
+num_client_threads=1
 
 # Check for netem support.
 lsmod_output=$(lsmod | grep sch_netem)
@@ -125,6 +127,13 @@ while [ $# -gt 0 ]; do
       shift
       netstack_opts="${netstack_opts} -memprofile=$1"
       ;;
+    --disable-linux-gso)
+      disable_linux_gso=1
+      ;;
+    --num-client-threads)
+      shift
+      num_client_threads=$1
+      ;;
     --helpers)
       shift
       [ "$#" -le 0 ] && echo "no helper dir provided" && exit 1
@@ -147,6 +156,8 @@ while [ $# -gt 0 ]; do
       echo " --loss                set the loss probability (%)"
       echo " --duplicate           set the duplicate probability (%)"
       echo " --helpers             set the helper directory"
+      echo " --num-client-threads  number of parallel client threads to run"
+      echo " --disable-linux-gso   disable segmentation offload in the Linux network stack"
       echo ""
       echo "The output will of the script will be:"
       echo "  <throughput> <client-cpu-usage> <server-cpu-usage>"
@@ -301,6 +312,14 @@ fi
 # Add client and server addresses, and bring everything up.
 ${nsjoin_binary} /tmp/client.netns ip addr add ${client_addr}/${mask} dev client.0
 ${nsjoin_binary} /tmp/server.netns ip addr add ${server_addr}/${mask} dev server.0
+if [ "${disable_linux_gso}" == "1" ]; then
+  ${nsjoin_binary} /tmp/client.netns ethtool -K client.0 tso off
+  ${nsjoin_binary} /tmp/client.netns ethtool -K client.0 gro off
+  ${nsjoin_binary} /tmp/client.netns ethtool -K client.0 gso off
+  ${nsjoin_binary} /tmp/server.netns ethtool -K server.0 tso off
+  ${nsjoin_binary} /tmp/server.netns ethtool -K server.0 gso off
+  ${nsjoin_binary} /tmp/server.netns ethtool -K server.0 gro off
+fi
 ${nsjoin_binary} /tmp/client.netns ip link set client.0 up
 ${nsjoin_binary} /tmp/client.netns ip link set lo up
 ${nsjoin_binary} /tmp/server.netns ip link set server.0 up
@@ -338,7 +357,7 @@ trap cleanup EXIT
 
 # Run the benchmark, recording the results file.
 while ${nsjoin_binary} /tmp/client.netns iperf \\
-    -p ${proxy_port} -c ${client_addr} -t ${duration} -f m 2>&1 \\
+    -p ${proxy_port} -c ${client_addr} -t ${duration} -f m -P ${num_client_threads} 2>&1 \\
     | tee \$results_file \\
     | grep "connect failed" >/dev/null; do
   sleep 0.1 # Wait for all services.
diff --git a/benchmarks/tcp/tcp_proxy.go b/benchmarks/tcp/tcp_proxy.go
index 361a56755..be0d7bdd6 100644
--- a/benchmarks/tcp/tcp_proxy.go
+++ b/benchmarks/tcp/tcp_proxy.go
@@ -94,11 +94,11 @@ type netstackImpl struct {
 	mode string
 }
 
-func setupNetwork(ifaceName string) (fd int, err error) {
+func setupNetwork(ifaceName string, numChannels int) (fds []int, err error) {
 	// Get all interfaces in the namespace.
 	ifaces, err := net.Interfaces()
 	if err != nil {
-		return -1, fmt.Errorf("querying interfaces: %v", err)
+		return nil, fmt.Errorf("querying interfaces: %v", err)
 	}
 
 	for _, iface := range ifaces {
@@ -107,39 +107,43 @@ func setupNetwork(ifaceName string) (fd int, err error) {
 		}
 		// Create the socket.
 		const protocol = 0x0300 // htons(ETH_P_ALL)
-		fd, err := syscall.Socket(syscall.AF_PACKET, syscall.SOCK_RAW, protocol)
-		if err != nil {
-			return -1, fmt.Errorf("unable to create raw socket: %v", err)
-		}
+		fds := make([]int, numChannels)
+		for i := range fds {
+			fd, err := syscall.Socket(syscall.AF_PACKET, syscall.SOCK_RAW, protocol)
+			if err != nil {
+				return nil, fmt.Errorf("unable to create raw socket: %v", err)
+			}
 
-		// Bind to the appropriate device.
-		ll := syscall.SockaddrLinklayer{
-			Protocol: protocol,
-			Ifindex:  iface.Index,
-			Pkttype:  syscall.PACKET_HOST,
-		}
-		if err := syscall.Bind(fd, &ll); err != nil {
-			return -1, fmt.Errorf("unable to bind to %q: %v", iface.Name, err)
-		}
+			// Bind to the appropriate device.
+			ll := syscall.SockaddrLinklayer{
+				Protocol: protocol,
+				Ifindex:  iface.Index,
+				Pkttype:  syscall.PACKET_HOST,
+			}
+			if err := syscall.Bind(fd, &ll); err != nil {
+				return nil, fmt.Errorf("unable to bind to %q: %v", iface.Name, err)
+			}
 
-		// RAW Sockets by default have a very small SO_RCVBUF of 256KB,
-		// up it to at least 1MB to reduce packet drops.
-		if err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, rcvBufSize); err != nil {
-			return -1, fmt.Errorf("setsockopt(..., SO_RCVBUF, %v,..) = %v", rcvBufSize, err)
-		}
+			// RAW Sockets by default have a very small SO_RCVBUF of 256KB,
+			// up it to at least 1MB to reduce packet drops.
+			if err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, rcvBufSize); err != nil {
+				return nil, fmt.Errorf("setsockopt(..., SO_RCVBUF, %v,..) = %v", rcvBufSize, err)
+			}
 
-		if !*swgso && *gso != 0 {
-			if err := syscall.SetsockoptInt(fd, syscall.SOL_PACKET, unix.PACKET_VNET_HDR, 1); err != nil {
-				return -1, fmt.Errorf("unable to enable the PACKET_VNET_HDR option: %v", err)
+			if !*swgso && *gso != 0 {
+				if err := syscall.SetsockoptInt(fd, syscall.SOL_PACKET, unix.PACKET_VNET_HDR, 1); err != nil {
+					return nil, fmt.Errorf("unable to enable the PACKET_VNET_HDR option: %v", err)
+				}
 			}
+			fds[i] = fd
 		}
-		return fd, nil
+		return fds, nil
 	}
-	return -1, fmt.Errorf("failed to find interface: %v", ifaceName)
+	return nil, fmt.Errorf("failed to find interface: %v", ifaceName)
 }
 
 func newNetstackImpl(mode string) (impl, error) {
-	fd, err := setupNetwork(*iface)
+	fds, err := setupNetwork(*iface, runtime.GOMAXPROCS(-1))
 	if err != nil {
 		return nil, err
 	}
@@ -177,7 +181,7 @@ func newNetstackImpl(mode string) (impl, error) {
 	mac[0] &^= 0x1 // Clear multicast bit.
 	mac[0] |= 0x2  // Set local assignment bit (IEEE802).
 	ep, err := fdbased.New(&fdbased.Options{
-		FDs:            []int{fd},
+		FDs:            fds,
 		MTU:            uint32(*mtu),
 		EthernetHeader: true,
 		Address:        tcpip.LinkAddress(mac),