diff options
Diffstat (limited to 'benchmarks/tcp')
| -rwxr-xr-x | benchmarks/tcp/tcp_benchmark.sh | 21 | ||||
| -rw-r--r-- | benchmarks/tcp/tcp_proxy.go | 66 |
2 files changed, 57 insertions, 30 deletions
diff --git a/benchmarks/tcp/tcp_benchmark.sh b/benchmarks/tcp/tcp_benchmark.sh index 69344c9c3..e65801a7b 100755 --- a/benchmarks/tcp/tcp_benchmark.sh +++ b/benchmarks/tcp/tcp_benchmark.sh @@ -41,6 +41,8 @@ duplicate=0.1 # 0.1% means duplicates are 1/10x as frequent as losses. duration=30 # 30s is enough time to consistent results (experimentally). helper_dir=$(dirname $0) netstack_opts= +disable_linux_gso= +num_client_threads=1 # Check for netem support. lsmod_output=$(lsmod | grep sch_netem) @@ -125,6 +127,13 @@ while [ $# -gt 0 ]; do shift netstack_opts="${netstack_opts} -memprofile=$1" ;; + --disable-linux-gso) + disable_linux_gso=1 + ;; + --num-client-threads) + shift + num_client_threads=$1 + ;; --helpers) shift [ "$#" -le 0 ] && echo "no helper dir provided" && exit 1 @@ -147,6 +156,8 @@ while [ $# -gt 0 ]; do echo " --loss set the loss probability (%)" echo " --duplicate set the duplicate probability (%)" echo " --helpers set the helper directory" + echo " --num-client-threads number of parallel client threads to run" + echo " --disable-linux-gso disable segmentation offload in the Linux network stack" echo "" echo "The output will of the script will be:" echo " <throughput> <client-cpu-usage> <server-cpu-usage>" @@ -301,6 +312,14 @@ fi # Add client and server addresses, and bring everything up. ${nsjoin_binary} /tmp/client.netns ip addr add ${client_addr}/${mask} dev client.0 ${nsjoin_binary} /tmp/server.netns ip addr add ${server_addr}/${mask} dev server.0 +if [ "${disable_linux_gso}" == "1" ]; then + ${nsjoin_binary} /tmp/client.netns ethtool -K client.0 tso off + ${nsjoin_binary} /tmp/client.netns ethtool -K client.0 gro off + ${nsjoin_binary} /tmp/client.netns ethtool -K client.0 gso off + ${nsjoin_binary} /tmp/server.netns ethtool -K server.0 tso off + ${nsjoin_binary} /tmp/server.netns ethtool -K server.0 gso off + ${nsjoin_binary} /tmp/server.netns ethtool -K server.0 gro off +fi ${nsjoin_binary} /tmp/client.netns ip link set client.0 up ${nsjoin_binary} /tmp/client.netns ip link set lo up ${nsjoin_binary} /tmp/server.netns ip link set server.0 up @@ -338,7 +357,7 @@ trap cleanup EXIT # Run the benchmark, recording the results file. while ${nsjoin_binary} /tmp/client.netns iperf \\ - -p ${proxy_port} -c ${client_addr} -t ${duration} -f m 2>&1 \\ + -p ${proxy_port} -c ${client_addr} -t ${duration} -f m -P ${num_client_threads} 2>&1 \\ | tee \$results_file \\ | grep "connect failed" >/dev/null; do sleep 0.1 # Wait for all services. diff --git a/benchmarks/tcp/tcp_proxy.go b/benchmarks/tcp/tcp_proxy.go index 361a56755..72ada5700 100644 --- a/benchmarks/tcp/tcp_proxy.go +++ b/benchmarks/tcp/tcp_proxy.go @@ -84,8 +84,8 @@ func (netImpl) printStats() { } const ( - nicID = 1 // Fixed. - rcvBufSize = 1 << 20 // 1MB. + nicID = 1 // Fixed. + bufSize = 4 << 20 // 4MB. ) type netstackImpl struct { @@ -94,11 +94,11 @@ type netstackImpl struct { mode string } -func setupNetwork(ifaceName string) (fd int, err error) { +func setupNetwork(ifaceName string, numChannels int) (fds []int, err error) { // Get all interfaces in the namespace. ifaces, err := net.Interfaces() if err != nil { - return -1, fmt.Errorf("querying interfaces: %v", err) + return nil, fmt.Errorf("querying interfaces: %v", err) } for _, iface := range ifaces { @@ -107,39 +107,47 @@ func setupNetwork(ifaceName string) (fd int, err error) { } // Create the socket. const protocol = 0x0300 // htons(ETH_P_ALL) - fd, err := syscall.Socket(syscall.AF_PACKET, syscall.SOCK_RAW, protocol) - if err != nil { - return -1, fmt.Errorf("unable to create raw socket: %v", err) - } + fds := make([]int, numChannels) + for i := range fds { + fd, err := syscall.Socket(syscall.AF_PACKET, syscall.SOCK_RAW, protocol) + if err != nil { + return nil, fmt.Errorf("unable to create raw socket: %v", err) + } - // Bind to the appropriate device. - ll := syscall.SockaddrLinklayer{ - Protocol: protocol, - Ifindex: iface.Index, - Pkttype: syscall.PACKET_HOST, - } - if err := syscall.Bind(fd, &ll); err != nil { - return -1, fmt.Errorf("unable to bind to %q: %v", iface.Name, err) - } + // Bind to the appropriate device. + ll := syscall.SockaddrLinklayer{ + Protocol: protocol, + Ifindex: iface.Index, + Pkttype: syscall.PACKET_HOST, + } + if err := syscall.Bind(fd, &ll); err != nil { + return nil, fmt.Errorf("unable to bind to %q: %v", iface.Name, err) + } - // RAW Sockets by default have a very small SO_RCVBUF of 256KB, - // up it to at least 1MB to reduce packet drops. - if err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, rcvBufSize); err != nil { - return -1, fmt.Errorf("setsockopt(..., SO_RCVBUF, %v,..) = %v", rcvBufSize, err) - } + // RAW Sockets by default have a very small SO_RCVBUF of 256KB, + // up it to at least 4MB to reduce packet drops. + if err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, bufSize); err != nil { + return nil, fmt.Errorf("setsockopt(..., SO_RCVBUF, %v,..) = %v", bufSize, err) + } - if !*swgso && *gso != 0 { - if err := syscall.SetsockoptInt(fd, syscall.SOL_PACKET, unix.PACKET_VNET_HDR, 1); err != nil { - return -1, fmt.Errorf("unable to enable the PACKET_VNET_HDR option: %v", err) + if err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_SNDBUF, bufSize); err != nil { + return nil, fmt.Errorf("setsockopt(..., SO_SNDBUF, %v,..) = %v", bufSize, err) + } + + if !*swgso && *gso != 0 { + if err := syscall.SetsockoptInt(fd, syscall.SOL_PACKET, unix.PACKET_VNET_HDR, 1); err != nil { + return nil, fmt.Errorf("unable to enable the PACKET_VNET_HDR option: %v", err) + } } + fds[i] = fd } - return fd, nil + return fds, nil } - return -1, fmt.Errorf("failed to find interface: %v", ifaceName) + return nil, fmt.Errorf("failed to find interface: %v", ifaceName) } func newNetstackImpl(mode string) (impl, error) { - fd, err := setupNetwork(*iface) + fds, err := setupNetwork(*iface, runtime.GOMAXPROCS(-1)) if err != nil { return nil, err } @@ -177,7 +185,7 @@ func newNetstackImpl(mode string) (impl, error) { mac[0] &^= 0x1 // Clear multicast bit. mac[0] |= 0x2 // Set local assignment bit (IEEE802). ep, err := fdbased.New(&fdbased.Options{ - FDs: []int{fd}, + FDs: fds, MTU: uint32(*mtu), EthernetHeader: true, Address: tcpip.LinkAddress(mac), |