Add network benchmarks jobs

Add httpd, nginx, node, and ruby benchmarks to continuous jobs.

PiperOrigin-RevId: 346629115

9 files changed, 286 insertions, 114 deletions

diff --git a/test/benchmarks/network/BUILD b/test/benchmarks/network/BUILD
index c75d1ce11..2741570f5 100644
--- a/test/benchmarks/network/BUILD
+++ b/test/benchmarks/network/BUILD
@@ -8,7 +8,6 @@ go_library(
     testonly = 1,
     srcs = [
         "network.go",
-        "static_server.go",
     ],
     deps = [
         "//pkg/test/dockerutil",
@@ -18,19 +17,76 @@ go_library(
 )
 
 benchmark_test(
-    name = "network_test",
-    size = "large",
+    name = "iperf_test",
+    size = "enormous",
     srcs = [
-        "httpd_test.go",
         "iperf_test.go",
-        "nginx_test.go",
+    ],
+    library = ":network",
+    visibility = ["//:sandbox"],
+    deps = [
+        "//pkg/test/dockerutil",
+        "//pkg/test/testutil",
+        "//test/benchmarks/harness",
+        "//test/benchmarks/tools",
+    ],
+)
+
+benchmark_test(
+    name = "node_test",
+    size = "enormous",
+    srcs = [
         "node_test.go",
+    ],
+    library = ":network",
+    visibility = ["//:sandbox"],
+    deps = [
+        "//pkg/test/dockerutil",
+        "//test/benchmarks/harness",
+        "//test/benchmarks/tools",
+    ],
+)
+
+benchmark_test(
+    name = "ruby_test",
+    size = "enormous",
+    srcs = [
         "ruby_test.go",
     ],
     library = ":network",
     visibility = ["//:sandbox"],
     deps = [
         "//pkg/test/dockerutil",
+        "//test/benchmarks/harness",
+        "//test/benchmarks/tools",
+    ],
+)
+
+benchmark_test(
+    name = "nginx_test",
+    size = "enormous",
+    srcs = [
+        "nginx_test.go",
+    ],
+    library = ":network",
+    visibility = ["//:sandbox"],
+    deps = [
+        "//pkg/test/dockerutil",
+        "//test/benchmarks/harness",
+        "//test/benchmarks/tools",
+    ],
+)
+
+benchmark_test(
+    name = "httpd_test",
+    size = "enormous",
+    srcs = [
+        "httpd_test.go",
+    ],
+    library = ":network",
+    visibility = ["//:sandbox"],
+    deps = [
+        "//pkg/test/dockerutil",
         "//pkg/test/testutil",
         "//test/benchmarks/harness",
         "//test/benchmarks/tools",
diff --git a/test/benchmarks/network/httpd_test.go b/test/benchmarks/network/httpd_test.go
index 8d7d5f750..aa300dabd 100644
--- a/test/benchmarks/network/httpd_test.go
+++ b/test/benchmarks/network/httpd_test.go
@@ -14,13 +14,17 @@
 package network
 
 import (
+	"os"
 	"strconv"
 	"testing"
 
 	"gvisor.dev/gvisor/pkg/test/dockerutil"
+	"gvisor.dev/gvisor/test/benchmarks/harness"
 	"gvisor.dev/gvisor/test/benchmarks/tools"
 )
 
+var h harness.Harness
+
 // see Dockerfile '//images/benchmarks/httpd'.
 var httpdDocs = map[string]string{
 	"notfound": "notfound",
@@ -43,6 +47,22 @@ func BenchmarkReverseHttpd(b *testing.B) {
 	benchmarkHttpdDocSize(b, true /* reverse */)
 }
 
+// BenchmarkContinuousHttpd runs specific benchmarks for continous jobs.
+// The runtime under test is the server serving a runc client.
+func BenchmarkContinuousHttpd(b *testing.B) {
+	sizes := []string{"10Kb", "100Kb", "1Mb"}
+	threads := []int{1, 25, 100, 1000}
+	benchmarkHttpdContinuous(b, threads, sizes, false /*reverse*/)
+}
+
+// BenchmarkContinuousHttpdReverse runs specific benchmarks for continous jobs.
+// The runtime under test is the client downloading from a runc server.
+func BenchmarkContinuousHttpdReverse(b *testing.B) {
+	sizes := []string{"10Kb", "100Kb", "1Mb"}
+	threads := []int{1, 25, 100, 1000}
+	benchmarkHttpdContinuous(b, threads, sizes, true /*reverse*/)
+}
+
 // benchmarkHttpdDocSize iterates through all doc sizes, running subbenchmarks
 // for each size.
 func benchmarkHttpdDocSize(b *testing.B, reverse bool) {
@@ -74,6 +94,42 @@ func benchmarkHttpdDocSize(b *testing.B, reverse bool) {
 	}
 }
 
+// benchmarkHttpdContinuous iterates through given sizes and concurrencies.
+func benchmarkHttpdContinuous(b *testing.B, concurrency []int, sizes []string, reverse bool) {
+	for _, size := range sizes {
+		filename := httpdDocs[size]
+		for _, c := range concurrency {
+			fsize := tools.Parameter{
+				Name:  "filesize",
+				Value: size,
+			}
+
+			threads := tools.Parameter{
+				Name:  "concurrency",
+				Value: strconv.Itoa(c),
+			}
+
+			name, err := tools.ParametersToName(fsize, threads)
+			if err != nil {
+				b.Fatalf("Failed to parse parameters: %v", err)
+			}
+
+			requests := b.N
+			if requests < c {
+				requests = c
+			}
+			b.Run(name, func(b *testing.B) {
+				hey := &tools.Hey{
+					Requests:    requests,
+					Concurrency: c,
+					Doc:         filename,
+				}
+				runHttpd(b, hey, reverse)
+			})
+		}
+	}
+}
+
 // runHttpd configures the static serving methods to run httpd.
 func runHttpd(b *testing.B, hey *tools.Hey, reverse bool) {
 	// httpd runs on port 80.
@@ -91,5 +147,10 @@ func runHttpd(b *testing.B, hey *tools.Hey, reverse bool) {
 		},
 	}
 	httpdCmd := []string{"sh", "-c", "mkdir -p /tmp/html; cp -r /local/* /tmp/html/.; apache2 -X"}
-	runStaticServer(b, httpdRunOpts, httpdCmd, port, hey, reverse)
+	runStaticServer(b, h, httpdRunOpts, httpdCmd, port, hey, reverse)
+}
+
+func TestMain(m *testing.M) {
+	h.Init()
+	os.Exit(m.Run())
 }
diff --git a/test/benchmarks/network/iperf_test.go b/test/benchmarks/network/iperf_test.go
index b8ab7dfb8..9d64db943 100644
--- a/test/benchmarks/network/iperf_test.go
+++ b/test/benchmarks/network/iperf_test.go
@@ -15,6 +15,7 @@ package network
 
 import (
 	"context"
+	"os"
 	"testing"
 
 	"gvisor.dev/gvisor/pkg/test/dockerutil"
@@ -23,9 +24,11 @@ import (
 	"gvisor.dev/gvisor/test/benchmarks/tools"
 )
 
+var h harness.Harness
+
 func BenchmarkIperf(b *testing.B) {
 	iperf := tools.Iperf{
-		Time: 10, // time in seconds to run client.
+		Time: b.N, // time in seconds to run client.
 	}
 
 	clientMachine, err := h.GetMachine()
@@ -97,17 +100,19 @@ func BenchmarkIperf(b *testing.B) {
 			// Restart the server profiles. If the server isn't being profiled
 			// this does nothing.
 			server.RestartProfiles()
-			for i := 0; i < b.N; i++ {
-				out, err := client.Run(ctx, dockerutil.RunOpts{
-					Image: "benchmarks/iperf",
-				}, iperf.MakeCmd(ip, servingPort)...)
-				if err != nil {
-					b.Fatalf("failed to run client: %v", err)
-				}
-				b.StopTimer()
-				iperf.Report(b, out)
-				b.StartTimer()
+			out, err := client.Run(ctx, dockerutil.RunOpts{
+				Image: "benchmarks/iperf",
+			}, iperf.MakeCmd(ip, servingPort)...)
+			if err != nil {
+				b.Fatalf("failed to run client: %v", err)
 			}
+			b.StopTimer()
+			iperf.Report(b, out)
 		})
 	}
 }
+
+func TestMain(m *testing.M) {
+	h.Init()
+	os.Exit(m.Run())
+}
diff --git a/test/benchmarks/network/network.go b/test/benchmarks/network/network.go
index ce17ddb94..b18bc2b3c 100644
--- a/test/benchmarks/network/network.go
+++ b/test/benchmarks/network/network.go
@@ -16,16 +16,73 @@
 package network
 
 import (
-	"os"
+	"context"
 	"testing"
 
+	"gvisor.dev/gvisor/pkg/test/dockerutil"
 	"gvisor.dev/gvisor/test/benchmarks/harness"
+	"gvisor.dev/gvisor/test/benchmarks/tools"
 )
 
-var h harness.Harness
+// runStaticServer runs static serving workloads (httpd, nginx).
+func runStaticServer(b *testing.B, h harness.Harness, serverOpts dockerutil.RunOpts, serverCmd []string, port int, hey *tools.Hey, reverse bool) {
+	ctx := context.Background()
 
-// TestMain is the main method for package network.
-func TestMain(m *testing.M) {
-	h.Init()
-	os.Exit(m.Run())
+	// Get two machines: a client and server.
+	clientMachine, err := h.GetMachine()
+	if err != nil {
+		b.Fatalf("failed to get machine: %v", err)
+	}
+	defer clientMachine.CleanUp()
+
+	serverMachine, err := h.GetMachine()
+	if err != nil {
+		b.Fatalf("failed to get machine: %v", err)
+	}
+	defer serverMachine.CleanUp()
+
+	// Make the containers. 'reverse=true' specifies that the client should use the
+	// runtime under test.
+	var client, server *dockerutil.Container
+	if reverse {
+		client = clientMachine.GetContainer(ctx, b)
+		server = serverMachine.GetNativeContainer(ctx, b)
+	} else {
+		client = clientMachine.GetNativeContainer(ctx, b)
+		server = serverMachine.GetContainer(ctx, b)
+	}
+	defer client.CleanUp(ctx)
+	defer server.CleanUp(ctx)
+
+	// Start the server.
+	if err := server.Spawn(ctx, serverOpts, serverCmd...); err != nil {
+		b.Fatalf("failed to start server: %v", err)
+	}
+
+	// Get its IP.
+	ip, err := serverMachine.IPAddress()
+	if err != nil {
+		b.Fatalf("failed to find server ip: %v", err)
+	}
+
+	// Get the published port.
+	servingPort, err := server.FindPort(ctx, port)
+	if err != nil {
+		b.Fatalf("failed to find server port %d: %v", port, err)
+	}
+
+	// Make sure the server is serving.
+	harness.WaitUntilServing(ctx, clientMachine, ip, servingPort)
+	b.ResetTimer()
+	server.RestartProfiles()
+	out, err := client.Run(ctx, dockerutil.RunOpts{
+		Image: "benchmarks/hey",
+	}, hey.MakeCmd(ip, servingPort)...)
+	if err != nil {
+		b.Fatalf("run failed with: %v", err)
+	}
+
+	b.StopTimer()
+	hey.Report(b, out)
+	b.StartTimer()
 }
diff --git a/test/benchmarks/network/nginx_test.go b/test/benchmarks/network/nginx_test.go
index 08565d0b2..6e7a4dec4 100644
--- a/test/benchmarks/network/nginx_test.go
+++ b/test/benchmarks/network/nginx_test.go
@@ -14,13 +14,17 @@
 package network
 
 import (
+	"os"
 	"strconv"
 	"testing"
 
 	"gvisor.dev/gvisor/pkg/test/dockerutil"
+	"gvisor.dev/gvisor/test/benchmarks/harness"
 	"gvisor.dev/gvisor/test/benchmarks/tools"
 )
 
+var h harness.Harness
+
 // see Dockerfile '//images/benchmarks/nginx'.
 var nginxDocs = map[string]string{
 	"notfound": "notfound",
@@ -44,6 +48,22 @@ func BenchmarkReverseNginxDocSize(b *testing.B) {
 	benchmarkNginxDocSize(b, true /* reverse */, true /* tmpfs */)
 }
 
+// BenchmarkContinuousNginx runs specific benchmarks for continous jobs.
+// The runtime under test is the sever serving a runc client.
+func BenchmarkContinuousNginx(b *testing.B) {
+	sizes := []string{"10Kb", "100Kb", "1Mb"}
+	threads := []int{1, 25, 100, 1000}
+	benchmarkNginxContinuous(b, threads, sizes, false /*reverse*/)
+}
+
+// BenchmarkContinuousNginxReverse runs specific benchmarks for continous jobs.
+// The runtime under test is the client downloading from a runc server.
+func BenchmarkContinuousNginxReverse(b *testing.B) {
+	sizes := []string{"10Kb", "100Kb", "1Mb"}
+	threads := []int{1, 25, 100, 1000}
+	benchmarkNginxContinuous(b, threads, sizes, true /*reverse*/)
+}
+
 // benchmarkNginxDocSize iterates through all doc sizes, running subbenchmarks
 // for each size.
 func benchmarkNginxDocSize(b *testing.B, reverse, tmpfs bool) {
@@ -84,6 +104,47 @@ func benchmarkNginxDocSize(b *testing.B, reverse, tmpfs bool) {
 	}
 }
 
+// benchmarkNginxContinuous iterates through given sizes and concurrencies on a tmpfs mount.
+func benchmarkNginxContinuous(b *testing.B, concurrency []int, sizes []string, reverse bool) {
+	for _, size := range sizes {
+		filename := nginxDocs[size]
+		for _, c := range concurrency {
+			fsize := tools.Parameter{
+				Name:  "filesize",
+				Value: size,
+			}
+
+			threads := tools.Parameter{
+				Name:  "concurrency",
+				Value: strconv.Itoa(c),
+			}
+
+			fs := tools.Parameter{
+				Name:  "filesystem",
+				Value: "tmpfs",
+			}
+
+			name, err := tools.ParametersToName(fsize, threads, fs)
+			if err != nil {
+				b.Fatalf("Failed to parse parameters: %v", err)
+			}
+
+			requests := b.N
+			if requests < c {
+				requests = c
+			}
+			b.Run(name, func(b *testing.B) {
+				hey := &tools.Hey{
+					Requests:    requests,
+					Concurrency: c,
+					Doc:         filename,
+				}
+				runNginx(b, hey, reverse, true /*tmpfs*/)
+			})
+		}
+	}
+}
+
 // runNginx configures the static serving methods to run httpd.
 func runNginx(b *testing.B, hey *tools.Hey, reverse, tmpfs bool) {
 	// nginx runs on port 80.
@@ -99,5 +160,10 @@ func runNginx(b *testing.B, hey *tools.Hey, reverse, tmpfs bool) {
 	}
 
 	// Command copies nginxDocs to tmpfs serving directory and runs nginx.
-	runStaticServer(b, nginxRunOpts, nginxCmd, port, hey, reverse)
+	runStaticServer(b, h, nginxRunOpts, nginxCmd, port, hey, reverse)
+}
+
+func TestMain(m *testing.M) {
+	h.Init()
+	os.Exit(m.Run())
 }
diff --git a/test/benchmarks/network/node_test.go b/test/benchmarks/network/node_test.go
index 254538899..246720d43 100644
--- a/test/benchmarks/network/node_test.go
+++ b/test/benchmarks/network/node_test.go
@@ -15,6 +15,7 @@ package network
 
 import (
 	"context"
+	"os"
 	"strconv"
 	"testing"
 	"time"
@@ -24,6 +25,8 @@ import (
 	"gvisor.dev/gvisor/test/benchmarks/tools"
 )
 
+var h harness.Harness
+
 // BenchmarkNode runs requests using 'hey' against a Node server run on
 // 'runtime'. The server responds to requests by grabbing some data in a
 // redis instance and returns the data in its reponse. The test loops through
@@ -131,5 +134,9 @@ func runNode(b *testing.B, hey *tools.Hey) {
 	// Stop the timer to parse the data and report stats.
 	b.StopTimer()
 	hey.Report(b, out)
-	b.StartTimer()
+}
+
+func TestMain(m *testing.M) {
+	h.Init()
+	os.Exit(m.Run())
 }
diff --git a/test/benchmarks/network/ruby_test.go b/test/benchmarks/network/ruby_test.go
index 0174ff3f3..6f5199480 100644
--- a/test/benchmarks/network/ruby_test.go
+++ b/test/benchmarks/network/ruby_test.go
@@ -16,6 +16,7 @@ package network
 import (
 	"context"
 	"fmt"
+	"os"
 	"strconv"
 	"testing"
 	"time"
@@ -25,6 +26,8 @@ import (
 	"gvisor.dev/gvisor/test/benchmarks/tools"
 )
 
+var h harness.Harness
+
 // BenchmarkRuby runs requests using 'hey' against a ruby application server.
 // On start, ruby app generates some random data and pushes it to a redis
 // instance. On a request, the app grabs for random entries from the redis
@@ -52,7 +55,6 @@ func BenchmarkRuby(b *testing.B) {
 
 // runRuby runs the test for a given # of requests and concurrency.
 func runRuby(b *testing.B, hey *tools.Hey) {
-	b.Helper()
 	// The machine to hold Redis and the Ruby Server.
 	serverMachine, err := h.GetMachine()
 	if err != nil {
@@ -141,3 +143,8 @@ func runRuby(b *testing.B, hey *tools.Hey) {
 	hey.Report(b, out)
 	b.StartTimer()
 }
+
+func TestMain(m *testing.M) {
+	h.Init()
+	os.Exit(m.Run())
+}
diff --git a/test/benchmarks/network/static_server.go b/test/benchmarks/network/static_server.go
deleted file mode 100644
index e747a1395..000000000
--- a/test/benchmarks/network/static_server.go
+++ /dev/null
@@ -1,87 +0,0 @@
-// Copyright 2020 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package network
-
-import (
-	"context"
-	"testing"
-
-	"gvisor.dev/gvisor/pkg/test/dockerutil"
-	"gvisor.dev/gvisor/test/benchmarks/harness"
-	"gvisor.dev/gvisor/test/benchmarks/tools"
-)
-
-// runStaticServer runs static serving workloads (httpd, nginx).
-func runStaticServer(b *testing.B, serverOpts dockerutil.RunOpts, serverCmd []string, port int, hey *tools.Hey, reverse bool) {
-	ctx := context.Background()
-
-	// Get two machines: a client and server.
-	clientMachine, err := h.GetMachine()
-	if err != nil {
-		b.Fatalf("failed to get machine: %v", err)
-	}
-	defer clientMachine.CleanUp()
-
-	serverMachine, err := h.GetMachine()
-	if err != nil {
-		b.Fatalf("failed to get machine: %v", err)
-	}
-	defer serverMachine.CleanUp()
-
-	// Make the containers. 'reverse=true' specifies that the client should use the
-	// runtime under test.
-	var client, server *dockerutil.Container
-	if reverse {
-		client = clientMachine.GetContainer(ctx, b)
-		server = serverMachine.GetNativeContainer(ctx, b)
-	} else {
-		client = clientMachine.GetNativeContainer(ctx, b)
-		server = serverMachine.GetContainer(ctx, b)
-	}
-	defer client.CleanUp(ctx)
-	defer server.CleanUp(ctx)
-
-	// Start the server.
-	if err := server.Spawn(ctx, serverOpts, serverCmd...); err != nil {
-		b.Fatalf("failed to start server: %v", err)
-	}
-
-	// Get its IP.
-	ip, err := serverMachine.IPAddress()
-	if err != nil {
-		b.Fatalf("failed to find server ip: %v", err)
-	}
-
-	// Get the published port.
-	servingPort, err := server.FindPort(ctx, port)
-	if err != nil {
-		b.Fatalf("failed to find server port %d: %v", port, err)
-	}
-
-	// Make sure the server is serving.
-	harness.WaitUntilServing(ctx, clientMachine, ip, servingPort)
-	b.ResetTimer()
-	server.RestartProfiles()
-	out, err := client.Run(ctx, dockerutil.RunOpts{
-		Image: "benchmarks/hey",
-	}, hey.MakeCmd(ip, servingPort)...)
-	if err != nil {
-		b.Fatalf("run failed with: %v", err)
-	}
-
-	b.StopTimer()
-	hey.Report(b, out)
-	b.StartTimer()
-}
diff --git a/test/benchmarks/tools/iperf.go b/test/benchmarks/tools/iperf.go
index 5c4e7125b..891d32704 100644
--- a/test/benchmarks/tools/iperf.go
+++ b/test/benchmarks/tools/iperf.go
@@ -31,7 +31,7 @@ type Iperf struct {
 // MakeCmd returns a iperf client command.
 func (i *Iperf) MakeCmd(ip net.IP, port int) []string {
 	// iperf report in Kb realtime
-	return strings.Split(fmt.Sprintf("iperf -f K --realtime --time %d -c %s -p %d", i.Time, ip, port), " ")
+	return strings.Split(fmt.Sprintf("iperf -f K --realtime --time %d --client %s --port %d", i.Time, ip, port), " ")
 }
 
 // Report parses output from iperf client and reports metrics.