Simplify profiling and benchmarks.

- Tweak the benchmarks to work with b.N where appropriate. In many cases,
  b.N was simply being ignored. This creates an implicit dependency in the
  user passing a reasonable benchtime (less than or equal to the actual
  runtime of the test, or using the X syntax) otherwise the test runs
  forever.
- In cases where the above is impossible, explicitly set benchtime from
  the test wrapper, to prevent the above behavior (tensorflow).
- Drop the *Reverse variants, which are simply hey benchmarks. We should
  just add a hey benchmark. The platforms benchmarks already include a
  native platform, and thus these benchmarks are incredibly confusing.
  (In other words, BenchmarkNginxReverse has nothing to do with an nginx
  benchmark for runsc.)
- Remove the redunant Harness object, which contains no state, in order
  to slightly simplify the code.
- Make Block and Heap profiling actually work, but setting appropriate
  runtime parameters (and plumbing them through the config).
- Split the profiling into two phases: start and stop, since some will
  need to be started early, and others will need to happen at the end.

PiperOrigin-RevId: 349495377

6 files changed, 46 insertions, 134 deletions

diff --git a/test/benchmarks/network/httpd_test.go b/test/benchmarks/network/httpd_test.go
index b07274662..629127250 100644
--- a/test/benchmarks/network/httpd_test.go
+++ b/test/benchmarks/network/httpd_test.go
@@ -23,8 +23,6 @@ import (
 	"gvisor.dev/gvisor/test/benchmarks/tools"
 )
 
-var h harness.Harness
-
 // see Dockerfile '//images/benchmarks/httpd'.
 var httpdDocs = map[string]string{
 	"notfound": "notfound",
@@ -38,13 +36,7 @@ var httpdDocs = map[string]string{
 // BenchmarkHttpd iterates over different sized payloads and concurrency, testing
 // how well the runtime handles sending different payload sizes.
 func BenchmarkHttpd(b *testing.B) {
-	benchmarkHttpdDocSize(b, false /* reverse */)
-}
-
-// BenchmarkReverseHttpd iterates over different sized payloads, testing
-// how well the runtime handles receiving different payload sizes.
-func BenchmarkReverseHttpd(b *testing.B) {
-	benchmarkHttpdDocSize(b, true /* reverse */)
+	benchmarkHttpdDocSize(b)
 }
 
 // BenchmarkContinuousHttpd runs specific benchmarks for continous jobs.
@@ -52,20 +44,12 @@ func BenchmarkReverseHttpd(b *testing.B) {
 func BenchmarkContinuousHttpd(b *testing.B) {
 	sizes := []string{"10Kb", "100Kb", "1Mb"}
 	threads := []int{1, 25, 100, 1000}
-	benchmarkHttpdContinuous(b, threads, sizes, false /*reverse*/)
-}
-
-// BenchmarkContinuousHttpdReverse runs specific benchmarks for continous jobs.
-// The runtime under test is the client downloading from a runc server.
-func BenchmarkContinuousHttpdReverse(b *testing.B) {
-	sizes := []string{"10Kb", "100Kb", "1Mb"}
-	threads := []int{1, 25, 100, 1000}
-	benchmarkHttpdContinuous(b, threads, sizes, true /*reverse*/)
+	benchmarkHttpdContinuous(b, threads, sizes)
 }
 
 // benchmarkHttpdDocSize iterates through all doc sizes, running subbenchmarks
 // for each size.
-func benchmarkHttpdDocSize(b *testing.B, reverse bool) {
+func benchmarkHttpdDocSize(b *testing.B) {
 	b.Helper()
 	for size, filename := range httpdDocs {
 		concurrency := []int{1, 25, 50, 100, 1000}
@@ -82,25 +66,20 @@ func benchmarkHttpdDocSize(b *testing.B, reverse bool) {
 			if err != nil {
 				b.Fatalf("Failed to parse parameters: %v", err)
 			}
-			requests := b.N
-			if requests < c {
-				b.Logf("b.N is %d must be greater than threads %d. Consider running with --test.benchtime=Nx where N >= %d", b.N, c, c)
-				requests = c
-			}
 			b.Run(name, func(b *testing.B) {
 				hey := &tools.Hey{
-					Requests:    requests,
+					Requests:    b.N,
 					Concurrency: c,
 					Doc:         filename,
 				}
-				runHttpd(b, hey, reverse)
+				runHttpd(b, hey)
 			})
 		}
 	}
 }
 
 // benchmarkHttpdContinuous iterates through given sizes and concurrencies.
-func benchmarkHttpdContinuous(b *testing.B, concurrency []int, sizes []string, reverse bool) {
+func benchmarkHttpdContinuous(b *testing.B, concurrency []int, sizes []string) {
 	for _, size := range sizes {
 		filename := httpdDocs[size]
 		for _, c := range concurrency {
@@ -118,26 +97,20 @@ func benchmarkHttpdContinuous(b *testing.B, concurrency []int, sizes []string, r
 			if err != nil {
 				b.Fatalf("Failed to parse parameters: %v", err)
 			}
-
-			requests := b.N
-			if requests < c {
-				b.Logf("b.N is %d must be greater than threads %d. Consider running with --test.benchtime=Nx where N >= %d", b.N, c, c)
-				requests = c
-			}
 			b.Run(name, func(b *testing.B) {
 				hey := &tools.Hey{
-					Requests:    requests,
+					Requests:    b.N,
 					Concurrency: c,
 					Doc:         filename,
 				}
-				runHttpd(b, hey, reverse)
+				runHttpd(b, hey)
 			})
 		}
 	}
 }
 
 // runHttpd configures the static serving methods to run httpd.
-func runHttpd(b *testing.B, hey *tools.Hey, reverse bool) {
+func runHttpd(b *testing.B, hey *tools.Hey) {
 	// httpd runs on port 80.
 	port := 80
 	httpdRunOpts := dockerutil.RunOpts{
@@ -153,10 +126,10 @@ func runHttpd(b *testing.B, hey *tools.Hey, reverse bool) {
 		},
 	}
 	httpdCmd := []string{"sh", "-c", "mkdir -p /tmp/html; cp -r /local/* /tmp/html/.; apache2 -X"}
-	runStaticServer(b, h, httpdRunOpts, httpdCmd, port, hey, reverse)
+	runStaticServer(b, httpdRunOpts, httpdCmd, port, hey)
 }
 
 func TestMain(m *testing.M) {
-	h.Init()
+	harness.Init()
 	os.Exit(m.Run())
 }
diff --git a/test/benchmarks/network/iperf_test.go b/test/benchmarks/network/iperf_test.go
index 9d64db943..5e81149fe 100644
--- a/test/benchmarks/network/iperf_test.go
+++ b/test/benchmarks/network/iperf_test.go
@@ -24,20 +24,18 @@ import (
 	"gvisor.dev/gvisor/test/benchmarks/tools"
 )
 
-var h harness.Harness
-
 func BenchmarkIperf(b *testing.B) {
 	iperf := tools.Iperf{
-		Time: b.N, // time in seconds to run client.
+		Num: b.N,
 	}
 
-	clientMachine, err := h.GetMachine()
+	clientMachine, err := harness.GetMachine()
 	if err != nil {
 		b.Fatalf("failed to get machine: %v", err)
 	}
 	defer clientMachine.CleanUp()
 
-	serverMachine, err := h.GetMachine()
+	serverMachine, err := harness.GetMachine()
 	if err != nil {
 		b.Fatalf("failed to get machine: %v", err)
 	}
@@ -94,12 +92,9 @@ func BenchmarkIperf(b *testing.B) {
 			if err := harness.WaitUntilServing(ctx, clientMachine, ip, servingPort); err != nil {
 				b.Fatalf("failed to wait for server: %v", err)
 			}
+
 			// Run the client.
 			b.ResetTimer()
-
-			// Restart the server profiles. If the server isn't being profiled
-			// this does nothing.
-			server.RestartProfiles()
 			out, err := client.Run(ctx, dockerutil.RunOpts{
 				Image: "benchmarks/iperf",
 			}, iperf.MakeCmd(ip, servingPort)...)
@@ -113,6 +108,6 @@ func BenchmarkIperf(b *testing.B) {
 }
 
 func TestMain(m *testing.M) {
-	h.Init()
+	harness.Init()
 	os.Exit(m.Run())
 }
diff --git a/test/benchmarks/network/network.go b/test/benchmarks/network/network.go
index b18bc2b3c..d61002cea 100644
--- a/test/benchmarks/network/network.go
+++ b/test/benchmarks/network/network.go
@@ -25,33 +25,26 @@ import (
 )
 
 // runStaticServer runs static serving workloads (httpd, nginx).
-func runStaticServer(b *testing.B, h harness.Harness, serverOpts dockerutil.RunOpts, serverCmd []string, port int, hey *tools.Hey, reverse bool) {
+func runStaticServer(b *testing.B, serverOpts dockerutil.RunOpts, serverCmd []string, port int, hey *tools.Hey) {
 	ctx := context.Background()
 
 	// Get two machines: a client and server.
-	clientMachine, err := h.GetMachine()
+	clientMachine, err := harness.GetMachine()
 	if err != nil {
 		b.Fatalf("failed to get machine: %v", err)
 	}
 	defer clientMachine.CleanUp()
 
-	serverMachine, err := h.GetMachine()
+	serverMachine, err := harness.GetMachine()
 	if err != nil {
 		b.Fatalf("failed to get machine: %v", err)
 	}
 	defer serverMachine.CleanUp()
 
-	// Make the containers. 'reverse=true' specifies that the client should use the
-	// runtime under test.
-	var client, server *dockerutil.Container
-	if reverse {
-		client = clientMachine.GetContainer(ctx, b)
-		server = serverMachine.GetNativeContainer(ctx, b)
-	} else {
-		client = clientMachine.GetNativeContainer(ctx, b)
-		server = serverMachine.GetContainer(ctx, b)
-	}
+	// Make the containers.
+	client := clientMachine.GetNativeContainer(ctx, b)
 	defer client.CleanUp(ctx)
+	server := serverMachine.GetContainer(ctx, b)
 	defer server.CleanUp(ctx)
 
 	// Start the server.
@@ -73,16 +66,15 @@ func runStaticServer(b *testing.B, h harness.Harness, serverOpts dockerutil.RunO
 
 	// Make sure the server is serving.
 	harness.WaitUntilServing(ctx, clientMachine, ip, servingPort)
+
+	// Run the client.
 	b.ResetTimer()
-	server.RestartProfiles()
 	out, err := client.Run(ctx, dockerutil.RunOpts{
 		Image: "benchmarks/hey",
 	}, hey.MakeCmd(ip, servingPort)...)
 	if err != nil {
 		b.Fatalf("run failed with: %v", err)
 	}
-
 	b.StopTimer()
 	hey.Report(b, out)
-	b.StartTimer()
 }
diff --git a/test/benchmarks/network/nginx_test.go b/test/benchmarks/network/nginx_test.go
index 87449612a..74f3578fc 100644
--- a/test/benchmarks/network/nginx_test.go
+++ b/test/benchmarks/network/nginx_test.go
@@ -23,8 +23,6 @@ import (
 	"gvisor.dev/gvisor/test/benchmarks/tools"
 )
 
-var h harness.Harness
-
 // see Dockerfile '//images/benchmarks/nginx'.
 var nginxDocs = map[string]string{
 	"notfound": "notfound",
@@ -38,14 +36,8 @@ var nginxDocs = map[string]string{
 // BenchmarkNginxDocSize iterates over different sized payloads, testing how
 // well the runtime handles sending different payload sizes.
 func BenchmarkNginxDocSize(b *testing.B) {
-	benchmarkNginxDocSize(b, false /* reverse */, true /* tmpfs */)
-	benchmarkNginxDocSize(b, false /* reverse */, false /* tmpfs */)
-}
-
-// BenchmarkReverseNginxDocSize iterates over different sized payloads, testing
-// how well the runtime handles receiving different payload sizes.
-func BenchmarkReverseNginxDocSize(b *testing.B) {
-	benchmarkNginxDocSize(b, true /* reverse */, true /* tmpfs */)
+	benchmarkNginxDocSize(b, true /* tmpfs */)
+	benchmarkNginxDocSize(b, false /* tmpfs */)
 }
 
 // BenchmarkContinuousNginx runs specific benchmarks for continous jobs.
@@ -53,20 +45,12 @@ func BenchmarkReverseNginxDocSize(b *testing.B) {
 func BenchmarkContinuousNginx(b *testing.B) {
 	sizes := []string{"10Kb", "100Kb", "1Mb"}
 	threads := []int{1, 25, 100, 1000}
-	benchmarkNginxContinuous(b, threads, sizes, false /*reverse*/)
-}
-
-// BenchmarkContinuousNginxReverse runs specific benchmarks for continous jobs.
-// The runtime under test is the client downloading from a runc server.
-func BenchmarkContinuousNginxReverse(b *testing.B) {
-	sizes := []string{"10Kb", "100Kb", "1Mb"}
-	threads := []int{1, 25, 100, 1000}
-	benchmarkNginxContinuous(b, threads, sizes, true /*reverse*/)
+	benchmarkNginxContinuous(b, threads, sizes)
 }
 
 // benchmarkNginxDocSize iterates through all doc sizes, running subbenchmarks
 // for each size.
-func benchmarkNginxDocSize(b *testing.B, reverse, tmpfs bool) {
+func benchmarkNginxDocSize(b *testing.B, tmpfs bool) {
 	for size, filename := range nginxDocs {
 		concurrency := []int{1, 25, 50, 100, 1000}
 		for _, c := range concurrency {
@@ -91,26 +75,20 @@ func benchmarkNginxDocSize(b *testing.B, reverse, tmpfs bool) {
 			if err != nil {
 				b.Fatalf("Failed to parse parameters: %v", err)
 			}
-
-			requests := b.N
-			if requests < c {
-				b.Logf("b.N is %d must be greater than threads %d. Consider running with --test.benchtime=Nx where N >= %d", b.N, c, c)
-				requests = c
-			}
 			b.Run(name, func(b *testing.B) {
 				hey := &tools.Hey{
-					Requests:    requests,
+					Requests:    b.N,
 					Concurrency: c,
 					Doc:         filename,
 				}
-				runNginx(b, hey, reverse, tmpfs)
+				runNginx(b, hey, tmpfs)
 			})
 		}
 	}
 }
 
 // benchmarkNginxContinuous iterates through given sizes and concurrencies on a tmpfs mount.
-func benchmarkNginxContinuous(b *testing.B, concurrency []int, sizes []string, reverse bool) {
+func benchmarkNginxContinuous(b *testing.B, concurrency []int, sizes []string) {
 	for _, size := range sizes {
 		filename := nginxDocs[size]
 		for _, c := range concurrency {
@@ -133,25 +111,20 @@ func benchmarkNginxContinuous(b *testing.B, concurrency []int, sizes []string, r
 			if err != nil {
 				b.Fatalf("Failed to parse parameters: %v", err)
 			}
-			requests := b.N
-			if requests < c {
-				b.Logf("b.N is %d must be greater than threads %d. Consider running with --test.benchtime=Nx where N >= %d", b.N, c, c)
-				requests = c
-			}
 			b.Run(name, func(b *testing.B) {
 				hey := &tools.Hey{
-					Requests:    requests,
+					Requests:    b.N,
 					Concurrency: c,
 					Doc:         filename,
 				}
-				runNginx(b, hey, reverse, true /*tmpfs*/)
+				runNginx(b, hey, true /*tmpfs*/)
 			})
 		}
 	}
 }
 
 // runNginx configures the static serving methods to run httpd.
-func runNginx(b *testing.B, hey *tools.Hey, reverse, tmpfs bool) {
+func runNginx(b *testing.B, hey *tools.Hey, tmpfs bool) {
 	// nginx runs on port 80.
 	port := 80
 	nginxRunOpts := dockerutil.RunOpts{
@@ -165,10 +138,10 @@ func runNginx(b *testing.B, hey *tools.Hey, reverse, tmpfs bool) {
 	}
 
 	// Command copies nginxDocs to tmpfs serving directory and runs nginx.
-	runStaticServer(b, h, nginxRunOpts, nginxCmd, port, hey, reverse)
+	runStaticServer(b, nginxRunOpts, nginxCmd, port, hey)
 }
 
 func TestMain(m *testing.M) {
-	h.Init()
+	harness.Init()
 	os.Exit(m.Run())
 }
diff --git a/test/benchmarks/network/node_test.go b/test/benchmarks/network/node_test.go
index 3e837a9e4..a1fc82f95 100644
--- a/test/benchmarks/network/node_test.go
+++ b/test/benchmarks/network/node_test.go
@@ -25,8 +25,6 @@ import (
 	"gvisor.dev/gvisor/test/benchmarks/tools"
 )
 
-var h harness.Harness
-
 // BenchmarkNode runs requests using 'hey' against a Node server run on
 // 'runtime'. The server responds to requests by grabbing some data in a
 // redis instance and returns the data in its reponse. The test loops through
@@ -42,14 +40,9 @@ func BenchmarkNode(b *testing.B) {
 		if err != nil {
 			b.Fatalf("Failed to parse parameters: %v", err)
 		}
-		requests := b.N
-		if requests < c {
-			b.Logf("b.N is %d must be greater than threads %d. Consider running with --test.benchtime=Nx where N >= %d", b.N, c, c)
-			requests = c
-		}
 		b.Run(name, func(b *testing.B) {
 			hey := &tools.Hey{
-				Requests:    requests,
+				Requests:    b.N,
 				Concurrency: c,
 			}
 			runNode(b, hey)
@@ -62,14 +55,14 @@ func runNode(b *testing.B, hey *tools.Hey) {
 	b.Helper()
 
 	// The machine to hold Redis and the Node Server.
-	serverMachine, err := h.GetMachine()
+	serverMachine, err := harness.GetMachine()
 	if err != nil {
 		b.Fatalf("failed to get machine with: %v", err)
 	}
 	defer serverMachine.CleanUp()
 
 	// The machine to run 'hey'.
-	clientMachine, err := h.GetMachine()
+	clientMachine, err := harness.GetMachine()
 	if err != nil {
 		b.Fatalf("failed to get machine with: %v", err)
 	}
@@ -124,10 +117,8 @@ func runNode(b *testing.B, hey *tools.Hey) {
 
 	heyCmd := hey.MakeCmd(servingIP, servingPort)
 
-	nodeApp.RestartProfiles()
-	b.ResetTimer()
-
 	// the client should run on Native.
+	b.ResetTimer()
 	client := clientMachine.GetNativeContainer(ctx, b)
 	out, err := client.Run(ctx, dockerutil.RunOpts{
 		Image: "benchmarks/hey",
@@ -137,11 +128,10 @@ func runNode(b *testing.B, hey *tools.Hey) {
 	}
 
 	// Stop the timer to parse the data and report stats.
-	b.StopTimer()
 	hey.Report(b, out)
 }
 
 func TestMain(m *testing.M) {
-	h.Init()
+	harness.Init()
 	os.Exit(m.Run())
 }
diff --git a/test/benchmarks/network/ruby_test.go b/test/benchmarks/network/ruby_test.go
index c89672873..b7ec16e0a 100644
--- a/test/benchmarks/network/ruby_test.go
+++ b/test/benchmarks/network/ruby_test.go
@@ -26,8 +26,6 @@ import (
 	"gvisor.dev/gvisor/test/benchmarks/tools"
 )
 
-var h harness.Harness
-
 // BenchmarkRuby runs requests using 'hey' against a ruby application server.
 // On start, ruby app generates some random data and pushes it to a redis
 // instance. On a request, the app grabs for random entries from the redis
@@ -43,14 +41,9 @@ func BenchmarkRuby(b *testing.B) {
 		if err != nil {
 			b.Fatalf("Failed to parse parameters: %v", err)
 		}
-		requests := b.N
-		if requests < c {
-			b.Logf("b.N is %d must be greater than threads %d. Consider running with --test.benchtime=Nx where N >= %d", b.N, c, c)
-			requests = c
-		}
 		b.Run(name, func(b *testing.B) {
 			hey := &tools.Hey{
-				Requests:    requests,
+				Requests:    b.N,
 				Concurrency: c,
 			}
 			runRuby(b, hey)
@@ -61,14 +54,14 @@ func BenchmarkRuby(b *testing.B) {
 // runRuby runs the test for a given # of requests and concurrency.
 func runRuby(b *testing.B, hey *tools.Hey) {
 	// The machine to hold Redis and the Ruby Server.
-	serverMachine, err := h.GetMachine()
+	serverMachine, err := harness.GetMachine()
 	if err != nil {
 		b.Fatalf("failed to get machine with: %v", err)
 	}
 	defer serverMachine.CleanUp()
 
 	// The machine to run 'hey'.
-	clientMachine, err := h.GetMachine()
+	clientMachine, err := harness.GetMachine()
 	if err != nil {
 		b.Fatalf("failed to get machine with: %v", err)
 	}
@@ -130,10 +123,9 @@ func runRuby(b *testing.B, hey *tools.Hey) {
 		b.Fatalf("failed to wait until  serving: %v", err)
 	}
 	heyCmd := hey.MakeCmd(servingIP, servingPort)
-	rubyApp.RestartProfiles()
-	b.ResetTimer()
 
 	// the client should run on Native.
+	b.ResetTimer()
 	client := clientMachine.GetNativeContainer(ctx, b)
 	defer client.CleanUp(ctx)
 	out, err := client.Run(ctx, dockerutil.RunOpts{
@@ -142,14 +134,11 @@ func runRuby(b *testing.B, hey *tools.Hey) {
 	if err != nil {
 		b.Fatalf("hey container failed: %v logs: %s", err, out)
 	}
-
-	// Stop the timer to parse the data and report stats.
 	b.StopTimer()
 	hey.Report(b, out)
-	b.StartTimer()
 }
 
 func TestMain(m *testing.M) {
-	h.Init()
+	harness.Init()
 	os.Exit(m.Run())
 }