Simplify profiling and benchmarks.

- Tweak the benchmarks to work with b.N where appropriate. In many cases,
  b.N was simply being ignored. This creates an implicit dependency in the
  user passing a reasonable benchtime (less than or equal to the actual
  runtime of the test, or using the X syntax) otherwise the test runs
  forever.
- In cases where the above is impossible, explicitly set benchtime from
  the test wrapper, to prevent the above behavior (tensorflow).
- Drop the *Reverse variants, which are simply hey benchmarks. We should
  just add a hey benchmark. The platforms benchmarks already include a
  native platform, and thus these benchmarks are incredibly confusing.
  (In other words, BenchmarkNginxReverse has nothing to do with an nginx
  benchmark for runsc.)
- Remove the redunant Harness object, which contains no state, in order
  to slightly simplify the code.
- Make Block and Heap profiling actually work, but setting appropriate
  runtime parameters (and plumbing them through the config).
- Split the profiling into two phases: start and stop, since some will
  need to be started early, and others will need to happen at the end.

PiperOrigin-RevId: 349495377

5 files changed, 85 insertions, 75 deletions

diff --git a/test/benchmarks/tools/fio.go b/test/benchmarks/tools/fio.go
index f5f60fa84..f6324c3ab 100644
--- a/test/benchmarks/tools/fio.go
+++ b/test/benchmarks/tools/fio.go
@@ -25,25 +25,20 @@ import (
 // Fio makes 'fio' commands and parses their output.
 type Fio struct {
 	Test      string // test to run: read, write, randread, randwrite.
-	Size      string // total size to be read/written of format N[GMK] (e.g. 5G).
-	Blocksize string // blocksize to be read/write of format N[GMK] (e.g. 4K).
-	Iodepth   int    // iodepth for reads/writes.
-	Time      int    // time to run the test in seconds, usually for rand(read/write).
+	Size      int    // total size to be read/written in megabytes.
+	BlockSize int    // block size to be read/written in kilobytes.
+	IODepth   int    // I/O depth for reads/writes.
 }
 
 // MakeCmd makes a 'fio' command.
 func (f *Fio) MakeCmd(filename string) []string {
 	cmd := []string{"fio", "--output-format=json", "--ioengine=sync"}
 	cmd = append(cmd, fmt.Sprintf("--name=%s", f.Test))
-	cmd = append(cmd, fmt.Sprintf("--size=%s", f.Size))
-	cmd = append(cmd, fmt.Sprintf("--blocksize=%s", f.Blocksize))
+	cmd = append(cmd, fmt.Sprintf("--size=%dM", f.Size))
+	cmd = append(cmd, fmt.Sprintf("--blocksize=%dK", f.BlockSize))
 	cmd = append(cmd, fmt.Sprintf("--filename=%s", filename))
-	cmd = append(cmd, fmt.Sprintf("--iodepth=%d", f.Iodepth))
+	cmd = append(cmd, fmt.Sprintf("--iodepth=%d", f.IODepth))
 	cmd = append(cmd, fmt.Sprintf("--rw=%s", f.Test))
-	if f.Time != 0 {
-		cmd = append(cmd, "--time_based")
-		cmd = append(cmd, fmt.Sprintf("--runtime=%d", f.Time))
-	}
 	return cmd
 }
 
diff --git a/test/benchmarks/tools/hey.go b/test/benchmarks/tools/hey.go
index b8cb938fe..de908feeb 100644
--- a/test/benchmarks/tools/hey.go
+++ b/test/benchmarks/tools/hey.go
@@ -19,7 +19,6 @@ import (
 	"net"
 	"regexp"
 	"strconv"
-	"strings"
 	"testing"
 )
 
@@ -32,8 +31,16 @@ type Hey struct {
 
 // MakeCmd returns a 'hey' command.
 func (h *Hey) MakeCmd(ip net.IP, port int) []string {
-	return strings.Split(fmt.Sprintf("hey -n %d -c %d http://%s:%d/%s",
-		h.Requests, h.Concurrency, ip, port, h.Doc), " ")
+	c := h.Concurrency
+	if c > h.Requests {
+		c = h.Requests
+	}
+	return []string{
+		"hey",
+		"-n", fmt.Sprintf("%d", h.Requests),
+		"-c", fmt.Sprintf("%d", c),
+		fmt.Sprintf("http://%s:%d/%s", ip.String(), port, h.Doc),
+	}
 }
 
 // Report parses output from 'hey' and reports metrics.
diff --git a/test/benchmarks/tools/iperf.go b/test/benchmarks/tools/iperf.go
index 891d32704..abf296731 100644
--- a/test/benchmarks/tools/iperf.go
+++ b/test/benchmarks/tools/iperf.go
@@ -19,19 +19,27 @@ import (
 	"net"
 	"regexp"
 	"strconv"
-	"strings"
 	"testing"
 )
 
+const length = 64 * 1024
+
 // Iperf is for the client side of `iperf`.
 type Iperf struct {
-	Time int
+	Num int
 }
 
 // MakeCmd returns a iperf client command.
 func (i *Iperf) MakeCmd(ip net.IP, port int) []string {
-	// iperf report in Kb realtime
-	return strings.Split(fmt.Sprintf("iperf -f K --realtime --time %d --client %s --port %d", i.Time, ip, port), " ")
+	return []string{
+		"iperf",
+		"--format", "K", // Output in KBytes.
+		"--realtime", // Measured in realtime.
+		"--num", fmt.Sprintf("%d", i.Num),
+		"--length", fmt.Sprintf("%d", length),
+		"--client", ip.String(),
+		"--port", fmt.Sprintf("%d", port),
+	}
 }
 
 // Report parses output from iperf client and reports metrics.
@@ -42,6 +50,7 @@ func (i *Iperf) Report(b *testing.B, output string) {
 	if err != nil {
 		b.Fatalf("failed to parse bandwitdth from %s: %v", output, err)
 	}
+	b.SetBytes(length) // Measure Bytes/sec for b.N, although below is iperf output.
 	ReportCustomMetric(b, bW*1024, "bandwidth" /*metric name*/, "bytes_per_second" /*unit*/)
 }
 
diff --git a/test/benchmarks/tools/redis.go b/test/benchmarks/tools/redis.go
index a42e3456e..12fdbc7cc 100644
--- a/test/benchmarks/tools/redis.go
+++ b/test/benchmarks/tools/redis.go
@@ -19,7 +19,6 @@ import (
 	"net"
 	"regexp"
 	"strconv"
-	"strings"
 	"testing"
 )
 
@@ -33,13 +32,25 @@ func (r *Redis) MakeCmd(ip net.IP, port, requests int) []string {
 	// There is no -t PING_BULK for redis-benchmark, so adjust the command in that case.
 	// Note that "ping" will run both PING_INLINE and PING_BULK.
 	if r.Operation == "PING_BULK" {
-		return strings.Split(
-			fmt.Sprintf("redis-benchmark --csv -t ping -h %s -p %d -n %d", ip, port, requests), " ")
+		return []string{
+			"redis-benchmark",
+			"--csv",
+			"-t", "ping",
+			"-h", ip.String(),
+			"-p", fmt.Sprintf("%d", port),
+			"-n", fmt.Sprintf("%d", requests),
+		}
 	}
 
 	// runs redis-benchmark -t operation for 100K requests against server.
-	return strings.Split(
-		fmt.Sprintf("redis-benchmark --csv -t %s -h %s -p %d -n %d", r.Operation, ip, port, requests), " ")
+	return []string{
+		"redis-benchmark",
+		"--csv",
+		"-t", r.Operation,
+		"-h", ip.String(),
+		"-p", fmt.Sprintf("%d", port),
+		"-n", fmt.Sprintf("%d", requests),
+	}
 }
 
 // Report parses output from redis-benchmark client and reports metrics.
diff --git a/test/benchmarks/tools/sysbench.go b/test/benchmarks/tools/sysbench.go
index 7ccacd8ff..2b8e6c8aa 100644
--- a/test/benchmarks/tools/sysbench.go
+++ b/test/benchmarks/tools/sysbench.go
@@ -18,58 +18,46 @@ import (
 	"fmt"
 	"regexp"
 	"strconv"
-	"strings"
 	"testing"
 )
 
-var warmup = "sysbench --threads=8 --memory-total-size=5G memory run > /dev/null &&"
-
 // Sysbench represents a 'sysbench' command.
 type Sysbench interface {
-	MakeCmd() []string // Makes a sysbench command.
-	flags() []string
-	Report(*testing.B, string) // Reports results contained in string.
+	// MakeCmd constructs the relevant command line.
+	MakeCmd(*testing.B) []string
+
+	// Report reports relevant custom metrics.
+	Report(*testing.B, string)
 }
 
 // SysbenchBase is the top level struct for sysbench and holds top-level arguments
 // for sysbench. See: 'sysbench --help'
 type SysbenchBase struct {
-	Threads int // number of Threads for the test.
-	Time    int // time limit for test in seconds.
+	// Threads is the number of threads for the test.
+	Threads int
 }
 
 // baseFlags returns top level flags.
-func (s *SysbenchBase) baseFlags() []string {
+func (s *SysbenchBase) baseFlags(b *testing.B) []string {
 	var ret []string
 	if s.Threads > 0 {
 		ret = append(ret, fmt.Sprintf("--threads=%d", s.Threads))
 	}
-	if s.Time > 0 {
-		ret = append(ret, fmt.Sprintf("--time=%d", s.Time))
-	}
+	ret = append(ret, "--time=0") // Ensure events is used.
+	ret = append(ret, fmt.Sprintf("--events=%d", b.N))
 	return ret
 }
 
 // SysbenchCPU is for 'sysbench [flags] cpu run' and holds CPU specific arguments.
 type SysbenchCPU struct {
-	Base     SysbenchBase
-	MaxPrime int // upper limit for primes generator [10000].
+	SysbenchBase
 }
 
 // MakeCmd makes commands for SysbenchCPU.
-func (s *SysbenchCPU) MakeCmd() []string {
-	cmd := []string{warmup, "sysbench"}
-	cmd = append(cmd, s.flags()...)
-	cmd = append(cmd, "cpu run")
-	return []string{"sh", "-c", strings.Join(cmd, " ")}
-}
-
-// flags makes flags for SysbenchCPU cmds.
-func (s *SysbenchCPU) flags() []string {
-	cmd := s.Base.baseFlags()
-	if s.MaxPrime > 0 {
-		return append(cmd, fmt.Sprintf("--cpu-max-prime=%d", s.MaxPrime))
-	}
+func (s *SysbenchCPU) MakeCmd(b *testing.B) []string {
+	cmd := []string{"sysbench"}
+	cmd = append(cmd, s.baseFlags(b)...)
+	cmd = append(cmd, "cpu", "run")
 	return cmd
 }
 
@@ -96,9 +84,9 @@ func (s *SysbenchCPU) parseEvents(data string) (float64, error) {
 
 // SysbenchMemory is for 'sysbench [FLAGS] memory run' and holds Memory specific arguments.
 type SysbenchMemory struct {
-	Base          SysbenchBase
-	BlockSize     string // size of test memory block [1K].
-	TotalSize     string // size of data to transfer [100G].
+	SysbenchBase
+	BlockSize     int    // size of test memory block in megabytes [1].
+	TotalSize     int    // size of data to transfer in gigabytes [100].
 	Scope         string // memory access scope {global, local} [global].
 	HugeTLB       bool   // allocate memory from HugeTLB [off].
 	OperationType string // type of memory ops {read, write, none} [write].
@@ -106,21 +94,21 @@ type SysbenchMemory struct {
 }
 
 // MakeCmd makes commands for SysbenchMemory.
-func (s *SysbenchMemory) MakeCmd() []string {
-	cmd := []string{warmup, "sysbench"}
-	cmd = append(cmd, s.flags()...)
-	cmd = append(cmd, "memory run")
-	return []string{"sh", "-c", strings.Join(cmd, " ")}
+func (s *SysbenchMemory) MakeCmd(b *testing.B) []string {
+	cmd := []string{"sysbench"}
+	cmd = append(cmd, s.flags(b)...)
+	cmd = append(cmd, "memory", "run")
+	return cmd
 }
 
 // flags makes flags for SysbenchMemory cmds.
-func (s *SysbenchMemory) flags() []string {
-	cmd := s.Base.baseFlags()
-	if s.BlockSize != "" {
-		cmd = append(cmd, fmt.Sprintf("--memory-block-size=%s", s.BlockSize))
+func (s *SysbenchMemory) flags(b *testing.B) []string {
+	cmd := s.baseFlags(b)
+	if s.BlockSize != 0 {
+		cmd = append(cmd, fmt.Sprintf("--memory-block-size=%dM", s.BlockSize))
 	}
-	if s.TotalSize != "" {
-		cmd = append(cmd, fmt.Sprintf("--memory-total-size=%s", s.TotalSize))
+	if s.TotalSize != 0 {
+		cmd = append(cmd, fmt.Sprintf("--memory-total-size=%dG", s.TotalSize))
 	}
 	if s.Scope != "" {
 		cmd = append(cmd, fmt.Sprintf("--memory-scope=%s", s.Scope))
@@ -147,7 +135,7 @@ func (s *SysbenchMemory) Report(b *testing.B, output string) {
 	ReportCustomMetric(b, result, "memory_operations" /*metric name*/, "ops_per_second" /*unit*/)
 }
 
-var memoryOperationsRE = regexp.MustCompile(`Total\soperations:\s+\d*\s*\((\d*\.\d*)\sper\ssecond\)`)
+var memoryOperationsRE = regexp.MustCompile(`Total\s+operations:\s+\d+\s+\((\s*\d+\.\d+\s*)\s+per\s+second\)`)
 
 // parseOperations parses memory operations per second form sysbench memory ouput.
 func (s *SysbenchMemory) parseOperations(data string) (float64, error) {
@@ -160,24 +148,24 @@ func (s *SysbenchMemory) parseOperations(data string) (float64, error) {
 
 // SysbenchMutex is for 'sysbench [FLAGS] mutex run' and holds Mutex specific arguments.
 type SysbenchMutex struct {
-	Base  SysbenchBase
+	SysbenchBase
 	Num   int // total size of mutex array [4096].
-	Locks int // number of mutex locks per thread [50K].
-	Loops int // number of loops to do outside mutex lock [10K].
+	Locks int // number of mutex locks per thread [50000].
+	Loops int // number of loops to do outside mutex lock [10000].
 }
 
 // MakeCmd makes commands for SysbenchMutex.
-func (s *SysbenchMutex) MakeCmd() []string {
-	cmd := []string{warmup, "sysbench"}
-	cmd = append(cmd, s.flags()...)
-	cmd = append(cmd, "mutex run")
-	return []string{"sh", "-c", strings.Join(cmd, " ")}
+func (s *SysbenchMutex) MakeCmd(b *testing.B) []string {
+	cmd := []string{"sysbench"}
+	cmd = append(cmd, s.flags(b)...)
+	cmd = append(cmd, "mutex", "run")
+	return cmd
 }
 
 // flags makes flags for SysbenchMutex commands.
-func (s *SysbenchMutex) flags() []string {
+func (s *SysbenchMutex) flags(b *testing.B) []string {
 	var cmd []string
-	cmd = append(cmd, s.Base.baseFlags()...)
+	cmd = append(cmd, s.baseFlags(b)...)
 	if s.Num > 0 {
 		cmd = append(cmd, fmt.Sprintf("--mutex-num=%d", s.Num))
 	}