Initial golang Benchmarks

PiperOrigin-RevId: 321021071

11 files changed, 611 insertions, 0 deletions

diff --git a/test/benchmarks/README.md b/test/benchmarks/README.md
new file mode 100644
index 000000000..9ff602cf1
--- /dev/null
+++ b/test/benchmarks/README.md
@@ -0,0 +1,118 @@
+# Benchmark tools
+
+This package and subpackages are for running macro benchmarks on `runsc`. They
+are meant to replace the previous //benchmarks benchmark-tools written in
+python.
+
+Benchmarks are meant to look like regular golang benchmarks using the testing.B
+library.
+
+## Setup
+
+To run benchmarks you will need:
+
+*   Docker installed (17.09.0 or greater).
+
+The easiest way to run benchmarks is to use the script at
+//scripts/benchmark.sh.
+
+If not using the script, you will need:
+
+*   `runsc` configured with docker
+
+Note: benchmarks call the runtime by name. If docker can run it with
+`--runtime=` flag, these tools should work.
+
+## Running benchmarks
+
+The easiest way to run is with the script at //scripts/benchmarks.sh. The script
+will run all benchmarks under //test/benchmarks if a target is not provided.
+
+```bash
+./script/benchmarks.sh //path/to/target
+```
+
+If you want to run benchmarks manually:
+
+*   Run `make load-all-images` from `//`
+*   Run with:
+
+```bash
+bazel test --test_arg=--runtime=RUNTIME -c opt --test_output=streamed --test_timeout=600 --test_arg=-test.bench=. --nocache_test_results //path/to/target
+```
+
+## Writing benchmarks
+
+Benchmarks consist of docker images as Dockerfiles and golang testing.B
+benchmarks.
+
+### Dockerfiles:
+
+*   Are stored at //images.
+*   New Dockerfiles go in an appropriately named directory at
+    `//images/benchmarks/my-cool-dockerfile`.
+*   Dockerfiles for benchmarks should:
+    *   Use explicitly versioned packages.
+    *   Not use ENV and CMD statements...it is easy to add these in the API.
+*   Note: A common pattern for getting access to a tmpfs mount is to copy files
+    there after container start. See: //test/benchmarks/build/bazel_test.go. You
+    can also make your own with `RunOpts.Mounts`.
+
+### testing.B packages
+
+In general, benchmarks should look like this:
+
+```golang
+
+var h harness.Harness
+
+func BenchmarkMyCoolOne(b *testing.B) {
+  machine, err := h.GetMachine()
+  // check err
+
+  ctx := context.Background()
+  container := machine.GetContainer(ctx, b)
+  defer container.CleanUp(ctx)
+
+  b.ResetTimer()
+
+  //Respect b.N.
+  for i := 0; i < b.N; i++ {
+    out, err := container.Run(ctx, dockerutil.RunOpts{
+      Image: "benchmarks/my-cool-image",
+      Env: []string{"MY_VAR=awesome"},
+      other options...see dockerutil
+    }, "sh", "-c", "echo MY_VAR" ...)
+    //check err
+    b.StopTimer()
+
+    // Do parsing and reporting outside of the timer.
+    number := parseMyMetric(out)
+    b.ReportMetric(number, "my-cool-custom-metric")
+
+    b.StartTimer()
+  }
+}
+
+func TestMain(m *testing.M) {
+    h.Init()
+    os.Exit(m.Run())
+}
+```
+
+Some notes on the above:
+
+*   The harness is initiated in the TestMain method and made global to test
+    module. The harness will handle any presetup that needs to happen with
+    flags, remote virtual machines (eventually), and other services.
+*   Respect `b.N` in that users of the benchmark may want to "run for an hour"
+    or something of the sort.
+*   Use the `b.ReportMetric` method to report custom metrics.
+*   Set the timer if time is useful for reporting. There isn't a way to turn off
+    default metrics in testing.B (B/op, allocs/op, ns/op).
+*   Take a look at dockerutil at //pkg/test/dockerutil to see all methods
+    available from containers. The API is based on the "official"
+    [docker API for golang](https://pkg.go.dev/mod/github.com/docker/docker).
+*   `harness.GetMachine` marks how many machines this tests needs. If you have a
+    client and server and to mark them as multiple machines, call it
+    `GetMachine` twice.
diff --git a/test/benchmarks/fs/BUILD b/test/benchmarks/fs/BUILD
new file mode 100644
index 000000000..606331895
--- /dev/null
+++ b/test/benchmarks/fs/BUILD
@@ -0,0 +1,24 @@
+load("//tools:defs.bzl", "go_library", "go_test")
+
+package(licenses = ["notice"])
+
+go_library(
+    name = "fs",
+    srcs = ["fs.go"],
+)
+
+go_test(
+    name = "fs_test",
+    size = "large",
+    srcs = ["bazel_test.go"],
+    library = ":fs",
+    tags = [
+        # Requires docker and runsc to be configured before test runs.
+        "local",
+        "manual",
+    ],
+    deps = [
+        "//pkg/test/dockerutil",
+        "//test/benchmarks/harness",
+    ],
+)
diff --git a/test/benchmarks/fs/bazel_test.go b/test/benchmarks/fs/bazel_test.go
new file mode 100644
index 000000000..aabcdbd87
--- /dev/null
+++ b/test/benchmarks/fs/bazel_test.go
@@ -0,0 +1,103 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package fs
+
+import (
+	"context"
+	"os"
+	"strings"
+	"testing"
+
+	"gvisor.dev/gvisor/pkg/test/dockerutil"
+	"gvisor.dev/gvisor/test/benchmarks/harness"
+)
+
+var h harness.Harness
+
+// Note: CleanCache versions of this test require running with root permissions.
+func BenchmarkABSL(b *testing.B) {
+	// Get a machine from the Harness on which to run.
+	machine, err := h.GetMachine()
+	if err != nil {
+		b.Fatalf("failed to get machine: %v", err)
+	}
+
+	// Dimensions here are clean/dirty cache (do or don't drop caches)
+	// and if the mount on which we are compiling is a tmpfs/bind mount.
+	benchmarks := []struct {
+		name       string
+		clearCache bool // clearCache drops caches before running.
+		tmpfs      bool // tmpfs will run compilation on a tmpfs.
+	}{
+		{name: "CleanCache", clearCache: true, tmpfs: false},
+		{name: "DirtyCache", clearCache: false, tmpfs: false},
+		{name: "CleanCacheTmpfs", clearCache: true, tmpfs: true},
+		{name: "DirtyCacheTmpfs", clearCache: false, tmpfs: true},
+	}
+	for _, bm := range benchmarks {
+		b.Run(bm.name, func(b *testing.B) {
+			// Grab a container.
+			ctx := context.Background()
+			container := machine.GetContainer(ctx, b)
+			defer container.CleanUp(ctx)
+
+			workdir := "/abseil-cpp"
+
+			// Start a container.
+			if err := container.Spawn(ctx, dockerutil.RunOpts{
+				Image: "benchmarks/absl",
+			}, "sleep", "1000"); err != nil {
+				b.Fatalf("run failed with: %v", err)
+			}
+
+			// If we are running on a tmpfs, copy to /tmp which is a tmpfs.
+			if bm.tmpfs {
+				if _, err := container.Exec(ctx, dockerutil.ExecOpts{},
+					"cp", "-r", "/abseil-cpp", "/tmp/."); err != nil {
+					b.Fatal("failed to copy directory: %v", err)
+				}
+				workdir = "/tmp" + workdir
+			}
+
+			// Drop Caches.
+			if bm.clearCache {
+				if out, err := machine.RunCommand("/bin/sh -c sync; echo 3 > /proc/sys/vm/drop_caches"); err != nil {
+					b.Fatalf("failed to drop caches: %v %s", err, out)
+				}
+			}
+
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				got, err := container.Exec(ctx, dockerutil.ExecOpts{
+					WorkDir: workdir,
+				}, "bazel", "build", "-c", "opt", "absl/base/...")
+				if err != nil {
+					b.Fatalf("build failed with: %v", err)
+				}
+				b.StopTimer()
+
+				want := "Build completed successfully"
+				if !strings.Contains(got, want) {
+					b.Fatalf("string %s not in: %s", want, got)
+				}
+				b.StartTimer()
+			}
+		})
+	}
+}
+
+func TestMain(m *testing.M) {
+	h.Init()
+	os.Exit(m.Run())
+}
diff --git a/test/benchmarks/fs/fs.go b/test/benchmarks/fs/fs.go
new file mode 100644
index 000000000..27eb6c56a
--- /dev/null
+++ b/test/benchmarks/fs/fs.go
@@ -0,0 +1,16 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package fs holds benchmarks around filesystem performance.
+package fs
diff --git a/test/benchmarks/harness/BUILD b/test/benchmarks/harness/BUILD
new file mode 100644
index 000000000..c2e316709
--- /dev/null
+++ b/test/benchmarks/harness/BUILD
@@ -0,0 +1,18 @@
+load("//tools:defs.bzl", "go_library")
+
+package(licenses = ["notice"])
+
+go_library(
+    name = "harness",
+    testonly = 1,
+    srcs = [
+        "harness.go",
+        "machine.go",
+        "util.go",
+    ],
+    visibility = ["//:sandbox"],
+    deps = [
+        "//pkg/test/dockerutil",
+        "//pkg/test/testutil",
+    ],
+)
diff --git a/test/benchmarks/harness/harness.go b/test/benchmarks/harness/harness.go
new file mode 100644
index 000000000..68bd7b4cf
--- /dev/null
+++ b/test/benchmarks/harness/harness.go
@@ -0,0 +1,38 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package harness holds utility code for running benchmarks on Docker.
+package harness
+
+import (
+	"flag"
+
+	"gvisor.dev/gvisor/pkg/test/dockerutil"
+)
+
+// Harness is a handle for managing state in benchmark runs.
+type Harness struct {
+}
+
+// Init performs any harness initilialization before runs.
+func (h *Harness) Init() error {
+	flag.Parse()
+	dockerutil.EnsureSupportedDockerVersion()
+	return nil
+}
+
+// GetMachine returns this run's implementation of machine.
+func (h *Harness) GetMachine() (Machine, error) {
+	return &localMachine{}, nil
+}
diff --git a/test/benchmarks/harness/machine.go b/test/benchmarks/harness/machine.go
new file mode 100644
index 000000000..032b387fc
--- /dev/null
+++ b/test/benchmarks/harness/machine.go
@@ -0,0 +1,64 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package harness
+
+import (
+	"context"
+	"net"
+	"os/exec"
+
+	"gvisor.dev/gvisor/pkg/test/dockerutil"
+	"gvisor.dev/gvisor/pkg/test/testutil"
+)
+
+// Machine describes a real machine for use in benchmarks.
+type Machine interface {
+	// GetContainer gets a container from the machine,
+	GetContainer(ctx context.Context, log testutil.Logger) *dockerutil.Container
+
+	// RunCommand runs cmd on this machine.
+	RunCommand(cmd string, args ...string) (string, error)
+
+	// Returns IP Address for the machine.
+	IPAddress() (net.IP, error)
+}
+
+// localMachine describes this machine.
+type localMachine struct {
+}
+
+// GetContainer implements Machine.GetContainer for localMachine.
+func (l *localMachine) GetContainer(ctx context.Context, logger testutil.Logger) *dockerutil.Container {
+	return dockerutil.MakeContainer(ctx, logger)
+}
+
+// RunCommand implements Machine.RunCommand for localMachine.
+func (l *localMachine) RunCommand(cmd string, args ...string) (string, error) {
+	c := exec.Command(cmd, args...)
+	out, err := c.CombinedOutput()
+	return string(out), err
+}
+
+// IPAddress implements Machine.IPAddress.
+func (l *localMachine) IPAddress() (net.IP, error) {
+	conn, err := net.Dial("udp", "8.8.8.8:80")
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+
+	addr := conn.LocalAddr().(*net.UDPAddr)
+	return addr.IP, nil
+}
diff --git a/test/benchmarks/harness/util.go b/test/benchmarks/harness/util.go
new file mode 100644
index 000000000..cc7de6426
--- /dev/null
+++ b/test/benchmarks/harness/util.go
@@ -0,0 +1,38 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package harness
+
+import (
+	"context"
+	"fmt"
+	"net"
+
+	"gvisor.dev/gvisor/pkg/test/dockerutil"
+	"gvisor.dev/gvisor/pkg/test/testutil"
+)
+
+// WaitUntilServing grabs a container from `machine` and waits for a server at
+// IP:port.
+func WaitUntilServing(ctx context.Context, machine Machine, server net.IP, port int) error {
+	var logger testutil.DefaultLogger = "netcat"
+	netcat := machine.GetContainer(ctx, logger)
+	defer netcat.CleanUp(ctx)
+
+	cmd := fmt.Sprintf("while ! nc -zv %s %d; do true; done", server.String(), port)
+	_, err := netcat.Run(ctx, dockerutil.RunOpts{
+		Image: "packetdrill",
+	}, "sh", "-c", cmd)
+	return err
+}
diff --git a/test/benchmarks/network/BUILD b/test/benchmarks/network/BUILD
new file mode 100644
index 000000000..57328456d
--- /dev/null
+++ b/test/benchmarks/network/BUILD
@@ -0,0 +1,25 @@
+load("//tools:defs.bzl", "go_library", "go_test")
+
+package(licenses = ["notice"])
+
+go_library(
+    name = "network",
+    testonly = 1,
+    srcs = ["network.go"],
+)
+
+go_test(
+    name = "network_test",
+    size = "large",
+    srcs = ["iperf_test.go"],
+    library = ":network",
+    tags = [
+        # Requires docker and runsc to be configured before test runs.
+        "manual",
+        "local",
+    ],
+    deps = [
+        "//pkg/test/dockerutil",
+        "//test/benchmarks/harness",
+    ],
+)
diff --git a/test/benchmarks/network/iperf_test.go b/test/benchmarks/network/iperf_test.go
new file mode 100644
index 000000000..72e9c99a8
--- /dev/null
+++ b/test/benchmarks/network/iperf_test.go
@@ -0,0 +1,151 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package network
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+
+	"gvisor.dev/gvisor/pkg/test/dockerutil"
+	"gvisor.dev/gvisor/test/benchmarks/harness"
+)
+
+var h harness.Harness
+
+func BenchmarkIperf(b *testing.B) {
+
+	// Get two machines
+	clientMachine, err := h.GetMachine()
+	if err != nil {
+		b.Fatalf("failed to get machine: %v", err)
+	}
+
+	serverMachine, err := h.GetMachine()
+	if err != nil {
+		b.Fatalf("failed to get machine: %v", err)
+	}
+
+	for _, bm := range []struct {
+		name          string
+		clientRuntime string
+		serverRuntime string
+	}{
+		// We are either measuring the server or the client. The other should be
+		// runc. e.g. Upload sees how fast the runtime under test uploads to a native
+		// server.
+		{name: "Upload", clientRuntime: dockerutil.Runtime(), serverRuntime: "runc"},
+		{name: "Download", clientRuntime: "runc", serverRuntime: dockerutil.Runtime()},
+	} {
+		b.Run(bm.name, func(b *testing.B) {
+
+			// Get a container from the server and set its runtime.
+			ctx := context.Background()
+			server := serverMachine.GetContainer(ctx, b)
+			defer server.CleanUp(ctx)
+			server.Runtime = bm.serverRuntime
+
+			// Get a container from the client and set its runtime.
+			client := clientMachine.GetContainer(ctx, b)
+			defer client.CleanUp(ctx)
+			client.Runtime = bm.clientRuntime
+
+			// iperf serves on port 5001 by default.
+			port := 5001
+
+			// Start the server.
+			if err := server.Spawn(ctx, dockerutil.RunOpts{
+				Image: "benchmarks/iperf",
+				Ports: []int{port},
+			}, "iperf", "-s"); err != nil {
+				b.Fatalf("failed to start server with: %v", err)
+			}
+
+			ip, err := serverMachine.IPAddress()
+			if err != nil {
+				b.Fatalf("failed to find server ip: %v", err)
+			}
+
+			servingPort, err := server.FindPort(ctx, port)
+			if err != nil {
+				b.Fatalf("failed to find port %d: %v", port, err)
+			}
+
+			// Make sure the server is up and serving before we run.
+			if err := harness.WaitUntilServing(ctx, clientMachine, ip, servingPort); err != nil {
+				b.Fatalf("failed to wait for server: %v", err)
+			}
+
+			// iperf report in Kb realtime
+			cmd := fmt.Sprintf("iperf -f K --realtime -c %s -p %d", ip.String(), servingPort)
+
+			// Run the client.
+			b.ResetTimer()
+
+			for i := 0; i < b.N; i++ {
+				out, err := client.Run(ctx, dockerutil.RunOpts{
+					Image: "benchmarks/iperf",
+				}, strings.Split(cmd, " ")...)
+				if err != nil {
+					b.Fatalf("failed to run client: %v", err)
+				}
+				b.StopTimer()
+
+				// Parse bandwidth and report it.
+				bW, err := bandwidth(out)
+				if err != nil {
+					b.Fatalf("failed to parse bandwitdth from %s: %v", out, err)
+				}
+				b.ReportMetric(bW, "KBytes/sec")
+				b.StartTimer()
+			}
+		})
+	}
+}
+
+// bandwidth parses the Bandwidth number from an iperf report. A sample is below.
+func bandwidth(data string) (float64, error) {
+	re := regexp.MustCompile(`\[\s*\d+\][^\n]+\s+(\d+\.?\d*)\s+KBytes/sec`)
+	match := re.FindStringSubmatch(data)
+	if len(match) < 1 {
+		return 0, fmt.Errorf("failed get bandwidth: %s", data)
+	}
+	return strconv.ParseFloat(match[1], 64)
+}
+
+func TestParser(t *testing.T) {
+	sampleData := `
+------------------------------------------------------------
+Client connecting to 10.138.15.215, TCP port 32779
+TCP window size: 45.0 KByte (default)
+------------------------------------------------------------
+[  3] local 10.138.15.216 port 32866 connected with 10.138.15.215 port 32779
+[ ID] Interval       Transfer     Bandwidth
+[  3]  0.0-10.0 sec  459520 KBytes  45900 KBytes/sec
+`
+	bandwidth, err := bandwidth(sampleData)
+	if err != nil || bandwidth != 45900 {
+		t.Fatalf("failed with: %v and %f", err, bandwidth)
+	}
+
+}
+
+func TestMain(m *testing.M) {
+	h.Init()
+	os.Exit(m.Run())
+}
diff --git a/test/benchmarks/network/network.go b/test/benchmarks/network/network.go
new file mode 100644
index 000000000..f480b5bcd
--- /dev/null
+++ b/test/benchmarks/network/network.go
@@ -0,0 +1,16 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package network holds benchmarks around raw network performance.
+package network