diff options
Diffstat (limited to 'test/benchmarks')
| -rw-r--r-- | test/benchmarks/README.md | 118 | ||||
| -rw-r--r-- | test/benchmarks/fs/BUILD | 24 | ||||
| -rw-r--r-- | test/benchmarks/fs/bazel_test.go | 103 | ||||
| -rw-r--r-- | test/benchmarks/fs/fs.go | 16 | ||||
| -rw-r--r-- | test/benchmarks/harness/BUILD | 18 | ||||
| -rw-r--r-- | test/benchmarks/harness/harness.go | 38 | ||||
| -rw-r--r-- | test/benchmarks/harness/machine.go | 64 | ||||
| -rw-r--r-- | test/benchmarks/harness/util.go | 38 | ||||
| -rw-r--r-- | test/benchmarks/network/BUILD | 25 | ||||
| -rw-r--r-- | test/benchmarks/network/iperf_test.go | 151 | ||||
| -rw-r--r-- | test/benchmarks/network/network.go | 16 |
11 files changed, 0 insertions, 611 deletions
diff --git a/test/benchmarks/README.md b/test/benchmarks/README.md deleted file mode 100644 index 9ff602cf1..000000000 --- a/test/benchmarks/README.md +++ /dev/null @@ -1,118 +0,0 @@ -# Benchmark tools - -This package and subpackages are for running macro benchmarks on `runsc`. They -are meant to replace the previous //benchmarks benchmark-tools written in -python. - -Benchmarks are meant to look like regular golang benchmarks using the testing.B -library. - -## Setup - -To run benchmarks you will need: - -* Docker installed (17.09.0 or greater). - -The easiest way to run benchmarks is to use the script at -//scripts/benchmark.sh. - -If not using the script, you will need: - -* `runsc` configured with docker - -Note: benchmarks call the runtime by name. If docker can run it with -`--runtime=` flag, these tools should work. - -## Running benchmarks - -The easiest way to run is with the script at //scripts/benchmarks.sh. The script -will run all benchmarks under //test/benchmarks if a target is not provided. - -```bash -./script/benchmarks.sh //path/to/target -``` - -If you want to run benchmarks manually: - -* Run `make load-all-images` from `//` -* Run with: - -```bash -bazel test --test_arg=--runtime=RUNTIME -c opt --test_output=streamed --test_timeout=600 --test_arg=-test.bench=. --nocache_test_results //path/to/target -``` - -## Writing benchmarks - -Benchmarks consist of docker images as Dockerfiles and golang testing.B -benchmarks. - -### Dockerfiles: - -* Are stored at //images. -* New Dockerfiles go in an appropriately named directory at - `//images/benchmarks/my-cool-dockerfile`. -* Dockerfiles for benchmarks should: - * Use explicitly versioned packages. - * Not use ENV and CMD statements...it is easy to add these in the API. -* Note: A common pattern for getting access to a tmpfs mount is to copy files - there after container start. See: //test/benchmarks/build/bazel_test.go. You - can also make your own with `RunOpts.Mounts`. - -### testing.B packages - -In general, benchmarks should look like this: - -```golang - -var h harness.Harness - -func BenchmarkMyCoolOne(b *testing.B) { - machine, err := h.GetMachine() - // check err - - ctx := context.Background() - container := machine.GetContainer(ctx, b) - defer container.CleanUp(ctx) - - b.ResetTimer() - - //Respect b.N. - for i := 0; i < b.N; i++ { - out, err := container.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/my-cool-image", - Env: []string{"MY_VAR=awesome"}, - other options...see dockerutil - }, "sh", "-c", "echo MY_VAR" ...) - //check err - b.StopTimer() - - // Do parsing and reporting outside of the timer. - number := parseMyMetric(out) - b.ReportMetric(number, "my-cool-custom-metric") - - b.StartTimer() - } -} - -func TestMain(m *testing.M) { - h.Init() - os.Exit(m.Run()) -} -``` - -Some notes on the above: - -* The harness is initiated in the TestMain method and made global to test - module. The harness will handle any presetup that needs to happen with - flags, remote virtual machines (eventually), and other services. -* Respect `b.N` in that users of the benchmark may want to "run for an hour" - or something of the sort. -* Use the `b.ReportMetric` method to report custom metrics. -* Set the timer if time is useful for reporting. There isn't a way to turn off - default metrics in testing.B (B/op, allocs/op, ns/op). -* Take a look at dockerutil at //pkg/test/dockerutil to see all methods - available from containers. The API is based on the "official" - [docker API for golang](https://pkg.go.dev/mod/github.com/docker/docker). -* `harness.GetMachine` marks how many machines this tests needs. If you have a - client and server and to mark them as multiple machines, call it - `GetMachine` twice. diff --git a/test/benchmarks/fs/BUILD b/test/benchmarks/fs/BUILD deleted file mode 100644 index 606331895..000000000 --- a/test/benchmarks/fs/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "fs", - srcs = ["fs.go"], -) - -go_test( - name = "fs_test", - size = "large", - srcs = ["bazel_test.go"], - library = ":fs", - tags = [ - # Requires docker and runsc to be configured before test runs. - "local", - "manual", - ], - deps = [ - "//pkg/test/dockerutil", - "//test/benchmarks/harness", - ], -) diff --git a/test/benchmarks/fs/bazel_test.go b/test/benchmarks/fs/bazel_test.go deleted file mode 100644 index aabcdbd87..000000000 --- a/test/benchmarks/fs/bazel_test.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package fs - -import ( - "context" - "os" - "strings" - "testing" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -var h harness.Harness - -// Note: CleanCache versions of this test require running with root permissions. -func BenchmarkABSL(b *testing.B) { - // Get a machine from the Harness on which to run. - machine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - - // Dimensions here are clean/dirty cache (do or don't drop caches) - // and if the mount on which we are compiling is a tmpfs/bind mount. - benchmarks := []struct { - name string - clearCache bool // clearCache drops caches before running. - tmpfs bool // tmpfs will run compilation on a tmpfs. - }{ - {name: "CleanCache", clearCache: true, tmpfs: false}, - {name: "DirtyCache", clearCache: false, tmpfs: false}, - {name: "CleanCacheTmpfs", clearCache: true, tmpfs: true}, - {name: "DirtyCacheTmpfs", clearCache: false, tmpfs: true}, - } - for _, bm := range benchmarks { - b.Run(bm.name, func(b *testing.B) { - // Grab a container. - ctx := context.Background() - container := machine.GetContainer(ctx, b) - defer container.CleanUp(ctx) - - workdir := "/abseil-cpp" - - // Start a container. - if err := container.Spawn(ctx, dockerutil.RunOpts{ - Image: "benchmarks/absl", - }, "sleep", "1000"); err != nil { - b.Fatalf("run failed with: %v", err) - } - - // If we are running on a tmpfs, copy to /tmp which is a tmpfs. - if bm.tmpfs { - if _, err := container.Exec(ctx, dockerutil.ExecOpts{}, - "cp", "-r", "/abseil-cpp", "/tmp/."); err != nil { - b.Fatal("failed to copy directory: %v", err) - } - workdir = "/tmp" + workdir - } - - // Drop Caches. - if bm.clearCache { - if out, err := machine.RunCommand("/bin/sh -c sync; echo 3 > /proc/sys/vm/drop_caches"); err != nil { - b.Fatalf("failed to drop caches: %v %s", err, out) - } - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - got, err := container.Exec(ctx, dockerutil.ExecOpts{ - WorkDir: workdir, - }, "bazel", "build", "-c", "opt", "absl/base/...") - if err != nil { - b.Fatalf("build failed with: %v", err) - } - b.StopTimer() - - want := "Build completed successfully" - if !strings.Contains(got, want) { - b.Fatalf("string %s not in: %s", want, got) - } - b.StartTimer() - } - }) - } -} - -func TestMain(m *testing.M) { - h.Init() - os.Exit(m.Run()) -} diff --git a/test/benchmarks/fs/fs.go b/test/benchmarks/fs/fs.go deleted file mode 100644 index 27eb6c56a..000000000 --- a/test/benchmarks/fs/fs.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package fs holds benchmarks around filesystem performance. -package fs diff --git a/test/benchmarks/harness/BUILD b/test/benchmarks/harness/BUILD deleted file mode 100644 index c2e316709..000000000 --- a/test/benchmarks/harness/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -package(licenses = ["notice"]) - -go_library( - name = "harness", - testonly = 1, - srcs = [ - "harness.go", - "machine.go", - "util.go", - ], - visibility = ["//:sandbox"], - deps = [ - "//pkg/test/dockerutil", - "//pkg/test/testutil", - ], -) diff --git a/test/benchmarks/harness/harness.go b/test/benchmarks/harness/harness.go deleted file mode 100644 index 68bd7b4cf..000000000 --- a/test/benchmarks/harness/harness.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package harness holds utility code for running benchmarks on Docker. -package harness - -import ( - "flag" - - "gvisor.dev/gvisor/pkg/test/dockerutil" -) - -// Harness is a handle for managing state in benchmark runs. -type Harness struct { -} - -// Init performs any harness initilialization before runs. -func (h *Harness) Init() error { - flag.Parse() - dockerutil.EnsureSupportedDockerVersion() - return nil -} - -// GetMachine returns this run's implementation of machine. -func (h *Harness) GetMachine() (Machine, error) { - return &localMachine{}, nil -} diff --git a/test/benchmarks/harness/machine.go b/test/benchmarks/harness/machine.go deleted file mode 100644 index 032b387fc..000000000 --- a/test/benchmarks/harness/machine.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package harness - -import ( - "context" - "net" - "os/exec" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/pkg/test/testutil" -) - -// Machine describes a real machine for use in benchmarks. -type Machine interface { - // GetContainer gets a container from the machine, - GetContainer(ctx context.Context, log testutil.Logger) *dockerutil.Container - - // RunCommand runs cmd on this machine. - RunCommand(cmd string, args ...string) (string, error) - - // Returns IP Address for the machine. - IPAddress() (net.IP, error) -} - -// localMachine describes this machine. -type localMachine struct { -} - -// GetContainer implements Machine.GetContainer for localMachine. -func (l *localMachine) GetContainer(ctx context.Context, logger testutil.Logger) *dockerutil.Container { - return dockerutil.MakeContainer(ctx, logger) -} - -// RunCommand implements Machine.RunCommand for localMachine. -func (l *localMachine) RunCommand(cmd string, args ...string) (string, error) { - c := exec.Command(cmd, args...) - out, err := c.CombinedOutput() - return string(out), err -} - -// IPAddress implements Machine.IPAddress. -func (l *localMachine) IPAddress() (net.IP, error) { - conn, err := net.Dial("udp", "8.8.8.8:80") - if err != nil { - return nil, err - } - defer conn.Close() - - addr := conn.LocalAddr().(*net.UDPAddr) - return addr.IP, nil -} diff --git a/test/benchmarks/harness/util.go b/test/benchmarks/harness/util.go deleted file mode 100644 index cc7de6426..000000000 --- a/test/benchmarks/harness/util.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package harness - -import ( - "context" - "fmt" - "net" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/pkg/test/testutil" -) - -// WaitUntilServing grabs a container from `machine` and waits for a server at -// IP:port. -func WaitUntilServing(ctx context.Context, machine Machine, server net.IP, port int) error { - var logger testutil.DefaultLogger = "netcat" - netcat := machine.GetContainer(ctx, logger) - defer netcat.CleanUp(ctx) - - cmd := fmt.Sprintf("while ! nc -zv %s %d; do true; done", server.String(), port) - _, err := netcat.Run(ctx, dockerutil.RunOpts{ - Image: "packetdrill", - }, "sh", "-c", cmd) - return err -} diff --git a/test/benchmarks/network/BUILD b/test/benchmarks/network/BUILD deleted file mode 100644 index 57328456d..000000000 --- a/test/benchmarks/network/BUILD +++ /dev/null @@ -1,25 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "network", - testonly = 1, - srcs = ["network.go"], -) - -go_test( - name = "network_test", - size = "large", - srcs = ["iperf_test.go"], - library = ":network", - tags = [ - # Requires docker and runsc to be configured before test runs. - "manual", - "local", - ], - deps = [ - "//pkg/test/dockerutil", - "//test/benchmarks/harness", - ], -) diff --git a/test/benchmarks/network/iperf_test.go b/test/benchmarks/network/iperf_test.go deleted file mode 100644 index 72e9c99a8..000000000 --- a/test/benchmarks/network/iperf_test.go +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package network - -import ( - "context" - "fmt" - "os" - "regexp" - "strconv" - "strings" - "testing" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -var h harness.Harness - -func BenchmarkIperf(b *testing.B) { - - // Get two machines - clientMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - - serverMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - - for _, bm := range []struct { - name string - clientRuntime string - serverRuntime string - }{ - // We are either measuring the server or the client. The other should be - // runc. e.g. Upload sees how fast the runtime under test uploads to a native - // server. - {name: "Upload", clientRuntime: dockerutil.Runtime(), serverRuntime: "runc"}, - {name: "Download", clientRuntime: "runc", serverRuntime: dockerutil.Runtime()}, - } { - b.Run(bm.name, func(b *testing.B) { - - // Get a container from the server and set its runtime. - ctx := context.Background() - server := serverMachine.GetContainer(ctx, b) - defer server.CleanUp(ctx) - server.Runtime = bm.serverRuntime - - // Get a container from the client and set its runtime. - client := clientMachine.GetContainer(ctx, b) - defer client.CleanUp(ctx) - client.Runtime = bm.clientRuntime - - // iperf serves on port 5001 by default. - port := 5001 - - // Start the server. - if err := server.Spawn(ctx, dockerutil.RunOpts{ - Image: "benchmarks/iperf", - Ports: []int{port}, - }, "iperf", "-s"); err != nil { - b.Fatalf("failed to start server with: %v", err) - } - - ip, err := serverMachine.IPAddress() - if err != nil { - b.Fatalf("failed to find server ip: %v", err) - } - - servingPort, err := server.FindPort(ctx, port) - if err != nil { - b.Fatalf("failed to find port %d: %v", port, err) - } - - // Make sure the server is up and serving before we run. - if err := harness.WaitUntilServing(ctx, clientMachine, ip, servingPort); err != nil { - b.Fatalf("failed to wait for server: %v", err) - } - - // iperf report in Kb realtime - cmd := fmt.Sprintf("iperf -f K --realtime -c %s -p %d", ip.String(), servingPort) - - // Run the client. - b.ResetTimer() - - for i := 0; i < b.N; i++ { - out, err := client.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/iperf", - }, strings.Split(cmd, " ")...) - if err != nil { - b.Fatalf("failed to run client: %v", err) - } - b.StopTimer() - - // Parse bandwidth and report it. - bW, err := bandwidth(out) - if err != nil { - b.Fatalf("failed to parse bandwitdth from %s: %v", out, err) - } - b.ReportMetric(bW, "KBytes/sec") - b.StartTimer() - } - }) - } -} - -// bandwidth parses the Bandwidth number from an iperf report. A sample is below. -func bandwidth(data string) (float64, error) { - re := regexp.MustCompile(`\[\s*\d+\][^\n]+\s+(\d+\.?\d*)\s+KBytes/sec`) - match := re.FindStringSubmatch(data) - if len(match) < 1 { - return 0, fmt.Errorf("failed get bandwidth: %s", data) - } - return strconv.ParseFloat(match[1], 64) -} - -func TestParser(t *testing.T) { - sampleData := ` ------------------------------------------------------------- -Client connecting to 10.138.15.215, TCP port 32779 -TCP window size: 45.0 KByte (default) ------------------------------------------------------------- -[ 3] local 10.138.15.216 port 32866 connected with 10.138.15.215 port 32779 -[ ID] Interval Transfer Bandwidth -[ 3] 0.0-10.0 sec 459520 KBytes 45900 KBytes/sec -` - bandwidth, err := bandwidth(sampleData) - if err != nil || bandwidth != 45900 { - t.Fatalf("failed with: %v and %f", err, bandwidth) - } - -} - -func TestMain(m *testing.M) { - h.Init() - os.Exit(m.Run()) -} diff --git a/test/benchmarks/network/network.go b/test/benchmarks/network/network.go deleted file mode 100644 index f480b5bcd..000000000 --- a/test/benchmarks/network/network.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package network holds benchmarks around raw network performance. -package network |