diff options
Diffstat (limited to 'test/benchmarks')
| -rw-r--r-- | test/benchmarks/README.md | 157 | ||||
| -rw-r--r-- | test/benchmarks/database/BUILD | 28 | ||||
| -rw-r--r-- | test/benchmarks/database/database.go | 31 | ||||
| -rw-r--r-- | test/benchmarks/database/redis_test.go | 197 | ||||
| -rw-r--r-- | test/benchmarks/fs/BUILD | 23 | ||||
| -rw-r--r-- | test/benchmarks/fs/bazel_test.go | 109 | ||||
| -rw-r--r-- | test/benchmarks/fs/fs.go | 31 | ||||
| -rw-r--r-- | test/benchmarks/harness/BUILD | 18 | ||||
| -rw-r--r-- | test/benchmarks/harness/harness.go | 38 | ||||
| -rw-r--r-- | test/benchmarks/harness/machine.go | 81 | ||||
| -rw-r--r-- | test/benchmarks/harness/util.go | 46 | ||||
| -rw-r--r-- | test/benchmarks/media/BUILD | 21 | ||||
| -rw-r--r-- | test/benchmarks/media/ffmpeg_test.go | 52 | ||||
| -rw-r--r-- | test/benchmarks/media/media.go | 31 | ||||
| -rw-r--r-- | test/benchmarks/network/BUILD | 30 | ||||
| -rw-r--r-- | test/benchmarks/network/httpd_test.go | 277 | ||||
| -rw-r--r-- | test/benchmarks/network/iperf_test.go | 150 | ||||
| -rw-r--r-- | test/benchmarks/network/network.go | 31 |
18 files changed, 0 insertions, 1351 deletions
diff --git a/test/benchmarks/README.md b/test/benchmarks/README.md deleted file mode 100644 index d1bbabf6f..000000000 --- a/test/benchmarks/README.md +++ /dev/null @@ -1,157 +0,0 @@ -# Benchmark tools - -This package and subpackages are for running macro benchmarks on `runsc`. They -are meant to replace the previous //benchmarks benchmark-tools written in -python. - -Benchmarks are meant to look like regular golang benchmarks using the testing.B -library. - -## Setup - -To run benchmarks you will need: - -* Docker installed (17.09.0 or greater). - -The easiest way to setup runsc for running benchmarks is to use the make file. -From the root directory: - -* Download images: `make load-all-images` -* Install runsc suitable for benchmarking, which should probably not have - strace or debug logs enabled. For example:`make configure RUNTIME=myrunsc - ARGS=--platform=kvm`. -* Restart docker: `sudo service docker restart` - -You should now have a runtime with the following options configured in -`/etc/docker/daemon.json` - -``` -"myrunsc": { - "path": "/tmp/myrunsc/runsc", - "runtimeArgs": [ - "--debug-log", - "/tmp/bench/logs/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND%", - "--platform=kvm" - ] - }, - -``` - -This runtime has been configured with a debugging off and strace logs off and is -using kvm for demonstration. - -## Running benchmarks - -Given the runtime above runtime `myrunsc`, run benchmarks with the following: - -``` -make sudo TARGETS=//path/to:target ARGS="--runtime=myrunsc -test.v \ - -test.bench=." OPTIONS="-c opt -``` - -For example, to run only the Iperf tests: - -``` -make sudo TARGETS=//test/benchmarks/network:network_test \ - ARGS="--runtime=myrunsc -test.v -test.bench=Iperf" OPTIONS="-c opt" -``` - -Benchmarks are run with root as some benchmarks require root privileges to do -things like drop caches. - -## Writing benchmarks - -Benchmarks consist of docker images as Dockerfiles and golang testing.B -benchmarks. - -### Dockerfiles: - -* Are stored at //images. -* New Dockerfiles go in an appropriately named directory at - `//images/benchmarks/my-cool-dockerfile`. -* Dockerfiles for benchmarks should: - * Use explicitly versioned packages. - * Not use ENV and CMD statements...it is easy to add these in the API. -* Note: A common pattern for getting access to a tmpfs mount is to copy files - there after container start. See: //test/benchmarks/build/bazel_test.go. You - can also make your own with `RunOpts.Mounts`. - -### testing.B packages - -In general, benchmarks should look like this: - -```golang - -var h harness.Harness - -func BenchmarkMyCoolOne(b *testing.B) { - machine, err := h.GetMachine() - // check err - defer machine.CleanUp() - - ctx := context.Background() - container := machine.GetContainer(ctx, b) - defer container.CleanUp(ctx) - - b.ResetTimer() - - //Respect b.N. - for i := 0; i < b.N; i++ { - out, err := container.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/my-cool-image", - Env: []string{"MY_VAR=awesome"}, - other options...see dockerutil - }, "sh", "-c", "echo MY_VAR") - //check err - b.StopTimer() - - // Do parsing and reporting outside of the timer. - number := parseMyMetric(out) - b.ReportMetric(number, "my-cool-custom-metric") - - b.StartTimer() - } -} - -func TestMain(m *testing.M) { - h.Init() - os.Exit(m.Run()) -} -``` - -Some notes on the above: - -* The harness is initiated in the TestMain method and made global to test - module. The harness will handle any presetup that needs to happen with - flags, remote virtual machines (eventually), and other services. -* Respect `b.N` in that users of the benchmark may want to "run for an hour" - or something of the sort. -* Use the `b.ReportMetric()` method to report custom metrics. -* Set the timer if time is useful for reporting. There isn't a way to turn off - default metrics in testing.B (B/op, allocs/op, ns/op). -* Take a look at dockerutil at //pkg/test/dockerutil to see all methods - available from containers. The API is based on the "official" - [docker API for golang](https://pkg.go.dev/mod/github.com/docker/docker). -* `harness.GetMachine()` marks how many machines this tests needs. If you have - a client and server and to mark them as multiple machines, call - `harness.GetMachine()` twice. - -## Profiling - -For profiling, the runtime is required to have the `--profile` flag enabled. -This flag loosens seccomp filters so that the runtime can write profile data to -disk. This configuration is not recommended for production. - -* Install runsc with the `--profile` flag: `make configure RUNTIME=myrunsc - ARGS="--profile --platform=kvm --vfs2"`. The kvm and vfs2 flags are not - required, but are included for demonstration. -* Restart docker: `sudo service docker restart` - -To run and generate CPU profiles fs_test test run: - -``` -make sudo TARGETS=//test/benchmarks/fs:fs_test \ - ARGS="--runtime=myrunsc -test.v -test.bench=. --pprof-cpu" OPTIONS="-c opt" -``` - -Profiles would be at: `/tmp/profile/myrunsc/CONTAINERNAME/cpu.pprof` diff --git a/test/benchmarks/database/BUILD b/test/benchmarks/database/BUILD deleted file mode 100644 index 5e33465cd..000000000 --- a/test/benchmarks/database/BUILD +++ /dev/null @@ -1,28 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "database", - testonly = 1, - srcs = ["database.go"], - deps = ["//test/benchmarks/harness"], -) - -go_test( - name = "database_test", - size = "enormous", - srcs = [ - "redis_test.go", - ], - library = ":database", - tags = [ - # Requires docker and runsc to be configured before test runs. - "manual", - "local", - ], - deps = [ - "//pkg/test/dockerutil", - "//test/benchmarks/harness", - ], -) diff --git a/test/benchmarks/database/database.go b/test/benchmarks/database/database.go deleted file mode 100644 index 9eeb59f9a..000000000 --- a/test/benchmarks/database/database.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package database holds benchmarks around database applications. -package database - -import ( - "os" - "testing" - - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -var h harness.Harness - -// TestMain is the main method for package database. -func TestMain(m *testing.M) { - h.Init() - os.Exit(m.Run()) -} diff --git a/test/benchmarks/database/redis_test.go b/test/benchmarks/database/redis_test.go deleted file mode 100644 index 6d39f4d66..000000000 --- a/test/benchmarks/database/redis_test.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package database - -import ( - "context" - "fmt" - "regexp" - "strconv" - "strings" - "testing" - "time" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -// All possible operations from redis. Note: "ping" will -// run both PING_INLINE and PING_BUILD. -var operations []string = []string{ - "PING_INLINE", - "PING_BULK", - "SET", - "GET", - "INCR", - "LPUSH", - "RPUSH", - "LPOP", - "RPOP", - "SADD", - "HSET", - "SPOP", - "LRANGE_100", - "LRANGE_300", - "LRANGE_500", - "LRANGE_600", - "MSET", -} - -// BenchmarkRedis runs redis-benchmark against a redis instance and reports -// data in queries per second. Each is reported by named operation (e.g. LPUSH). -func BenchmarkRedis(b *testing.B) { - clientMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer clientMachine.CleanUp() - - serverMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer serverMachine.CleanUp() - - // Redis runs on port 6379 by default. - port := 6379 - ctx := context.Background() - - for _, operation := range operations { - b.Run(operation, func(b *testing.B) { - server := serverMachine.GetContainer(ctx, b) - defer server.CleanUp(ctx) - - // The redis docker container takes no arguments to run a redis server. - if err := server.Spawn(ctx, dockerutil.RunOpts{ - Image: "benchmarks/redis", - Ports: []int{port}, - }); err != nil { - b.Fatalf("failed to start redis server with: %v", err) - } - - if out, err := server.WaitForOutput(ctx, "Ready to accept connections", 3*time.Second); err != nil { - b.Fatalf("failed to start redis server: %v %s", err, out) - } - - ip, err := serverMachine.IPAddress() - if err != nil { - b.Fatal("failed to get IP from server: %v", err) - } - - serverPort, err := server.FindPort(ctx, port) - if err != nil { - b.Fatal("failed to get IP from server: %v", err) - } - - if err = harness.WaitUntilServing(ctx, clientMachine, ip, serverPort); err != nil { - b.Fatalf("failed to start redis with: %v", err) - } - - // runs redis benchmark -t operation for 100K requests against server. - cmd := strings.Split( - fmt.Sprintf("redis-benchmark --csv -t %s -h %s -p %d", operation, ip, serverPort), " ") - - // There is no -t PING_BULK for redis-benchmark, so adjust the command in that case. - // Note that "ping" will run both PING_INLINE and PING_BULK. - if operation == "PING_BULK" { - cmd = strings.Split( - fmt.Sprintf("redis-benchmark --csv -t ping -h %s -p %d", ip, serverPort), " ") - } - // Reset profiles and timer to begin the measurement. - server.RestartProfiles() - b.ResetTimer() - for i := 0; i < b.N; i++ { - client := clientMachine.GetNativeContainer(ctx, b) - defer client.CleanUp(ctx) - out, err := client.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/redis", - }, cmd...) - if err != nil { - b.Fatalf("redis-benchmark failed with: %v", err) - } - - // Stop time while we parse results. - b.StopTimer() - result, err := parseOperation(operation, out) - if err != nil { - b.Fatalf("parsing result %s failed with err: %v", out, err) - } - b.ReportMetric(result, operation) // operations per second - b.StartTimer() - } - }) - } -} - -// parseOperation grabs the metric operations per second from redis-benchmark output. -func parseOperation(operation, data string) (float64, error) { - re := regexp.MustCompile(fmt.Sprintf(`"%s( .*)?","(\d*\.\d*)"`, operation)) - match := re.FindStringSubmatch(data) - // If no match, simply don't add it to the result map. - if len(match) < 3 { - return 0.0, fmt.Errorf("could not find %s in %s", operation, data) - } - return strconv.ParseFloat(match[2], 64) -} - -// TestParser tests the parser on sample data. -func TestParser(t *testing.T) { - sampleData := ` - "PING_INLINE","48661.80" - "PING_BULK","50301.81" - "SET","48923.68" - "GET","49382.71" - "INCR","49975.02" - "LPUSH","49875.31" - "RPUSH","50276.52" - "LPOP","50327.12" - "RPOP","50556.12" - "SADD","49504.95" - "HSET","49504.95" - "SPOP","50025.02" - "LPUSH (needed to benchmark LRANGE)","48875.86" - "LRANGE_100 (first 100 elements)","33955.86" - "LRANGE_300 (first 300 elements)","16550.81" - "LRANGE_500 (first 450 elements)","13653.74" - "LRANGE_600 (first 600 elements)","11219.57" - "MSET (10 keys)","44682.75" - ` - wants := map[string]float64{ - "PING_INLINE": 48661.80, - "PING_BULK": 50301.81, - "SET": 48923.68, - "GET": 49382.71, - "INCR": 49975.02, - "LPUSH": 49875.31, - "RPUSH": 50276.52, - "LPOP": 50327.12, - "RPOP": 50556.12, - "SADD": 49504.95, - "HSET": 49504.95, - "SPOP": 50025.02, - "LRANGE_100": 33955.86, - "LRANGE_300": 16550.81, - "LRANGE_500": 13653.74, - "LRANGE_600": 11219.57, - "MSET": 44682.75, - } - for op, want := range wants { - if got, err := parseOperation(op, sampleData); err != nil { - t.Fatalf("failed to parse %s: %v", op, err) - } else if want != got { - t.Fatalf("wanted %f for op %s, got %f", want, op, got) - } - } -} diff --git a/test/benchmarks/fs/BUILD b/test/benchmarks/fs/BUILD deleted file mode 100644 index 2874cdbb3..000000000 --- a/test/benchmarks/fs/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "fs", - testonly = 1, - srcs = ["fs.go"], - deps = ["//test/benchmarks/harness"], -) - -go_test( - name = "fs_test", - size = "large", - srcs = ["bazel_test.go"], - library = ":fs", - tags = [ - # Requires docker and runsc to be configured before test runs. - "local", - "manual", - ], - deps = ["//pkg/test/dockerutil"], -) diff --git a/test/benchmarks/fs/bazel_test.go b/test/benchmarks/fs/bazel_test.go deleted file mode 100644 index 9b652fd43..000000000 --- a/test/benchmarks/fs/bazel_test.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package fs - -import ( - "context" - "fmt" - "strings" - "testing" - - "gvisor.dev/gvisor/pkg/test/dockerutil" -) - -// Note: CleanCache versions of this test require running with root permissions. -func BenchmarkABSL(b *testing.B) { - // Get a machine from the Harness on which to run. - machine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer machine.CleanUp() - - // Dimensions here are clean/dirty cache (do or don't drop caches) - // and if the mount on which we are compiling is a tmpfs/bind mount. - benchmarks := []struct { - name string - clearCache bool // clearCache drops caches before running. - tmpfs bool // tmpfs will run compilation on a tmpfs. - }{ - {name: "CleanCache", clearCache: true, tmpfs: false}, - {name: "DirtyCache", clearCache: false, tmpfs: false}, - {name: "CleanCacheTmpfs", clearCache: true, tmpfs: true}, - {name: "DirtyCacheTmpfs", clearCache: false, tmpfs: true}, - } - for _, bm := range benchmarks { - b.Run(bm.name, func(b *testing.B) { - // Grab a container. - ctx := context.Background() - container := machine.GetContainer(ctx, b) - defer container.CleanUp(ctx) - - workdir := "/abseil-cpp" - - // Start a container and sleep by an order of b.N. - if err := container.Spawn(ctx, dockerutil.RunOpts{ - Image: "benchmarks/absl", - }, "sleep", fmt.Sprintf("%d", 1000000)); err != nil { - b.Fatalf("run failed with: %v", err) - } - - // If we are running on a tmpfs, copy to /tmp which is a tmpfs. - if bm.tmpfs { - if _, err := container.Exec(ctx, dockerutil.ExecOpts{}, - "cp", "-r", "/abseil-cpp", "/tmp/."); err != nil { - b.Fatal("failed to copy directory: %v", err) - } - workdir = "/tmp" + workdir - } - - // Restart profiles after the copy. - container.RestartProfiles() - b.ResetTimer() - // Drop Caches and bazel clean should happen inside the loop as we may use - // time options with b.N. (e.g. Run for an hour.) - for i := 0; i < b.N; i++ { - b.StopTimer() - // Drop Caches for clear cache runs. - if bm.clearCache { - if out, err := machine.RunCommand("/bin/sh", "-c", "sync && sysctl vm.drop_caches=3"); err != nil { - b.Skipf("failed to drop caches: %v %s. You probably need root.", err, out) - } - } - b.StartTimer() - - got, err := container.Exec(ctx, dockerutil.ExecOpts{ - WorkDir: workdir, - }, "bazel", "build", "-c", "opt", "absl/base/...") - if err != nil { - b.Fatalf("build failed with: %v", err) - } - b.StopTimer() - - want := "Build completed successfully" - if !strings.Contains(got, want) { - b.Fatalf("string %s not in: %s", want, got) - } - // Clean bazel in case we use b.N. - _, err = container.Exec(ctx, dockerutil.ExecOpts{ - WorkDir: workdir, - }, "bazel", "clean") - if err != nil { - b.Fatalf("build failed with: %v", err) - } - b.StartTimer() - } - }) - } -} diff --git a/test/benchmarks/fs/fs.go b/test/benchmarks/fs/fs.go deleted file mode 100644 index e5ca28c3b..000000000 --- a/test/benchmarks/fs/fs.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package fs holds benchmarks around filesystem performance. -package fs - -import ( - "os" - "testing" - - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -var h harness.Harness - -// TestMain is the main method for package fs. -func TestMain(m *testing.M) { - h.Init() - os.Exit(m.Run()) -} diff --git a/test/benchmarks/harness/BUILD b/test/benchmarks/harness/BUILD deleted file mode 100644 index c2e316709..000000000 --- a/test/benchmarks/harness/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -package(licenses = ["notice"]) - -go_library( - name = "harness", - testonly = 1, - srcs = [ - "harness.go", - "machine.go", - "util.go", - ], - visibility = ["//:sandbox"], - deps = [ - "//pkg/test/dockerutil", - "//pkg/test/testutil", - ], -) diff --git a/test/benchmarks/harness/harness.go b/test/benchmarks/harness/harness.go deleted file mode 100644 index 68bd7b4cf..000000000 --- a/test/benchmarks/harness/harness.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package harness holds utility code for running benchmarks on Docker. -package harness - -import ( - "flag" - - "gvisor.dev/gvisor/pkg/test/dockerutil" -) - -// Harness is a handle for managing state in benchmark runs. -type Harness struct { -} - -// Init performs any harness initilialization before runs. -func (h *Harness) Init() error { - flag.Parse() - dockerutil.EnsureSupportedDockerVersion() - return nil -} - -// GetMachine returns this run's implementation of machine. -func (h *Harness) GetMachine() (Machine, error) { - return &localMachine{}, nil -} diff --git a/test/benchmarks/harness/machine.go b/test/benchmarks/harness/machine.go deleted file mode 100644 index 88e5e841b..000000000 --- a/test/benchmarks/harness/machine.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package harness - -import ( - "context" - "net" - "os/exec" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/pkg/test/testutil" -) - -// Machine describes a real machine for use in benchmarks. -type Machine interface { - // GetContainer gets a container from the machine. The container uses the - // runtime under test and is profiled if requested by flags. - GetContainer(ctx context.Context, log testutil.Logger) *dockerutil.Container - - // GetNativeContainer gets a native container from the machine. Native containers - // use runc by default and are not profiled. - GetNativeContainer(ctx context.Context, log testutil.Logger) *dockerutil.Container - - // RunCommand runs cmd on this machine. - RunCommand(cmd string, args ...string) (string, error) - - // Returns IP Address for the machine. - IPAddress() (net.IP, error) - - // CleanUp cleans up this machine. - CleanUp() -} - -// localMachine describes this machine. -type localMachine struct { -} - -// GetContainer implements Machine.GetContainer for localMachine. -func (l *localMachine) GetContainer(ctx context.Context, logger testutil.Logger) *dockerutil.Container { - return dockerutil.MakeContainer(ctx, logger) -} - -// GetContainer implements Machine.GetContainer for localMachine. -func (l *localMachine) GetNativeContainer(ctx context.Context, logger testutil.Logger) *dockerutil.Container { - return dockerutil.MakeNativeContainer(ctx, logger) -} - -// RunCommand implements Machine.RunCommand for localMachine. -func (l *localMachine) RunCommand(cmd string, args ...string) (string, error) { - c := exec.Command(cmd, args...) - out, err := c.CombinedOutput() - return string(out), err -} - -// IPAddress implements Machine.IPAddress. -func (l *localMachine) IPAddress() (net.IP, error) { - conn, err := net.Dial("udp", "8.8.8.8:80") - if err != nil { - return nil, err - } - defer conn.Close() - - addr := conn.LocalAddr().(*net.UDPAddr) - return addr.IP, nil -} - -// CleanUp implements Machine.CleanUp and does nothing for localMachine. -func (*localMachine) CleanUp() { -} diff --git a/test/benchmarks/harness/util.go b/test/benchmarks/harness/util.go deleted file mode 100644 index bc551c582..000000000 --- a/test/benchmarks/harness/util.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package harness - -import ( - "context" - "fmt" - "net" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/pkg/test/testutil" -) - -// WaitUntilServing grabs a container from `machine` and waits for a server at -// IP:port. -func WaitUntilServing(ctx context.Context, machine Machine, server net.IP, port int) error { - var logger testutil.DefaultLogger = "netcat" - netcat := machine.GetNativeContainer(ctx, logger) - defer netcat.CleanUp(ctx) - - cmd := fmt.Sprintf("while ! nc -zv %s %d; do true; done", server, port) - _, err := netcat.Run(ctx, dockerutil.RunOpts{ - Image: "packetdrill", - }, "sh", "-c", cmd) - return err -} - -// DropCaches drops caches on the provided machine. Requires root. -func DropCaches(machine Machine) error { - if out, err := machine.RunCommand("/bin/sh", "-c", "sync | sysctl vm.drop_caches=3"); err != nil { - return fmt.Errorf("failed to drop caches: %v logs: %s", err, out) - } - return nil -} diff --git a/test/benchmarks/media/BUILD b/test/benchmarks/media/BUILD deleted file mode 100644 index 6c41fc4f6..000000000 --- a/test/benchmarks/media/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "media", - testonly = 1, - srcs = ["media.go"], - deps = ["//test/benchmarks/harness"], -) - -go_test( - name = "media_test", - size = "large", - srcs = ["ffmpeg_test.go"], - library = ":media", - deps = [ - "//pkg/test/dockerutil", - "//test/benchmarks/harness", - ], -) diff --git a/test/benchmarks/media/ffmpeg_test.go b/test/benchmarks/media/ffmpeg_test.go deleted file mode 100644 index bfcfbab80..000000000 --- a/test/benchmarks/media/ffmpeg_test.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package media - -import ( - "context" - "strings" - "testing" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -// BenchmarkFfmpeg runs ffmpeg in a container and records runtime. -// BenchmarkFfmpeg should run as root to drop caches. -func BenchmarkFfmpeg(b *testing.B) { - machine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer machine.CleanUp() - - ctx := context.Background() - container := machine.GetContainer(ctx, b) - cmd := strings.Split("ffmpeg -i video.mp4 -c:v libx264 -preset veryslow output.mp4", " ") - - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - if err := harness.DropCaches(machine); err != nil { - b.Skipf("failed to drop caches: %v. You probably need root.", err) - } - b.StartTimer() - - if _, err := container.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/ffmpeg", - }, cmd...); err != nil { - b.Fatalf("failed to run container: %v", err) - } - } -} diff --git a/test/benchmarks/media/media.go b/test/benchmarks/media/media.go deleted file mode 100644 index c7b35b758..000000000 --- a/test/benchmarks/media/media.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package media holds benchmarks around media processing applications. -package media - -import ( - "os" - "testing" - - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -var h harness.Harness - -// TestMain is the main method for package media. -func TestMain(m *testing.M) { - h.Init() - os.Exit(m.Run()) -} diff --git a/test/benchmarks/network/BUILD b/test/benchmarks/network/BUILD deleted file mode 100644 index 363041fb7..000000000 --- a/test/benchmarks/network/BUILD +++ /dev/null @@ -1,30 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "network", - testonly = 1, - srcs = ["network.go"], - deps = ["//test/benchmarks/harness"], -) - -go_test( - name = "network_test", - size = "large", - srcs = [ - "httpd_test.go", - "iperf_test.go", - ], - library = ":network", - tags = [ - # Requires docker and runsc to be configured before test runs. - "manual", - "local", - ], - deps = [ - "//pkg/test/dockerutil", - "//pkg/test/testutil", - "//test/benchmarks/harness", - ], -) diff --git a/test/benchmarks/network/httpd_test.go b/test/benchmarks/network/httpd_test.go deleted file mode 100644 index fe23ca949..000000000 --- a/test/benchmarks/network/httpd_test.go +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package network - -import ( - "context" - "fmt" - "regexp" - "strconv" - "testing" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -// see Dockerfile '//images/benchmarks/httpd'. -var docs = map[string]string{ - "notfound": "notfound", - "1Kb": "latin1k.txt", - "10Kb": "latin10k.txt", - "100Kb": "latin100k.txt", - "1000Kb": "latin1000k.txt", - "1Mb": "latin1024k.txt", - "10Mb": "latin10240k.txt", -} - -// BenchmarkHttpdConcurrency iterates the concurrency argument and tests -// how well the runtime under test handles requests in parallel. -func BenchmarkHttpdConcurrency(b *testing.B) { - // Grab a machine for the client and server. - clientMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get client: %v", err) - } - defer clientMachine.CleanUp() - - serverMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get server: %v", err) - } - defer serverMachine.CleanUp() - - // The test iterates over client concurrency, so set other parameters. - requests := 10000 - concurrency := []int{1, 5, 10, 25} - doc := docs["10Kb"] - - for _, c := range concurrency { - b.Run(fmt.Sprintf("%d", c), func(b *testing.B) { - runHttpd(b, clientMachine, serverMachine, doc, requests, c) - }) - } -} - -// BenchmarkHttpdDocSize iterates over different sized payloads, testing how -// well the runtime handles different payload sizes. -func BenchmarkHttpdDocSize(b *testing.B) { - clientMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer clientMachine.CleanUp() - - serverMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer serverMachine.CleanUp() - - requests := 10000 - concurrency := 1 - - for name, filename := range docs { - b.Run(name, func(b *testing.B) { - runHttpd(b, clientMachine, serverMachine, filename, requests, concurrency) - }) - } -} - -// runHttpd runs a single test run. -func runHttpd(b *testing.B, clientMachine, serverMachine harness.Machine, doc string, requests, concurrency int) { - b.Helper() - - // Grab a container from the server. - ctx := context.Background() - server := serverMachine.GetContainer(ctx, b) - defer server.CleanUp(ctx) - - // Copy the docs to /tmp and serve from there. - cmd := "mkdir -p /tmp/html; cp -r /local /tmp/html/.; apache2 -X" - port := 80 - - // Start the server. - server.Spawn(ctx, dockerutil.RunOpts{ - Image: "benchmarks/httpd", - Ports: []int{port}, - Env: []string{ - // Standard environmental variables for httpd. - "APACHE_RUN_DIR=/tmp", - "APACHE_RUN_USER=nobody", - "APACHE_RUN_GROUP=nogroup", - "APACHE_LOG_DIR=/tmp", - "APACHE_PID_FILE=/tmp/apache.pid", - }, - }, "sh", "-c", cmd) - - ip, err := serverMachine.IPAddress() - if err != nil { - b.Fatalf("failed to find server ip: %v", err) - } - - servingPort, err := server.FindPort(ctx, port) - if err != nil { - b.Fatalf("failed to find server port %d: %v", port, err) - } - - // Check the server is serving. - harness.WaitUntilServing(ctx, clientMachine, ip, servingPort) - - // Grab a client. - client := clientMachine.GetNativeContainer(ctx, b) - defer client.CleanUp(ctx) - - path := fmt.Sprintf("http://%s:%d/%s", ip, servingPort, doc) - // See apachebench (ab) for flags. - cmd = fmt.Sprintf("ab -n %d -c %d %s", requests, concurrency, path) - - b.ResetTimer() - server.RestartProfiles() - for i := 0; i < b.N; i++ { - out, err := client.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/ab", - }, "sh", "-c", cmd) - if err != nil { - b.Fatalf("run failed with: %v", err) - } - - b.StopTimer() - - // Parse and report custom metrics. - transferRate, err := parseTransferRate(out) - if err != nil { - b.Logf("failed to parse transferrate: %v", err) - } - b.ReportMetric(transferRate*1024, "transfer_rate") // Convert from Kb/s to b/s. - - latency, err := parseLatency(out) - if err != nil { - b.Logf("failed to parse latency: %v", err) - } - b.ReportMetric(latency/1000, "mean_latency") // Convert from ms to s. - - reqPerSecond, err := parseRequestsPerSecond(out) - if err != nil { - b.Logf("failed to parse requests per second: %v", err) - } - b.ReportMetric(reqPerSecond, "requests_per_second") - - b.StartTimer() - } -} - -var transferRateRE = regexp.MustCompile(`Transfer rate:\s+(\d+\.?\d+?)\s+\[Kbytes/sec\]\s+received`) - -// parseTransferRate parses transfer rate from apachebench output. -func parseTransferRate(data string) (float64, error) { - match := transferRateRE.FindStringSubmatch(data) - if len(match) < 2 { - return 0, fmt.Errorf("failed get bandwidth: %s", data) - } - return strconv.ParseFloat(match[1], 64) -} - -var latencyRE = regexp.MustCompile(`Total:\s+\d+\s+(\d+)\s+(\d+\.?\d+?)\s+\d+\s+\d+\s`) - -// parseLatency parses latency from apachebench output. -func parseLatency(data string) (float64, error) { - match := latencyRE.FindStringSubmatch(data) - if len(match) < 2 { - return 0, fmt.Errorf("failed get bandwidth: %s", data) - } - return strconv.ParseFloat(match[1], 64) -} - -var requestsPerSecondRE = regexp.MustCompile(`Requests per second:\s+(\d+\.?\d+?)\s+`) - -// parseRequestsPerSecond parses requests per second from apachebench output. -func parseRequestsPerSecond(data string) (float64, error) { - match := requestsPerSecondRE.FindStringSubmatch(data) - if len(match) < 2 { - return 0, fmt.Errorf("failed get bandwidth: %s", data) - } - return strconv.ParseFloat(match[1], 64) -} - -// Sample output from apachebench. -const sampleData = `This is ApacheBench, Version 2.3 <$Revision: 1826891 $> -Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ -Licensed to The Apache Software Foundation, http://www.apache.org/ - -Benchmarking 10.10.10.10 (be patient).....done - - -Server Software: Apache/2.4.38 -Server Hostname: 10.10.10.10 -Server Port: 80 - -Document Path: /latin10k.txt -Document Length: 210 bytes - -Concurrency Level: 1 -Time taken for tests: 0.180 seconds -Complete requests: 100 -Failed requests: 0 -Non-2xx responses: 100 -Total transferred: 38800 bytes -HTML transferred: 21000 bytes -Requests per second: 556.44 [#/sec] (mean) -Time per request: 1.797 [ms] (mean) -Time per request: 1.797 [ms] (mean, across all concurrent requests) -Transfer rate: 210.84 [Kbytes/sec] received - -Connection Times (ms) - min mean[+/-sd] median max -Connect: 0 0 0.2 0 2 -Processing: 1 2 1.0 1 8 -Waiting: 1 1 1.0 1 7 -Total: 1 2 1.2 1 10 - -Percentage of the requests served within a certain time (ms) - 50% 1 - 66% 2 - 75% 2 - 80% 2 - 90% 2 - 95% 3 - 98% 7 - 99% 10 - 100% 10 (longest request)` - -// TestParsers checks the parsers work. -func TestParsers(t *testing.T) { - want := 210.84 - got, err := parseTransferRate(sampleData) - if err != nil { - t.Fatalf("failed to parse transfer rate with error: %v", err) - } else if got != want { - t.Fatalf("parseTransferRate got: %f, want: %f", got, want) - } - - want = 2.0 - got, err = parseLatency(sampleData) - if err != nil { - t.Fatalf("failed to parse transfer rate with error: %v", err) - } else if got != want { - t.Fatalf("parseLatency got: %f, want: %f", got, want) - } - - want = 556.44 - got, err = parseRequestsPerSecond(sampleData) - if err != nil { - t.Fatalf("failed to parse transfer rate with error: %v", err) - } else if got != want { - t.Fatalf("parseRequestsPerSecond got: %f, want: %f", got, want) - } -} diff --git a/test/benchmarks/network/iperf_test.go b/test/benchmarks/network/iperf_test.go deleted file mode 100644 index a5e198e14..000000000 --- a/test/benchmarks/network/iperf_test.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package network - -import ( - "context" - "fmt" - "regexp" - "strconv" - "strings" - "testing" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/pkg/test/testutil" - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -func BenchmarkIperf(b *testing.B) { - const time = 10 // time in seconds to run the client. - - clientMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer clientMachine.CleanUp() - - serverMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer serverMachine.CleanUp() - ctx := context.Background() - for _, bm := range []struct { - name string - clientFunc func(context.Context, testutil.Logger) *dockerutil.Container - serverFunc func(context.Context, testutil.Logger) *dockerutil.Container - }{ - // We are either measuring the server or the client. The other should be - // runc. e.g. Upload sees how fast the runtime under test uploads to a native - // server. - { - name: "Upload", - clientFunc: clientMachine.GetContainer, - serverFunc: serverMachine.GetNativeContainer, - }, - { - name: "Download", - clientFunc: clientMachine.GetNativeContainer, - serverFunc: serverMachine.GetContainer, - }, - } { - b.Run(bm.name, func(b *testing.B) { - // Set up the containers. - server := bm.serverFunc(ctx, b) - defer server.CleanUp(ctx) - client := bm.clientFunc(ctx, b) - defer client.CleanUp(ctx) - - // iperf serves on port 5001 by default. - port := 5001 - - // Start the server. - if err := server.Spawn(ctx, dockerutil.RunOpts{ - Image: "benchmarks/iperf", - Ports: []int{port}, - }, "iperf", "-s"); err != nil { - b.Fatalf("failed to start server with: %v", err) - } - - ip, err := serverMachine.IPAddress() - if err != nil { - b.Fatalf("failed to find server ip: %v", err) - } - - servingPort, err := server.FindPort(ctx, port) - if err != nil { - b.Fatalf("failed to find port %d: %v", port, err) - } - - // Make sure the server is up and serving before we run. - if err := harness.WaitUntilServing(ctx, clientMachine, ip, servingPort); err != nil { - b.Fatalf("failed to wait for server: %v", err) - } - - // iperf report in Kb realtime - cmd := fmt.Sprintf("iperf -f K --realtime --time %d -c %s -p %d", time, ip.String(), servingPort) - - // Run the client. - b.ResetTimer() - - // Restart the server profiles. If the server isn't being profiled - // this does nothing. - server.RestartProfiles() - for i := 0; i < b.N; i++ { - out, err := client.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/iperf", - }, strings.Split(cmd, " ")...) - if err != nil { - b.Fatalf("failed to run client: %v", err) - } - b.StopTimer() - - // Parse bandwidth and report it. - bW, err := bandwidth(out) - if err != nil { - b.Fatalf("failed to parse bandwitdth from %s: %v", out, err) - } - b.ReportMetric(bW*1024, "bandwidth") // Convert from Kb/s to b/s. - b.StartTimer() - } - }) - } -} - -// bandwidth parses the Bandwidth number from an iperf report. A sample is below. -func bandwidth(data string) (float64, error) { - re := regexp.MustCompile(`\[\s*\d+\][^\n]+\s+(\d+\.?\d*)\s+KBytes/sec`) - match := re.FindStringSubmatch(data) - if len(match) < 1 { - return 0, fmt.Errorf("failed get bandwidth: %s", data) - } - return strconv.ParseFloat(match[1], 64) -} - -func TestParser(t *testing.T) { - sampleData := ` ------------------------------------------------------------- -Client connecting to 10.138.15.215, TCP port 32779 -TCP window size: 45.0 KByte (default) ------------------------------------------------------------- -[ 3] local 10.138.15.216 port 32866 connected with 10.138.15.215 port 32779 -[ ID] Interval Transfer Bandwidth -[ 3] 0.0-10.0 sec 459520 KBytes 45900 KBytes/sec -` - bandwidth, err := bandwidth(sampleData) - if err != nil || bandwidth != 45900 { - t.Fatalf("failed with: %v and %f", err, bandwidth) - } -} diff --git a/test/benchmarks/network/network.go b/test/benchmarks/network/network.go deleted file mode 100644 index ce17ddb94..000000000 --- a/test/benchmarks/network/network.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package network holds benchmarks around raw network performance. -package network - -import ( - "os" - "testing" - - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -var h harness.Harness - -// TestMain is the main method for package network. -func TestMain(m *testing.M) { - h.Init() - os.Exit(m.Run()) -} |