diff options
Diffstat (limited to 'test/benchmarks')
36 files changed, 0 insertions, 2566 deletions
diff --git a/test/benchmarks/README.md b/test/benchmarks/README.md deleted file mode 100644 index d1bbabf6f..000000000 --- a/test/benchmarks/README.md +++ /dev/null @@ -1,157 +0,0 @@ -# Benchmark tools - -This package and subpackages are for running macro benchmarks on `runsc`. They -are meant to replace the previous //benchmarks benchmark-tools written in -python. - -Benchmarks are meant to look like regular golang benchmarks using the testing.B -library. - -## Setup - -To run benchmarks you will need: - -* Docker installed (17.09.0 or greater). - -The easiest way to setup runsc for running benchmarks is to use the make file. -From the root directory: - -* Download images: `make load-all-images` -* Install runsc suitable for benchmarking, which should probably not have - strace or debug logs enabled. For example:`make configure RUNTIME=myrunsc - ARGS=--platform=kvm`. -* Restart docker: `sudo service docker restart` - -You should now have a runtime with the following options configured in -`/etc/docker/daemon.json` - -``` -"myrunsc": { - "path": "/tmp/myrunsc/runsc", - "runtimeArgs": [ - "--debug-log", - "/tmp/bench/logs/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND%", - "--platform=kvm" - ] - }, - -``` - -This runtime has been configured with a debugging off and strace logs off and is -using kvm for demonstration. - -## Running benchmarks - -Given the runtime above runtime `myrunsc`, run benchmarks with the following: - -``` -make sudo TARGETS=//path/to:target ARGS="--runtime=myrunsc -test.v \ - -test.bench=." OPTIONS="-c opt -``` - -For example, to run only the Iperf tests: - -``` -make sudo TARGETS=//test/benchmarks/network:network_test \ - ARGS="--runtime=myrunsc -test.v -test.bench=Iperf" OPTIONS="-c opt" -``` - -Benchmarks are run with root as some benchmarks require root privileges to do -things like drop caches. - -## Writing benchmarks - -Benchmarks consist of docker images as Dockerfiles and golang testing.B -benchmarks. - -### Dockerfiles: - -* Are stored at //images. -* New Dockerfiles go in an appropriately named directory at - `//images/benchmarks/my-cool-dockerfile`. -* Dockerfiles for benchmarks should: - * Use explicitly versioned packages. - * Not use ENV and CMD statements...it is easy to add these in the API. -* Note: A common pattern for getting access to a tmpfs mount is to copy files - there after container start. See: //test/benchmarks/build/bazel_test.go. You - can also make your own with `RunOpts.Mounts`. - -### testing.B packages - -In general, benchmarks should look like this: - -```golang - -var h harness.Harness - -func BenchmarkMyCoolOne(b *testing.B) { - machine, err := h.GetMachine() - // check err - defer machine.CleanUp() - - ctx := context.Background() - container := machine.GetContainer(ctx, b) - defer container.CleanUp(ctx) - - b.ResetTimer() - - //Respect b.N. - for i := 0; i < b.N; i++ { - out, err := container.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/my-cool-image", - Env: []string{"MY_VAR=awesome"}, - other options...see dockerutil - }, "sh", "-c", "echo MY_VAR") - //check err - b.StopTimer() - - // Do parsing and reporting outside of the timer. - number := parseMyMetric(out) - b.ReportMetric(number, "my-cool-custom-metric") - - b.StartTimer() - } -} - -func TestMain(m *testing.M) { - h.Init() - os.Exit(m.Run()) -} -``` - -Some notes on the above: - -* The harness is initiated in the TestMain method and made global to test - module. The harness will handle any presetup that needs to happen with - flags, remote virtual machines (eventually), and other services. -* Respect `b.N` in that users of the benchmark may want to "run for an hour" - or something of the sort. -* Use the `b.ReportMetric()` method to report custom metrics. -* Set the timer if time is useful for reporting. There isn't a way to turn off - default metrics in testing.B (B/op, allocs/op, ns/op). -* Take a look at dockerutil at //pkg/test/dockerutil to see all methods - available from containers. The API is based on the "official" - [docker API for golang](https://pkg.go.dev/mod/github.com/docker/docker). -* `harness.GetMachine()` marks how many machines this tests needs. If you have - a client and server and to mark them as multiple machines, call - `harness.GetMachine()` twice. - -## Profiling - -For profiling, the runtime is required to have the `--profile` flag enabled. -This flag loosens seccomp filters so that the runtime can write profile data to -disk. This configuration is not recommended for production. - -* Install runsc with the `--profile` flag: `make configure RUNTIME=myrunsc - ARGS="--profile --platform=kvm --vfs2"`. The kvm and vfs2 flags are not - required, but are included for demonstration. -* Restart docker: `sudo service docker restart` - -To run and generate CPU profiles fs_test test run: - -``` -make sudo TARGETS=//test/benchmarks/fs:fs_test \ - ARGS="--runtime=myrunsc -test.v -test.bench=. --pprof-cpu" OPTIONS="-c opt" -``` - -Profiles would be at: `/tmp/profile/myrunsc/CONTAINERNAME/cpu.pprof` diff --git a/test/benchmarks/database/BUILD b/test/benchmarks/database/BUILD deleted file mode 100644 index 572db665f..000000000 --- a/test/benchmarks/database/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "database", - testonly = 1, - srcs = ["database.go"], - deps = ["//test/benchmarks/harness"], -) - -go_test( - name = "database_test", - size = "enormous", - srcs = [ - "redis_test.go", - ], - library = ":database", - tags = [ - # Requires docker and runsc to be configured before test runs. - "manual", - "local", - ], - deps = [ - "//pkg/test/dockerutil", - "//test/benchmarks/harness", - "//test/benchmarks/tools", - ], -) diff --git a/test/benchmarks/database/database.go b/test/benchmarks/database/database.go deleted file mode 100644 index 9eeb59f9a..000000000 --- a/test/benchmarks/database/database.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package database holds benchmarks around database applications. -package database - -import ( - "os" - "testing" - - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -var h harness.Harness - -// TestMain is the main method for package database. -func TestMain(m *testing.M) { - h.Init() - os.Exit(m.Run()) -} diff --git a/test/benchmarks/database/redis_test.go b/test/benchmarks/database/redis_test.go deleted file mode 100644 index 394fce820..000000000 --- a/test/benchmarks/database/redis_test.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package database - -import ( - "context" - "testing" - "time" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/test/benchmarks/harness" - "gvisor.dev/gvisor/test/benchmarks/tools" -) - -// All possible operations from redis. Note: "ping" will -// run both PING_INLINE and PING_BUILD. -var operations []string = []string{ - "PING_INLINE", - "PING_BULK", - "SET", - "GET", - "INCR", - "LPUSH", - "RPUSH", - "LPOP", - "RPOP", - "SADD", - "HSET", - "SPOP", - "LRANGE_100", - "LRANGE_300", - "LRANGE_500", - "LRANGE_600", - "MSET", -} - -// BenchmarkRedis runs redis-benchmark against a redis instance and reports -// data in queries per second. Each is reported by named operation (e.g. LPUSH). -func BenchmarkRedis(b *testing.B) { - clientMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer clientMachine.CleanUp() - - serverMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer serverMachine.CleanUp() - - // Redis runs on port 6379 by default. - port := 6379 - ctx := context.Background() - - for _, operation := range operations { - b.Run(operation, func(b *testing.B) { - server := serverMachine.GetContainer(ctx, b) - defer server.CleanUp(ctx) - - // The redis docker container takes no arguments to run a redis server. - if err := server.Spawn(ctx, dockerutil.RunOpts{ - Image: "benchmarks/redis", - Ports: []int{port}, - }); err != nil { - b.Fatalf("failed to start redis server with: %v", err) - } - - if out, err := server.WaitForOutput(ctx, "Ready to accept connections", 3*time.Second); err != nil { - b.Fatalf("failed to start redis server: %v %s", err, out) - } - - ip, err := serverMachine.IPAddress() - if err != nil { - b.Fatal("failed to get IP from server: %v", err) - } - - serverPort, err := server.FindPort(ctx, port) - if err != nil { - b.Fatal("failed to get IP from server: %v", err) - } - - if err = harness.WaitUntilServing(ctx, clientMachine, ip, serverPort); err != nil { - b.Fatalf("failed to start redis with: %v", err) - } - - redis := tools.Redis{ - Operation: operation, - } - - // Reset profiles and timer to begin the measurement. - server.RestartProfiles() - b.ResetTimer() - for i := 0; i < b.N; i++ { - client := clientMachine.GetNativeContainer(ctx, b) - defer client.CleanUp(ctx) - out, err := client.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/redis", - }, redis.MakeCmd(ip, serverPort)...) - if err != nil { - b.Fatalf("redis-benchmark failed with: %v", err) - } - - // Stop time while we parse results. - b.StopTimer() - redis.Report(b, out) - b.StartTimer() - } - }) - } -} diff --git a/test/benchmarks/fs/BUILD b/test/benchmarks/fs/BUILD deleted file mode 100644 index 20654d88f..000000000 --- a/test/benchmarks/fs/BUILD +++ /dev/null @@ -1,31 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "fs", - testonly = 1, - srcs = ["fs.go"], - deps = ["//test/benchmarks/harness"], -) - -go_test( - name = "fs_test", - size = "large", - srcs = [ - "bazel_test.go", - "fio_test.go", - ], - library = ":fs", - tags = [ - # Requires docker and runsc to be configured before test runs. - "local", - "manual", - ], - deps = [ - "//pkg/test/dockerutil", - "//test/benchmarks/harness", - "//test/benchmarks/tools", - "@com_github_docker_docker//api/types/mount:go_default_library", - ], -) diff --git a/test/benchmarks/fs/bazel_test.go b/test/benchmarks/fs/bazel_test.go deleted file mode 100644 index f4236ba37..000000000 --- a/test/benchmarks/fs/bazel_test.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package fs - -import ( - "context" - "fmt" - "strings" - "testing" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -// Note: CleanCache versions of this test require running with root permissions. -func BenchmarkBuildABSL(b *testing.B) { - runBuildBenchmark(b, "benchmarks/absl", "/abseil-cpp", "absl/base/...") -} - -// Note: CleanCache versions of this test require running with root permissions. -// Note: This test takes on the order of 10m per permutation for runsc on kvm. -func BenchmarkBuildRunsc(b *testing.B) { - runBuildBenchmark(b, "benchmarks/runsc", "/gvisor", "runsc:runsc") -} - -func runBuildBenchmark(b *testing.B, image, workdir, target string) { - b.Helper() - // Get a machine from the Harness on which to run. - machine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer machine.CleanUp() - - // Dimensions here are clean/dirty cache (do or don't drop caches) - // and if the mount on which we are compiling is a tmpfs/bind mount. - benchmarks := []struct { - name string - clearCache bool // clearCache drops caches before running. - tmpfs bool // tmpfs will run compilation on a tmpfs. - }{ - {name: "CleanCache", clearCache: true, tmpfs: false}, - {name: "DirtyCache", clearCache: false, tmpfs: false}, - {name: "CleanCacheTmpfs", clearCache: true, tmpfs: true}, - {name: "DirtyCacheTmpfs", clearCache: false, tmpfs: true}, - } - for _, bm := range benchmarks { - b.Run(bm.name, func(b *testing.B) { - // Grab a container. - ctx := context.Background() - container := machine.GetContainer(ctx, b) - defer container.CleanUp(ctx) - - // Start a container and sleep by an order of b.N. - if err := container.Spawn(ctx, dockerutil.RunOpts{ - Image: image, - }, "sleep", fmt.Sprintf("%d", 1000000)); err != nil { - b.Fatalf("run failed with: %v", err) - } - - // If we are running on a tmpfs, copy to /tmp which is a tmpfs. - if bm.tmpfs { - if out, err := container.Exec(ctx, dockerutil.ExecOpts{}, - "cp", "-r", workdir, "/tmp/."); err != nil { - b.Fatal("failed to copy directory: %v %s", err, out) - } - workdir = "/tmp" + workdir - } - - // Restart profiles after the copy. - container.RestartProfiles() - b.ResetTimer() - // Drop Caches and bazel clean should happen inside the loop as we may use - // time options with b.N. (e.g. Run for an hour.) - for i := 0; i < b.N; i++ { - b.StopTimer() - // Drop Caches for clear cache runs. - if bm.clearCache { - if err := harness.DropCaches(machine); err != nil { - b.Skipf("failed to drop caches: %v. You probably need root.", err) - } - } - b.StartTimer() - - got, err := container.Exec(ctx, dockerutil.ExecOpts{ - WorkDir: workdir, - }, "bazel", "build", "-c", "opt", target) - if err != nil { - b.Fatalf("build failed with: %v", err) - } - b.StopTimer() - - want := "Build completed successfully" - if !strings.Contains(got, want) { - b.Fatalf("string %s not in: %s", want, got) - } - // Clean bazel in case we use b.N. - _, err = container.Exec(ctx, dockerutil.ExecOpts{ - WorkDir: workdir, - }, "bazel", "clean") - if err != nil { - b.Fatalf("build failed with: %v", err) - } - b.StartTimer() - } - }) - } -} diff --git a/test/benchmarks/fs/fio_test.go b/test/benchmarks/fs/fio_test.go deleted file mode 100644 index 65874ed8b..000000000 --- a/test/benchmarks/fs/fio_test.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package fs - -import ( - "context" - "fmt" - "path/filepath" - "strings" - "testing" - - "github.com/docker/docker/api/types/mount" - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/test/benchmarks/harness" - "gvisor.dev/gvisor/test/benchmarks/tools" -) - -// BenchmarkFio runs fio on the runtime under test. There are 4 basic test -// cases each run on a tmpfs mount and a bind mount. Fio requires root so that -// caches can be dropped. -func BenchmarkFio(b *testing.B) { - testCases := []tools.Fio{ - tools.Fio{ - Test: "write", - Size: "5G", - Blocksize: "1M", - Iodepth: 4, - }, - tools.Fio{ - Test: "read", - Size: "5G", - Blocksize: "1M", - Iodepth: 4, - }, - tools.Fio{ - Test: "randwrite", - Size: "5G", - Blocksize: "4K", - Iodepth: 4, - Time: 30, - }, - tools.Fio{ - Test: "randread", - Size: "5G", - Blocksize: "4K", - Iodepth: 4, - Time: 30, - }, - } - - machine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine with: %v", err) - } - defer machine.CleanUp() - - for _, fsType := range []mount.Type{mount.TypeBind, mount.TypeTmpfs} { - for _, tc := range testCases { - testName := strings.Title(tc.Test) + strings.Title(string(fsType)) - b.Run(testName, func(b *testing.B) { - ctx := context.Background() - container := machine.GetContainer(ctx, b) - defer container.CleanUp(ctx) - - // Directory and filename inside container where fio will read/write. - outdir := "/data" - outfile := filepath.Join(outdir, "test.txt") - - // Make the required mount and grab a cleanup for bind mounts - // as they are backed by a temp directory (mktemp). - mnt, mountCleanup, err := makeMount(machine, fsType, outdir) - if err != nil { - b.Fatalf("failed to make mount: %v", err) - } - defer mountCleanup() - - // Start the container with the mount. - if err := container.Spawn( - ctx, - dockerutil.RunOpts{ - Image: "benchmarks/fio", - Mounts: []mount.Mount{ - mnt, - }, - }, - // Sleep on the order of b.N. - "sleep", fmt.Sprintf("%d", 1000*b.N), - ); err != nil { - b.Fatalf("failed to start fio container with: %v", err) - } - - // For reads, we need a file to read so make one inside the container. - if strings.Contains(tc.Test, "read") { - fallocateCmd := fmt.Sprintf("fallocate -l %s %s", tc.Size, outfile) - if out, err := container.Exec(ctx, dockerutil.ExecOpts{}, - strings.Split(fallocateCmd, " ")...); err != nil { - b.Fatalf("failed to create readable file on mount: %v, %s", err, out) - } - } - - // Drop caches just before running. - if err := harness.DropCaches(machine); err != nil { - b.Skipf("failed to drop caches with %v. You probably need root.", err) - } - cmd := tc.MakeCmd(outfile) - container.RestartProfiles() - b.ResetTimer() - for i := 0; i < b.N; i++ { - // Run fio. - data, err := container.Exec(ctx, dockerutil.ExecOpts{}, cmd...) - if err != nil { - b.Fatalf("failed to run cmd %v: %v", cmd, err) - } - b.StopTimer() - tc.Report(b, data) - // If b.N is used (i.e. we run for an hour), we should drop caches - // after each run. - if err := harness.DropCaches(machine); err != nil { - b.Fatalf("failed to drop caches: %v", err) - } - b.StartTimer() - } - }) - } - } -} - -// makeMount makes a mount and cleanup based on the requested type. Bind -// and volume mounts are backed by a temp directory made with mktemp. -// tmpfs mounts require no such backing and are just made. -// It is up to the caller to call the returned cleanup. -func makeMount(machine harness.Machine, mountType mount.Type, target string) (mount.Mount, func(), error) { - switch mountType { - case mount.TypeVolume, mount.TypeBind: - dir, err := machine.RunCommand("mktemp", "-d") - if err != nil { - return mount.Mount{}, func() {}, fmt.Errorf("failed to create tempdir: %v", err) - } - dir = strings.TrimSuffix(dir, "\n") - - out, err := machine.RunCommand("chmod", "777", dir) - if err != nil { - machine.RunCommand("rm", "-rf", dir) - return mount.Mount{}, func() {}, fmt.Errorf("failed modify directory: %v %s", err, out) - } - return mount.Mount{ - Target: target, - Source: dir, - Type: mount.TypeBind, - }, func() { machine.RunCommand("rm", "-rf", dir) }, nil - case mount.TypeTmpfs: - return mount.Mount{ - Target: target, - Type: mount.TypeTmpfs, - }, func() {}, nil - default: - return mount.Mount{}, func() {}, fmt.Errorf("illegal mount time not supported: %v", mountType) - } -} diff --git a/test/benchmarks/fs/fs.go b/test/benchmarks/fs/fs.go deleted file mode 100644 index e5ca28c3b..000000000 --- a/test/benchmarks/fs/fs.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package fs holds benchmarks around filesystem performance. -package fs - -import ( - "os" - "testing" - - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -var h harness.Harness - -// TestMain is the main method for package fs. -func TestMain(m *testing.M) { - h.Init() - os.Exit(m.Run()) -} diff --git a/test/benchmarks/harness/BUILD b/test/benchmarks/harness/BUILD deleted file mode 100644 index c2e316709..000000000 --- a/test/benchmarks/harness/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -package(licenses = ["notice"]) - -go_library( - name = "harness", - testonly = 1, - srcs = [ - "harness.go", - "machine.go", - "util.go", - ], - visibility = ["//:sandbox"], - deps = [ - "//pkg/test/dockerutil", - "//pkg/test/testutil", - ], -) diff --git a/test/benchmarks/harness/harness.go b/test/benchmarks/harness/harness.go deleted file mode 100644 index 68bd7b4cf..000000000 --- a/test/benchmarks/harness/harness.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package harness holds utility code for running benchmarks on Docker. -package harness - -import ( - "flag" - - "gvisor.dev/gvisor/pkg/test/dockerutil" -) - -// Harness is a handle for managing state in benchmark runs. -type Harness struct { -} - -// Init performs any harness initilialization before runs. -func (h *Harness) Init() error { - flag.Parse() - dockerutil.EnsureSupportedDockerVersion() - return nil -} - -// GetMachine returns this run's implementation of machine. -func (h *Harness) GetMachine() (Machine, error) { - return &localMachine{}, nil -} diff --git a/test/benchmarks/harness/machine.go b/test/benchmarks/harness/machine.go deleted file mode 100644 index 88e5e841b..000000000 --- a/test/benchmarks/harness/machine.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package harness - -import ( - "context" - "net" - "os/exec" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/pkg/test/testutil" -) - -// Machine describes a real machine for use in benchmarks. -type Machine interface { - // GetContainer gets a container from the machine. The container uses the - // runtime under test and is profiled if requested by flags. - GetContainer(ctx context.Context, log testutil.Logger) *dockerutil.Container - - // GetNativeContainer gets a native container from the machine. Native containers - // use runc by default and are not profiled. - GetNativeContainer(ctx context.Context, log testutil.Logger) *dockerutil.Container - - // RunCommand runs cmd on this machine. - RunCommand(cmd string, args ...string) (string, error) - - // Returns IP Address for the machine. - IPAddress() (net.IP, error) - - // CleanUp cleans up this machine. - CleanUp() -} - -// localMachine describes this machine. -type localMachine struct { -} - -// GetContainer implements Machine.GetContainer for localMachine. -func (l *localMachine) GetContainer(ctx context.Context, logger testutil.Logger) *dockerutil.Container { - return dockerutil.MakeContainer(ctx, logger) -} - -// GetContainer implements Machine.GetContainer for localMachine. -func (l *localMachine) GetNativeContainer(ctx context.Context, logger testutil.Logger) *dockerutil.Container { - return dockerutil.MakeNativeContainer(ctx, logger) -} - -// RunCommand implements Machine.RunCommand for localMachine. -func (l *localMachine) RunCommand(cmd string, args ...string) (string, error) { - c := exec.Command(cmd, args...) - out, err := c.CombinedOutput() - return string(out), err -} - -// IPAddress implements Machine.IPAddress. -func (l *localMachine) IPAddress() (net.IP, error) { - conn, err := net.Dial("udp", "8.8.8.8:80") - if err != nil { - return nil, err - } - defer conn.Close() - - addr := conn.LocalAddr().(*net.UDPAddr) - return addr.IP, nil -} - -// CleanUp implements Machine.CleanUp and does nothing for localMachine. -func (*localMachine) CleanUp() { -} diff --git a/test/benchmarks/harness/util.go b/test/benchmarks/harness/util.go deleted file mode 100644 index bc551c582..000000000 --- a/test/benchmarks/harness/util.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package harness - -import ( - "context" - "fmt" - "net" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/pkg/test/testutil" -) - -// WaitUntilServing grabs a container from `machine` and waits for a server at -// IP:port. -func WaitUntilServing(ctx context.Context, machine Machine, server net.IP, port int) error { - var logger testutil.DefaultLogger = "netcat" - netcat := machine.GetNativeContainer(ctx, logger) - defer netcat.CleanUp(ctx) - - cmd := fmt.Sprintf("while ! nc -zv %s %d; do true; done", server, port) - _, err := netcat.Run(ctx, dockerutil.RunOpts{ - Image: "packetdrill", - }, "sh", "-c", cmd) - return err -} - -// DropCaches drops caches on the provided machine. Requires root. -func DropCaches(machine Machine) error { - if out, err := machine.RunCommand("/bin/sh", "-c", "sync | sysctl vm.drop_caches=3"); err != nil { - return fmt.Errorf("failed to drop caches: %v logs: %s", err, out) - } - return nil -} diff --git a/test/benchmarks/media/BUILD b/test/benchmarks/media/BUILD deleted file mode 100644 index 6c41fc4f6..000000000 --- a/test/benchmarks/media/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "media", - testonly = 1, - srcs = ["media.go"], - deps = ["//test/benchmarks/harness"], -) - -go_test( - name = "media_test", - size = "large", - srcs = ["ffmpeg_test.go"], - library = ":media", - deps = [ - "//pkg/test/dockerutil", - "//test/benchmarks/harness", - ], -) diff --git a/test/benchmarks/media/ffmpeg_test.go b/test/benchmarks/media/ffmpeg_test.go deleted file mode 100644 index 7822dfad7..000000000 --- a/test/benchmarks/media/ffmpeg_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package media - -import ( - "context" - "strings" - "testing" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -// BenchmarkFfmpeg runs ffmpeg in a container and records runtime. -// BenchmarkFfmpeg should run as root to drop caches. -func BenchmarkFfmpeg(b *testing.B) { - machine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer machine.CleanUp() - - ctx := context.Background() - container := machine.GetContainer(ctx, b) - defer container.CleanUp(ctx) - cmd := strings.Split("ffmpeg -i video.mp4 -c:v libx264 -preset veryslow output.mp4", " ") - - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - if err := harness.DropCaches(machine); err != nil { - b.Skipf("failed to drop caches: %v. You probably need root.", err) - } - b.StartTimer() - - if _, err := container.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/ffmpeg", - }, cmd...); err != nil { - b.Fatalf("failed to run container: %v", err) - } - } -} diff --git a/test/benchmarks/media/media.go b/test/benchmarks/media/media.go deleted file mode 100644 index c7b35b758..000000000 --- a/test/benchmarks/media/media.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package media holds benchmarks around media processing applications. -package media - -import ( - "os" - "testing" - - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -var h harness.Harness - -// TestMain is the main method for package media. -func TestMain(m *testing.M) { - h.Init() - os.Exit(m.Run()) -} diff --git a/test/benchmarks/ml/BUILD b/test/benchmarks/ml/BUILD deleted file mode 100644 index 2430b60a7..000000000 --- a/test/benchmarks/ml/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "ml", - testonly = 1, - srcs = ["ml.go"], - deps = ["//test/benchmarks/harness"], -) - -go_test( - name = "ml_test", - size = "large", - srcs = ["tensorflow_test.go"], - library = ":ml", - deps = [ - "//pkg/test/dockerutil", - "//test/benchmarks/harness", - ], -) diff --git a/test/benchmarks/ml/ml.go b/test/benchmarks/ml/ml.go deleted file mode 100644 index 13282d7bb..000000000 --- a/test/benchmarks/ml/ml.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package ml holds benchmarks around machine learning performance. -package ml - -import ( - "os" - "testing" - - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -var h harness.Harness - -// TestMain is the main method for package ml. -func TestMain(m *testing.M) { - h.Init() - os.Exit(m.Run()) -} diff --git a/test/benchmarks/ml/tensorflow_test.go b/test/benchmarks/ml/tensorflow_test.go deleted file mode 100644 index f7746897d..000000000 --- a/test/benchmarks/ml/tensorflow_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package ml - -import ( - "context" - "testing" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -// BenchmarkTensorflow runs workloads from a TensorFlow tutorial. -// See: https://github.com/aymericdamien/TensorFlow-Examples -func BenchmarkTensorflow(b *testing.B) { - workloads := map[string]string{ - "GradientDecisionTree": "2_BasicModels/gradient_boosted_decision_tree.py", - "Kmeans": "2_BasicModels/kmeans.py", - "LogisticRegression": "2_BasicModels/logistic_regression.py", - "NearestNeighbor": "2_BasicModels/nearest_neighbor.py", - "RandomForest": "2_BasicModels/random_forest.py", - "ConvolutionalNetwork": "3_NeuralNetworks/convolutional_network.py", - "MultilayerPerceptron": "3_NeuralNetworks/multilayer_perceptron.py", - "NeuralNetwork": "3_NeuralNetworks/neural_network.py", - } - - machine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer machine.CleanUp() - - for name, workload := range workloads { - b.Run(name, func(b *testing.B) { - ctx := context.Background() - container := machine.GetContainer(ctx, b) - defer container.CleanUp(ctx) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - if err := harness.DropCaches(machine); err != nil { - b.Skipf("failed to drop caches: %v. You probably need root.", err) - } - b.StartTimer() - - if out, err := container.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/tensorflow", - Env: []string{"PYTHONPATH=$PYTHONPATH:/TensorFlow-Examples/examples"}, - WorkDir: "/TensorFlow-Examples/examples", - }, "python", workload); err != nil { - b.Fatalf("failed to run container: %v logs: %s", err, out) - } - } - }) - } - -} diff --git a/test/benchmarks/network/BUILD b/test/benchmarks/network/BUILD deleted file mode 100644 index d15cd55ee..000000000 --- a/test/benchmarks/network/BUILD +++ /dev/null @@ -1,33 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "network", - testonly = 1, - srcs = ["network.go"], - deps = ["//test/benchmarks/harness"], -) - -go_test( - name = "network_test", - size = "large", - srcs = [ - "httpd_test.go", - "iperf_test.go", - "nginx_test.go", - "node_test.go", - ], - library = ":network", - tags = [ - # Requires docker and runsc to be configured before test runs. - "manual", - "local", - ], - deps = [ - "//pkg/test/dockerutil", - "//pkg/test/testutil", - "//test/benchmarks/harness", - "//test/benchmarks/tools", - ], -) diff --git a/test/benchmarks/network/httpd_test.go b/test/benchmarks/network/httpd_test.go deleted file mode 100644 index 21cb0b804..000000000 --- a/test/benchmarks/network/httpd_test.go +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package network - -import ( - "context" - "fmt" - "testing" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/test/benchmarks/harness" - "gvisor.dev/gvisor/test/benchmarks/tools" -) - -// see Dockerfile '//images/benchmarks/httpd'. -var docs = map[string]string{ - "notfound": "notfound", - "1Kb": "latin1k.txt", - "10Kb": "latin10k.txt", - "100Kb": "latin100k.txt", - "1000Kb": "latin1000k.txt", - "1Mb": "latin1024k.txt", - "10Mb": "latin10240k.txt", -} - -// BenchmarkHttpdConcurrency iterates the concurrency argument and tests -// how well the runtime under test handles requests in parallel. -func BenchmarkHttpdConcurrency(b *testing.B) { - // Grab a machine for the client and server. - clientMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get client: %v", err) - } - defer clientMachine.CleanUp() - - serverMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get server: %v", err) - } - defer serverMachine.CleanUp() - - // The test iterates over client concurrency, so set other parameters. - concurrency := []int{1, 25, 50, 100, 1000} - - for _, c := range concurrency { - b.Run(fmt.Sprintf("%d", c), func(b *testing.B) { - hey := &tools.Hey{ - Requests: 10000, - Concurrency: c, - Doc: docs["10Kb"], - } - runHttpd(b, clientMachine, serverMachine, hey, false /* reverse */) - }) - } -} - -// BenchmarkHttpdDocSize iterates over different sized payloads, testing how -// well the runtime handles sending different payload sizes. -func BenchmarkHttpdDocSize(b *testing.B) { - benchmarkHttpDocSize(b, false /* reverse */) -} - -// BenchmarkReverseHttpdDocSize iterates over different sized payloads, testing -// how well the runtime handles receiving different payload sizes. -func BenchmarkReverseHttpdDocSize(b *testing.B) { - benchmarkHttpDocSize(b, true /* reverse */) -} - -func benchmarkHttpdDocSize(b *testing.B, reverse bool) { - b.Helper() - - clientMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer clientMachine.CleanUp() - - serverMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer serverMachine.CleanUp() - - for name, filename := range docs { - concurrency := []int{1, 25, 50, 100, 1000} - for _, c := range concurrency { - b.Run(fmt.Sprintf("%s_%d", name, c), func(b *testing.B) { - hey := &tools.Hey{ - Requests: 10000, - Concurrency: c, - Doc: filename, - } - runHttpd(b, clientMachine, serverMachine, hey, reverse) - }) - } - } -} - -// runHttpd runs a single test run. -func runHttpd(b *testing.B, clientMachine, serverMachine harness.Machine, hey *tools.Hey, reverse bool) { - b.Helper() - - // Grab a container from the server. - ctx := context.Background() - var server *dockerutil.Container - if reverse { - server = serverMachine.GetNativeContainer(ctx, b) - } else { - server = serverMachine.GetContainer(ctx, b) - } - - defer server.CleanUp(ctx) - - // Copy the docs to /tmp and serve from there. - cmd := "mkdir -p /tmp/html; cp -r /local/* /tmp/html/.; apache2 -X" - port := 80 - - // Start the server. - if err := server.Spawn(ctx, dockerutil.RunOpts{ - Image: "benchmarks/httpd", - Ports: []int{port}, - Env: []string{ - // Standard environmental variables for httpd. - "APACHE_RUN_DIR=/tmp", - "APACHE_RUN_USER=nobody", - "APACHE_RUN_GROUP=nogroup", - "APACHE_LOG_DIR=/tmp", - "APACHE_PID_FILE=/tmp/apache.pid", - }, - }, "sh", "-c", cmd); err != nil { - b.Fatalf("failed to start server: %v", err) - } - - ip, err := serverMachine.IPAddress() - if err != nil { - b.Fatalf("failed to find server ip: %v", err) - } - - servingPort, err := server.FindPort(ctx, port) - if err != nil { - b.Fatalf("failed to find server port %d: %v", port, err) - } - - // Check the server is serving. - harness.WaitUntilServing(ctx, clientMachine, ip, servingPort) - - var client *dockerutil.Container - // Grab a client. - if reverse { - client = clientMachine.GetContainer(ctx, b) - } else { - client = clientMachine.GetNativeContainer(ctx, b) - } - defer client.CleanUp(ctx) - - b.ResetTimer() - server.RestartProfiles() - for i := 0; i < b.N; i++ { - out, err := client.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/hey", - }, hey.MakeCmd(ip, servingPort)...) - if err != nil { - b.Fatalf("run failed with: %v", err) - } - - b.StopTimer() - hey.Report(b, out) - b.StartTimer() - } -} diff --git a/test/benchmarks/network/iperf_test.go b/test/benchmarks/network/iperf_test.go deleted file mode 100644 index b8ab7dfb8..000000000 --- a/test/benchmarks/network/iperf_test.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package network - -import ( - "context" - "testing" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/pkg/test/testutil" - "gvisor.dev/gvisor/test/benchmarks/harness" - "gvisor.dev/gvisor/test/benchmarks/tools" -) - -func BenchmarkIperf(b *testing.B) { - iperf := tools.Iperf{ - Time: 10, // time in seconds to run client. - } - - clientMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer clientMachine.CleanUp() - - serverMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer serverMachine.CleanUp() - ctx := context.Background() - for _, bm := range []struct { - name string - clientFunc func(context.Context, testutil.Logger) *dockerutil.Container - serverFunc func(context.Context, testutil.Logger) *dockerutil.Container - }{ - // We are either measuring the server or the client. The other should be - // runc. e.g. Upload sees how fast the runtime under test uploads to a native - // server. - { - name: "Upload", - clientFunc: clientMachine.GetContainer, - serverFunc: serverMachine.GetNativeContainer, - }, - { - name: "Download", - clientFunc: clientMachine.GetNativeContainer, - serverFunc: serverMachine.GetContainer, - }, - } { - b.Run(bm.name, func(b *testing.B) { - // Set up the containers. - server := bm.serverFunc(ctx, b) - defer server.CleanUp(ctx) - client := bm.clientFunc(ctx, b) - defer client.CleanUp(ctx) - - // iperf serves on port 5001 by default. - port := 5001 - - // Start the server. - if err := server.Spawn(ctx, dockerutil.RunOpts{ - Image: "benchmarks/iperf", - Ports: []int{port}, - }, "iperf", "-s"); err != nil { - b.Fatalf("failed to start server with: %v", err) - } - - ip, err := serverMachine.IPAddress() - if err != nil { - b.Fatalf("failed to find server ip: %v", err) - } - - servingPort, err := server.FindPort(ctx, port) - if err != nil { - b.Fatalf("failed to find port %d: %v", port, err) - } - - // Make sure the server is up and serving before we run. - if err := harness.WaitUntilServing(ctx, clientMachine, ip, servingPort); err != nil { - b.Fatalf("failed to wait for server: %v", err) - } - // Run the client. - b.ResetTimer() - - // Restart the server profiles. If the server isn't being profiled - // this does nothing. - server.RestartProfiles() - for i := 0; i < b.N; i++ { - out, err := client.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/iperf", - }, iperf.MakeCmd(ip, servingPort)...) - if err != nil { - b.Fatalf("failed to run client: %v", err) - } - b.StopTimer() - iperf.Report(b, out) - b.StartTimer() - } - }) - } -} diff --git a/test/benchmarks/network/network.go b/test/benchmarks/network/network.go deleted file mode 100644 index ce17ddb94..000000000 --- a/test/benchmarks/network/network.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package network holds benchmarks around raw network performance. -package network - -import ( - "os" - "testing" - - "gvisor.dev/gvisor/test/benchmarks/harness" -) - -var h harness.Harness - -// TestMain is the main method for package network. -func TestMain(m *testing.M) { - h.Init() - os.Exit(m.Run()) -} diff --git a/test/benchmarks/network/nginx_test.go b/test/benchmarks/network/nginx_test.go deleted file mode 100644 index 5965652a5..000000000 --- a/test/benchmarks/network/nginx_test.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package network - -import ( - "context" - "fmt" - "testing" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/test/benchmarks/harness" - "gvisor.dev/gvisor/test/benchmarks/tools" -) - -// BenchmarkNginxConcurrency iterates the concurrency argument and tests -// how well the runtime under test handles requests in parallel. -// TODO(zkoopmans): Update with different doc sizes like Httpd. -func BenchmarkNginxConcurrency(b *testing.B) { - // Grab a machine for the client and server. - clientMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get client: %v", err) - } - defer clientMachine.CleanUp() - - serverMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get server: %v", err) - } - defer serverMachine.CleanUp() - - concurrency := []int{1, 5, 10, 25} - for _, c := range concurrency { - b.Run(fmt.Sprintf("%d", c), func(b *testing.B) { - hey := &tools.Hey{ - Requests: 10000, - Concurrency: c, - } - runNginx(b, clientMachine, serverMachine, hey) - }) - } -} - -// runHttpd runs a single test run. -func runNginx(b *testing.B, clientMachine, serverMachine harness.Machine, hey *tools.Hey) { - b.Helper() - - // Grab a container from the server. - ctx := context.Background() - server := serverMachine.GetContainer(ctx, b) - defer server.CleanUp(ctx) - - port := 80 - // Start the server. - if err := server.Spawn(ctx, - dockerutil.RunOpts{ - Image: "benchmarks/nginx", - Ports: []int{port}, - }); err != nil { - b.Fatalf("server failed to start: %v", err) - } - - ip, err := serverMachine.IPAddress() - if err != nil { - b.Fatalf("failed to find server ip: %v", err) - } - - servingPort, err := server.FindPort(ctx, port) - if err != nil { - b.Fatalf("failed to find server port %d: %v", port, err) - } - - // Check the server is serving. - harness.WaitUntilServing(ctx, clientMachine, ip, servingPort) - - // Grab a client. - client := clientMachine.GetNativeContainer(ctx, b) - defer client.CleanUp(ctx) - - b.ResetTimer() - server.RestartProfiles() - for i := 0; i < b.N; i++ { - out, err := client.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/hey", - }, hey.MakeCmd(ip, servingPort)...) - if err != nil { - b.Fatalf("run failed with: %v", err) - } - b.StopTimer() - hey.Report(b, out) - b.StartTimer() - } -} diff --git a/test/benchmarks/network/node_test.go b/test/benchmarks/network/node_test.go deleted file mode 100644 index 5b568cfe5..000000000 --- a/test/benchmarks/network/node_test.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package network - -import ( - "context" - "fmt" - "testing" - "time" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/test/benchmarks/harness" - "gvisor.dev/gvisor/test/benchmarks/tools" -) - -// BenchmarkNode runs 10K requests using 'hey' against a Node server run on -// 'runtime'. The server responds to requests by grabbing some data in a -// redis instance and returns the data in its reponse. The test loops through -// increasing amounts of concurency for requests. -func BenchmarkNode(b *testing.B) { - requests := 10000 - concurrency := []int{1, 5, 10, 25} - - for _, c := range concurrency { - b.Run(fmt.Sprintf("Concurrency%d", c), func(b *testing.B) { - hey := &tools.Hey{ - Requests: requests, - Concurrency: c, - } - runNode(b, hey) - }) - } -} - -// runNode runs the test for a given # of requests and concurrency. -func runNode(b *testing.B, hey *tools.Hey) { - b.Helper() - - // The machine to hold Redis and the Node Server. - serverMachine, err := h.GetMachine() - if err != nil { - b.Fatal("failed to get machine with: %v", err) - } - defer serverMachine.CleanUp() - - // The machine to run 'hey'. - clientMachine, err := h.GetMachine() - if err != nil { - b.Fatal("failed to get machine with: %v", err) - } - defer clientMachine.CleanUp() - - ctx := context.Background() - - // Spawn a redis instance for the app to use. - redis := serverMachine.GetNativeContainer(ctx, b) - if err := redis.Spawn(ctx, dockerutil.RunOpts{ - Image: "benchmarks/redis", - }); err != nil { - b.Fatalf("failed to spwan redis instance: %v", err) - } - defer redis.CleanUp(ctx) - - if out, err := redis.WaitForOutput(ctx, "Ready to accept connections", 3*time.Second); err != nil { - b.Fatalf("failed to start redis server: %v %s", err, out) - } - redisIP, err := redis.FindIP(ctx, false) - if err != nil { - b.Fatalf("failed to get IP from redis instance: %v", err) - } - - // Node runs on port 8080. - port := 8080 - - // Start-up the Node server. - nodeApp := serverMachine.GetContainer(ctx, b) - if err := nodeApp.Spawn(ctx, dockerutil.RunOpts{ - Image: "benchmarks/node", - WorkDir: "/usr/src/app", - Links: []string{redis.MakeLink("redis")}, - Ports: []int{port}, - }, "node", "index.js", redisIP.String()); err != nil { - b.Fatalf("failed to spawn node instance: %v", err) - } - defer nodeApp.CleanUp(ctx) - - servingIP, err := serverMachine.IPAddress() - if err != nil { - b.Fatalf("failed to get ip from server: %v", err) - } - - servingPort, err := nodeApp.FindPort(ctx, port) - if err != nil { - b.Fatalf("failed to port from node instance: %v", err) - } - - // Wait until the Client sees the server as up. - harness.WaitUntilServing(ctx, clientMachine, servingIP, servingPort) - - heyCmd := hey.MakeCmd(servingIP, servingPort) - - nodeApp.RestartProfiles() - b.ResetTimer() - - for i := 0; i < b.N; i++ { - // the client should run on Native. - client := clientMachine.GetNativeContainer(ctx, b) - out, err := client.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/hey", - }, heyCmd...) - if err != nil { - b.Fatalf("hey container failed: %v logs: %s", err, out) - } - - // Stop the timer to parse the data and report stats. - b.StopTimer() - hey.Report(b, out) - b.StartTimer() - } -} diff --git a/test/benchmarks/tools/BUILD b/test/benchmarks/tools/BUILD deleted file mode 100644 index 4358551bc..000000000 --- a/test/benchmarks/tools/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "tools", - srcs = [ - "ab.go", - "fio.go", - "hey.go", - "iperf.go", - "redis.go", - "tools.go", - ], - visibility = ["//:sandbox"], -) - -go_test( - name = "tools_test", - size = "small", - srcs = [ - "ab_test.go", - "fio_test.go", - "hey_test.go", - "iperf_test.go", - "redis_test.go", - ], - library = ":tools", -) diff --git a/test/benchmarks/tools/ab.go b/test/benchmarks/tools/ab.go deleted file mode 100644 index 4cc9c3bce..000000000 --- a/test/benchmarks/tools/ab.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tools - -import ( - "fmt" - "net" - "regexp" - "strconv" - "testing" -) - -// ApacheBench is for the client application ApacheBench. -type ApacheBench struct { - Requests int - Concurrency int - Doc string - // TODO(zkoopmans): support KeepAlive and pass option to enable. -} - -// MakeCmd makes an ApacheBench command. -func (a *ApacheBench) MakeCmd(ip net.IP, port int) []string { - path := fmt.Sprintf("http://%s:%d/%s", ip, port, a.Doc) - // See apachebench (ab) for flags. - cmd := fmt.Sprintf("ab -n %d -c %d %s", a.Requests, a.Concurrency, path) - return []string{"sh", "-c", cmd} -} - -// Report parses and reports metrics from ApacheBench output. -func (a *ApacheBench) Report(b *testing.B, output string) { - // Parse and report custom metrics. - transferRate, err := a.parseTransferRate(output) - if err != nil { - b.Logf("failed to parse transferrate: %v", err) - } - b.ReportMetric(transferRate*1024, "transfer_rate_b/s") // Convert from Kb/s to b/s. - - latency, err := a.parseLatency(output) - if err != nil { - b.Logf("failed to parse latency: %v", err) - } - b.ReportMetric(latency/1000, "mean_latency_secs") // Convert from ms to s. - - reqPerSecond, err := a.parseRequestsPerSecond(output) - if err != nil { - b.Logf("failed to parse requests per second: %v", err) - } - b.ReportMetric(reqPerSecond, "requests_per_second") -} - -var transferRateRE = regexp.MustCompile(`Transfer rate:\s+(\d+\.?\d+?)\s+\[Kbytes/sec\]\s+received`) - -// parseTransferRate parses transfer rate from ApacheBench output. -func (a *ApacheBench) parseTransferRate(data string) (float64, error) { - match := transferRateRE.FindStringSubmatch(data) - if len(match) < 2 { - return 0, fmt.Errorf("failed get bandwidth: %s", data) - } - return strconv.ParseFloat(match[1], 64) -} - -var latencyRE = regexp.MustCompile(`Total:\s+\d+\s+(\d+)\s+(\d+\.?\d+?)\s+\d+\s+\d+\s`) - -// parseLatency parses latency from ApacheBench output. -func (a *ApacheBench) parseLatency(data string) (float64, error) { - match := latencyRE.FindStringSubmatch(data) - if len(match) < 2 { - return 0, fmt.Errorf("failed get bandwidth: %s", data) - } - return strconv.ParseFloat(match[1], 64) -} - -var requestsPerSecondRE = regexp.MustCompile(`Requests per second:\s+(\d+\.?\d+?)\s+`) - -// parseRequestsPerSecond parses requests per second from ApacheBench output. -func (a *ApacheBench) parseRequestsPerSecond(data string) (float64, error) { - match := requestsPerSecondRE.FindStringSubmatch(data) - if len(match) < 2 { - return 0, fmt.Errorf("failed get bandwidth: %s", data) - } - return strconv.ParseFloat(match[1], 64) -} diff --git a/test/benchmarks/tools/ab_test.go b/test/benchmarks/tools/ab_test.go deleted file mode 100644 index 28ee66ec1..000000000 --- a/test/benchmarks/tools/ab_test.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tools - -import "testing" - -// TestApacheBench checks the ApacheBench parsers on sample output. -func TestApacheBench(t *testing.T) { - // Sample output from apachebench. - sampleData := `This is ApacheBench, Version 2.3 <$Revision: 1826891 $> -Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/ -Licensed to The Apache Software Foundation, http://www.apache.org/ - -Benchmarking 10.10.10.10 (be patient).....done - - -Server Software: Apache/2.4.38 -Server Hostname: 10.10.10.10 -Server Port: 80 - -Document Path: /latin10k.txt -Document Length: 210 bytes - -Concurrency Level: 1 -Time taken for tests: 0.180 seconds -Complete requests: 100 -Failed requests: 0 -Non-2xx responses: 100 -Total transferred: 38800 bytes -HTML transferred: 21000 bytes -Requests per second: 556.44 [#/sec] (mean) -Time per request: 1.797 [ms] (mean) -Time per request: 1.797 [ms] (mean, across all concurrent requests) -Transfer rate: 210.84 [Kbytes/sec] received - -Connection Times (ms) - min mean[+/-sd] median max -Connect: 0 0 0.2 0 2 -Processing: 1 2 1.0 1 8 -Waiting: 1 1 1.0 1 7 -Total: 1 2 1.2 1 10 - -Percentage of the requests served within a certain time (ms) - 50% 1 - 66% 2 - 75% 2 - 80% 2 - 90% 2 - 95% 3 - 98% 7 - 99% 10 - 100% 10 (longest request)` - - ab := ApacheBench{} - want := 210.84 - got, err := ab.parseTransferRate(sampleData) - if err != nil { - t.Fatalf("failed to parse transfer rate with error: %v", err) - } else if got != want { - t.Fatalf("parseTransferRate got: %f, want: %f", got, want) - } - - want = 2.0 - got, err = ab.parseLatency(sampleData) - if err != nil { - t.Fatalf("failed to parse transfer rate with error: %v", err) - } else if got != want { - t.Fatalf("parseLatency got: %f, want: %f", got, want) - } - - want = 556.44 - got, err = ab.parseRequestsPerSecond(sampleData) - if err != nil { - t.Fatalf("failed to parse transfer rate with error: %v", err) - } else if got != want { - t.Fatalf("parseRequestsPerSecond got: %f, want: %f", got, want) - } -} diff --git a/test/benchmarks/tools/fio.go b/test/benchmarks/tools/fio.go deleted file mode 100644 index 20000db16..000000000 --- a/test/benchmarks/tools/fio.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tools - -import ( - "encoding/json" - "fmt" - "strconv" - "strings" - "testing" -) - -// Fio makes 'fio' commands and parses their output. -type Fio struct { - Test string // test to run: read, write, randread, randwrite. - Size string // total size to be read/written of format N[GMK] (e.g. 5G). - Blocksize string // blocksize to be read/write of format N[GMK] (e.g. 4K). - Iodepth int // iodepth for reads/writes. - Time int // time to run the test in seconds, usually for rand(read/write). -} - -// MakeCmd makes a 'fio' command. -func (f *Fio) MakeCmd(filename string) []string { - cmd := []string{"fio", "--output-format=json", "--ioengine=sync"} - cmd = append(cmd, fmt.Sprintf("--name=%s", f.Test)) - cmd = append(cmd, fmt.Sprintf("--size=%s", f.Size)) - cmd = append(cmd, fmt.Sprintf("--blocksize=%s", f.Blocksize)) - cmd = append(cmd, fmt.Sprintf("--filename=%s", filename)) - cmd = append(cmd, fmt.Sprintf("--iodepth=%d", f.Iodepth)) - cmd = append(cmd, fmt.Sprintf("--rw=%s", f.Test)) - if f.Time != 0 { - cmd = append(cmd, "--time_based") - cmd = append(cmd, fmt.Sprintf("--runtime=%d", f.Time)) - } - return cmd -} - -// Report reports metrics based on output from an 'fio' command. -func (f *Fio) Report(b *testing.B, output string) { - b.Helper() - // Parse the output and report the metrics. - isRead := strings.Contains(f.Test, "read") - bw, err := f.parseBandwidth(output, isRead) - if err != nil { - b.Fatalf("failed to parse bandwidth from %s with: %v", output, err) - } - b.ReportMetric(bw, "bandwidth_b/s") // in b/s. - - iops, err := f.parseIOps(output, isRead) - if err != nil { - b.Fatalf("failed to parse iops from %s with: %v", output, err) - } - b.ReportMetric(iops, "iops") -} - -// parseBandwidth reports the bandwidth in b/s. -func (f *Fio) parseBandwidth(data string, isRead bool) (float64, error) { - if isRead { - result, err := f.parseFioJSON(data, "read", "bw") - if err != nil { - return 0, err - } - return 1024 * result, nil - } - result, err := f.parseFioJSON(data, "write", "bw") - if err != nil { - return 0, err - } - return 1024 * result, nil -} - -// parseIOps reports the write IO per second metric. -func (f *Fio) parseIOps(data string, isRead bool) (float64, error) { - if isRead { - return f.parseFioJSON(data, "read", "iops") - } - return f.parseFioJSON(data, "write", "iops") -} - -// fioResult is for parsing FioJSON. -type fioResult struct { - Jobs []fioJob -} - -// fioJob is for parsing FioJSON. -type fioJob map[string]json.RawMessage - -// fioMetrics is for parsing FioJSON. -type fioMetrics map[string]json.RawMessage - -// parseFioJSON parses data and grabs "op" (read or write) and "metric" -// (bw or iops) from the JSON. -func (f *Fio) parseFioJSON(data, op, metric string) (float64, error) { - var result fioResult - if err := json.Unmarshal([]byte(data), &result); err != nil { - return 0, fmt.Errorf("could not unmarshal data: %v", err) - } - - if len(result.Jobs) < 1 { - return 0, fmt.Errorf("no jobs present to parse") - } - - var metrics fioMetrics - if err := json.Unmarshal(result.Jobs[0][op], &metrics); err != nil { - return 0, fmt.Errorf("could not unmarshal jobs: %v", err) - } - - if _, ok := metrics[metric]; !ok { - return 0, fmt.Errorf("no metric found for op: %s", op) - } - return strconv.ParseFloat(string(metrics[metric]), 64) -} diff --git a/test/benchmarks/tools/fio_test.go b/test/benchmarks/tools/fio_test.go deleted file mode 100644 index a98277150..000000000 --- a/test/benchmarks/tools/fio_test.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tools - -import "testing" - -// TestFio checks the Fio parsers on sample output. -func TestFio(t *testing.T) { - sampleData := ` -{ - "fio version" : "fio-3.1", - "timestamp" : 1554837456, - "timestamp_ms" : 1554837456621, - "time" : "Tue Apr 9 19:17:36 2019", - "jobs" : [ - { - "jobname" : "test", - "groupid" : 0, - "error" : 0, - "eta" : 2147483647, - "elapsed" : 1, - "job options" : { - "name" : "test", - "ioengine" : "sync", - "size" : "1073741824", - "filename" : "/disk/file.dat", - "iodepth" : "4", - "bs" : "4096", - "rw" : "write" - }, - "read" : { - "io_bytes" : 0, - "io_kbytes" : 0, - "bw" : 123456, - "iops" : 1234.5678, - "runtime" : 0, - "total_ios" : 0, - "short_ios" : 0, - "bw_min" : 0, - "bw_max" : 0, - "bw_agg" : 0.000000, - "bw_mean" : 0.000000, - "bw_dev" : 0.000000, - "bw_samples" : 0, - "iops_min" : 0, - "iops_max" : 0, - "iops_mean" : 0.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 0 - }, - "write" : { - "io_bytes" : 1073741824, - "io_kbytes" : 1048576, - "bw" : 1753471, - "iops" : 438367.892977, - "runtime" : 598, - "total_ios" : 262144, - "bw_min" : 1731120, - "bw_max" : 1731120, - "bw_agg" : 98.725328, - "bw_mean" : 1731120.000000, - "bw_dev" : 0.000000, - "bw_samples" : 1, - "iops_min" : 432780, - "iops_max" : 432780, - "iops_mean" : 432780.000000, - "iops_stddev" : 0.000000, - "iops_samples" : 1 - } - } - ] -} -` - fio := Fio{} - // WriteBandwidth. - got, err := fio.parseBandwidth(sampleData, false) - var want float64 = 1753471.0 * 1024 - if err != nil { - t.Fatalf("parse failed with err: %v", err) - } else if got != want { - t.Fatalf("got: %f, want: %f", got, want) - } - - // ReadBandwidth. - got, err = fio.parseBandwidth(sampleData, true) - want = 123456 * 1024 - if err != nil { - t.Fatalf("parse failed with err: %v", err) - } else if got != want { - t.Fatalf("got: %f, want: %f", got, want) - } - - // WriteIOps. - got, err = fio.parseIOps(sampleData, false) - want = 438367.892977 - if err != nil { - t.Fatalf("parse failed with err: %v", err) - } else if got != want { - t.Fatalf("got: %f, want: %f", got, want) - } - - // ReadIOps. - got, err = fio.parseIOps(sampleData, true) - want = 1234.5678 - if err != nil { - t.Fatalf("parse failed with err: %v", err) - } else if got != want { - t.Fatalf("got: %f, want: %f", got, want) - } -} diff --git a/test/benchmarks/tools/hey.go b/test/benchmarks/tools/hey.go deleted file mode 100644 index 699497c64..000000000 --- a/test/benchmarks/tools/hey.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tools - -import ( - "fmt" - "net" - "regexp" - "strconv" - "strings" - "testing" -) - -// Hey is for the client application 'hey'. -type Hey struct { - Requests int - Concurrency int - Doc string -} - -// MakeCmd returns a 'hey' command. -func (h *Hey) MakeCmd(ip net.IP, port int) []string { - return strings.Split(fmt.Sprintf("hey -n %d -c %d http://%s:%d/%s", - h.Requests, h.Concurrency, ip, port, h.Doc), " ") -} - -// Report parses output from 'hey' and reports metrics. -func (h *Hey) Report(b *testing.B, output string) { - b.Helper() - requests, err := h.parseRequestsPerSecond(output) - if err != nil { - b.Fatalf("failed to parse requests per second: %v", err) - } - b.ReportMetric(requests, "requests_per_second") - - ave, err := h.parseAverageLatency(output) - if err != nil { - b.Fatalf("failed to parse average latency: %v", err) - } - b.ReportMetric(ave, "average_latency_secs") -} - -var heyReqPerSecondRE = regexp.MustCompile(`Requests/sec:\s*(\d+\.?\d+?)\s+`) - -// parseRequestsPerSecond finds requests per second from 'hey' output. -func (h *Hey) parseRequestsPerSecond(data string) (float64, error) { - match := heyReqPerSecondRE.FindStringSubmatch(data) - if len(match) < 2 { - return 0, fmt.Errorf("failed get bandwidth: %s", data) - } - return strconv.ParseFloat(match[1], 64) -} - -var heyAverageLatencyRE = regexp.MustCompile(`Average:\s*(\d+\.?\d+?)\s+secs`) - -// parseHeyAverageLatency finds Average Latency in seconds form 'hey' output. -func (h *Hey) parseAverageLatency(data string) (float64, error) { - match := heyAverageLatencyRE.FindStringSubmatch(data) - if len(match) < 2 { - return 0, fmt.Errorf("failed get average latency match%d : %s", len(match), data) - } - return strconv.ParseFloat(match[1], 64) -} diff --git a/test/benchmarks/tools/hey_test.go b/test/benchmarks/tools/hey_test.go deleted file mode 100644 index e0cab1f52..000000000 --- a/test/benchmarks/tools/hey_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tools - -import "testing" - -// TestHey checks the Hey parsers on sample output. -func TestHey(t *testing.T) { - sampleData := ` - Summary: - Total: 2.2391 secs - Slowest: 1.6292 secs - Fastest: 0.0066 secs - Average: 0.5351 secs - Requests/sec: 89.3202 - - Total data: 841200 bytes - Size/request: 4206 bytes - - Response time histogram: - 0.007 [1] | - 0.169 [0] | - 0.331 [149] |■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■ - 0.493 [0] | - 0.656 [0] | - 0.818 [0] | - 0.980 [0] | - 1.142 [0] | - 1.305 [0] | - 1.467 [49] |■■■■■■■■■■■■■ - 1.629 [1] | - - - Latency distribution: - 10% in 0.2149 secs - 25% in 0.2449 secs - 50% in 0.2703 secs - 75% in 1.3315 secs - 90% in 1.4045 secs - 95% in 1.4232 secs - 99% in 1.4362 secs - - Details (average, fastest, slowest): - DNS+dialup: 0.0002 secs, 0.0066 secs, 1.6292 secs - DNS-lookup: 0.0000 secs, 0.0000 secs, 0.0000 secs - req write: 0.0000 secs, 0.0000 secs, 0.0012 secs - resp wait: 0.5225 secs, 0.0064 secs, 1.4346 secs - resp read: 0.0122 secs, 0.0001 secs, 0.2006 secs - - Status code distribution: - [200] 200 responses - ` - hey := Hey{} - want := 89.3202 - got, err := hey.parseRequestsPerSecond(sampleData) - if err != nil { - t.Fatalf("failed to parse request per second with: %v", err) - } else if got != want { - t.Fatalf("got: %f, want: %f", got, want) - } - - want = 0.5351 - got, err = hey.parseAverageLatency(sampleData) - if err != nil { - t.Fatalf("failed to parse average latency with: %v", err) - } else if got != want { - t.Fatalf("got: %f, want: %f", got, want) - } -} diff --git a/test/benchmarks/tools/iperf.go b/test/benchmarks/tools/iperf.go deleted file mode 100644 index df3d9349b..000000000 --- a/test/benchmarks/tools/iperf.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tools - -import ( - "fmt" - "net" - "regexp" - "strconv" - "strings" - "testing" -) - -// Iperf is for the client side of `iperf`. -type Iperf struct { - Time int -} - -// MakeCmd returns a iperf client command. -func (i *Iperf) MakeCmd(ip net.IP, port int) []string { - // iperf report in Kb realtime - return strings.Split(fmt.Sprintf("iperf -f K --realtime --time %d -c %s -p %d", i.Time, ip, port), " ") -} - -// Report parses output from iperf client and reports metrics. -func (i *Iperf) Report(b *testing.B, output string) { - b.Helper() - // Parse bandwidth and report it. - bW, err := i.bandwidth(output) - if err != nil { - b.Fatalf("failed to parse bandwitdth from %s: %v", output, err) - } - b.ReportMetric(bW*1024, "bandwidth_b/s") // Convert from Kb/s to b/s. -} - -// bandwidth parses the Bandwidth number from an iperf report. A sample is below. -func (i *Iperf) bandwidth(data string) (float64, error) { - re := regexp.MustCompile(`\[\s*\d+\][^\n]+\s+(\d+\.?\d*)\s+KBytes/sec`) - match := re.FindStringSubmatch(data) - if len(match) < 1 { - return 0, fmt.Errorf("failed get bandwidth: %s", data) - } - return strconv.ParseFloat(match[1], 64) -} diff --git a/test/benchmarks/tools/iperf_test.go b/test/benchmarks/tools/iperf_test.go deleted file mode 100644 index 03bb30d05..000000000 --- a/test/benchmarks/tools/iperf_test.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package tools - -import "testing" - -// TestIperf checks the Iperf parsers on sample output. -func TestIperf(t *testing.T) { - sampleData := ` ------------------------------------------------------------- -Client connecting to 10.138.15.215, TCP port 32779 -TCP window size: 45.0 KByte (default) ------------------------------------------------------------- -[ 3] local 10.138.15.216 port 32866 connected with 10.138.15.215 port 32779 -[ ID] Interval Transfer Bandwidth -[ 3] 0.0-10.0 sec 459520 KBytes 45900 KBytes/sec -` - i := Iperf{} - bandwidth, err := i.bandwidth(sampleData) - if err != nil || bandwidth != 45900 { - t.Fatalf("failed with: %v and %f", err, bandwidth) - } -} diff --git a/test/benchmarks/tools/redis.go b/test/benchmarks/tools/redis.go deleted file mode 100644 index db32460ec..000000000 --- a/test/benchmarks/tools/redis.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tools - -import ( - "fmt" - "net" - "regexp" - "strconv" - "strings" - "testing" -) - -// Redis is for the client 'redis-benchmark'. -type Redis struct { - Operation string -} - -// MakeCmd returns a redis-benchmark client command. -func (r *Redis) MakeCmd(ip net.IP, port int) []string { - // There is no -t PING_BULK for redis-benchmark, so adjust the command in that case. - // Note that "ping" will run both PING_INLINE and PING_BULK. - if r.Operation == "PING_BULK" { - return strings.Split( - fmt.Sprintf("redis-benchmark --csv -t ping -h %s -p %d", ip, port), " ") - } - - // runs redis-benchmark -t operation for 100K requests against server. - return strings.Split( - fmt.Sprintf("redis-benchmark --csv -t %s -h %s -p %d", r.Operation, ip, port), " ") -} - -// Report parses output from redis-benchmark client and reports metrics. -func (r *Redis) Report(b *testing.B, output string) { - b.Helper() - result, err := r.parseOperation(output) - if err != nil { - b.Fatalf("parsing result %s failed with err: %v", output, err) - } - b.ReportMetric(result, r.Operation) // operations per second -} - -// parseOperation grabs the metric operations per second from redis-benchmark output. -func (r *Redis) parseOperation(data string) (float64, error) { - re := regexp.MustCompile(fmt.Sprintf(`"%s( .*)?","(\d*\.\d*)"`, r.Operation)) - match := re.FindStringSubmatch(data) - // If no match, simply don't add it to the result map. - if len(match) < 3 { - return 0.0, fmt.Errorf("could not find %s in %s", r.Operation, data) - } - return strconv.ParseFloat(match[2], 64) -} diff --git a/test/benchmarks/tools/redis_test.go b/test/benchmarks/tools/redis_test.go deleted file mode 100644 index 4bafda66f..000000000 --- a/test/benchmarks/tools/redis_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tools - -import ( - "testing" -) - -// TestRedis checks the Redis parsers on sample output. -func TestRedis(t *testing.T) { - sampleData := ` - "PING_INLINE","48661.80" - "PING_BULK","50301.81" - "SET","48923.68" - "GET","49382.71" - "INCR","49975.02" - "LPUSH","49875.31" - "RPUSH","50276.52" - "LPOP","50327.12" - "RPOP","50556.12" - "SADD","49504.95" - "HSET","49504.95" - "SPOP","50025.02" - "LPUSH (needed to benchmark LRANGE)","48875.86" - "LRANGE_100 (first 100 elements)","33955.86" - "LRANGE_300 (first 300 elements)","16550.81"// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tools - - "LRANGE_500 (first 450 elements)","13653.74" - "LRANGE_600 (first 600 elements)","11219.57" - "MSET (10 keys)","44682.75" - ` - wants := map[string]float64{ - "PING_INLINE": 48661.80, - "PING_BULK": 50301.81, - "SET": 48923.68, - "GET": 49382.71, - "INCR": 49975.02, - "LPUSH": 49875.31, - "RPUSH": 50276.52, - "LPOP": 50327.12, - "RPOP": 50556.12, - "SADD": 49504.95, - "HSET": 49504.95, - "SPOP": 50025.02, - "LRANGE_100": 33955.86, - "LRANGE_300": 16550.81, - "LRANGE_500": 13653.74, - "LRANGE_600": 11219.57, - "MSET": 44682.75, - } - for op, want := range wants { - redis := Redis{ - Operation: op, - } - if got, err := redis.parseOperation(sampleData); err != nil { - t.Fatalf("failed to parse %s: %v", op, err) - } else if want != got { - t.Fatalf("wanted %f for op %s, got %f", want, op, got) - } - } -} diff --git a/test/benchmarks/tools/tools.go b/test/benchmarks/tools/tools.go deleted file mode 100644 index eb61c0136..000000000 --- a/test/benchmarks/tools/tools.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package tools holds tooling to couple command formatting and output parsers -// together. -package tools |