From ffa9a715aaf4ebc4c8d8bef948649af358e4f4e4 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Tue, 29 Dec 2020 18:26:46 -0800 Subject: Simplify profiling and benchmarks. - Tweak the benchmarks to work with b.N where appropriate. In many cases, b.N was simply being ignored. This creates an implicit dependency in the user passing a reasonable benchtime (less than or equal to the actual runtime of the test, or using the X syntax) otherwise the test runs forever. - In cases where the above is impossible, explicitly set benchtime from the test wrapper, to prevent the above behavior (tensorflow). - Drop the *Reverse variants, which are simply hey benchmarks. We should just add a hey benchmark. The platforms benchmarks already include a native platform, and thus these benchmarks are incredibly confusing. (In other words, BenchmarkNginxReverse has nothing to do with an nginx benchmark for runsc.) - Remove the redunant Harness object, which contains no state, in order to slightly simplify the code. - Make Block and Heap profiling actually work, but setting appropriate runtime parameters (and plumbing them through the config). - Split the profiling into two phases: start and stop, since some will need to be started early, and others will need to happen at the end. PiperOrigin-RevId: 349495377 --- test/benchmarks/fs/bazel_test.go | 15 ++++----- test/benchmarks/fs/fio_test.go | 70 +++++++++++++++++++++++----------------- 2 files changed, 46 insertions(+), 39 deletions(-) (limited to 'test/benchmarks/fs') diff --git a/test/benchmarks/fs/bazel_test.go b/test/benchmarks/fs/bazel_test.go index 3fb4da9d1..8baeff0db 100644 --- a/test/benchmarks/fs/bazel_test.go +++ b/test/benchmarks/fs/bazel_test.go @@ -25,8 +25,6 @@ import ( "gvisor.dev/gvisor/test/benchmarks/tools" ) -var h harness.Harness - // Note: CleanCache versions of this test require running with root permissions. func BenchmarkBuildABSL(b *testing.B) { runBuildBenchmark(b, "benchmarks/absl", "/abseil-cpp", "absl/base/...") @@ -41,7 +39,7 @@ func BenchmarkBuildRunsc(b *testing.B) { func runBuildBenchmark(b *testing.B, image, workdir, target string) { b.Helper() // Get a machine from the Harness on which to run. - machine, err := h.GetMachine() + machine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine: %v", err) } @@ -102,21 +100,20 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) { prefix = "/tmp" } - // Restart profiles after the copy. - container.RestartProfiles() b.ResetTimer() + b.StopTimer() + // Drop Caches and bazel clean should happen inside the loop as we may use // time options with b.N. (e.g. Run for an hour.) for i := 0; i < b.N; i++ { - b.StopTimer() // Drop Caches for clear cache runs. if bm.clearCache { if err := harness.DropCaches(machine); err != nil { b.Skipf("failed to drop caches: %v. You probably need root.", err) } } - b.StartTimer() + b.StartTimer() got, err := container.Exec(ctx, dockerutil.ExecOpts{ WorkDir: prefix + workdir, }, "bazel", "build", "-c", "opt", target) @@ -138,7 +135,6 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) { b.Fatalf("build failed with: %v", err) } } - b.StartTimer() } }) } @@ -146,6 +142,7 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) { // TestMain is the main method for package fs. func TestMain(m *testing.M) { - h.Init() + harness.Init() + harness.SetFixedBenchmarks() os.Exit(m.Run()) } diff --git a/test/benchmarks/fs/fio_test.go b/test/benchmarks/fs/fio_test.go index 96340373c..0c772b768 100644 --- a/test/benchmarks/fs/fio_test.go +++ b/test/benchmarks/fs/fio_test.go @@ -27,42 +27,50 @@ import ( "gvisor.dev/gvisor/test/benchmarks/tools" ) -var h harness.Harness - // BenchmarkFio runs fio on the runtime under test. There are 4 basic test // cases each run on a tmpfs mount and a bind mount. Fio requires root so that // caches can be dropped. func BenchmarkFio(b *testing.B) { testCases := []tools.Fio{ tools.Fio{ - Test: "write", - Size: "5G", - Blocksize: "1M", - Iodepth: 4, + Test: "write4K", + Size: b.N, + BlockSize: 4, + IODepth: 4, + }, + tools.Fio{ + Test: "write1M", + Size: b.N, + BlockSize: 1024, + IODepth: 4, + }, + tools.Fio{ + Test: "read4K", + Size: b.N, + BlockSize: 4, + IODepth: 4, }, tools.Fio{ - Test: "read", - Size: "5G", - Blocksize: "1M", - Iodepth: 4, + Test: "read1M", + Size: b.N, + BlockSize: 1024, + IODepth: 4, }, tools.Fio{ - Test: "randwrite", - Size: "5G", - Blocksize: "4K", - Iodepth: 4, - Time: 30, + Test: "randwrite4K", + Size: b.N, + BlockSize: 4, + IODepth: 4, }, tools.Fio{ - Test: "randread", - Size: "5G", - Blocksize: "4K", - Iodepth: 4, - Time: 30, + Test: "randread4K", + Size: b.N, + BlockSize: 4, + IODepth: 4, }, } - machine, err := h.GetMachine() + machine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine with: %v", err) } @@ -116,7 +124,7 @@ func BenchmarkFio(b *testing.B) { // For reads, we need a file to read so make one inside the container. if strings.Contains(tc.Test, "read") { - fallocateCmd := fmt.Sprintf("fallocate -l %s %s", tc.Size, outfile) + fallocateCmd := fmt.Sprintf("fallocate -l %dK %s", tc.Size, outfile) if out, err := container.Exec(ctx, dockerutil.ExecOpts{}, strings.Split(fallocateCmd, " ")...); err != nil { b.Fatalf("failed to create readable file on mount: %v, %s", err, out) @@ -128,22 +136,24 @@ func BenchmarkFio(b *testing.B) { b.Skipf("failed to drop caches with %v. You probably need root.", err) } cmd := tc.MakeCmd(outfile) - container.RestartProfiles() + b.ResetTimer() + b.StopTimer() + for i := 0; i < b.N; i++ { + if err := harness.DropCaches(machine); err != nil { + b.Fatalf("failed to drop caches: %v", err) + } + // Run fio. + b.StartTimer() data, err := container.Exec(ctx, dockerutil.ExecOpts{}, cmd...) if err != nil { b.Fatalf("failed to run cmd %v: %v", cmd, err) } b.StopTimer() + b.SetBytes(1024 * 1024) // Bytes for go reporting (Size is in megabytes). tc.Report(b, data) - // If b.N is used (i.e. we run for an hour), we should drop caches - // after each run. - if err := harness.DropCaches(machine); err != nil { - b.Fatalf("failed to drop caches: %v", err) - } - b.StartTimer() } }) } @@ -185,6 +195,6 @@ func makeMount(machine harness.Machine, mountType mount.Type, target string) (mo // TestMain is the main method for package fs. func TestMain(m *testing.M) { - h.Init() + harness.Init() os.Exit(m.Run()) } -- cgit v1.2.3