diff options
25 files changed, 379 insertions, 481 deletions
diff --git a/pkg/test/dockerutil/container.go b/pkg/test/dockerutil/container.go index 2bf0a22ff..7b5fcef9c 100644 --- a/pkg/test/dockerutil/container.go +++ b/pkg/test/dockerutil/container.go @@ -55,11 +55,8 @@ type Container struct { copyErr error cleanups []func() - // Profiles are profiles added to this container. They contain methods - // that are run after Creation, Start, and Cleanup of this Container, along - // a handle to restart the profile. Generally, tests/benchmarks using - // profiles need to run as root. - profiles []Profile + // profile is the profiling hook associated with this container. + profile *profile } // RunOpts are options for running a container. @@ -105,22 +102,7 @@ type RunOpts struct { Links []string } -// MakeContainer sets up the struct for a Docker container. -// -// Names of containers will be unique. -// Containers will check flags for profiling requests. -func MakeContainer(ctx context.Context, logger testutil.Logger) *Container { - c := MakeNativeContainer(ctx, logger) - c.runtime = *runtime - if p := MakePprofFromFlags(c); p != nil { - c.AddProfile(p) - } - return c -} - -// MakeNativeContainer sets up the struct for a DockerContainer using runc. Native -// containers aren't profiled. -func MakeNativeContainer(ctx context.Context, logger testutil.Logger) *Container { +func makeContainer(ctx context.Context, logger testutil.Logger, runtime string) *Container { // Slashes are not allowed in container names. name := testutil.RandomID(logger.Name()) name = strings.ReplaceAll(name, "/", "-") @@ -132,24 +114,29 @@ func MakeNativeContainer(ctx context.Context, logger testutil.Logger) *Container return &Container{ logger: logger, Name: name, - runtime: "", + runtime: runtime, client: client, } } -// AddProfile adds a profile to this container. -func (c *Container) AddProfile(p Profile) { - c.profiles = append(c.profiles, p) +// MakeContainer constructs a suitable Container object. +// +// The runtime used is determined by the runtime flag. +// +// Containers will check flags for profiling requests. +func MakeContainer(ctx context.Context, logger testutil.Logger) *Container { + c := makeContainer(ctx, logger, *runtime) + c.profileInit() + return c } -// RestartProfiles calls Restart on all profiles for this container. -func (c *Container) RestartProfiles() error { - for _, profile := range c.profiles { - if err := profile.Restart(c); err != nil { - return err - } - } - return nil +// MakeNativeContainer constructs a suitable Container object. +// +// The runtime used will be the system default. +// +// Native containers aren't profiled. +func MakeNativeContainer(ctx context.Context, logger testutil.Logger) *Container { + return makeContainer(ctx, logger, "" /*runtime*/) } // Spawn is analogous to 'docker run -d'. @@ -206,6 +193,8 @@ func (c *Container) Run(ctx context.Context, r RunOpts, args ...string) (string, return "", err } + c.stopProfiling() + return c.Logs(ctx) } @@ -236,11 +225,6 @@ func (c *Container) create(ctx context.Context, conf *container.Config, hostconf return err } c.id = cont.ID - for _, profile := range c.profiles { - if err := profile.OnCreate(c); err != nil { - return fmt.Errorf("OnCreate method failed with: %v", err) - } - } return nil } @@ -286,11 +270,13 @@ func (c *Container) Start(ctx context.Context) error { if err := c.client.ContainerStart(ctx, c.id, types.ContainerStartOptions{}); err != nil { return fmt.Errorf("ContainerStart failed: %v", err) } - for _, profile := range c.profiles { - if err := profile.OnStart(c); err != nil { - return fmt.Errorf("OnStart method failed: %v", err) + + if c.profile != nil { + if err := c.profile.Start(c); err != nil { + c.logger.Logf("profile.Start failed: %v", err) } } + return nil } @@ -499,8 +485,18 @@ func (c *Container) WaitForOutputSubmatch(ctx context.Context, pattern string, t } } +// stopProfiling stops profiling. +func (c *Container) stopProfiling() { + if c.profile != nil { + if err := c.profile.Stop(c); err != nil { + c.logger.Logf("profile.Stop failed: %v", err) + } + } +} + // Kill kills the container. func (c *Container) Kill(ctx context.Context) error { + c.stopProfiling() return c.client.ContainerKill(ctx, c.id, "") } @@ -517,14 +513,6 @@ func (c *Container) Remove(ctx context.Context) error { // CleanUp kills and deletes the container (best effort). func (c *Container) CleanUp(ctx context.Context) { - // Execute profile cleanups before the container goes down. - for _, profile := range c.profiles { - profile.OnCleanUp(c) - } - - // Forget profiles. - c.profiles = nil - // Execute all cleanups. We execute cleanups here to close any // open connections to the container before closing. Open connections // can cause Kill and Remove to hang. @@ -538,10 +526,12 @@ func (c *Container) CleanUp(ctx context.Context) { // Just log; can't do anything here. c.logger.Logf("error killing container %q: %v", c.Name, err) } + // Remove the image. if err := c.Remove(ctx); err != nil { c.logger.Logf("error removing container %q: %v", c.Name, err) } + // Forget all mounts. c.mounts = nil } diff --git a/pkg/test/dockerutil/dockerutil.go b/pkg/test/dockerutil/dockerutil.go index 7027df1a5..a40005799 100644 --- a/pkg/test/dockerutil/dockerutil.go +++ b/pkg/test/dockerutil/dockerutil.go @@ -49,15 +49,11 @@ var ( // pprofBaseDir allows the user to change the directory to which profiles are // written. By default, profiles will appear under: // /tmp/profile/RUNTIME/CONTAINER_NAME/*.pprof. - pprofBaseDir = flag.String("pprof-dir", "/tmp/profile", "base directory in: BASEDIR/RUNTIME/CONTINER_NAME/FILENAME (e.g. /tmp/profile/runtime/mycontainer/cpu.pprof)") - - // duration is the max duration `runsc debug` will run and capture profiles. - // If the container's clean up method is called prior to duration, the - // profiling process will be killed. - duration = flag.Duration("pprof-duration", 10*time.Second, "duration to run the profile in seconds") + pprofBaseDir = flag.String("pprof-dir", "/tmp/profile", "base directory in: BASEDIR/RUNTIME/CONTINER_NAME/FILENAME (e.g. /tmp/profile/runtime/mycontainer/cpu.pprof)") + pprofDuration = flag.Duration("pprof-duration", time.Hour, "profiling duration (automatically stopped at container exit)") // The below flags enable each type of profile. Multiple profiles can be - // enabled for each run. + // enabled for each run. The profile will be collected from the start. pprofBlock = flag.Bool("pprof-block", false, "enables block profiling with runsc debug") pprofCPU = flag.Bool("pprof-cpu", false, "enables CPU profiling with runsc debug") pprofHeap = flag.Bool("pprof-heap", false, "enables heap profiling with runsc debug") diff --git a/pkg/test/dockerutil/profile.go b/pkg/test/dockerutil/profile.go index 55f9496cd..f1103eb6e 100644 --- a/pkg/test/dockerutil/profile.go +++ b/pkg/test/dockerutil/profile.go @@ -17,72 +17,57 @@ package dockerutil import ( "context" "fmt" - "io" "os" "os/exec" "path/filepath" + "syscall" "time" ) -// Profile represents profile-like operations on a container, -// such as running perf or pprof. It is meant to be added to containers -// such that the container type calls the Profile during its lifecycle. -type Profile interface { - // OnCreate is called just after the container is created when the container - // has a valid ID (e.g. c.ID()). - OnCreate(c *Container) error - - // OnStart is called just after the container is started when the container - // has a valid Pid (e.g. c.SandboxPid()). - OnStart(c *Container) error - - // Restart restarts the Profile on request. - Restart(c *Container) error - - // OnCleanUp is called during the container's cleanup method. - // Cleanups should just log errors if they have them. - OnCleanUp(c *Container) error -} - -// Pprof is for running profiles with 'runsc debug'. Pprof workloads -// should be run as root and ONLY against runsc sandboxes. The runtime -// should have --profile set as an option in /etc/docker/daemon.json in -// order for profiling to work with Pprof. -type Pprof struct { - BasePath string // path to put profiles - BlockProfile bool - CPUProfile bool - HeapProfile bool - MutexProfile bool - Duration time.Duration // duration to run profiler e.g. '10s' or '1m'. - shouldRun bool - cmd *exec.Cmd - stdout io.ReadCloser - stderr io.ReadCloser +// profile represents profile-like operations on a container. +// +// It is meant to be added to containers such that the container type calls +// the profile during its lifecycle. Standard implementations are below. + +// profile is for running profiles with 'runsc debug'. +type profile struct { + BasePath string + Types []string + Duration time.Duration + cmd *exec.Cmd } -// MakePprofFromFlags makes a Pprof profile from flags. -func MakePprofFromFlags(c *Container) *Pprof { - if !(*pprofBlock || *pprofCPU || *pprofHeap || *pprofMutex) { - return nil +// profileInit initializes a profile object, if required. +func (c *Container) profileInit() { + if !*pprofBlock && !*pprofCPU && !*pprofMutex && !*pprofHeap { + return // Nothing to do. + } + c.profile = &profile{ + BasePath: filepath.Join(*pprofBaseDir, c.runtime, c.Name), + Duration: *pprofDuration, } - return &Pprof{ - BasePath: filepath.Join(*pprofBaseDir, c.runtime, c.Name), - BlockProfile: *pprofBlock, - CPUProfile: *pprofCPU, - HeapProfile: *pprofHeap, - MutexProfile: *pprofMutex, - Duration: *duration, + if *pprofCPU { + c.profile.Types = append(c.profile.Types, "cpu") + } + if *pprofHeap { + c.profile.Types = append(c.profile.Types, "heap") + } + if *pprofMutex { + c.profile.Types = append(c.profile.Types, "mutex") + } + if *pprofBlock { + c.profile.Types = append(c.profile.Types, "block") } } -// OnCreate implements Profile.OnCreate. -func (p *Pprof) OnCreate(c *Container) error { - return os.MkdirAll(p.BasePath, 0755) -} +// createProcess creates the collection process. +func (p *profile) createProcess(c *Container) error { + // Ensure our directory exists. + if err := os.MkdirAll(p.BasePath, 0755); err != nil { + return err + } -// OnStart implements Profile.OnStart. -func (p *Pprof) OnStart(c *Container) error { + // Find the runtime to invoke. path, err := RuntimePath() if err != nil { return fmt.Errorf("failed to get runtime path: %v", err) @@ -90,58 +75,66 @@ func (p *Pprof) OnStart(c *Container) error { // The root directory of this container's runtime. root := fmt.Sprintf("--root=/var/run/docker/runtime-%s/moby", c.runtime) - // Format is `runsc --root=rootdir debug --profile-*=file --duration=* containerID`. + + // Format is `runsc --root=rootdir debug --profile-*=file --duration=24h containerID`. args := []string{root, "debug"} - args = append(args, p.makeProfileArgs(c)...) + for _, profileArg := range p.Types { + outputPath := filepath.Join(p.BasePath, fmt.Sprintf("%s.pprof", profileArg)) + args = append(args, fmt.Sprintf("--profile-%s=%s", profileArg, outputPath)) + } + args = append(args, fmt.Sprintf("--duration=%s", p.Duration)) // Or until container exits. args = append(args, c.ID()) // Best effort wait until container is running. for now := time.Now(); time.Since(now) < 5*time.Second; { if status, err := c.Status(context.Background()); err != nil { return fmt.Errorf("failed to get status with: %v", err) - } else if status.Running { break } - time.Sleep(500 * time.Millisecond) + time.Sleep(100 * time.Millisecond) } p.cmd = exec.Command(path, args...) + p.cmd.Stderr = os.Stderr // Pass through errors. if err := p.cmd.Start(); err != nil { - return fmt.Errorf("process failed: %v", err) + return fmt.Errorf("start process failed: %v", err) } + return nil } -// Restart implements Profile.Restart. -func (p *Pprof) Restart(c *Container) error { - p.OnCleanUp(c) - return p.OnStart(c) +// killProcess kills the process, if running. +// +// Precondition: mu must be held. +func (p *profile) killProcess() error { + if p.cmd != nil && p.cmd.Process != nil { + return p.cmd.Process.Signal(syscall.SIGTERM) + } + return nil } -// OnCleanUp implements Profile.OnCleanup -func (p *Pprof) OnCleanUp(c *Container) error { +// waitProcess waits for the process, if running. +// +// Precondition: mu must be held. +func (p *profile) waitProcess() error { defer func() { p.cmd = nil }() - if p.cmd != nil && p.cmd.Process != nil && p.cmd.ProcessState != nil && !p.cmd.ProcessState.Exited() { - return p.cmd.Process.Kill() + if p.cmd != nil { + return p.cmd.Wait() } return nil } -// makeProfileArgs turns Pprof fields into runsc debug flags. -func (p *Pprof) makeProfileArgs(c *Container) []string { - var ret []string - if p.BlockProfile { - ret = append(ret, fmt.Sprintf("--profile-block=%s", filepath.Join(p.BasePath, "block.pprof"))) - } - if p.CPUProfile { - ret = append(ret, fmt.Sprintf("--profile-cpu=%s", filepath.Join(p.BasePath, "cpu.pprof"))) - } - if p.HeapProfile { - ret = append(ret, fmt.Sprintf("--profile-heap=%s", filepath.Join(p.BasePath, "heap.pprof"))) - } - if p.MutexProfile { - ret = append(ret, fmt.Sprintf("--profile-mutex=%s", filepath.Join(p.BasePath, "mutex.pprof"))) +// Start is called when profiling is started. +func (p *profile) Start(c *Container) error { + return p.createProcess(c) +} + +// Stop is called when profiling is started. +func (p *profile) Stop(c *Container) error { + killErr := p.killProcess() + waitErr := p.waitProcess() + if waitErr != nil && killErr != nil { + return killErr } - ret = append(ret, fmt.Sprintf("--duration=%s", p.Duration)) - return ret + return waitErr // Ignore okay wait, err kill. } diff --git a/pkg/test/dockerutil/profile_test.go b/pkg/test/dockerutil/profile_test.go index 8c4ffe483..4fe9ce15c 100644 --- a/pkg/test/dockerutil/profile_test.go +++ b/pkg/test/dockerutil/profile_test.go @@ -17,6 +17,7 @@ package dockerutil import ( "context" "fmt" + "io/ioutil" "os" "path/filepath" "testing" @@ -25,52 +26,60 @@ import ( type testCase struct { name string - pprof Pprof + profile profile expectedFiles []string } -func TestPprof(t *testing.T) { +func TestProfile(t *testing.T) { // Basepath and expected file names for each type of profile. - basePath := "/tmp/test/profile" + tmpDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("unable to create temporary directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + // All expected names. + basePath := tmpDir block := "block.pprof" cpu := "cpu.pprof" - goprofle := "go.pprof" heap := "heap.pprof" mutex := "mutex.pprof" testCases := []testCase{ { - name: "Cpu", - pprof: Pprof{ - BasePath: basePath, - CPUProfile: true, - Duration: 2 * time.Second, + name: "One", + profile: profile{ + BasePath: basePath, + Types: []string{"cpu"}, + Duration: 2 * time.Second, }, expectedFiles: []string{cpu}, }, { name: "All", - pprof: Pprof{ - BasePath: basePath, - BlockProfile: true, - CPUProfile: true, - HeapProfile: true, - MutexProfile: true, - Duration: 2 * time.Second, + profile: profile{ + BasePath: basePath, + Types: []string{"block", "cpu", "heap", "mutex"}, + Duration: 2 * time.Second, }, - expectedFiles: []string{block, cpu, goprofle, heap, mutex}, + expectedFiles: []string{block, cpu, heap, mutex}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() c := MakeContainer(ctx, t) + // Set basepath to include the container name so there are no conflicts. - tc.pprof.BasePath = filepath.Join(tc.pprof.BasePath, c.Name) - c.AddProfile(&tc.pprof) + localProfile := tc.profile // Copy it. + localProfile.BasePath = filepath.Join(localProfile.BasePath, tc.name) + + // Set directly on the container, to avoid flags. + c.profile = &localProfile func() { defer c.CleanUp(ctx) + // Start a container. if err := c.Spawn(ctx, RunOpts{ Image: "basic/alpine", @@ -83,24 +92,24 @@ func TestPprof(t *testing.T) { } // End early if the expected files exist and have data. - for start := time.Now(); time.Since(start) < tc.pprof.Duration; time.Sleep(500 * time.Millisecond) { - if err := checkFiles(tc); err == nil { + for start := time.Now(); time.Since(start) < localProfile.Duration; time.Sleep(100 * time.Millisecond) { + if err := checkFiles(localProfile.BasePath, tc.expectedFiles); err == nil { break } } }() // Check all expected files exist and have data. - if err := checkFiles(tc); err != nil { + if err := checkFiles(localProfile.BasePath, tc.expectedFiles); err != nil { t.Fatalf(err.Error()) } }) } } -func checkFiles(tc testCase) error { - for _, file := range tc.expectedFiles { - stat, err := os.Stat(filepath.Join(tc.pprof.BasePath, file)) +func checkFiles(basePath string, expectedFiles []string) error { + for _, file := range expectedFiles { + stat, err := os.Stat(filepath.Join(basePath, file)) if err != nil { return fmt.Errorf("stat failed with: %v", err) } else if stat.Size() < 1 { diff --git a/test/benchmarks/README.md b/test/benchmarks/README.md index d1bbabf6f..1bfb4a129 100644 --- a/test/benchmarks/README.md +++ b/test/benchmarks/README.md @@ -81,11 +81,8 @@ benchmarks. In general, benchmarks should look like this: ```golang - -var h harness.Harness - func BenchmarkMyCoolOne(b *testing.B) { - machine, err := h.GetMachine() + machine, err := harness.GetMachine() // check err defer machine.CleanUp() @@ -95,14 +92,14 @@ func BenchmarkMyCoolOne(b *testing.B) { b.ResetTimer() - //Respect b.N. + // Respect b.N. for i := 0; i < b.N; i++ { out, err := container.Run(ctx, dockerutil.RunOpts{ Image: "benchmarks/my-cool-image", Env: []string{"MY_VAR=awesome"}, other options...see dockerutil }, "sh", "-c", "echo MY_VAR") - //check err + // check err... b.StopTimer() // Do parsing and reporting outside of the timer. @@ -114,16 +111,13 @@ func BenchmarkMyCoolOne(b *testing.B) { } func TestMain(m *testing.M) { - h.Init() + harness.Init() os.Exit(m.Run()) } ``` Some notes on the above: -* The harness is initiated in the TestMain method and made global to test - module. The harness will handle any presetup that needs to happen with - flags, remote virtual machines (eventually), and other services. * Respect `b.N` in that users of the benchmark may want to "run for an hour" or something of the sort. * Use the `b.ReportMetric()` method to report custom metrics. diff --git a/test/benchmarks/base/size_test.go b/test/benchmarks/base/size_test.go index acc49cc7c..452926e5f 100644 --- a/test/benchmarks/base/size_test.go +++ b/test/benchmarks/base/size_test.go @@ -26,12 +26,10 @@ import ( "gvisor.dev/gvisor/test/benchmarks/tools" ) -var testHarness harness.Harness - // BenchmarkSizeEmpty creates N empty containers and reads memory usage from // /proc/meminfo. func BenchmarkSizeEmpty(b *testing.B) { - machine, err := testHarness.GetMachine() + machine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine: %v", err) } @@ -81,7 +79,7 @@ func BenchmarkSizeEmpty(b *testing.B) { // BenchmarkSizeNginx starts N containers running Nginx, checks that they're // serving, and checks memory used based on /proc/meminfo. func BenchmarkSizeNginx(b *testing.B) { - machine, err := testHarness.GetMachine() + machine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine with: %v", err) } @@ -126,7 +124,7 @@ func BenchmarkSizeNginx(b *testing.B) { // BenchmarkSizeNode starts N containers running a Node app, checks that // they're serving, and checks memory used based on /proc/meminfo. func BenchmarkSizeNode(b *testing.B) { - machine, err := testHarness.GetMachine() + machine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine with: %v", err) } @@ -178,6 +176,6 @@ func BenchmarkSizeNode(b *testing.B) { // TestMain is the main method for package network. func TestMain(m *testing.M) { - testHarness.Init() + harness.Init() os.Exit(m.Run()) } diff --git a/test/benchmarks/base/startup_test.go b/test/benchmarks/base/startup_test.go index 8ef9f99c4..05a43ad17 100644 --- a/test/benchmarks/base/startup_test.go +++ b/test/benchmarks/base/startup_test.go @@ -25,11 +25,9 @@ import ( "gvisor.dev/gvisor/test/benchmarks/harness" ) -var testHarness harness.Harness - // BenchmarkStartEmpty times startup time for an empty container. func BenchmarkStartupEmpty(b *testing.B) { - machine, err := testHarness.GetMachine() + machine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine: %v", err) } @@ -53,7 +51,7 @@ func BenchmarkStartupEmpty(b *testing.B) { // Time is measured from start until the first request is served. func BenchmarkStartupNginx(b *testing.B) { // The machine to hold Nginx and the Node Server. - machine, err := testHarness.GetMachine() + machine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine with: %v", err) } @@ -76,7 +74,7 @@ func BenchmarkStartupNginx(b *testing.B) { // Time is measured from start until the first request is served. // Note that the Node app connects to a Redis instance before serving. func BenchmarkStartupNode(b *testing.B) { - machine, err := testHarness.GetMachine() + machine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine with: %v", err) } @@ -126,8 +124,8 @@ func runServerWorkload(ctx context.Context, b *testing.B, args base.ServerArgs) return fmt.Errorf("failed to get ip from server: %v", err) } - harness.DebugLog(b, "Waiting for container to start.") // Wait until the Client sees the server as up. + harness.DebugLog(b, "Waiting for container to start.") if err := harness.WaitUntilServing(ctx, args.Machine, servingIP, args.Port); err != nil { return fmt.Errorf("failed to wait for serving: %v", err) } @@ -141,6 +139,6 @@ func runServerWorkload(ctx context.Context, b *testing.B, args base.ServerArgs) // TestMain is the main method for package network. func TestMain(m *testing.M) { - testHarness.Init() + harness.Init() os.Exit(m.Run()) } diff --git a/test/benchmarks/base/sysbench_test.go b/test/benchmarks/base/sysbench_test.go index bbb797e14..80569687c 100644 --- a/test/benchmarks/base/sysbench_test.go +++ b/test/benchmarks/base/sysbench_test.go @@ -23,8 +23,6 @@ import ( "gvisor.dev/gvisor/test/benchmarks/tools" ) -var testHarness harness.Harness - type testCase struct { name string test tools.Sysbench @@ -32,42 +30,34 @@ type testCase struct { // BenchmarSysbench runs sysbench on the runtime. func BenchmarkSysbench(b *testing.B) { - testCases := []testCase{ testCase{ name: "CPU", test: &tools.SysbenchCPU{ - Base: tools.SysbenchBase{ + SysbenchBase: tools.SysbenchBase{ Threads: 1, - Time: 5, }, - MaxPrime: 50000, }, }, testCase{ name: "Memory", test: &tools.SysbenchMemory{ - Base: tools.SysbenchBase{ + SysbenchBase: tools.SysbenchBase{ Threads: 1, }, - BlockSize: "1M", - TotalSize: "500G", }, }, testCase{ name: "Mutex", test: &tools.SysbenchMutex{ - Base: tools.SysbenchBase{ + SysbenchBase: tools.SysbenchBase{ Threads: 8, }, - Loops: 1, - Locks: 10000000, - Num: 4, }, }, } - machine, err := testHarness.GetMachine() + machine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine: %v", err) } @@ -87,12 +77,15 @@ func BenchmarkSysbench(b *testing.B) { sysbench := machine.GetContainer(ctx, b) defer sysbench.CleanUp(ctx) + cmd := tc.test.MakeCmd(b) + b.ResetTimer() out, err := sysbench.Run(ctx, dockerutil.RunOpts{ Image: "benchmarks/sysbench", - }, tc.test.MakeCmd()...) + }, cmd...) if err != nil { b.Fatalf("failed to run sysbench: %v: logs:%s", err, out) } + b.StopTimer() tc.test.Report(b, out) }) } diff --git a/test/benchmarks/database/redis_test.go b/test/benchmarks/database/redis_test.go index f8075a04b..f3c4522ac 100644 --- a/test/benchmarks/database/redis_test.go +++ b/test/benchmarks/database/redis_test.go @@ -25,8 +25,6 @@ import ( "gvisor.dev/gvisor/test/benchmarks/tools" ) -var h harness.Harness - // All possible operations from redis. Note: "ping" will // run both PING_INLINE and PING_BUILD. var operations []string = []string{ @@ -52,13 +50,13 @@ var operations []string = []string{ // BenchmarkRedis runs redis-benchmark against a redis instance and reports // data in queries per second. Each is reported by named operation (e.g. LPUSH). func BenchmarkRedis(b *testing.B) { - clientMachine, err := h.GetMachine() + clientMachine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine: %v", err) } defer clientMachine.CleanUp() - serverMachine, err := h.GetMachine() + serverMachine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine: %v", err) } @@ -67,7 +65,6 @@ func BenchmarkRedis(b *testing.B) { // Redis runs on port 6379 by default. port := 6379 ctx := context.Background() - for _, operation := range operations { param := tools.Parameter{ Name: "operation", @@ -107,23 +104,19 @@ func BenchmarkRedis(b *testing.B) { b.Fatalf("failed to start redis with: %v", err) } + client := clientMachine.GetNativeContainer(ctx, b) + defer client.CleanUp(ctx) + redis := tools.Redis{ Operation: operation, } - - // Reset profiles and timer to begin the measurement. - server.RestartProfiles() b.ResetTimer() - client := clientMachine.GetNativeContainer(ctx, b) - defer client.CleanUp(ctx) out, err := client.Run(ctx, dockerutil.RunOpts{ Image: "benchmarks/redis", }, redis.MakeCmd(ip, serverPort, b.N /*requests*/)...) if err != nil { b.Fatalf("redis-benchmark failed with: %v", err) } - - // Stop time while we parse results. b.StopTimer() redis.Report(b, out) }) @@ -131,6 +124,6 @@ func BenchmarkRedis(b *testing.B) { } func TestMain(m *testing.M) { - h.Init() + harness.Init() os.Exit(m.Run()) } diff --git a/test/benchmarks/fs/bazel_test.go b/test/benchmarks/fs/bazel_test.go index 3fb4da9d1..8baeff0db 100644 --- a/test/benchmarks/fs/bazel_test.go +++ b/test/benchmarks/fs/bazel_test.go @@ -25,8 +25,6 @@ import ( "gvisor.dev/gvisor/test/benchmarks/tools" ) -var h harness.Harness - // Note: CleanCache versions of this test require running with root permissions. func BenchmarkBuildABSL(b *testing.B) { runBuildBenchmark(b, "benchmarks/absl", "/abseil-cpp", "absl/base/...") @@ -41,7 +39,7 @@ func BenchmarkBuildRunsc(b *testing.B) { func runBuildBenchmark(b *testing.B, image, workdir, target string) { b.Helper() // Get a machine from the Harness on which to run. - machine, err := h.GetMachine() + machine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine: %v", err) } @@ -102,21 +100,20 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) { prefix = "/tmp" } - // Restart profiles after the copy. - container.RestartProfiles() b.ResetTimer() + b.StopTimer() + // Drop Caches and bazel clean should happen inside the loop as we may use // time options with b.N. (e.g. Run for an hour.) for i := 0; i < b.N; i++ { - b.StopTimer() // Drop Caches for clear cache runs. if bm.clearCache { if err := harness.DropCaches(machine); err != nil { b.Skipf("failed to drop caches: %v. You probably need root.", err) } } - b.StartTimer() + b.StartTimer() got, err := container.Exec(ctx, dockerutil.ExecOpts{ WorkDir: prefix + workdir, }, "bazel", "build", "-c", "opt", target) @@ -138,7 +135,6 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) { b.Fatalf("build failed with: %v", err) } } - b.StartTimer() } }) } @@ -146,6 +142,7 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) { // TestMain is the main method for package fs. func TestMain(m *testing.M) { - h.Init() + harness.Init() + harness.SetFixedBenchmarks() os.Exit(m.Run()) } diff --git a/test/benchmarks/fs/fio_test.go b/test/benchmarks/fs/fio_test.go index 96340373c..0c772b768 100644 --- a/test/benchmarks/fs/fio_test.go +++ b/test/benchmarks/fs/fio_test.go @@ -27,42 +27,50 @@ import ( "gvisor.dev/gvisor/test/benchmarks/tools" ) -var h harness.Harness - // BenchmarkFio runs fio on the runtime under test. There are 4 basic test // cases each run on a tmpfs mount and a bind mount. Fio requires root so that // caches can be dropped. func BenchmarkFio(b *testing.B) { testCases := []tools.Fio{ tools.Fio{ - Test: "write", - Size: "5G", - Blocksize: "1M", - Iodepth: 4, + Test: "write4K", + Size: b.N, + BlockSize: 4, + IODepth: 4, + }, + tools.Fio{ + Test: "write1M", + Size: b.N, + BlockSize: 1024, + IODepth: 4, + }, + tools.Fio{ + Test: "read4K", + Size: b.N, + BlockSize: 4, + IODepth: 4, }, tools.Fio{ - Test: "read", - Size: "5G", - Blocksize: "1M", - Iodepth: 4, + Test: "read1M", + Size: b.N, + BlockSize: 1024, + IODepth: 4, }, tools.Fio{ - Test: "randwrite", - Size: "5G", - Blocksize: "4K", - Iodepth: 4, - Time: 30, + Test: "randwrite4K", + Size: b.N, + BlockSize: 4, + IODepth: 4, }, tools.Fio{ - Test: "randread", - Size: "5G", - Blocksize: "4K", - Iodepth: 4, - Time: 30, + Test: "randread4K", + Size: b.N, + BlockSize: 4, + IODepth: 4, }, } - machine, err := h.GetMachine() + machine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine with: %v", err) } @@ -116,7 +124,7 @@ func BenchmarkFio(b *testing.B) { // For reads, we need a file to read so make one inside the container. if strings.Contains(tc.Test, "read") { - fallocateCmd := fmt.Sprintf("fallocate -l %s %s", tc.Size, outfile) + fallocateCmd := fmt.Sprintf("fallocate -l %dK %s", tc.Size, outfile) if out, err := container.Exec(ctx, dockerutil.ExecOpts{}, strings.Split(fallocateCmd, " ")...); err != nil { b.Fatalf("failed to create readable file on mount: %v, %s", err, out) @@ -128,22 +136,24 @@ func BenchmarkFio(b *testing.B) { b.Skipf("failed to drop caches with %v. You probably need root.", err) } cmd := tc.MakeCmd(outfile) - container.RestartProfiles() + b.ResetTimer() + b.StopTimer() + for i := 0; i < b.N; i++ { + if err := harness.DropCaches(machine); err != nil { + b.Fatalf("failed to drop caches: %v", err) + } + // Run fio. + b.StartTimer() data, err := container.Exec(ctx, dockerutil.ExecOpts{}, cmd...) if err != nil { b.Fatalf("failed to run cmd %v: %v", cmd, err) } b.StopTimer() + b.SetBytes(1024 * 1024) // Bytes for go reporting (Size is in megabytes). tc.Report(b, data) - // If b.N is used (i.e. we run for an hour), we should drop caches - // after each run. - if err := harness.DropCaches(machine); err != nil { - b.Fatalf("failed to drop caches: %v", err) - } - b.StartTimer() } }) } @@ -185,6 +195,6 @@ func makeMount(machine harness.Machine, mountType mount.Type, target string) (mo // TestMain is the main method for package fs. func TestMain(m *testing.M) { - h.Init() + harness.Init() os.Exit(m.Run()) } diff --git a/test/benchmarks/harness/harness.go b/test/benchmarks/harness/harness.go index 4c6e724aa..a853b7ba8 100644 --- a/test/benchmarks/harness/harness.go +++ b/test/benchmarks/harness/harness.go @@ -28,12 +28,8 @@ var ( debug = flag.Bool("debug", false, "turns on debug messages for individual benchmarks") ) -// Harness is a handle for managing state in benchmark runs. -type Harness struct { -} - // Init performs any harness initilialization before runs. -func (h *Harness) Init() error { +func Init() error { flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage: %s -- --test.bench=<regex>\n", os.Args[0]) flag.PrintDefaults() @@ -47,7 +43,15 @@ func (h *Harness) Init() error { return nil } +// SetFixedBenchmarks causes all benchmarks to run once. +// +// This must be set if they cannot scale with N. Note that this uses 1ns +// instead of 1x due to https://github.com/golang/go/issues/32051. +func SetFixedBenchmarks() { + flag.Set("test.benchtime", "1ns") +} + // GetMachine returns this run's implementation of machine. -func (h *Harness) GetMachine() (Machine, error) { +func GetMachine() (Machine, error) { return &localMachine{}, nil } diff --git a/test/benchmarks/media/ffmpeg_test.go b/test/benchmarks/media/ffmpeg_test.go index a462ec2a6..1b99a319a 100644 --- a/test/benchmarks/media/ffmpeg_test.go +++ b/test/benchmarks/media/ffmpeg_test.go @@ -23,12 +23,10 @@ import ( "gvisor.dev/gvisor/test/benchmarks/harness" ) -var h harness.Harness - // BenchmarkFfmpeg runs ffmpeg in a container and records runtime. // BenchmarkFfmpeg should run as root to drop caches. func BenchmarkFfmpeg(b *testing.B) { - machine, err := h.GetMachine() + machine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine: %v", err) } @@ -38,24 +36,26 @@ func BenchmarkFfmpeg(b *testing.B) { cmd := strings.Split("ffmpeg -i video.mp4 -c:v libx264 -preset veryslow output.mp4", " ") b.ResetTimer() + b.StopTimer() + for i := 0; i < b.N; i++ { - b.StopTimer() container := machine.GetContainer(ctx, b) defer container.CleanUp(ctx) if err := harness.DropCaches(machine); err != nil { b.Skipf("failed to drop caches: %v. You probably need root.", err) } - b.StartTimer() + b.StartTimer() if _, err := container.Run(ctx, dockerutil.RunOpts{ Image: "benchmarks/ffmpeg", }, cmd...); err != nil { b.Fatalf("failed to run container: %v", err) } + b.StopTimer() } } func TestMain(m *testing.M) { - h.Init() + harness.Init() os.Exit(m.Run()) } diff --git a/test/benchmarks/ml/tensorflow_test.go b/test/benchmarks/ml/tensorflow_test.go index a55329d82..b0e0c4720 100644 --- a/test/benchmarks/ml/tensorflow_test.go +++ b/test/benchmarks/ml/tensorflow_test.go @@ -22,8 +22,6 @@ import ( "gvisor.dev/gvisor/test/benchmarks/harness" ) -var h harness.Harness - // BenchmarkTensorflow runs workloads from a TensorFlow tutorial. // See: https://github.com/aymericdamien/TensorFlow-Examples func BenchmarkTensorflow(b *testing.B) { @@ -38,7 +36,7 @@ func BenchmarkTensorflow(b *testing.B) { "NeuralNetwork": "3_NeuralNetworks/neural_network.py", } - machine, err := h.GetMachine() + machine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine: %v", err) } @@ -49,15 +47,17 @@ func BenchmarkTensorflow(b *testing.B) { ctx := context.Background() b.ResetTimer() + b.StopTimer() + for i := 0; i < b.N; i++ { - b.StopTimer() container := machine.GetContainer(ctx, b) defer container.CleanUp(ctx) if err := harness.DropCaches(machine); err != nil { b.Skipf("failed to drop caches: %v. You probably need root.", err) } - b.StartTimer() + // Run tensorflow. + b.StartTimer() if out, err := container.Run(ctx, dockerutil.RunOpts{ Image: "benchmarks/tensorflow", Env: []string{"PYTHONPATH=$PYTHONPATH:/TensorFlow-Examples/examples"}, @@ -65,13 +65,14 @@ func BenchmarkTensorflow(b *testing.B) { }, "python", workload); err != nil { b.Fatalf("failed to run container: %v logs: %s", err, out) } + b.StopTimer() } }) } - } func TestMain(m *testing.M) { - h.Init() + harness.Init() + harness.SetFixedBenchmarks() os.Exit(m.Run()) } diff --git a/test/benchmarks/network/httpd_test.go b/test/benchmarks/network/httpd_test.go index b07274662..629127250 100644 --- a/test/benchmarks/network/httpd_test.go +++ b/test/benchmarks/network/httpd_test.go @@ -23,8 +23,6 @@ import ( "gvisor.dev/gvisor/test/benchmarks/tools" ) -var h harness.Harness - // see Dockerfile '//images/benchmarks/httpd'. var httpdDocs = map[string]string{ "notfound": "notfound", @@ -38,13 +36,7 @@ var httpdDocs = map[string]string{ // BenchmarkHttpd iterates over different sized payloads and concurrency, testing // how well the runtime handles sending different payload sizes. func BenchmarkHttpd(b *testing.B) { - benchmarkHttpdDocSize(b, false /* reverse */) -} - -// BenchmarkReverseHttpd iterates over different sized payloads, testing -// how well the runtime handles receiving different payload sizes. -func BenchmarkReverseHttpd(b *testing.B) { - benchmarkHttpdDocSize(b, true /* reverse */) + benchmarkHttpdDocSize(b) } // BenchmarkContinuousHttpd runs specific benchmarks for continous jobs. @@ -52,20 +44,12 @@ func BenchmarkReverseHttpd(b *testing.B) { func BenchmarkContinuousHttpd(b *testing.B) { sizes := []string{"10Kb", "100Kb", "1Mb"} threads := []int{1, 25, 100, 1000} - benchmarkHttpdContinuous(b, threads, sizes, false /*reverse*/) -} - -// BenchmarkContinuousHttpdReverse runs specific benchmarks for continous jobs. -// The runtime under test is the client downloading from a runc server. -func BenchmarkContinuousHttpdReverse(b *testing.B) { - sizes := []string{"10Kb", "100Kb", "1Mb"} - threads := []int{1, 25, 100, 1000} - benchmarkHttpdContinuous(b, threads, sizes, true /*reverse*/) + benchmarkHttpdContinuous(b, threads, sizes) } // benchmarkHttpdDocSize iterates through all doc sizes, running subbenchmarks // for each size. -func benchmarkHttpdDocSize(b *testing.B, reverse bool) { +func benchmarkHttpdDocSize(b *testing.B) { b.Helper() for size, filename := range httpdDocs { concurrency := []int{1, 25, 50, 100, 1000} @@ -82,25 +66,20 @@ func benchmarkHttpdDocSize(b *testing.B, reverse bool) { if err != nil { b.Fatalf("Failed to parse parameters: %v", err) } - requests := b.N - if requests < c { - b.Logf("b.N is %d must be greater than threads %d. Consider running with --test.benchtime=Nx where N >= %d", b.N, c, c) - requests = c - } b.Run(name, func(b *testing.B) { hey := &tools.Hey{ - Requests: requests, + Requests: b.N, Concurrency: c, Doc: filename, } - runHttpd(b, hey, reverse) + runHttpd(b, hey) }) } } } // benchmarkHttpdContinuous iterates through given sizes and concurrencies. -func benchmarkHttpdContinuous(b *testing.B, concurrency []int, sizes []string, reverse bool) { +func benchmarkHttpdContinuous(b *testing.B, concurrency []int, sizes []string) { for _, size := range sizes { filename := httpdDocs[size] for _, c := range concurrency { @@ -118,26 +97,20 @@ func benchmarkHttpdContinuous(b *testing.B, concurrency []int, sizes []string, r if err != nil { b.Fatalf("Failed to parse parameters: %v", err) } - - requests := b.N - if requests < c { - b.Logf("b.N is %d must be greater than threads %d. Consider running with --test.benchtime=Nx where N >= %d", b.N, c, c) - requests = c - } b.Run(name, func(b *testing.B) { hey := &tools.Hey{ - Requests: requests, + Requests: b.N, Concurrency: c, Doc: filename, } - runHttpd(b, hey, reverse) + runHttpd(b, hey) }) } } } // runHttpd configures the static serving methods to run httpd. -func runHttpd(b *testing.B, hey *tools.Hey, reverse bool) { +func runHttpd(b *testing.B, hey *tools.Hey) { // httpd runs on port 80. port := 80 httpdRunOpts := dockerutil.RunOpts{ @@ -153,10 +126,10 @@ func runHttpd(b *testing.B, hey *tools.Hey, reverse bool) { }, } httpdCmd := []string{"sh", "-c", "mkdir -p /tmp/html; cp -r /local/* /tmp/html/.; apache2 -X"} - runStaticServer(b, h, httpdRunOpts, httpdCmd, port, hey, reverse) + runStaticServer(b, httpdRunOpts, httpdCmd, port, hey) } func TestMain(m *testing.M) { - h.Init() + harness.Init() os.Exit(m.Run()) } diff --git a/test/benchmarks/network/iperf_test.go b/test/benchmarks/network/iperf_test.go index 9d64db943..5e81149fe 100644 --- a/test/benchmarks/network/iperf_test.go +++ b/test/benchmarks/network/iperf_test.go @@ -24,20 +24,18 @@ import ( "gvisor.dev/gvisor/test/benchmarks/tools" ) -var h harness.Harness - func BenchmarkIperf(b *testing.B) { iperf := tools.Iperf{ - Time: b.N, // time in seconds to run client. + Num: b.N, } - clientMachine, err := h.GetMachine() + clientMachine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine: %v", err) } defer clientMachine.CleanUp() - serverMachine, err := h.GetMachine() + serverMachine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine: %v", err) } @@ -94,12 +92,9 @@ func BenchmarkIperf(b *testing.B) { if err := harness.WaitUntilServing(ctx, clientMachine, ip, servingPort); err != nil { b.Fatalf("failed to wait for server: %v", err) } + // Run the client. b.ResetTimer() - - // Restart the server profiles. If the server isn't being profiled - // this does nothing. - server.RestartProfiles() out, err := client.Run(ctx, dockerutil.RunOpts{ Image: "benchmarks/iperf", }, iperf.MakeCmd(ip, servingPort)...) @@ -113,6 +108,6 @@ func BenchmarkIperf(b *testing.B) { } func TestMain(m *testing.M) { - h.Init() + harness.Init() os.Exit(m.Run()) } diff --git a/test/benchmarks/network/network.go b/test/benchmarks/network/network.go index b18bc2b3c..d61002cea 100644 --- a/test/benchmarks/network/network.go +++ b/test/benchmarks/network/network.go @@ -25,33 +25,26 @@ import ( ) // runStaticServer runs static serving workloads (httpd, nginx). -func runStaticServer(b *testing.B, h harness.Harness, serverOpts dockerutil.RunOpts, serverCmd []string, port int, hey *tools.Hey, reverse bool) { +func runStaticServer(b *testing.B, serverOpts dockerutil.RunOpts, serverCmd []string, port int, hey *tools.Hey) { ctx := context.Background() // Get two machines: a client and server. - clientMachine, err := h.GetMachine() + clientMachine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine: %v", err) } defer clientMachine.CleanUp() - serverMachine, err := h.GetMachine() + serverMachine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine: %v", err) } defer serverMachine.CleanUp() - // Make the containers. 'reverse=true' specifies that the client should use the - // runtime under test. - var client, server *dockerutil.Container - if reverse { - client = clientMachine.GetContainer(ctx, b) - server = serverMachine.GetNativeContainer(ctx, b) - } else { - client = clientMachine.GetNativeContainer(ctx, b) - server = serverMachine.GetContainer(ctx, b) - } + // Make the containers. + client := clientMachine.GetNativeContainer(ctx, b) defer client.CleanUp(ctx) + server := serverMachine.GetContainer(ctx, b) defer server.CleanUp(ctx) // Start the server. @@ -73,16 +66,15 @@ func runStaticServer(b *testing.B, h harness.Harness, serverOpts dockerutil.RunO // Make sure the server is serving. harness.WaitUntilServing(ctx, clientMachine, ip, servingPort) + + // Run the client. b.ResetTimer() - server.RestartProfiles() out, err := client.Run(ctx, dockerutil.RunOpts{ Image: "benchmarks/hey", }, hey.MakeCmd(ip, servingPort)...) if err != nil { b.Fatalf("run failed with: %v", err) } - b.StopTimer() hey.Report(b, out) - b.StartTimer() } diff --git a/test/benchmarks/network/nginx_test.go b/test/benchmarks/network/nginx_test.go index 87449612a..74f3578fc 100644 --- a/test/benchmarks/network/nginx_test.go +++ b/test/benchmarks/network/nginx_test.go @@ -23,8 +23,6 @@ import ( "gvisor.dev/gvisor/test/benchmarks/tools" ) -var h harness.Harness - // see Dockerfile '//images/benchmarks/nginx'. var nginxDocs = map[string]string{ "notfound": "notfound", @@ -38,14 +36,8 @@ var nginxDocs = map[string]string{ // BenchmarkNginxDocSize iterates over different sized payloads, testing how // well the runtime handles sending different payload sizes. func BenchmarkNginxDocSize(b *testing.B) { - benchmarkNginxDocSize(b, false /* reverse */, true /* tmpfs */) - benchmarkNginxDocSize(b, false /* reverse */, false /* tmpfs */) -} - -// BenchmarkReverseNginxDocSize iterates over different sized payloads, testing -// how well the runtime handles receiving different payload sizes. -func BenchmarkReverseNginxDocSize(b *testing.B) { - benchmarkNginxDocSize(b, true /* reverse */, true /* tmpfs */) + benchmarkNginxDocSize(b, true /* tmpfs */) + benchmarkNginxDocSize(b, false /* tmpfs */) } // BenchmarkContinuousNginx runs specific benchmarks for continous jobs. @@ -53,20 +45,12 @@ func BenchmarkReverseNginxDocSize(b *testing.B) { func BenchmarkContinuousNginx(b *testing.B) { sizes := []string{"10Kb", "100Kb", "1Mb"} threads := []int{1, 25, 100, 1000} - benchmarkNginxContinuous(b, threads, sizes, false /*reverse*/) -} - -// BenchmarkContinuousNginxReverse runs specific benchmarks for continous jobs. -// The runtime under test is the client downloading from a runc server. -func BenchmarkContinuousNginxReverse(b *testing.B) { - sizes := []string{"10Kb", "100Kb", "1Mb"} - threads := []int{1, 25, 100, 1000} - benchmarkNginxContinuous(b, threads, sizes, true /*reverse*/) + benchmarkNginxContinuous(b, threads, sizes) } // benchmarkNginxDocSize iterates through all doc sizes, running subbenchmarks // for each size. -func benchmarkNginxDocSize(b *testing.B, reverse, tmpfs bool) { +func benchmarkNginxDocSize(b *testing.B, tmpfs bool) { for size, filename := range nginxDocs { concurrency := []int{1, 25, 50, 100, 1000} for _, c := range concurrency { @@ -91,26 +75,20 @@ func benchmarkNginxDocSize(b *testing.B, reverse, tmpfs bool) { if err != nil { b.Fatalf("Failed to parse parameters: %v", err) } - - requests := b.N - if requests < c { - b.Logf("b.N is %d must be greater than threads %d. Consider running with --test.benchtime=Nx where N >= %d", b.N, c, c) - requests = c - } b.Run(name, func(b *testing.B) { hey := &tools.Hey{ - Requests: requests, + Requests: b.N, Concurrency: c, Doc: filename, } - runNginx(b, hey, reverse, tmpfs) + runNginx(b, hey, tmpfs) }) } } } // benchmarkNginxContinuous iterates through given sizes and concurrencies on a tmpfs mount. -func benchmarkNginxContinuous(b *testing.B, concurrency []int, sizes []string, reverse bool) { +func benchmarkNginxContinuous(b *testing.B, concurrency []int, sizes []string) { for _, size := range sizes { filename := nginxDocs[size] for _, c := range concurrency { @@ -133,25 +111,20 @@ func benchmarkNginxContinuous(b *testing.B, concurrency []int, sizes []string, r if err != nil { b.Fatalf("Failed to parse parameters: %v", err) } - requests := b.N - if requests < c { - b.Logf("b.N is %d must be greater than threads %d. Consider running with --test.benchtime=Nx where N >= %d", b.N, c, c) - requests = c - } b.Run(name, func(b *testing.B) { hey := &tools.Hey{ - Requests: requests, + Requests: b.N, Concurrency: c, Doc: filename, } - runNginx(b, hey, reverse, true /*tmpfs*/) + runNginx(b, hey, true /*tmpfs*/) }) } } } // runNginx configures the static serving methods to run httpd. -func runNginx(b *testing.B, hey *tools.Hey, reverse, tmpfs bool) { +func runNginx(b *testing.B, hey *tools.Hey, tmpfs bool) { // nginx runs on port 80. port := 80 nginxRunOpts := dockerutil.RunOpts{ @@ -165,10 +138,10 @@ func runNginx(b *testing.B, hey *tools.Hey, reverse, tmpfs bool) { } // Command copies nginxDocs to tmpfs serving directory and runs nginx. - runStaticServer(b, h, nginxRunOpts, nginxCmd, port, hey, reverse) + runStaticServer(b, nginxRunOpts, nginxCmd, port, hey) } func TestMain(m *testing.M) { - h.Init() + harness.Init() os.Exit(m.Run()) } diff --git a/test/benchmarks/network/node_test.go b/test/benchmarks/network/node_test.go index 3e837a9e4..a1fc82f95 100644 --- a/test/benchmarks/network/node_test.go +++ b/test/benchmarks/network/node_test.go @@ -25,8 +25,6 @@ import ( "gvisor.dev/gvisor/test/benchmarks/tools" ) -var h harness.Harness - // BenchmarkNode runs requests using 'hey' against a Node server run on // 'runtime'. The server responds to requests by grabbing some data in a // redis instance and returns the data in its reponse. The test loops through @@ -42,14 +40,9 @@ func BenchmarkNode(b *testing.B) { if err != nil { b.Fatalf("Failed to parse parameters: %v", err) } - requests := b.N - if requests < c { - b.Logf("b.N is %d must be greater than threads %d. Consider running with --test.benchtime=Nx where N >= %d", b.N, c, c) - requests = c - } b.Run(name, func(b *testing.B) { hey := &tools.Hey{ - Requests: requests, + Requests: b.N, Concurrency: c, } runNode(b, hey) @@ -62,14 +55,14 @@ func runNode(b *testing.B, hey *tools.Hey) { b.Helper() // The machine to hold Redis and the Node Server. - serverMachine, err := h.GetMachine() + serverMachine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine with: %v", err) } defer serverMachine.CleanUp() // The machine to run 'hey'. - clientMachine, err := h.GetMachine() + clientMachine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine with: %v", err) } @@ -124,10 +117,8 @@ func runNode(b *testing.B, hey *tools.Hey) { heyCmd := hey.MakeCmd(servingIP, servingPort) - nodeApp.RestartProfiles() - b.ResetTimer() - // the client should run on Native. + b.ResetTimer() client := clientMachine.GetNativeContainer(ctx, b) out, err := client.Run(ctx, dockerutil.RunOpts{ Image: "benchmarks/hey", @@ -137,11 +128,10 @@ func runNode(b *testing.B, hey *tools.Hey) { } // Stop the timer to parse the data and report stats. - b.StopTimer() hey.Report(b, out) } func TestMain(m *testing.M) { - h.Init() + harness.Init() os.Exit(m.Run()) } diff --git a/test/benchmarks/network/ruby_test.go b/test/benchmarks/network/ruby_test.go index c89672873..b7ec16e0a 100644 --- a/test/benchmarks/network/ruby_test.go +++ b/test/benchmarks/network/ruby_test.go @@ -26,8 +26,6 @@ import ( "gvisor.dev/gvisor/test/benchmarks/tools" ) -var h harness.Harness - // BenchmarkRuby runs requests using 'hey' against a ruby application server. // On start, ruby app generates some random data and pushes it to a redis // instance. On a request, the app grabs for random entries from the redis @@ -43,14 +41,9 @@ func BenchmarkRuby(b *testing.B) { if err != nil { b.Fatalf("Failed to parse parameters: %v", err) } - requests := b.N - if requests < c { - b.Logf("b.N is %d must be greater than threads %d. Consider running with --test.benchtime=Nx where N >= %d", b.N, c, c) - requests = c - } b.Run(name, func(b *testing.B) { hey := &tools.Hey{ - Requests: requests, + Requests: b.N, Concurrency: c, } runRuby(b, hey) @@ -61,14 +54,14 @@ func BenchmarkRuby(b *testing.B) { // runRuby runs the test for a given # of requests and concurrency. func runRuby(b *testing.B, hey *tools.Hey) { // The machine to hold Redis and the Ruby Server. - serverMachine, err := h.GetMachine() + serverMachine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine with: %v", err) } defer serverMachine.CleanUp() // The machine to run 'hey'. - clientMachine, err := h.GetMachine() + clientMachine, err := harness.GetMachine() if err != nil { b.Fatalf("failed to get machine with: %v", err) } @@ -130,10 +123,9 @@ func runRuby(b *testing.B, hey *tools.Hey) { b.Fatalf("failed to wait until serving: %v", err) } heyCmd := hey.MakeCmd(servingIP, servingPort) - rubyApp.RestartProfiles() - b.ResetTimer() // the client should run on Native. + b.ResetTimer() client := clientMachine.GetNativeContainer(ctx, b) defer client.CleanUp(ctx) out, err := client.Run(ctx, dockerutil.RunOpts{ @@ -142,14 +134,11 @@ func runRuby(b *testing.B, hey *tools.Hey) { if err != nil { b.Fatalf("hey container failed: %v logs: %s", err, out) } - - // Stop the timer to parse the data and report stats. b.StopTimer() hey.Report(b, out) - b.StartTimer() } func TestMain(m *testing.M) { - h.Init() + harness.Init() os.Exit(m.Run()) } diff --git a/test/benchmarks/tools/fio.go b/test/benchmarks/tools/fio.go index f5f60fa84..f6324c3ab 100644 --- a/test/benchmarks/tools/fio.go +++ b/test/benchmarks/tools/fio.go @@ -25,25 +25,20 @@ import ( // Fio makes 'fio' commands and parses their output. type Fio struct { Test string // test to run: read, write, randread, randwrite. - Size string // total size to be read/written of format N[GMK] (e.g. 5G). - Blocksize string // blocksize to be read/write of format N[GMK] (e.g. 4K). - Iodepth int // iodepth for reads/writes. - Time int // time to run the test in seconds, usually for rand(read/write). + Size int // total size to be read/written in megabytes. + BlockSize int // block size to be read/written in kilobytes. + IODepth int // I/O depth for reads/writes. } // MakeCmd makes a 'fio' command. func (f *Fio) MakeCmd(filename string) []string { cmd := []string{"fio", "--output-format=json", "--ioengine=sync"} cmd = append(cmd, fmt.Sprintf("--name=%s", f.Test)) - cmd = append(cmd, fmt.Sprintf("--size=%s", f.Size)) - cmd = append(cmd, fmt.Sprintf("--blocksize=%s", f.Blocksize)) + cmd = append(cmd, fmt.Sprintf("--size=%dM", f.Size)) + cmd = append(cmd, fmt.Sprintf("--blocksize=%dK", f.BlockSize)) cmd = append(cmd, fmt.Sprintf("--filename=%s", filename)) - cmd = append(cmd, fmt.Sprintf("--iodepth=%d", f.Iodepth)) + cmd = append(cmd, fmt.Sprintf("--iodepth=%d", f.IODepth)) cmd = append(cmd, fmt.Sprintf("--rw=%s", f.Test)) - if f.Time != 0 { - cmd = append(cmd, "--time_based") - cmd = append(cmd, fmt.Sprintf("--runtime=%d", f.Time)) - } return cmd } diff --git a/test/benchmarks/tools/hey.go b/test/benchmarks/tools/hey.go index b8cb938fe..de908feeb 100644 --- a/test/benchmarks/tools/hey.go +++ b/test/benchmarks/tools/hey.go @@ -19,7 +19,6 @@ import ( "net" "regexp" "strconv" - "strings" "testing" ) @@ -32,8 +31,16 @@ type Hey struct { // MakeCmd returns a 'hey' command. func (h *Hey) MakeCmd(ip net.IP, port int) []string { - return strings.Split(fmt.Sprintf("hey -n %d -c %d http://%s:%d/%s", - h.Requests, h.Concurrency, ip, port, h.Doc), " ") + c := h.Concurrency + if c > h.Requests { + c = h.Requests + } + return []string{ + "hey", + "-n", fmt.Sprintf("%d", h.Requests), + "-c", fmt.Sprintf("%d", c), + fmt.Sprintf("http://%s:%d/%s", ip.String(), port, h.Doc), + } } // Report parses output from 'hey' and reports metrics. diff --git a/test/benchmarks/tools/iperf.go b/test/benchmarks/tools/iperf.go index 891d32704..abf296731 100644 --- a/test/benchmarks/tools/iperf.go +++ b/test/benchmarks/tools/iperf.go @@ -19,19 +19,27 @@ import ( "net" "regexp" "strconv" - "strings" "testing" ) +const length = 64 * 1024 + // Iperf is for the client side of `iperf`. type Iperf struct { - Time int + Num int } // MakeCmd returns a iperf client command. func (i *Iperf) MakeCmd(ip net.IP, port int) []string { - // iperf report in Kb realtime - return strings.Split(fmt.Sprintf("iperf -f K --realtime --time %d --client %s --port %d", i.Time, ip, port), " ") + return []string{ + "iperf", + "--format", "K", // Output in KBytes. + "--realtime", // Measured in realtime. + "--num", fmt.Sprintf("%d", i.Num), + "--length", fmt.Sprintf("%d", length), + "--client", ip.String(), + "--port", fmt.Sprintf("%d", port), + } } // Report parses output from iperf client and reports metrics. @@ -42,6 +50,7 @@ func (i *Iperf) Report(b *testing.B, output string) { if err != nil { b.Fatalf("failed to parse bandwitdth from %s: %v", output, err) } + b.SetBytes(length) // Measure Bytes/sec for b.N, although below is iperf output. ReportCustomMetric(b, bW*1024, "bandwidth" /*metric name*/, "bytes_per_second" /*unit*/) } diff --git a/test/benchmarks/tools/redis.go b/test/benchmarks/tools/redis.go index a42e3456e..12fdbc7cc 100644 --- a/test/benchmarks/tools/redis.go +++ b/test/benchmarks/tools/redis.go @@ -19,7 +19,6 @@ import ( "net" "regexp" "strconv" - "strings" "testing" ) @@ -33,13 +32,25 @@ func (r *Redis) MakeCmd(ip net.IP, port, requests int) []string { // There is no -t PING_BULK for redis-benchmark, so adjust the command in that case. // Note that "ping" will run both PING_INLINE and PING_BULK. if r.Operation == "PING_BULK" { - return strings.Split( - fmt.Sprintf("redis-benchmark --csv -t ping -h %s -p %d -n %d", ip, port, requests), " ") + return []string{ + "redis-benchmark", + "--csv", + "-t", "ping", + "-h", ip.String(), + "-p", fmt.Sprintf("%d", port), + "-n", fmt.Sprintf("%d", requests), + } } // runs redis-benchmark -t operation for 100K requests against server. - return strings.Split( - fmt.Sprintf("redis-benchmark --csv -t %s -h %s -p %d -n %d", r.Operation, ip, port, requests), " ") + return []string{ + "redis-benchmark", + "--csv", + "-t", r.Operation, + "-h", ip.String(), + "-p", fmt.Sprintf("%d", port), + "-n", fmt.Sprintf("%d", requests), + } } // Report parses output from redis-benchmark client and reports metrics. diff --git a/test/benchmarks/tools/sysbench.go b/test/benchmarks/tools/sysbench.go index 7ccacd8ff..2b8e6c8aa 100644 --- a/test/benchmarks/tools/sysbench.go +++ b/test/benchmarks/tools/sysbench.go @@ -18,58 +18,46 @@ import ( "fmt" "regexp" "strconv" - "strings" "testing" ) -var warmup = "sysbench --threads=8 --memory-total-size=5G memory run > /dev/null &&" - // Sysbench represents a 'sysbench' command. type Sysbench interface { - MakeCmd() []string // Makes a sysbench command. - flags() []string - Report(*testing.B, string) // Reports results contained in string. + // MakeCmd constructs the relevant command line. + MakeCmd(*testing.B) []string + + // Report reports relevant custom metrics. + Report(*testing.B, string) } // SysbenchBase is the top level struct for sysbench and holds top-level arguments // for sysbench. See: 'sysbench --help' type SysbenchBase struct { - Threads int // number of Threads for the test. - Time int // time limit for test in seconds. + // Threads is the number of threads for the test. + Threads int } // baseFlags returns top level flags. -func (s *SysbenchBase) baseFlags() []string { +func (s *SysbenchBase) baseFlags(b *testing.B) []string { var ret []string if s.Threads > 0 { ret = append(ret, fmt.Sprintf("--threads=%d", s.Threads)) } - if s.Time > 0 { - ret = append(ret, fmt.Sprintf("--time=%d", s.Time)) - } + ret = append(ret, "--time=0") // Ensure events is used. + ret = append(ret, fmt.Sprintf("--events=%d", b.N)) return ret } // SysbenchCPU is for 'sysbench [flags] cpu run' and holds CPU specific arguments. type SysbenchCPU struct { - Base SysbenchBase - MaxPrime int // upper limit for primes generator [10000]. + SysbenchBase } // MakeCmd makes commands for SysbenchCPU. -func (s *SysbenchCPU) MakeCmd() []string { - cmd := []string{warmup, "sysbench"} - cmd = append(cmd, s.flags()...) - cmd = append(cmd, "cpu run") - return []string{"sh", "-c", strings.Join(cmd, " ")} -} - -// flags makes flags for SysbenchCPU cmds. -func (s *SysbenchCPU) flags() []string { - cmd := s.Base.baseFlags() - if s.MaxPrime > 0 { - return append(cmd, fmt.Sprintf("--cpu-max-prime=%d", s.MaxPrime)) - } +func (s *SysbenchCPU) MakeCmd(b *testing.B) []string { + cmd := []string{"sysbench"} + cmd = append(cmd, s.baseFlags(b)...) + cmd = append(cmd, "cpu", "run") return cmd } @@ -96,9 +84,9 @@ func (s *SysbenchCPU) parseEvents(data string) (float64, error) { // SysbenchMemory is for 'sysbench [FLAGS] memory run' and holds Memory specific arguments. type SysbenchMemory struct { - Base SysbenchBase - BlockSize string // size of test memory block [1K]. - TotalSize string // size of data to transfer [100G]. + SysbenchBase + BlockSize int // size of test memory block in megabytes [1]. + TotalSize int // size of data to transfer in gigabytes [100]. Scope string // memory access scope {global, local} [global]. HugeTLB bool // allocate memory from HugeTLB [off]. OperationType string // type of memory ops {read, write, none} [write]. @@ -106,21 +94,21 @@ type SysbenchMemory struct { } // MakeCmd makes commands for SysbenchMemory. -func (s *SysbenchMemory) MakeCmd() []string { - cmd := []string{warmup, "sysbench"} - cmd = append(cmd, s.flags()...) - cmd = append(cmd, "memory run") - return []string{"sh", "-c", strings.Join(cmd, " ")} +func (s *SysbenchMemory) MakeCmd(b *testing.B) []string { + cmd := []string{"sysbench"} + cmd = append(cmd, s.flags(b)...) + cmd = append(cmd, "memory", "run") + return cmd } // flags makes flags for SysbenchMemory cmds. -func (s *SysbenchMemory) flags() []string { - cmd := s.Base.baseFlags() - if s.BlockSize != "" { - cmd = append(cmd, fmt.Sprintf("--memory-block-size=%s", s.BlockSize)) +func (s *SysbenchMemory) flags(b *testing.B) []string { + cmd := s.baseFlags(b) + if s.BlockSize != 0 { + cmd = append(cmd, fmt.Sprintf("--memory-block-size=%dM", s.BlockSize)) } - if s.TotalSize != "" { - cmd = append(cmd, fmt.Sprintf("--memory-total-size=%s", s.TotalSize)) + if s.TotalSize != 0 { + cmd = append(cmd, fmt.Sprintf("--memory-total-size=%dG", s.TotalSize)) } if s.Scope != "" { cmd = append(cmd, fmt.Sprintf("--memory-scope=%s", s.Scope)) @@ -147,7 +135,7 @@ func (s *SysbenchMemory) Report(b *testing.B, output string) { ReportCustomMetric(b, result, "memory_operations" /*metric name*/, "ops_per_second" /*unit*/) } -var memoryOperationsRE = regexp.MustCompile(`Total\soperations:\s+\d*\s*\((\d*\.\d*)\sper\ssecond\)`) +var memoryOperationsRE = regexp.MustCompile(`Total\s+operations:\s+\d+\s+\((\s*\d+\.\d+\s*)\s+per\s+second\)`) // parseOperations parses memory operations per second form sysbench memory ouput. func (s *SysbenchMemory) parseOperations(data string) (float64, error) { @@ -160,24 +148,24 @@ func (s *SysbenchMemory) parseOperations(data string) (float64, error) { // SysbenchMutex is for 'sysbench [FLAGS] mutex run' and holds Mutex specific arguments. type SysbenchMutex struct { - Base SysbenchBase + SysbenchBase Num int // total size of mutex array [4096]. - Locks int // number of mutex locks per thread [50K]. - Loops int // number of loops to do outside mutex lock [10K]. + Locks int // number of mutex locks per thread [50000]. + Loops int // number of loops to do outside mutex lock [10000]. } // MakeCmd makes commands for SysbenchMutex. -func (s *SysbenchMutex) MakeCmd() []string { - cmd := []string{warmup, "sysbench"} - cmd = append(cmd, s.flags()...) - cmd = append(cmd, "mutex run") - return []string{"sh", "-c", strings.Join(cmd, " ")} +func (s *SysbenchMutex) MakeCmd(b *testing.B) []string { + cmd := []string{"sysbench"} + cmd = append(cmd, s.flags(b)...) + cmd = append(cmd, "mutex", "run") + return cmd } // flags makes flags for SysbenchMutex commands. -func (s *SysbenchMutex) flags() []string { +func (s *SysbenchMutex) flags(b *testing.B) []string { var cmd []string - cmd = append(cmd, s.Base.baseFlags()...) + cmd = append(cmd, s.baseFlags(b)...) if s.Num > 0 { cmd = append(cmd, fmt.Sprintf("--mutex-num=%d", s.Num)) } |