From b4af9d4572707718514e66b7e537bc51216b60f6 Mon Sep 17 00:00:00 2001 From: Zach Koopmans Date: Wed, 9 Dec 2020 13:49:41 -0800 Subject: Add network benchmarks jobs Add httpd, nginx, node, and ruby benchmarks to continuous jobs. PiperOrigin-RevId: 346629115 --- test/benchmarks/network/BUILD | 66 ++++++++++++++++++++++-- test/benchmarks/network/httpd_test.go | 63 ++++++++++++++++++++++- test/benchmarks/network/iperf_test.go | 27 ++++++---- test/benchmarks/network/network.go | 69 ++++++++++++++++++++++--- test/benchmarks/network/nginx_test.go | 68 ++++++++++++++++++++++++- test/benchmarks/network/node_test.go | 9 +++- test/benchmarks/network/ruby_test.go | 9 +++- test/benchmarks/network/static_server.go | 87 -------------------------------- test/benchmarks/tools/iperf.go | 2 +- 9 files changed, 286 insertions(+), 114 deletions(-) delete mode 100644 test/benchmarks/network/static_server.go (limited to 'test/benchmarks') diff --git a/test/benchmarks/network/BUILD b/test/benchmarks/network/BUILD index c75d1ce11..2741570f5 100644 --- a/test/benchmarks/network/BUILD +++ b/test/benchmarks/network/BUILD @@ -8,7 +8,6 @@ go_library( testonly = 1, srcs = [ "network.go", - "static_server.go", ], deps = [ "//pkg/test/dockerutil", @@ -18,17 +17,74 @@ go_library( ) benchmark_test( - name = "network_test", - size = "large", + name = "iperf_test", + size = "enormous", srcs = [ - "httpd_test.go", "iperf_test.go", - "nginx_test.go", + ], + library = ":network", + visibility = ["//:sandbox"], + deps = [ + "//pkg/test/dockerutil", + "//pkg/test/testutil", + "//test/benchmarks/harness", + "//test/benchmarks/tools", + ], +) + +benchmark_test( + name = "node_test", + size = "enormous", + srcs = [ "node_test.go", + ], + library = ":network", + visibility = ["//:sandbox"], + deps = [ + "//pkg/test/dockerutil", + "//test/benchmarks/harness", + "//test/benchmarks/tools", + ], +) + +benchmark_test( + name = "ruby_test", + size = "enormous", + srcs = [ "ruby_test.go", ], library = ":network", visibility = ["//:sandbox"], + deps = [ + "//pkg/test/dockerutil", + "//test/benchmarks/harness", + "//test/benchmarks/tools", + ], +) + +benchmark_test( + name = "nginx_test", + size = "enormous", + srcs = [ + "nginx_test.go", + ], + library = ":network", + visibility = ["//:sandbox"], + deps = [ + "//pkg/test/dockerutil", + "//test/benchmarks/harness", + "//test/benchmarks/tools", + ], +) + +benchmark_test( + name = "httpd_test", + size = "enormous", + srcs = [ + "httpd_test.go", + ], + library = ":network", + visibility = ["//:sandbox"], deps = [ "//pkg/test/dockerutil", "//pkg/test/testutil", diff --git a/test/benchmarks/network/httpd_test.go b/test/benchmarks/network/httpd_test.go index 8d7d5f750..aa300dabd 100644 --- a/test/benchmarks/network/httpd_test.go +++ b/test/benchmarks/network/httpd_test.go @@ -14,13 +14,17 @@ package network import ( + "os" "strconv" "testing" "gvisor.dev/gvisor/pkg/test/dockerutil" + "gvisor.dev/gvisor/test/benchmarks/harness" "gvisor.dev/gvisor/test/benchmarks/tools" ) +var h harness.Harness + // see Dockerfile '//images/benchmarks/httpd'. var httpdDocs = map[string]string{ "notfound": "notfound", @@ -43,6 +47,22 @@ func BenchmarkReverseHttpd(b *testing.B) { benchmarkHttpdDocSize(b, true /* reverse */) } +// BenchmarkContinuousHttpd runs specific benchmarks for continous jobs. +// The runtime under test is the server serving a runc client. +func BenchmarkContinuousHttpd(b *testing.B) { + sizes := []string{"10Kb", "100Kb", "1Mb"} + threads := []int{1, 25, 100, 1000} + benchmarkHttpdContinuous(b, threads, sizes, false /*reverse*/) +} + +// BenchmarkContinuousHttpdReverse runs specific benchmarks for continous jobs. +// The runtime under test is the client downloading from a runc server. +func BenchmarkContinuousHttpdReverse(b *testing.B) { + sizes := []string{"10Kb", "100Kb", "1Mb"} + threads := []int{1, 25, 100, 1000} + benchmarkHttpdContinuous(b, threads, sizes, true /*reverse*/) +} + // benchmarkHttpdDocSize iterates through all doc sizes, running subbenchmarks // for each size. func benchmarkHttpdDocSize(b *testing.B, reverse bool) { @@ -74,6 +94,42 @@ func benchmarkHttpdDocSize(b *testing.B, reverse bool) { } } +// benchmarkHttpdContinuous iterates through given sizes and concurrencies. +func benchmarkHttpdContinuous(b *testing.B, concurrency []int, sizes []string, reverse bool) { + for _, size := range sizes { + filename := httpdDocs[size] + for _, c := range concurrency { + fsize := tools.Parameter{ + Name: "filesize", + Value: size, + } + + threads := tools.Parameter{ + Name: "concurrency", + Value: strconv.Itoa(c), + } + + name, err := tools.ParametersToName(fsize, threads) + if err != nil { + b.Fatalf("Failed to parse parameters: %v", err) + } + + requests := b.N + if requests < c { + requests = c + } + b.Run(name, func(b *testing.B) { + hey := &tools.Hey{ + Requests: requests, + Concurrency: c, + Doc: filename, + } + runHttpd(b, hey, reverse) + }) + } + } +} + // runHttpd configures the static serving methods to run httpd. func runHttpd(b *testing.B, hey *tools.Hey, reverse bool) { // httpd runs on port 80. @@ -91,5 +147,10 @@ func runHttpd(b *testing.B, hey *tools.Hey, reverse bool) { }, } httpdCmd := []string{"sh", "-c", "mkdir -p /tmp/html; cp -r /local/* /tmp/html/.; apache2 -X"} - runStaticServer(b, httpdRunOpts, httpdCmd, port, hey, reverse) + runStaticServer(b, h, httpdRunOpts, httpdCmd, port, hey, reverse) +} + +func TestMain(m *testing.M) { + h.Init() + os.Exit(m.Run()) } diff --git a/test/benchmarks/network/iperf_test.go b/test/benchmarks/network/iperf_test.go index b8ab7dfb8..9d64db943 100644 --- a/test/benchmarks/network/iperf_test.go +++ b/test/benchmarks/network/iperf_test.go @@ -15,6 +15,7 @@ package network import ( "context" + "os" "testing" "gvisor.dev/gvisor/pkg/test/dockerutil" @@ -23,9 +24,11 @@ import ( "gvisor.dev/gvisor/test/benchmarks/tools" ) +var h harness.Harness + func BenchmarkIperf(b *testing.B) { iperf := tools.Iperf{ - Time: 10, // time in seconds to run client. + Time: b.N, // time in seconds to run client. } clientMachine, err := h.GetMachine() @@ -97,17 +100,19 @@ func BenchmarkIperf(b *testing.B) { // Restart the server profiles. If the server isn't being profiled // this does nothing. server.RestartProfiles() - for i := 0; i < b.N; i++ { - out, err := client.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/iperf", - }, iperf.MakeCmd(ip, servingPort)...) - if err != nil { - b.Fatalf("failed to run client: %v", err) - } - b.StopTimer() - iperf.Report(b, out) - b.StartTimer() + out, err := client.Run(ctx, dockerutil.RunOpts{ + Image: "benchmarks/iperf", + }, iperf.MakeCmd(ip, servingPort)...) + if err != nil { + b.Fatalf("failed to run client: %v", err) } + b.StopTimer() + iperf.Report(b, out) }) } } + +func TestMain(m *testing.M) { + h.Init() + os.Exit(m.Run()) +} diff --git a/test/benchmarks/network/network.go b/test/benchmarks/network/network.go index ce17ddb94..b18bc2b3c 100644 --- a/test/benchmarks/network/network.go +++ b/test/benchmarks/network/network.go @@ -16,16 +16,73 @@ package network import ( - "os" + "context" "testing" + "gvisor.dev/gvisor/pkg/test/dockerutil" "gvisor.dev/gvisor/test/benchmarks/harness" + "gvisor.dev/gvisor/test/benchmarks/tools" ) -var h harness.Harness +// runStaticServer runs static serving workloads (httpd, nginx). +func runStaticServer(b *testing.B, h harness.Harness, serverOpts dockerutil.RunOpts, serverCmd []string, port int, hey *tools.Hey, reverse bool) { + ctx := context.Background() -// TestMain is the main method for package network. -func TestMain(m *testing.M) { - h.Init() - os.Exit(m.Run()) + // Get two machines: a client and server. + clientMachine, err := h.GetMachine() + if err != nil { + b.Fatalf("failed to get machine: %v", err) + } + defer clientMachine.CleanUp() + + serverMachine, err := h.GetMachine() + if err != nil { + b.Fatalf("failed to get machine: %v", err) + } + defer serverMachine.CleanUp() + + // Make the containers. 'reverse=true' specifies that the client should use the + // runtime under test. + var client, server *dockerutil.Container + if reverse { + client = clientMachine.GetContainer(ctx, b) + server = serverMachine.GetNativeContainer(ctx, b) + } else { + client = clientMachine.GetNativeContainer(ctx, b) + server = serverMachine.GetContainer(ctx, b) + } + defer client.CleanUp(ctx) + defer server.CleanUp(ctx) + + // Start the server. + if err := server.Spawn(ctx, serverOpts, serverCmd...); err != nil { + b.Fatalf("failed to start server: %v", err) + } + + // Get its IP. + ip, err := serverMachine.IPAddress() + if err != nil { + b.Fatalf("failed to find server ip: %v", err) + } + + // Get the published port. + servingPort, err := server.FindPort(ctx, port) + if err != nil { + b.Fatalf("failed to find server port %d: %v", port, err) + } + + // Make sure the server is serving. + harness.WaitUntilServing(ctx, clientMachine, ip, servingPort) + b.ResetTimer() + server.RestartProfiles() + out, err := client.Run(ctx, dockerutil.RunOpts{ + Image: "benchmarks/hey", + }, hey.MakeCmd(ip, servingPort)...) + if err != nil { + b.Fatalf("run failed with: %v", err) + } + + b.StopTimer() + hey.Report(b, out) + b.StartTimer() } diff --git a/test/benchmarks/network/nginx_test.go b/test/benchmarks/network/nginx_test.go index 08565d0b2..6e7a4dec4 100644 --- a/test/benchmarks/network/nginx_test.go +++ b/test/benchmarks/network/nginx_test.go @@ -14,13 +14,17 @@ package network import ( + "os" "strconv" "testing" "gvisor.dev/gvisor/pkg/test/dockerutil" + "gvisor.dev/gvisor/test/benchmarks/harness" "gvisor.dev/gvisor/test/benchmarks/tools" ) +var h harness.Harness + // see Dockerfile '//images/benchmarks/nginx'. var nginxDocs = map[string]string{ "notfound": "notfound", @@ -44,6 +48,22 @@ func BenchmarkReverseNginxDocSize(b *testing.B) { benchmarkNginxDocSize(b, true /* reverse */, true /* tmpfs */) } +// BenchmarkContinuousNginx runs specific benchmarks for continous jobs. +// The runtime under test is the sever serving a runc client. +func BenchmarkContinuousNginx(b *testing.B) { + sizes := []string{"10Kb", "100Kb", "1Mb"} + threads := []int{1, 25, 100, 1000} + benchmarkNginxContinuous(b, threads, sizes, false /*reverse*/) +} + +// BenchmarkContinuousNginxReverse runs specific benchmarks for continous jobs. +// The runtime under test is the client downloading from a runc server. +func BenchmarkContinuousNginxReverse(b *testing.B) { + sizes := []string{"10Kb", "100Kb", "1Mb"} + threads := []int{1, 25, 100, 1000} + benchmarkNginxContinuous(b, threads, sizes, true /*reverse*/) +} + // benchmarkNginxDocSize iterates through all doc sizes, running subbenchmarks // for each size. func benchmarkNginxDocSize(b *testing.B, reverse, tmpfs bool) { @@ -84,6 +104,47 @@ func benchmarkNginxDocSize(b *testing.B, reverse, tmpfs bool) { } } +// benchmarkNginxContinuous iterates through given sizes and concurrencies on a tmpfs mount. +func benchmarkNginxContinuous(b *testing.B, concurrency []int, sizes []string, reverse bool) { + for _, size := range sizes { + filename := nginxDocs[size] + for _, c := range concurrency { + fsize := tools.Parameter{ + Name: "filesize", + Value: size, + } + + threads := tools.Parameter{ + Name: "concurrency", + Value: strconv.Itoa(c), + } + + fs := tools.Parameter{ + Name: "filesystem", + Value: "tmpfs", + } + + name, err := tools.ParametersToName(fsize, threads, fs) + if err != nil { + b.Fatalf("Failed to parse parameters: %v", err) + } + + requests := b.N + if requests < c { + requests = c + } + b.Run(name, func(b *testing.B) { + hey := &tools.Hey{ + Requests: requests, + Concurrency: c, + Doc: filename, + } + runNginx(b, hey, reverse, true /*tmpfs*/) + }) + } + } +} + // runNginx configures the static serving methods to run httpd. func runNginx(b *testing.B, hey *tools.Hey, reverse, tmpfs bool) { // nginx runs on port 80. @@ -99,5 +160,10 @@ func runNginx(b *testing.B, hey *tools.Hey, reverse, tmpfs bool) { } // Command copies nginxDocs to tmpfs serving directory and runs nginx. - runStaticServer(b, nginxRunOpts, nginxCmd, port, hey, reverse) + runStaticServer(b, h, nginxRunOpts, nginxCmd, port, hey, reverse) +} + +func TestMain(m *testing.M) { + h.Init() + os.Exit(m.Run()) } diff --git a/test/benchmarks/network/node_test.go b/test/benchmarks/network/node_test.go index 254538899..246720d43 100644 --- a/test/benchmarks/network/node_test.go +++ b/test/benchmarks/network/node_test.go @@ -15,6 +15,7 @@ package network import ( "context" + "os" "strconv" "testing" "time" @@ -24,6 +25,8 @@ import ( "gvisor.dev/gvisor/test/benchmarks/tools" ) +var h harness.Harness + // BenchmarkNode runs requests using 'hey' against a Node server run on // 'runtime'. The server responds to requests by grabbing some data in a // redis instance and returns the data in its reponse. The test loops through @@ -131,5 +134,9 @@ func runNode(b *testing.B, hey *tools.Hey) { // Stop the timer to parse the data and report stats. b.StopTimer() hey.Report(b, out) - b.StartTimer() +} + +func TestMain(m *testing.M) { + h.Init() + os.Exit(m.Run()) } diff --git a/test/benchmarks/network/ruby_test.go b/test/benchmarks/network/ruby_test.go index 0174ff3f3..6f5199480 100644 --- a/test/benchmarks/network/ruby_test.go +++ b/test/benchmarks/network/ruby_test.go @@ -16,6 +16,7 @@ package network import ( "context" "fmt" + "os" "strconv" "testing" "time" @@ -25,6 +26,8 @@ import ( "gvisor.dev/gvisor/test/benchmarks/tools" ) +var h harness.Harness + // BenchmarkRuby runs requests using 'hey' against a ruby application server. // On start, ruby app generates some random data and pushes it to a redis // instance. On a request, the app grabs for random entries from the redis @@ -52,7 +55,6 @@ func BenchmarkRuby(b *testing.B) { // runRuby runs the test for a given # of requests and concurrency. func runRuby(b *testing.B, hey *tools.Hey) { - b.Helper() // The machine to hold Redis and the Ruby Server. serverMachine, err := h.GetMachine() if err != nil { @@ -141,3 +143,8 @@ func runRuby(b *testing.B, hey *tools.Hey) { hey.Report(b, out) b.StartTimer() } + +func TestMain(m *testing.M) { + h.Init() + os.Exit(m.Run()) +} diff --git a/test/benchmarks/network/static_server.go b/test/benchmarks/network/static_server.go deleted file mode 100644 index e747a1395..000000000 --- a/test/benchmarks/network/static_server.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package network - -import ( - "context" - "testing" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/test/benchmarks/harness" - "gvisor.dev/gvisor/test/benchmarks/tools" -) - -// runStaticServer runs static serving workloads (httpd, nginx). -func runStaticServer(b *testing.B, serverOpts dockerutil.RunOpts, serverCmd []string, port int, hey *tools.Hey, reverse bool) { - ctx := context.Background() - - // Get two machines: a client and server. - clientMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer clientMachine.CleanUp() - - serverMachine, err := h.GetMachine() - if err != nil { - b.Fatalf("failed to get machine: %v", err) - } - defer serverMachine.CleanUp() - - // Make the containers. 'reverse=true' specifies that the client should use the - // runtime under test. - var client, server *dockerutil.Container - if reverse { - client = clientMachine.GetContainer(ctx, b) - server = serverMachine.GetNativeContainer(ctx, b) - } else { - client = clientMachine.GetNativeContainer(ctx, b) - server = serverMachine.GetContainer(ctx, b) - } - defer client.CleanUp(ctx) - defer server.CleanUp(ctx) - - // Start the server. - if err := server.Spawn(ctx, serverOpts, serverCmd...); err != nil { - b.Fatalf("failed to start server: %v", err) - } - - // Get its IP. - ip, err := serverMachine.IPAddress() - if err != nil { - b.Fatalf("failed to find server ip: %v", err) - } - - // Get the published port. - servingPort, err := server.FindPort(ctx, port) - if err != nil { - b.Fatalf("failed to find server port %d: %v", port, err) - } - - // Make sure the server is serving. - harness.WaitUntilServing(ctx, clientMachine, ip, servingPort) - b.ResetTimer() - server.RestartProfiles() - out, err := client.Run(ctx, dockerutil.RunOpts{ - Image: "benchmarks/hey", - }, hey.MakeCmd(ip, servingPort)...) - if err != nil { - b.Fatalf("run failed with: %v", err) - } - - b.StopTimer() - hey.Report(b, out) - b.StartTimer() -} diff --git a/test/benchmarks/tools/iperf.go b/test/benchmarks/tools/iperf.go index 5c4e7125b..891d32704 100644 --- a/test/benchmarks/tools/iperf.go +++ b/test/benchmarks/tools/iperf.go @@ -31,7 +31,7 @@ type Iperf struct { // MakeCmd returns a iperf client command. func (i *Iperf) MakeCmd(ip net.IP, port int) []string { // iperf report in Kb realtime - return strings.Split(fmt.Sprintf("iperf -f K --realtime --time %d -c %s -p %d", i.Time, ip, port), " ") + return strings.Split(fmt.Sprintf("iperf -f K --realtime --time %d --client %s --port %d", i.Time, ip, port), " ") } // Report parses output from iperf client and reports metrics. -- cgit v1.2.3