summaryrefslogtreecommitdiffhomepage
path: root/test/benchmarks/network
diff options
context:
space:
mode:
authorZach Koopmans <zkoopmans@google.com>2020-07-30 21:15:34 -0700
committergVisor bot <gvisor-bot@google.com>2020-07-30 21:17:45 -0700
commit98f9527c04dfc4af242080b5ea29e6da09290098 (patch)
treea4fac06097579a7ce7ecf9a5c4a1756fbfcdf06d /test/benchmarks/network
parent6a4bcbdb288842633528b10b17950d97d756b2e0 (diff)
Port nginx and move parsers to own package.
This change: - Ports the nginx benchmark. - Switches the Httpd benchmark to use 'hey' as a client. - Moves all parsers to their own package 'tools'. Parsers are moved to their own package because 1) parsing output of a command is often dependent on the format of the command (e.g. 'fio --json'), 2) to enable easier reuse, and 3) clean up and simplify actual running benchmarks (no TestParser functions and ugly sample output in benchmark files). PiperOrigin-RevId: 324144165
Diffstat (limited to 'test/benchmarks/network')
-rw-r--r--test/benchmarks/network/BUILD2
-rw-r--r--test/benchmarks/network/httpd_test.go166
-rw-r--r--test/benchmarks/network/iperf_test.go49
-rw-r--r--test/benchmarks/network/nginx_test.go104
-rw-r--r--test/benchmarks/network/node_test.go148
5 files changed, 143 insertions, 326 deletions
diff --git a/test/benchmarks/network/BUILD b/test/benchmarks/network/BUILD
index b47400590..d15cd55ee 100644
--- a/test/benchmarks/network/BUILD
+++ b/test/benchmarks/network/BUILD
@@ -15,6 +15,7 @@ go_test(
srcs = [
"httpd_test.go",
"iperf_test.go",
+ "nginx_test.go",
"node_test.go",
],
library = ":network",
@@ -27,5 +28,6 @@ go_test(
"//pkg/test/dockerutil",
"//pkg/test/testutil",
"//test/benchmarks/harness",
+ "//test/benchmarks/tools",
],
)
diff --git a/test/benchmarks/network/httpd_test.go b/test/benchmarks/network/httpd_test.go
index fe23ca949..07833f9cd 100644
--- a/test/benchmarks/network/httpd_test.go
+++ b/test/benchmarks/network/httpd_test.go
@@ -16,12 +16,11 @@ package network
import (
"context"
"fmt"
- "regexp"
- "strconv"
"testing"
"gvisor.dev/gvisor/pkg/test/dockerutil"
"gvisor.dev/gvisor/test/benchmarks/harness"
+ "gvisor.dev/gvisor/test/benchmarks/tools"
)
// see Dockerfile '//images/benchmarks/httpd'.
@@ -52,13 +51,16 @@ func BenchmarkHttpdConcurrency(b *testing.B) {
defer serverMachine.CleanUp()
// The test iterates over client concurrency, so set other parameters.
- requests := 10000
concurrency := []int{1, 5, 10, 25}
- doc := docs["10Kb"]
for _, c := range concurrency {
b.Run(fmt.Sprintf("%d", c), func(b *testing.B) {
- runHttpd(b, clientMachine, serverMachine, doc, requests, c)
+ hey := &tools.Hey{
+ Requests: 10000,
+ Concurrency: c,
+ Doc: docs["10Kb"],
+ }
+ runHttpd(b, clientMachine, serverMachine, hey)
})
}
}
@@ -78,18 +80,20 @@ func BenchmarkHttpdDocSize(b *testing.B) {
}
defer serverMachine.CleanUp()
- requests := 10000
- concurrency := 1
-
for name, filename := range docs {
b.Run(name, func(b *testing.B) {
- runHttpd(b, clientMachine, serverMachine, filename, requests, concurrency)
+ hey := &tools.Hey{
+ Requests: 10000,
+ Concurrency: 1,
+ Doc: filename,
+ }
+ runHttpd(b, clientMachine, serverMachine, hey)
})
}
}
// runHttpd runs a single test run.
-func runHttpd(b *testing.B, clientMachine, serverMachine harness.Machine, doc string, requests, concurrency int) {
+func runHttpd(b *testing.B, clientMachine, serverMachine harness.Machine, hey *tools.Hey) {
b.Helper()
// Grab a container from the server.
@@ -98,11 +102,11 @@ func runHttpd(b *testing.B, clientMachine, serverMachine harness.Machine, doc st
defer server.CleanUp(ctx)
// Copy the docs to /tmp and serve from there.
- cmd := "mkdir -p /tmp/html; cp -r /local /tmp/html/.; apache2 -X"
+ cmd := "mkdir -p /tmp/html; cp -r /local/* /tmp/html/.; apache2 -X"
port := 80
// Start the server.
- server.Spawn(ctx, dockerutil.RunOpts{
+ if err := server.Spawn(ctx, dockerutil.RunOpts{
Image: "benchmarks/httpd",
Ports: []int{port},
Env: []string{
@@ -113,7 +117,9 @@ func runHttpd(b *testing.B, clientMachine, serverMachine harness.Machine, doc st
"APACHE_LOG_DIR=/tmp",
"APACHE_PID_FILE=/tmp/apache.pid",
},
- }, "sh", "-c", cmd)
+ }, "sh", "-c", cmd); err != nil {
+ b.Fatalf("failed to start server: %v")
+ }
ip, err := serverMachine.IPAddress()
if err != nil {
@@ -132,146 +138,18 @@ func runHttpd(b *testing.B, clientMachine, serverMachine harness.Machine, doc st
client := clientMachine.GetNativeContainer(ctx, b)
defer client.CleanUp(ctx)
- path := fmt.Sprintf("http://%s:%d/%s", ip, servingPort, doc)
- // See apachebench (ab) for flags.
- cmd = fmt.Sprintf("ab -n %d -c %d %s", requests, concurrency, path)
-
b.ResetTimer()
server.RestartProfiles()
for i := 0; i < b.N; i++ {
out, err := client.Run(ctx, dockerutil.RunOpts{
- Image: "benchmarks/ab",
- }, "sh", "-c", cmd)
+ Image: "benchmarks/hey",
+ }, hey.MakeCmd(ip, servingPort)...)
if err != nil {
b.Fatalf("run failed with: %v", err)
}
b.StopTimer()
-
- // Parse and report custom metrics.
- transferRate, err := parseTransferRate(out)
- if err != nil {
- b.Logf("failed to parse transferrate: %v", err)
- }
- b.ReportMetric(transferRate*1024, "transfer_rate") // Convert from Kb/s to b/s.
-
- latency, err := parseLatency(out)
- if err != nil {
- b.Logf("failed to parse latency: %v", err)
- }
- b.ReportMetric(latency/1000, "mean_latency") // Convert from ms to s.
-
- reqPerSecond, err := parseRequestsPerSecond(out)
- if err != nil {
- b.Logf("failed to parse requests per second: %v", err)
- }
- b.ReportMetric(reqPerSecond, "requests_per_second")
-
+ hey.Report(b, out)
b.StartTimer()
}
}
-
-var transferRateRE = regexp.MustCompile(`Transfer rate:\s+(\d+\.?\d+?)\s+\[Kbytes/sec\]\s+received`)
-
-// parseTransferRate parses transfer rate from apachebench output.
-func parseTransferRate(data string) (float64, error) {
- match := transferRateRE.FindStringSubmatch(data)
- if len(match) < 2 {
- return 0, fmt.Errorf("failed get bandwidth: %s", data)
- }
- return strconv.ParseFloat(match[1], 64)
-}
-
-var latencyRE = regexp.MustCompile(`Total:\s+\d+\s+(\d+)\s+(\d+\.?\d+?)\s+\d+\s+\d+\s`)
-
-// parseLatency parses latency from apachebench output.
-func parseLatency(data string) (float64, error) {
- match := latencyRE.FindStringSubmatch(data)
- if len(match) < 2 {
- return 0, fmt.Errorf("failed get bandwidth: %s", data)
- }
- return strconv.ParseFloat(match[1], 64)
-}
-
-var requestsPerSecondRE = regexp.MustCompile(`Requests per second:\s+(\d+\.?\d+?)\s+`)
-
-// parseRequestsPerSecond parses requests per second from apachebench output.
-func parseRequestsPerSecond(data string) (float64, error) {
- match := requestsPerSecondRE.FindStringSubmatch(data)
- if len(match) < 2 {
- return 0, fmt.Errorf("failed get bandwidth: %s", data)
- }
- return strconv.ParseFloat(match[1], 64)
-}
-
-// Sample output from apachebench.
-const sampleData = `This is ApacheBench, Version 2.3 <$Revision: 1826891 $>
-Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
-Licensed to The Apache Software Foundation, http://www.apache.org/
-
-Benchmarking 10.10.10.10 (be patient).....done
-
-
-Server Software: Apache/2.4.38
-Server Hostname: 10.10.10.10
-Server Port: 80
-
-Document Path: /latin10k.txt
-Document Length: 210 bytes
-
-Concurrency Level: 1
-Time taken for tests: 0.180 seconds
-Complete requests: 100
-Failed requests: 0
-Non-2xx responses: 100
-Total transferred: 38800 bytes
-HTML transferred: 21000 bytes
-Requests per second: 556.44 [#/sec] (mean)
-Time per request: 1.797 [ms] (mean)
-Time per request: 1.797 [ms] (mean, across all concurrent requests)
-Transfer rate: 210.84 [Kbytes/sec] received
-
-Connection Times (ms)
- min mean[+/-sd] median max
-Connect: 0 0 0.2 0 2
-Processing: 1 2 1.0 1 8
-Waiting: 1 1 1.0 1 7
-Total: 1 2 1.2 1 10
-
-Percentage of the requests served within a certain time (ms)
- 50% 1
- 66% 2
- 75% 2
- 80% 2
- 90% 2
- 95% 3
- 98% 7
- 99% 10
- 100% 10 (longest request)`
-
-// TestParsers checks the parsers work.
-func TestParsers(t *testing.T) {
- want := 210.84
- got, err := parseTransferRate(sampleData)
- if err != nil {
- t.Fatalf("failed to parse transfer rate with error: %v", err)
- } else if got != want {
- t.Fatalf("parseTransferRate got: %f, want: %f", got, want)
- }
-
- want = 2.0
- got, err = parseLatency(sampleData)
- if err != nil {
- t.Fatalf("failed to parse transfer rate with error: %v", err)
- } else if got != want {
- t.Fatalf("parseLatency got: %f, want: %f", got, want)
- }
-
- want = 556.44
- got, err = parseRequestsPerSecond(sampleData)
- if err != nil {
- t.Fatalf("failed to parse transfer rate with error: %v", err)
- } else if got != want {
- t.Fatalf("parseRequestsPerSecond got: %f, want: %f", got, want)
- }
-}
diff --git a/test/benchmarks/network/iperf_test.go b/test/benchmarks/network/iperf_test.go
index a5e198e14..b8ab7dfb8 100644
--- a/test/benchmarks/network/iperf_test.go
+++ b/test/benchmarks/network/iperf_test.go
@@ -15,19 +15,18 @@ package network
import (
"context"
- "fmt"
- "regexp"
- "strconv"
- "strings"
"testing"
"gvisor.dev/gvisor/pkg/test/dockerutil"
"gvisor.dev/gvisor/pkg/test/testutil"
"gvisor.dev/gvisor/test/benchmarks/harness"
+ "gvisor.dev/gvisor/test/benchmarks/tools"
)
func BenchmarkIperf(b *testing.B) {
- const time = 10 // time in seconds to run the client.
+ iperf := tools.Iperf{
+ Time: 10, // time in seconds to run client.
+ }
clientMachine, err := h.GetMachine()
if err != nil {
@@ -92,10 +91,6 @@ func BenchmarkIperf(b *testing.B) {
if err := harness.WaitUntilServing(ctx, clientMachine, ip, servingPort); err != nil {
b.Fatalf("failed to wait for server: %v", err)
}
-
- // iperf report in Kb realtime
- cmd := fmt.Sprintf("iperf -f K --realtime --time %d -c %s -p %d", time, ip.String(), servingPort)
-
// Run the client.
b.ResetTimer()
@@ -105,46 +100,14 @@ func BenchmarkIperf(b *testing.B) {
for i := 0; i < b.N; i++ {
out, err := client.Run(ctx, dockerutil.RunOpts{
Image: "benchmarks/iperf",
- }, strings.Split(cmd, " ")...)
+ }, iperf.MakeCmd(ip, servingPort)...)
if err != nil {
b.Fatalf("failed to run client: %v", err)
}
b.StopTimer()
-
- // Parse bandwidth and report it.
- bW, err := bandwidth(out)
- if err != nil {
- b.Fatalf("failed to parse bandwitdth from %s: %v", out, err)
- }
- b.ReportMetric(bW*1024, "bandwidth") // Convert from Kb/s to b/s.
+ iperf.Report(b, out)
b.StartTimer()
}
})
}
}
-
-// bandwidth parses the Bandwidth number from an iperf report. A sample is below.
-func bandwidth(data string) (float64, error) {
- re := regexp.MustCompile(`\[\s*\d+\][^\n]+\s+(\d+\.?\d*)\s+KBytes/sec`)
- match := re.FindStringSubmatch(data)
- if len(match) < 1 {
- return 0, fmt.Errorf("failed get bandwidth: %s", data)
- }
- return strconv.ParseFloat(match[1], 64)
-}
-
-func TestParser(t *testing.T) {
- sampleData := `
-------------------------------------------------------------
-Client connecting to 10.138.15.215, TCP port 32779
-TCP window size: 45.0 KByte (default)
-------------------------------------------------------------
-[ 3] local 10.138.15.216 port 32866 connected with 10.138.15.215 port 32779
-[ ID] Interval Transfer Bandwidth
-[ 3] 0.0-10.0 sec 459520 KBytes 45900 KBytes/sec
-`
- bandwidth, err := bandwidth(sampleData)
- if err != nil || bandwidth != 45900 {
- t.Fatalf("failed with: %v and %f", err, bandwidth)
- }
-}
diff --git a/test/benchmarks/network/nginx_test.go b/test/benchmarks/network/nginx_test.go
new file mode 100644
index 000000000..5965652a5
--- /dev/null
+++ b/test/benchmarks/network/nginx_test.go
@@ -0,0 +1,104 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package network
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "gvisor.dev/gvisor/pkg/test/dockerutil"
+ "gvisor.dev/gvisor/test/benchmarks/harness"
+ "gvisor.dev/gvisor/test/benchmarks/tools"
+)
+
+// BenchmarkNginxConcurrency iterates the concurrency argument and tests
+// how well the runtime under test handles requests in parallel.
+// TODO(zkoopmans): Update with different doc sizes like Httpd.
+func BenchmarkNginxConcurrency(b *testing.B) {
+ // Grab a machine for the client and server.
+ clientMachine, err := h.GetMachine()
+ if err != nil {
+ b.Fatalf("failed to get client: %v", err)
+ }
+ defer clientMachine.CleanUp()
+
+ serverMachine, err := h.GetMachine()
+ if err != nil {
+ b.Fatalf("failed to get server: %v", err)
+ }
+ defer serverMachine.CleanUp()
+
+ concurrency := []int{1, 5, 10, 25}
+ for _, c := range concurrency {
+ b.Run(fmt.Sprintf("%d", c), func(b *testing.B) {
+ hey := &tools.Hey{
+ Requests: 10000,
+ Concurrency: c,
+ }
+ runNginx(b, clientMachine, serverMachine, hey)
+ })
+ }
+}
+
+// runHttpd runs a single test run.
+func runNginx(b *testing.B, clientMachine, serverMachine harness.Machine, hey *tools.Hey) {
+ b.Helper()
+
+ // Grab a container from the server.
+ ctx := context.Background()
+ server := serverMachine.GetContainer(ctx, b)
+ defer server.CleanUp(ctx)
+
+ port := 80
+ // Start the server.
+ if err := server.Spawn(ctx,
+ dockerutil.RunOpts{
+ Image: "benchmarks/nginx",
+ Ports: []int{port},
+ }); err != nil {
+ b.Fatalf("server failed to start: %v", err)
+ }
+
+ ip, err := serverMachine.IPAddress()
+ if err != nil {
+ b.Fatalf("failed to find server ip: %v", err)
+ }
+
+ servingPort, err := server.FindPort(ctx, port)
+ if err != nil {
+ b.Fatalf("failed to find server port %d: %v", port, err)
+ }
+
+ // Check the server is serving.
+ harness.WaitUntilServing(ctx, clientMachine, ip, servingPort)
+
+ // Grab a client.
+ client := clientMachine.GetNativeContainer(ctx, b)
+ defer client.CleanUp(ctx)
+
+ b.ResetTimer()
+ server.RestartProfiles()
+ for i := 0; i < b.N; i++ {
+ out, err := client.Run(ctx, dockerutil.RunOpts{
+ Image: "benchmarks/hey",
+ }, hey.MakeCmd(ip, servingPort)...)
+ if err != nil {
+ b.Fatalf("run failed with: %v", err)
+ }
+ b.StopTimer()
+ hey.Report(b, out)
+ b.StartTimer()
+ }
+}
diff --git a/test/benchmarks/network/node_test.go b/test/benchmarks/network/node_test.go
index f9278ab66..5b568cfe5 100644
--- a/test/benchmarks/network/node_test.go
+++ b/test/benchmarks/network/node_test.go
@@ -16,14 +16,12 @@ package network
import (
"context"
"fmt"
- "regexp"
- "strconv"
- "strings"
"testing"
"time"
"gvisor.dev/gvisor/pkg/test/dockerutil"
"gvisor.dev/gvisor/test/benchmarks/harness"
+ "gvisor.dev/gvisor/test/benchmarks/tools"
)
// BenchmarkNode runs 10K requests using 'hey' against a Node server run on
@@ -36,13 +34,17 @@ func BenchmarkNode(b *testing.B) {
for _, c := range concurrency {
b.Run(fmt.Sprintf("Concurrency%d", c), func(b *testing.B) {
- runNode(b, requests, c)
+ hey := &tools.Hey{
+ Requests: requests,
+ Concurrency: c,
+ }
+ runNode(b, hey)
})
}
}
// runNode runs the test for a given # of requests and concurrency.
-func runNode(b *testing.B, requests, concurrency int) {
+func runNode(b *testing.B, hey *tools.Hey) {
b.Helper()
// The machine to hold Redis and the Node Server.
@@ -106,7 +108,7 @@ func runNode(b *testing.B, requests, concurrency int) {
// Wait until the Client sees the server as up.
harness.WaitUntilServing(ctx, clientMachine, servingIP, servingPort)
- heyCmd := strings.Split(fmt.Sprintf("hey -n %d -c %d http://%s:%d/", requests, concurrency, servingIP, servingPort), " ")
+ heyCmd := hey.MakeCmd(servingIP, servingPort)
nodeApp.RestartProfiles()
b.ResetTimer()
@@ -123,139 +125,7 @@ func runNode(b *testing.B, requests, concurrency int) {
// Stop the timer to parse the data and report stats.
b.StopTimer()
- requests, err := parseHeyRequestsPerSecond(out)
- if err != nil {
- b.Fatalf("failed to parse requests per second: %v", err)
- }
- b.ReportMetric(requests, "requests_per_second")
-
- bw, err := parseHeyBandwidth(out)
- if err != nil {
- b.Fatalf("failed to parse bandwidth: %v", err)
- }
- b.ReportMetric(bw, "bandwidth")
-
- ave, err := parseHeyAverageLatency(out)
- if err != nil {
- b.Fatalf("failed to parse average latency: %v", err)
- }
- b.ReportMetric(ave, "average_latency")
+ hey.Report(b, out)
b.StartTimer()
}
}
-
-var heyReqPerSecondRE = regexp.MustCompile(`Requests/sec:\s*(\d+\.?\d+?)\s+`)
-
-// parseHeyRequestsPerSecond finds requests per second from hey output.
-func parseHeyRequestsPerSecond(data string) (float64, error) {
- match := heyReqPerSecondRE.FindStringSubmatch(data)
- if len(match) < 2 {
- return 0, fmt.Errorf("failed get bandwidth: %s", data)
- }
- return strconv.ParseFloat(match[1], 64)
-}
-
-var heyAverageLatencyRE = regexp.MustCompile(`Average:\s*(\d+\.?\d+?)\s+secs`)
-
-// parseHeyAverageLatency finds Average Latency in seconds form hey output.
-func parseHeyAverageLatency(data string) (float64, error) {
- match := heyAverageLatencyRE.FindStringSubmatch(data)
- if len(match) < 2 {
- return 0, fmt.Errorf("failed get average latency match%d : %s", len(match), data)
- }
- return strconv.ParseFloat(match[1], 64)
-}
-
-var heySizePerRequestRE = regexp.MustCompile(`Size/request:\s*(\d+\.?\d+?)\s+bytes`)
-
-// parseHeyBandwidth computes bandwidth from request/sec * bytes/request
-// and reports in bytes/second.
-func parseHeyBandwidth(data string) (float64, error) {
- match := heyReqPerSecondRE.FindStringSubmatch(data)
- if len(match) < 2 {
- return 0, fmt.Errorf("failed get requests per second: %s", data)
- }
- reqPerSecond, err := strconv.ParseFloat(match[1], 64)
- if err != nil {
- return 0, fmt.Errorf("failed to convert %s to float", match[1])
- }
-
- match = heySizePerRequestRE.FindStringSubmatch(data)
- if len(match) < 2 {
- return 0, fmt.Errorf("failed get average latency: %s", data)
- }
- requestSize, err := strconv.ParseFloat(match[1], 64)
- return requestSize * reqPerSecond, err
-}
-
-// TestHeyParsers tests that the parsers work with sample output.
-func TestHeyParsers(t *testing.T) {
- sampleData := `
- Summary:
- Total: 2.2391 secs
- Slowest: 1.6292 secs
- Fastest: 0.0066 secs
- Average: 0.5351 secs
- Requests/sec: 89.3202
-
- Total data: 841200 bytes
- Size/request: 4206 bytes
-
- Response time histogram:
- 0.007 [1] |
- 0.169 [0] |
- 0.331 [149] |■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■
- 0.493 [0] |
- 0.656 [0] |
- 0.818 [0] |
- 0.980 [0] |
- 1.142 [0] |
- 1.305 [0] |
- 1.467 [49] |■■■■■■■■■■■■■
- 1.629 [1] |
-
-
- Latency distribution:
- 10% in 0.2149 secs
- 25% in 0.2449 secs
- 50% in 0.2703 secs
- 75% in 1.3315 secs
- 90% in 1.4045 secs
- 95% in 1.4232 secs
- 99% in 1.4362 secs
-
- Details (average, fastest, slowest):
- DNS+dialup: 0.0002 secs, 0.0066 secs, 1.6292 secs
- DNS-lookup: 0.0000 secs, 0.0000 secs, 0.0000 secs
- req write: 0.0000 secs, 0.0000 secs, 0.0012 secs
- resp wait: 0.5225 secs, 0.0064 secs, 1.4346 secs
- resp read: 0.0122 secs, 0.0001 secs, 0.2006 secs
-
- Status code distribution:
- [200] 200 responses
- `
- want := 89.3202
- got, err := parseHeyRequestsPerSecond(sampleData)
- if err != nil {
- t.Fatalf("failed to parse request per second with: %v", err)
- } else if got != want {
- t.Fatalf("got: %f, want: %f", got, want)
- }
-
- want = 89.3202 * 4206
- got, err = parseHeyBandwidth(sampleData)
- if err != nil {
- t.Fatalf("failed to parse bandwidth with: %v", err)
- } else if got != want {
- t.Fatalf("got: %f, want: %f", got, want)
- }
-
- want = 0.5351
- got, err = parseHeyAverageLatency(sampleData)
- if err != nil {
- t.Fatalf("failed to parse average latency with: %v", err)
- } else if got != want {
- t.Fatalf("got: %f, want: %f", got, want)
- }
-
-}