summaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--test/benchmarks/base/sysbench_test.go11
-rw-r--r--test/benchmarks/database/redis_test.go10
-rw-r--r--test/benchmarks/fs/bazel_test.go32
-rw-r--r--test/benchmarks/fs/fio_test.go15
-rw-r--r--test/benchmarks/network/httpd_test.go46
-rw-r--r--test/benchmarks/network/nginx_test.go53
-rw-r--r--test/benchmarks/network/node_test.go12
-rw-r--r--test/benchmarks/network/ruby_test.go11
-rw-r--r--test/benchmarks/tools/BUILD2
-rw-r--r--test/benchmarks/tools/ab.go3
-rw-r--r--test/benchmarks/tools/fio.go4
-rw-r--r--test/benchmarks/tools/hey.go4
-rw-r--r--test/benchmarks/tools/iperf.go2
-rw-r--r--test/benchmarks/tools/meminfo.go2
-rw-r--r--test/benchmarks/tools/parser_util.go101
-rw-r--r--test/benchmarks/tools/redis.go2
-rw-r--r--test/benchmarks/tools/sysbench.go10
-rw-r--r--tools/bigquery/BUILD3
-rw-r--r--tools/bigquery/bigquery.go31
-rw-r--r--tools/parsers/BUILD27
-rw-r--r--tools/parsers/go_parser.go151
-rw-r--r--tools/parsers/go_parser_test.go171
22 files changed, 614 insertions, 89 deletions
diff --git a/test/benchmarks/base/sysbench_test.go b/test/benchmarks/base/sysbench_test.go
index 6fb813640..39ced3dab 100644
--- a/test/benchmarks/base/sysbench_test.go
+++ b/test/benchmarks/base/sysbench_test.go
@@ -71,8 +71,15 @@ func BenchmarkSysbench(b *testing.B) {
defer machine.CleanUp()
for _, tc := range testCases {
- b.Run(tc.name, func(b *testing.B) {
-
+ param := tools.Parameter{
+ Name: "testname",
+ Value: tc.name,
+ }
+ name, err := tools.ParametersToName(param)
+ if err != nil {
+ b.Fatalf("Failed to parse params: %v", err)
+ }
+ b.Run(name, func(b *testing.B) {
ctx := context.Background()
sysbench := machine.GetContainer(ctx, b)
defer sysbench.CleanUp(ctx)
diff --git a/test/benchmarks/database/redis_test.go b/test/benchmarks/database/redis_test.go
index 6671a4969..02e67154e 100644
--- a/test/benchmarks/database/redis_test.go
+++ b/test/benchmarks/database/redis_test.go
@@ -66,7 +66,15 @@ func BenchmarkRedis(b *testing.B) {
ctx := context.Background()
for _, operation := range operations {
- b.Run(operation, func(b *testing.B) {
+ param := tools.Parameter{
+ Name: "operation",
+ Value: operation,
+ }
+ name, err := tools.ParametersToName(param)
+ if err != nil {
+ b.Fatalf("Failed to parse paramaters: %v", err)
+ }
+ b.Run(name, func(b *testing.B) {
server := serverMachine.GetContainer(ctx, b)
defer server.CleanUp(ctx)
diff --git a/test/benchmarks/fs/bazel_test.go b/test/benchmarks/fs/bazel_test.go
index ef1b8e4ea..56103639d 100644
--- a/test/benchmarks/fs/bazel_test.go
+++ b/test/benchmarks/fs/bazel_test.go
@@ -21,6 +21,7 @@ import (
"gvisor.dev/gvisor/pkg/test/dockerutil"
"gvisor.dev/gvisor/test/benchmarks/harness"
+ "gvisor.dev/gvisor/test/benchmarks/tools"
)
// Note: CleanCache versions of this test require running with root permissions.
@@ -46,17 +47,36 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) {
// Dimensions here are clean/dirty cache (do or don't drop caches)
// and if the mount on which we are compiling is a tmpfs/bind mount.
benchmarks := []struct {
- name string
clearCache bool // clearCache drops caches before running.
tmpfs bool // tmpfs will run compilation on a tmpfs.
}{
- {name: "CleanCache", clearCache: true, tmpfs: false},
- {name: "DirtyCache", clearCache: false, tmpfs: false},
- {name: "CleanCacheTmpfs", clearCache: true, tmpfs: true},
- {name: "DirtyCacheTmpfs", clearCache: false, tmpfs: true},
+ {clearCache: true, tmpfs: false},
+ {clearCache: false, tmpfs: false},
+ {clearCache: true, tmpfs: true},
+ {clearCache: false, tmpfs: true},
}
for _, bm := range benchmarks {
- b.Run(bm.name, func(b *testing.B) {
+ pageCache := tools.Parameter{
+ Name: "page_cache",
+ Value: "clean",
+ }
+ if bm.clearCache {
+ pageCache.Value = "dirty"
+ }
+
+ filesystem := tools.Parameter{
+ Name: "filesystem",
+ Value: "bind",
+ }
+ if bm.tmpfs {
+ filesystem.Value = "tmpfs"
+ }
+ name, err := tools.ParametersToName(pageCache, filesystem)
+ if err != nil {
+ b.Fatalf("Failed to parse parameters: %v", err)
+ }
+
+ b.Run(name, func(b *testing.B) {
// Grab a container.
ctx := context.Background()
container := machine.GetContainer(ctx, b)
diff --git a/test/benchmarks/fs/fio_test.go b/test/benchmarks/fs/fio_test.go
index 65874ed8b..5ca191404 100644
--- a/test/benchmarks/fs/fio_test.go
+++ b/test/benchmarks/fs/fio_test.go
@@ -67,8 +67,19 @@ func BenchmarkFio(b *testing.B) {
for _, fsType := range []mount.Type{mount.TypeBind, mount.TypeTmpfs} {
for _, tc := range testCases {
- testName := strings.Title(tc.Test) + strings.Title(string(fsType))
- b.Run(testName, func(b *testing.B) {
+ operation := tools.Parameter{
+ Name: "operation",
+ Value: tc.Test,
+ }
+ filesystem := tools.Parameter{
+ Name: "filesystem",
+ Value: string(fsType),
+ }
+ name, err := tools.ParametersToName(operation, filesystem)
+ if err != nil {
+ b.Fatalf("Failed to parser paramters: %v", err)
+ }
+ b.Run(name, func(b *testing.B) {
ctx := context.Background()
container := machine.GetContainer(ctx, b)
defer container.CleanUp(ctx)
diff --git a/test/benchmarks/network/httpd_test.go b/test/benchmarks/network/httpd_test.go
index 369ab326e..8d7d5f750 100644
--- a/test/benchmarks/network/httpd_test.go
+++ b/test/benchmarks/network/httpd_test.go
@@ -14,7 +14,7 @@
package network
import (
- "fmt"
+ "strconv"
"testing"
"gvisor.dev/gvisor/pkg/test/dockerutil"
@@ -31,33 +31,15 @@ var httpdDocs = map[string]string{
"10Mb": "latin10240k.txt",
}
-// BenchmarkHttpdConcurrency iterates the concurrency argument and tests
-// how well the runtime under test handles requests in parallel.
-func BenchmarkHttpdConcurrency(b *testing.B) {
- // The test iterates over client concurrency, so set other parameters.
- concurrency := []int{1, 25, 50, 100, 1000}
-
- for _, c := range concurrency {
- b.Run(fmt.Sprintf("%d", c), func(b *testing.B) {
- hey := &tools.Hey{
- Requests: c * b.N,
- Concurrency: c,
- Doc: httpdDocs["10Kb"],
- }
- runHttpd(b, hey, false /* reverse */)
- })
- }
-}
-
-// BenchmarkHttpdDocSize iterates over different sized payloads, testing how
-// well the runtime handles sending different payload sizes.
-func BenchmarkHttpdDocSize(b *testing.B) {
+// BenchmarkHttpd iterates over different sized payloads and concurrency, testing
+// how well the runtime handles sending different payload sizes.
+func BenchmarkHttpd(b *testing.B) {
benchmarkHttpdDocSize(b, false /* reverse */)
}
-// BenchmarkReverseHttpdDocSize iterates over different sized payloads, testing
+// BenchmarkReverseHttpd iterates over different sized payloads, testing
// how well the runtime handles receiving different payload sizes.
-func BenchmarkReverseHttpdDocSize(b *testing.B) {
+func BenchmarkReverseHttpd(b *testing.B) {
benchmarkHttpdDocSize(b, true /* reverse */)
}
@@ -65,10 +47,22 @@ func BenchmarkReverseHttpdDocSize(b *testing.B) {
// for each size.
func benchmarkHttpdDocSize(b *testing.B, reverse bool) {
b.Helper()
- for name, filename := range httpdDocs {
+ for size, filename := range httpdDocs {
concurrency := []int{1, 25, 50, 100, 1000}
for _, c := range concurrency {
- b.Run(fmt.Sprintf("%s_%d", name, c), func(b *testing.B) {
+ fsize := tools.Parameter{
+ Name: "filesize",
+ Value: size,
+ }
+ concurrency := tools.Parameter{
+ Name: "concurrency",
+ Value: strconv.Itoa(c),
+ }
+ name, err := tools.ParametersToName(fsize, concurrency)
+ if err != nil {
+ b.Fatalf("Failed to parse parameters: %v", err)
+ }
+ b.Run(name, func(b *testing.B) {
hey := &tools.Hey{
Requests: c * b.N,
Concurrency: c,
diff --git a/test/benchmarks/network/nginx_test.go b/test/benchmarks/network/nginx_test.go
index 9ec70369b..08565d0b2 100644
--- a/test/benchmarks/network/nginx_test.go
+++ b/test/benchmarks/network/nginx_test.go
@@ -14,7 +14,7 @@
package network
import (
- "fmt"
+ "strconv"
"testing"
"gvisor.dev/gvisor/pkg/test/dockerutil"
@@ -31,30 +31,6 @@ var nginxDocs = map[string]string{
"10Mb": "latin10240k.txt",
}
-// BenchmarkNginxConcurrency iterates the concurrency argument and tests
-// how well the runtime under test handles requests in parallel.
-func BenchmarkNginxConcurrency(b *testing.B) {
- concurrency := []int{1, 25, 100, 1000}
- for _, c := range concurrency {
- for _, tmpfs := range []bool{true, false} {
- fs := "Gofer"
- if tmpfs {
- fs = "Tmpfs"
- }
- name := fmt.Sprintf("%d_%s", c, fs)
- b.Run(name, func(b *testing.B) {
- hey := &tools.Hey{
- Requests: c * b.N,
- Concurrency: c,
- Doc: nginxDocs["10kb"], // see Dockerfile '//images/benchmarks/nginx' and httpd_test.
- }
- runNginx(b, hey, false /* reverse */, tmpfs /* tmpfs */)
- })
- }
-
- }
-}
-
// BenchmarkNginxDocSize iterates over different sized payloads, testing how
// well the runtime handles sending different payload sizes.
func BenchmarkNginxDocSize(b *testing.B) {
@@ -71,15 +47,32 @@ func BenchmarkReverseNginxDocSize(b *testing.B) {
// benchmarkNginxDocSize iterates through all doc sizes, running subbenchmarks
// for each size.
func benchmarkNginxDocSize(b *testing.B, reverse, tmpfs bool) {
- for name, filename := range nginxDocs {
+ for size, filename := range nginxDocs {
concurrency := []int{1, 25, 50, 100, 1000}
for _, c := range concurrency {
- fs := "Gofer"
+ fsize := tools.Parameter{
+ Name: "filesize",
+ Value: size,
+ }
+
+ threads := tools.Parameter{
+ Name: "concurrency",
+ Value: strconv.Itoa(c),
+ }
+
+ fs := tools.Parameter{
+ Name: "filesystem",
+ Value: "bind",
+ }
if tmpfs {
- fs = "Tmpfs"
+ fs.Value = "tmpfs"
+ }
+ name, err := tools.ParametersToName(fsize, threads, fs)
+ if err != nil {
+ b.Fatalf("Failed to parse parameters: %v", err)
}
- benchName := fmt.Sprintf("%s_%d_%s", name, c, fs)
- b.Run(benchName, func(b *testing.B) {
+
+ b.Run(name, func(b *testing.B) {
hey := &tools.Hey{
Requests: c * b.N,
Concurrency: c,
diff --git a/test/benchmarks/network/node_test.go b/test/benchmarks/network/node_test.go
index 0f4a205b6..254538899 100644
--- a/test/benchmarks/network/node_test.go
+++ b/test/benchmarks/network/node_test.go
@@ -15,7 +15,7 @@ package network
import (
"context"
- "fmt"
+ "strconv"
"testing"
"time"
@@ -31,7 +31,15 @@ import (
func BenchmarkNode(b *testing.B) {
concurrency := []int{1, 5, 10, 25}
for _, c := range concurrency {
- b.Run(fmt.Sprintf("Concurrency%d", c), func(b *testing.B) {
+ param := tools.Parameter{
+ Name: "concurrency",
+ Value: strconv.Itoa(c),
+ }
+ name, err := tools.ParametersToName(param)
+ if err != nil {
+ b.Fatalf("Failed to parse parameters: %v", err)
+ }
+ b.Run(name, func(b *testing.B) {
hey := &tools.Hey{
Requests: b.N * c, // Requests b.N requests per thread.
Concurrency: c,
diff --git a/test/benchmarks/network/ruby_test.go b/test/benchmarks/network/ruby_test.go
index 67f63f76a..0174ff3f3 100644
--- a/test/benchmarks/network/ruby_test.go
+++ b/test/benchmarks/network/ruby_test.go
@@ -16,6 +16,7 @@ package network
import (
"context"
"fmt"
+ "strconv"
"testing"
"time"
@@ -31,7 +32,15 @@ import (
func BenchmarkRuby(b *testing.B) {
concurrency := []int{1, 5, 10, 25}
for _, c := range concurrency {
- b.Run(fmt.Sprintf("Concurrency%d", c), func(b *testing.B) {
+ param := tools.Parameter{
+ Name: "concurrency",
+ Value: strconv.Itoa(c),
+ }
+ name, err := tools.ParametersToName(param)
+ if err != nil {
+ b.Fatalf("Failed to parse parameters: %v", err)
+ }
+ b.Run(name, func(b *testing.B) {
hey := &tools.Hey{
Requests: b.N * c, // b.N requests per thread.
Concurrency: c,
diff --git a/test/benchmarks/tools/BUILD b/test/benchmarks/tools/BUILD
index e5734d85c..9290830d7 100644
--- a/test/benchmarks/tools/BUILD
+++ b/test/benchmarks/tools/BUILD
@@ -4,12 +4,14 @@ package(licenses = ["notice"])
go_library(
name = "tools",
+ testonly = 1,
srcs = [
"ab.go",
"fio.go",
"hey.go",
"iperf.go",
"meminfo.go",
+ "parser_util.go",
"redis.go",
"sysbench.go",
"tools.go",
diff --git a/test/benchmarks/tools/ab.go b/test/benchmarks/tools/ab.go
index 4cc9c3bce..d9abf0763 100644
--- a/test/benchmarks/tools/ab.go
+++ b/test/benchmarks/tools/ab.go
@@ -46,18 +46,21 @@ func (a *ApacheBench) Report(b *testing.B, output string) {
b.Logf("failed to parse transferrate: %v", err)
}
b.ReportMetric(transferRate*1024, "transfer_rate_b/s") // Convert from Kb/s to b/s.
+ ReportCustomMetric(b, transferRate*1024, "transfer_rate" /*metric name*/, "bytes_per_second" /*unit*/)
latency, err := a.parseLatency(output)
if err != nil {
b.Logf("failed to parse latency: %v", err)
}
b.ReportMetric(latency/1000, "mean_latency_secs") // Convert from ms to s.
+ ReportCustomMetric(b, latency/1000, "mean_latency" /*metric name*/, "s" /*unit*/)
reqPerSecond, err := a.parseRequestsPerSecond(output)
if err != nil {
b.Logf("failed to parse requests per second: %v", err)
}
b.ReportMetric(reqPerSecond, "requests_per_second")
+ ReportCustomMetric(b, reqPerSecond, "requests_per_second" /*metric name*/, "QPS" /*unit*/)
}
var transferRateRE = regexp.MustCompile(`Transfer rate:\s+(\d+\.?\d+?)\s+\[Kbytes/sec\]\s+received`)
diff --git a/test/benchmarks/tools/fio.go b/test/benchmarks/tools/fio.go
index 20000db16..f5f60fa84 100644
--- a/test/benchmarks/tools/fio.go
+++ b/test/benchmarks/tools/fio.go
@@ -56,13 +56,13 @@ func (f *Fio) Report(b *testing.B, output string) {
if err != nil {
b.Fatalf("failed to parse bandwidth from %s with: %v", output, err)
}
- b.ReportMetric(bw, "bandwidth_b/s") // in b/s.
+ ReportCustomMetric(b, bw, "bandwidth" /*metric name*/, "bytes_per_second" /*unit*/)
iops, err := f.parseIOps(output, isRead)
if err != nil {
b.Fatalf("failed to parse iops from %s with: %v", output, err)
}
- b.ReportMetric(iops, "iops")
+ ReportCustomMetric(b, iops, "io_ops" /*metric name*/, "ops_per_second" /*unit*/)
}
// parseBandwidth reports the bandwidth in b/s.
diff --git a/test/benchmarks/tools/hey.go b/test/benchmarks/tools/hey.go
index b1e20e356..b8cb938fe 100644
--- a/test/benchmarks/tools/hey.go
+++ b/test/benchmarks/tools/hey.go
@@ -43,13 +43,13 @@ func (h *Hey) Report(b *testing.B, output string) {
if err != nil {
b.Fatalf("failed to parse requests per second: %v", err)
}
- b.ReportMetric(requests, "requests_per_second")
+ ReportCustomMetric(b, requests, "requests_per_second" /*metric name*/, "QPS" /*unit*/)
ave, err := h.parseAverageLatency(output)
if err != nil {
b.Fatalf("failed to parse average latency: %v", err)
}
- b.ReportMetric(ave, "average_latency_secs")
+ ReportCustomMetric(b, ave, "average_latency" /*metric name*/, "s" /*unit*/)
}
var heyReqPerSecondRE = regexp.MustCompile(`Requests/sec:\s*(\d+\.?\d+?)\s+`)
diff --git a/test/benchmarks/tools/iperf.go b/test/benchmarks/tools/iperf.go
index df3d9349b..5c4e7125b 100644
--- a/test/benchmarks/tools/iperf.go
+++ b/test/benchmarks/tools/iperf.go
@@ -42,7 +42,7 @@ func (i *Iperf) Report(b *testing.B, output string) {
if err != nil {
b.Fatalf("failed to parse bandwitdth from %s: %v", output, err)
}
- b.ReportMetric(bW*1024, "bandwidth_b/s") // Convert from Kb/s to b/s.
+ ReportCustomMetric(b, bW*1024, "bandwidth" /*metric name*/, "bytes_per_second" /*unit*/)
}
// bandwidth parses the Bandwidth number from an iperf report. A sample is below.
diff --git a/test/benchmarks/tools/meminfo.go b/test/benchmarks/tools/meminfo.go
index 2414a96a7..b5786fe11 100644
--- a/test/benchmarks/tools/meminfo.go
+++ b/test/benchmarks/tools/meminfo.go
@@ -45,7 +45,7 @@ func (*Meminfo) Report(b *testing.B, before, after string) {
b.Fatalf("could not parse before value %s: %v", before, err)
}
val := 1024 * ((beforeVal - afterVal) / float64(b.N))
- b.ReportMetric(val, "average_container_size_bytes")
+ ReportCustomMetric(b, val, "average_container_size" /*metric name*/, "bytes" /*units*/)
}
var memInfoRE = regexp.MustCompile(`MemAvailable:\s*(\d+)\skB\n`)
diff --git a/test/benchmarks/tools/parser_util.go b/test/benchmarks/tools/parser_util.go
new file mode 100644
index 000000000..a4555c7dd
--- /dev/null
+++ b/test/benchmarks/tools/parser_util.go
@@ -0,0 +1,101 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tools
+
+import (
+ "fmt"
+ "regexp"
+ "strconv"
+ "strings"
+ "testing"
+)
+
+// Parameter is a test parameter.
+type Parameter struct {
+ Name string
+ Value string
+}
+
+// Output is parsed and split by these values. Make them illegal in input methods.
+// We are constrained on what characters these can be by 1) docker's allowable
+// container names, 2) golang allowable benchmark names, and 3) golangs allowable
+// charecters in b.ReportMetric calls.
+var illegalChars = regexp.MustCompile(`[/\.]`)
+
+// ParametersToName joins parameters into a string format for parsing.
+// It is meant to be used for t.Run() calls in benchmark tools.
+func ParametersToName(params ...Parameter) (string, error) {
+ var strs []string
+ for _, param := range params {
+ if illegalChars.MatchString(param.Name) || illegalChars.MatchString(param.Value) {
+ return "", fmt.Errorf("params Name: %q and Value: %q cannot container '.' or '/'", param.Name, param.Value)
+ }
+ strs = append(strs, strings.Join([]string{param.Name, param.Value}, "."))
+ }
+ return strings.Join(strs, "/"), nil
+}
+
+// NameToParameters parses the string created by ParametersToName and returns
+// it as a set of Parameters.
+// Example: BenchmarkRuby/server_threads.1/doc_size.16KB-6
+// The parameter part of this benchmark is:
+// "server_threads.1/doc_size.16KB" (BenchmarkRuby is the name, and 6 is GOMAXPROCS)
+// This function will return a slice with two parameters ->
+// {Name: server_threads, Value: 1}, {Name: doc_size, Value: 16KB}
+func NameToParameters(name string) ([]*Parameter, error) {
+ var params []*Parameter
+ for _, cond := range strings.Split(name, "/") {
+ cs := strings.Split(cond, ".")
+ switch len(cs) {
+ case 1:
+ params = append(params, &Parameter{Name: cond, Value: cond})
+ case 2:
+ params = append(params, &Parameter{Name: cs[0], Value: cs[1]})
+ default:
+ return nil, fmt.Errorf("failed to parse param: %s", cond)
+ }
+ }
+ return params, nil
+}
+
+// ReportCustomMetric reports a metric in a set format for parsing.
+func ReportCustomMetric(b *testing.B, value float64, name, unit string) {
+ if illegalChars.MatchString(name) || illegalChars.MatchString(unit) {
+ b.Fatalf("name: %q and unit: %q cannot contain '/' or '.'", name, unit)
+ }
+ nameUnit := strings.Join([]string{name, unit}, ".")
+ b.ReportMetric(value, nameUnit)
+}
+
+// Metric holds metric data parsed from a string based on the format
+// ReportMetric.
+type Metric struct {
+ Name string
+ Unit string
+ Sample float64
+}
+
+// ParseCustomMetric parses a metric reported with ReportCustomMetric.
+func ParseCustomMetric(value, metric string) (*Metric, error) {
+ sample, err := strconv.ParseFloat(value, 64)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse value: %v", err)
+ }
+ nameUnit := strings.Split(metric, ".")
+ if len(nameUnit) != 2 {
+ return nil, fmt.Errorf("failed to parse metric: %s", metric)
+ }
+ return &Metric{Name: nameUnit[0], Unit: nameUnit[1], Sample: sample}, nil
+}
diff --git a/test/benchmarks/tools/redis.go b/test/benchmarks/tools/redis.go
index c899ae0d4..e35886437 100644
--- a/test/benchmarks/tools/redis.go
+++ b/test/benchmarks/tools/redis.go
@@ -49,7 +49,7 @@ func (r *Redis) Report(b *testing.B, output string) {
if err != nil {
b.Fatalf("parsing result %s failed with err: %v", output, err)
}
- b.ReportMetric(result, r.Operation) // operations per second
+ ReportCustomMetric(b, result, r.Operation /*metric_name*/, "QPS" /*unit*/)
}
// parseOperation grabs the metric operations per second from redis-benchmark output.
diff --git a/test/benchmarks/tools/sysbench.go b/test/benchmarks/tools/sysbench.go
index 6b2f75ca2..7ccacd8ff 100644
--- a/test/benchmarks/tools/sysbench.go
+++ b/test/benchmarks/tools/sysbench.go
@@ -80,7 +80,7 @@ func (s *SysbenchCPU) Report(b *testing.B, output string) {
if err != nil {
b.Fatalf("parsing CPU events from %s failed: %v", output, err)
}
- b.ReportMetric(result, "cpu_events_per_second")
+ ReportCustomMetric(b, result, "cpu_events" /*metric name*/, "events_per_second" /*unit*/)
}
var cpuEventsPerSecondRE = regexp.MustCompile(`events per second:\s*(\d*.?\d*)\n`)
@@ -144,7 +144,7 @@ func (s *SysbenchMemory) Report(b *testing.B, output string) {
if err != nil {
b.Fatalf("parsing result %s failed with err: %v", output, err)
}
- b.ReportMetric(result, "operations_per_second")
+ ReportCustomMetric(b, result, "memory_operations" /*metric name*/, "ops_per_second" /*unit*/)
}
var memoryOperationsRE = regexp.MustCompile(`Total\soperations:\s+\d*\s*\((\d*\.\d*)\sper\ssecond\)`)
@@ -198,19 +198,19 @@ func (s *SysbenchMutex) Report(b *testing.B, output string) {
if err != nil {
b.Fatalf("parsing result %s failed with err: %v", output, err)
}
- b.ReportMetric(result, "average_execution_time_secs")
+ ReportCustomMetric(b, result, "average_execution_time" /*metric name*/, "s" /*unit*/)
result, err = s.parseDeviation(output)
if err != nil {
b.Fatalf("parsing result %s failed with err: %v", output, err)
}
- b.ReportMetric(result, "stdev_execution_time_secs")
+ ReportCustomMetric(b, result, "stddev_execution_time" /*metric name*/, "s" /*unit*/)
result, err = s.parseLatency(output)
if err != nil {
b.Fatalf("parsing result %s failed with err: %v", output, err)
}
- b.ReportMetric(result/1000, "average_latency_secs")
+ ReportCustomMetric(b, result/1000, "average_latency" /*metric name*/, "s" /*unit*/)
}
var executionTimeRE = regexp.MustCompile(`execution time \(avg/stddev\):\s*(\d*.?\d*)/(\d*.?\d*)`)
diff --git a/tools/bigquery/BUILD b/tools/bigquery/BUILD
index 5748fb390..2b0062a63 100644
--- a/tools/bigquery/BUILD
+++ b/tools/bigquery/BUILD
@@ -6,5 +6,8 @@ go_library(
name = "bigquery",
testonly = 1,
srcs = ["bigquery.go"],
+ visibility = [
+ "//:sandbox",
+ ],
deps = ["@com_google_cloud_go_bigquery//:go_default_library"],
)
diff --git a/tools/bigquery/bigquery.go b/tools/bigquery/bigquery.go
index 56f0dc5c9..5f1a882de 100644
--- a/tools/bigquery/bigquery.go
+++ b/tools/bigquery/bigquery.go
@@ -30,11 +30,20 @@ import (
// Benchmark is the top level structure of recorded benchmark data. BigQuery
// will infer the schema from this.
type Benchmark struct {
- Name string `bq:"name"`
- Timestamp time.Time `bq:"timestamp"`
- Official bool `bq:"official"`
- Metric []*Metric `bq:"metric"`
- Metadata *Metadata `bq:"metadata"`
+ Name string `bq:"name"`
+ Condition []*Condition `bq:"condition"`
+ Timestamp time.Time `bq:"timestamp"`
+ Official bool `bq:"official"`
+ Metric []*Metric `bq:"metric"`
+ Metadata *Metadata `bq:"metadata"`
+}
+
+// Condition represents qualifiers for the benchmark. For example:
+// Get_Pid/1/real_time would have Benchmark Name "Get_Pid" with "1"
+// and "real_time" parameters as conditions.
+type Condition struct {
+ Name string `bq:"name"`
+ Value string `bq:"value"`
}
// Metric holds the actual metric data and unit information for this benchmark.
@@ -79,6 +88,14 @@ func InitBigQuery(ctx context.Context, projectID, datasetID, tableID string) err
return nil
}
+// AddCondition adds a condition to an existing Benchmark.
+func (bm *Benchmark) AddCondition(name, value string) {
+ bm.Condition = append(bm.Condition, &Condition{
+ Name: name,
+ Value: value,
+ })
+}
+
// AddMetric adds a metric to an existing Benchmark.
func (bm *Benchmark) AddMetric(metricName, unit string, sample float64) {
m := &Metric{
@@ -90,7 +107,7 @@ func (bm *Benchmark) AddMetric(metricName, unit string, sample float64) {
}
// NewBenchmark initializes a new benchmark.
-func NewBenchmark(name string, official bool) *Benchmark {
+func NewBenchmark(name string, iters int, official bool) *Benchmark {
return &Benchmark{
Name: name,
Timestamp: time.Now().UTC(),
@@ -103,7 +120,7 @@ func NewBenchmark(name string, official bool) *Benchmark {
func SendBenchmarks(ctx context.Context, benchmarks []*Benchmark, projectID, datasetID, tableID string) error {
client, err := bq.NewClient(ctx, projectID)
if err != nil {
- return fmt.Errorf("Failed to initialize client on project: %s: %v", projectID, err)
+ return fmt.Errorf("failed to initialize client on project: %s: %v", projectID, err)
}
defer client.Close()
diff --git a/tools/parsers/BUILD b/tools/parsers/BUILD
new file mode 100644
index 000000000..7d9c9a3fb
--- /dev/null
+++ b/tools/parsers/BUILD
@@ -0,0 +1,27 @@
+load("//tools:defs.bzl", "go_library", "go_test")
+
+package(licenses = ["notice"])
+
+go_test(
+ name = "parsers_test",
+ size = "small",
+ srcs = ["go_parser_test.go"],
+ library = ":parsers",
+ deps = [
+ "//tools/bigquery",
+ "@com_github_google_go_cmp//cmp:go_default_library",
+ ],
+)
+
+go_library(
+ name = "parsers",
+ testonly = 1,
+ srcs = [
+ "go_parser.go",
+ ],
+ visibility = ["//:sandbox"],
+ deps = [
+ "//test/benchmarks/tools",
+ "//tools/bigquery",
+ ],
+)
diff --git a/tools/parsers/go_parser.go b/tools/parsers/go_parser.go
new file mode 100644
index 000000000..2cf74c883
--- /dev/null
+++ b/tools/parsers/go_parser.go
@@ -0,0 +1,151 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package parsers holds parsers to parse Benchmark test output.
+//
+// Parsers parse Benchmark test output and place it in BigQuery
+// structs for sending to BigQuery databases.
+package parsers
+
+import (
+ "fmt"
+ "strconv"
+ "strings"
+
+ "gvisor.dev/gvisor/test/benchmarks/tools"
+ "gvisor.dev/gvisor/tools/bigquery"
+)
+
+// parseOutput expects golang benchmark output returns a Benchmark struct formatted for BigQuery.
+func parseOutput(output string, metadata *bigquery.Metadata, official bool) ([]*bigquery.Benchmark, error) {
+ var benchmarks []*bigquery.Benchmark
+ lines := strings.Split(output, "\n")
+ for _, line := range lines {
+ bm, err := parseLine(line, metadata, official)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse line '%s': %v", line, err)
+ }
+ if bm != nil {
+ benchmarks = append(benchmarks, bm)
+ }
+ }
+ return benchmarks, nil
+}
+
+// parseLine handles parsing a benchmark line into a bigquery.Benchmark.
+//
+// Example: "BenchmarkRuby/server_threads.1-6 1 1397875880 ns/op 140 requests_per_second.QPS"
+//
+// This function will return the following benchmark:
+// *bigquery.Benchmark{
+// Name: BenchmarkRuby
+// []*bigquery.Condition{
+// {Name: GOMAXPROCS, 6}
+// {Name: server_threads, 1}
+// }
+// []*bigquery.Metric{
+// {Name: ns/op, Unit: ns/op, Sample: 1397875880}
+// {Name: requests_per_second, Unit: QPS, Sample: 140 }
+// }
+// Metadata: metadata
+//}
+func parseLine(line string, metadata *bigquery.Metadata, official bool) (*bigquery.Benchmark, error) {
+ fields := strings.Fields(line)
+
+ // Check if this line is a Benchmark line. Otherwise ignore the line.
+ if len(fields) < 2 || !strings.HasPrefix(fields[0], "Benchmark") {
+ return nil, nil
+ }
+
+ iters, err := strconv.Atoi(fields[1])
+ if err != nil {
+ return nil, fmt.Errorf("expecting number of runs, got %s: %v", fields[1], err)
+ }
+
+ name, params, err := parseNameParams(fields[0])
+ if err != nil {
+ return nil, fmt.Errorf("parse name/params: %v", err)
+ }
+
+ bm := bigquery.NewBenchmark(name, iters, official)
+ bm.Metadata = metadata
+ for _, p := range params {
+ bm.AddCondition(p.Name, p.Value)
+ }
+
+ for i := 1; i < len(fields)/2; i++ {
+ value := fields[2*i]
+ metric := fields[2*i+1]
+ if err := makeMetric(bm, value, metric); err != nil {
+ return nil, fmt.Errorf("makeMetric on metric %q value: %s: %v", metric, value, err)
+ }
+ }
+ return bm, nil
+}
+
+// parseNameParams parses the Name, GOMAXPROCS, and Params from the test.
+// Field here should be of the format TESTNAME/PARAMS-GOMAXPROCS.
+// Parameters will be separated by a "/" with individual params being
+// "name.value".
+func parseNameParams(field string) (string, []*tools.Parameter, error) {
+ var params []*tools.Parameter
+ // Remove GOMAXPROCS from end.
+ maxIndex := strings.LastIndex(field, "-")
+ if maxIndex < 0 {
+ return "", nil, fmt.Errorf("GOMAXPROCS not found: %s", field)
+ }
+ maxProcs := field[maxIndex+1:]
+ params = append(params, &tools.Parameter{
+ Name: "GOMAXPROCS",
+ Value: maxProcs,
+ })
+
+ remainder := field[0:maxIndex]
+ index := strings.Index(remainder, "/")
+ if index == -1 {
+ return remainder, params, nil
+ }
+
+ name := remainder[0:index]
+ p := remainder[index+1:]
+
+ ps, err := tools.NameToParameters(p)
+ if err != nil {
+ return "", nil, fmt.Errorf("NameToParameters %s: %v", field, err)
+ }
+ params = append(params, ps...)
+ return name, params, nil
+}
+
+// makeMetric parses metrics and adds them to the passed Benchmark.
+func makeMetric(bm *bigquery.Benchmark, value, metric string) error {
+ switch metric {
+ // Ignore most output from golang benchmarks.
+ case "MB/s", "B/op", "allocs/op":
+ return nil
+ case "ns/op":
+ val, err := strconv.ParseFloat(value, 64)
+ if err != nil {
+ return fmt.Errorf("ParseFloat %s: %v", value, err)
+ }
+ bm.AddMetric(metric /*metric name*/, metric /*unit*/, val /*sample*/)
+ default:
+ m, err := tools.ParseCustomMetric(value, metric)
+ if err != nil {
+ return fmt.Errorf("ParseCustomMetric %s: %v ", metric, err)
+ }
+ bm.AddMetric(m.Name, m.Unit, m.Sample)
+ }
+ return nil
+}
diff --git a/tools/parsers/go_parser_test.go b/tools/parsers/go_parser_test.go
new file mode 100644
index 000000000..36996b7c8
--- /dev/null
+++ b/tools/parsers/go_parser_test.go
@@ -0,0 +1,171 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parsers
+
+import (
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "gvisor.dev/gvisor/tools/bigquery"
+)
+
+func TestParseLine(t *testing.T) {
+ testCases := []struct {
+ name string
+ data string
+ want *bigquery.Benchmark
+ }{
+ {
+ name: "Iperf",
+ data: "BenchmarkIperf/Upload-6 1 11094914892 ns/op 4751711232 bandwidth.bytes_per_second",
+ want: &bigquery.Benchmark{
+ Name: "BenchmarkIperf",
+ Condition: []*bigquery.Condition{
+ {
+ Name: "GOMAXPROCS",
+ Value: "6",
+ },
+ {
+ Name: "Upload",
+ Value: "Upload",
+ },
+ },
+ Metric: []*bigquery.Metric{
+ {
+ Name: "ns/op",
+ Unit: "ns/op",
+ Sample: 11094914892.0,
+ },
+ {
+ Name: "bandwidth",
+ Unit: "bytes_per_second",
+ Sample: 4751711232.0,
+ },
+ },
+ },
+ },
+ {
+ name: "Ruby",
+ data: "BenchmarkRuby/server_threads.1-6 1 1397875880 ns/op 0.00710 average_latency.s 140 requests_per_second.QPS",
+ want: &bigquery.Benchmark{
+ Name: "BenchmarkRuby",
+ Condition: []*bigquery.Condition{
+ {
+ Name: "GOMAXPROCS",
+ Value: "6",
+ },
+ {
+ Name: "server_threads",
+ Value: "1",
+ },
+ },
+ Metric: []*bigquery.Metric{
+ {
+ Name: "ns/op",
+ Unit: "ns/op",
+ Sample: 1397875880.0,
+ },
+ {
+ Name: "average_latency",
+ Unit: "s",
+ Sample: 0.00710,
+ },
+ {
+ Name: "requests_per_second",
+ Unit: "QPS",
+ Sample: 140.0,
+ },
+ },
+ },
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := parseLine(tc.data, nil, false)
+ if err != nil {
+ t.Fatalf("parseLine failed with: %v", err)
+ }
+
+ tc.want.Timestamp = got.Timestamp
+
+ if !cmp.Equal(tc.want, got, nil) {
+ for _, c := range got.Condition {
+ t.Logf("Cond: %+v", c)
+ }
+ for _, m := range got.Metric {
+ t.Logf("Metric: %+v", m)
+ }
+ t.Fatalf("Compare failed want: %+v got: %+v", tc.want, got)
+ }
+ })
+
+ }
+}
+
+func TestParseOutput(t *testing.T) {
+ testCases := []struct {
+ name string
+ data string
+ numBenchmarks int
+ numMetrics int
+ numConditions int
+ }{
+ {
+ name: "Startup",
+ data: `
+ BenchmarkStartupEmpty
+ BenchmarkStartupEmpty-6 2 766377884 ns/op 1 allocs/op
+ BenchmarkStartupNode
+ BenchmarkStartupNode-6 1 1752158409 ns/op 1 allocs/op
+ `,
+ numBenchmarks: 2,
+ numMetrics: 1,
+ numConditions: 1,
+ },
+ {
+ name: "Ruby",
+ data: `BenchmarkRuby
+BenchmarkRuby/server_threads.1
+BenchmarkRuby/server_threads.1-6 1 1397875880 ns/op 0.00710 average_latency.s 140 requests_per_second.QPS
+BenchmarkRuby/server_threads.5
+BenchmarkRuby/server_threads.5-6 1 1416003331 ns/op 0.00950 average_latency.s 465 requests_per_second.QPS`,
+ numBenchmarks: 2,
+ numMetrics: 3,
+ numConditions: 2,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ bms, err := parseOutput(tc.data, nil, false)
+ if err != nil {
+ t.Fatalf("parseOutput failed: %v", err)
+ } else if len(bms) != tc.numBenchmarks {
+ t.Fatalf("NumBenchmarks failed want: %d got: %d %+v", tc.numBenchmarks, len(bms), bms)
+ }
+
+ for _, bm := range bms {
+ if len(bm.Metric) != tc.numMetrics {
+ t.Fatalf("NumMetrics failed want: %d got: %d %+v", tc.numMetrics, len(bm.Metric), bm.Metric)
+ }
+
+ if len(bm.Condition) != tc.numConditions {
+ t.Fatalf("NumConditions failed want: %d got: %d %+v", tc.numConditions, len(bm.Condition), bm.Condition)
+ }
+ }
+ })
+ }
+}