From b576de907cb42b8df11695cc58792631f4c059ae Mon Sep 17 00:00:00 2001 From: Zach Koopmans Date: Fri, 9 Oct 2020 14:26:55 -0700 Subject: Add parsers golang benchmarks. Add parser and formatting for golang benchmarks for docker benchmarks. Change adds a library for printing and parsing Test parameters and metrics. Benchmarks use the library to print parameters in the Benchmark title (e.g. the name field in b.Run()), and to report CustomMetrics. Parser uses the library to parse printed data from benchmark output and put it into BigQuery structs. PiperOrigin-RevId: 336365628 --- test/benchmarks/base/sysbench_test.go | 11 ++- test/benchmarks/database/redis_test.go | 10 +- test/benchmarks/fs/bazel_test.go | 32 ++++-- test/benchmarks/fs/fio_test.go | 15 ++- test/benchmarks/network/httpd_test.go | 46 ++++----- test/benchmarks/network/nginx_test.go | 53 +++++----- test/benchmarks/network/node_test.go | 12 ++- test/benchmarks/network/ruby_test.go | 11 ++- test/benchmarks/tools/BUILD | 2 + test/benchmarks/tools/ab.go | 3 + test/benchmarks/tools/fio.go | 4 +- test/benchmarks/tools/hey.go | 4 +- test/benchmarks/tools/iperf.go | 2 +- test/benchmarks/tools/meminfo.go | 2 +- test/benchmarks/tools/parser_util.go | 101 +++++++++++++++++++ test/benchmarks/tools/redis.go | 2 +- test/benchmarks/tools/sysbench.go | 10 +- tools/bigquery/BUILD | 3 + tools/bigquery/bigquery.go | 31 ++++-- tools/parsers/BUILD | 27 ++++++ tools/parsers/go_parser.go | 151 +++++++++++++++++++++++++++++ tools/parsers/go_parser_test.go | 171 +++++++++++++++++++++++++++++++++ 22 files changed, 614 insertions(+), 89 deletions(-) create mode 100644 test/benchmarks/tools/parser_util.go create mode 100644 tools/parsers/BUILD create mode 100644 tools/parsers/go_parser.go create mode 100644 tools/parsers/go_parser_test.go diff --git a/test/benchmarks/base/sysbench_test.go b/test/benchmarks/base/sysbench_test.go index 6fb813640..39ced3dab 100644 --- a/test/benchmarks/base/sysbench_test.go +++ b/test/benchmarks/base/sysbench_test.go @@ -71,8 +71,15 @@ func BenchmarkSysbench(b *testing.B) { defer machine.CleanUp() for _, tc := range testCases { - b.Run(tc.name, func(b *testing.B) { - + param := tools.Parameter{ + Name: "testname", + Value: tc.name, + } + name, err := tools.ParametersToName(param) + if err != nil { + b.Fatalf("Failed to parse params: %v", err) + } + b.Run(name, func(b *testing.B) { ctx := context.Background() sysbench := machine.GetContainer(ctx, b) defer sysbench.CleanUp(ctx) diff --git a/test/benchmarks/database/redis_test.go b/test/benchmarks/database/redis_test.go index 6671a4969..02e67154e 100644 --- a/test/benchmarks/database/redis_test.go +++ b/test/benchmarks/database/redis_test.go @@ -66,7 +66,15 @@ func BenchmarkRedis(b *testing.B) { ctx := context.Background() for _, operation := range operations { - b.Run(operation, func(b *testing.B) { + param := tools.Parameter{ + Name: "operation", + Value: operation, + } + name, err := tools.ParametersToName(param) + if err != nil { + b.Fatalf("Failed to parse paramaters: %v", err) + } + b.Run(name, func(b *testing.B) { server := serverMachine.GetContainer(ctx, b) defer server.CleanUp(ctx) diff --git a/test/benchmarks/fs/bazel_test.go b/test/benchmarks/fs/bazel_test.go index ef1b8e4ea..56103639d 100644 --- a/test/benchmarks/fs/bazel_test.go +++ b/test/benchmarks/fs/bazel_test.go @@ -21,6 +21,7 @@ import ( "gvisor.dev/gvisor/pkg/test/dockerutil" "gvisor.dev/gvisor/test/benchmarks/harness" + "gvisor.dev/gvisor/test/benchmarks/tools" ) // Note: CleanCache versions of this test require running with root permissions. @@ -46,17 +47,36 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) { // Dimensions here are clean/dirty cache (do or don't drop caches) // and if the mount on which we are compiling is a tmpfs/bind mount. benchmarks := []struct { - name string clearCache bool // clearCache drops caches before running. tmpfs bool // tmpfs will run compilation on a tmpfs. }{ - {name: "CleanCache", clearCache: true, tmpfs: false}, - {name: "DirtyCache", clearCache: false, tmpfs: false}, - {name: "CleanCacheTmpfs", clearCache: true, tmpfs: true}, - {name: "DirtyCacheTmpfs", clearCache: false, tmpfs: true}, + {clearCache: true, tmpfs: false}, + {clearCache: false, tmpfs: false}, + {clearCache: true, tmpfs: true}, + {clearCache: false, tmpfs: true}, } for _, bm := range benchmarks { - b.Run(bm.name, func(b *testing.B) { + pageCache := tools.Parameter{ + Name: "page_cache", + Value: "clean", + } + if bm.clearCache { + pageCache.Value = "dirty" + } + + filesystem := tools.Parameter{ + Name: "filesystem", + Value: "bind", + } + if bm.tmpfs { + filesystem.Value = "tmpfs" + } + name, err := tools.ParametersToName(pageCache, filesystem) + if err != nil { + b.Fatalf("Failed to parse parameters: %v", err) + } + + b.Run(name, func(b *testing.B) { // Grab a container. ctx := context.Background() container := machine.GetContainer(ctx, b) diff --git a/test/benchmarks/fs/fio_test.go b/test/benchmarks/fs/fio_test.go index 65874ed8b..5ca191404 100644 --- a/test/benchmarks/fs/fio_test.go +++ b/test/benchmarks/fs/fio_test.go @@ -67,8 +67,19 @@ func BenchmarkFio(b *testing.B) { for _, fsType := range []mount.Type{mount.TypeBind, mount.TypeTmpfs} { for _, tc := range testCases { - testName := strings.Title(tc.Test) + strings.Title(string(fsType)) - b.Run(testName, func(b *testing.B) { + operation := tools.Parameter{ + Name: "operation", + Value: tc.Test, + } + filesystem := tools.Parameter{ + Name: "filesystem", + Value: string(fsType), + } + name, err := tools.ParametersToName(operation, filesystem) + if err != nil { + b.Fatalf("Failed to parser paramters: %v", err) + } + b.Run(name, func(b *testing.B) { ctx := context.Background() container := machine.GetContainer(ctx, b) defer container.CleanUp(ctx) diff --git a/test/benchmarks/network/httpd_test.go b/test/benchmarks/network/httpd_test.go index 369ab326e..8d7d5f750 100644 --- a/test/benchmarks/network/httpd_test.go +++ b/test/benchmarks/network/httpd_test.go @@ -14,7 +14,7 @@ package network import ( - "fmt" + "strconv" "testing" "gvisor.dev/gvisor/pkg/test/dockerutil" @@ -31,33 +31,15 @@ var httpdDocs = map[string]string{ "10Mb": "latin10240k.txt", } -// BenchmarkHttpdConcurrency iterates the concurrency argument and tests -// how well the runtime under test handles requests in parallel. -func BenchmarkHttpdConcurrency(b *testing.B) { - // The test iterates over client concurrency, so set other parameters. - concurrency := []int{1, 25, 50, 100, 1000} - - for _, c := range concurrency { - b.Run(fmt.Sprintf("%d", c), func(b *testing.B) { - hey := &tools.Hey{ - Requests: c * b.N, - Concurrency: c, - Doc: httpdDocs["10Kb"], - } - runHttpd(b, hey, false /* reverse */) - }) - } -} - -// BenchmarkHttpdDocSize iterates over different sized payloads, testing how -// well the runtime handles sending different payload sizes. -func BenchmarkHttpdDocSize(b *testing.B) { +// BenchmarkHttpd iterates over different sized payloads and concurrency, testing +// how well the runtime handles sending different payload sizes. +func BenchmarkHttpd(b *testing.B) { benchmarkHttpdDocSize(b, false /* reverse */) } -// BenchmarkReverseHttpdDocSize iterates over different sized payloads, testing +// BenchmarkReverseHttpd iterates over different sized payloads, testing // how well the runtime handles receiving different payload sizes. -func BenchmarkReverseHttpdDocSize(b *testing.B) { +func BenchmarkReverseHttpd(b *testing.B) { benchmarkHttpdDocSize(b, true /* reverse */) } @@ -65,10 +47,22 @@ func BenchmarkReverseHttpdDocSize(b *testing.B) { // for each size. func benchmarkHttpdDocSize(b *testing.B, reverse bool) { b.Helper() - for name, filename := range httpdDocs { + for size, filename := range httpdDocs { concurrency := []int{1, 25, 50, 100, 1000} for _, c := range concurrency { - b.Run(fmt.Sprintf("%s_%d", name, c), func(b *testing.B) { + fsize := tools.Parameter{ + Name: "filesize", + Value: size, + } + concurrency := tools.Parameter{ + Name: "concurrency", + Value: strconv.Itoa(c), + } + name, err := tools.ParametersToName(fsize, concurrency) + if err != nil { + b.Fatalf("Failed to parse parameters: %v", err) + } + b.Run(name, func(b *testing.B) { hey := &tools.Hey{ Requests: c * b.N, Concurrency: c, diff --git a/test/benchmarks/network/nginx_test.go b/test/benchmarks/network/nginx_test.go index 9ec70369b..08565d0b2 100644 --- a/test/benchmarks/network/nginx_test.go +++ b/test/benchmarks/network/nginx_test.go @@ -14,7 +14,7 @@ package network import ( - "fmt" + "strconv" "testing" "gvisor.dev/gvisor/pkg/test/dockerutil" @@ -31,30 +31,6 @@ var nginxDocs = map[string]string{ "10Mb": "latin10240k.txt", } -// BenchmarkNginxConcurrency iterates the concurrency argument and tests -// how well the runtime under test handles requests in parallel. -func BenchmarkNginxConcurrency(b *testing.B) { - concurrency := []int{1, 25, 100, 1000} - for _, c := range concurrency { - for _, tmpfs := range []bool{true, false} { - fs := "Gofer" - if tmpfs { - fs = "Tmpfs" - } - name := fmt.Sprintf("%d_%s", c, fs) - b.Run(name, func(b *testing.B) { - hey := &tools.Hey{ - Requests: c * b.N, - Concurrency: c, - Doc: nginxDocs["10kb"], // see Dockerfile '//images/benchmarks/nginx' and httpd_test. - } - runNginx(b, hey, false /* reverse */, tmpfs /* tmpfs */) - }) - } - - } -} - // BenchmarkNginxDocSize iterates over different sized payloads, testing how // well the runtime handles sending different payload sizes. func BenchmarkNginxDocSize(b *testing.B) { @@ -71,15 +47,32 @@ func BenchmarkReverseNginxDocSize(b *testing.B) { // benchmarkNginxDocSize iterates through all doc sizes, running subbenchmarks // for each size. func benchmarkNginxDocSize(b *testing.B, reverse, tmpfs bool) { - for name, filename := range nginxDocs { + for size, filename := range nginxDocs { concurrency := []int{1, 25, 50, 100, 1000} for _, c := range concurrency { - fs := "Gofer" + fsize := tools.Parameter{ + Name: "filesize", + Value: size, + } + + threads := tools.Parameter{ + Name: "concurrency", + Value: strconv.Itoa(c), + } + + fs := tools.Parameter{ + Name: "filesystem", + Value: "bind", + } if tmpfs { - fs = "Tmpfs" + fs.Value = "tmpfs" + } + name, err := tools.ParametersToName(fsize, threads, fs) + if err != nil { + b.Fatalf("Failed to parse parameters: %v", err) } - benchName := fmt.Sprintf("%s_%d_%s", name, c, fs) - b.Run(benchName, func(b *testing.B) { + + b.Run(name, func(b *testing.B) { hey := &tools.Hey{ Requests: c * b.N, Concurrency: c, diff --git a/test/benchmarks/network/node_test.go b/test/benchmarks/network/node_test.go index 0f4a205b6..254538899 100644 --- a/test/benchmarks/network/node_test.go +++ b/test/benchmarks/network/node_test.go @@ -15,7 +15,7 @@ package network import ( "context" - "fmt" + "strconv" "testing" "time" @@ -31,7 +31,15 @@ import ( func BenchmarkNode(b *testing.B) { concurrency := []int{1, 5, 10, 25} for _, c := range concurrency { - b.Run(fmt.Sprintf("Concurrency%d", c), func(b *testing.B) { + param := tools.Parameter{ + Name: "concurrency", + Value: strconv.Itoa(c), + } + name, err := tools.ParametersToName(param) + if err != nil { + b.Fatalf("Failed to parse parameters: %v", err) + } + b.Run(name, func(b *testing.B) { hey := &tools.Hey{ Requests: b.N * c, // Requests b.N requests per thread. Concurrency: c, diff --git a/test/benchmarks/network/ruby_test.go b/test/benchmarks/network/ruby_test.go index 67f63f76a..0174ff3f3 100644 --- a/test/benchmarks/network/ruby_test.go +++ b/test/benchmarks/network/ruby_test.go @@ -16,6 +16,7 @@ package network import ( "context" "fmt" + "strconv" "testing" "time" @@ -31,7 +32,15 @@ import ( func BenchmarkRuby(b *testing.B) { concurrency := []int{1, 5, 10, 25} for _, c := range concurrency { - b.Run(fmt.Sprintf("Concurrency%d", c), func(b *testing.B) { + param := tools.Parameter{ + Name: "concurrency", + Value: strconv.Itoa(c), + } + name, err := tools.ParametersToName(param) + if err != nil { + b.Fatalf("Failed to parse parameters: %v", err) + } + b.Run(name, func(b *testing.B) { hey := &tools.Hey{ Requests: b.N * c, // b.N requests per thread. Concurrency: c, diff --git a/test/benchmarks/tools/BUILD b/test/benchmarks/tools/BUILD index e5734d85c..9290830d7 100644 --- a/test/benchmarks/tools/BUILD +++ b/test/benchmarks/tools/BUILD @@ -4,12 +4,14 @@ package(licenses = ["notice"]) go_library( name = "tools", + testonly = 1, srcs = [ "ab.go", "fio.go", "hey.go", "iperf.go", "meminfo.go", + "parser_util.go", "redis.go", "sysbench.go", "tools.go", diff --git a/test/benchmarks/tools/ab.go b/test/benchmarks/tools/ab.go index 4cc9c3bce..d9abf0763 100644 --- a/test/benchmarks/tools/ab.go +++ b/test/benchmarks/tools/ab.go @@ -46,18 +46,21 @@ func (a *ApacheBench) Report(b *testing.B, output string) { b.Logf("failed to parse transferrate: %v", err) } b.ReportMetric(transferRate*1024, "transfer_rate_b/s") // Convert from Kb/s to b/s. + ReportCustomMetric(b, transferRate*1024, "transfer_rate" /*metric name*/, "bytes_per_second" /*unit*/) latency, err := a.parseLatency(output) if err != nil { b.Logf("failed to parse latency: %v", err) } b.ReportMetric(latency/1000, "mean_latency_secs") // Convert from ms to s. + ReportCustomMetric(b, latency/1000, "mean_latency" /*metric name*/, "s" /*unit*/) reqPerSecond, err := a.parseRequestsPerSecond(output) if err != nil { b.Logf("failed to parse requests per second: %v", err) } b.ReportMetric(reqPerSecond, "requests_per_second") + ReportCustomMetric(b, reqPerSecond, "requests_per_second" /*metric name*/, "QPS" /*unit*/) } var transferRateRE = regexp.MustCompile(`Transfer rate:\s+(\d+\.?\d+?)\s+\[Kbytes/sec\]\s+received`) diff --git a/test/benchmarks/tools/fio.go b/test/benchmarks/tools/fio.go index 20000db16..f5f60fa84 100644 --- a/test/benchmarks/tools/fio.go +++ b/test/benchmarks/tools/fio.go @@ -56,13 +56,13 @@ func (f *Fio) Report(b *testing.B, output string) { if err != nil { b.Fatalf("failed to parse bandwidth from %s with: %v", output, err) } - b.ReportMetric(bw, "bandwidth_b/s") // in b/s. + ReportCustomMetric(b, bw, "bandwidth" /*metric name*/, "bytes_per_second" /*unit*/) iops, err := f.parseIOps(output, isRead) if err != nil { b.Fatalf("failed to parse iops from %s with: %v", output, err) } - b.ReportMetric(iops, "iops") + ReportCustomMetric(b, iops, "io_ops" /*metric name*/, "ops_per_second" /*unit*/) } // parseBandwidth reports the bandwidth in b/s. diff --git a/test/benchmarks/tools/hey.go b/test/benchmarks/tools/hey.go index b1e20e356..b8cb938fe 100644 --- a/test/benchmarks/tools/hey.go +++ b/test/benchmarks/tools/hey.go @@ -43,13 +43,13 @@ func (h *Hey) Report(b *testing.B, output string) { if err != nil { b.Fatalf("failed to parse requests per second: %v", err) } - b.ReportMetric(requests, "requests_per_second") + ReportCustomMetric(b, requests, "requests_per_second" /*metric name*/, "QPS" /*unit*/) ave, err := h.parseAverageLatency(output) if err != nil { b.Fatalf("failed to parse average latency: %v", err) } - b.ReportMetric(ave, "average_latency_secs") + ReportCustomMetric(b, ave, "average_latency" /*metric name*/, "s" /*unit*/) } var heyReqPerSecondRE = regexp.MustCompile(`Requests/sec:\s*(\d+\.?\d+?)\s+`) diff --git a/test/benchmarks/tools/iperf.go b/test/benchmarks/tools/iperf.go index df3d9349b..5c4e7125b 100644 --- a/test/benchmarks/tools/iperf.go +++ b/test/benchmarks/tools/iperf.go @@ -42,7 +42,7 @@ func (i *Iperf) Report(b *testing.B, output string) { if err != nil { b.Fatalf("failed to parse bandwitdth from %s: %v", output, err) } - b.ReportMetric(bW*1024, "bandwidth_b/s") // Convert from Kb/s to b/s. + ReportCustomMetric(b, bW*1024, "bandwidth" /*metric name*/, "bytes_per_second" /*unit*/) } // bandwidth parses the Bandwidth number from an iperf report. A sample is below. diff --git a/test/benchmarks/tools/meminfo.go b/test/benchmarks/tools/meminfo.go index 2414a96a7..b5786fe11 100644 --- a/test/benchmarks/tools/meminfo.go +++ b/test/benchmarks/tools/meminfo.go @@ -45,7 +45,7 @@ func (*Meminfo) Report(b *testing.B, before, after string) { b.Fatalf("could not parse before value %s: %v", before, err) } val := 1024 * ((beforeVal - afterVal) / float64(b.N)) - b.ReportMetric(val, "average_container_size_bytes") + ReportCustomMetric(b, val, "average_container_size" /*metric name*/, "bytes" /*units*/) } var memInfoRE = regexp.MustCompile(`MemAvailable:\s*(\d+)\skB\n`) diff --git a/test/benchmarks/tools/parser_util.go b/test/benchmarks/tools/parser_util.go new file mode 100644 index 000000000..a4555c7dd --- /dev/null +++ b/test/benchmarks/tools/parser_util.go @@ -0,0 +1,101 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tools + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "testing" +) + +// Parameter is a test parameter. +type Parameter struct { + Name string + Value string +} + +// Output is parsed and split by these values. Make them illegal in input methods. +// We are constrained on what characters these can be by 1) docker's allowable +// container names, 2) golang allowable benchmark names, and 3) golangs allowable +// charecters in b.ReportMetric calls. +var illegalChars = regexp.MustCompile(`[/\.]`) + +// ParametersToName joins parameters into a string format for parsing. +// It is meant to be used for t.Run() calls in benchmark tools. +func ParametersToName(params ...Parameter) (string, error) { + var strs []string + for _, param := range params { + if illegalChars.MatchString(param.Name) || illegalChars.MatchString(param.Value) { + return "", fmt.Errorf("params Name: %q and Value: %q cannot container '.' or '/'", param.Name, param.Value) + } + strs = append(strs, strings.Join([]string{param.Name, param.Value}, ".")) + } + return strings.Join(strs, "/"), nil +} + +// NameToParameters parses the string created by ParametersToName and returns +// it as a set of Parameters. +// Example: BenchmarkRuby/server_threads.1/doc_size.16KB-6 +// The parameter part of this benchmark is: +// "server_threads.1/doc_size.16KB" (BenchmarkRuby is the name, and 6 is GOMAXPROCS) +// This function will return a slice with two parameters -> +// {Name: server_threads, Value: 1}, {Name: doc_size, Value: 16KB} +func NameToParameters(name string) ([]*Parameter, error) { + var params []*Parameter + for _, cond := range strings.Split(name, "/") { + cs := strings.Split(cond, ".") + switch len(cs) { + case 1: + params = append(params, &Parameter{Name: cond, Value: cond}) + case 2: + params = append(params, &Parameter{Name: cs[0], Value: cs[1]}) + default: + return nil, fmt.Errorf("failed to parse param: %s", cond) + } + } + return params, nil +} + +// ReportCustomMetric reports a metric in a set format for parsing. +func ReportCustomMetric(b *testing.B, value float64, name, unit string) { + if illegalChars.MatchString(name) || illegalChars.MatchString(unit) { + b.Fatalf("name: %q and unit: %q cannot contain '/' or '.'", name, unit) + } + nameUnit := strings.Join([]string{name, unit}, ".") + b.ReportMetric(value, nameUnit) +} + +// Metric holds metric data parsed from a string based on the format +// ReportMetric. +type Metric struct { + Name string + Unit string + Sample float64 +} + +// ParseCustomMetric parses a metric reported with ReportCustomMetric. +func ParseCustomMetric(value, metric string) (*Metric, error) { + sample, err := strconv.ParseFloat(value, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse value: %v", err) + } + nameUnit := strings.Split(metric, ".") + if len(nameUnit) != 2 { + return nil, fmt.Errorf("failed to parse metric: %s", metric) + } + return &Metric{Name: nameUnit[0], Unit: nameUnit[1], Sample: sample}, nil +} diff --git a/test/benchmarks/tools/redis.go b/test/benchmarks/tools/redis.go index c899ae0d4..e35886437 100644 --- a/test/benchmarks/tools/redis.go +++ b/test/benchmarks/tools/redis.go @@ -49,7 +49,7 @@ func (r *Redis) Report(b *testing.B, output string) { if err != nil { b.Fatalf("parsing result %s failed with err: %v", output, err) } - b.ReportMetric(result, r.Operation) // operations per second + ReportCustomMetric(b, result, r.Operation /*metric_name*/, "QPS" /*unit*/) } // parseOperation grabs the metric operations per second from redis-benchmark output. diff --git a/test/benchmarks/tools/sysbench.go b/test/benchmarks/tools/sysbench.go index 6b2f75ca2..7ccacd8ff 100644 --- a/test/benchmarks/tools/sysbench.go +++ b/test/benchmarks/tools/sysbench.go @@ -80,7 +80,7 @@ func (s *SysbenchCPU) Report(b *testing.B, output string) { if err != nil { b.Fatalf("parsing CPU events from %s failed: %v", output, err) } - b.ReportMetric(result, "cpu_events_per_second") + ReportCustomMetric(b, result, "cpu_events" /*metric name*/, "events_per_second" /*unit*/) } var cpuEventsPerSecondRE = regexp.MustCompile(`events per second:\s*(\d*.?\d*)\n`) @@ -144,7 +144,7 @@ func (s *SysbenchMemory) Report(b *testing.B, output string) { if err != nil { b.Fatalf("parsing result %s failed with err: %v", output, err) } - b.ReportMetric(result, "operations_per_second") + ReportCustomMetric(b, result, "memory_operations" /*metric name*/, "ops_per_second" /*unit*/) } var memoryOperationsRE = regexp.MustCompile(`Total\soperations:\s+\d*\s*\((\d*\.\d*)\sper\ssecond\)`) @@ -198,19 +198,19 @@ func (s *SysbenchMutex) Report(b *testing.B, output string) { if err != nil { b.Fatalf("parsing result %s failed with err: %v", output, err) } - b.ReportMetric(result, "average_execution_time_secs") + ReportCustomMetric(b, result, "average_execution_time" /*metric name*/, "s" /*unit*/) result, err = s.parseDeviation(output) if err != nil { b.Fatalf("parsing result %s failed with err: %v", output, err) } - b.ReportMetric(result, "stdev_execution_time_secs") + ReportCustomMetric(b, result, "stddev_execution_time" /*metric name*/, "s" /*unit*/) result, err = s.parseLatency(output) if err != nil { b.Fatalf("parsing result %s failed with err: %v", output, err) } - b.ReportMetric(result/1000, "average_latency_secs") + ReportCustomMetric(b, result/1000, "average_latency" /*metric name*/, "s" /*unit*/) } var executionTimeRE = regexp.MustCompile(`execution time \(avg/stddev\):\s*(\d*.?\d*)/(\d*.?\d*)`) diff --git a/tools/bigquery/BUILD b/tools/bigquery/BUILD index 5748fb390..2b0062a63 100644 --- a/tools/bigquery/BUILD +++ b/tools/bigquery/BUILD @@ -6,5 +6,8 @@ go_library( name = "bigquery", testonly = 1, srcs = ["bigquery.go"], + visibility = [ + "//:sandbox", + ], deps = ["@com_google_cloud_go_bigquery//:go_default_library"], ) diff --git a/tools/bigquery/bigquery.go b/tools/bigquery/bigquery.go index 56f0dc5c9..5f1a882de 100644 --- a/tools/bigquery/bigquery.go +++ b/tools/bigquery/bigquery.go @@ -30,11 +30,20 @@ import ( // Benchmark is the top level structure of recorded benchmark data. BigQuery // will infer the schema from this. type Benchmark struct { - Name string `bq:"name"` - Timestamp time.Time `bq:"timestamp"` - Official bool `bq:"official"` - Metric []*Metric `bq:"metric"` - Metadata *Metadata `bq:"metadata"` + Name string `bq:"name"` + Condition []*Condition `bq:"condition"` + Timestamp time.Time `bq:"timestamp"` + Official bool `bq:"official"` + Metric []*Metric `bq:"metric"` + Metadata *Metadata `bq:"metadata"` +} + +// Condition represents qualifiers for the benchmark. For example: +// Get_Pid/1/real_time would have Benchmark Name "Get_Pid" with "1" +// and "real_time" parameters as conditions. +type Condition struct { + Name string `bq:"name"` + Value string `bq:"value"` } // Metric holds the actual metric data and unit information for this benchmark. @@ -79,6 +88,14 @@ func InitBigQuery(ctx context.Context, projectID, datasetID, tableID string) err return nil } +// AddCondition adds a condition to an existing Benchmark. +func (bm *Benchmark) AddCondition(name, value string) { + bm.Condition = append(bm.Condition, &Condition{ + Name: name, + Value: value, + }) +} + // AddMetric adds a metric to an existing Benchmark. func (bm *Benchmark) AddMetric(metricName, unit string, sample float64) { m := &Metric{ @@ -90,7 +107,7 @@ func (bm *Benchmark) AddMetric(metricName, unit string, sample float64) { } // NewBenchmark initializes a new benchmark. -func NewBenchmark(name string, official bool) *Benchmark { +func NewBenchmark(name string, iters int, official bool) *Benchmark { return &Benchmark{ Name: name, Timestamp: time.Now().UTC(), @@ -103,7 +120,7 @@ func NewBenchmark(name string, official bool) *Benchmark { func SendBenchmarks(ctx context.Context, benchmarks []*Benchmark, projectID, datasetID, tableID string) error { client, err := bq.NewClient(ctx, projectID) if err != nil { - return fmt.Errorf("Failed to initialize client on project: %s: %v", projectID, err) + return fmt.Errorf("failed to initialize client on project: %s: %v", projectID, err) } defer client.Close() diff --git a/tools/parsers/BUILD b/tools/parsers/BUILD new file mode 100644 index 000000000..7d9c9a3fb --- /dev/null +++ b/tools/parsers/BUILD @@ -0,0 +1,27 @@ +load("//tools:defs.bzl", "go_library", "go_test") + +package(licenses = ["notice"]) + +go_test( + name = "parsers_test", + size = "small", + srcs = ["go_parser_test.go"], + library = ":parsers", + deps = [ + "//tools/bigquery", + "@com_github_google_go_cmp//cmp:go_default_library", + ], +) + +go_library( + name = "parsers", + testonly = 1, + srcs = [ + "go_parser.go", + ], + visibility = ["//:sandbox"], + deps = [ + "//test/benchmarks/tools", + "//tools/bigquery", + ], +) diff --git a/tools/parsers/go_parser.go b/tools/parsers/go_parser.go new file mode 100644 index 000000000..2cf74c883 --- /dev/null +++ b/tools/parsers/go_parser.go @@ -0,0 +1,151 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package parsers holds parsers to parse Benchmark test output. +// +// Parsers parse Benchmark test output and place it in BigQuery +// structs for sending to BigQuery databases. +package parsers + +import ( + "fmt" + "strconv" + "strings" + + "gvisor.dev/gvisor/test/benchmarks/tools" + "gvisor.dev/gvisor/tools/bigquery" +) + +// parseOutput expects golang benchmark output returns a Benchmark struct formatted for BigQuery. +func parseOutput(output string, metadata *bigquery.Metadata, official bool) ([]*bigquery.Benchmark, error) { + var benchmarks []*bigquery.Benchmark + lines := strings.Split(output, "\n") + for _, line := range lines { + bm, err := parseLine(line, metadata, official) + if err != nil { + return nil, fmt.Errorf("failed to parse line '%s': %v", line, err) + } + if bm != nil { + benchmarks = append(benchmarks, bm) + } + } + return benchmarks, nil +} + +// parseLine handles parsing a benchmark line into a bigquery.Benchmark. +// +// Example: "BenchmarkRuby/server_threads.1-6 1 1397875880 ns/op 140 requests_per_second.QPS" +// +// This function will return the following benchmark: +// *bigquery.Benchmark{ +// Name: BenchmarkRuby +// []*bigquery.Condition{ +// {Name: GOMAXPROCS, 6} +// {Name: server_threads, 1} +// } +// []*bigquery.Metric{ +// {Name: ns/op, Unit: ns/op, Sample: 1397875880} +// {Name: requests_per_second, Unit: QPS, Sample: 140 } +// } +// Metadata: metadata +//} +func parseLine(line string, metadata *bigquery.Metadata, official bool) (*bigquery.Benchmark, error) { + fields := strings.Fields(line) + + // Check if this line is a Benchmark line. Otherwise ignore the line. + if len(fields) < 2 || !strings.HasPrefix(fields[0], "Benchmark") { + return nil, nil + } + + iters, err := strconv.Atoi(fields[1]) + if err != nil { + return nil, fmt.Errorf("expecting number of runs, got %s: %v", fields[1], err) + } + + name, params, err := parseNameParams(fields[0]) + if err != nil { + return nil, fmt.Errorf("parse name/params: %v", err) + } + + bm := bigquery.NewBenchmark(name, iters, official) + bm.Metadata = metadata + for _, p := range params { + bm.AddCondition(p.Name, p.Value) + } + + for i := 1; i < len(fields)/2; i++ { + value := fields[2*i] + metric := fields[2*i+1] + if err := makeMetric(bm, value, metric); err != nil { + return nil, fmt.Errorf("makeMetric on metric %q value: %s: %v", metric, value, err) + } + } + return bm, nil +} + +// parseNameParams parses the Name, GOMAXPROCS, and Params from the test. +// Field here should be of the format TESTNAME/PARAMS-GOMAXPROCS. +// Parameters will be separated by a "/" with individual params being +// "name.value". +func parseNameParams(field string) (string, []*tools.Parameter, error) { + var params []*tools.Parameter + // Remove GOMAXPROCS from end. + maxIndex := strings.LastIndex(field, "-") + if maxIndex < 0 { + return "", nil, fmt.Errorf("GOMAXPROCS not found: %s", field) + } + maxProcs := field[maxIndex+1:] + params = append(params, &tools.Parameter{ + Name: "GOMAXPROCS", + Value: maxProcs, + }) + + remainder := field[0:maxIndex] + index := strings.Index(remainder, "/") + if index == -1 { + return remainder, params, nil + } + + name := remainder[0:index] + p := remainder[index+1:] + + ps, err := tools.NameToParameters(p) + if err != nil { + return "", nil, fmt.Errorf("NameToParameters %s: %v", field, err) + } + params = append(params, ps...) + return name, params, nil +} + +// makeMetric parses metrics and adds them to the passed Benchmark. +func makeMetric(bm *bigquery.Benchmark, value, metric string) error { + switch metric { + // Ignore most output from golang benchmarks. + case "MB/s", "B/op", "allocs/op": + return nil + case "ns/op": + val, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("ParseFloat %s: %v", value, err) + } + bm.AddMetric(metric /*metric name*/, metric /*unit*/, val /*sample*/) + default: + m, err := tools.ParseCustomMetric(value, metric) + if err != nil { + return fmt.Errorf("ParseCustomMetric %s: %v ", metric, err) + } + bm.AddMetric(m.Name, m.Unit, m.Sample) + } + return nil +} diff --git a/tools/parsers/go_parser_test.go b/tools/parsers/go_parser_test.go new file mode 100644 index 000000000..36996b7c8 --- /dev/null +++ b/tools/parsers/go_parser_test.go @@ -0,0 +1,171 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parsers + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "gvisor.dev/gvisor/tools/bigquery" +) + +func TestParseLine(t *testing.T) { + testCases := []struct { + name string + data string + want *bigquery.Benchmark + }{ + { + name: "Iperf", + data: "BenchmarkIperf/Upload-6 1 11094914892 ns/op 4751711232 bandwidth.bytes_per_second", + want: &bigquery.Benchmark{ + Name: "BenchmarkIperf", + Condition: []*bigquery.Condition{ + { + Name: "GOMAXPROCS", + Value: "6", + }, + { + Name: "Upload", + Value: "Upload", + }, + }, + Metric: []*bigquery.Metric{ + { + Name: "ns/op", + Unit: "ns/op", + Sample: 11094914892.0, + }, + { + Name: "bandwidth", + Unit: "bytes_per_second", + Sample: 4751711232.0, + }, + }, + }, + }, + { + name: "Ruby", + data: "BenchmarkRuby/server_threads.1-6 1 1397875880 ns/op 0.00710 average_latency.s 140 requests_per_second.QPS", + want: &bigquery.Benchmark{ + Name: "BenchmarkRuby", + Condition: []*bigquery.Condition{ + { + Name: "GOMAXPROCS", + Value: "6", + }, + { + Name: "server_threads", + Value: "1", + }, + }, + Metric: []*bigquery.Metric{ + { + Name: "ns/op", + Unit: "ns/op", + Sample: 1397875880.0, + }, + { + Name: "average_latency", + Unit: "s", + Sample: 0.00710, + }, + { + Name: "requests_per_second", + Unit: "QPS", + Sample: 140.0, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := parseLine(tc.data, nil, false) + if err != nil { + t.Fatalf("parseLine failed with: %v", err) + } + + tc.want.Timestamp = got.Timestamp + + if !cmp.Equal(tc.want, got, nil) { + for _, c := range got.Condition { + t.Logf("Cond: %+v", c) + } + for _, m := range got.Metric { + t.Logf("Metric: %+v", m) + } + t.Fatalf("Compare failed want: %+v got: %+v", tc.want, got) + } + }) + + } +} + +func TestParseOutput(t *testing.T) { + testCases := []struct { + name string + data string + numBenchmarks int + numMetrics int + numConditions int + }{ + { + name: "Startup", + data: ` + BenchmarkStartupEmpty + BenchmarkStartupEmpty-6 2 766377884 ns/op 1 allocs/op + BenchmarkStartupNode + BenchmarkStartupNode-6 1 1752158409 ns/op 1 allocs/op + `, + numBenchmarks: 2, + numMetrics: 1, + numConditions: 1, + }, + { + name: "Ruby", + data: `BenchmarkRuby +BenchmarkRuby/server_threads.1 +BenchmarkRuby/server_threads.1-6 1 1397875880 ns/op 0.00710 average_latency.s 140 requests_per_second.QPS +BenchmarkRuby/server_threads.5 +BenchmarkRuby/server_threads.5-6 1 1416003331 ns/op 0.00950 average_latency.s 465 requests_per_second.QPS`, + numBenchmarks: 2, + numMetrics: 3, + numConditions: 2, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + bms, err := parseOutput(tc.data, nil, false) + if err != nil { + t.Fatalf("parseOutput failed: %v", err) + } else if len(bms) != tc.numBenchmarks { + t.Fatalf("NumBenchmarks failed want: %d got: %d %+v", tc.numBenchmarks, len(bms), bms) + } + + for _, bm := range bms { + if len(bm.Metric) != tc.numMetrics { + t.Fatalf("NumMetrics failed want: %d got: %d %+v", tc.numMetrics, len(bm.Metric), bm.Metric) + } + + if len(bm.Condition) != tc.numConditions { + t.Fatalf("NumConditions failed want: %d got: %d %+v", tc.numConditions, len(bm.Condition), bm.Condition) + } + } + }) + } +} -- cgit v1.2.3