diff options
-rw-r--r-- | test/benchmarks/base/sysbench_test.go | 11 | ||||
-rw-r--r-- | test/benchmarks/database/redis_test.go | 10 | ||||
-rw-r--r-- | test/benchmarks/fs/bazel_test.go | 32 | ||||
-rw-r--r-- | test/benchmarks/fs/fio_test.go | 15 | ||||
-rw-r--r-- | test/benchmarks/network/httpd_test.go | 46 | ||||
-rw-r--r-- | test/benchmarks/network/nginx_test.go | 53 | ||||
-rw-r--r-- | test/benchmarks/network/node_test.go | 12 | ||||
-rw-r--r-- | test/benchmarks/network/ruby_test.go | 11 | ||||
-rw-r--r-- | test/benchmarks/tools/BUILD | 2 | ||||
-rw-r--r-- | test/benchmarks/tools/ab.go | 3 | ||||
-rw-r--r-- | test/benchmarks/tools/fio.go | 4 | ||||
-rw-r--r-- | test/benchmarks/tools/hey.go | 4 | ||||
-rw-r--r-- | test/benchmarks/tools/iperf.go | 2 | ||||
-rw-r--r-- | test/benchmarks/tools/meminfo.go | 2 | ||||
-rw-r--r-- | test/benchmarks/tools/parser_util.go | 101 | ||||
-rw-r--r-- | test/benchmarks/tools/redis.go | 2 | ||||
-rw-r--r-- | test/benchmarks/tools/sysbench.go | 10 | ||||
-rw-r--r-- | tools/bigquery/BUILD | 3 | ||||
-rw-r--r-- | tools/bigquery/bigquery.go | 31 | ||||
-rw-r--r-- | tools/parsers/BUILD | 27 | ||||
-rw-r--r-- | tools/parsers/go_parser.go | 151 | ||||
-rw-r--r-- | tools/parsers/go_parser_test.go | 171 |
22 files changed, 614 insertions, 89 deletions
diff --git a/test/benchmarks/base/sysbench_test.go b/test/benchmarks/base/sysbench_test.go index 6fb813640..39ced3dab 100644 --- a/test/benchmarks/base/sysbench_test.go +++ b/test/benchmarks/base/sysbench_test.go @@ -71,8 +71,15 @@ func BenchmarkSysbench(b *testing.B) { defer machine.CleanUp() for _, tc := range testCases { - b.Run(tc.name, func(b *testing.B) { - + param := tools.Parameter{ + Name: "testname", + Value: tc.name, + } + name, err := tools.ParametersToName(param) + if err != nil { + b.Fatalf("Failed to parse params: %v", err) + } + b.Run(name, func(b *testing.B) { ctx := context.Background() sysbench := machine.GetContainer(ctx, b) defer sysbench.CleanUp(ctx) diff --git a/test/benchmarks/database/redis_test.go b/test/benchmarks/database/redis_test.go index 6671a4969..02e67154e 100644 --- a/test/benchmarks/database/redis_test.go +++ b/test/benchmarks/database/redis_test.go @@ -66,7 +66,15 @@ func BenchmarkRedis(b *testing.B) { ctx := context.Background() for _, operation := range operations { - b.Run(operation, func(b *testing.B) { + param := tools.Parameter{ + Name: "operation", + Value: operation, + } + name, err := tools.ParametersToName(param) + if err != nil { + b.Fatalf("Failed to parse paramaters: %v", err) + } + b.Run(name, func(b *testing.B) { server := serverMachine.GetContainer(ctx, b) defer server.CleanUp(ctx) diff --git a/test/benchmarks/fs/bazel_test.go b/test/benchmarks/fs/bazel_test.go index ef1b8e4ea..56103639d 100644 --- a/test/benchmarks/fs/bazel_test.go +++ b/test/benchmarks/fs/bazel_test.go @@ -21,6 +21,7 @@ import ( "gvisor.dev/gvisor/pkg/test/dockerutil" "gvisor.dev/gvisor/test/benchmarks/harness" + "gvisor.dev/gvisor/test/benchmarks/tools" ) // Note: CleanCache versions of this test require running with root permissions. @@ -46,17 +47,36 @@ func runBuildBenchmark(b *testing.B, image, workdir, target string) { // Dimensions here are clean/dirty cache (do or don't drop caches) // and if the mount on which we are compiling is a tmpfs/bind mount. benchmarks := []struct { - name string clearCache bool // clearCache drops caches before running. tmpfs bool // tmpfs will run compilation on a tmpfs. }{ - {name: "CleanCache", clearCache: true, tmpfs: false}, - {name: "DirtyCache", clearCache: false, tmpfs: false}, - {name: "CleanCacheTmpfs", clearCache: true, tmpfs: true}, - {name: "DirtyCacheTmpfs", clearCache: false, tmpfs: true}, + {clearCache: true, tmpfs: false}, + {clearCache: false, tmpfs: false}, + {clearCache: true, tmpfs: true}, + {clearCache: false, tmpfs: true}, } for _, bm := range benchmarks { - b.Run(bm.name, func(b *testing.B) { + pageCache := tools.Parameter{ + Name: "page_cache", + Value: "clean", + } + if bm.clearCache { + pageCache.Value = "dirty" + } + + filesystem := tools.Parameter{ + Name: "filesystem", + Value: "bind", + } + if bm.tmpfs { + filesystem.Value = "tmpfs" + } + name, err := tools.ParametersToName(pageCache, filesystem) + if err != nil { + b.Fatalf("Failed to parse parameters: %v", err) + } + + b.Run(name, func(b *testing.B) { // Grab a container. ctx := context.Background() container := machine.GetContainer(ctx, b) diff --git a/test/benchmarks/fs/fio_test.go b/test/benchmarks/fs/fio_test.go index 65874ed8b..5ca191404 100644 --- a/test/benchmarks/fs/fio_test.go +++ b/test/benchmarks/fs/fio_test.go @@ -67,8 +67,19 @@ func BenchmarkFio(b *testing.B) { for _, fsType := range []mount.Type{mount.TypeBind, mount.TypeTmpfs} { for _, tc := range testCases { - testName := strings.Title(tc.Test) + strings.Title(string(fsType)) - b.Run(testName, func(b *testing.B) { + operation := tools.Parameter{ + Name: "operation", + Value: tc.Test, + } + filesystem := tools.Parameter{ + Name: "filesystem", + Value: string(fsType), + } + name, err := tools.ParametersToName(operation, filesystem) + if err != nil { + b.Fatalf("Failed to parser paramters: %v", err) + } + b.Run(name, func(b *testing.B) { ctx := context.Background() container := machine.GetContainer(ctx, b) defer container.CleanUp(ctx) diff --git a/test/benchmarks/network/httpd_test.go b/test/benchmarks/network/httpd_test.go index 369ab326e..8d7d5f750 100644 --- a/test/benchmarks/network/httpd_test.go +++ b/test/benchmarks/network/httpd_test.go @@ -14,7 +14,7 @@ package network import ( - "fmt" + "strconv" "testing" "gvisor.dev/gvisor/pkg/test/dockerutil" @@ -31,33 +31,15 @@ var httpdDocs = map[string]string{ "10Mb": "latin10240k.txt", } -// BenchmarkHttpdConcurrency iterates the concurrency argument and tests -// how well the runtime under test handles requests in parallel. -func BenchmarkHttpdConcurrency(b *testing.B) { - // The test iterates over client concurrency, so set other parameters. - concurrency := []int{1, 25, 50, 100, 1000} - - for _, c := range concurrency { - b.Run(fmt.Sprintf("%d", c), func(b *testing.B) { - hey := &tools.Hey{ - Requests: c * b.N, - Concurrency: c, - Doc: httpdDocs["10Kb"], - } - runHttpd(b, hey, false /* reverse */) - }) - } -} - -// BenchmarkHttpdDocSize iterates over different sized payloads, testing how -// well the runtime handles sending different payload sizes. -func BenchmarkHttpdDocSize(b *testing.B) { +// BenchmarkHttpd iterates over different sized payloads and concurrency, testing +// how well the runtime handles sending different payload sizes. +func BenchmarkHttpd(b *testing.B) { benchmarkHttpdDocSize(b, false /* reverse */) } -// BenchmarkReverseHttpdDocSize iterates over different sized payloads, testing +// BenchmarkReverseHttpd iterates over different sized payloads, testing // how well the runtime handles receiving different payload sizes. -func BenchmarkReverseHttpdDocSize(b *testing.B) { +func BenchmarkReverseHttpd(b *testing.B) { benchmarkHttpdDocSize(b, true /* reverse */) } @@ -65,10 +47,22 @@ func BenchmarkReverseHttpdDocSize(b *testing.B) { // for each size. func benchmarkHttpdDocSize(b *testing.B, reverse bool) { b.Helper() - for name, filename := range httpdDocs { + for size, filename := range httpdDocs { concurrency := []int{1, 25, 50, 100, 1000} for _, c := range concurrency { - b.Run(fmt.Sprintf("%s_%d", name, c), func(b *testing.B) { + fsize := tools.Parameter{ + Name: "filesize", + Value: size, + } + concurrency := tools.Parameter{ + Name: "concurrency", + Value: strconv.Itoa(c), + } + name, err := tools.ParametersToName(fsize, concurrency) + if err != nil { + b.Fatalf("Failed to parse parameters: %v", err) + } + b.Run(name, func(b *testing.B) { hey := &tools.Hey{ Requests: c * b.N, Concurrency: c, diff --git a/test/benchmarks/network/nginx_test.go b/test/benchmarks/network/nginx_test.go index 9ec70369b..08565d0b2 100644 --- a/test/benchmarks/network/nginx_test.go +++ b/test/benchmarks/network/nginx_test.go @@ -14,7 +14,7 @@ package network import ( - "fmt" + "strconv" "testing" "gvisor.dev/gvisor/pkg/test/dockerutil" @@ -31,30 +31,6 @@ var nginxDocs = map[string]string{ "10Mb": "latin10240k.txt", } -// BenchmarkNginxConcurrency iterates the concurrency argument and tests -// how well the runtime under test handles requests in parallel. -func BenchmarkNginxConcurrency(b *testing.B) { - concurrency := []int{1, 25, 100, 1000} - for _, c := range concurrency { - for _, tmpfs := range []bool{true, false} { - fs := "Gofer" - if tmpfs { - fs = "Tmpfs" - } - name := fmt.Sprintf("%d_%s", c, fs) - b.Run(name, func(b *testing.B) { - hey := &tools.Hey{ - Requests: c * b.N, - Concurrency: c, - Doc: nginxDocs["10kb"], // see Dockerfile '//images/benchmarks/nginx' and httpd_test. - } - runNginx(b, hey, false /* reverse */, tmpfs /* tmpfs */) - }) - } - - } -} - // BenchmarkNginxDocSize iterates over different sized payloads, testing how // well the runtime handles sending different payload sizes. func BenchmarkNginxDocSize(b *testing.B) { @@ -71,15 +47,32 @@ func BenchmarkReverseNginxDocSize(b *testing.B) { // benchmarkNginxDocSize iterates through all doc sizes, running subbenchmarks // for each size. func benchmarkNginxDocSize(b *testing.B, reverse, tmpfs bool) { - for name, filename := range nginxDocs { + for size, filename := range nginxDocs { concurrency := []int{1, 25, 50, 100, 1000} for _, c := range concurrency { - fs := "Gofer" + fsize := tools.Parameter{ + Name: "filesize", + Value: size, + } + + threads := tools.Parameter{ + Name: "concurrency", + Value: strconv.Itoa(c), + } + + fs := tools.Parameter{ + Name: "filesystem", + Value: "bind", + } if tmpfs { - fs = "Tmpfs" + fs.Value = "tmpfs" + } + name, err := tools.ParametersToName(fsize, threads, fs) + if err != nil { + b.Fatalf("Failed to parse parameters: %v", err) } - benchName := fmt.Sprintf("%s_%d_%s", name, c, fs) - b.Run(benchName, func(b *testing.B) { + + b.Run(name, func(b *testing.B) { hey := &tools.Hey{ Requests: c * b.N, Concurrency: c, diff --git a/test/benchmarks/network/node_test.go b/test/benchmarks/network/node_test.go index 0f4a205b6..254538899 100644 --- a/test/benchmarks/network/node_test.go +++ b/test/benchmarks/network/node_test.go @@ -15,7 +15,7 @@ package network import ( "context" - "fmt" + "strconv" "testing" "time" @@ -31,7 +31,15 @@ import ( func BenchmarkNode(b *testing.B) { concurrency := []int{1, 5, 10, 25} for _, c := range concurrency { - b.Run(fmt.Sprintf("Concurrency%d", c), func(b *testing.B) { + param := tools.Parameter{ + Name: "concurrency", + Value: strconv.Itoa(c), + } + name, err := tools.ParametersToName(param) + if err != nil { + b.Fatalf("Failed to parse parameters: %v", err) + } + b.Run(name, func(b *testing.B) { hey := &tools.Hey{ Requests: b.N * c, // Requests b.N requests per thread. Concurrency: c, diff --git a/test/benchmarks/network/ruby_test.go b/test/benchmarks/network/ruby_test.go index 67f63f76a..0174ff3f3 100644 --- a/test/benchmarks/network/ruby_test.go +++ b/test/benchmarks/network/ruby_test.go @@ -16,6 +16,7 @@ package network import ( "context" "fmt" + "strconv" "testing" "time" @@ -31,7 +32,15 @@ import ( func BenchmarkRuby(b *testing.B) { concurrency := []int{1, 5, 10, 25} for _, c := range concurrency { - b.Run(fmt.Sprintf("Concurrency%d", c), func(b *testing.B) { + param := tools.Parameter{ + Name: "concurrency", + Value: strconv.Itoa(c), + } + name, err := tools.ParametersToName(param) + if err != nil { + b.Fatalf("Failed to parse parameters: %v", err) + } + b.Run(name, func(b *testing.B) { hey := &tools.Hey{ Requests: b.N * c, // b.N requests per thread. Concurrency: c, diff --git a/test/benchmarks/tools/BUILD b/test/benchmarks/tools/BUILD index e5734d85c..9290830d7 100644 --- a/test/benchmarks/tools/BUILD +++ b/test/benchmarks/tools/BUILD @@ -4,12 +4,14 @@ package(licenses = ["notice"]) go_library( name = "tools", + testonly = 1, srcs = [ "ab.go", "fio.go", "hey.go", "iperf.go", "meminfo.go", + "parser_util.go", "redis.go", "sysbench.go", "tools.go", diff --git a/test/benchmarks/tools/ab.go b/test/benchmarks/tools/ab.go index 4cc9c3bce..d9abf0763 100644 --- a/test/benchmarks/tools/ab.go +++ b/test/benchmarks/tools/ab.go @@ -46,18 +46,21 @@ func (a *ApacheBench) Report(b *testing.B, output string) { b.Logf("failed to parse transferrate: %v", err) } b.ReportMetric(transferRate*1024, "transfer_rate_b/s") // Convert from Kb/s to b/s. + ReportCustomMetric(b, transferRate*1024, "transfer_rate" /*metric name*/, "bytes_per_second" /*unit*/) latency, err := a.parseLatency(output) if err != nil { b.Logf("failed to parse latency: %v", err) } b.ReportMetric(latency/1000, "mean_latency_secs") // Convert from ms to s. + ReportCustomMetric(b, latency/1000, "mean_latency" /*metric name*/, "s" /*unit*/) reqPerSecond, err := a.parseRequestsPerSecond(output) if err != nil { b.Logf("failed to parse requests per second: %v", err) } b.ReportMetric(reqPerSecond, "requests_per_second") + ReportCustomMetric(b, reqPerSecond, "requests_per_second" /*metric name*/, "QPS" /*unit*/) } var transferRateRE = regexp.MustCompile(`Transfer rate:\s+(\d+\.?\d+?)\s+\[Kbytes/sec\]\s+received`) diff --git a/test/benchmarks/tools/fio.go b/test/benchmarks/tools/fio.go index 20000db16..f5f60fa84 100644 --- a/test/benchmarks/tools/fio.go +++ b/test/benchmarks/tools/fio.go @@ -56,13 +56,13 @@ func (f *Fio) Report(b *testing.B, output string) { if err != nil { b.Fatalf("failed to parse bandwidth from %s with: %v", output, err) } - b.ReportMetric(bw, "bandwidth_b/s") // in b/s. + ReportCustomMetric(b, bw, "bandwidth" /*metric name*/, "bytes_per_second" /*unit*/) iops, err := f.parseIOps(output, isRead) if err != nil { b.Fatalf("failed to parse iops from %s with: %v", output, err) } - b.ReportMetric(iops, "iops") + ReportCustomMetric(b, iops, "io_ops" /*metric name*/, "ops_per_second" /*unit*/) } // parseBandwidth reports the bandwidth in b/s. diff --git a/test/benchmarks/tools/hey.go b/test/benchmarks/tools/hey.go index b1e20e356..b8cb938fe 100644 --- a/test/benchmarks/tools/hey.go +++ b/test/benchmarks/tools/hey.go @@ -43,13 +43,13 @@ func (h *Hey) Report(b *testing.B, output string) { if err != nil { b.Fatalf("failed to parse requests per second: %v", err) } - b.ReportMetric(requests, "requests_per_second") + ReportCustomMetric(b, requests, "requests_per_second" /*metric name*/, "QPS" /*unit*/) ave, err := h.parseAverageLatency(output) if err != nil { b.Fatalf("failed to parse average latency: %v", err) } - b.ReportMetric(ave, "average_latency_secs") + ReportCustomMetric(b, ave, "average_latency" /*metric name*/, "s" /*unit*/) } var heyReqPerSecondRE = regexp.MustCompile(`Requests/sec:\s*(\d+\.?\d+?)\s+`) diff --git a/test/benchmarks/tools/iperf.go b/test/benchmarks/tools/iperf.go index df3d9349b..5c4e7125b 100644 --- a/test/benchmarks/tools/iperf.go +++ b/test/benchmarks/tools/iperf.go @@ -42,7 +42,7 @@ func (i *Iperf) Report(b *testing.B, output string) { if err != nil { b.Fatalf("failed to parse bandwitdth from %s: %v", output, err) } - b.ReportMetric(bW*1024, "bandwidth_b/s") // Convert from Kb/s to b/s. + ReportCustomMetric(b, bW*1024, "bandwidth" /*metric name*/, "bytes_per_second" /*unit*/) } // bandwidth parses the Bandwidth number from an iperf report. A sample is below. diff --git a/test/benchmarks/tools/meminfo.go b/test/benchmarks/tools/meminfo.go index 2414a96a7..b5786fe11 100644 --- a/test/benchmarks/tools/meminfo.go +++ b/test/benchmarks/tools/meminfo.go @@ -45,7 +45,7 @@ func (*Meminfo) Report(b *testing.B, before, after string) { b.Fatalf("could not parse before value %s: %v", before, err) } val := 1024 * ((beforeVal - afterVal) / float64(b.N)) - b.ReportMetric(val, "average_container_size_bytes") + ReportCustomMetric(b, val, "average_container_size" /*metric name*/, "bytes" /*units*/) } var memInfoRE = regexp.MustCompile(`MemAvailable:\s*(\d+)\skB\n`) diff --git a/test/benchmarks/tools/parser_util.go b/test/benchmarks/tools/parser_util.go new file mode 100644 index 000000000..a4555c7dd --- /dev/null +++ b/test/benchmarks/tools/parser_util.go @@ -0,0 +1,101 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tools + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "testing" +) + +// Parameter is a test parameter. +type Parameter struct { + Name string + Value string +} + +// Output is parsed and split by these values. Make them illegal in input methods. +// We are constrained on what characters these can be by 1) docker's allowable +// container names, 2) golang allowable benchmark names, and 3) golangs allowable +// charecters in b.ReportMetric calls. +var illegalChars = regexp.MustCompile(`[/\.]`) + +// ParametersToName joins parameters into a string format for parsing. +// It is meant to be used for t.Run() calls in benchmark tools. +func ParametersToName(params ...Parameter) (string, error) { + var strs []string + for _, param := range params { + if illegalChars.MatchString(param.Name) || illegalChars.MatchString(param.Value) { + return "", fmt.Errorf("params Name: %q and Value: %q cannot container '.' or '/'", param.Name, param.Value) + } + strs = append(strs, strings.Join([]string{param.Name, param.Value}, ".")) + } + return strings.Join(strs, "/"), nil +} + +// NameToParameters parses the string created by ParametersToName and returns +// it as a set of Parameters. +// Example: BenchmarkRuby/server_threads.1/doc_size.16KB-6 +// The parameter part of this benchmark is: +// "server_threads.1/doc_size.16KB" (BenchmarkRuby is the name, and 6 is GOMAXPROCS) +// This function will return a slice with two parameters -> +// {Name: server_threads, Value: 1}, {Name: doc_size, Value: 16KB} +func NameToParameters(name string) ([]*Parameter, error) { + var params []*Parameter + for _, cond := range strings.Split(name, "/") { + cs := strings.Split(cond, ".") + switch len(cs) { + case 1: + params = append(params, &Parameter{Name: cond, Value: cond}) + case 2: + params = append(params, &Parameter{Name: cs[0], Value: cs[1]}) + default: + return nil, fmt.Errorf("failed to parse param: %s", cond) + } + } + return params, nil +} + +// ReportCustomMetric reports a metric in a set format for parsing. +func ReportCustomMetric(b *testing.B, value float64, name, unit string) { + if illegalChars.MatchString(name) || illegalChars.MatchString(unit) { + b.Fatalf("name: %q and unit: %q cannot contain '/' or '.'", name, unit) + } + nameUnit := strings.Join([]string{name, unit}, ".") + b.ReportMetric(value, nameUnit) +} + +// Metric holds metric data parsed from a string based on the format +// ReportMetric. +type Metric struct { + Name string + Unit string + Sample float64 +} + +// ParseCustomMetric parses a metric reported with ReportCustomMetric. +func ParseCustomMetric(value, metric string) (*Metric, error) { + sample, err := strconv.ParseFloat(value, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse value: %v", err) + } + nameUnit := strings.Split(metric, ".") + if len(nameUnit) != 2 { + return nil, fmt.Errorf("failed to parse metric: %s", metric) + } + return &Metric{Name: nameUnit[0], Unit: nameUnit[1], Sample: sample}, nil +} diff --git a/test/benchmarks/tools/redis.go b/test/benchmarks/tools/redis.go index c899ae0d4..e35886437 100644 --- a/test/benchmarks/tools/redis.go +++ b/test/benchmarks/tools/redis.go @@ -49,7 +49,7 @@ func (r *Redis) Report(b *testing.B, output string) { if err != nil { b.Fatalf("parsing result %s failed with err: %v", output, err) } - b.ReportMetric(result, r.Operation) // operations per second + ReportCustomMetric(b, result, r.Operation /*metric_name*/, "QPS" /*unit*/) } // parseOperation grabs the metric operations per second from redis-benchmark output. diff --git a/test/benchmarks/tools/sysbench.go b/test/benchmarks/tools/sysbench.go index 6b2f75ca2..7ccacd8ff 100644 --- a/test/benchmarks/tools/sysbench.go +++ b/test/benchmarks/tools/sysbench.go @@ -80,7 +80,7 @@ func (s *SysbenchCPU) Report(b *testing.B, output string) { if err != nil { b.Fatalf("parsing CPU events from %s failed: %v", output, err) } - b.ReportMetric(result, "cpu_events_per_second") + ReportCustomMetric(b, result, "cpu_events" /*metric name*/, "events_per_second" /*unit*/) } var cpuEventsPerSecondRE = regexp.MustCompile(`events per second:\s*(\d*.?\d*)\n`) @@ -144,7 +144,7 @@ func (s *SysbenchMemory) Report(b *testing.B, output string) { if err != nil { b.Fatalf("parsing result %s failed with err: %v", output, err) } - b.ReportMetric(result, "operations_per_second") + ReportCustomMetric(b, result, "memory_operations" /*metric name*/, "ops_per_second" /*unit*/) } var memoryOperationsRE = regexp.MustCompile(`Total\soperations:\s+\d*\s*\((\d*\.\d*)\sper\ssecond\)`) @@ -198,19 +198,19 @@ func (s *SysbenchMutex) Report(b *testing.B, output string) { if err != nil { b.Fatalf("parsing result %s failed with err: %v", output, err) } - b.ReportMetric(result, "average_execution_time_secs") + ReportCustomMetric(b, result, "average_execution_time" /*metric name*/, "s" /*unit*/) result, err = s.parseDeviation(output) if err != nil { b.Fatalf("parsing result %s failed with err: %v", output, err) } - b.ReportMetric(result, "stdev_execution_time_secs") + ReportCustomMetric(b, result, "stddev_execution_time" /*metric name*/, "s" /*unit*/) result, err = s.parseLatency(output) if err != nil { b.Fatalf("parsing result %s failed with err: %v", output, err) } - b.ReportMetric(result/1000, "average_latency_secs") + ReportCustomMetric(b, result/1000, "average_latency" /*metric name*/, "s" /*unit*/) } var executionTimeRE = regexp.MustCompile(`execution time \(avg/stddev\):\s*(\d*.?\d*)/(\d*.?\d*)`) diff --git a/tools/bigquery/BUILD b/tools/bigquery/BUILD index 5748fb390..2b0062a63 100644 --- a/tools/bigquery/BUILD +++ b/tools/bigquery/BUILD @@ -6,5 +6,8 @@ go_library( name = "bigquery", testonly = 1, srcs = ["bigquery.go"], + visibility = [ + "//:sandbox", + ], deps = ["@com_google_cloud_go_bigquery//:go_default_library"], ) diff --git a/tools/bigquery/bigquery.go b/tools/bigquery/bigquery.go index 56f0dc5c9..5f1a882de 100644 --- a/tools/bigquery/bigquery.go +++ b/tools/bigquery/bigquery.go @@ -30,11 +30,20 @@ import ( // Benchmark is the top level structure of recorded benchmark data. BigQuery // will infer the schema from this. type Benchmark struct { - Name string `bq:"name"` - Timestamp time.Time `bq:"timestamp"` - Official bool `bq:"official"` - Metric []*Metric `bq:"metric"` - Metadata *Metadata `bq:"metadata"` + Name string `bq:"name"` + Condition []*Condition `bq:"condition"` + Timestamp time.Time `bq:"timestamp"` + Official bool `bq:"official"` + Metric []*Metric `bq:"metric"` + Metadata *Metadata `bq:"metadata"` +} + +// Condition represents qualifiers for the benchmark. For example: +// Get_Pid/1/real_time would have Benchmark Name "Get_Pid" with "1" +// and "real_time" parameters as conditions. +type Condition struct { + Name string `bq:"name"` + Value string `bq:"value"` } // Metric holds the actual metric data and unit information for this benchmark. @@ -79,6 +88,14 @@ func InitBigQuery(ctx context.Context, projectID, datasetID, tableID string) err return nil } +// AddCondition adds a condition to an existing Benchmark. +func (bm *Benchmark) AddCondition(name, value string) { + bm.Condition = append(bm.Condition, &Condition{ + Name: name, + Value: value, + }) +} + // AddMetric adds a metric to an existing Benchmark. func (bm *Benchmark) AddMetric(metricName, unit string, sample float64) { m := &Metric{ @@ -90,7 +107,7 @@ func (bm *Benchmark) AddMetric(metricName, unit string, sample float64) { } // NewBenchmark initializes a new benchmark. -func NewBenchmark(name string, official bool) *Benchmark { +func NewBenchmark(name string, iters int, official bool) *Benchmark { return &Benchmark{ Name: name, Timestamp: time.Now().UTC(), @@ -103,7 +120,7 @@ func NewBenchmark(name string, official bool) *Benchmark { func SendBenchmarks(ctx context.Context, benchmarks []*Benchmark, projectID, datasetID, tableID string) error { client, err := bq.NewClient(ctx, projectID) if err != nil { - return fmt.Errorf("Failed to initialize client on project: %s: %v", projectID, err) + return fmt.Errorf("failed to initialize client on project: %s: %v", projectID, err) } defer client.Close() diff --git a/tools/parsers/BUILD b/tools/parsers/BUILD new file mode 100644 index 000000000..7d9c9a3fb --- /dev/null +++ b/tools/parsers/BUILD @@ -0,0 +1,27 @@ +load("//tools:defs.bzl", "go_library", "go_test") + +package(licenses = ["notice"]) + +go_test( + name = "parsers_test", + size = "small", + srcs = ["go_parser_test.go"], + library = ":parsers", + deps = [ + "//tools/bigquery", + "@com_github_google_go_cmp//cmp:go_default_library", + ], +) + +go_library( + name = "parsers", + testonly = 1, + srcs = [ + "go_parser.go", + ], + visibility = ["//:sandbox"], + deps = [ + "//test/benchmarks/tools", + "//tools/bigquery", + ], +) diff --git a/tools/parsers/go_parser.go b/tools/parsers/go_parser.go new file mode 100644 index 000000000..2cf74c883 --- /dev/null +++ b/tools/parsers/go_parser.go @@ -0,0 +1,151 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package parsers holds parsers to parse Benchmark test output. +// +// Parsers parse Benchmark test output and place it in BigQuery +// structs for sending to BigQuery databases. +package parsers + +import ( + "fmt" + "strconv" + "strings" + + "gvisor.dev/gvisor/test/benchmarks/tools" + "gvisor.dev/gvisor/tools/bigquery" +) + +// parseOutput expects golang benchmark output returns a Benchmark struct formatted for BigQuery. +func parseOutput(output string, metadata *bigquery.Metadata, official bool) ([]*bigquery.Benchmark, error) { + var benchmarks []*bigquery.Benchmark + lines := strings.Split(output, "\n") + for _, line := range lines { + bm, err := parseLine(line, metadata, official) + if err != nil { + return nil, fmt.Errorf("failed to parse line '%s': %v", line, err) + } + if bm != nil { + benchmarks = append(benchmarks, bm) + } + } + return benchmarks, nil +} + +// parseLine handles parsing a benchmark line into a bigquery.Benchmark. +// +// Example: "BenchmarkRuby/server_threads.1-6 1 1397875880 ns/op 140 requests_per_second.QPS" +// +// This function will return the following benchmark: +// *bigquery.Benchmark{ +// Name: BenchmarkRuby +// []*bigquery.Condition{ +// {Name: GOMAXPROCS, 6} +// {Name: server_threads, 1} +// } +// []*bigquery.Metric{ +// {Name: ns/op, Unit: ns/op, Sample: 1397875880} +// {Name: requests_per_second, Unit: QPS, Sample: 140 } +// } +// Metadata: metadata +//} +func parseLine(line string, metadata *bigquery.Metadata, official bool) (*bigquery.Benchmark, error) { + fields := strings.Fields(line) + + // Check if this line is a Benchmark line. Otherwise ignore the line. + if len(fields) < 2 || !strings.HasPrefix(fields[0], "Benchmark") { + return nil, nil + } + + iters, err := strconv.Atoi(fields[1]) + if err != nil { + return nil, fmt.Errorf("expecting number of runs, got %s: %v", fields[1], err) + } + + name, params, err := parseNameParams(fields[0]) + if err != nil { + return nil, fmt.Errorf("parse name/params: %v", err) + } + + bm := bigquery.NewBenchmark(name, iters, official) + bm.Metadata = metadata + for _, p := range params { + bm.AddCondition(p.Name, p.Value) + } + + for i := 1; i < len(fields)/2; i++ { + value := fields[2*i] + metric := fields[2*i+1] + if err := makeMetric(bm, value, metric); err != nil { + return nil, fmt.Errorf("makeMetric on metric %q value: %s: %v", metric, value, err) + } + } + return bm, nil +} + +// parseNameParams parses the Name, GOMAXPROCS, and Params from the test. +// Field here should be of the format TESTNAME/PARAMS-GOMAXPROCS. +// Parameters will be separated by a "/" with individual params being +// "name.value". +func parseNameParams(field string) (string, []*tools.Parameter, error) { + var params []*tools.Parameter + // Remove GOMAXPROCS from end. + maxIndex := strings.LastIndex(field, "-") + if maxIndex < 0 { + return "", nil, fmt.Errorf("GOMAXPROCS not found: %s", field) + } + maxProcs := field[maxIndex+1:] + params = append(params, &tools.Parameter{ + Name: "GOMAXPROCS", + Value: maxProcs, + }) + + remainder := field[0:maxIndex] + index := strings.Index(remainder, "/") + if index == -1 { + return remainder, params, nil + } + + name := remainder[0:index] + p := remainder[index+1:] + + ps, err := tools.NameToParameters(p) + if err != nil { + return "", nil, fmt.Errorf("NameToParameters %s: %v", field, err) + } + params = append(params, ps...) + return name, params, nil +} + +// makeMetric parses metrics and adds them to the passed Benchmark. +func makeMetric(bm *bigquery.Benchmark, value, metric string) error { + switch metric { + // Ignore most output from golang benchmarks. + case "MB/s", "B/op", "allocs/op": + return nil + case "ns/op": + val, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("ParseFloat %s: %v", value, err) + } + bm.AddMetric(metric /*metric name*/, metric /*unit*/, val /*sample*/) + default: + m, err := tools.ParseCustomMetric(value, metric) + if err != nil { + return fmt.Errorf("ParseCustomMetric %s: %v ", metric, err) + } + bm.AddMetric(m.Name, m.Unit, m.Sample) + } + return nil +} diff --git a/tools/parsers/go_parser_test.go b/tools/parsers/go_parser_test.go new file mode 100644 index 000000000..36996b7c8 --- /dev/null +++ b/tools/parsers/go_parser_test.go @@ -0,0 +1,171 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parsers + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "gvisor.dev/gvisor/tools/bigquery" +) + +func TestParseLine(t *testing.T) { + testCases := []struct { + name string + data string + want *bigquery.Benchmark + }{ + { + name: "Iperf", + data: "BenchmarkIperf/Upload-6 1 11094914892 ns/op 4751711232 bandwidth.bytes_per_second", + want: &bigquery.Benchmark{ + Name: "BenchmarkIperf", + Condition: []*bigquery.Condition{ + { + Name: "GOMAXPROCS", + Value: "6", + }, + { + Name: "Upload", + Value: "Upload", + }, + }, + Metric: []*bigquery.Metric{ + { + Name: "ns/op", + Unit: "ns/op", + Sample: 11094914892.0, + }, + { + Name: "bandwidth", + Unit: "bytes_per_second", + Sample: 4751711232.0, + }, + }, + }, + }, + { + name: "Ruby", + data: "BenchmarkRuby/server_threads.1-6 1 1397875880 ns/op 0.00710 average_latency.s 140 requests_per_second.QPS", + want: &bigquery.Benchmark{ + Name: "BenchmarkRuby", + Condition: []*bigquery.Condition{ + { + Name: "GOMAXPROCS", + Value: "6", + }, + { + Name: "server_threads", + Value: "1", + }, + }, + Metric: []*bigquery.Metric{ + { + Name: "ns/op", + Unit: "ns/op", + Sample: 1397875880.0, + }, + { + Name: "average_latency", + Unit: "s", + Sample: 0.00710, + }, + { + Name: "requests_per_second", + Unit: "QPS", + Sample: 140.0, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := parseLine(tc.data, nil, false) + if err != nil { + t.Fatalf("parseLine failed with: %v", err) + } + + tc.want.Timestamp = got.Timestamp + + if !cmp.Equal(tc.want, got, nil) { + for _, c := range got.Condition { + t.Logf("Cond: %+v", c) + } + for _, m := range got.Metric { + t.Logf("Metric: %+v", m) + } + t.Fatalf("Compare failed want: %+v got: %+v", tc.want, got) + } + }) + + } +} + +func TestParseOutput(t *testing.T) { + testCases := []struct { + name string + data string + numBenchmarks int + numMetrics int + numConditions int + }{ + { + name: "Startup", + data: ` + BenchmarkStartupEmpty + BenchmarkStartupEmpty-6 2 766377884 ns/op 1 allocs/op + BenchmarkStartupNode + BenchmarkStartupNode-6 1 1752158409 ns/op 1 allocs/op + `, + numBenchmarks: 2, + numMetrics: 1, + numConditions: 1, + }, + { + name: "Ruby", + data: `BenchmarkRuby +BenchmarkRuby/server_threads.1 +BenchmarkRuby/server_threads.1-6 1 1397875880 ns/op 0.00710 average_latency.s 140 requests_per_second.QPS +BenchmarkRuby/server_threads.5 +BenchmarkRuby/server_threads.5-6 1 1416003331 ns/op 0.00950 average_latency.s 465 requests_per_second.QPS`, + numBenchmarks: 2, + numMetrics: 3, + numConditions: 2, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + bms, err := parseOutput(tc.data, nil, false) + if err != nil { + t.Fatalf("parseOutput failed: %v", err) + } else if len(bms) != tc.numBenchmarks { + t.Fatalf("NumBenchmarks failed want: %d got: %d %+v", tc.numBenchmarks, len(bms), bms) + } + + for _, bm := range bms { + if len(bm.Metric) != tc.numMetrics { + t.Fatalf("NumMetrics failed want: %d got: %d %+v", tc.numMetrics, len(bm.Metric), bm.Metric) + } + + if len(bm.Condition) != tc.numConditions { + t.Fatalf("NumConditions failed want: %d got: %d %+v", tc.numConditions, len(bm.Condition), bm.Condition) + } + } + }) + } +} |