summaryrefslogtreecommitdiffhomepage
path: root/tools/parsers
diff options
context:
space:
mode:
Diffstat (limited to 'tools/parsers')
-rw-r--r--tools/parsers/BUILD41
-rw-r--r--tools/parsers/go_parser.go150
-rw-r--r--tools/parsers/go_parser_test.go169
-rw-r--r--tools/parsers/parser_main.go129
4 files changed, 489 insertions, 0 deletions
diff --git a/tools/parsers/BUILD b/tools/parsers/BUILD
new file mode 100644
index 000000000..dab954e25
--- /dev/null
+++ b/tools/parsers/BUILD
@@ -0,0 +1,41 @@
+load("//tools:defs.bzl", "go_binary", "go_library", "go_test")
+
+package(licenses = ["notice"])
+
+go_test(
+ name = "parsers_test",
+ size = "small",
+ srcs = ["go_parser_test.go"],
+ library = ":parsers",
+ nogo = False,
+ deps = [
+ "//tools/bigquery",
+ "@com_github_google_go_cmp//cmp:go_default_library",
+ ],
+)
+
+go_library(
+ name = "parsers",
+ testonly = 1,
+ srcs = [
+ "go_parser.go",
+ ],
+ nogo = False,
+ visibility = ["//:sandbox"],
+ deps = [
+ "//test/benchmarks/tools",
+ "//tools/bigquery",
+ ],
+)
+
+go_binary(
+ name = "parser",
+ testonly = 1,
+ srcs = ["parser_main.go"],
+ nogo = False,
+ deps = [
+ ":parsers",
+ "//runsc/flag",
+ "//tools/bigquery",
+ ],
+)
diff --git a/tools/parsers/go_parser.go b/tools/parsers/go_parser.go
new file mode 100644
index 000000000..df4875e6a
--- /dev/null
+++ b/tools/parsers/go_parser.go
@@ -0,0 +1,150 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package parsers holds parsers to parse Benchmark test output.
+//
+// Parsers parse Benchmark test output and place it in BigQuery
+// structs for sending to BigQuery databases.
+package parsers
+
+import (
+ "fmt"
+ "strconv"
+ "strings"
+
+ "gvisor.dev/gvisor/test/benchmarks/tools"
+ "gvisor.dev/gvisor/tools/bigquery"
+)
+
+// ParseOutput expects golang benchmark output and returns a struct formatted
+// for BigQuery.
+func ParseOutput(output string, name string, official bool) (*bigquery.Suite, error) {
+ suite := bigquery.NewSuite(name)
+ lines := strings.Split(output, "\n")
+ for _, line := range lines {
+ bm, err := parseLine(line, official)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse line '%s': %v", line, err)
+ }
+ if bm != nil {
+ suite.Benchmarks = append(suite.Benchmarks, bm)
+ }
+ }
+ return suite, nil
+}
+
+// parseLine handles parsing a benchmark line into a bigquery.Benchmark.
+//
+// Example: "BenchmarkRuby/server_threads.1-6 1 1397875880 ns/op 140 requests_per_second.QPS"
+//
+// This function will return the following benchmark:
+// *bigquery.Benchmark{
+// Name: BenchmarkRuby
+// []*bigquery.Condition{
+// {Name: GOMAXPROCS, 6}
+// {Name: server_threads, 1}
+// }
+// []*bigquery.Metric{
+// {Name: ns/op, Unit: ns/op, Sample: 1397875880}
+// {Name: requests_per_second, Unit: QPS, Sample: 140 }
+// }
+//}
+func parseLine(line string, official bool) (*bigquery.Benchmark, error) {
+ fields := strings.Fields(line)
+
+ // Check if this line is a Benchmark line. Otherwise ignore the line.
+ if len(fields) < 2 || !strings.HasPrefix(fields[0], "Benchmark") {
+ return nil, nil
+ }
+
+ iters, err := strconv.Atoi(fields[1])
+ if err != nil {
+ return nil, fmt.Errorf("expecting number of runs, got %s: %v", fields[1], err)
+ }
+
+ name, params, err := parseNameParams(fields[0])
+ if err != nil {
+ return nil, fmt.Errorf("parse name/params: %v", err)
+ }
+
+ bm := bigquery.NewBenchmark(name, iters, official)
+ for _, p := range params {
+ bm.AddCondition(p.Name, p.Value)
+ }
+
+ for i := 1; i < len(fields)/2; i++ {
+ value := fields[2*i]
+ metric := fields[2*i+1]
+ if err := makeMetric(bm, value, metric); err != nil {
+ return nil, fmt.Errorf("makeMetric on metric %q value: %s: %v", metric, value, err)
+ }
+ }
+ return bm, nil
+}
+
+// parseNameParams parses the Name, GOMAXPROCS, and Params from the test.
+// Field here should be of the format TESTNAME/PARAMS-GOMAXPROCS.
+// Parameters will be separated by a "/" with individual params being
+// "name.value".
+func parseNameParams(field string) (string, []*tools.Parameter, error) {
+ var params []*tools.Parameter
+ // Remove GOMAXPROCS from end.
+ maxIndex := strings.LastIndex(field, "-")
+ if maxIndex < 0 {
+ return "", nil, fmt.Errorf("GOMAXPROCS not found: %s", field)
+ }
+ maxProcs := field[maxIndex+1:]
+ params = append(params, &tools.Parameter{
+ Name: "GOMAXPROCS",
+ Value: maxProcs,
+ })
+
+ remainder := field[0:maxIndex]
+ index := strings.Index(remainder, "/")
+ if index == -1 {
+ return remainder, params, nil
+ }
+
+ name := remainder[0:index]
+ p := remainder[index+1:]
+
+ ps, err := tools.NameToParameters(p)
+ if err != nil {
+ return "", nil, fmt.Errorf("NameToParameters %s: %v", field, err)
+ }
+ params = append(params, ps...)
+ return name, params, nil
+}
+
+// makeMetric parses metrics and adds them to the passed Benchmark.
+func makeMetric(bm *bigquery.Benchmark, value, metric string) error {
+ switch metric {
+ // Ignore most output from golang benchmarks.
+ case "MB/s", "B/op", "allocs/op":
+ return nil
+ case "ns/op":
+ val, err := strconv.ParseFloat(value, 64)
+ if err != nil {
+ return fmt.Errorf("ParseFloat %s: %v", value, err)
+ }
+ bm.AddMetric(metric /*metric name*/, metric /*unit*/, val /*sample*/)
+ default:
+ m, err := tools.ParseCustomMetric(value, metric)
+ if err != nil {
+ return fmt.Errorf("ParseCustomMetric %s: %v ", metric, err)
+ }
+ bm.AddMetric(m.Name, m.Unit, m.Sample)
+ }
+ return nil
+}
diff --git a/tools/parsers/go_parser_test.go b/tools/parsers/go_parser_test.go
new file mode 100644
index 000000000..0aa1152a2
--- /dev/null
+++ b/tools/parsers/go_parser_test.go
@@ -0,0 +1,169 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parsers
+
+import (
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "gvisor.dev/gvisor/tools/bigquery"
+)
+
+func TestParseLine(t *testing.T) {
+ testCases := []struct {
+ name string
+ data string
+ want *bigquery.Benchmark
+ }{
+ {
+ name: "Iperf",
+ data: "BenchmarkIperf/Upload-6 1 11094914892 ns/op 4751711232 bandwidth.bytes_per_second",
+ want: &bigquery.Benchmark{
+ Name: "BenchmarkIperf",
+ Condition: []*bigquery.Condition{
+ {
+ Name: "GOMAXPROCS",
+ Value: "6",
+ },
+ {
+ Name: "Upload",
+ Value: "Upload",
+ },
+ },
+ Metric: []*bigquery.Metric{
+ {
+ Name: "ns/op",
+ Unit: "ns/op",
+ Sample: 11094914892.0,
+ },
+ {
+ Name: "bandwidth",
+ Unit: "bytes_per_second",
+ Sample: 4751711232.0,
+ },
+ },
+ },
+ },
+ {
+ name: "Ruby",
+ data: "BenchmarkRuby/server_threads.1-6 1 1397875880 ns/op 0.00710 average_latency.s 140 requests_per_second.QPS",
+ want: &bigquery.Benchmark{
+ Name: "BenchmarkRuby",
+ Condition: []*bigquery.Condition{
+ {
+ Name: "GOMAXPROCS",
+ Value: "6",
+ },
+ {
+ Name: "server_threads",
+ Value: "1",
+ },
+ },
+ Metric: []*bigquery.Metric{
+ {
+ Name: "ns/op",
+ Unit: "ns/op",
+ Sample: 1397875880.0,
+ },
+ {
+ Name: "average_latency",
+ Unit: "s",
+ Sample: 0.00710,
+ },
+ {
+ Name: "requests_per_second",
+ Unit: "QPS",
+ Sample: 140.0,
+ },
+ },
+ },
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := parseLine(tc.data, false)
+ if err != nil {
+ t.Fatalf("parseLine failed with: %v", err)
+ }
+
+ if !cmp.Equal(tc.want, got, nil) {
+ for _, c := range got.Condition {
+ t.Logf("Cond: %+v", c)
+ }
+ for _, m := range got.Metric {
+ t.Logf("Metric: %+v", m)
+ }
+ t.Fatalf("Compare failed want: %+v got: %+v", tc.want, got)
+ }
+ })
+
+ }
+}
+
+func TestParseOutput(t *testing.T) {
+ testCases := []struct {
+ name string
+ data string
+ numBenchmarks int
+ numMetrics int
+ numConditions int
+ }{
+ {
+ name: "Startup",
+ data: `
+ BenchmarkStartupEmpty
+ BenchmarkStartupEmpty-6 2 766377884 ns/op 1 allocs/op
+ BenchmarkStartupNode
+ BenchmarkStartupNode-6 1 1752158409 ns/op 1 allocs/op
+ `,
+ numBenchmarks: 2,
+ numMetrics: 1,
+ numConditions: 1,
+ },
+ {
+ name: "Ruby",
+ data: `BenchmarkRuby
+BenchmarkRuby/server_threads.1
+BenchmarkRuby/server_threads.1-6 1 1397875880 ns/op 0.00710 average_latency.s 140 requests_per_second.QPS
+BenchmarkRuby/server_threads.5
+BenchmarkRuby/server_threads.5-6 1 1416003331 ns/op 0.00950 average_latency.s 465 requests_per_second.QPS`,
+ numBenchmarks: 2,
+ numMetrics: 3,
+ numConditions: 2,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ suite, err := ParseOutput(tc.data, "", false)
+ if err != nil {
+ t.Fatalf("parseOutput failed: %v", err)
+ } else if len(suite.Benchmarks) != tc.numBenchmarks {
+ t.Fatalf("NumBenchmarks failed want: %d got: %d %+v", tc.numBenchmarks, len(suite.Benchmarks), suite.Benchmarks)
+ }
+
+ for _, bm := range suite.Benchmarks {
+ if len(bm.Metric) != tc.numMetrics {
+ t.Fatalf("NumMetrics failed want: %d got: %d %+v", tc.numMetrics, len(bm.Metric), bm.Metric)
+ }
+
+ if len(bm.Condition) != tc.numConditions {
+ t.Fatalf("NumConditions failed want: %d got: %d %+v", tc.numConditions, len(bm.Condition), bm.Condition)
+ }
+ }
+ })
+ }
+}
diff --git a/tools/parsers/parser_main.go b/tools/parsers/parser_main.go
new file mode 100644
index 000000000..6c6182464
--- /dev/null
+++ b/tools/parsers/parser_main.go
@@ -0,0 +1,129 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Binary parser parses Benchmark data from golang benchmarks,
+// puts it into a Schema for BigQuery, and sends it to BigQuery.
+// parser will also initialize a table with the Benchmarks BigQuery schema.
+package main
+
+import (
+ "context"
+ "fmt"
+ "io/ioutil"
+ "os"
+
+ "gvisor.dev/gvisor/runsc/flag"
+ bq "gvisor.dev/gvisor/tools/bigquery"
+ "gvisor.dev/gvisor/tools/parsers"
+)
+
+const (
+ initString = "init"
+ initDescription = "initializes a new table with benchmarks schema"
+ parseString = "parse"
+ parseDescription = "parses given benchmarks file and sends it to BigQuery table."
+)
+
+var (
+ // The init command will create a new dataset/table in the given project and initialize
+ // the table with the schema in //tools/bigquery/bigquery.go. If the table/dataset exists
+ // or has been initialized, init has no effect and successfully returns.
+ initCmd = flag.NewFlagSet(initString, flag.ContinueOnError)
+ initProject = initCmd.String("project", "", "GCP project to send benchmarks.")
+ initDataset = initCmd.String("dataset", "", "dataset to send benchmarks data.")
+ initTable = initCmd.String("table", "", "table to send benchmarks data.")
+
+ // The parse command parses benchmark data in `file` and sends it to the
+ // requested table.
+ parseCmd = flag.NewFlagSet(parseString, flag.ContinueOnError)
+ file = parseCmd.String("file", "", "file to parse for benchmarks")
+ name = parseCmd.String("suite_name", "", "name of the benchmark suite")
+ clNumber = parseCmd.String("cl", "", "changelist number of this run")
+ gitCommit = parseCmd.String("git_commit", "", "git commit sha for this run")
+ parseProject = parseCmd.String("project", "", "GCP project to send benchmarks.")
+ parseDataset = parseCmd.String("dataset", "", "dataset to send benchmarks data.")
+ parseTable = parseCmd.String("table", "", "table to send benchmarks data.")
+ official = parseCmd.Bool("official", false, "mark input data as official.")
+)
+
+// initBenchmarks initializes a dataset/table in a BigQuery project.
+func initBenchmarks(ctx context.Context) error {
+ return bq.InitBigQuery(ctx, *initProject, *initDataset, *initTable, nil)
+}
+
+// parseBenchmarks parses the given file into the BigQuery schema,
+// adds some custom data for the commit, and sends the data to BigQuery.
+func parseBenchmarks(ctx context.Context) error {
+ data, err := ioutil.ReadFile(*file)
+ if err != nil {
+ return fmt.Errorf("failed to read file: %v", err)
+ }
+ suite, err := parsers.ParseOutput(string(data), *name, *official)
+ if err != nil {
+ return fmt.Errorf("failed parse data: %v", err)
+ }
+ extraConditions := []*bq.Condition{
+ {
+ Name: "change_list",
+ Value: *clNumber,
+ },
+ {
+ Name: "commit",
+ Value: *gitCommit,
+ },
+ }
+
+ suite.Conditions = append(suite.Conditions, extraConditions...)
+ return bq.SendBenchmarks(ctx, suite, *parseProject, *parseDataset, *parseTable, nil)
+}
+
+func main() {
+ ctx := context.Background()
+ switch {
+ // the "init" command
+ case len(os.Args) >= 2 && os.Args[1] == initString:
+ if err := initCmd.Parse(os.Args[2:]); err != nil {
+ fmt.Fprintf(os.Stderr, "failed parse flags: %v", err)
+ os.Exit(1)
+ }
+ if err := initBenchmarks(ctx); err != nil {
+ failure := "failed to initialize project: %s dataset: %s table: %s: %v"
+ fmt.Fprintf(os.Stderr, failure, *parseProject, *parseDataset, *parseTable, err)
+ os.Exit(1)
+ }
+ // the "parse" command.
+ case len(os.Args) >= 2 && os.Args[1] == parseString:
+ if err := parseCmd.Parse(os.Args[2:]); err != nil {
+ fmt.Fprintf(os.Stderr, "failed parse flags: %v", err)
+ os.Exit(1)
+ }
+ if err := parseBenchmarks(ctx); err != nil {
+ fmt.Fprintf(os.Stderr, "failed parse benchmarks: %v", err)
+ os.Exit(1)
+ }
+ default:
+ printUsage()
+ }
+}
+
+// printUsage prints the top level usage string.
+func printUsage() {
+ usage := `Usage: parser <command> <flags> ...
+
+Available commands:
+ %s %s
+ %s %s
+`
+ fmt.Fprintf(os.Stderr, usage, initCmd.Name(), initDescription, parseCmd.Name(), parseDescription)
+}
generated by cgit v1.2.3 (git 2.39.1) at 2025-04-11 09:14:57 +0000