diff options
| -rw-r--r-- | Makefile | 55 | ||||
| -rw-r--r-- | tools/bigquery/bigquery.go | 5 | ||||
| -rw-r--r-- | tools/parsers/BUILD | 6 | ||||
| -rw-r--r-- | tools/parsers/go_parser.go | 8 | ||||
| -rw-r--r-- | tools/parsers/go_parser_test.go | 2 | ||||
| -rw-r--r-- | tools/parsers/parser_main.go | 28 | ||||
| -rw-r--r-- | tools/parsers/version.go | 18 |
7 files changed, 103 insertions, 19 deletions
@@ -249,6 +249,61 @@ containerd-tests: containerd-test-1.3.4 containerd-tests: containerd-test-1.4.0-beta.0 ## +## Benchmarks. +## +## Targets to run benchmarks. See //test/benchmarks for details. +## +## common arguments: +## RUNTIME_ARGS - arguments to runsc placed in /etc/docker/daemon.json +## e.g. "--platform=ptrace" +## BENCHMARKS_PROJECT - BigQuery project to which to send data. +## BENCHMARKS_DATASET - BigQuery dataset to which to send data. +## BENCHMARKS_TABLE - BigQuery table to which to send data. +## BENCHMARKS_SUITE - name of the benchmark suite. See //tools/bigquery/bigquery.go. +## BENCHMARKS_UPLOAD - if true, upload benchmark data from the run. +## BENCHMARKS_OFFICIAL - marks the data as official. +## BENCHMARKS_PLATFORMS - platforms to run benchmarks (e.g. ptrace kvm). +## +RUNTIME_ARGS := --net-raw --platform=ptrace +BENCHMARKS_PROJECT := gvisor-benchmarks +BENCHMARKS_DATASET := kokoro +BENCHMARKS_TABLE := benchmarks +BENCHMARKS_SUITE := start +BENCHMARKS_UPLOAD := false +BENCHMARKS_OFFICIAL := false +BENCHMARKS_PLATFORMS := ptrace + +init-benchmark-table: ## Initializes a BigQuery table with the benchmark schema +## (see //tools/bigquery/bigquery.go). If the table alread exists, this is a noop. + $(call submake, run TARGETS=//tools/parsers:parser ARGS="init --project=$(BENCHMARKS_PROJECT) \ + --dataset=$(BENCHMARKS_DATASET) --table=$(BENCHMARKS_TABLE)") +.PHONY: init-benchmark-table + +benchmark-platforms: ## Runs benchmarks for runc and all given platforms in BENCHMARK_PLATFORMS. + $(call submake, run-benchmark RUNTIME="runc") + $(foreach PLATFORM,$(BENCHMARKS_PLATFORMS),\ + $(call submake,benchmark-platform RUNTIME="$(PLATFORM)" RUNTIME_ARGS="--platform=$(PLATFORM) --net-raw --vfs2") && \ + $(call submake,benchmark-platform RUNTIME="$(PLATFORM)_vfs1" RUNTIME_ARGS="--platform=$(PLATFORM) --net-raw")) +.PHONY: benchmark-platforms + +benchmark-platform: ## Installs a runtime with the given platform args. + @$(call submake,install-test-runtime ARGS="$(RUNTIME_ARGS)") + @$(call submake, run-benchmark) +.PHONY: benchmark-platform + +run-benchmark: ## Runs single benchmark and optionally sends data to BigQuery. + $(eval T := $(shell mktemp /tmp/logs.$(RUNTIME).XXXXXX)) + $(call submake,sudo TARGETS="$(TARGETS)" ARGS="--runtime=$(RUNTIME) $(ARGS)" | tee $(T)) + @if [[ "$(BENCHMARKS_UPLOAD)" == "true" ]]; then \ + @$(call submake,run TARGETS=tools/parsers:parser ARGS="parse --file=$(T) \ + --runtime=$(RUNTIME) --suite_name=$(BENCHMARKS_SUITE) \ + --project=$(BENCHMARKS_PROJECT) --dataset=$(BENCHMARKS_DATASET) \ + --table=$(BENCHMARKS_TABLE) --official=$(BENCHMARKS_OFFICIAL)"); \ + fi; + rm -rf $T +.PHONY: run-benchmark + +## ## Website & documentation helpers. ## ## The website is built from repository documentation and wrappers, using diff --git a/tools/bigquery/bigquery.go b/tools/bigquery/bigquery.go index 34b270cc0..544af3876 100644 --- a/tools/bigquery/bigquery.go +++ b/tools/bigquery/bigquery.go @@ -105,7 +105,7 @@ func (bm *Benchmark) AddMetric(metricName, unit string, sample float64) { } // NewBenchmark initializes a new benchmark. -func NewBenchmark(name string, iters int, official bool) *Benchmark { +func NewBenchmark(name string, iters int) *Benchmark { return &Benchmark{ Name: name, Metric: make([]*Metric, 0), @@ -113,12 +113,13 @@ func NewBenchmark(name string, iters int, official bool) *Benchmark { } // NewSuite initializes a new Suite. -func NewSuite(name string) *Suite { +func NewSuite(name string, official bool) *Suite { return &Suite{ Name: name, Timestamp: time.Now().UTC(), Benchmarks: make([]*Benchmark, 0), Conditions: make([]*Condition, 0), + Official: official, } } diff --git a/tools/parsers/BUILD b/tools/parsers/BUILD index dab954e25..6932bba9a 100644 --- a/tools/parsers/BUILD +++ b/tools/parsers/BUILD @@ -31,8 +31,12 @@ go_library( go_binary( name = "parser", testonly = 1, - srcs = ["parser_main.go"], + srcs = [ + "parser_main.go", + "version.go", + ], nogo = False, + x_defs = {"main.version": "{STABLE_VERSION}"}, deps = [ ":parsers", "//runsc/flag", diff --git a/tools/parsers/go_parser.go b/tools/parsers/go_parser.go index df4875e6a..57e538149 100644 --- a/tools/parsers/go_parser.go +++ b/tools/parsers/go_parser.go @@ -30,10 +30,10 @@ import ( // ParseOutput expects golang benchmark output and returns a struct formatted // for BigQuery. func ParseOutput(output string, name string, official bool) (*bigquery.Suite, error) { - suite := bigquery.NewSuite(name) + suite := bigquery.NewSuite(name, official) lines := strings.Split(output, "\n") for _, line := range lines { - bm, err := parseLine(line, official) + bm, err := parseLine(line) if err != nil { return nil, fmt.Errorf("failed to parse line '%s': %v", line, err) } @@ -60,7 +60,7 @@ func ParseOutput(output string, name string, official bool) (*bigquery.Suite, er // {Name: requests_per_second, Unit: QPS, Sample: 140 } // } //} -func parseLine(line string, official bool) (*bigquery.Benchmark, error) { +func parseLine(line string) (*bigquery.Benchmark, error) { fields := strings.Fields(line) // Check if this line is a Benchmark line. Otherwise ignore the line. @@ -78,7 +78,7 @@ func parseLine(line string, official bool) (*bigquery.Benchmark, error) { return nil, fmt.Errorf("parse name/params: %v", err) } - bm := bigquery.NewBenchmark(name, iters, official) + bm := bigquery.NewBenchmark(name, iters) for _, p := range params { bm.AddCondition(p.Name, p.Value) } diff --git a/tools/parsers/go_parser_test.go b/tools/parsers/go_parser_test.go index 0aa1152a2..f0737d46b 100644 --- a/tools/parsers/go_parser_test.go +++ b/tools/parsers/go_parser_test.go @@ -94,7 +94,7 @@ func TestParseLine(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - got, err := parseLine(tc.data, false) + got, err := parseLine(tc.data) if err != nil { t.Fatalf("parseLine failed with: %v", err) } diff --git a/tools/parsers/parser_main.go b/tools/parsers/parser_main.go index 6c6182464..7cce69e03 100644 --- a/tools/parsers/parser_main.go +++ b/tools/parsers/parser_main.go @@ -49,12 +49,11 @@ var ( parseCmd = flag.NewFlagSet(parseString, flag.ContinueOnError) file = parseCmd.String("file", "", "file to parse for benchmarks") name = parseCmd.String("suite_name", "", "name of the benchmark suite") - clNumber = parseCmd.String("cl", "", "changelist number of this run") - gitCommit = parseCmd.String("git_commit", "", "git commit sha for this run") parseProject = parseCmd.String("project", "", "GCP project to send benchmarks.") parseDataset = parseCmd.String("dataset", "", "dataset to send benchmarks data.") parseTable = parseCmd.String("table", "", "table to send benchmarks data.") official = parseCmd.Bool("official", false, "mark input data as official.") + runtime = parseCmd.String("runtime", "", "runtime used to run the benchmark") ) // initBenchmarks initializes a dataset/table in a BigQuery project. @@ -67,23 +66,29 @@ func initBenchmarks(ctx context.Context) error { func parseBenchmarks(ctx context.Context) error { data, err := ioutil.ReadFile(*file) if err != nil { - return fmt.Errorf("failed to read file: %v", err) + return fmt.Errorf("failed to read file %s: %v", *file, err) } suite, err := parsers.ParseOutput(string(data), *name, *official) if err != nil { return fmt.Errorf("failed parse data: %v", err) } + if len(suite.Benchmarks) < 1 { + fmt.Fprintf(os.Stderr, "Failed to find benchmarks for file: %s", *file) + return nil + } + extraConditions := []*bq.Condition{ { - Name: "change_list", - Value: *clNumber, + Name: "runtime", + Value: *runtime, }, { - Name: "commit", - Value: *gitCommit, + Name: "version", + Value: version, }, } + suite.Official = *official suite.Conditions = append(suite.Conditions, extraConditions...) return bq.SendBenchmarks(ctx, suite, *parseProject, *parseDataset, *parseTable, nil) } @@ -94,26 +99,27 @@ func main() { // the "init" command case len(os.Args) >= 2 && os.Args[1] == initString: if err := initCmd.Parse(os.Args[2:]); err != nil { - fmt.Fprintf(os.Stderr, "failed parse flags: %v", err) + fmt.Fprintf(os.Stderr, "failed parse flags: %v\n", err) os.Exit(1) } if err := initBenchmarks(ctx); err != nil { - failure := "failed to initialize project: %s dataset: %s table: %s: %v" + failure := "failed to initialize project: %s dataset: %s table: %s: %v\n" fmt.Fprintf(os.Stderr, failure, *parseProject, *parseDataset, *parseTable, err) os.Exit(1) } // the "parse" command. case len(os.Args) >= 2 && os.Args[1] == parseString: if err := parseCmd.Parse(os.Args[2:]); err != nil { - fmt.Fprintf(os.Stderr, "failed parse flags: %v", err) + fmt.Fprintf(os.Stderr, "failed parse flags: %v\n", err) os.Exit(1) } if err := parseBenchmarks(ctx); err != nil { - fmt.Fprintf(os.Stderr, "failed parse benchmarks: %v", err) + fmt.Fprintf(os.Stderr, "failed parse benchmarks: %v\n", err) os.Exit(1) } default: printUsage() + os.Exit(1) } } diff --git a/tools/parsers/version.go b/tools/parsers/version.go new file mode 100644 index 000000000..ab9194b9d --- /dev/null +++ b/tools/parsers/version.go @@ -0,0 +1,18 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +// version is set during linking. +var version = "VERSION_MISSING" |