Internal Change

PiperOrigin-RevId: 340941898

7 files changed, 103 insertions, 19 deletions

diff --git a/Makefile b/Makefile
index f8098b539..8bd8c6c6e 100644
--- a/Makefile
+++ b/Makefile
@@ -249,6 +249,61 @@ containerd-tests: containerd-test-1.3.4
 containerd-tests: containerd-test-1.4.0-beta.0
 
 ##
+## Benchmarks.
+##
+## Targets to run benchmarks. See //test/benchmarks for details.
+##
+##   common arguments:
+##     RUNTIME_ARGS - arguments to runsc placed in /etc/docker/daemon.json
+##       e.g. "--platform=ptrace"
+##     BENCHMARKS_PROJECT - BigQuery project to which to send data.
+##     BENCHMARKS_DATASET - BigQuery dataset to which to send data.
+##     BENCHMARKS_TABLE - BigQuery table to which to send data.
+##     BENCHMARKS_SUITE - name of the benchmark suite. See //tools/bigquery/bigquery.go.
+##     BENCHMARKS_UPLOAD - if true, upload benchmark data from the run.
+##     BENCHMARKS_OFFICIAL - marks the data as official.
+##     BENCHMARKS_PLATFORMS - platforms to run benchmarks (e.g. ptrace kvm).
+##
+RUNTIME_ARGS := --net-raw --platform=ptrace
+BENCHMARKS_PROJECT := gvisor-benchmarks
+BENCHMARKS_DATASET := kokoro
+BENCHMARKS_TABLE := benchmarks
+BENCHMARKS_SUITE := start
+BENCHMARKS_UPLOAD := false
+BENCHMARKS_OFFICIAL := false
+BENCHMARKS_PLATFORMS := ptrace
+
+init-benchmark-table: ## Initializes a BigQuery table with the benchmark schema
+## (see //tools/bigquery/bigquery.go). If the table alread exists, this is a noop.
+	$(call submake, run TARGETS=//tools/parsers:parser ARGS="init --project=$(BENCHMARKS_PROJECT) \
+	--dataset=$(BENCHMARKS_DATASET) --table=$(BENCHMARKS_TABLE)")
+.PHONY: init-benchmark-table
+
+benchmark-platforms: ## Runs benchmarks for runc and all given platforms in BENCHMARK_PLATFORMS.
+	$(call submake, run-benchmark RUNTIME="runc")
+	$(foreach PLATFORM,$(BENCHMARKS_PLATFORMS),\
+	$(call submake,benchmark-platform RUNTIME="$(PLATFORM)" RUNTIME_ARGS="--platform=$(PLATFORM) --net-raw --vfs2") && \
+	$(call submake,benchmark-platform RUNTIME="$(PLATFORM)_vfs1" RUNTIME_ARGS="--platform=$(PLATFORM) --net-raw"))
+.PHONY: benchmark-platforms
+
+benchmark-platform: ## Installs a runtime with the given platform args.
+	@$(call submake,install-test-runtime ARGS="$(RUNTIME_ARGS)")
+	@$(call submake, run-benchmark)
+.PHONY: benchmark-platform
+
+run-benchmark: ## Runs single benchmark and optionally sends data to BigQuery.
+	$(eval T := $(shell mktemp /tmp/logs.$(RUNTIME).XXXXXX))
+	$(call submake,sudo TARGETS="$(TARGETS)" ARGS="--runtime=$(RUNTIME) $(ARGS)" | tee $(T))
+	@if [[ "$(BENCHMARKS_UPLOAD)" == "true" ]]; then \
+		@$(call submake,run TARGETS=tools/parsers:parser ARGS="parse --file=$(T) \
+		--runtime=$(RUNTIME) --suite_name=$(BENCHMARKS_SUITE) \
+		--project=$(BENCHMARKS_PROJECT) --dataset=$(BENCHMARKS_DATASET) \
+		--table=$(BENCHMARKS_TABLE) --official=$(BENCHMARKS_OFFICIAL)"); \
+	fi;
+	rm -rf $T
+.PHONY: run-benchmark
+
+##
 ## Website & documentation helpers.
 ##
 ##   The website is built from repository documentation and wrappers, using
diff --git a/tools/bigquery/bigquery.go b/tools/bigquery/bigquery.go
index 34b270cc0..544af3876 100644
--- a/tools/bigquery/bigquery.go
+++ b/tools/bigquery/bigquery.go
@@ -105,7 +105,7 @@ func (bm *Benchmark) AddMetric(metricName, unit string, sample float64) {
 }
 
 // NewBenchmark initializes a new benchmark.
-func NewBenchmark(name string, iters int, official bool) *Benchmark {
+func NewBenchmark(name string, iters int) *Benchmark {
 	return &Benchmark{
 		Name:   name,
 		Metric: make([]*Metric, 0),
@@ -113,12 +113,13 @@ func NewBenchmark(name string, iters int, official bool) *Benchmark {
 }
 
 // NewSuite initializes a new Suite.
-func NewSuite(name string) *Suite {
+func NewSuite(name string, official bool) *Suite {
 	return &Suite{
 		Name:       name,
 		Timestamp:  time.Now().UTC(),
 		Benchmarks: make([]*Benchmark, 0),
 		Conditions: make([]*Condition, 0),
+		Official:   official,
 	}
 }
 
diff --git a/tools/parsers/BUILD b/tools/parsers/BUILD
index dab954e25..6932bba9a 100644
--- a/tools/parsers/BUILD
+++ b/tools/parsers/BUILD
@@ -31,8 +31,12 @@ go_library(
 go_binary(
     name = "parser",
     testonly = 1,
-    srcs = ["parser_main.go"],
+    srcs = [
+        "parser_main.go",
+        "version.go",
+    ],
     nogo = False,
+    x_defs = {"main.version": "{STABLE_VERSION}"},
     deps = [
         ":parsers",
         "//runsc/flag",
diff --git a/tools/parsers/go_parser.go b/tools/parsers/go_parser.go
index df4875e6a..57e538149 100644
--- a/tools/parsers/go_parser.go
+++ b/tools/parsers/go_parser.go
@@ -30,10 +30,10 @@ import (
 // ParseOutput expects golang benchmark output and returns a struct formatted
 // for BigQuery.
 func ParseOutput(output string, name string, official bool) (*bigquery.Suite, error) {
-	suite := bigquery.NewSuite(name)
+	suite := bigquery.NewSuite(name, official)
 	lines := strings.Split(output, "\n")
 	for _, line := range lines {
-		bm, err := parseLine(line, official)
+		bm, err := parseLine(line)
 		if err != nil {
 			return nil, fmt.Errorf("failed to parse line '%s': %v", line, err)
 		}
@@ -60,7 +60,7 @@ func ParseOutput(output string, name string, official bool) (*bigquery.Suite, er
 //		{Name: requests_per_second, Unit: QPS, Sample: 140 }
 //  }
 //}
-func parseLine(line string, official bool) (*bigquery.Benchmark, error) {
+func parseLine(line string) (*bigquery.Benchmark, error) {
 	fields := strings.Fields(line)
 
 	// Check if this line is a Benchmark line. Otherwise ignore the line.
@@ -78,7 +78,7 @@ func parseLine(line string, official bool) (*bigquery.Benchmark, error) {
 		return nil, fmt.Errorf("parse name/params: %v", err)
 	}
 
-	bm := bigquery.NewBenchmark(name, iters, official)
+	bm := bigquery.NewBenchmark(name, iters)
 	for _, p := range params {
 		bm.AddCondition(p.Name, p.Value)
 	}
diff --git a/tools/parsers/go_parser_test.go b/tools/parsers/go_parser_test.go
index 0aa1152a2..f0737d46b 100644
--- a/tools/parsers/go_parser_test.go
+++ b/tools/parsers/go_parser_test.go
@@ -94,7 +94,7 @@ func TestParseLine(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			got, err := parseLine(tc.data, false)
+			got, err := parseLine(tc.data)
 			if err != nil {
 				t.Fatalf("parseLine failed with: %v", err)
 			}
diff --git a/tools/parsers/parser_main.go b/tools/parsers/parser_main.go
index 6c6182464..7cce69e03 100644
--- a/tools/parsers/parser_main.go
+++ b/tools/parsers/parser_main.go
@@ -49,12 +49,11 @@ var (
 	parseCmd     = flag.NewFlagSet(parseString, flag.ContinueOnError)
 	file         = parseCmd.String("file", "", "file to parse for benchmarks")
 	name         = parseCmd.String("suite_name", "", "name of the benchmark suite")
-	clNumber     = parseCmd.String("cl", "", "changelist number of this run")
-	gitCommit    = parseCmd.String("git_commit", "", "git commit sha for this run")
 	parseProject = parseCmd.String("project", "", "GCP project to send benchmarks.")
 	parseDataset = parseCmd.String("dataset", "", "dataset to send benchmarks data.")
 	parseTable   = parseCmd.String("table", "", "table to send benchmarks data.")
 	official     = parseCmd.Bool("official", false, "mark input data as official.")
+	runtime      = parseCmd.String("runtime", "", "runtime used to run the benchmark")
 )
 
 // initBenchmarks initializes a dataset/table in a BigQuery project.
@@ -67,23 +66,29 @@ func initBenchmarks(ctx context.Context) error {
 func parseBenchmarks(ctx context.Context) error {
 	data, err := ioutil.ReadFile(*file)
 	if err != nil {
-		return fmt.Errorf("failed to read file: %v", err)
+		return fmt.Errorf("failed to read file %s: %v", *file, err)
 	}
 	suite, err := parsers.ParseOutput(string(data), *name, *official)
 	if err != nil {
 		return fmt.Errorf("failed parse data: %v", err)
 	}
+	if len(suite.Benchmarks) < 1 {
+		fmt.Fprintf(os.Stderr, "Failed to find benchmarks for file: %s", *file)
+		return nil
+	}
+
 	extraConditions := []*bq.Condition{
 		{
-			Name:  "change_list",
-			Value: *clNumber,
+			Name:  "runtime",
+			Value: *runtime,
 		},
 		{
-			Name:  "commit",
-			Value: *gitCommit,
+			Name:  "version",
+			Value: version,
 		},
 	}
 
+	suite.Official = *official
 	suite.Conditions = append(suite.Conditions, extraConditions...)
 	return bq.SendBenchmarks(ctx, suite, *parseProject, *parseDataset, *parseTable, nil)
 }
@@ -94,26 +99,27 @@ func main() {
 	// the "init" command
 	case len(os.Args) >= 2 && os.Args[1] == initString:
 		if err := initCmd.Parse(os.Args[2:]); err != nil {
-			fmt.Fprintf(os.Stderr, "failed parse flags: %v", err)
+			fmt.Fprintf(os.Stderr, "failed parse flags: %v\n", err)
 			os.Exit(1)
 		}
 		if err := initBenchmarks(ctx); err != nil {
-			failure := "failed to initialize project: %s dataset: %s table: %s: %v"
+			failure := "failed to initialize project: %s dataset: %s table: %s: %v\n"
 			fmt.Fprintf(os.Stderr, failure, *parseProject, *parseDataset, *parseTable, err)
 			os.Exit(1)
 		}
 	// the "parse" command.
 	case len(os.Args) >= 2 && os.Args[1] == parseString:
 		if err := parseCmd.Parse(os.Args[2:]); err != nil {
-			fmt.Fprintf(os.Stderr, "failed parse flags: %v", err)
+			fmt.Fprintf(os.Stderr, "failed parse flags: %v\n", err)
 			os.Exit(1)
 		}
 		if err := parseBenchmarks(ctx); err != nil {
-			fmt.Fprintf(os.Stderr, "failed parse benchmarks: %v", err)
+			fmt.Fprintf(os.Stderr, "failed parse benchmarks: %v\n", err)
 			os.Exit(1)
 		}
 	default:
 		printUsage()
+		os.Exit(1)
 	}
 }
 
diff --git a/tools/parsers/version.go b/tools/parsers/version.go
new file mode 100644
index 000000000..ab9194b9d
--- /dev/null
+++ b/tools/parsers/version.go
@@ -0,0 +1,18 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+// version is set during linking.
+var version = "VERSION_MISSING"