From 30794512d3977ebb2b185e5e9cfb969d558a07a4 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Wed, 19 Feb 2020 18:20:52 -0800 Subject: Add basic microbenchmarks. PiperOrigin-RevId: 296104390 --- test/runner/gtest/gtest.go | 154 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 test/runner/gtest/gtest.go (limited to 'test/runner/gtest/gtest.go') diff --git a/test/runner/gtest/gtest.go b/test/runner/gtest/gtest.go new file mode 100644 index 000000000..23bf7b5f6 --- /dev/null +++ b/test/runner/gtest/gtest.go @@ -0,0 +1,154 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gtest contains helpers for running google-test tests from Go. +package gtest + +import ( + "fmt" + "os/exec" + "strings" +) + +var ( + // listTestFlag is the flag that will list tests in gtest binaries. + listTestFlag = "--gtest_list_tests" + + // filterTestFlag is the flag that will filter tests in gtest binaries. + filterTestFlag = "--gtest_filter" + + // listBechmarkFlag is the flag that will list benchmarks in gtest binaries. + listBenchmarkFlag = "--benchmark_list_tests" + + // filterBenchmarkFlag is the flag that will run specified benchmarks. + filterBenchmarkFlag = "--benchmark_filter" +) + +// TestCase is a single gtest test case. +type TestCase struct { + // Suite is the suite for this test. + Suite string + + // Name is the name of this individual test. + Name string + + // benchmark indicates that this is a benchmark. In this case, the + // suite will be empty, and we will use the appropriate test and + // benchmark flags. + benchmark bool +} + +// FullName returns the name of the test including the suite. It is suitable to +// pass to "-gtest_filter". +func (tc TestCase) FullName() string { + return fmt.Sprintf("%s.%s", tc.Suite, tc.Name) +} + +// Args returns arguments to be passed when invoking the test. +func (tc TestCase) Args() []string { + if tc.benchmark { + return []string{ + fmt.Sprintf("%s=^$", filterTestFlag), + fmt.Sprintf("%s=^%s$", filterBenchmarkFlag, tc.Name), + } + } + return []string{ + fmt.Sprintf("%s=^%s$", filterTestFlag, tc.FullName()), + fmt.Sprintf("%s=^$", filterBenchmarkFlag), + } +} + +// ParseTestCases calls a gtest test binary to list its test and returns a +// slice with the name and suite of each test. +// +// If benchmarks is true, then benchmarks will be included in the list of test +// cases provided. Note that this requires the binary to support the +// benchmarks_list_tests flag. +func ParseTestCases(testBin string, benchmarks bool, extraArgs ...string) ([]TestCase, error) { + // Run to extract test cases. + args := append([]string{listTestFlag}, extraArgs...) + cmd := exec.Command(testBin, args...) + out, err := cmd.Output() + if err != nil { + exitErr, ok := err.(*exec.ExitError) + if !ok { + return nil, fmt.Errorf("could not enumerate gtest tests: %v", err) + } + return nil, fmt.Errorf("could not enumerate gtest tests: %v\nstderr:\n%s", err, exitErr.Stderr) + } + + // Parse test output. + var t []TestCase + var suite string + for _, line := range strings.Split(string(out), "\n") { + // Strip comments. + line = strings.Split(line, "#")[0] + + // New suite? + if !strings.HasPrefix(line, " ") { + suite = strings.TrimSuffix(strings.TrimSpace(line), ".") + continue + } + + // Individual test. + name := strings.TrimSpace(line) + + // Do we have a suite yet? + if suite == "" { + return nil, fmt.Errorf("test without a suite: %v", name) + } + + // Add this individual test. + t = append(t, TestCase{ + Suite: suite, + Name: name, + }) + + } + + // Finished? + if !benchmarks { + return t, nil + } + + // Run again to extract benchmarks. + args = append([]string{listBenchmarkFlag}, extraArgs...) + cmd = exec.Command(testBin, args...) + out, err = cmd.Output() + if err != nil { + exitErr, ok := err.(*exec.ExitError) + if !ok { + return nil, fmt.Errorf("could not enumerate gtest benchmarks: %v", err) + } + return nil, fmt.Errorf("could not enumerate gtest benchmarks: %v\nstderr\n%s", err, exitErr.Stderr) + } + + // Parse benchmark output. + for _, line := range strings.Split(string(out), "\n") { + // Strip comments. + line = strings.Split(line, "#")[0] + + // Single benchmark. + name := strings.TrimSpace(line) + + // Add the single benchmark. + t = append(t, TestCase{ + Suite: "Benchmarks", + Name: name, + benchmark: true, + }) + } + + return t, nil +} -- cgit v1.2.3 From 160d5751ab6a06c22aed7d829a17c88344cc7cf2 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Mon, 24 Feb 2020 17:28:27 -0800 Subject: Add default behavior for gtest runner. PiperOrigin-RevId: 297009116 --- test/perf/BUILD | 8 ++++---- test/perf/linux/getdents_benchmark.cc | 2 +- test/runner/gtest/gtest.go | 25 +++++++++++++++++++------ 3 files changed, 24 insertions(+), 11 deletions(-) (limited to 'test/runner/gtest/gtest.go') diff --git a/test/perf/BUILD b/test/perf/BUILD index 7a2bf10ed..346a28e16 100644 --- a/test/perf/BUILD +++ b/test/perf/BUILD @@ -29,7 +29,7 @@ syscall_test( ) syscall_test( - size = "large", + size = "enormous", test = "//test/perf/linux:getdents_benchmark", ) @@ -39,7 +39,7 @@ syscall_test( ) syscall_test( - size = "large", + size = "enormous", test = "//test/perf/linux:gettid_benchmark", ) @@ -87,7 +87,7 @@ syscall_test( ) syscall_test( - size = "large", + size = "enormous", test = "//test/perf/linux:signal_benchmark", ) @@ -102,7 +102,7 @@ syscall_test( ) syscall_test( - size = "large", + size = "enormous", add_overlay = True, test = "//test/perf/linux:unlink_benchmark", ) diff --git a/test/perf/linux/getdents_benchmark.cc b/test/perf/linux/getdents_benchmark.cc index 0e03975b4..afc599ad2 100644 --- a/test/perf/linux/getdents_benchmark.cc +++ b/test/perf/linux/getdents_benchmark.cc @@ -141,7 +141,7 @@ void BM_GetdentsNewFD(benchmark::State& state) { state.SetItemsProcessed(state.iterations()); } -BENCHMARK(BM_GetdentsNewFD)->Range(1, 1 << 16)->UseRealTime(); +BENCHMARK(BM_GetdentsNewFD)->Range(1, 1 << 12)->UseRealTime(); } // namespace diff --git a/test/runner/gtest/gtest.go b/test/runner/gtest/gtest.go index 23bf7b5f6..f96e2415e 100644 --- a/test/runner/gtest/gtest.go +++ b/test/runner/gtest/gtest.go @@ -43,6 +43,10 @@ type TestCase struct { // Name is the name of this individual test. Name string + // all indicates that this will run without flags. This takes + // precendence over benchmark below. + all bool + // benchmark indicates that this is a benchmark. In this case, the // suite will be empty, and we will use the appropriate test and // benchmark flags. @@ -57,6 +61,9 @@ func (tc TestCase) FullName() string { // Args returns arguments to be passed when invoking the test. func (tc TestCase) Args() []string { + if tc.all { + return []string{} // No arguments. + } if tc.benchmark { return []string{ fmt.Sprintf("%s=^$", filterTestFlag), @@ -81,11 +88,16 @@ func ParseTestCases(testBin string, benchmarks bool, extraArgs ...string) ([]Tes cmd := exec.Command(testBin, args...) out, err := cmd.Output() if err != nil { - exitErr, ok := err.(*exec.ExitError) - if !ok { - return nil, fmt.Errorf("could not enumerate gtest tests: %v", err) - } - return nil, fmt.Errorf("could not enumerate gtest tests: %v\nstderr:\n%s", err, exitErr.Stderr) + // We failed to list tests with the given flags. Just + // return something that will run the binary with no + // flags, which should execute all tests. + return []TestCase{ + TestCase{ + Suite: "Default", + Name: "All", + all: true, + }, + }, nil } // Parse test output. @@ -114,7 +126,6 @@ func ParseTestCases(testBin string, benchmarks bool, extraArgs ...string) ([]Tes Suite: suite, Name: name, }) - } // Finished? @@ -127,6 +138,8 @@ func ParseTestCases(testBin string, benchmarks bool, extraArgs ...string) ([]Tes cmd = exec.Command(testBin, args...) out, err = cmd.Output() if err != nil { + // We were able to enumerate tests above, but not benchmarks? + // We requested them, so we return an error in this case. exitErr, ok := err.(*exec.ExitError) if !ok { return nil, fmt.Errorf("could not enumerate gtest benchmarks: %v", err) -- cgit v1.2.3 From 504c9e14d61a9ca9fa3615290a05471684019ecc Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Tue, 3 Mar 2020 15:53:48 -0800 Subject: test/runner: use proper filters for test cases The benchmark_filter options accepts regex-s, but the gtest-filter option accepts shell-like wildcards. Fixes #2034 Signed-off-by: Andrei Vagin --- test/runner/gtest/gtest.go | 7 ++++--- test/syscalls/linux/tuntap_hostinet.cc | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'test/runner/gtest/gtest.go') diff --git a/test/runner/gtest/gtest.go b/test/runner/gtest/gtest.go index f96e2415e..869169ad5 100644 --- a/test/runner/gtest/gtest.go +++ b/test/runner/gtest/gtest.go @@ -66,13 +66,12 @@ func (tc TestCase) Args() []string { } if tc.benchmark { return []string{ - fmt.Sprintf("%s=^$", filterTestFlag), fmt.Sprintf("%s=^%s$", filterBenchmarkFlag, tc.Name), + fmt.Sprintf("%s=", filterTestFlag), } } return []string{ - fmt.Sprintf("%s=^%s$", filterTestFlag, tc.FullName()), - fmt.Sprintf("%s=^$", filterBenchmarkFlag), + fmt.Sprintf("%s=%s", filterTestFlag, tc.FullName()), } } @@ -147,6 +146,8 @@ func ParseTestCases(testBin string, benchmarks bool, extraArgs ...string) ([]Tes return nil, fmt.Errorf("could not enumerate gtest benchmarks: %v\nstderr\n%s", err, exitErr.Stderr) } + out = []byte(strings.Trim(string(out), "\n")) + // Parse benchmark output. for _, line := range strings.Split(string(out), "\n") { // Strip comments. diff --git a/test/syscalls/linux/tuntap_hostinet.cc b/test/syscalls/linux/tuntap_hostinet.cc index 0c527419e..1513fb9d5 100644 --- a/test/syscalls/linux/tuntap_hostinet.cc +++ b/test/syscalls/linux/tuntap_hostinet.cc @@ -26,6 +26,7 @@ namespace { TEST(TuntapHostInetTest, NoNetTun) { SKIP_IF(!IsRunningOnGvisor()); + SKIP_IF(!IsRunningWithHostinet()); struct stat statbuf; ASSERT_THAT(stat("/dev/net/tun", &statbuf), SyscallFailsWithErrno(ENOENT)); -- cgit v1.2.3