From 30794512d3977ebb2b185e5e9cfb969d558a07a4 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Wed, 19 Feb 2020 18:20:52 -0800 Subject: Add basic microbenchmarks. PiperOrigin-RevId: 296104390 --- test/runner/gtest/gtest.go | 154 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 test/runner/gtest/gtest.go (limited to 'test/runner/gtest/gtest.go') diff --git a/test/runner/gtest/gtest.go b/test/runner/gtest/gtest.go new file mode 100644 index 000000000..23bf7b5f6 --- /dev/null +++ b/test/runner/gtest/gtest.go @@ -0,0 +1,154 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gtest contains helpers for running google-test tests from Go. +package gtest + +import ( + "fmt" + "os/exec" + "strings" +) + +var ( + // listTestFlag is the flag that will list tests in gtest binaries. + listTestFlag = "--gtest_list_tests" + + // filterTestFlag is the flag that will filter tests in gtest binaries. + filterTestFlag = "--gtest_filter" + + // listBechmarkFlag is the flag that will list benchmarks in gtest binaries. + listBenchmarkFlag = "--benchmark_list_tests" + + // filterBenchmarkFlag is the flag that will run specified benchmarks. + filterBenchmarkFlag = "--benchmark_filter" +) + +// TestCase is a single gtest test case. +type TestCase struct { + // Suite is the suite for this test. + Suite string + + // Name is the name of this individual test. + Name string + + // benchmark indicates that this is a benchmark. In this case, the + // suite will be empty, and we will use the appropriate test and + // benchmark flags. + benchmark bool +} + +// FullName returns the name of the test including the suite. It is suitable to +// pass to "-gtest_filter". +func (tc TestCase) FullName() string { + return fmt.Sprintf("%s.%s", tc.Suite, tc.Name) +} + +// Args returns arguments to be passed when invoking the test. +func (tc TestCase) Args() []string { + if tc.benchmark { + return []string{ + fmt.Sprintf("%s=^$", filterTestFlag), + fmt.Sprintf("%s=^%s$", filterBenchmarkFlag, tc.Name), + } + } + return []string{ + fmt.Sprintf("%s=^%s$", filterTestFlag, tc.FullName()), + fmt.Sprintf("%s=^$", filterBenchmarkFlag), + } +} + +// ParseTestCases calls a gtest test binary to list its test and returns a +// slice with the name and suite of each test. +// +// If benchmarks is true, then benchmarks will be included in the list of test +// cases provided. Note that this requires the binary to support the +// benchmarks_list_tests flag. +func ParseTestCases(testBin string, benchmarks bool, extraArgs ...string) ([]TestCase, error) { + // Run to extract test cases. + args := append([]string{listTestFlag}, extraArgs...) + cmd := exec.Command(testBin, args...) + out, err := cmd.Output() + if err != nil { + exitErr, ok := err.(*exec.ExitError) + if !ok { + return nil, fmt.Errorf("could not enumerate gtest tests: %v", err) + } + return nil, fmt.Errorf("could not enumerate gtest tests: %v\nstderr:\n%s", err, exitErr.Stderr) + } + + // Parse test output. + var t []TestCase + var suite string + for _, line := range strings.Split(string(out), "\n") { + // Strip comments. + line = strings.Split(line, "#")[0] + + // New suite? + if !strings.HasPrefix(line, " ") { + suite = strings.TrimSuffix(strings.TrimSpace(line), ".") + continue + } + + // Individual test. + name := strings.TrimSpace(line) + + // Do we have a suite yet? + if suite == "" { + return nil, fmt.Errorf("test without a suite: %v", name) + } + + // Add this individual test. + t = append(t, TestCase{ + Suite: suite, + Name: name, + }) + + } + + // Finished? + if !benchmarks { + return t, nil + } + + // Run again to extract benchmarks. + args = append([]string{listBenchmarkFlag}, extraArgs...) + cmd = exec.Command(testBin, args...) + out, err = cmd.Output() + if err != nil { + exitErr, ok := err.(*exec.ExitError) + if !ok { + return nil, fmt.Errorf("could not enumerate gtest benchmarks: %v", err) + } + return nil, fmt.Errorf("could not enumerate gtest benchmarks: %v\nstderr\n%s", err, exitErr.Stderr) + } + + // Parse benchmark output. + for _, line := range strings.Split(string(out), "\n") { + // Strip comments. + line = strings.Split(line, "#")[0] + + // Single benchmark. + name := strings.TrimSpace(line) + + // Add the single benchmark. + t = append(t, TestCase{ + Suite: "Benchmarks", + Name: name, + benchmark: true, + }) + } + + return t, nil +} -- cgit v1.2.3 From 160d5751ab6a06c22aed7d829a17c88344cc7cf2 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Mon, 24 Feb 2020 17:28:27 -0800 Subject: Add default behavior for gtest runner. PiperOrigin-RevId: 297009116 --- test/perf/BUILD | 8 ++++---- test/perf/linux/getdents_benchmark.cc | 2 +- test/runner/gtest/gtest.go | 25 +++++++++++++++++++------ 3 files changed, 24 insertions(+), 11 deletions(-) (limited to 'test/runner/gtest/gtest.go') diff --git a/test/perf/BUILD b/test/perf/BUILD index 7a2bf10ed..346a28e16 100644 --- a/test/perf/BUILD +++ b/test/perf/BUILD @@ -29,7 +29,7 @@ syscall_test( ) syscall_test( - size = "large", + size = "enormous", test = "//test/perf/linux:getdents_benchmark", ) @@ -39,7 +39,7 @@ syscall_test( ) syscall_test( - size = "large", + size = "enormous", test = "//test/perf/linux:gettid_benchmark", ) @@ -87,7 +87,7 @@ syscall_test( ) syscall_test( - size = "large", + size = "enormous", test = "//test/perf/linux:signal_benchmark", ) @@ -102,7 +102,7 @@ syscall_test( ) syscall_test( - size = "large", + size = "enormous", add_overlay = True, test = "//test/perf/linux:unlink_benchmark", ) diff --git a/test/perf/linux/getdents_benchmark.cc b/test/perf/linux/getdents_benchmark.cc index 0e03975b4..afc599ad2 100644 --- a/test/perf/linux/getdents_benchmark.cc +++ b/test/perf/linux/getdents_benchmark.cc @@ -141,7 +141,7 @@ void BM_GetdentsNewFD(benchmark::State& state) { state.SetItemsProcessed(state.iterations()); } -BENCHMARK(BM_GetdentsNewFD)->Range(1, 1 << 16)->UseRealTime(); +BENCHMARK(BM_GetdentsNewFD)->Range(1, 1 << 12)->UseRealTime(); } // namespace diff --git a/test/runner/gtest/gtest.go b/test/runner/gtest/gtest.go index 23bf7b5f6..f96e2415e 100644 --- a/test/runner/gtest/gtest.go +++ b/test/runner/gtest/gtest.go @@ -43,6 +43,10 @@ type TestCase struct { // Name is the name of this individual test. Name string + // all indicates that this will run without flags. This takes + // precendence over benchmark below. + all bool + // benchmark indicates that this is a benchmark. In this case, the // suite will be empty, and we will use the appropriate test and // benchmark flags. @@ -57,6 +61,9 @@ func (tc TestCase) FullName() string { // Args returns arguments to be passed when invoking the test. func (tc TestCase) Args() []string { + if tc.all { + return []string{} // No arguments. + } if tc.benchmark { return []string{ fmt.Sprintf("%s=^$", filterTestFlag), @@ -81,11 +88,16 @@ func ParseTestCases(testBin string, benchmarks bool, extraArgs ...string) ([]Tes cmd := exec.Command(testBin, args...) out, err := cmd.Output() if err != nil { - exitErr, ok := err.(*exec.ExitError) - if !ok { - return nil, fmt.Errorf("could not enumerate gtest tests: %v", err) - } - return nil, fmt.Errorf("could not enumerate gtest tests: %v\nstderr:\n%s", err, exitErr.Stderr) + // We failed to list tests with the given flags. Just + // return something that will run the binary with no + // flags, which should execute all tests. + return []TestCase{ + TestCase{ + Suite: "Default", + Name: "All", + all: true, + }, + }, nil } // Parse test output. @@ -114,7 +126,6 @@ func ParseTestCases(testBin string, benchmarks bool, extraArgs ...string) ([]Tes Suite: suite, Name: name, }) - } // Finished? @@ -127,6 +138,8 @@ func ParseTestCases(testBin string, benchmarks bool, extraArgs ...string) ([]Tes cmd = exec.Command(testBin, args...) out, err = cmd.Output() if err != nil { + // We were able to enumerate tests above, but not benchmarks? + // We requested them, so we return an error in this case. exitErr, ok := err.(*exec.ExitError) if !ok { return nil, fmt.Errorf("could not enumerate gtest benchmarks: %v", err) -- cgit v1.2.3 From 504c9e14d61a9ca9fa3615290a05471684019ecc Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Tue, 3 Mar 2020 15:53:48 -0800 Subject: test/runner: use proper filters for test cases The benchmark_filter options accepts regex-s, but the gtest-filter option accepts shell-like wildcards. Fixes #2034 Signed-off-by: Andrei Vagin --- test/runner/gtest/gtest.go | 7 ++++--- test/syscalls/linux/tuntap_hostinet.cc | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'test/runner/gtest/gtest.go') diff --git a/test/runner/gtest/gtest.go b/test/runner/gtest/gtest.go index f96e2415e..869169ad5 100644 --- a/test/runner/gtest/gtest.go +++ b/test/runner/gtest/gtest.go @@ -66,13 +66,12 @@ func (tc TestCase) Args() []string { } if tc.benchmark { return []string{ - fmt.Sprintf("%s=^$", filterTestFlag), fmt.Sprintf("%s=^%s$", filterBenchmarkFlag, tc.Name), + fmt.Sprintf("%s=", filterTestFlag), } } return []string{ - fmt.Sprintf("%s=^%s$", filterTestFlag, tc.FullName()), - fmt.Sprintf("%s=^$", filterBenchmarkFlag), + fmt.Sprintf("%s=%s", filterTestFlag, tc.FullName()), } } @@ -147,6 +146,8 @@ func ParseTestCases(testBin string, benchmarks bool, extraArgs ...string) ([]Tes return nil, fmt.Errorf("could not enumerate gtest benchmarks: %v\nstderr\n%s", err, exitErr.Stderr) } + out = []byte(strings.Trim(string(out), "\n")) + // Parse benchmark output. for _, line := range strings.Split(string(out), "\n") { // Strip comments. diff --git a/test/syscalls/linux/tuntap_hostinet.cc b/test/syscalls/linux/tuntap_hostinet.cc index 0c527419e..1513fb9d5 100644 --- a/test/syscalls/linux/tuntap_hostinet.cc +++ b/test/syscalls/linux/tuntap_hostinet.cc @@ -26,6 +26,7 @@ namespace { TEST(TuntapHostInetTest, NoNetTun) { SKIP_IF(!IsRunningOnGvisor()); + SKIP_IF(!IsRunningWithHostinet()); struct stat statbuf; ASSERT_THAT(stat("/dev/net/tun", &statbuf), SyscallFailsWithErrno(ENOENT)); -- cgit v1.2.3 From 00b684ea7fd96beb32d1517dd06c5d736621ff70 Mon Sep 17 00:00:00 2001 From: Fabricio Voznika Date: Wed, 12 Aug 2020 14:21:04 -0700 Subject: Limit the scope when deleting test log directory on success The code was deleting logs for all tests when a single test passed. Change it to delete only the logs relevant to the test at hand. Also fixed the benchmark lookup code, which was always generating a single empty benchmark entry if there were not benchmarks. PiperOrigin-RevId: 326311477 --- test/runner/gtest/gtest.go | 8 +++++--- test/runner/runner.go | 31 ++++++++++++++++--------------- 2 files changed, 21 insertions(+), 18 deletions(-) (limited to 'test/runner/gtest/gtest.go') diff --git a/test/runner/gtest/gtest.go b/test/runner/gtest/gtest.go index 869169ad5..e4445e01b 100644 --- a/test/runner/gtest/gtest.go +++ b/test/runner/gtest/gtest.go @@ -146,10 +146,13 @@ func ParseTestCases(testBin string, benchmarks bool, extraArgs ...string) ([]Tes return nil, fmt.Errorf("could not enumerate gtest benchmarks: %v\nstderr\n%s", err, exitErr.Stderr) } - out = []byte(strings.Trim(string(out), "\n")) + benches := strings.Trim(string(out), "\n") + if len(benches) == 0 { + return t, nil + } // Parse benchmark output. - for _, line := range strings.Split(string(out), "\n") { + for _, line := range strings.Split(benches, "\n") { // Strip comments. line = strings.Split(line, "#")[0] @@ -163,6 +166,5 @@ func ParseTestCases(testBin string, benchmarks bool, extraArgs ...string) ([]Tes benchmark: true, }) } - return t, nil } diff --git a/test/runner/runner.go b/test/runner/runner.go index bc4b39cbb..5ac91310d 100644 --- a/test/runner/runner.go +++ b/test/runner/runner.go @@ -172,13 +172,14 @@ func runRunsc(tc gtest.TestCase, spec *specs.Spec) error { args = append(args, "-fsgofer-host-uds") } - undeclaredOutputsDir, ok := syscall.Getenv("TEST_UNDECLARED_OUTPUTS_DIR") - if ok { - tdir := filepath.Join(undeclaredOutputsDir, strings.Replace(name, "/", "_", -1)) - if err := os.MkdirAll(tdir, 0755); err != nil { + testLogDir := "" + if undeclaredOutputsDir, ok := syscall.Getenv("TEST_UNDECLARED_OUTPUTS_DIR"); ok { + // Create log directory dedicated for this test. + testLogDir = filepath.Join(undeclaredOutputsDir, strings.Replace(name, "/", "_", -1)) + if err := os.MkdirAll(testLogDir, 0755); err != nil { return fmt.Errorf("could not create test dir: %v", err) } - debugLogDir, err := ioutil.TempDir(tdir, "runsc") + debugLogDir, err := ioutil.TempDir(testLogDir, "runsc") if err != nil { return fmt.Errorf("could not create temp dir: %v", err) } @@ -227,10 +228,10 @@ func runRunsc(tc gtest.TestCase, spec *specs.Spec) error { dArgs := append([]string{}, args...) dArgs = append(dArgs, "-alsologtostderr=true", "debug", "--stacks", id) go func(dArgs []string) { - cmd := exec.Command(*runscPath, dArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Run() + debug := exec.Command(*runscPath, dArgs...) + debug.Stdout = os.Stdout + debug.Stderr = os.Stderr + debug.Run() done <- true }(dArgs) @@ -245,17 +246,17 @@ func runRunsc(tc gtest.TestCase, spec *specs.Spec) error { dArgs = append(args, "debug", fmt.Sprintf("--signal=%d", syscall.SIGTERM), id) - cmd := exec.Command(*runscPath, dArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Run() + signal := exec.Command(*runscPath, dArgs...) + signal.Stdout = os.Stdout + signal.Stderr = os.Stderr + signal.Run() }() err = cmd.Run() - if err == nil { + if err == nil && len(testLogDir) > 0 { // If the test passed, then we erase the log directory. This speeds up // uploading logs in continuous integration & saves on disk space. - os.RemoveAll(undeclaredOutputsDir) + os.RemoveAll(testLogDir) } return err -- cgit v1.2.3