diff options
Diffstat (limited to 'test/runner')
-rw-r--r-- | test/runner/gtest/gtest.go | 50 | ||||
-rw-r--r-- | test/runner/runner.go | 210 |
2 files changed, 129 insertions, 131 deletions
diff --git a/test/runner/gtest/gtest.go b/test/runner/gtest/gtest.go index 2ad5f58ef..38e57d62f 100644 --- a/test/runner/gtest/gtest.go +++ b/test/runner/gtest/gtest.go @@ -35,39 +35,6 @@ var ( filterBenchmarkFlag = "--benchmark_filter" ) -// BuildTestArgs builds arguments to be passed to the test binary to execute -// only the test cases in `indices`. -func BuildTestArgs(indices []int, testCases []TestCase) []string { - var testFilter, benchFilter string - for _, tci := range indices { - tc := testCases[tci] - if tc.all { - // No argument will make all tests run. - return nil - } - if tc.benchmark { - if len(benchFilter) > 0 { - benchFilter += "|" - } - benchFilter += "^" + tc.Name + "$" - } else { - if len(testFilter) > 0 { - testFilter += ":" - } - testFilter += tc.FullName() - } - } - - var args []string - if len(testFilter) > 0 { - args = append(args, fmt.Sprintf("%s=%s", filterTestFlag, testFilter)) - } - if len(benchFilter) > 0 { - args = append(args, fmt.Sprintf("%s=%s", filterBenchmarkFlag, benchFilter)) - } - return args -} - // TestCase is a single gtest test case. type TestCase struct { // Suite is the suite for this test. @@ -92,6 +59,22 @@ func (tc TestCase) FullName() string { return fmt.Sprintf("%s.%s", tc.Suite, tc.Name) } +// Args returns arguments to be passed when invoking the test. +func (tc TestCase) Args() []string { + if tc.all { + return []string{} // No arguments. + } + if tc.benchmark { + return []string{ + fmt.Sprintf("%s=^%s$", filterBenchmarkFlag, tc.Name), + fmt.Sprintf("%s=", filterTestFlag), + } + } + return []string{ + fmt.Sprintf("%s=%s", filterTestFlag, tc.FullName()), + } +} + // ParseTestCases calls a gtest test binary to list its test and returns a // slice with the name and suite of each test. // @@ -107,7 +90,6 @@ func ParseTestCases(testBin string, benchmarks bool, extraArgs ...string) ([]Tes // We failed to list tests with the given flags. Just // return something that will run the binary with no // flags, which should execute all tests. - fmt.Printf("failed to get test list: %v\n", err) return []TestCase{ { Suite: "Default", diff --git a/test/runner/runner.go b/test/runner/runner.go index d314a5036..7e8e88ba2 100644 --- a/test/runner/runner.go +++ b/test/runner/runner.go @@ -26,6 +26,7 @@ import ( "path/filepath" "strings" "syscall" + "testing" "time" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -56,82 +57,13 @@ var ( leakCheck = flag.Bool("leak-check", false, "check for reference leaks") ) -func main() { - flag.Parse() - if flag.NArg() != 1 { - fatalf("test must be provided") - } - - log.SetLevel(log.Info) - if *debug { - log.SetLevel(log.Debug) - } - - if *platform != "native" && *runscPath == "" { - if err := testutil.ConfigureExePath(); err != nil { - panic(err.Error()) - } - *runscPath = specutils.ExePath - } - - // Make sure stdout and stderr are opened with O_APPEND, otherwise logs - // from outside the sandbox can (and will) stomp on logs from inside - // the sandbox. - for _, f := range []*os.File{os.Stdout, os.Stderr} { - flags, err := unix.FcntlInt(f.Fd(), unix.F_GETFL, 0) - if err != nil { - fatalf("error getting file flags for %v: %v", f, err) - } - if flags&unix.O_APPEND == 0 { - flags |= unix.O_APPEND - if _, err := unix.FcntlInt(f.Fd(), unix.F_SETFL, flags); err != nil { - fatalf("error setting file flags for %v: %v", f, err) - } - } - } - - // Resolve the absolute path for the binary. - testBin, err := filepath.Abs(flag.Args()[0]) - if err != nil { - fatalf("Abs(%q) failed: %v", flag.Args()[0], err) - } - - // Get all test cases in each binary. - testCases, err := gtest.ParseTestCases(testBin, true) - if err != nil { - fatalf("ParseTestCases(%q) failed: %v", testBin, err) - } - - // Get subset of tests corresponding to shard. - indices, err := testutil.TestIndicesForShard(len(testCases)) - if err != nil { - fatalf("TestsForShard() failed: %v", err) - } - if len(indices) == 0 { - log.Warningf("No tests to run in this shard") - return - } - args := gtest.BuildTestArgs(indices, testCases) - - switch *platform { - case "native": - if err := runTestCaseNative(testBin, args); err != nil { - fatalf(err.Error()) - } - default: - if err := runTestCaseRunsc(testBin, args); err != nil { - fatalf(err.Error()) - } - } -} - // runTestCaseNative runs the test case directly on the host machine. -func runTestCaseNative(testBin string, args []string) error { +func runTestCaseNative(testBin string, tc gtest.TestCase, t *testing.T) { // These tests might be running in parallel, so make sure they have a // unique test temp dir. tmpDir, err := ioutil.TempDir(testutil.TmpDir(), "") if err != nil { - return fmt.Errorf("could not create temp dir: %v", err) + t.Fatalf("could not create temp dir: %v", err) } defer os.RemoveAll(tmpDir) @@ -152,12 +84,12 @@ func runTestCaseNative(testBin string, args []string) error { } // Remove shard env variables so that the gunit binary does not try to // interpret them. - env = filterEnv(env, "TEST_SHARD_INDEX", "TEST_TOTAL_SHARDS", "GTEST_SHARD_INDEX", "GTEST_TOTAL_SHARDS") + env = filterEnv(env, []string{"TEST_SHARD_INDEX", "TEST_TOTAL_SHARDS", "GTEST_SHARD_INDEX", "GTEST_TOTAL_SHARDS"}) if *addUDSTree { socketDir, cleanup, err := uds.CreateSocketTree("/tmp") if err != nil { - return fmt.Errorf("failed to create socket tree: %v", err) + t.Fatalf("failed to create socket tree: %v", err) } defer cleanup() @@ -167,7 +99,7 @@ func runTestCaseNative(testBin string, args []string) error { env = append(env, "TEST_UDS_ATTACH_TREE="+socketDir) } - cmd := exec.Command(testBin, args...) + cmd := exec.Command(testBin, tc.Args()...) cmd.Env = env cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -183,9 +115,8 @@ func runTestCaseNative(testBin string, args []string) error { if err := cmd.Run(); err != nil { ws := err.(*exec.ExitError).Sys().(syscall.WaitStatus) - return fmt.Errorf("test exited with status %d, want 0", ws.ExitStatus()) + t.Errorf("test %q exited with status %d, want 0", tc.FullName(), ws.ExitStatus()) } - return nil } // runRunsc runs spec in runsc in a standard test configuration. @@ -193,7 +124,7 @@ func runTestCaseNative(testBin string, args []string) error { // runsc logs will be saved to a path in TEST_UNDECLARED_OUTPUTS_DIR. // // Returns an error if the sandboxed application exits non-zero. -func runRunsc(spec *specs.Spec) error { +func runRunsc(tc gtest.TestCase, spec *specs.Spec) error { bundleDir, cleanup, err := testutil.SetupBundleDir(spec) if err != nil { return fmt.Errorf("SetupBundleDir failed: %v", err) @@ -206,8 +137,9 @@ func runRunsc(spec *specs.Spec) error { } defer cleanup() + name := tc.FullName() id := testutil.RandomContainerID() - log.Infof("Running test in container %q", id) + log.Infof("Running test %q in container %q", name, id) specutils.LogSpec(spec) args := []string{ @@ -243,8 +175,13 @@ func runRunsc(spec *specs.Spec) error { args = append(args, "-ref-leak-mode=log-names") } - testLogDir := os.Getenv("TEST_UNDECLARED_OUTPUTS_DIR") - if len(testLogDir) > 0 { + testLogDir := "" + if undeclaredOutputsDir, ok := unix.Getenv("TEST_UNDECLARED_OUTPUTS_DIR"); ok { + // Create log directory dedicated for this test. + testLogDir = filepath.Join(undeclaredOutputsDir, strings.Replace(name, "/", "_", -1)) + if err := os.MkdirAll(testLogDir, 0755); err != nil { + return fmt.Errorf("could not create test dir: %v", err) + } debugLogDir, err := ioutil.TempDir(testLogDir, "runsc") if err != nil { return fmt.Errorf("could not create temp dir: %v", err) @@ -290,7 +227,7 @@ func runRunsc(spec *specs.Spec) error { if !ok { return } - log.Warningf("Got signal: %v", s) + log.Warningf("%s: Got signal: %v", name, s) done := make(chan bool, 1) dArgs := append([]string{}, args...) dArgs = append(dArgs, "-alsologtostderr=true", "debug", "--stacks", id) @@ -323,7 +260,7 @@ func runRunsc(spec *specs.Spec) error { if err == nil && len(testLogDir) > 0 { // If the test passed, then we erase the log directory. This speeds up // uploading logs in continuous integration & saves on disk space. - _ = os.RemoveAll(testLogDir) + os.RemoveAll(testLogDir) } return err @@ -378,10 +315,10 @@ func setupUDSTree(spec *specs.Spec) (cleanup func(), err error) { } // runsTestCaseRunsc runs the test case in runsc. -func runTestCaseRunsc(testBin string, args []string) error { +func runTestCaseRunsc(testBin string, tc gtest.TestCase, t *testing.T) { // Run a new container with the test executable and filter for the // given test suite and name. - spec := testutil.NewSpecWithArgs(append([]string{testBin}, args...)...) + spec := testutil.NewSpecWithArgs(append([]string{testBin}, tc.Args()...)...) // Mark the root as writeable, as some tests attempt to // write to the rootfs, and expect EACCES, not EROFS. @@ -407,12 +344,12 @@ func runTestCaseRunsc(testBin string, args []string) error { // users, so make sure it is world-accessible. tmpDir, err := ioutil.TempDir(testutil.TmpDir(), "") if err != nil { - return fmt.Errorf("could not create temp dir: %v", err) + t.Fatalf("could not create temp dir: %v", err) } defer os.RemoveAll(tmpDir) if err := os.Chmod(tmpDir, 0777); err != nil { - return fmt.Errorf("could not chmod temp dir: %v", err) + t.Fatalf("could not chmod temp dir: %v", err) } // "/tmp" is not replaced with a tmpfs mount inside the sandbox @@ -432,12 +369,13 @@ func runTestCaseRunsc(testBin string, args []string) error { // Set environment variables that indicate we are running in gVisor with // the given platform, network, and filesystem stack. - env := []string{"TEST_ON_GVISOR=" + *platform, "GVISOR_NETWORK=" + *network} - env = append(env, os.Environ()...) - const vfsVar = "GVISOR_VFS" + platformVar := "TEST_ON_GVISOR" + networkVar := "GVISOR_NETWORK" + env := append(os.Environ(), platformVar+"="+*platform, networkVar+"="+*network) + vfsVar := "GVISOR_VFS" if *vfs2 { env = append(env, vfsVar+"=VFS2") - const fuseVar = "FUSE_ENABLED" + fuseVar := "FUSE_ENABLED" if *fuse { env = append(env, fuseVar+"=TRUE") } else { @@ -449,11 +387,11 @@ func runTestCaseRunsc(testBin string, args []string) error { // Remove shard env variables so that the gunit binary does not try to // interpret them. - env = filterEnv(env, "TEST_SHARD_INDEX", "TEST_TOTAL_SHARDS", "GTEST_SHARD_INDEX", "GTEST_TOTAL_SHARDS") + env = filterEnv(env, []string{"TEST_SHARD_INDEX", "TEST_TOTAL_SHARDS", "GTEST_SHARD_INDEX", "GTEST_TOTAL_SHARDS"}) // Set TEST_TMPDIR to /tmp, as some of the syscall tests require it to // be backed by tmpfs. - env = filterEnv(env, "TEST_TMPDIR") + env = filterEnv(env, []string{"TEST_TMPDIR"}) env = append(env, fmt.Sprintf("TEST_TMPDIR=%s", testTmpDir)) spec.Process.Env = env @@ -461,19 +399,18 @@ func runTestCaseRunsc(testBin string, args []string) error { if *addUDSTree { cleanup, err := setupUDSTree(spec) if err != nil { - return fmt.Errorf("error creating UDS tree: %v", err) + t.Fatalf("error creating UDS tree: %v", err) } defer cleanup() } - if err := runRunsc(spec); err != nil { - return fmt.Errorf("test failed with error %v, want nil", err) + if err := runRunsc(tc, spec); err != nil { + t.Errorf("test %q failed with error %v, want nil", tc.FullName(), err) } - return nil } // filterEnv returns an environment with the excluded variables removed. -func filterEnv(env []string, exclude ...string) []string { +func filterEnv(env, exclude []string) []string { var out []string for _, kv := range env { ok := true @@ -494,3 +431,82 @@ func fatalf(s string, args ...interface{}) { fmt.Fprintf(os.Stderr, s+"\n", args...) os.Exit(1) } + +func matchString(a, b string) (bool, error) { + return a == b, nil +} + +func main() { + flag.Parse() + if flag.NArg() != 1 { + fatalf("test must be provided") + } + testBin := flag.Args()[0] // Only argument. + + log.SetLevel(log.Info) + if *debug { + log.SetLevel(log.Debug) + } + + if *platform != "native" && *runscPath == "" { + if err := testutil.ConfigureExePath(); err != nil { + panic(err.Error()) + } + *runscPath = specutils.ExePath + } + + // Make sure stdout and stderr are opened with O_APPEND, otherwise logs + // from outside the sandbox can (and will) stomp on logs from inside + // the sandbox. + for _, f := range []*os.File{os.Stdout, os.Stderr} { + flags, err := unix.FcntlInt(f.Fd(), unix.F_GETFL, 0) + if err != nil { + fatalf("error getting file flags for %v: %v", f, err) + } + if flags&unix.O_APPEND == 0 { + flags |= unix.O_APPEND + if _, err := unix.FcntlInt(f.Fd(), unix.F_SETFL, flags); err != nil { + fatalf("error setting file flags for %v: %v", f, err) + } + } + } + + // Get all test cases in each binary. + testCases, err := gtest.ParseTestCases(testBin, true) + if err != nil { + fatalf("ParseTestCases(%q) failed: %v", testBin, err) + } + + // Get subset of tests corresponding to shard. + indices, err := testutil.TestIndicesForShard(len(testCases)) + if err != nil { + fatalf("TestsForShard() failed: %v", err) + } + + // Resolve the absolute path for the binary. + testBin, err = filepath.Abs(testBin) + if err != nil { + fatalf("Abs() failed: %v", err) + } + + // Run the tests. + var tests []testing.InternalTest + for _, tci := range indices { + // Capture tc. + tc := testCases[tci] + tests = append(tests, testing.InternalTest{ + Name: fmt.Sprintf("%s_%s", tc.Suite, tc.Name), + F: func(t *testing.T) { + if *platform == "native" { + // Run the test case on host. + runTestCaseNative(testBin, tc, t) + } else { + // Run the test case in runsc. + runTestCaseRunsc(testBin, tc, t) + } + }, + }) + } + + testing.Main(matchString, tests, nil, nil) +} |