diff options
Diffstat (limited to 'test/runtimes')
| -rw-r--r-- | test/runtimes/BUILD | 1 | ||||
| -rw-r--r-- | test/runtimes/blacklist_nodejs12.4.0.csv | 47 | ||||
| -rw-r--r-- | test/runtimes/build_defs.bzl | 27 | ||||
| -rw-r--r-- | test/runtimes/runner.go | 60 |
4 files changed, 121 insertions, 14 deletions
diff --git a/test/runtimes/BUILD b/test/runtimes/BUILD index dfb4e2a97..1cde74cfc 100644 --- a/test/runtimes/BUILD +++ b/test/runtimes/BUILD @@ -26,6 +26,7 @@ runtime_test( ) runtime_test( + blacklist_file = "blacklist_nodejs12.4.0.csv", image = "gcr.io/gvisor-presubmit/nodejs12.4.0", lang = "nodejs", ) diff --git a/test/runtimes/blacklist_nodejs12.4.0.csv b/test/runtimes/blacklist_nodejs12.4.0.csv new file mode 100644 index 000000000..9135d763c --- /dev/null +++ b/test/runtimes/blacklist_nodejs12.4.0.csv @@ -0,0 +1,47 @@ +test name,bug id,comment +benchmark/test-benchmark-fs.js,, +benchmark/test-benchmark-module.js,, +benchmark/test-benchmark-napi.js,, +doctool/test-make-doc.js,b/68848110,Expected to fail. +fixtures/test-error-first-line-offset.js,, +fixtures/test-fs-readfile-error.js,, +fixtures/test-fs-stat-sync-overflow.js,, +internet/test-dgram-broadcast-multi-process.js,, +internet/test-dgram-multicast-multi-process.js,, +internet/test-dgram-multicast-set-interface-lo.js,, +parallel/test-cluster-dgram-reuse.js,b/64024294, +parallel/test-dgram-bind-fd.js,b/132447356, +parallel/test-dgram-create-socket-handle-fd.js,b/132447238, +parallel/test-dgram-createSocket-type.js,b/68847739, +parallel/test-dgram-socket-buffer-size.js,b/68847921, +parallel/test-fs-access.js,, +parallel/test-fs-write-stream-double-close.js +parallel/test-fs-write-stream-throw-type-error.js,b/110226209, +parallel/test-fs-write-stream.js,, +parallel/test-http2-respond-file-error-pipe-offset.js,, +parallel/test-os.js,, +parallel/test-process-uid-gid.js,, +pseudo-tty/test-assert-colors.js,, +pseudo-tty/test-assert-no-color.js,, +pseudo-tty/test-assert-position-indicator.js,, +pseudo-tty/test-async-wrap-getasyncid-tty.js,, +pseudo-tty/test-fatal-error.js,, +pseudo-tty/test-handle-wrap-isrefed-tty.js,, +pseudo-tty/test-readable-tty-keepalive.js,, +pseudo-tty/test-set-raw-mode-reset-process-exit.js,, +pseudo-tty/test-set-raw-mode-reset-signal.js,, +pseudo-tty/test-set-raw-mode-reset.js,, +pseudo-tty/test-stderr-stdout-handle-sigwinch.js,, +pseudo-tty/test-stdout-read.js,, +pseudo-tty/test-tty-color-support.js,, +pseudo-tty/test-tty-isatty.js,, +pseudo-tty/test-tty-stdin-call-end.js,, +pseudo-tty/test-tty-stdin-end.js,, +pseudo-tty/test-stdin-write.js,, +pseudo-tty/test-tty-stdout-end.js,, +pseudo-tty/test-tty-stdout-resize.js,, +pseudo-tty/test-tty-stream-constructors.js,, +pseudo-tty/test-tty-window-size.js,, +pseudo-tty/test-tty-wrap.js,, +pummel/test-net-pingpong.js,, +pummel/test-vm-memleak.js,, diff --git a/test/runtimes/build_defs.bzl b/test/runtimes/build_defs.bzl index 5e3065342..7edd12c17 100644 --- a/test/runtimes/build_defs.bzl +++ b/test/runtimes/build_defs.bzl @@ -6,19 +6,26 @@ def runtime_test( lang, image, shard_count = 50, - size = "enormous"): + size = "enormous", + blacklist_file = ""): + args = [ + "--lang", + lang, + "--image", + image, + ] + data = [ + ":runner", + ] + if blacklist_file != "": + args += ["--blacklist_file", "test/runtimes/" + blacklist_file] + data += [blacklist_file] + sh_test( name = lang + "_test", srcs = ["runner.sh"], - args = [ - "--lang", - lang, - "--image", - image, - ], - data = [ - ":runner", - ], + args = args, + data = data, size = size, shard_count = shard_count, tags = [ diff --git a/test/runtimes/runner.go b/test/runtimes/runner.go index 3a15f59a7..bec37c69d 100644 --- a/test/runtimes/runner.go +++ b/test/runtimes/runner.go @@ -16,6 +16,7 @@ package main import ( + "encoding/csv" "flag" "fmt" "io" @@ -31,8 +32,9 @@ import ( ) var ( - lang = flag.String("lang", "", "language runtime to test") - image = flag.String("image", "", "docker image with runtime tests") + lang = flag.String("lang", "", "language runtime to test") + image = flag.String("image", "", "docker image with runtime tests") + blacklistFile = flag.String("blacklist_file", "", "file containing blacklist of tests to exclude, in CSV format with fields: test name, bug id, comment") ) // Wait time for each test to run. @@ -52,6 +54,13 @@ func main() { // defered functions before exiting. It returns an exit code that should be // passed to os.Exit. func runTests() int { + // Get tests to blacklist. + blacklist, err := getBlacklist() + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting blacklist: %s\n", err.Error()) + return 1 + } + // Create a single docker container that will be used for all tests. d := dockerutil.MakeDocker("gvisor-" + *lang) defer d.CleanUp() @@ -59,7 +68,7 @@ func runTests() int { // Get a slice of tests to run. This will also start a single Docker // container that will be used to run each test. The final test will // stop the Docker container. - tests, err := getTests(d) + tests, err := getTests(d, blacklist) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", err.Error()) return 1 @@ -71,7 +80,7 @@ func runTests() int { // getTests returns a slice of tests to run, subject to the shard size and // index. -func getTests(d dockerutil.Docker) ([]testing.InternalTest, error) { +func getTests(d dockerutil.Docker, blacklist map[string]struct{}) ([]testing.InternalTest, error) { // Pull the image. if err := dockerutil.Pull(*image); err != nil { return nil, fmt.Errorf("docker pull %q failed: %v", *image, err) @@ -106,12 +115,18 @@ func getTests(d dockerutil.Docker) ([]testing.InternalTest, error) { itests = append(itests, testing.InternalTest{ Name: tc, F: func(t *testing.T) { + // Is the test blacklisted? + if _, ok := blacklist[tc]; ok { + t.Skip("SKIP: blacklisted test %q", tc) + } + var ( now = time.Now() done = make(chan struct{}) output string err error ) + go func() { fmt.Printf("RUNNING %s...\n", tc) output, err = d.Exec("/proctor", "--runtime", *lang, "--test", tc) @@ -134,6 +149,43 @@ func getTests(d dockerutil.Docker) ([]testing.InternalTest, error) { return itests, nil } +// getBlacklist reads the blacklist file and returns a set of test names to +// exclude. +func getBlacklist() (map[string]struct{}, error) { + blacklist := make(map[string]struct{}) + if *blacklistFile == "" { + return blacklist, nil + } + file, err := testutil.FindFile(*blacklistFile) + if err != nil { + return nil, err + } + f, err := os.Open(file) + if err != nil { + return nil, err + } + defer f.Close() + + r := csv.NewReader(f) + + // First line is header. Skip it. + if _, err := r.Read(); err != nil { + return nil, err + } + + for { + record, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + blacklist[record[0]] = struct{}{} + } + return blacklist, nil +} + // testDeps implements testing.testDeps (an unexported interface), and is // required to use testing.MainStart. type testDeps struct{} |