summaryrefslogtreecommitdiffhomepage
path: root/test/runtimes
diff options
context:
space:
mode:
Diffstat (limited to 'test/runtimes')
-rw-r--r--test/runtimes/BUILD1
-rw-r--r--test/runtimes/blacklist_nodejs12.4.0.csv47
-rw-r--r--test/runtimes/build_defs.bzl29
-rw-r--r--test/runtimes/images/proctor/proctor.go31
-rw-r--r--test/runtimes/runner.go154
5 files changed, 218 insertions, 44 deletions
diff --git a/test/runtimes/BUILD b/test/runtimes/BUILD
index dfb4e2a97..1cde74cfc 100644
--- a/test/runtimes/BUILD
+++ b/test/runtimes/BUILD
@@ -26,6 +26,7 @@ runtime_test(
)
runtime_test(
+ blacklist_file = "blacklist_nodejs12.4.0.csv",
image = "gcr.io/gvisor-presubmit/nodejs12.4.0",
lang = "nodejs",
)
diff --git a/test/runtimes/blacklist_nodejs12.4.0.csv b/test/runtimes/blacklist_nodejs12.4.0.csv
new file mode 100644
index 000000000..9135d763c
--- /dev/null
+++ b/test/runtimes/blacklist_nodejs12.4.0.csv
@@ -0,0 +1,47 @@
+test name,bug id,comment
+benchmark/test-benchmark-fs.js,,
+benchmark/test-benchmark-module.js,,
+benchmark/test-benchmark-napi.js,,
+doctool/test-make-doc.js,b/68848110,Expected to fail.
+fixtures/test-error-first-line-offset.js,,
+fixtures/test-fs-readfile-error.js,,
+fixtures/test-fs-stat-sync-overflow.js,,
+internet/test-dgram-broadcast-multi-process.js,,
+internet/test-dgram-multicast-multi-process.js,,
+internet/test-dgram-multicast-set-interface-lo.js,,
+parallel/test-cluster-dgram-reuse.js,b/64024294,
+parallel/test-dgram-bind-fd.js,b/132447356,
+parallel/test-dgram-create-socket-handle-fd.js,b/132447238,
+parallel/test-dgram-createSocket-type.js,b/68847739,
+parallel/test-dgram-socket-buffer-size.js,b/68847921,
+parallel/test-fs-access.js,,
+parallel/test-fs-write-stream-double-close.js
+parallel/test-fs-write-stream-throw-type-error.js,b/110226209,
+parallel/test-fs-write-stream.js,,
+parallel/test-http2-respond-file-error-pipe-offset.js,,
+parallel/test-os.js,,
+parallel/test-process-uid-gid.js,,
+pseudo-tty/test-assert-colors.js,,
+pseudo-tty/test-assert-no-color.js,,
+pseudo-tty/test-assert-position-indicator.js,,
+pseudo-tty/test-async-wrap-getasyncid-tty.js,,
+pseudo-tty/test-fatal-error.js,,
+pseudo-tty/test-handle-wrap-isrefed-tty.js,,
+pseudo-tty/test-readable-tty-keepalive.js,,
+pseudo-tty/test-set-raw-mode-reset-process-exit.js,,
+pseudo-tty/test-set-raw-mode-reset-signal.js,,
+pseudo-tty/test-set-raw-mode-reset.js,,
+pseudo-tty/test-stderr-stdout-handle-sigwinch.js,,
+pseudo-tty/test-stdout-read.js,,
+pseudo-tty/test-tty-color-support.js,,
+pseudo-tty/test-tty-isatty.js,,
+pseudo-tty/test-tty-stdin-call-end.js,,
+pseudo-tty/test-tty-stdin-end.js,,
+pseudo-tty/test-stdin-write.js,,
+pseudo-tty/test-tty-stdout-end.js,,
+pseudo-tty/test-tty-stdout-resize.js,,
+pseudo-tty/test-tty-stream-constructors.js,,
+pseudo-tty/test-tty-window-size.js,,
+pseudo-tty/test-tty-wrap.js,,
+pummel/test-net-pingpong.js,,
+pummel/test-vm-memleak.js,,
diff --git a/test/runtimes/build_defs.bzl b/test/runtimes/build_defs.bzl
index 19aceb6fb..7edd12c17 100644
--- a/test/runtimes/build_defs.bzl
+++ b/test/runtimes/build_defs.bzl
@@ -5,20 +5,27 @@
def runtime_test(
lang,
image,
- shard_count = 20,
- size = "enormous"):
+ shard_count = 50,
+ size = "enormous",
+ blacklist_file = ""):
+ args = [
+ "--lang",
+ lang,
+ "--image",
+ image,
+ ]
+ data = [
+ ":runner",
+ ]
+ if blacklist_file != "":
+ args += ["--blacklist_file", "test/runtimes/" + blacklist_file]
+ data += [blacklist_file]
+
sh_test(
name = lang + "_test",
srcs = ["runner.sh"],
- args = [
- "--lang",
- lang,
- "--image",
- image,
- ],
- data = [
- ":runner",
- ],
+ args = args,
+ data = data,
size = size,
shard_count = shard_count,
tags = [
diff --git a/test/runtimes/images/proctor/proctor.go b/test/runtimes/images/proctor/proctor.go
index e2c198b46..e6178e82b 100644
--- a/test/runtimes/images/proctor/proctor.go
+++ b/test/runtimes/images/proctor/proctor.go
@@ -22,8 +22,10 @@ import (
"log"
"os"
"os/exec"
+ "os/signal"
"path/filepath"
"regexp"
+ "syscall"
)
// TestRunner is an interface that must be implemented for each runtime
@@ -40,11 +42,17 @@ var (
runtime = flag.String("runtime", "", "name of runtime")
list = flag.Bool("list", false, "list all available tests")
test = flag.String("test", "", "run a single test from the list of available tests")
+ pause = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children")
)
func main() {
flag.Parse()
+ if *pause {
+ pauseAndReap()
+ panic("pauseAndReap should never return")
+ }
+
if *runtime == "" {
log.Fatalf("runtime flag must be provided")
}
@@ -73,7 +81,7 @@ func main() {
cmd := tr.TestCmd(*test)
cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
if err := cmd.Run(); err != nil {
- log.Fatalf("FAIL %q: %v", err)
+ log.Fatalf("FAIL: %v", err)
}
}
@@ -94,6 +102,27 @@ func testRunnerForRuntime(runtime string) (TestRunner, error) {
return nil, fmt.Errorf("invalid runtime %q", runtime)
}
+// pauseAndReap is like init. It runs forever and reaps any children.
+func pauseAndReap() {
+ // Get notified of any new children.
+ ch := make(chan os.Signal, 1)
+ signal.Notify(ch, syscall.SIGCHLD)
+
+ for {
+ if _, ok := <-ch; !ok {
+ // Channel closed. This should not happen.
+ panic("signal channel closed")
+ }
+
+ // Reap the child.
+ for {
+ if cpid, _ := syscall.Wait4(-1, nil, syscall.WNOHANG, nil); cpid < 1 {
+ break
+ }
+ }
+ }
+}
+
// search is a helper function to find tests in the given directory that match
// the regex.
func search(root string, testFilter *regexp.Regexp) ([]string, error) {
diff --git a/test/runtimes/runner.go b/test/runtimes/runner.go
index 3111963eb..bec37c69d 100644
--- a/test/runtimes/runner.go
+++ b/test/runtimes/runner.go
@@ -16,8 +16,10 @@
package main
import (
+ "encoding/csv"
"flag"
"fmt"
+ "io"
"log"
"os"
"sort"
@@ -30,8 +32,9 @@ import (
)
var (
- lang = flag.String("lang", "", "language runtime to test")
- image = flag.String("image", "", "docker image with runtime tests")
+ lang = flag.String("lang", "", "language runtime to test")
+ image = flag.String("image", "", "docker image with runtime tests")
+ blacklistFile = flag.String("blacklist_file", "", "file containing blacklist of tests to exclude, in CSV format with fields: test name, bug id, comment")
)
// Wait time for each test to run.
@@ -43,30 +46,59 @@ func main() {
fmt.Fprintf(os.Stderr, "lang and image flags must not be empty\n")
os.Exit(1)
}
- tests, err := testsForImage(*lang, *image)
+
+ os.Exit(runTests())
+}
+
+// runTests is a helper that is called by main. It exists so that we can run
+// defered functions before exiting. It returns an exit code that should be
+// passed to os.Exit.
+func runTests() int {
+ // Get tests to blacklist.
+ blacklist, err := getBlacklist()
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error getting blacklist: %s\n", err.Error())
+ return 1
+ }
+
+ // Create a single docker container that will be used for all tests.
+ d := dockerutil.MakeDocker("gvisor-" + *lang)
+ defer d.CleanUp()
+
+ // Get a slice of tests to run. This will also start a single Docker
+ // container that will be used to run each test. The final test will
+ // stop the Docker container.
+ tests, err := getTests(d, blacklist)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err.Error())
- os.Exit(1)
+ return 1
}
- testing.Main(func(a, b string) (bool, error) {
- return a == b, nil
- }, tests, nil, nil)
+ m := testing.MainStart(testDeps{}, tests, nil, nil)
+ return m.Run()
}
-func testsForImage(lang, image string) ([]testing.InternalTest, error) {
- if err := dockerutil.Pull(image); err != nil {
- return nil, fmt.Errorf("docker pull failed: %v", err)
+// getTests returns a slice of tests to run, subject to the shard size and
+// index.
+func getTests(d dockerutil.Docker, blacklist map[string]struct{}) ([]testing.InternalTest, error) {
+ // Pull the image.
+ if err := dockerutil.Pull(*image); err != nil {
+ return nil, fmt.Errorf("docker pull %q failed: %v", *image, err)
}
- c := dockerutil.MakeDocker("gvisor-list")
- list, err := c.RunFg(image, "--runtime", lang, "--list")
- defer c.CleanUp()
- if err != nil {
+ // Run proctor with --pause flag to keep container alive forever.
+ if err := d.Run(*image, "--pause"); err != nil {
return nil, fmt.Errorf("docker run failed: %v", err)
}
- // Get subset of tests corresponding to shard.
+ // Get a list of all tests in the image.
+ list, err := d.Exec("/proctor", "--runtime", *lang, "--list")
+ if err != nil {
+ return nil, fmt.Errorf("docker exec failed: %v", err)
+ }
+
+ // Calculate a subset of tests to run corresponding to the current
+ // shard.
tests := strings.Fields(list)
sort.Strings(tests)
begin, end, err := testutil.TestBoundsForShard(len(tests))
@@ -77,33 +109,91 @@ func testsForImage(lang, image string) ([]testing.InternalTest, error) {
tests = tests[begin:end]
var itests []testing.InternalTest
- for i, tc := range tests {
+ for _, tc := range tests {
// Capture tc in this scope.
tc := tc
itests = append(itests, testing.InternalTest{
Name: tc,
F: func(t *testing.T) {
- d := dockerutil.MakeDocker(fmt.Sprintf("gvisor-test-%d", i))
- defer d.CleanUp()
- if err := d.Run(image, "--runtime", lang, "--test", tc); err != nil {
- t.Fatalf("docker test %q failed to run: %v", tc, err)
+ // Is the test blacklisted?
+ if _, ok := blacklist[tc]; ok {
+ t.Skip("SKIP: blacklisted test %q", tc)
}
- status, err := d.Wait(timeout)
- if err != nil {
- t.Fatalf("docker test %q failed to wait: %v", tc, err)
- }
- logs, err := d.Logs()
- if err != nil {
- t.Fatalf("docker test %q failed to supply logs: %v", tc, err)
- }
- if status == 0 {
- t.Logf("test %q passed", tc)
- return
+ var (
+ now = time.Now()
+ done = make(chan struct{})
+ output string
+ err error
+ )
+
+ go func() {
+ fmt.Printf("RUNNING %s...\n", tc)
+ output, err = d.Exec("/proctor", "--runtime", *lang, "--test", tc)
+ close(done)
+ }()
+
+ select {
+ case <-done:
+ if err == nil {
+ fmt.Printf("PASS: %s (%v)\n\n", tc, time.Since(now))
+ return
+ }
+ t.Errorf("FAIL: %s (%v):\n%s\n", tc, time.Since(now), output)
+ case <-time.After(timeout):
+ t.Errorf("TIMEOUT: %s (%v):\n%s\n", tc, time.Since(now), output)
}
- t.Errorf("test %q failed: %v", tc, logs)
},
})
}
return itests, nil
}
+
+// getBlacklist reads the blacklist file and returns a set of test names to
+// exclude.
+func getBlacklist() (map[string]struct{}, error) {
+ blacklist := make(map[string]struct{})
+ if *blacklistFile == "" {
+ return blacklist, nil
+ }
+ file, err := testutil.FindFile(*blacklistFile)
+ if err != nil {
+ return nil, err
+ }
+ f, err := os.Open(file)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ r := csv.NewReader(f)
+
+ // First line is header. Skip it.
+ if _, err := r.Read(); err != nil {
+ return nil, err
+ }
+
+ for {
+ record, err := r.Read()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+ blacklist[record[0]] = struct{}{}
+ }
+ return blacklist, nil
+}
+
+// testDeps implements testing.testDeps (an unexported interface), and is
+// required to use testing.MainStart.
+type testDeps struct{}
+
+func (f testDeps) MatchString(a, b string) (bool, error) { return a == b, nil }
+func (f testDeps) StartCPUProfile(io.Writer) error { return nil }
+func (f testDeps) StopCPUProfile() {}
+func (f testDeps) WriteProfileTo(string, io.Writer, int) error { return nil }
+func (f testDeps) ImportPath() string { return "" }
+func (f testDeps) StartTestLog(io.Writer) {}
+func (f testDeps) StopTestLog() error { return nil }
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-26 07:04:03 +0000