summaryrefslogtreecommitdiffhomepage
path: root/test/runtimes
diff options
context:
space:
mode:
Diffstat (limited to 'test/runtimes')
-rw-r--r--test/runtimes/BUILD5
-rw-r--r--test/runtimes/exclude_nodejs12.4.0.csv3
-rw-r--r--test/runtimes/exclude_php7.3.6.csv6
-rw-r--r--test/runtimes/proctor/go.go29
-rw-r--r--test/runtimes/proctor/java.go18
-rw-r--r--test/runtimes/proctor/nodejs.go8
-rw-r--r--test/runtimes/proctor/php.go9
-rw-r--r--test/runtimes/proctor/proctor.go28
-rw-r--r--test/runtimes/proctor/python.go8
-rw-r--r--test/runtimes/runner/BUILD1
-rw-r--r--test/runtimes/runner/exclude_test.go2
-rw-r--r--test/runtimes/runner/main.go57
12 files changed, 111 insertions, 63 deletions
diff --git a/test/runtimes/BUILD b/test/runtimes/BUILD
index 022de5ff7..f98d02e00 100644
--- a/test/runtimes/BUILD
+++ b/test/runtimes/BUILD
@@ -6,28 +6,33 @@ runtime_test(
name = "go1.12",
exclude_file = "exclude_go1.12.csv",
lang = "go",
+ shard_count = 5,
)
runtime_test(
name = "java11",
exclude_file = "exclude_java11.csv",
lang = "java",
+ shard_count = 10,
)
runtime_test(
name = "nodejs12.4.0",
exclude_file = "exclude_nodejs12.4.0.csv",
lang = "nodejs",
+ shard_count = 5,
)
runtime_test(
name = "php7.3.6",
exclude_file = "exclude_php7.3.6.csv",
lang = "php",
+ shard_count = 5,
)
runtime_test(
name = "python3.7.3",
exclude_file = "exclude_python3.7.3.csv",
lang = "python",
+ shard_count = 5,
)
diff --git a/test/runtimes/exclude_nodejs12.4.0.csv b/test/runtimes/exclude_nodejs12.4.0.csv
index 4ab4e2927..e7edfa0a5 100644
--- a/test/runtimes/exclude_nodejs12.4.0.csv
+++ b/test/runtimes/exclude_nodejs12.4.0.csv
@@ -9,6 +9,8 @@ fixtures/test-fs-stat-sync-overflow.js,,
internet/test-dgram-broadcast-multi-process.js,,
internet/test-dgram-multicast-multi-process.js,,
internet/test-dgram-multicast-set-interface-lo.js,,
+internet/test-doctool-versions.js,,
+internet/test-uv-threadpool-schedule.js,,
parallel/test-cluster-dgram-reuse.js,b/64024294,
parallel/test-dgram-bind-fd.js,b/132447356,
parallel/test-dgram-create-socket-handle-fd.js,b/132447238,
@@ -45,3 +47,4 @@ pseudo-tty/test-tty-window-size.js,,
pseudo-tty/test-tty-wrap.js,,
pummel/test-net-pingpong.js,,
pummel/test-vm-memleak.js,,
+tick-processor/test-tick-processor-builtin.js,,
diff --git a/test/runtimes/exclude_php7.3.6.csv b/test/runtimes/exclude_php7.3.6.csv
index 456bf7487..f3606bfe8 100644
--- a/test/runtimes/exclude_php7.3.6.csv
+++ b/test/runtimes/exclude_php7.3.6.csv
@@ -8,6 +8,9 @@ ext/mbstring/tests/bug77165.phpt,,
ext/mbstring/tests/bug77454.phpt,,
ext/mbstring/tests/mb_convert_encoding_leak.phpt,,
ext/mbstring/tests/mb_strrpos_encoding_3rd_param.phpt,,
+ext/session/tests/session_set_save_handler_class_018.phpt,,
+ext/session/tests/session_set_save_handler_iface_003.phpt,,
+ext/session/tests/session_set_save_handler_variation4.phpt,,
ext/standard/tests/file/filetype_variation.phpt,,
ext/standard/tests/file/fopen_variation19.phpt,,
ext/standard/tests/file/php_fd_wrapper_01.phpt,,
@@ -21,9 +24,12 @@ ext/standard/tests/file/symlink_link_linkinfo_is_link_variation8.phpt,,
ext/standard/tests/general_functions/escapeshellarg_bug71270.phpt,,
ext/standard/tests/general_functions/escapeshellcmd_bug71270.phpt,,
ext/standard/tests/network/bug20134.phpt,,
+ext/standard/tests/streams/stream_socket_sendto.phpt,,
+ext/standard/tests/strings/007.phpt,,
tests/output/stream_isatty_err.phpt,b/68720279,
tests/output/stream_isatty_in-err.phpt,b/68720282,
tests/output/stream_isatty_in-out-err.phpt,,
tests/output/stream_isatty_in-out.phpt,b/68720299,
tests/output/stream_isatty_out-err.phpt,b/68720311,
tests/output/stream_isatty_out.phpt,b/68720325,
+Zend/tests/concat_003.phpt,,
diff --git a/test/runtimes/proctor/go.go b/test/runtimes/proctor/go.go
index 3e2d5d8db..073c2959d 100644
--- a/test/runtimes/proctor/go.go
+++ b/test/runtimes/proctor/go.go
@@ -74,17 +74,26 @@ func (goRunner) ListTests() ([]string, error) {
return append(toolSlice, diskFiltered...), nil
}
-// TestCmd implements TestRunner.TestCmd.
-func (goRunner) TestCmd(test string) *exec.Cmd {
- // Check if test exists on disk by searching for file of the same name.
- // This will determine whether or not it is a Go test on disk.
- if strings.HasSuffix(test, ".go") {
- // Test has suffix ".go" which indicates a disk test, run it as such.
- cmd := exec.Command("go", "run", "run.go", "-v", "--", test)
+// TestCmds implements TestRunner.TestCmds.
+func (goRunner) TestCmds(tests []string) []*exec.Cmd {
+ var toolTests, onDiskTests []string
+ for _, test := range tests {
+ if strings.HasSuffix(test, ".go") {
+ onDiskTests = append(onDiskTests, test)
+ } else {
+ toolTests = append(toolTests, test)
+ }
+ }
+
+ var cmds []*exec.Cmd
+ if len(toolTests) > 0 {
+ cmds = append(cmds, exec.Command("go", "tool", "dist", "test", "-run", strings.Join(toolTests, "\\|")))
+ }
+ if len(onDiskTests) > 0 {
+ cmd := exec.Command("go", append([]string{"run", "run.go", "-v", "--"}, onDiskTests...)...)
cmd.Dir = goTestDir
- return cmd
+ cmds = append(cmds, cmd)
}
- // No ".go" suffix, run as a tool test.
- return exec.Command("go", "tool", "dist", "test", "-run", test)
+ return cmds
}
diff --git a/test/runtimes/proctor/java.go b/test/runtimes/proctor/java.go
index 8b362029d..737fbe23e 100644
--- a/test/runtimes/proctor/java.go
+++ b/test/runtimes/proctor/java.go
@@ -60,12 +60,14 @@ func (javaRunner) ListTests() ([]string, error) {
return testSlice, nil
}
-// TestCmd implements TestRunner.TestCmd.
-func (javaRunner) TestCmd(test string) *exec.Cmd {
- args := []string{
- "-noreport",
- "-dir:" + javaTestDir,
- test,
- }
- return exec.Command("jtreg", args...)
+// TestCmds implements TestRunner.TestCmds.
+func (javaRunner) TestCmds(tests []string) []*exec.Cmd {
+ args := append(
+ []string{
+ "-noreport",
+ "-dir:" + javaTestDir,
+ },
+ tests...,
+ )
+ return []*exec.Cmd{exec.Command("jtreg", args...)}
}
diff --git a/test/runtimes/proctor/nodejs.go b/test/runtimes/proctor/nodejs.go
index bd57db444..23d6edc72 100644
--- a/test/runtimes/proctor/nodejs.go
+++ b/test/runtimes/proctor/nodejs.go
@@ -39,8 +39,8 @@ func (nodejsRunner) ListTests() ([]string, error) {
return testSlice, nil
}
-// TestCmd implements TestRunner.TestCmd.
-func (nodejsRunner) TestCmd(test string) *exec.Cmd {
- args := []string{filepath.Join("tools", "test.py"), test}
- return exec.Command("/usr/bin/python", args...)
+// TestCmds implements TestRunner.TestCmds.
+func (nodejsRunner) TestCmds(tests []string) []*exec.Cmd {
+ args := append([]string{filepath.Join("tools", "test.py")}, tests...)
+ return []*exec.Cmd{exec.Command("/usr/bin/python", args...)}
}
diff --git a/test/runtimes/proctor/php.go b/test/runtimes/proctor/php.go
index 9115040e1..6a83d64e3 100644
--- a/test/runtimes/proctor/php.go
+++ b/test/runtimes/proctor/php.go
@@ -17,6 +17,7 @@ package main
import (
"os/exec"
"regexp"
+ "strings"
)
var phpTestRegEx = regexp.MustCompile(`^.+\.phpt$`)
@@ -35,8 +36,8 @@ func (phpRunner) ListTests() ([]string, error) {
return testSlice, nil
}
-// TestCmd implements TestRunner.TestCmd.
-func (phpRunner) TestCmd(test string) *exec.Cmd {
- args := []string{"test", "TESTS=" + test}
- return exec.Command("make", args...)
+// TestCmds implements TestRunner.TestCmds.
+func (phpRunner) TestCmds(tests []string) []*exec.Cmd {
+ args := []string{"test", "TESTS=" + strings.Join(tests, " ")}
+ return []*exec.Cmd{exec.Command("make", args...)}
}
diff --git a/test/runtimes/proctor/proctor.go b/test/runtimes/proctor/proctor.go
index b54abe434..9e0642424 100644
--- a/test/runtimes/proctor/proctor.go
+++ b/test/runtimes/proctor/proctor.go
@@ -25,6 +25,7 @@ import (
"os/signal"
"path/filepath"
"regexp"
+ "strings"
"syscall"
)
@@ -34,15 +35,18 @@ type TestRunner interface {
// ListTests returns a string slice of tests available to run.
ListTests() ([]string, error)
- // TestCmd returns an *exec.Cmd that will run the given test.
- TestCmd(test string) *exec.Cmd
+ // TestCmds returns a slice of *exec.Cmd that will run the given tests.
+ // There is no correlation between the number of exec.Cmds returned and the
+ // number of tests. It could return one command to run all tests or a few
+ // commands that collectively run all.
+ TestCmds(tests []string) []*exec.Cmd
}
var (
- runtime = flag.String("runtime", "", "name of runtime")
- list = flag.Bool("list", false, "list all available tests")
- testName = flag.String("test", "", "run a single test from the list of available tests")
- pause = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children")
+ runtime = flag.String("runtime", "", "name of runtime")
+ list = flag.Bool("list", false, "list all available tests")
+ testNames = flag.String("tests", "", "run a subset of the available tests")
+ pause = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children")
)
func main() {
@@ -75,18 +79,20 @@ func main() {
}
var tests []string
- if *testName == "" {
+ if *testNames == "" {
// Run every test.
tests, err = tr.ListTests()
if err != nil {
log.Fatalf("failed to get all tests: %v", err)
}
} else {
- // Run a single test.
- tests = []string{*testName}
+ // Run subset of test.
+ tests = strings.Split(*testNames, ",")
}
- for _, test := range tests {
- cmd := tr.TestCmd(test)
+
+ // Run tests.
+ cmds := tr.TestCmds(tests)
+ for _, cmd := range cmds {
cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
if err := cmd.Run(); err != nil {
log.Fatalf("FAIL: %v", err)
diff --git a/test/runtimes/proctor/python.go b/test/runtimes/proctor/python.go
index b9e0fbe6f..7c598801b 100644
--- a/test/runtimes/proctor/python.go
+++ b/test/runtimes/proctor/python.go
@@ -42,8 +42,8 @@ func (pythonRunner) ListTests() ([]string, error) {
return toolSlice, nil
}
-// TestCmd implements TestRunner.TestCmd.
-func (pythonRunner) TestCmd(test string) *exec.Cmd {
- args := []string{"-m", "test", test}
- return exec.Command("./python", args...)
+// TestCmds implements TestRunner.TestCmds.
+func (pythonRunner) TestCmds(tests []string) []*exec.Cmd {
+ args := append([]string{"-m", "test"}, tests...)
+ return []*exec.Cmd{exec.Command("./python", args...)}
}
diff --git a/test/runtimes/runner/BUILD b/test/runtimes/runner/BUILD
index 3972244b9..dc0d5d5b4 100644
--- a/test/runtimes/runner/BUILD
+++ b/test/runtimes/runner/BUILD
@@ -8,6 +8,7 @@ go_binary(
srcs = ["main.go"],
visibility = ["//test/runtimes:__pkg__"],
deps = [
+ "//pkg/log",
"//pkg/test/dockerutil",
"//pkg/test/testutil",
],
diff --git a/test/runtimes/runner/exclude_test.go b/test/runtimes/runner/exclude_test.go
index c08755894..67c2170c8 100644
--- a/test/runtimes/runner/exclude_test.go
+++ b/test/runtimes/runner/exclude_test.go
@@ -26,7 +26,7 @@ func TestMain(m *testing.M) {
}
// Test that the exclude file parses without error.
-func TestBlacklists(t *testing.T) {
+func TestExcludelist(t *testing.T) {
ex, err := getExcludes()
if err != nil {
t.Fatalf("error parsing exclude file: %v", err)
diff --git a/test/runtimes/runner/main.go b/test/runtimes/runner/main.go
index 54d1169ef..e230912c9 100644
--- a/test/runtimes/runner/main.go
+++ b/test/runtimes/runner/main.go
@@ -16,6 +16,7 @@
package main
import (
+ "context"
"encoding/csv"
"flag"
"fmt"
@@ -26,6 +27,7 @@ import (
"testing"
"time"
+ "gvisor.dev/gvisor/pkg/log"
"gvisor.dev/gvisor/pkg/test/dockerutil"
"gvisor.dev/gvisor/pkg/test/testutil"
)
@@ -34,10 +36,11 @@ var (
lang = flag.String("lang", "", "language runtime to test")
image = flag.String("image", "", "docker image with runtime tests")
excludeFile = flag.String("exclude_file", "", "file containing list of tests to exclude, in CSV format with fields: test name, bug id, comment")
+ batchSize = flag.Int("batch", 50, "number of test cases run in one command")
)
// Wait time for each test to run.
-const timeout = 5 * time.Minute
+const timeout = 45 * time.Minute
func main() {
flag.Parse()
@@ -60,13 +63,19 @@ func runTests() int {
}
// Construct the shared docker instance.
- d := dockerutil.MakeDocker(testutil.DefaultLogger(*lang))
- defer d.CleanUp()
+ ctx := context.Background()
+ d := dockerutil.MakeContainer(ctx, testutil.DefaultLogger(*lang))
+ defer d.CleanUp(ctx)
+
+ if err := testutil.TouchShardStatusFile(); err != nil {
+ fmt.Fprintf(os.Stderr, "error touching status shard file: %v\n", err)
+ return 1
+ }
// Get a slice of tests to run. This will also start a single Docker
// container that will be used to run each test. The final test will
// stop the Docker container.
- tests, err := getTests(d, excludes)
+ tests, err := getTests(ctx, d, excludes)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err.Error())
return 1
@@ -77,18 +86,18 @@ func runTests() int {
}
// getTests executes all tests as table tests.
-func getTests(d *dockerutil.Docker, excludes map[string]struct{}) ([]testing.InternalTest, error) {
+func getTests(ctx context.Context, d *dockerutil.Container, excludes map[string]struct{}) ([]testing.InternalTest, error) {
// Start the container.
opts := dockerutil.RunOpts{
Image: fmt.Sprintf("runtimes/%s", *image),
}
d.CopyFiles(&opts, "/proctor", "test/runtimes/proctor/proctor")
- if err := d.Spawn(opts, "/proctor/proctor", "--pause"); err != nil {
+ if err := d.Spawn(ctx, opts, "/proctor/proctor", "--pause"); err != nil {
return nil, fmt.Errorf("docker run failed: %v", err)
}
// Get a list of all tests in the image.
- list, err := d.Exec(dockerutil.RunOpts{}, "/proctor/proctor", "--runtime", *lang, "--list")
+ list, err := d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", *lang, "--list")
if err != nil {
return nil, fmt.Errorf("docker exec failed: %v", err)
}
@@ -103,17 +112,23 @@ func getTests(d *dockerutil.Docker, excludes map[string]struct{}) ([]testing.Int
}
var itests []testing.InternalTest
- for _, tci := range indices {
- // Capture tc in this scope.
- tc := tests[tci]
+ for i := 0; i < len(indices); i += *batchSize {
+ var tcs []string
+ end := i + *batchSize
+ if end > len(indices) {
+ end = len(indices)
+ }
+ for _, tc := range indices[i:end] {
+ // Add test if not excluded.
+ if _, ok := excludes[tests[tc]]; ok {
+ log.Infof("Skipping test case %s\n", tests[tc])
+ continue
+ }
+ tcs = append(tcs, tests[tc])
+ }
itests = append(itests, testing.InternalTest{
- Name: tc,
+ Name: strings.Join(tcs, ", "),
F: func(t *testing.T) {
- // Is the test excluded?
- if _, ok := excludes[tc]; ok {
- t.Skipf("SKIP: excluded test %q", tc)
- }
-
var (
now = time.Now()
done = make(chan struct{})
@@ -122,20 +137,20 @@ func getTests(d *dockerutil.Docker, excludes map[string]struct{}) ([]testing.Int
)
go func() {
- fmt.Printf("RUNNING %s...\n", tc)
- output, err = d.Exec(dockerutil.RunOpts{}, "/proctor/proctor", "--runtime", *lang, "--test", tc)
+ fmt.Printf("RUNNING the following in a batch\n%s\n", strings.Join(tcs, "\n"))
+ output, err = d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", *lang, "--tests", strings.Join(tcs, ","))
close(done)
}()
select {
case <-done:
if err == nil {
- fmt.Printf("PASS: %s (%v)\n\n", tc, time.Since(now))
+ fmt.Printf("PASS: (%v)\n\n", time.Since(now))
return
}
- t.Errorf("FAIL: %s (%v):\n%s\n", tc, time.Since(now), output)
+ t.Errorf("FAIL: (%v):\n%s\n", time.Since(now), output)
case <-time.After(timeout):
- t.Errorf("TIMEOUT: %s (%v):\n%s\n", tc, time.Since(now), output)
+ t.Errorf("TIMEOUT: (%v):\n%s\n", time.Since(now), output)
}
},
})
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-20 08:37:47 +0000