Runtime test batch executor

Earlier we were docker exec-ing each test at a time. However invoking the test
framework has a fixed overhead which made it infeasible to make the runtime
tests run as presubmits. This change now executes tests in batches of 50 (can
be altered). This really speeds up testing process.

With this change, the following tests can be run in reasonable times:
- Go
- Nodejs
- Php
- Python

PiperOrigin-RevId: 320763916

12 files changed, 98 insertions, 57 deletions

diff --git a/test/runtimes/BUILD b/test/runtimes/BUILD
index 022de5ff7..f98d02e00 100644
--- a/test/runtimes/BUILD
+++ b/test/runtimes/BUILD
@@ -6,28 +6,33 @@ runtime_test(
     name = "go1.12",
     exclude_file = "exclude_go1.12.csv",
     lang = "go",
+    shard_count = 5,
 )
 
 runtime_test(
     name = "java11",
     exclude_file = "exclude_java11.csv",
     lang = "java",
+    shard_count = 10,
 )
 
 runtime_test(
     name = "nodejs12.4.0",
     exclude_file = "exclude_nodejs12.4.0.csv",
     lang = "nodejs",
+    shard_count = 5,
 )
 
 runtime_test(
     name = "php7.3.6",
     exclude_file = "exclude_php7.3.6.csv",
     lang = "php",
+    shard_count = 5,
 )
 
 runtime_test(
     name = "python3.7.3",
     exclude_file = "exclude_python3.7.3.csv",
     lang = "python",
+    shard_count = 5,
 )
diff --git a/test/runtimes/exclude_nodejs12.4.0.csv b/test/runtimes/exclude_nodejs12.4.0.csv
index 4ab4e2927..e7edfa0a5 100644
--- a/test/runtimes/exclude_nodejs12.4.0.csv
+++ b/test/runtimes/exclude_nodejs12.4.0.csv
@@ -9,6 +9,8 @@ fixtures/test-fs-stat-sync-overflow.js,,
 internet/test-dgram-broadcast-multi-process.js,,
 internet/test-dgram-multicast-multi-process.js,,
 internet/test-dgram-multicast-set-interface-lo.js,,
+internet/test-doctool-versions.js,,
+internet/test-uv-threadpool-schedule.js,,
 parallel/test-cluster-dgram-reuse.js,b/64024294,
 parallel/test-dgram-bind-fd.js,b/132447356,
 parallel/test-dgram-create-socket-handle-fd.js,b/132447238,
@@ -45,3 +47,4 @@ pseudo-tty/test-tty-window-size.js,,
 pseudo-tty/test-tty-wrap.js,,
 pummel/test-net-pingpong.js,,
 pummel/test-vm-memleak.js,,
+tick-processor/test-tick-processor-builtin.js,,
diff --git a/test/runtimes/exclude_php7.3.6.csv b/test/runtimes/exclude_php7.3.6.csv
index 456bf7487..f3606bfe8 100644
--- a/test/runtimes/exclude_php7.3.6.csv
+++ b/test/runtimes/exclude_php7.3.6.csv
@@ -8,6 +8,9 @@ ext/mbstring/tests/bug77165.phpt,,
 ext/mbstring/tests/bug77454.phpt,,
 ext/mbstring/tests/mb_convert_encoding_leak.phpt,,
 ext/mbstring/tests/mb_strrpos_encoding_3rd_param.phpt,,
+ext/session/tests/session_set_save_handler_class_018.phpt,,
+ext/session/tests/session_set_save_handler_iface_003.phpt,,
+ext/session/tests/session_set_save_handler_variation4.phpt,,
 ext/standard/tests/file/filetype_variation.phpt,,
 ext/standard/tests/file/fopen_variation19.phpt,,
 ext/standard/tests/file/php_fd_wrapper_01.phpt,,
@@ -21,9 +24,12 @@ ext/standard/tests/file/symlink_link_linkinfo_is_link_variation8.phpt,,
 ext/standard/tests/general_functions/escapeshellarg_bug71270.phpt,,
 ext/standard/tests/general_functions/escapeshellcmd_bug71270.phpt,,
 ext/standard/tests/network/bug20134.phpt,,
+ext/standard/tests/streams/stream_socket_sendto.phpt,,
+ext/standard/tests/strings/007.phpt,,
 tests/output/stream_isatty_err.phpt,b/68720279,
 tests/output/stream_isatty_in-err.phpt,b/68720282,
 tests/output/stream_isatty_in-out-err.phpt,,
 tests/output/stream_isatty_in-out.phpt,b/68720299,
 tests/output/stream_isatty_out-err.phpt,b/68720311,
 tests/output/stream_isatty_out.phpt,b/68720325,
+Zend/tests/concat_003.phpt,,
diff --git a/test/runtimes/proctor/go.go b/test/runtimes/proctor/go.go
index 3e2d5d8db..073c2959d 100644
--- a/test/runtimes/proctor/go.go
+++ b/test/runtimes/proctor/go.go
@@ -74,17 +74,26 @@ func (goRunner) ListTests() ([]string, error) {
 	return append(toolSlice, diskFiltered...), nil
 }
 
-// TestCmd implements TestRunner.TestCmd.
-func (goRunner) TestCmd(test string) *exec.Cmd {
-	// Check if test exists on disk by searching for file of the same name.
-	// This will determine whether or not it is a Go test on disk.
-	if strings.HasSuffix(test, ".go") {
-		// Test has suffix ".go" which indicates a disk test, run it as such.
-		cmd := exec.Command("go", "run", "run.go", "-v", "--", test)
+// TestCmds implements TestRunner.TestCmds.
+func (goRunner) TestCmds(tests []string) []*exec.Cmd {
+	var toolTests, onDiskTests []string
+	for _, test := range tests {
+		if strings.HasSuffix(test, ".go") {
+			onDiskTests = append(onDiskTests, test)
+		} else {
+			toolTests = append(toolTests, test)
+		}
+	}
+
+	var cmds []*exec.Cmd
+	if len(toolTests) > 0 {
+		cmds = append(cmds, exec.Command("go", "tool", "dist", "test", "-run", strings.Join(toolTests, "\\|")))
+	}
+	if len(onDiskTests) > 0 {
+		cmd := exec.Command("go", append([]string{"run", "run.go", "-v", "--"}, onDiskTests...)...)
 		cmd.Dir = goTestDir
-		return cmd
+		cmds = append(cmds, cmd)
 	}
 
-	// No ".go" suffix, run as a tool test.
-	return exec.Command("go", "tool", "dist", "test", "-run", test)
+	return cmds
 }
diff --git a/test/runtimes/proctor/java.go b/test/runtimes/proctor/java.go
index 8b362029d..737fbe23e 100644
--- a/test/runtimes/proctor/java.go
+++ b/test/runtimes/proctor/java.go
@@ -60,12 +60,14 @@ func (javaRunner) ListTests() ([]string, error) {
 	return testSlice, nil
 }
 
-// TestCmd implements TestRunner.TestCmd.
-func (javaRunner) TestCmd(test string) *exec.Cmd {
-	args := []string{
-		"-noreport",
-		"-dir:" + javaTestDir,
-		test,
-	}
-	return exec.Command("jtreg", args...)
+// TestCmds implements TestRunner.TestCmds.
+func (javaRunner) TestCmds(tests []string) []*exec.Cmd {
+	args := append(
+		[]string{
+			"-noreport",
+			"-dir:" + javaTestDir,
+		},
+		tests...,
+	)
+	return []*exec.Cmd{exec.Command("jtreg", args...)}
 }
diff --git a/test/runtimes/proctor/nodejs.go b/test/runtimes/proctor/nodejs.go
index bd57db444..23d6edc72 100644
--- a/test/runtimes/proctor/nodejs.go
+++ b/test/runtimes/proctor/nodejs.go
@@ -39,8 +39,8 @@ func (nodejsRunner) ListTests() ([]string, error) {
 	return testSlice, nil
 }
 
-// TestCmd implements TestRunner.TestCmd.
-func (nodejsRunner) TestCmd(test string) *exec.Cmd {
-	args := []string{filepath.Join("tools", "test.py"), test}
-	return exec.Command("/usr/bin/python", args...)
+// TestCmds implements TestRunner.TestCmds.
+func (nodejsRunner) TestCmds(tests []string) []*exec.Cmd {
+	args := append([]string{filepath.Join("tools", "test.py")}, tests...)
+	return []*exec.Cmd{exec.Command("/usr/bin/python", args...)}
 }
diff --git a/test/runtimes/proctor/php.go b/test/runtimes/proctor/php.go
index 9115040e1..6a83d64e3 100644
--- a/test/runtimes/proctor/php.go
+++ b/test/runtimes/proctor/php.go
@@ -17,6 +17,7 @@ package main
 import (
 	"os/exec"
 	"regexp"
+	"strings"
 )
 
 var phpTestRegEx = regexp.MustCompile(`^.+\.phpt$`)
@@ -35,8 +36,8 @@ func (phpRunner) ListTests() ([]string, error) {
 	return testSlice, nil
 }
 
-// TestCmd implements TestRunner.TestCmd.
-func (phpRunner) TestCmd(test string) *exec.Cmd {
-	args := []string{"test", "TESTS=" + test}
-	return exec.Command("make", args...)
+// TestCmds implements TestRunner.TestCmds.
+func (phpRunner) TestCmds(tests []string) []*exec.Cmd {
+	args := []string{"test", "TESTS=" + strings.Join(tests, " ")}
+	return []*exec.Cmd{exec.Command("make", args...)}
 }
diff --git a/test/runtimes/proctor/proctor.go b/test/runtimes/proctor/proctor.go
index b54abe434..9e0642424 100644
--- a/test/runtimes/proctor/proctor.go
+++ b/test/runtimes/proctor/proctor.go
@@ -25,6 +25,7 @@ import (
 	"os/signal"
 	"path/filepath"
 	"regexp"
+	"strings"
 	"syscall"
 )
 
@@ -34,15 +35,18 @@ type TestRunner interface {
 	// ListTests returns a string slice of tests available to run.
 	ListTests() ([]string, error)
 
-	// TestCmd returns an *exec.Cmd that will run the given test.
-	TestCmd(test string) *exec.Cmd
+	// TestCmds returns a slice of *exec.Cmd that will run the given tests.
+	// There is no correlation between the number of exec.Cmds returned and the
+	// number of tests. It could return one command to run all tests or a few
+	// commands that collectively run all.
+	TestCmds(tests []string) []*exec.Cmd
 }
 
 var (
-	runtime  = flag.String("runtime", "", "name of runtime")
-	list     = flag.Bool("list", false, "list all available tests")
-	testName = flag.String("test", "", "run a single test from the list of available tests")
-	pause    = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children")
+	runtime   = flag.String("runtime", "", "name of runtime")
+	list      = flag.Bool("list", false, "list all available tests")
+	testNames = flag.String("tests", "", "run a subset of the available tests")
+	pause     = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children")
 )
 
 func main() {
@@ -75,18 +79,20 @@ func main() {
 	}
 
 	var tests []string
-	if *testName == "" {
+	if *testNames == "" {
 		// Run every test.
 		tests, err = tr.ListTests()
 		if err != nil {
 			log.Fatalf("failed to get all tests: %v", err)
 		}
 	} else {
-		// Run a single test.
-		tests = []string{*testName}
+		// Run subset of test.
+		tests = strings.Split(*testNames, ",")
 	}
-	for _, test := range tests {
-		cmd := tr.TestCmd(test)
+
+	// Run tests.
+	cmds := tr.TestCmds(tests)
+	for _, cmd := range cmds {
 		cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
 		if err := cmd.Run(); err != nil {
 			log.Fatalf("FAIL: %v", err)
diff --git a/test/runtimes/proctor/python.go b/test/runtimes/proctor/python.go
index b9e0fbe6f..7c598801b 100644
--- a/test/runtimes/proctor/python.go
+++ b/test/runtimes/proctor/python.go
@@ -42,8 +42,8 @@ func (pythonRunner) ListTests() ([]string, error) {
 	return toolSlice, nil
 }
 
-// TestCmd implements TestRunner.TestCmd.
-func (pythonRunner) TestCmd(test string) *exec.Cmd {
-	args := []string{"-m", "test", test}
-	return exec.Command("./python", args...)
+// TestCmds implements TestRunner.TestCmds.
+func (pythonRunner) TestCmds(tests []string) []*exec.Cmd {
+	args := append([]string{"-m", "test"}, tests...)
+	return []*exec.Cmd{exec.Command("./python", args...)}
 }
diff --git a/test/runtimes/runner/BUILD b/test/runtimes/runner/BUILD
index 3972244b9..dc0d5d5b4 100644
--- a/test/runtimes/runner/BUILD
+++ b/test/runtimes/runner/BUILD
@@ -8,6 +8,7 @@ go_binary(
     srcs = ["main.go"],
     visibility = ["//test/runtimes:__pkg__"],
     deps = [
+        "//pkg/log",
         "//pkg/test/dockerutil",
         "//pkg/test/testutil",
     ],
diff --git a/test/runtimes/runner/exclude_test.go b/test/runtimes/runner/exclude_test.go
index c08755894..67c2170c8 100644
--- a/test/runtimes/runner/exclude_test.go
+++ b/test/runtimes/runner/exclude_test.go
@@ -26,7 +26,7 @@ func TestMain(m *testing.M) {
 }
 
 // Test that the exclude file parses without error.
-func TestBlacklists(t *testing.T) {
+func TestExcludelist(t *testing.T) {
 	ex, err := getExcludes()
 	if err != nil {
 		t.Fatalf("error parsing exclude file: %v", err)
diff --git a/test/runtimes/runner/main.go b/test/runtimes/runner/main.go
index 2a0f62c73..e230912c9 100644
--- a/test/runtimes/runner/main.go
+++ b/test/runtimes/runner/main.go
@@ -27,6 +27,7 @@ import (
 	"testing"
 	"time"
 
+	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/test/dockerutil"
 	"gvisor.dev/gvisor/pkg/test/testutil"
 )
@@ -35,10 +36,11 @@ var (
 	lang        = flag.String("lang", "", "language runtime to test")
 	image       = flag.String("image", "", "docker image with runtime tests")
 	excludeFile = flag.String("exclude_file", "", "file containing list of tests to exclude, in CSV format with fields: test name, bug id, comment")
+	batchSize   = flag.Int("batch", 50, "number of test cases run in one command")
 )
 
 // Wait time for each test to run.
-const timeout = 5 * time.Minute
+const timeout = 45 * time.Minute
 
 func main() {
 	flag.Parse()
@@ -110,17 +112,23 @@ func getTests(ctx context.Context, d *dockerutil.Container, excludes map[string]
 	}
 
 	var itests []testing.InternalTest
-	for _, tci := range indices {
-		// Capture tc in this scope.
-		tc := tests[tci]
+	for i := 0; i < len(indices); i += *batchSize {
+		var tcs []string
+		end := i + *batchSize
+		if end > len(indices) {
+			end = len(indices)
+		}
+		for _, tc := range indices[i:end] {
+			// Add test if not excluded.
+			if _, ok := excludes[tests[tc]]; ok {
+				log.Infof("Skipping test case %s\n", tests[tc])
+				continue
+			}
+			tcs = append(tcs, tests[tc])
+		}
 		itests = append(itests, testing.InternalTest{
-			Name: tc,
+			Name: strings.Join(tcs, ", "),
 			F: func(t *testing.T) {
-				// Is the test excluded?
-				if _, ok := excludes[tc]; ok {
-					t.Skipf("SKIP: excluded test %q", tc)
-				}
-
 				var (
 					now    = time.Now()
 					done   = make(chan struct{})
@@ -129,20 +137,20 @@ func getTests(ctx context.Context, d *dockerutil.Container, excludes map[string]
 				)
 
 				go func() {
-					fmt.Printf("RUNNING %s...\n", tc)
-					output, err = d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", *lang, "--test", tc)
+					fmt.Printf("RUNNING the following in a batch\n%s\n", strings.Join(tcs, "\n"))
+					output, err = d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", *lang, "--tests", strings.Join(tcs, ","))
 					close(done)
 				}()
 
 				select {
 				case <-done:
 					if err == nil {
-						fmt.Printf("PASS: %s (%v)\n\n", tc, time.Since(now))
+						fmt.Printf("PASS: (%v)\n\n", time.Since(now))
 						return
 					}
-					t.Errorf("FAIL: %s (%v):\n%s\n", tc, time.Since(now), output)
+					t.Errorf("FAIL: (%v):\n%s\n", time.Since(now), output)
 				case <-time.After(timeout):
-					t.Errorf("TIMEOUT: %s (%v):\n%s\n", tc, time.Since(now), output)
+					t.Errorf("TIMEOUT: (%v):\n%s\n", time.Since(now), output)
 				}
 			},
 		})