From 1481499fe27157ad2716c00682f6ad819115a6c7 Mon Sep 17 00:00:00 2001
From: Adin Scannell <ascannell@google.com>
Date: Thu, 23 Apr 2020 11:32:08 -0700
Subject: Simplify Docker test infrastructure.

This change adds a layer of abstraction around the internal Docker APIs,
and eliminates all direct dependencies on Dockerfiles in the infrastructure.

A subsequent change will automated the generation of local images (with
efficient caching). Note that this change drops the use of bazel container
rules, as that experiment does not seem to be viable.

PiperOrigin-RevId: 308095430
---
 test/runtimes/proctor/BUILD           |  27 ++++++
 test/runtimes/proctor/go.go           |  90 +++++++++++++++++++
 test/runtimes/proctor/java.go         |  71 +++++++++++++++
 test/runtimes/proctor/nodejs.go       |  46 ++++++++++
 test/runtimes/proctor/php.go          |  42 +++++++++
 test/runtimes/proctor/proctor.go      | 163 ++++++++++++++++++++++++++++++++++
 test/runtimes/proctor/proctor_test.go | 127 ++++++++++++++++++++++++++
 test/runtimes/proctor/python.go       |  49 ++++++++++
 8 files changed, 615 insertions(+)
 create mode 100644 test/runtimes/proctor/BUILD
 create mode 100644 test/runtimes/proctor/go.go
 create mode 100644 test/runtimes/proctor/java.go
 create mode 100644 test/runtimes/proctor/nodejs.go
 create mode 100644 test/runtimes/proctor/php.go
 create mode 100644 test/runtimes/proctor/proctor.go
 create mode 100644 test/runtimes/proctor/proctor_test.go
 create mode 100644 test/runtimes/proctor/python.go

(limited to 'test/runtimes/proctor')

diff --git a/test/runtimes/proctor/BUILD b/test/runtimes/proctor/BUILD
new file mode 100644
index 000000000..50a26d182
--- /dev/null
+++ b/test/runtimes/proctor/BUILD
@@ -0,0 +1,27 @@
+load("//tools:defs.bzl", "go_binary", "go_test")
+
+package(licenses = ["notice"])
+
+go_binary(
+    name = "proctor",
+    srcs = [
+        "go.go",
+        "java.go",
+        "nodejs.go",
+        "php.go",
+        "proctor.go",
+        "python.go",
+    ],
+    pure = True,
+    visibility = ["//test/runtimes:__pkg__"],
+)
+
+go_test(
+    name = "proctor_test",
+    size = "small",
+    srcs = ["proctor_test.go"],
+    library = ":proctor",
+    deps = [
+        "//pkg/test/testutil",
+    ],
+)
diff --git a/test/runtimes/proctor/go.go b/test/runtimes/proctor/go.go
new file mode 100644
index 000000000..3e2d5d8db
--- /dev/null
+++ b/test/runtimes/proctor/go.go
@@ -0,0 +1,90 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+)
+
+var (
+	goTestRegEx = regexp.MustCompile(`^.+\.go$`)
+
+	// Directories with .dir contain helper files for tests.
+	// Exclude benchmarks and stress tests.
+	goDirFilter = regexp.MustCompile(`^(bench|stress)\/.+$|^.+\.dir.+$`)
+)
+
+// Location of Go tests on disk.
+const goTestDir = "/usr/local/go/test"
+
+// goRunner implements TestRunner for Go.
+//
+// There are two types of Go tests: "Go tool tests" and "Go tests on disk".
+// "Go tool tests" are found and executed using `go tool dist test`. "Go tests
+// on disk" are found in the /usr/local/go/test directory and are executed
+// using `go run run.go`.
+type goRunner struct{}
+
+var _ TestRunner = goRunner{}
+
+// ListTests implements TestRunner.ListTests.
+func (goRunner) ListTests() ([]string, error) {
+	// Go tool dist test tests.
+	args := []string{"tool", "dist", "test", "-list"}
+	cmd := exec.Command("go", args...)
+	cmd.Stderr = os.Stderr
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list: %v", err)
+	}
+	var toolSlice []string
+	for _, test := range strings.Split(string(out), "\n") {
+		toolSlice = append(toolSlice, test)
+	}
+
+	// Go tests on disk.
+	diskSlice, err := search(goTestDir, goTestRegEx)
+	if err != nil {
+		return nil, err
+	}
+	// Remove items from /bench/, /stress/ and .dir files
+	diskFiltered := diskSlice[:0]
+	for _, file := range diskSlice {
+		if !goDirFilter.MatchString(file) {
+			diskFiltered = append(diskFiltered, file)
+		}
+	}
+
+	return append(toolSlice, diskFiltered...), nil
+}
+
+// TestCmd implements TestRunner.TestCmd.
+func (goRunner) TestCmd(test string) *exec.Cmd {
+	// Check if test exists on disk by searching for file of the same name.
+	// This will determine whether or not it is a Go test on disk.
+	if strings.HasSuffix(test, ".go") {
+		// Test has suffix ".go" which indicates a disk test, run it as such.
+		cmd := exec.Command("go", "run", "run.go", "-v", "--", test)
+		cmd.Dir = goTestDir
+		return cmd
+	}
+
+	// No ".go" suffix, run as a tool test.
+	return exec.Command("go", "tool", "dist", "test", "-run", test)
+}
diff --git a/test/runtimes/proctor/java.go b/test/runtimes/proctor/java.go
new file mode 100644
index 000000000..8b362029d
--- /dev/null
+++ b/test/runtimes/proctor/java.go
@@ -0,0 +1,71 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+)
+
+// Directories to exclude from tests.
+var javaExclDirs = regexp.MustCompile(`(^(sun\/security)|(java\/util\/stream)|(java\/time)| )`)
+
+// Location of java tests.
+const javaTestDir = "/root/test/jdk"
+
+// javaRunner implements TestRunner for Java.
+type javaRunner struct{}
+
+var _ TestRunner = javaRunner{}
+
+// ListTests implements TestRunner.ListTests.
+func (javaRunner) ListTests() ([]string, error) {
+	args := []string{
+		"-dir:" + javaTestDir,
+		"-ignore:quiet",
+		"-a",
+		"-listtests",
+		":jdk_core",
+		":jdk_svc",
+		":jdk_sound",
+		":jdk_imageio",
+	}
+	cmd := exec.Command("jtreg", args...)
+	cmd.Stderr = os.Stderr
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("jtreg -listtests : %v", err)
+	}
+	var testSlice []string
+	for _, test := range strings.Split(string(out), "\n") {
+		if !javaExclDirs.MatchString(test) {
+			testSlice = append(testSlice, test)
+		}
+	}
+	return testSlice, nil
+}
+
+// TestCmd implements TestRunner.TestCmd.
+func (javaRunner) TestCmd(test string) *exec.Cmd {
+	args := []string{
+		"-noreport",
+		"-dir:" + javaTestDir,
+		test,
+	}
+	return exec.Command("jtreg", args...)
+}
diff --git a/test/runtimes/proctor/nodejs.go b/test/runtimes/proctor/nodejs.go
new file mode 100644
index 000000000..bd57db444
--- /dev/null
+++ b/test/runtimes/proctor/nodejs.go
@@ -0,0 +1,46 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"os/exec"
+	"path/filepath"
+	"regexp"
+)
+
+var nodejsTestRegEx = regexp.MustCompile(`^test-[^-].+\.js$`)
+
+// Location of nodejs tests relative to working dir.
+const nodejsTestDir = "test"
+
+// nodejsRunner implements TestRunner for NodeJS.
+type nodejsRunner struct{}
+
+var _ TestRunner = nodejsRunner{}
+
+// ListTests implements TestRunner.ListTests.
+func (nodejsRunner) ListTests() ([]string, error) {
+	testSlice, err := search(nodejsTestDir, nodejsTestRegEx)
+	if err != nil {
+		return nil, err
+	}
+	return testSlice, nil
+}
+
+// TestCmd implements TestRunner.TestCmd.
+func (nodejsRunner) TestCmd(test string) *exec.Cmd {
+	args := []string{filepath.Join("tools", "test.py"), test}
+	return exec.Command("/usr/bin/python", args...)
+}
diff --git a/test/runtimes/proctor/php.go b/test/runtimes/proctor/php.go
new file mode 100644
index 000000000..9115040e1
--- /dev/null
+++ b/test/runtimes/proctor/php.go
@@ -0,0 +1,42 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"os/exec"
+	"regexp"
+)
+
+var phpTestRegEx = regexp.MustCompile(`^.+\.phpt$`)
+
+// phpRunner implements TestRunner for PHP.
+type phpRunner struct{}
+
+var _ TestRunner = phpRunner{}
+
+// ListTests implements TestRunner.ListTests.
+func (phpRunner) ListTests() ([]string, error) {
+	testSlice, err := search(".", phpTestRegEx)
+	if err != nil {
+		return nil, err
+	}
+	return testSlice, nil
+}
+
+// TestCmd implements TestRunner.TestCmd.
+func (phpRunner) TestCmd(test string) *exec.Cmd {
+	args := []string{"test", "TESTS=" + test}
+	return exec.Command("make", args...)
+}
diff --git a/test/runtimes/proctor/proctor.go b/test/runtimes/proctor/proctor.go
new file mode 100644
index 000000000..b54abe434
--- /dev/null
+++ b/test/runtimes/proctor/proctor.go
@@ -0,0 +1,163 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Binary proctor runs the test for a particular runtime. It is meant to be
+// included in Docker images for all runtime tests.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"regexp"
+	"syscall"
+)
+
+// TestRunner is an interface that must be implemented for each runtime
+// integrated with proctor.
+type TestRunner interface {
+	// ListTests returns a string slice of tests available to run.
+	ListTests() ([]string, error)
+
+	// TestCmd returns an *exec.Cmd that will run the given test.
+	TestCmd(test string) *exec.Cmd
+}
+
+var (
+	runtime  = flag.String("runtime", "", "name of runtime")
+	list     = flag.Bool("list", false, "list all available tests")
+	testName = flag.String("test", "", "run a single test from the list of available tests")
+	pause    = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children")
+)
+
+func main() {
+	flag.Parse()
+
+	if *pause {
+		pauseAndReap()
+		panic("pauseAndReap should never return")
+	}
+
+	if *runtime == "" {
+		log.Fatalf("runtime flag must be provided")
+	}
+
+	tr, err := testRunnerForRuntime(*runtime)
+	if err != nil {
+		log.Fatalf("%v", err)
+	}
+
+	// List tests.
+	if *list {
+		tests, err := tr.ListTests()
+		if err != nil {
+			log.Fatalf("failed to list tests: %v", err)
+		}
+		for _, test := range tests {
+			fmt.Println(test)
+		}
+		return
+	}
+
+	var tests []string
+	if *testName == "" {
+		// Run every test.
+		tests, err = tr.ListTests()
+		if err != nil {
+			log.Fatalf("failed to get all tests: %v", err)
+		}
+	} else {
+		// Run a single test.
+		tests = []string{*testName}
+	}
+	for _, test := range tests {
+		cmd := tr.TestCmd(test)
+		cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
+		if err := cmd.Run(); err != nil {
+			log.Fatalf("FAIL: %v", err)
+		}
+	}
+}
+
+// testRunnerForRuntime returns a new TestRunner for the given runtime.
+func testRunnerForRuntime(runtime string) (TestRunner, error) {
+	switch runtime {
+	case "go":
+		return goRunner{}, nil
+	case "java":
+		return javaRunner{}, nil
+	case "nodejs":
+		return nodejsRunner{}, nil
+	case "php":
+		return phpRunner{}, nil
+	case "python":
+		return pythonRunner{}, nil
+	}
+	return nil, fmt.Errorf("invalid runtime %q", runtime)
+}
+
+// pauseAndReap is like init. It runs forever and reaps any children.
+func pauseAndReap() {
+	// Get notified of any new children.
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, syscall.SIGCHLD)
+
+	for {
+		if _, ok := <-ch; !ok {
+			// Channel closed. This should not happen.
+			panic("signal channel closed")
+		}
+
+		// Reap the child.
+		for {
+			if cpid, _ := syscall.Wait4(-1, nil, syscall.WNOHANG, nil); cpid < 1 {
+				break
+			}
+		}
+	}
+}
+
+// search is a helper function to find tests in the given directory that match
+// the regex.
+func search(root string, testFilter *regexp.Regexp) ([]string, error) {
+	var testSlice []string
+
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		name := filepath.Base(path)
+
+		if info.IsDir() || !testFilter.MatchString(name) {
+			return nil
+		}
+
+		relPath, err := filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+		testSlice = append(testSlice, relPath)
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("walking %q: %v", root, err)
+	}
+
+	return testSlice, nil
+}
diff --git a/test/runtimes/proctor/proctor_test.go b/test/runtimes/proctor/proctor_test.go
new file mode 100644
index 000000000..6ef2de085
--- /dev/null
+++ b/test/runtimes/proctor/proctor_test.go
@@ -0,0 +1,127 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"regexp"
+	"strings"
+	"testing"
+
+	"gvisor.dev/gvisor/pkg/test/testutil"
+)
+
+func touch(t *testing.T, name string) {
+	t.Helper()
+	f, err := os.Create(name)
+	if err != nil {
+		t.Fatalf("error creating file %q: %v", name, err)
+	}
+	if err := f.Close(); err != nil {
+		t.Fatalf("error closing file %q: %v", name, err)
+	}
+}
+
+func TestSearchEmptyDir(t *testing.T) {
+	td, err := ioutil.TempDir(testutil.TmpDir(), "searchtest")
+	if err != nil {
+		t.Fatalf("error creating searchtest: %v", err)
+	}
+	defer os.RemoveAll(td)
+
+	var want []string
+
+	testFilter := regexp.MustCompile(`^test-[^-].+\.tc$`)
+	got, err := search(td, testFilter)
+	if err != nil {
+		t.Errorf("search error: %v", err)
+	}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("Found %#v; want %#v", got, want)
+	}
+}
+
+func TestSearch(t *testing.T) {
+	td, err := ioutil.TempDir(testutil.TmpDir(), "searchtest")
+	if err != nil {
+		t.Fatalf("error creating searchtest: %v", err)
+	}
+	defer os.RemoveAll(td)
+
+	// Creating various files similar to the test filter regex.
+	files := []string{
+		"emp/",
+		"tee/",
+		"test-foo.tc",
+		"test-foo.tc",
+		"test-bar.tc",
+		"test-sam.tc",
+		"Test-que.tc",
+		"test-brett",
+		"test--abc.tc",
+		"test---xyz.tc",
+		"test-bool.TC",
+		"--test-gvs.tc",
+		" test-pew.tc",
+		"dir/test_baz.tc",
+		"dir/testsnap.tc",
+		"dir/test-luk.tc",
+		"dir/nest/test-ok.tc",
+		"dir/dip/diz/goog/test-pack.tc",
+		"dir/dip/diz/wobble/thud/test-cas.e",
+		"dir/dip/diz/wobble/thud/test-cas.tc",
+	}
+	want := []string{
+		"dir/dip/diz/goog/test-pack.tc",
+		"dir/dip/diz/wobble/thud/test-cas.tc",
+		"dir/nest/test-ok.tc",
+		"dir/test-luk.tc",
+		"test-bar.tc",
+		"test-foo.tc",
+		"test-sam.tc",
+	}
+
+	for _, item := range files {
+		if strings.HasSuffix(item, "/") {
+			// This item is a directory, create it.
+			if err := os.MkdirAll(filepath.Join(td, item), 0755); err != nil {
+				t.Fatalf("error making directory: %v", err)
+			}
+		} else {
+			// This item is a file, create the directory and touch file.
+			// Create directory in which file should be created
+			fullDirPath := filepath.Join(td, filepath.Dir(item))
+			if err := os.MkdirAll(fullDirPath, 0755); err != nil {
+				t.Fatalf("error making directory: %v", err)
+			}
+			// Create file with full path to file.
+			touch(t, filepath.Join(td, item))
+		}
+	}
+
+	testFilter := regexp.MustCompile(`^test-[^-].+\.tc$`)
+	got, err := search(td, testFilter)
+	if err != nil {
+		t.Errorf("search error: %v", err)
+	}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("Found %#v; want %#v", got, want)
+	}
+}
diff --git a/test/runtimes/proctor/python.go b/test/runtimes/proctor/python.go
new file mode 100644
index 000000000..b9e0fbe6f
--- /dev/null
+++ b/test/runtimes/proctor/python.go
@@ -0,0 +1,49 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+// pythonRunner implements TestRunner for Python.
+type pythonRunner struct{}
+
+var _ TestRunner = pythonRunner{}
+
+// ListTests implements TestRunner.ListTests.
+func (pythonRunner) ListTests() ([]string, error) {
+	args := []string{"-m", "test", "--list-tests"}
+	cmd := exec.Command("./python", args...)
+	cmd.Stderr = os.Stderr
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list: %v", err)
+	}
+	var toolSlice []string
+	for _, test := range strings.Split(string(out), "\n") {
+		toolSlice = append(toolSlice, test)
+	}
+	return toolSlice, nil
+}
+
+// TestCmd implements TestRunner.TestCmd.
+func (pythonRunner) TestCmd(test string) *exec.Cmd {
+	args := []string{"-m", "test", test}
+	return exec.Command("./python", args...)
+}
-- 
cgit v1.2.3


From e5d9e7c3b2920ef6f7034aa5180f059b6ccecb54 Mon Sep 17 00:00:00 2001
From: gVisor bot <gvisor-bot@google.com>
Date: Tue, 5 May 2020 12:41:59 -0700
Subject: Internal change.

PiperOrigin-RevId: 310001058
---
 test/runtimes/proctor/BUILD | 1 +
 1 file changed, 1 insertion(+)

(limited to 'test/runtimes/proctor')

diff --git a/test/runtimes/proctor/BUILD b/test/runtimes/proctor/BUILD
index 50a26d182..da1e331e1 100644
--- a/test/runtimes/proctor/BUILD
+++ b/test/runtimes/proctor/BUILD
@@ -21,6 +21,7 @@ go_test(
     size = "small",
     srcs = ["proctor_test.go"],
     library = ":proctor",
+    nocgo = 1,
     deps = [
         "//pkg/test/testutil",
     ],
-- 
cgit v1.2.3


From 12f90947612bc5d1d7122871e8c7c1f6b9e4e303 Mon Sep 17 00:00:00 2001
From: Andrei Vagin <avagin@google.com>
Date: Mon, 8 Jun 2020 14:27:41 -0700
Subject: test/runtimes/proctor: remove an unknown nocgo attribute from go_test
 rule

PiperOrigin-RevId: 315353408
---
 test/runtimes/proctor/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'test/runtimes/proctor')

diff --git a/test/runtimes/proctor/BUILD b/test/runtimes/proctor/BUILD
index da1e331e1..f76e2ddc0 100644
--- a/test/runtimes/proctor/BUILD
+++ b/test/runtimes/proctor/BUILD
@@ -21,7 +21,7 @@ go_test(
     size = "small",
     srcs = ["proctor_test.go"],
     library = ":proctor",
-    nocgo = 1,
+    pure = True,
     deps = [
         "//pkg/test/testutil",
     ],
-- 
cgit v1.2.3


From 69f2059e5d38bacac4bcda7912cca580ab70914d Mon Sep 17 00:00:00 2001
From: Ayush Ranjan <ayushranjan@google.com>
Date: Sat, 11 Jul 2020 08:17:07 -0700
Subject: Runtime test batch executor

Earlier we were docker exec-ing each test at a time. However invoking the test
framework has a fixed overhead which made it infeasible to make the runtime
tests run as presubmits. This change now executes tests in batches of 50 (can
be altered). This really speeds up testing process.

With this change, the following tests can be run in reasonable times:
- Go
- Nodejs
- Php
- Python

PiperOrigin-RevId: 320763916
---
 test/runtimes/BUILD                    |  5 +++++
 test/runtimes/exclude_nodejs12.4.0.csv |  3 +++
 test/runtimes/exclude_php7.3.6.csv     |  6 ++++++
 test/runtimes/proctor/go.go            | 29 +++++++++++++++++---------
 test/runtimes/proctor/java.go          | 18 +++++++++-------
 test/runtimes/proctor/nodejs.go        |  8 +++----
 test/runtimes/proctor/php.go           |  9 ++++----
 test/runtimes/proctor/proctor.go       | 28 +++++++++++++++----------
 test/runtimes/proctor/python.go        |  8 +++----
 test/runtimes/runner/BUILD             |  1 +
 test/runtimes/runner/exclude_test.go   |  2 +-
 test/runtimes/runner/main.go           | 38 ++++++++++++++++++++--------------
 12 files changed, 98 insertions(+), 57 deletions(-)

(limited to 'test/runtimes/proctor')

diff --git a/test/runtimes/BUILD b/test/runtimes/BUILD
index 022de5ff7..f98d02e00 100644
--- a/test/runtimes/BUILD
+++ b/test/runtimes/BUILD
@@ -6,28 +6,33 @@ runtime_test(
     name = "go1.12",
     exclude_file = "exclude_go1.12.csv",
     lang = "go",
+    shard_count = 5,
 )
 
 runtime_test(
     name = "java11",
     exclude_file = "exclude_java11.csv",
     lang = "java",
+    shard_count = 10,
 )
 
 runtime_test(
     name = "nodejs12.4.0",
     exclude_file = "exclude_nodejs12.4.0.csv",
     lang = "nodejs",
+    shard_count = 5,
 )
 
 runtime_test(
     name = "php7.3.6",
     exclude_file = "exclude_php7.3.6.csv",
     lang = "php",
+    shard_count = 5,
 )
 
 runtime_test(
     name = "python3.7.3",
     exclude_file = "exclude_python3.7.3.csv",
     lang = "python",
+    shard_count = 5,
 )
diff --git a/test/runtimes/exclude_nodejs12.4.0.csv b/test/runtimes/exclude_nodejs12.4.0.csv
index 4ab4e2927..e7edfa0a5 100644
--- a/test/runtimes/exclude_nodejs12.4.0.csv
+++ b/test/runtimes/exclude_nodejs12.4.0.csv
@@ -9,6 +9,8 @@ fixtures/test-fs-stat-sync-overflow.js,,
 internet/test-dgram-broadcast-multi-process.js,,
 internet/test-dgram-multicast-multi-process.js,,
 internet/test-dgram-multicast-set-interface-lo.js,,
+internet/test-doctool-versions.js,,
+internet/test-uv-threadpool-schedule.js,,
 parallel/test-cluster-dgram-reuse.js,b/64024294,
 parallel/test-dgram-bind-fd.js,b/132447356,
 parallel/test-dgram-create-socket-handle-fd.js,b/132447238,
@@ -45,3 +47,4 @@ pseudo-tty/test-tty-window-size.js,,
 pseudo-tty/test-tty-wrap.js,,
 pummel/test-net-pingpong.js,,
 pummel/test-vm-memleak.js,,
+tick-processor/test-tick-processor-builtin.js,,
diff --git a/test/runtimes/exclude_php7.3.6.csv b/test/runtimes/exclude_php7.3.6.csv
index 456bf7487..f3606bfe8 100644
--- a/test/runtimes/exclude_php7.3.6.csv
+++ b/test/runtimes/exclude_php7.3.6.csv
@@ -8,6 +8,9 @@ ext/mbstring/tests/bug77165.phpt,,
 ext/mbstring/tests/bug77454.phpt,,
 ext/mbstring/tests/mb_convert_encoding_leak.phpt,,
 ext/mbstring/tests/mb_strrpos_encoding_3rd_param.phpt,,
+ext/session/tests/session_set_save_handler_class_018.phpt,,
+ext/session/tests/session_set_save_handler_iface_003.phpt,,
+ext/session/tests/session_set_save_handler_variation4.phpt,,
 ext/standard/tests/file/filetype_variation.phpt,,
 ext/standard/tests/file/fopen_variation19.phpt,,
 ext/standard/tests/file/php_fd_wrapper_01.phpt,,
@@ -21,9 +24,12 @@ ext/standard/tests/file/symlink_link_linkinfo_is_link_variation8.phpt,,
 ext/standard/tests/general_functions/escapeshellarg_bug71270.phpt,,
 ext/standard/tests/general_functions/escapeshellcmd_bug71270.phpt,,
 ext/standard/tests/network/bug20134.phpt,,
+ext/standard/tests/streams/stream_socket_sendto.phpt,,
+ext/standard/tests/strings/007.phpt,,
 tests/output/stream_isatty_err.phpt,b/68720279,
 tests/output/stream_isatty_in-err.phpt,b/68720282,
 tests/output/stream_isatty_in-out-err.phpt,,
 tests/output/stream_isatty_in-out.phpt,b/68720299,
 tests/output/stream_isatty_out-err.phpt,b/68720311,
 tests/output/stream_isatty_out.phpt,b/68720325,
+Zend/tests/concat_003.phpt,,
diff --git a/test/runtimes/proctor/go.go b/test/runtimes/proctor/go.go
index 3e2d5d8db..073c2959d 100644
--- a/test/runtimes/proctor/go.go
+++ b/test/runtimes/proctor/go.go
@@ -74,17 +74,26 @@ func (goRunner) ListTests() ([]string, error) {
 	return append(toolSlice, diskFiltered...), nil
 }
 
-// TestCmd implements TestRunner.TestCmd.
-func (goRunner) TestCmd(test string) *exec.Cmd {
-	// Check if test exists on disk by searching for file of the same name.
-	// This will determine whether or not it is a Go test on disk.
-	if strings.HasSuffix(test, ".go") {
-		// Test has suffix ".go" which indicates a disk test, run it as such.
-		cmd := exec.Command("go", "run", "run.go", "-v", "--", test)
+// TestCmds implements TestRunner.TestCmds.
+func (goRunner) TestCmds(tests []string) []*exec.Cmd {
+	var toolTests, onDiskTests []string
+	for _, test := range tests {
+		if strings.HasSuffix(test, ".go") {
+			onDiskTests = append(onDiskTests, test)
+		} else {
+			toolTests = append(toolTests, test)
+		}
+	}
+
+	var cmds []*exec.Cmd
+	if len(toolTests) > 0 {
+		cmds = append(cmds, exec.Command("go", "tool", "dist", "test", "-run", strings.Join(toolTests, "\\|")))
+	}
+	if len(onDiskTests) > 0 {
+		cmd := exec.Command("go", append([]string{"run", "run.go", "-v", "--"}, onDiskTests...)...)
 		cmd.Dir = goTestDir
-		return cmd
+		cmds = append(cmds, cmd)
 	}
 
-	// No ".go" suffix, run as a tool test.
-	return exec.Command("go", "tool", "dist", "test", "-run", test)
+	return cmds
 }
diff --git a/test/runtimes/proctor/java.go b/test/runtimes/proctor/java.go
index 8b362029d..737fbe23e 100644
--- a/test/runtimes/proctor/java.go
+++ b/test/runtimes/proctor/java.go
@@ -60,12 +60,14 @@ func (javaRunner) ListTests() ([]string, error) {
 	return testSlice, nil
 }
 
-// TestCmd implements TestRunner.TestCmd.
-func (javaRunner) TestCmd(test string) *exec.Cmd {
-	args := []string{
-		"-noreport",
-		"-dir:" + javaTestDir,
-		test,
-	}
-	return exec.Command("jtreg", args...)
+// TestCmds implements TestRunner.TestCmds.
+func (javaRunner) TestCmds(tests []string) []*exec.Cmd {
+	args := append(
+		[]string{
+			"-noreport",
+			"-dir:" + javaTestDir,
+		},
+		tests...,
+	)
+	return []*exec.Cmd{exec.Command("jtreg", args...)}
 }
diff --git a/test/runtimes/proctor/nodejs.go b/test/runtimes/proctor/nodejs.go
index bd57db444..23d6edc72 100644
--- a/test/runtimes/proctor/nodejs.go
+++ b/test/runtimes/proctor/nodejs.go
@@ -39,8 +39,8 @@ func (nodejsRunner) ListTests() ([]string, error) {
 	return testSlice, nil
 }
 
-// TestCmd implements TestRunner.TestCmd.
-func (nodejsRunner) TestCmd(test string) *exec.Cmd {
-	args := []string{filepath.Join("tools", "test.py"), test}
-	return exec.Command("/usr/bin/python", args...)
+// TestCmds implements TestRunner.TestCmds.
+func (nodejsRunner) TestCmds(tests []string) []*exec.Cmd {
+	args := append([]string{filepath.Join("tools", "test.py")}, tests...)
+	return []*exec.Cmd{exec.Command("/usr/bin/python", args...)}
 }
diff --git a/test/runtimes/proctor/php.go b/test/runtimes/proctor/php.go
index 9115040e1..6a83d64e3 100644
--- a/test/runtimes/proctor/php.go
+++ b/test/runtimes/proctor/php.go
@@ -17,6 +17,7 @@ package main
 import (
 	"os/exec"
 	"regexp"
+	"strings"
 )
 
 var phpTestRegEx = regexp.MustCompile(`^.+\.phpt$`)
@@ -35,8 +36,8 @@ func (phpRunner) ListTests() ([]string, error) {
 	return testSlice, nil
 }
 
-// TestCmd implements TestRunner.TestCmd.
-func (phpRunner) TestCmd(test string) *exec.Cmd {
-	args := []string{"test", "TESTS=" + test}
-	return exec.Command("make", args...)
+// TestCmds implements TestRunner.TestCmds.
+func (phpRunner) TestCmds(tests []string) []*exec.Cmd {
+	args := []string{"test", "TESTS=" + strings.Join(tests, " ")}
+	return []*exec.Cmd{exec.Command("make", args...)}
 }
diff --git a/test/runtimes/proctor/proctor.go b/test/runtimes/proctor/proctor.go
index b54abe434..9e0642424 100644
--- a/test/runtimes/proctor/proctor.go
+++ b/test/runtimes/proctor/proctor.go
@@ -25,6 +25,7 @@ import (
 	"os/signal"
 	"path/filepath"
 	"regexp"
+	"strings"
 	"syscall"
 )
 
@@ -34,15 +35,18 @@ type TestRunner interface {
 	// ListTests returns a string slice of tests available to run.
 	ListTests() ([]string, error)
 
-	// TestCmd returns an *exec.Cmd that will run the given test.
-	TestCmd(test string) *exec.Cmd
+	// TestCmds returns a slice of *exec.Cmd that will run the given tests.
+	// There is no correlation between the number of exec.Cmds returned and the
+	// number of tests. It could return one command to run all tests or a few
+	// commands that collectively run all.
+	TestCmds(tests []string) []*exec.Cmd
 }
 
 var (
-	runtime  = flag.String("runtime", "", "name of runtime")
-	list     = flag.Bool("list", false, "list all available tests")
-	testName = flag.String("test", "", "run a single test from the list of available tests")
-	pause    = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children")
+	runtime   = flag.String("runtime", "", "name of runtime")
+	list      = flag.Bool("list", false, "list all available tests")
+	testNames = flag.String("tests", "", "run a subset of the available tests")
+	pause     = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children")
 )
 
 func main() {
@@ -75,18 +79,20 @@ func main() {
 	}
 
 	var tests []string
-	if *testName == "" {
+	if *testNames == "" {
 		// Run every test.
 		tests, err = tr.ListTests()
 		if err != nil {
 			log.Fatalf("failed to get all tests: %v", err)
 		}
 	} else {
-		// Run a single test.
-		tests = []string{*testName}
+		// Run subset of test.
+		tests = strings.Split(*testNames, ",")
 	}
-	for _, test := range tests {
-		cmd := tr.TestCmd(test)
+
+	// Run tests.
+	cmds := tr.TestCmds(tests)
+	for _, cmd := range cmds {
 		cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
 		if err := cmd.Run(); err != nil {
 			log.Fatalf("FAIL: %v", err)
diff --git a/test/runtimes/proctor/python.go b/test/runtimes/proctor/python.go
index b9e0fbe6f..7c598801b 100644
--- a/test/runtimes/proctor/python.go
+++ b/test/runtimes/proctor/python.go
@@ -42,8 +42,8 @@ func (pythonRunner) ListTests() ([]string, error) {
 	return toolSlice, nil
 }
 
-// TestCmd implements TestRunner.TestCmd.
-func (pythonRunner) TestCmd(test string) *exec.Cmd {
-	args := []string{"-m", "test", test}
-	return exec.Command("./python", args...)
+// TestCmds implements TestRunner.TestCmds.
+func (pythonRunner) TestCmds(tests []string) []*exec.Cmd {
+	args := append([]string{"-m", "test"}, tests...)
+	return []*exec.Cmd{exec.Command("./python", args...)}
 }
diff --git a/test/runtimes/runner/BUILD b/test/runtimes/runner/BUILD
index 3972244b9..dc0d5d5b4 100644
--- a/test/runtimes/runner/BUILD
+++ b/test/runtimes/runner/BUILD
@@ -8,6 +8,7 @@ go_binary(
     srcs = ["main.go"],
     visibility = ["//test/runtimes:__pkg__"],
     deps = [
+        "//pkg/log",
         "//pkg/test/dockerutil",
         "//pkg/test/testutil",
     ],
diff --git a/test/runtimes/runner/exclude_test.go b/test/runtimes/runner/exclude_test.go
index c08755894..67c2170c8 100644
--- a/test/runtimes/runner/exclude_test.go
+++ b/test/runtimes/runner/exclude_test.go
@@ -26,7 +26,7 @@ func TestMain(m *testing.M) {
 }
 
 // Test that the exclude file parses without error.
-func TestBlacklists(t *testing.T) {
+func TestExcludelist(t *testing.T) {
 	ex, err := getExcludes()
 	if err != nil {
 		t.Fatalf("error parsing exclude file: %v", err)
diff --git a/test/runtimes/runner/main.go b/test/runtimes/runner/main.go
index 2a0f62c73..e230912c9 100644
--- a/test/runtimes/runner/main.go
+++ b/test/runtimes/runner/main.go
@@ -27,6 +27,7 @@ import (
 	"testing"
 	"time"
 
+	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/test/dockerutil"
 	"gvisor.dev/gvisor/pkg/test/testutil"
 )
@@ -35,10 +36,11 @@ var (
 	lang        = flag.String("lang", "", "language runtime to test")
 	image       = flag.String("image", "", "docker image with runtime tests")
 	excludeFile = flag.String("exclude_file", "", "file containing list of tests to exclude, in CSV format with fields: test name, bug id, comment")
+	batchSize   = flag.Int("batch", 50, "number of test cases run in one command")
 )
 
 // Wait time for each test to run.
-const timeout = 5 * time.Minute
+const timeout = 45 * time.Minute
 
 func main() {
 	flag.Parse()
@@ -110,17 +112,23 @@ func getTests(ctx context.Context, d *dockerutil.Container, excludes map[string]
 	}
 
 	var itests []testing.InternalTest
-	for _, tci := range indices {
-		// Capture tc in this scope.
-		tc := tests[tci]
+	for i := 0; i < len(indices); i += *batchSize {
+		var tcs []string
+		end := i + *batchSize
+		if end > len(indices) {
+			end = len(indices)
+		}
+		for _, tc := range indices[i:end] {
+			// Add test if not excluded.
+			if _, ok := excludes[tests[tc]]; ok {
+				log.Infof("Skipping test case %s\n", tests[tc])
+				continue
+			}
+			tcs = append(tcs, tests[tc])
+		}
 		itests = append(itests, testing.InternalTest{
-			Name: tc,
+			Name: strings.Join(tcs, ", "),
 			F: func(t *testing.T) {
-				// Is the test excluded?
-				if _, ok := excludes[tc]; ok {
-					t.Skipf("SKIP: excluded test %q", tc)
-				}
-
 				var (
 					now    = time.Now()
 					done   = make(chan struct{})
@@ -129,20 +137,20 @@ func getTests(ctx context.Context, d *dockerutil.Container, excludes map[string]
 				)
 
 				go func() {
-					fmt.Printf("RUNNING %s...\n", tc)
-					output, err = d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", *lang, "--test", tc)
+					fmt.Printf("RUNNING the following in a batch\n%s\n", strings.Join(tcs, "\n"))
+					output, err = d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", *lang, "--tests", strings.Join(tcs, ","))
 					close(done)
 				}()
 
 				select {
 				case <-done:
 					if err == nil {
-						fmt.Printf("PASS: %s (%v)\n\n", tc, time.Since(now))
+						fmt.Printf("PASS: (%v)\n\n", time.Since(now))
 						return
 					}
-					t.Errorf("FAIL: %s (%v):\n%s\n", tc, time.Since(now), output)
+					t.Errorf("FAIL: (%v):\n%s\n", time.Since(now), output)
 				case <-time.After(timeout):
-					t.Errorf("TIMEOUT: %s (%v):\n%s\n", tc, time.Since(now), output)
+					t.Errorf("TIMEOUT: (%v):\n%s\n", time.Since(now), output)
 				}
 			},
 		})
-- 
cgit v1.2.3


From f7281902f8f6f748e661c2b18a0bd3b227a55ce8 Mon Sep 17 00:00:00 2001
From: Ayush Ranjan <ayushranjan@google.com>
Date: Thu, 30 Jul 2020 10:19:36 -0700
Subject: [runtime tests] go language test enhancement

- Unexported some passing tests. This will increase the testing surface and
  will be especially helpful when this is enabled for vfs2.
- Run tool tests with -v (verbose output). We only print the output when a test
  fails so this should not clutter the output.
- Run tool tests with "-no-rebuild" flag.
- Surround test name with appropriate regex, i.e. ^testname$. This will ensure
  that only that test is run. Earlier running go_test:os would also run
  go_test:os/exec, go_test:os/signal, go_test:os/user. This should help speed
  up the tests as we do not run the same test multiple times anymore.
- Updated bugs.

Updates #3191

PiperOrigin-RevId: 324028878
---
 test/runtimes/exclude_go1.12.csv | 11 ++++-------
 test/runtimes/proctor/go.go      |  4 ++--
 2 files changed, 6 insertions(+), 9 deletions(-)

(limited to 'test/runtimes/proctor')

diff --git a/test/runtimes/exclude_go1.12.csv b/test/runtimes/exclude_go1.12.csv
index 8c8ae0c5d..81e02cf64 100644
--- a/test/runtimes/exclude_go1.12.csv
+++ b/test/runtimes/exclude_go1.12.csv
@@ -2,15 +2,12 @@ test name,bug id,comment
 cgo_errors,,FLAKY
 cgo_test,,FLAKY
 go_test:cmd/go,,FLAKY
-go_test:cmd/vendor/golang.org/x/sys/unix,b/118783622,/dev devices missing
-go_test:net,b/118784196,socket: invalid argument. Works as intended: see bug.
+go_test:net,b/162473575,setsockopt: protocol not available.
 go_test:os,b/118780122,we have a pollable filesystem but that's a surprise
-go_test:os/signal,b/118780860,/dev/pts not properly supported
-go_test:runtime,b/118782341,sigtrap not reported or caught or something
-go_test:syscall,b/118781998,bad bytes -- bad mem addr
-race,b/118782931,thread sanitizer. Works as intended: b/62219744.
+go_test:os/signal,b/118780860,/dev/pts not properly supported. Also being tracked in b/29356795.
+go_test:runtime,b/118782341,sigtrap not reported or caught or something. Also being tracked in b/33003106.
+go_test:syscall,b/118781998,bad bytes -- bad mem addr; FcntlFlock(F_GETLK) not supported.
 runtime:cpu124,b/118778254,segmentation fault
 test:0_1,,FLAKY
-testasan,,
 testcarchive,b/118782924,no sigpipe
 testshared,,FLAKY
diff --git a/test/runtimes/proctor/go.go b/test/runtimes/proctor/go.go
index 073c2959d..d0ae844e6 100644
--- a/test/runtimes/proctor/go.go
+++ b/test/runtimes/proctor/go.go
@@ -81,13 +81,13 @@ func (goRunner) TestCmds(tests []string) []*exec.Cmd {
 		if strings.HasSuffix(test, ".go") {
 			onDiskTests = append(onDiskTests, test)
 		} else {
-			toolTests = append(toolTests, test)
+			toolTests = append(toolTests, "^"+test+"$")
 		}
 	}
 
 	var cmds []*exec.Cmd
 	if len(toolTests) > 0 {
-		cmds = append(cmds, exec.Command("go", "tool", "dist", "test", "-run", strings.Join(toolTests, "\\|")))
+		cmds = append(cmds, exec.Command("go", "tool", "dist", "test", "-v", "-no-rebuild", "-run", strings.Join(toolTests, "\\|")))
 	}
 	if len(onDiskTests) > 0 {
 		cmd := exec.Command("go", append([]string{"run", "run.go", "-v", "--"}, onDiskTests...)...)
-- 
cgit v1.2.3


From 12a6657d9c037b4359923069c67377d516b15194 Mon Sep 17 00:00:00 2001
From: Ayush Ranjan <ayushranjan@google.com>
Date: Fri, 31 Jul 2020 15:59:28 -0700
Subject: [runtime tests] Enhance java runtime test.

- Added a bunch of helpful options which help in speeding up the test and
  providing useful output.
- Unexcluded passing tests and updated bugs. Excluded tests which were failing.
- Increased the batch size for java tests so that we can take advantage of
  the shared JVMs.

The running time of the tests decreased from 3+ hours (I don't know the exact
running time because this test has always timed out after 3 hours) to 1 hour
15 minutes. We can reliably run this a CI kokoro job.

PiperOrigin-RevId: 324301503
---
 test/runtimes/BUILD              |  1 +
 test/runtimes/defs.bzl           |  6 +++
 test/runtimes/exclude_java11.csv | 93 ++++++++++++++++++++++++++++++++++------
 test/runtimes/proctor/java.go    |  7 ++-
 test/runtimes/runner/main.go     |  2 +-
 5 files changed, 94 insertions(+), 15 deletions(-)

(limited to 'test/runtimes/proctor')

diff --git a/test/runtimes/BUILD b/test/runtimes/BUILD
index 1728161ce..3be123d94 100644
--- a/test/runtimes/BUILD
+++ b/test/runtimes/BUILD
@@ -11,6 +11,7 @@ runtime_test(
 
 runtime_test(
     name = "java11",
+    batch = 100,
     exclude_file = "exclude_java11.csv",
     lang = "java",
     shard_count = 20,
diff --git a/test/runtimes/defs.bzl b/test/runtimes/defs.bzl
index 5779b9591..db22029a8 100644
--- a/test/runtimes/defs.bzl
+++ b/test/runtimes/defs.bzl
@@ -9,6 +9,8 @@ def _runtime_test_impl(ctx):
         ctx.attr.lang,
         "--image",
         ctx.attr.image,
+        "--batch",
+        str(ctx.attr.batch),
     ]
     if ctx.attr.exclude_file:
         args += [
@@ -47,6 +49,10 @@ _runtime_test = rule(
             mandatory = False,
             allow_single_file = True,
         ),
+        "batch": attr.int(
+            default = 50,
+            mandatory = False,
+        ),
         "_runner": attr.label(
             default = "//test/runtimes/runner:runner",
         ),
diff --git a/test/runtimes/exclude_java11.csv b/test/runtimes/exclude_java11.csv
index c012e5a56..4d62f7d3a 100644
--- a/test/runtimes/exclude_java11.csv
+++ b/test/runtimes/exclude_java11.csv
@@ -15,10 +15,9 @@ java/lang/Character/CheckScript.java,,Fails in Docker
 java/lang/Character/CheckUnicode.java,,Fails in Docker
 java/lang/Class/GetPackageBootLoaderChildLayer.java,,
 java/lang/ClassLoader/nativeLibrary/NativeLibraryTest.java,,Fails in Docker
+java/lang/module/ModuleDescriptorTest.java,,
 java/lang/String/nativeEncoding/StringPlatformChars.java,,
-java/net/DatagramSocket/ReuseAddressTest.java,,
-java/net/DatagramSocket/SendDatagramToBadAddress.java,b/78473345,
-java/net/Inet4Address/PingThis.java,,
+java/net/CookieHandler/B6791927.java,,java.lang.RuntimeException: Expiration date shouldn't be 0
 java/net/InterfaceAddress/NetworkPrefixLength.java,b/78507103,
 java/net/MulticastSocket/MulticastTTL.java,,
 java/net/MulticastSocket/Promiscuous.java,,
@@ -29,23 +28,16 @@ java/net/MulticastSocket/TestDefaults.java,,
 java/net/MulticastSocket/TimeToLive.java,,
 java/net/NetworkInterface/NetworkInterfaceStreamTest.java,,
 java/net/Socket/SetSoLinger.java,b/78527327,SO_LINGER is not yet supported
-java/net/Socket/TrafficClass.java,b/78527818,Not supported on gVisor
 java/net/Socket/UrgentDataTest.java,b/111515323,
-java/net/Socket/setReuseAddress/Basic.java,b/78519214,SO_REUSEADDR enabled by default
 java/net/SocketOption/OptionsTest.java,,Fails in Docker
-java/net/SocketOption/TcpKeepAliveTest.java,,
 java/net/SocketPermission/SocketPermissionTest.java,,
 java/net/URLConnection/6212146/TestDriver.java,,Fails in Docker
 java/net/httpclient/RequestBuilderTest.java,,Fails in Docker
-java/net/httpclient/ShortResponseBody.java,,
-java/net/httpclient/ShortResponseBodyWithRetry.java,,
-java/nio/channels/AsyncCloseAndInterrupt.java,,
-java/nio/channels/AsynchronousServerSocketChannel/Basic.java,,
-java/nio/channels/AsynchronousSocketChannel/Basic.java,b/77921528,SO_KEEPALIVE is not settable
 java/nio/channels/DatagramChannel/BasicMulticastTests.java,,
-java/nio/channels/DatagramChannel/SocketOptionTests.java,,Fails in Docker
+java/nio/channels/DatagramChannel/SocketOptionTests.java,,java.net.SocketException: Invalid argument
 java/nio/channels/DatagramChannel/UseDGWithIPv6.java,,
 java/nio/channels/FileChannel/directio/DirectIOTest.java,,Fails in Docker
+java/nio/channels/FileChannel/directio/PwriteDirect.java,,java.io.IOException: Invalid argument
 java/nio/channels/Selector/OutOfBand.java,,
 java/nio/channels/Selector/SelectWithConsumer.java,,Flaky
 java/nio/channels/ServerSocketChannel/SocketOptionTests.java,,
@@ -59,15 +51,89 @@ java/text/Format/NumberFormat/CurrencyFormat.java,,Fails in Docker
 java/util/Calendar/JapaneseEraNameTest.java,,
 java/util/Currency/CurrencyTest.java,,Fails in Docker
 java/util/Currency/ValidateISO4217.java,,Fails in Docker
+java/util/EnumSet/BogusEnumSet.java,,"java.io.InvalidClassException: java.util.EnumSet; local class incompatible: stream classdesc serialVersionUID = -2409567991088730183, local class serialVersionUID = 1009687484059888093"
+java/util/Locale/Bug8040211.java,,java.lang.RuntimeException: Failed.
 java/util/Locale/LSRDataTest.java,,
+java/util/Properties/CompatibilityTest.java,,"java.lang.RuntimeException: jdk.internal.org.xml.sax.SAXParseException; Internal DTD subset is not allowed. The Properties XML document must have the following DOCTYPE declaration: <!DOCTYPE properties SYSTEM ""http://java.sun.com/dtd/properties.dtd"">"
+java/util/ResourceBundle/Control/XMLResourceBundleTest.java,,java.util.MissingResourceException: Can't find bundle for base name XmlRB locale
+java/util/ResourceBundle/modules/xmlformat/xmlformat.sh,,Timeout reached: 60000. Process is not alive!
+java/util/TimeZone/TimeZoneTest.java,,Uncaught exception thrown in test method TestShortZoneIDs
 java/util/concurrent/locks/Lock/TimedAcquireLeak.java,,
 java/util/jar/JarFile/mrjar/MultiReleaseJarAPI.java,,Fails in Docker
 java/util/logging/LogManager/Configuration/updateConfiguration/SimpleUpdateConfigWithInputStreamTest.java,,
 java/util/logging/TestLoggerWeakRefLeak.java,,
+java/util/spi/ResourceBundleControlProvider/UserDefaultControlTest.java,,java.util.MissingResourceException: Can't find bundle for base name com.foo.XmlRB locale
 javax/imageio/AppletResourceTest.java,,
+javax/imageio/plugins/jpeg/JPEGsNotAcceleratedTest.java,,java.awt.HeadlessException: No X11 DISPLAY variable was set but this program performed an operation which requires it.
 javax/management/security/HashedPasswordFileTest.java,,
+javax/net/ssl/DTLS/DTLSBufferOverflowUnderflowTest.java,,Compilation failed
+javax/net/ssl/DTLS/DTLSDataExchangeTest.java,,Compilation failed
+javax/net/ssl/DTLS/DTLSEnginesClosureTest.java,,Compilation failed
+javax/net/ssl/DTLS/DTLSHandshakeTest.java,,Compilation failed
+javax/net/ssl/DTLS/DTLSHandshakeWithReplicatedPacketsTest.java,,Compilation failed
+javax/net/ssl/DTLS/DTLSIncorrectAppDataTest.java,,Compilation failed
+javax/net/ssl/DTLS/DTLSMFLNTest.java,,Compilation failed
+javax/net/ssl/DTLS/DTLSNotEnabledRC4Test.java,,Compilation failed
+javax/net/ssl/DTLS/DTLSRehandshakeTest.java,,Compilation failed
+javax/net/ssl/DTLS/DTLSRehandshakeWithCipherChangeTest.java,,Compilation failed
+javax/net/ssl/DTLS/DTLSRehandshakeWithDataExTest.java,,Compilation failed
+javax/net/ssl/DTLS/DTLSSequenceNumberTest.java,,Compilation failed
+javax/net/ssl/DTLS/DTLSUnsupportedCiphersTest.java,,Compilation failed
+javax/net/ssl/DTLSv10/DTLSv10BufferOverflowUnderflowTest.java,,Compilation failed
+javax/net/ssl/DTLSv10/DTLSv10DataExchangeTest.java,,Compilation failed
+javax/net/ssl/DTLSv10/DTLSv10EnginesClosureTest.java,,Compilation failed
+javax/net/ssl/DTLSv10/DTLSv10HandshakeTest.java,,Compilation failed
+javax/net/ssl/DTLSv10/DTLSv10HandshakeWithReplicatedPacketsTest.java,,Compilation failed
+javax/net/ssl/DTLSv10/DTLSv10IncorrectAppDataTest.java,,Compilation failed
+javax/net/ssl/DTLSv10/DTLSv10MFLNTest.java,,Compilation failed
+javax/net/ssl/DTLSv10/DTLSv10NotEnabledRC4Test.java,,Compilation failed
+javax/net/ssl/DTLSv10/DTLSv10RehandshakeTest.java,,Compilation failed
+javax/net/ssl/DTLSv10/DTLSv10RehandshakeWithCipherChangeTest.java,,Compilation failed
+javax/net/ssl/DTLSv10/DTLSv10RehandshakeWithDataExTest.java,,Compilation failed
+javax/net/ssl/DTLSv10/DTLSv10SequenceNumberTest.java,,Compilation failed
+javax/net/ssl/DTLSv10/DTLSv10UnsupportedCiphersTest.java,,Compilation failed
 javax/net/ssl/SSLSession/JSSERenegotiate.java,,Fails in Docker
+javax/net/ssl/TLS/TLSDataExchangeTest.java,,Compilation failed
+javax/net/ssl/TLS/TLSEnginesClosureTest.java,,Compilation failed
+javax/net/ssl/TLS/TLSHandshakeTest.java,,Compilation failed
+javax/net/ssl/TLS/TLSMFLNTest.java,,Compilation failed
+javax/net/ssl/TLS/TLSNotEnabledRC4Test.java,,Compilation failed
+javax/net/ssl/TLS/TLSRehandshakeTest.java,,Compilation failed
+javax/net/ssl/TLS/TLSRehandshakeWithCipherChangeTest.java,,Compilation failed
+javax/net/ssl/TLS/TLSRehandshakeWithDataExTest.java,,Compilation failed
+javax/net/ssl/TLS/TLSUnsupportedCiphersTest.java,,Compilation failed
+javax/net/ssl/TLSv1/TLSDataExchangeTest.java,,Compilation failed
+javax/net/ssl/TLSv1/TLSEnginesClosureTest.java,,Compilation failed
+javax/net/ssl/TLSv1/TLSHandshakeTest.java,,Compilation failed
+javax/net/ssl/TLSv1/TLSMFLNTest.java,,Compilation failed
+javax/net/ssl/TLSv1/TLSNotEnabledRC4Test.java,,Compilation failed
+javax/net/ssl/TLSv1/TLSRehandshakeTest.java,,Compilation failed
+javax/net/ssl/TLSv1/TLSRehandshakeWithCipherChangeTest.java,,Compilation failed
+javax/net/ssl/TLSv1/TLSRehandshakeWithDataExTest.java,,Compilation failed
+javax/net/ssl/TLSv1/TLSUnsupportedCiphersTest.java,,Compilation failed
+javax/net/ssl/TLSv11/TLSDataExchangeTest.java,,Compilation failed
+javax/net/ssl/TLSv11/TLSEnginesClosureTest.java,,Compilation failed
+javax/net/ssl/TLSv11/TLSHandshakeTest.java,,Compilation failed
+javax/net/ssl/TLSv11/TLSMFLNTest.java,,Compilation failed
+javax/net/ssl/TLSv11/TLSNotEnabledRC4Test.java,,Compilation failed
+javax/net/ssl/TLSv11/TLSRehandshakeTest.java,,Compilation failed
+javax/net/ssl/TLSv11/TLSRehandshakeWithCipherChangeTest.java,,Compilation failed
+javax/net/ssl/TLSv11/TLSRehandshakeWithDataExTest.java,,Compilation failed
+javax/net/ssl/TLSv11/TLSUnsupportedCiphersTest.java,,Compilation failed
+javax/net/ssl/TLSv12/TLSEnginesClosureTest.java,,Compilation failed
 javax/sound/sampled/AudioInputStream/FrameLengthAfterConversion.java,,
+jdk/jfr/cmd/TestHelp.java,,java.lang.RuntimeException: 'Available commands are:' missing from stdout/stderr
+jdk/jfr/cmd/TestPrint.java,,Missing file' missing from stdout/stderr
+jdk/jfr/cmd/TestPrintDefault.java,,java.lang.RuntimeException: 'JVMInformation' missing from stdout/stderr
+jdk/jfr/cmd/TestPrintJSON.java,,javax.script.ScriptException: <eval>:1:17 Expected an operand but found eof var jsonObject = ^ in <eval> at line number 1 at column number 17
+jdk/jfr/cmd/TestPrintXML.java,,org.xml.sax.SAXParseException; lineNumber: 1; columnNumber: 1; Premature end of file.
+jdk/jfr/cmd/TestReconstruct.java,,java.lang.RuntimeException: 'Too few arguments' missing from stdout/stderr
+jdk/jfr/cmd/TestSplit.java,,java.lang.RuntimeException: 'Missing file' missing from stdout/stderr
+jdk/jfr/cmd/TestSummary.java,,java.lang.RuntimeException: 'Missing file' missing from stdout/stderr
+jdk/jfr/event/compiler/TestCompilerStats.java,,java.lang.RuntimeException: Field nmetodsSize not in event
+jdk/jfr/event/metadata/TestDefaultConfigurations.java,,Setting 'threshold' in event 'jdk.SecurityPropertyModification' was not configured in the configuration 'default'
+jdk/jfr/event/runtime/TestActiveSettingEvent.java,,java.lang.Exception: Could not find setting with name jdk.X509Validation#threshold
+jdk/jfr/event/runtime/TestModuleEvents.java,,java.lang.RuntimeException: assertEquals: expected jdk.proxy1 to equal java.base
 jdk/jfr/event/runtime/TestNetworkUtilizationEvent.java,,
 jdk/jfr/event/runtime/TestThreadParkEvent.java,,
 jdk/jfr/event/sampling/TestNative.java,,
@@ -86,6 +152,7 @@ jdk/jfr/jcmd/TestJcmdStartStopDefault.java,,
 jdk/jfr/jcmd/TestJcmdStartWithOptions.java,,
 jdk/jfr/jcmd/TestJcmdStartWithSettings.java,,
 jdk/jfr/jcmd/TestJcmdStopInvalidFile.java,,
+jdk/jfr/jvm/TestGetAllEventClasses.java,,Compilation failed
 jdk/jfr/jvm/TestJfrJavaBase.java,,
 jdk/jfr/startupargs/TestStartRecording.java,,
 jdk/modules/incubator/ImageModules.java,,
@@ -123,4 +190,6 @@ tools/jlink/JLinkTest.java,,
 tools/jlink/plugins/IncludeLocalesPluginTest.java,,
 tools/jmod/hashes/HashesTest.java,,
 tools/launcher/BigJar.java,b/111611473,
+tools/launcher/HelpFlagsTest.java,,java.lang.AssertionError: HelpFlagsTest failed: Tool jfr not covered by this test. Add specification to jdkTools array!
+tools/launcher/VersionCheck.java,,java.lang.AssertionError: VersionCheck failed: testToolVersion: [jfr];
 tools/launcher/modules/patch/systemmodules/PatchSystemModules.java,,
diff --git a/test/runtimes/proctor/java.go b/test/runtimes/proctor/java.go
index 737fbe23e..d456fa681 100644
--- a/test/runtimes/proctor/java.go
+++ b/test/runtimes/proctor/java.go
@@ -64,8 +64,11 @@ func (javaRunner) ListTests() ([]string, error) {
 func (javaRunner) TestCmds(tests []string) []*exec.Cmd {
 	args := append(
 		[]string{
-			"-noreport",
-			"-dir:" + javaTestDir,
+			"-agentvm",            // Execute each action using a pool of reusable JVMs.
+			"-dir:" + javaTestDir, // Base directory for test files and directories.
+			"-noreport",           // Do not generate a final report.
+			"-timeoutFactor:20",   // Extend the default timeout (2 min) of all tests by this factor.
+			"-verbose:nopass",     // Verbose output but supress it for tests that passed.
 		},
 		tests...,
 	)
diff --git a/test/runtimes/runner/main.go b/test/runtimes/runner/main.go
index e230912c9..948e7cf9c 100644
--- a/test/runtimes/runner/main.go
+++ b/test/runtimes/runner/main.go
@@ -40,7 +40,7 @@ var (
 )
 
 // Wait time for each test to run.
-const timeout = 45 * time.Minute
+const timeout = 90 * time.Minute
 
 func main() {
 	flag.Parse()
-- 
cgit v1.2.3


From 7dfcf727a5730b15020f136282a1ede15144a268 Mon Sep 17 00:00:00 2001
From: Ayush Ranjan <ayushranjan@google.com>
Date: Tue, 4 Aug 2020 09:49:33 -0700
Subject: [runtime tests] Fix nodejs runtime tests.

- Exclude flaky tests.
- Bump timeout.
- Un-exclude passing tests to increase testing surface.
- Create/Update bugs for tests that pass on runc but fail on runsc.

PiperOrigin-RevId: 324830840
---
 test/runtimes/exclude_nodejs12.4.0.csv | 69 ++++++++++++++++------------------
 test/runtimes/proctor/nodejs.go        |  2 +-
 2 files changed, 34 insertions(+), 37 deletions(-)

(limited to 'test/runtimes/proctor')

diff --git a/test/runtimes/exclude_nodejs12.4.0.csv b/test/runtimes/exclude_nodejs12.4.0.csv
index 3b1d1bc9d..9aaf1e647 100644
--- a/test/runtimes/exclude_nodejs12.4.0.csv
+++ b/test/runtimes/exclude_nodejs12.4.0.csv
@@ -1,52 +1,49 @@
 test name,bug id,comment
 benchmark/test-benchmark-fs.js,,
-benchmark/test-benchmark-module.js,,
 benchmark/test-benchmark-napi.js,,
 doctool/test-make-doc.js,b/68848110,Expected to fail.
-fixtures/test-error-first-line-offset.js,,
-fixtures/test-fs-readfile-error.js,,
-fixtures/test-fs-stat-sync-overflow.js,,
-internet/test-dgram-broadcast-multi-process.js,,
-internet/test-dgram-multicast-multi-process.js,,
-internet/test-dgram-multicast-set-interface-lo.js,,
+internet/test-dgram-multicast-set-interface-lo.js,b/162798882,
 internet/test-doctool-versions.js,,
 internet/test-uv-threadpool-schedule.js,,
 parallel/test-cluster-dgram-reuse.js,b/64024294,
 parallel/test-dgram-bind-fd.js,b/132447356,
-parallel/test-dgram-create-socket-handle-fd.js,b/132447238,
-parallel/test-dgram-createSocket-type.js,b/68847739,
 parallel/test-dgram-socket-buffer-size.js,b/68847921,
 parallel/test-dns-channel-timeout.js,b/161893056,
 parallel/test-fs-access.js,,
-parallel/test-fs-write-stream-double-close.js,,
+parallel/test-fs-watchfile.js,,Flaky - File already exists error
 parallel/test-fs-write-stream-throw-type-error.js,b/110226209,
-parallel/test-fs-write-stream.js,,
-parallel/test-http2-respond-file-error-pipe-offset.js,,
-parallel/test-os.js,,
+parallel/test-http-writable-true-after-close.js,,Flaky - Mismatched <anonymous> function calls. Expected exactly 1 actual 2
+parallel/test-os.js,b/63997097,
 parallel/test-process-uid-gid.js,,
-pseudo-tty/test-assert-colors.js,,
-pseudo-tty/test-assert-no-color.js,,
-pseudo-tty/test-assert-position-indicator.js,,
-pseudo-tty/test-async-wrap-getasyncid-tty.js,,
-pseudo-tty/test-fatal-error.js,,
-pseudo-tty/test-handle-wrap-isrefed-tty.js,,
-pseudo-tty/test-readable-tty-keepalive.js,,
-pseudo-tty/test-set-raw-mode-reset-process-exit.js,,
-pseudo-tty/test-set-raw-mode-reset-signal.js,,
-pseudo-tty/test-set-raw-mode-reset.js,,
-pseudo-tty/test-stderr-stdout-handle-sigwinch.js,,
-pseudo-tty/test-stdout-read.js,,
-pseudo-tty/test-tty-color-support.js,,
-pseudo-tty/test-tty-isatty.js,,
-pseudo-tty/test-tty-stdin-call-end.js,,
-pseudo-tty/test-tty-stdin-end.js,,
-pseudo-tty/test-stdin-write.js,,
-pseudo-tty/test-tty-stdout-end.js,,
-pseudo-tty/test-tty-stdout-resize.js,,
-pseudo-tty/test-tty-stream-constructors.js,,
-pseudo-tty/test-tty-window-size.js,,
-pseudo-tty/test-tty-wrap.js,,
+parallel/test-tls-cli-min-version-1.0.js,,Flaky - EADDRINUSE
+parallel/test-tls-cli-min-version-1.3.js,,Flaky - EADDRINUSE
+parallel/test-tls-cli-max-version-1.2,,Flaky - EADDRINUSE
+parallel/test-tls-min-max-version.js,,Flaky - EADDRINUSE
+pseudo-tty/test-assert-colors.js,b/162801321,
+pseudo-tty/test-assert-no-color.js,b/162801321,
+pseudo-tty/test-assert-position-indicator.js,b/162801321,
+pseudo-tty/test-async-wrap-getasyncid-tty.js,b/162801321,
+pseudo-tty/test-fatal-error.js,b/162801321,
+pseudo-tty/test-handle-wrap-isrefed-tty.js,b/162801321,
+pseudo-tty/test-readable-tty-keepalive.js,b/162801321,
+pseudo-tty/test-set-raw-mode-reset-process-exit.js,b/162801321,
+pseudo-tty/test-set-raw-mode-reset-signal.js,b/162801321,
+pseudo-tty/test-set-raw-mode-reset.js,b/162801321,
+pseudo-tty/test-stderr-stdout-handle-sigwinch.js,b/162801321,
+pseudo-tty/test-stdout-read.js,b/162801321,
+pseudo-tty/test-tty-color-support.js,b/162801321,
+pseudo-tty/test-tty-isatty.js,b/162801321,
+pseudo-tty/test-tty-stdin-call-end.js,b/162801321,
+pseudo-tty/test-tty-stdin-end.js,b/162801321,
+pseudo-tty/test-stdin-write.js,b/162801321,
+pseudo-tty/test-tty-stdout-end.js,b/162801321,
+pseudo-tty/test-tty-stdout-resize.js,b/162801321,
+pseudo-tty/test-tty-stream-constructors.js,b/162801321,
+pseudo-tty/test-tty-window-size.js,b/162801321,
+pseudo-tty/test-tty-wrap.js,b/162801321,
 pummel/test-net-pingpong.js,,
-pummel/test-vm-memleak.js,,
+pummel/test-vm-memleak.js,b/162799436,
+sequential/test-child-process-pass-fd.js,b/63926391,Flaky
+sequential/test-https-connect-localport.js,,Flaky - EADDRINUSE
 sequential/test-net-bytes-per-incoming-chunk-overhead.js,,flaky - timeout
 tick-processor/test-tick-processor-builtin.js,,
diff --git a/test/runtimes/proctor/nodejs.go b/test/runtimes/proctor/nodejs.go
index 23d6edc72..dead5af4f 100644
--- a/test/runtimes/proctor/nodejs.go
+++ b/test/runtimes/proctor/nodejs.go
@@ -41,6 +41,6 @@ func (nodejsRunner) ListTests() ([]string, error) {
 
 // TestCmds implements TestRunner.TestCmds.
 func (nodejsRunner) TestCmds(tests []string) []*exec.Cmd {
-	args := append([]string{filepath.Join("tools", "test.py")}, tests...)
+	args := append([]string{filepath.Join("tools", "test.py"), "--timeout=180"}, tests...)
 	return []*exec.Cmd{exec.Command("/usr/bin/python", args...)}
 }
-- 
cgit v1.2.3