Refactor and clean up image tests.

* Use multi-stage builds in Dockerfiles.
* Combine all proctor binaries into a single binary.
* Change the TestRunner interface to reduce code duplication.

PiperOrigin-RevId: 269462101

8 files changed, 573 insertions, 0 deletions

diff --git a/test/runtimes/images/proctor/BUILD b/test/runtimes/images/proctor/BUILD
new file mode 100644
index 000000000..09dc6c42f
--- /dev/null
+++ b/test/runtimes/images/proctor/BUILD
@@ -0,0 +1,26 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_test")
+
+package(licenses = ["notice"])
+
+go_binary(
+    name = "proctor",
+    srcs = [
+        "go.go",
+        "java.go",
+        "nodejs.go",
+        "php.go",
+        "proctor.go",
+        "python.go",
+    ],
+    visibility = ["//test/runtimes/images:__subpackages__"],
+)
+
+go_test(
+    name = "proctor_test",
+    size = "small",
+    srcs = ["proctor_test.go"],
+    embed = [":proctor"],
+    deps = [
+        "//runsc/testutil",
+    ],
+)
diff --git a/test/runtimes/images/proctor/go.go b/test/runtimes/images/proctor/go.go
new file mode 100644
index 000000000..3e2d5d8db
--- /dev/null
+++ b/test/runtimes/images/proctor/go.go
@@ -0,0 +1,90 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+)
+
+var (
+	goTestRegEx = regexp.MustCompile(`^.+\.go$`)
+
+	// Directories with .dir contain helper files for tests.
+	// Exclude benchmarks and stress tests.
+	goDirFilter = regexp.MustCompile(`^(bench|stress)\/.+$|^.+\.dir.+$`)
+)
+
+// Location of Go tests on disk.
+const goTestDir = "/usr/local/go/test"
+
+// goRunner implements TestRunner for Go.
+//
+// There are two types of Go tests: "Go tool tests" and "Go tests on disk".
+// "Go tool tests" are found and executed using `go tool dist test`. "Go tests
+// on disk" are found in the /usr/local/go/test directory and are executed
+// using `go run run.go`.
+type goRunner struct{}
+
+var _ TestRunner = goRunner{}
+
+// ListTests implements TestRunner.ListTests.
+func (goRunner) ListTests() ([]string, error) {
+	// Go tool dist test tests.
+	args := []string{"tool", "dist", "test", "-list"}
+	cmd := exec.Command("go", args...)
+	cmd.Stderr = os.Stderr
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list: %v", err)
+	}
+	var toolSlice []string
+	for _, test := range strings.Split(string(out), "\n") {
+		toolSlice = append(toolSlice, test)
+	}
+
+	// Go tests on disk.
+	diskSlice, err := search(goTestDir, goTestRegEx)
+	if err != nil {
+		return nil, err
+	}
+	// Remove items from /bench/, /stress/ and .dir files
+	diskFiltered := diskSlice[:0]
+	for _, file := range diskSlice {
+		if !goDirFilter.MatchString(file) {
+			diskFiltered = append(diskFiltered, file)
+		}
+	}
+
+	return append(toolSlice, diskFiltered...), nil
+}
+
+// TestCmd implements TestRunner.TestCmd.
+func (goRunner) TestCmd(test string) *exec.Cmd {
+	// Check if test exists on disk by searching for file of the same name.
+	// This will determine whether or not it is a Go test on disk.
+	if strings.HasSuffix(test, ".go") {
+		// Test has suffix ".go" which indicates a disk test, run it as such.
+		cmd := exec.Command("go", "run", "run.go", "-v", "--", test)
+		cmd.Dir = goTestDir
+		return cmd
+	}
+
+	// No ".go" suffix, run as a tool test.
+	return exec.Command("go", "tool", "dist", "test", "-run", test)
+}
diff --git a/test/runtimes/images/proctor/java.go b/test/runtimes/images/proctor/java.go
new file mode 100644
index 000000000..594dc6cc6
--- /dev/null
+++ b/test/runtimes/images/proctor/java.go
@@ -0,0 +1,71 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+)
+
+// Directories to exclude from tests.
+var javaExclDirs = regexp.MustCompile(`(^(sun\/security)|(java\/util\/stream)|(java\/time)| )`)
+
+// Location of java tests.
+const javaTestDir = "/root/test"
+
+// javaRunner implements TestRunner for Java.
+type javaRunner struct{}
+
+var _ TestRunner = javaRunner{}
+
+// ListTests implements TestRunner.ListTests.
+func (javaRunner) ListTests() ([]string, error) {
+	args := []string{
+		"-dir:" + javaTestDir,
+		"-ignore:quiet",
+		"-a",
+		"-listtests",
+		":jdk_core",
+		":jdk_svc",
+		":jdk_sound",
+		":jdk_imageio",
+	}
+	cmd := exec.Command("jtreg", args...)
+	cmd.Stderr = os.Stderr
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("jtreg -listtests : %v", err)
+	}
+	var testSlice []string
+	for _, test := range strings.Split(string(out), "\n") {
+		if !javaExclDirs.MatchString(test) {
+			testSlice = append(testSlice, test)
+		}
+	}
+	return testSlice, nil
+}
+
+// TestCmd implements TestRunner.TestCmd.
+func (javaRunner) TestCmd(test string) *exec.Cmd {
+	args := []string{
+		"-noreport",
+		"-dir:" + javaTestDir,
+		test,
+	}
+	return exec.Command("jtreg", args...)
+}
diff --git a/test/runtimes/images/proctor/nodejs.go b/test/runtimes/images/proctor/nodejs.go
new file mode 100644
index 000000000..4ef1afe63
--- /dev/null
+++ b/test/runtimes/images/proctor/nodejs.go
@@ -0,0 +1,46 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"os/exec"
+	"path/filepath"
+	"regexp"
+)
+
+var nodejsTestRegEx = regexp.MustCompile(`^test-[^-].+\.js$`)
+
+// Location of nodejs tests relative to working dir.
+const nodejsTestDir = "test"
+
+// nodejsRunner implements TestRunner for NodeJS.
+type nodejsRunner struct{}
+
+var _ TestRunner = nodejsRunner{}
+
+// ListTests implements TestRunner.ListTests.
+func (nodejsRunner) ListTests() ([]string, error) {
+	testSlice, err := search(nodejsTestDir, nodejsTestRegEx)
+	if err != nil {
+		return nil, err
+	}
+	return testSlice, nil
+}
+
+// TestCmd implements TestRunner.TestCmd.
+func (nodejsRunner) TestCmd(test string) *exec.Cmd {
+	args := []string{filepath.Join(nodejsTestDir, "tools", "test.py"), test}
+	return exec.Command("/usr/bin/python", args...)
+}
diff --git a/test/runtimes/images/proctor/php.go b/test/runtimes/images/proctor/php.go
new file mode 100644
index 000000000..9115040e1
--- /dev/null
+++ b/test/runtimes/images/proctor/php.go
@@ -0,0 +1,42 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"os/exec"
+	"regexp"
+)
+
+var phpTestRegEx = regexp.MustCompile(`^.+\.phpt$`)
+
+// phpRunner implements TestRunner for PHP.
+type phpRunner struct{}
+
+var _ TestRunner = phpRunner{}
+
+// ListTests implements TestRunner.ListTests.
+func (phpRunner) ListTests() ([]string, error) {
+	testSlice, err := search(".", phpTestRegEx)
+	if err != nil {
+		return nil, err
+	}
+	return testSlice, nil
+}
+
+// TestCmd implements TestRunner.TestCmd.
+func (phpRunner) TestCmd(test string) *exec.Cmd {
+	args := []string{"test", "TESTS=" + test}
+	return exec.Command("make", args...)
+}
diff --git a/test/runtimes/images/proctor/proctor.go b/test/runtimes/images/proctor/proctor.go
new file mode 100644
index 000000000..45b3212de
--- /dev/null
+++ b/test/runtimes/images/proctor/proctor.go
@@ -0,0 +1,122 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Binary proctor runs the test for a particular runtime. It is meant to be
+// included in Docker images for all runtime tests.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+)
+
+// TestRunner is an interface that must be implemented for each runtime
+// integrated with proctor.
+type TestRunner interface {
+	// ListTests returns a string slice of tests available to run.
+	ListTests() ([]string, error)
+
+	// TestCmd returns an *exec.Cmd that will run the given test.
+	TestCmd(test string) *exec.Cmd
+}
+
+var (
+	runtime = flag.String("runtime", "", "name of runtime")
+	list    = flag.Bool("list", false, "list all available tests")
+	test    = flag.String("test", "", "run a single test from the list of available tests")
+)
+
+func main() {
+	flag.Parse()
+
+	if *runtime == "" {
+		log.Fatalf("runtime flag must be provided")
+	}
+
+	tr, err := testRunnerForRuntime(*runtime)
+	if err != nil {
+		log.Fatalf("%v", err)
+	}
+
+	// List tests.
+	if *list {
+		tests, err := tr.ListTests()
+		if err != nil {
+			log.Fatalf("failed to list tests: %v", err)
+		}
+		for _, test := range tests {
+			fmt.Println(test)
+		}
+		return
+	}
+
+	// Run a single test.
+	if *test == "" {
+		log.Fatalf("test flag must be provided")
+	}
+	cmd := tr.TestCmd(*test)
+	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
+	if err := cmd.Run(); err != nil {
+		log.Fatalf("FAIL %q: %v", err)
+	}
+}
+
+// testRunnerForRuntime returns a new TestRunner for the given runtime.
+func testRunnerForRuntime(runtime string) (TestRunner, error) {
+	switch runtime {
+	case "go":
+		return goRunner{}, nil
+	case "java":
+		return javaRunner{}, nil
+	case "nodejs":
+		return nodejsRunner{}, nil
+	case "php":
+		return phpRunner{}, nil
+	case "python":
+		return pythonRunner{}, nil
+	}
+	return nil, fmt.Errorf("invalid runtime %q", runtime)
+}
+
+// search is a helper function to find tests in the given directory that match
+// the regex.
+func search(root string, testFilter *regexp.Regexp) ([]string, error) {
+	var testSlice []string
+
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		name := filepath.Base(path)
+
+		if info.IsDir() || !testFilter.MatchString(name) {
+			return nil
+		}
+
+		relPath, err := filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+		testSlice = append(testSlice, relPath)
+		return nil
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("walking %q: %v", root, err)
+	}
+
+	return testSlice, nil
+}
diff --git a/test/runtimes/images/proctor/proctor_test.go b/test/runtimes/images/proctor/proctor_test.go
new file mode 100644
index 000000000..6bb61d142
--- /dev/null
+++ b/test/runtimes/images/proctor/proctor_test.go
@@ -0,0 +1,127 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"regexp"
+	"strings"
+	"testing"
+
+	"gvisor.dev/gvisor/runsc/testutil"
+)
+
+func touch(t *testing.T, name string) {
+	t.Helper()
+	f, err := os.Create(name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestSearchEmptyDir(t *testing.T) {
+	td, err := ioutil.TempDir(testutil.TmpDir(), "searchtest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(td)
+
+	var want []string
+
+	testFilter := regexp.MustCompile(`^test-[^-].+\.tc$`)
+	got, err := search(td, testFilter)
+	if err != nil {
+		t.Errorf("search error: %v", err)
+	}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("Found %#v; want %#v", got, want)
+	}
+}
+
+func TestSearch(t *testing.T) {
+	td, err := ioutil.TempDir(testutil.TmpDir(), "searchtest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(td)
+
+	// Creating various files similar to the test filter regex.
+	files := []string{
+		"emp/",
+		"tee/",
+		"test-foo.tc",
+		"test-foo.tc",
+		"test-bar.tc",
+		"test-sam.tc",
+		"Test-que.tc",
+		"test-brett",
+		"test--abc.tc",
+		"test---xyz.tc",
+		"test-bool.TC",
+		"--test-gvs.tc",
+		" test-pew.tc",
+		"dir/test_baz.tc",
+		"dir/testsnap.tc",
+		"dir/test-luk.tc",
+		"dir/nest/test-ok.tc",
+		"dir/dip/diz/goog/test-pack.tc",
+		"dir/dip/diz/wobble/thud/test-cas.e",
+		"dir/dip/diz/wobble/thud/test-cas.tc",
+	}
+	want := []string{
+		"dir/dip/diz/goog/test-pack.tc",
+		"dir/dip/diz/wobble/thud/test-cas.tc",
+		"dir/nest/test-ok.tc",
+		"dir/test-luk.tc",
+		"test-bar.tc",
+		"test-foo.tc",
+		"test-sam.tc",
+	}
+
+	for _, item := range files {
+		if strings.HasSuffix(item, "/") {
+			// This item is a directory, create it.
+			if err := os.MkdirAll(filepath.Join(td, item), 0755); err != nil {
+				t.Fatal(err)
+			}
+		} else {
+			// This item is a file, create the directory and touch file.
+			// Create directory in which file should be created
+			fullDirPath := filepath.Join(td, filepath.Dir(item))
+			if err := os.MkdirAll(fullDirPath, 0755); err != nil {
+				t.Fatal(err)
+			}
+			// Create file with full path to file.
+			touch(t, filepath.Join(td, item))
+		}
+	}
+
+	testFilter := regexp.MustCompile(`^test-[^-].+\.tc$`)
+	got, err := search(td, testFilter)
+	if err != nil {
+		t.Errorf("search error: %v", err)
+	}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("Found %#v; want %#v", got, want)
+	}
+}
diff --git a/test/runtimes/images/proctor/python.go b/test/runtimes/images/proctor/python.go
new file mode 100644
index 000000000..b9e0fbe6f
--- /dev/null
+++ b/test/runtimes/images/proctor/python.go
@@ -0,0 +1,49 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+// pythonRunner implements TestRunner for Python.
+type pythonRunner struct{}
+
+var _ TestRunner = pythonRunner{}
+
+// ListTests implements TestRunner.ListTests.
+func (pythonRunner) ListTests() ([]string, error) {
+	args := []string{"-m", "test", "--list-tests"}
+	cmd := exec.Command("./python", args...)
+	cmd.Stderr = os.Stderr
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list: %v", err)
+	}
+	var toolSlice []string
+	for _, test := range strings.Split(string(out), "\n") {
+		toolSlice = append(toolSlice, test)
+	}
+	return toolSlice, nil
+}
+
+// TestCmd implements TestRunner.TestCmd.
+func (pythonRunner) TestCmd(test string) *exec.Cmd {
+	args := []string{"-m", "test", test}
+	return exec.Command("./python", args...)
+}