12 files changed, 217 insertions, 151 deletions
diff --git a/test/runtimes/BUILD b/test/runtimes/BUILD
index 5616a8b7b..dfb4e2a97 100644
--- a/test/runtimes/BUILD
+++ b/test/runtimes/BUILD
@@ -1,25 +1,41 @@
 # These packages are used to run language runtime tests inside gVisor sandboxes.
 
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary")
 load("//test/runtimes:build_defs.bzl", "runtime_test")
 
 package(licenses = ["notice"])
 
-go_library(
-    name = "runtimes",
-    srcs = ["runtimes.go"],
-    importpath = "gvisor.dev/gvisor/test/runtimes",
+go_binary(
+    name = "runner",
+    testonly = 1,
+    srcs = ["runner.go"],
+    deps = [
+        "//runsc/dockerutil",
+        "//runsc/testutil",
+    ],
 )
 
 runtime_test(
-    name = "runtimes_test",
-    size = "small",
-    srcs = ["runtimes_test.go"],
-    embed = [":runtimes"],
-    tags = [
-        # Requires docker and runsc to be configured before the test runs.
-        "manual",
-        "local",
-    ],
-    deps = ["//runsc/testutil"],
+    image = "gcr.io/gvisor-presubmit/go1.12",
+    lang = "go",
+)
+
+runtime_test(
+    image = "gcr.io/gvisor-presubmit/java11",
+    lang = "java",
+)
+
+runtime_test(
+    image = "gcr.io/gvisor-presubmit/nodejs12.4.0",
+    lang = "nodejs",
+)
+
+runtime_test(
+    image = "gcr.io/gvisor-presubmit/php7.3.6",
+    lang = "php",
+)
+
+runtime_test(
+    image = "gcr.io/gvisor-presubmit/python3.7.3",
+    lang = "python",
 )
diff --git a/test/runtimes/build_defs.bzl b/test/runtimes/build_defs.bzl
index ac28cc037..19aceb6fb 100644
--- a/test/runtimes/build_defs.bzl
+++ b/test/runtimes/build_defs.bzl
@@ -1,19 +1,35 @@
 """Defines a rule for runsc test targets."""
 
-load("@io_bazel_rules_go//go:def.bzl", _go_test = "go_test")
-
 # runtime_test is a macro that will create targets to run the given test target
 # with different runtime options.
-def runtime_test(**kwargs):
-    """Runs the given test target with different runtime options."""
-    name = kwargs["name"]
-    _go_test(**kwargs)
-    kwargs["name"] = name + "_hostnet"
-    kwargs["args"] = ["--runtime-type=hostnet"]
-    _go_test(**kwargs)
-    kwargs["name"] = name + "_kvm"
-    kwargs["args"] = ["--runtime-type=kvm"]
-    _go_test(**kwargs)
-    kwargs["name"] = name + "_overlay"
-    kwargs["args"] = ["--runtime-type=overlay"]
-    _go_test(**kwargs)
+def runtime_test(
+        lang,
+        image,
+        shard_count = 20,
+        size = "enormous"):
+    sh_test(
+        name = lang + "_test",
+        srcs = ["runner.sh"],
+        args = [
+            "--lang",
+            lang,
+            "--image",
+            image,
+        ],
+        data = [
+            ":runner",
+        ],
+        size = size,
+        shard_count = shard_count,
+        tags = [
+            # Requires docker and runsc to be configured before the test runs.
+            "manual",
+            "local",
+        ],
+    )
+
+def sh_test(**kwargs):
+    """Wraps the standard sh_test."""
+    native.sh_test(
+        **kwargs
+    )
diff --git a/test/runtimes/images/Dockerfile_nodejs12.4.0 b/test/runtimes/images/Dockerfile_nodejs12.4.0
index 387824910..26f68b487 100644
--- a/test/runtimes/images/Dockerfile_nodejs12.4.0
+++ b/test/runtimes/images/Dockerfile_nodejs12.4.0
@@ -12,8 +12,8 @@ RUN apt-get update && apt-get install -y \
   python
 
 WORKDIR /root
-ARG VERSION v12.4.0
-RUN curl https://nodejs.org/dist/${VERSION}/node-${VERSION}.tar.gz
+ARG VERSION=v12.4.0
+RUN curl -o node-${VERSION}.tar.gz https://nodejs.org/dist/${VERSION}/node-${VERSION}.tar.gz
 RUN tar -zxf node-${VERSION}.tar.gz
 
 WORKDIR /root/node-${VERSION}
diff --git a/test/runtimes/images/Dockerfile_php7.3.6 b/test/runtimes/images/Dockerfile_php7.3.6
index 272b491b4..e6b4c6329 100644
--- a/test/runtimes/images/Dockerfile_php7.3.6
+++ b/test/runtimes/images/Dockerfile_php7.3.6
@@ -15,8 +15,8 @@ RUN apt-get update && apt-get install -y \
   re2c
 
 WORKDIR /root
-ARG VERSION 7.3.6
-RUN curl -o https://www.php.net/distributions/php-${VERSION}.tar.gz
+ARG VERSION=7.3.6
+RUN curl -o php-${VERSION}.tar.gz https://www.php.net/distributions/php-${VERSION}.tar.gz
 RUN tar -zxf php-${VERSION}.tar.gz
 
 WORKDIR /root/php-${VERSION}
diff --git a/test/runtimes/images/Dockerfile_python3.7.3 b/test/runtimes/images/Dockerfile_python3.7.3
index c9cc52d3a..905cd22d7 100644
--- a/test/runtimes/images/Dockerfile_python3.7.3
+++ b/test/runtimes/images/Dockerfile_python3.7.3
@@ -18,7 +18,7 @@ RUN apt-get update && apt-get install -y \
 
 # Use flags -LJO to follow the html redirect and download .tar.gz.
 WORKDIR /root
-ARG VERSION 3.7.3
+ARG VERSION=3.7.3
 RUN curl -LJO https://github.com/python/cpython/archive/v${VERSION}.tar.gz
 RUN tar -zxf cpython-${VERSION}.tar.gz
 
diff --git a/test/runtimes/images/proctor/java.go b/test/runtimes/images/proctor/java.go
index 594dc6cc6..8b362029d 100644
--- a/test/runtimes/images/proctor/java.go
+++ b/test/runtimes/images/proctor/java.go
@@ -26,7 +26,7 @@ import (
 var javaExclDirs = regexp.MustCompile(`(^(sun\/security)|(java\/util\/stream)|(java\/time)| )`)
 
 // Location of java tests.
-const javaTestDir = "/root/test"
+const javaTestDir = "/root/test/jdk"
 
 // javaRunner implements TestRunner for Java.
 type javaRunner struct{}
diff --git a/test/runtimes/images/proctor/nodejs.go b/test/runtimes/images/proctor/nodejs.go
index 4ef1afe63..bd57db444 100644
--- a/test/runtimes/images/proctor/nodejs.go
+++ b/test/runtimes/images/proctor/nodejs.go
@@ -41,6 +41,6 @@ func (nodejsRunner) ListTests() ([]string, error) {
 
 // TestCmd implements TestRunner.TestCmd.
 func (nodejsRunner) TestCmd(test string) *exec.Cmd {
-	args := []string{filepath.Join(nodejsTestDir, "tools", "test.py"), test}
+	args := []string{filepath.Join("tools", "test.py"), test}
 	return exec.Command("/usr/bin/python", args...)
 }
diff --git a/test/runtimes/images/proctor/proctor.go b/test/runtimes/images/proctor/proctor.go
index 45b3212de..e2c198b46 100644
--- a/test/runtimes/images/proctor/proctor.go
+++ b/test/runtimes/images/proctor/proctor.go
@@ -100,6 +100,10 @@ func search(root string, testFilter *regexp.Regexp) ([]string, error) {
 	var testSlice []string
 
 	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
 		name := filepath.Base(path)
 
 		if info.IsDir() || !testFilter.MatchString(name) {
@@ -113,7 +117,6 @@ func search(root string, testFilter *regexp.Regexp) ([]string, error) {
 		testSlice = append(testSlice, relPath)
 		return nil
 	})
-
 	if err != nil {
 		return nil, fmt.Errorf("walking %q: %v", root, err)
 	}
diff --git a/test/runtimes/runner.go b/test/runtimes/runner.go
new file mode 100644
index 000000000..3111963eb
--- /dev/null
+++ b/test/runtimes/runner.go
@@ -0,0 +1,109 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Binary runner runs the runtime tests in a Docker container.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"gvisor.dev/gvisor/runsc/dockerutil"
+	"gvisor.dev/gvisor/runsc/testutil"
+)
+
+var (
+	lang  = flag.String("lang", "", "language runtime to test")
+	image = flag.String("image", "", "docker image with runtime tests")
+)
+
+// Wait time for each test to run.
+const timeout = 5 * time.Minute
+
+func main() {
+	flag.Parse()
+	if *lang == "" || *image == "" {
+		fmt.Fprintf(os.Stderr, "lang and image flags must not be empty\n")
+		os.Exit(1)
+	}
+	tests, err := testsForImage(*lang, *image)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
+		os.Exit(1)
+	}
+
+	testing.Main(func(a, b string) (bool, error) {
+		return a == b, nil
+	}, tests, nil, nil)
+}
+
+func testsForImage(lang, image string) ([]testing.InternalTest, error) {
+	if err := dockerutil.Pull(image); err != nil {
+		return nil, fmt.Errorf("docker pull failed: %v", err)
+	}
+
+	c := dockerutil.MakeDocker("gvisor-list")
+	list, err := c.RunFg(image, "--runtime", lang, "--list")
+	defer c.CleanUp()
+	if err != nil {
+		return nil, fmt.Errorf("docker run failed: %v", err)
+	}
+
+	// Get subset of tests corresponding to shard.
+	tests := strings.Fields(list)
+	sort.Strings(tests)
+	begin, end, err := testutil.TestBoundsForShard(len(tests))
+	if err != nil {
+		return nil, fmt.Errorf("TestsForShard() failed: %v", err)
+	}
+	log.Printf("Got bounds [%d:%d) for shard out of %d total tests", begin, end, len(tests))
+	tests = tests[begin:end]
+
+	var itests []testing.InternalTest
+	for i, tc := range tests {
+		// Capture tc in this scope.
+		tc := tc
+		itests = append(itests, testing.InternalTest{
+			Name: tc,
+			F: func(t *testing.T) {
+				d := dockerutil.MakeDocker(fmt.Sprintf("gvisor-test-%d", i))
+				defer d.CleanUp()
+				if err := d.Run(image, "--runtime", lang, "--test", tc); err != nil {
+					t.Fatalf("docker test %q failed to run: %v", tc, err)
+				}
+
+				status, err := d.Wait(timeout)
+				if err != nil {
+					t.Fatalf("docker test %q failed to wait: %v", tc, err)
+				}
+				logs, err := d.Logs()
+				if err != nil {
+					t.Fatalf("docker test %q failed to supply logs: %v", tc, err)
+				}
+				if status == 0 {
+					t.Logf("test %q passed", tc)
+					return
+				}
+				t.Errorf("test %q failed: %v", tc, logs)
+			},
+		})
+	}
+	return itests, nil
+}
diff --git a/test/runtimes/runner.sh b/test/runtimes/runner.sh
new file mode 100755
index 000000000..a8d9a3460
--- /dev/null
+++ b/test/runtimes/runner.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Copyright 2018 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euf -x -o pipefail
+
+echo -- "$@"
+
+# Create outputs dir if it does not exist.
+if [[ -n "${TEST_UNDECLARED_OUTPUTS_DIR}" ]]; then
+  mkdir -p "${TEST_UNDECLARED_OUTPUTS_DIR}"
+  chmod a+rwx "${TEST_UNDECLARED_OUTPUTS_DIR}"
+fi
+
+# Update the timestamp on the shard status file. Bazel looks for this.
+touch "${TEST_SHARD_STATUS_FILE}"
+
+# Get location of runner binary.
+readonly runner=$(find "${TEST_SRCDIR}" -name runner)
+
+# Pass the arguments of this script directly to the runner.
+exec "${runner}" "$@"
+
diff --git a/test/runtimes/runtimes.go b/test/runtimes/runtimes.go
deleted file mode 100644
index 2568e07fe..000000000
--- a/test/runtimes/runtimes.go
+++ /dev/null
@@ -1,20 +0,0 @@
-// Copyright 2019 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package runtimes provides language tests for runsc runtimes.
-// Each test calls docker commands to start up a container for each supported runtime,
-// and tests that its respective language tests are behaving as expected, like
-// connecting to a port or looking at the output. The container is killed and deleted
-// at the end.
-package runtimes
diff --git a/test/runtimes/runtimes_test.go b/test/runtimes/runtimes_test.go
deleted file mode 100644
index 0ff5dda02..000000000
--- a/test/runtimes/runtimes_test.go
+++ /dev/null
@@ -1,93 +0,0 @@
-// Copyright 2019 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package runtimes
-
-import (
-	"strings"
-	"testing"
-	"time"
-
-	"gvisor.dev/gvisor/runsc/testutil"
-)
-
-// Wait time for each test to run.
-const timeout = 180 * time.Second
-
-// Helper function to execute the docker container associated with the
-// language passed.  Captures the output of the list function and executes
-// each test individually, supplying any errors recieved.
-func testLang(t *testing.T, lang string) {
-	t.Helper()
-
-	img := "gcr.io/gvisor-presubmit/" + lang
-	if err := testutil.Pull(img); err != nil {
-		t.Fatalf("docker pull failed: %v", err)
-	}
-
-	c := testutil.MakeDocker("gvisor-list")
-
-	list, err := c.RunFg(img, "--list")
-	if err != nil {
-		t.Fatalf("docker run failed: %v", err)
-	}
-	c.CleanUp()
-
-	tests := strings.Fields(list)
-
-	for _, tc := range tests {
-		tc := tc
-		t.Run(tc, func(t *testing.T) {
-			d := testutil.MakeDocker("gvisor-test")
-			if err := d.Run(img, "--test", tc); err != nil {
-				t.Fatalf("docker test %q failed to run: %v", tc, err)
-			}
-			defer d.CleanUp()
-
-			status, err := d.Wait(timeout)
-			if err != nil {
-				t.Fatalf("docker test %q failed to wait: %v", tc, err)
-			}
-			if status == 0 {
-				t.Logf("test %q passed", tc)
-				return
-			}
-			logs, err := d.Logs()
-			if err != nil {
-				t.Fatalf("docker test %q failed to supply logs: %v", tc, err)
-			}
-			t.Errorf("test %q failed: %v", tc, logs)
-		})
-	}
-}
-
-func TestGo(t *testing.T) {
-	testLang(t, "go")
-}
-
-func TestJava(t *testing.T) {
-	testLang(t, "java")
-}
-
-func TestNodejs(t *testing.T) {
-	testLang(t, "nodejs")
-}
-
-func TestPhp(t *testing.T) {
-	testLang(t, "php")
-}
-
-func TestPython(t *testing.T) {
-	testLang(t, "python")
-}