Shard the runtime tests.

Default of 20 shards was arbitrary and will need fine-tuning in later CLs.

PiperOrigin-RevId: 269922871

8 files changed, 255 insertions, 169 deletions

diff --git a/runsc/testutil/testutil.go b/runsc/testutil/testutil.go
index 57ab73d97..edf8b126c 100644
--- a/runsc/testutil/testutil.go
+++ b/runsc/testutil/testutil.go
@@ -26,12 +26,14 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"math"
 	"math/rand"
 	"net/http"
 	"os"
 	"os/exec"
 	"os/signal"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -438,3 +440,44 @@ func IsStatic(filename string) (bool, error) {
 	}
 	return true, nil
 }
+
+// TestBoundsForShard calculates the beginning and end indices for the test
+// based on the TEST_SHARD_INDEX and TEST_TOTAL_SHARDS environment vars. The
+// returned ints are the beginning (inclusive) and end (exclusive) of the
+// subslice corresponding to the shard. If either of the env vars are not
+// present, then the function will return bounds that include all tests. If
+// there are more shards than there are tests, then the returned list may be
+// empty.
+func TestBoundsForShard(numTests int) (int, int, error) {
+	var (
+		begin = 0
+		end   = numTests
+	)
+	indexStr, totalStr := os.Getenv("TEST_SHARD_INDEX"), os.Getenv("TEST_TOTAL_SHARDS")
+	if indexStr == "" || totalStr == "" {
+		return begin, end, nil
+	}
+
+	// Parse index and total to ints.
+	shardIndex, err := strconv.Atoi(indexStr)
+	if err != nil {
+		return 0, 0, fmt.Errorf("invalid TEST_SHARD_INDEX %q: %v", indexStr, err)
+	}
+	shardTotal, err := strconv.Atoi(totalStr)
+	if err != nil {
+		return 0, 0, fmt.Errorf("invalid TEST_TOTAL_SHARDS %q: %v", totalStr, err)
+	}
+
+	// Calculate!
+	shardSize := int(math.Ceil(float64(numTests) / float64(shardTotal)))
+	begin = shardIndex * shardSize
+	end = ((shardIndex + 1) * shardSize)
+	if begin > numTests {
+		// Nothing to run.
+		return 0, 0, nil
+	}
+	if end > numTests {
+		end = numTests
+	}
+	return begin, end, nil
+}
diff --git a/test/runtimes/BUILD b/test/runtimes/BUILD
index 5616a8b7b..dfb4e2a97 100644
--- a/test/runtimes/BUILD
+++ b/test/runtimes/BUILD
@@ -1,25 +1,41 @@
 # These packages are used to run language runtime tests inside gVisor sandboxes.
 
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary")
 load("//test/runtimes:build_defs.bzl", "runtime_test")
 
 package(licenses = ["notice"])
 
-go_library(
-    name = "runtimes",
-    srcs = ["runtimes.go"],
-    importpath = "gvisor.dev/gvisor/test/runtimes",
+go_binary(
+    name = "runner",
+    testonly = 1,
+    srcs = ["runner.go"],
+    deps = [
+        "//runsc/dockerutil",
+        "//runsc/testutil",
+    ],
 )
 
 runtime_test(
-    name = "runtimes_test",
-    size = "small",
-    srcs = ["runtimes_test.go"],
-    embed = [":runtimes"],
-    tags = [
-        # Requires docker and runsc to be configured before the test runs.
-        "manual",
-        "local",
-    ],
-    deps = ["//runsc/testutil"],
+    image = "gcr.io/gvisor-presubmit/go1.12",
+    lang = "go",
+)
+
+runtime_test(
+    image = "gcr.io/gvisor-presubmit/java11",
+    lang = "java",
+)
+
+runtime_test(
+    image = "gcr.io/gvisor-presubmit/nodejs12.4.0",
+    lang = "nodejs",
+)
+
+runtime_test(
+    image = "gcr.io/gvisor-presubmit/php7.3.6",
+    lang = "php",
+)
+
+runtime_test(
+    image = "gcr.io/gvisor-presubmit/python3.7.3",
+    lang = "python",
 )
diff --git a/test/runtimes/build_defs.bzl b/test/runtimes/build_defs.bzl
index ac28cc037..19aceb6fb 100644
--- a/test/runtimes/build_defs.bzl
+++ b/test/runtimes/build_defs.bzl
@@ -1,19 +1,35 @@
 """Defines a rule for runsc test targets."""
 
-load("@io_bazel_rules_go//go:def.bzl", _go_test = "go_test")
-
 # runtime_test is a macro that will create targets to run the given test target
 # with different runtime options.
-def runtime_test(**kwargs):
-    """Runs the given test target with different runtime options."""
-    name = kwargs["name"]
-    _go_test(**kwargs)
-    kwargs["name"] = name + "_hostnet"
-    kwargs["args"] = ["--runtime-type=hostnet"]
-    _go_test(**kwargs)
-    kwargs["name"] = name + "_kvm"
-    kwargs["args"] = ["--runtime-type=kvm"]
-    _go_test(**kwargs)
-    kwargs["name"] = name + "_overlay"
-    kwargs["args"] = ["--runtime-type=overlay"]
-    _go_test(**kwargs)
+def runtime_test(
+        lang,
+        image,
+        shard_count = 20,
+        size = "enormous"):
+    sh_test(
+        name = lang + "_test",
+        srcs = ["runner.sh"],
+        args = [
+            "--lang",
+            lang,
+            "--image",
+            image,
+        ],
+        data = [
+            ":runner",
+        ],
+        size = size,
+        shard_count = shard_count,
+        tags = [
+            # Requires docker and runsc to be configured before the test runs.
+            "manual",
+            "local",
+        ],
+    )
+
+def sh_test(**kwargs):
+    """Wraps the standard sh_test."""
+    native.sh_test(
+        **kwargs
+    )
diff --git a/test/runtimes/runner.go b/test/runtimes/runner.go
new file mode 100644
index 000000000..3111963eb
--- /dev/null
+++ b/test/runtimes/runner.go
@@ -0,0 +1,109 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Binary runner runs the runtime tests in a Docker container.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"gvisor.dev/gvisor/runsc/dockerutil"
+	"gvisor.dev/gvisor/runsc/testutil"
+)
+
+var (
+	lang  = flag.String("lang", "", "language runtime to test")
+	image = flag.String("image", "", "docker image with runtime tests")
+)
+
+// Wait time for each test to run.
+const timeout = 5 * time.Minute
+
+func main() {
+	flag.Parse()
+	if *lang == "" || *image == "" {
+		fmt.Fprintf(os.Stderr, "lang and image flags must not be empty\n")
+		os.Exit(1)
+	}
+	tests, err := testsForImage(*lang, *image)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
+		os.Exit(1)
+	}
+
+	testing.Main(func(a, b string) (bool, error) {
+		return a == b, nil
+	}, tests, nil, nil)
+}
+
+func testsForImage(lang, image string) ([]testing.InternalTest, error) {
+	if err := dockerutil.Pull(image); err != nil {
+		return nil, fmt.Errorf("docker pull failed: %v", err)
+	}
+
+	c := dockerutil.MakeDocker("gvisor-list")
+	list, err := c.RunFg(image, "--runtime", lang, "--list")
+	defer c.CleanUp()
+	if err != nil {
+		return nil, fmt.Errorf("docker run failed: %v", err)
+	}
+
+	// Get subset of tests corresponding to shard.
+	tests := strings.Fields(list)
+	sort.Strings(tests)
+	begin, end, err := testutil.TestBoundsForShard(len(tests))
+	if err != nil {
+		return nil, fmt.Errorf("TestsForShard() failed: %v", err)
+	}
+	log.Printf("Got bounds [%d:%d) for shard out of %d total tests", begin, end, len(tests))
+	tests = tests[begin:end]
+
+	var itests []testing.InternalTest
+	for i, tc := range tests {
+		// Capture tc in this scope.
+		tc := tc
+		itests = append(itests, testing.InternalTest{
+			Name: tc,
+			F: func(t *testing.T) {
+				d := dockerutil.MakeDocker(fmt.Sprintf("gvisor-test-%d", i))
+				defer d.CleanUp()
+				if err := d.Run(image, "--runtime", lang, "--test", tc); err != nil {
+					t.Fatalf("docker test %q failed to run: %v", tc, err)
+				}
+
+				status, err := d.Wait(timeout)
+				if err != nil {
+					t.Fatalf("docker test %q failed to wait: %v", tc, err)
+				}
+				logs, err := d.Logs()
+				if err != nil {
+					t.Fatalf("docker test %q failed to supply logs: %v", tc, err)
+				}
+				if status == 0 {
+					t.Logf("test %q passed", tc)
+					return
+				}
+				t.Errorf("test %q failed: %v", tc, logs)
+			},
+		})
+	}
+	return itests, nil
+}
diff --git a/test/runtimes/runner.sh b/test/runtimes/runner.sh
new file mode 100755
index 000000000..a8d9a3460
--- /dev/null
+++ b/test/runtimes/runner.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Copyright 2018 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euf -x -o pipefail
+
+echo -- "$@"
+
+# Create outputs dir if it does not exist.
+if [[ -n "${TEST_UNDECLARED_OUTPUTS_DIR}" ]]; then
+  mkdir -p "${TEST_UNDECLARED_OUTPUTS_DIR}"
+  chmod a+rwx "${TEST_UNDECLARED_OUTPUTS_DIR}"
+fi
+
+# Update the timestamp on the shard status file. Bazel looks for this.
+touch "${TEST_SHARD_STATUS_FILE}"
+
+# Get location of runner binary.
+readonly runner=$(find "${TEST_SRCDIR}" -name runner)
+
+# Pass the arguments of this script directly to the runner.
+exec "${runner}" "$@"
+
diff --git a/test/runtimes/runtimes.go b/test/runtimes/runtimes.go
deleted file mode 100644
index 2568e07fe..000000000
--- a/test/runtimes/runtimes.go
+++ /dev/null
@@ -1,20 +0,0 @@
-// Copyright 2019 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package runtimes provides language tests for runsc runtimes.
-// Each test calls docker commands to start up a container for each supported runtime,
-// and tests that its respective language tests are behaving as expected, like
-// connecting to a port or looking at the output. The container is killed and deleted
-// at the end.
-package runtimes
diff --git a/test/runtimes/runtimes_test.go b/test/runtimes/runtimes_test.go
deleted file mode 100644
index 0ff5dda02..000000000
--- a/test/runtimes/runtimes_test.go
+++ /dev/null
@@ -1,93 +0,0 @@
-// Copyright 2019 The gVisor Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package runtimes
-
-import (
-	"strings"
-	"testing"
-	"time"
-
-	"gvisor.dev/gvisor/runsc/testutil"
-)
-
-// Wait time for each test to run.
-const timeout = 180 * time.Second
-
-// Helper function to execute the docker container associated with the
-// language passed.  Captures the output of the list function and executes
-// each test individually, supplying any errors recieved.
-func testLang(t *testing.T, lang string) {
-	t.Helper()
-
-	img := "gcr.io/gvisor-presubmit/" + lang
-	if err := testutil.Pull(img); err != nil {
-		t.Fatalf("docker pull failed: %v", err)
-	}
-
-	c := testutil.MakeDocker("gvisor-list")
-
-	list, err := c.RunFg(img, "--list")
-	if err != nil {
-		t.Fatalf("docker run failed: %v", err)
-	}
-	c.CleanUp()
-
-	tests := strings.Fields(list)
-
-	for _, tc := range tests {
-		tc := tc
-		t.Run(tc, func(t *testing.T) {
-			d := testutil.MakeDocker("gvisor-test")
-			if err := d.Run(img, "--test", tc); err != nil {
-				t.Fatalf("docker test %q failed to run: %v", tc, err)
-			}
-			defer d.CleanUp()
-
-			status, err := d.Wait(timeout)
-			if err != nil {
-				t.Fatalf("docker test %q failed to wait: %v", tc, err)
-			}
-			if status == 0 {
-				t.Logf("test %q passed", tc)
-				return
-			}
-			logs, err := d.Logs()
-			if err != nil {
-				t.Fatalf("docker test %q failed to supply logs: %v", tc, err)
-			}
-			t.Errorf("test %q failed: %v", tc, logs)
-		})
-	}
-}
-
-func TestGo(t *testing.T) {
-	testLang(t, "go")
-}
-
-func TestJava(t *testing.T) {
-	testLang(t, "java")
-}
-
-func TestNodejs(t *testing.T) {
-	testLang(t, "nodejs")
-}
-
-func TestPhp(t *testing.T) {
-	testLang(t, "php")
-}
-
-func TestPython(t *testing.T) {
-	testLang(t, "python")
-}
diff --git a/test/syscalls/syscall_test_runner.go b/test/syscalls/syscall_test_runner.go
index e900f8abc..c1e9ce22c 100644
--- a/test/syscalls/syscall_test_runner.go
+++ b/test/syscalls/syscall_test_runner.go
@@ -20,12 +20,10 @@ import (
 	"flag"
 	"fmt"
 	"io/ioutil"
-	"math"
 	"os"
 	"os/exec"
 	"os/signal"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"syscall"
 	"testing"
@@ -358,32 +356,14 @@ func main() {
 		fatalf("ParseTestCases(%q) failed: %v", testBin, err)
 	}
 
-	// If sharding, then get the subset of tests to run based on the shard index.
-	if indexStr, totalStr := os.Getenv("TEST_SHARD_INDEX"), os.Getenv("TEST_TOTAL_SHARDS"); indexStr != "" && totalStr != "" {
-		// Parse index and total to ints.
-		index, err := strconv.Atoi(indexStr)
-		if err != nil {
-			fatalf("invalid TEST_SHARD_INDEX %q: %v", indexStr, err)
-		}
-		total, err := strconv.Atoi(totalStr)
-		if err != nil {
-			fatalf("invalid TEST_TOTAL_SHARDS %q: %v", totalStr, err)
-		}
-		// Calculate subslice of tests to run.
-		shardSize := int(math.Ceil(float64(len(testCases)) / float64(total)))
-		begin := index * shardSize
-		// Set end as begin of next subslice.
-		end := ((index + 1) * shardSize)
-		if begin > len(testCases) {
-			// Nothing to run.
-			return
-		}
-		if end > len(testCases) {
-			end = len(testCases)
-		}
-		testCases = testCases[begin:end]
+	// Get subset of tests corresponding to shard.
+	begin, end, err := testutil.TestBoundsForShard(len(testCases))
+	if err != nil {
+		fatalf("TestsForShard() failed: %v", err)
 	}
+	testCases = testCases[begin:end]
 
+	// Run the tests.
 	var tests []testing.InternalTest
 	for _, tc := range testCases {
 		// Capture tc.