Shard the syscall tests.

PiperOrigin-RevId: 225574278
Change-Id: If5060a37e8a9b0120bec2b5de4037354f0eaba16

3 files changed, 161 insertions, 99 deletions

diff --git a/test/syscalls/BUILD b/test/syscalls/BUILD
index 318d80393..f3a7cc715 100644
--- a/test/syscalls/BUILD
+++ b/test/syscalls/BUILD
@@ -1,15 +1,15 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_test")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary")
+load("//test/syscalls:build_defs.bzl", "syscall_test")
 
 package(licenses = ["notice"])  # Apache 2.0
 
-load("//test/syscalls:build_defs.bzl", "syscall_test")
-
 syscall_test(test = "//test/syscalls/linux:32bit_test")
 
 syscall_test(test = "//test/syscalls/linux:accept_bind_stream_test")
 
 syscall_test(
-    size = "enormous",
+    size = "large",
+    shard_count = 50,
     test = "//test/syscalls/linux:accept_bind_test",
 )
 
@@ -106,6 +106,7 @@ syscall_test(test = "//test/syscalls/linux:fsync_test")
 
 syscall_test(
     size = "medium",
+    shard_count = 20,
     test = "//test/syscalls/linux:futex_test",
 )
 
@@ -154,6 +155,7 @@ syscall_test(test = "//test/syscalls/linux:mknod_test")
 
 syscall_test(
     size = "medium",
+    shard_count = 10,
     test = "//test/syscalls/linux:mmap_test",
 )
 
@@ -248,7 +250,10 @@ syscall_test(test = "//test/syscalls/linux:seccomp_test")
 
 syscall_test(test = "//test/syscalls/linux:select_test")
 
-syscall_test(test = "//test/syscalls/linux:semaphore_test")
+syscall_test(
+    shard_count = 20,
+    test = "//test/syscalls/linux:semaphore_test",
+)
 
 syscall_test(test = "//test/syscalls/linux:sendfile_socket_test")
 
@@ -281,7 +286,8 @@ syscall_test(
 )
 
 syscall_test(
-    size = "enormous",
+    size = "large",
+    shard_count = 50,
     test = "//test/syscalls/linux:socket_abstract_test",
 )
 
@@ -291,7 +297,8 @@ syscall_test(
 )
 
 syscall_test(
-    size = "enormous",
+    size = "large",
+    shard_count = 50,
     test = "//test/syscalls/linux:socket_domain_test",
 )
 
@@ -301,7 +308,8 @@ syscall_test(
 )
 
 syscall_test(
-    size = "enormous",
+    size = "large",
+    shard_count = 50,
     test = "//test/syscalls/linux:socket_filesystem_test",
 )
 
@@ -312,6 +320,7 @@ syscall_test(
 
 syscall_test(
     size = "large",
+    shard_count = 50,
     test = "//test/syscalls/linux:socket_ip_tcp_generic_loopback_test",
 )
 
@@ -322,11 +331,13 @@ syscall_test(
 
 syscall_test(
     size = "large",
+    shard_count = 50,
     test = "//test/syscalls/linux:socket_ip_tcp_loopback_test",
 )
 
 syscall_test(
     size = "medium",
+    shard_count = 50,
     test = "//test/syscalls/linux:socket_ip_tcp_udp_generic_loopback_test",
 )
 
@@ -337,6 +348,7 @@ syscall_test(
 
 syscall_test(
     size = "large",
+    shard_count = 50,
     test = "//test/syscalls/linux:socket_ip_udp_loopback_test",
 )
 
@@ -369,7 +381,8 @@ syscall_test(
 )
 
 syscall_test(
-    size = "enormous",
+    size = "large",
+    shard_count = 50,
     test = "//test/syscalls/linux:socket_unix_abstract_test",
 )
 
@@ -385,12 +398,14 @@ syscall_test(
 )
 
 syscall_test(
-    size = "enormous",
+    size = "large",
+    shard_count = 50,
     test = "//test/syscalls/linux:socket_unix_filesystem_test",
 )
 
 syscall_test(
-    size = "enormous",
+    size = "large",
+    shard_count = 50,
     test = "//test/syscalls/linux:socket_unix_pair_test",
 )
 
@@ -415,15 +430,20 @@ syscall_test(
     test = "//test/syscalls/linux:socket_unix_unbound_dgram_test",
 )
 
-syscall_test(test = "//test/syscalls/linux:socket_unix_unbound_filesystem_test")
+syscall_test(
+    size = "medium",
+    test = "//test/syscalls/linux:socket_unix_unbound_filesystem_test",
+)
 
 syscall_test(
     size = "medium",
+    shard_count = 50,
     test = "//test/syscalls/linux:socket_unix_unbound_seqpacket_test",
 )
 
 syscall_test(
     size = "large",
+    shard_count = 50,
     test = "//test/syscalls/linux:socket_unix_unbound_stream_test",
 )
 
@@ -449,6 +469,7 @@ syscall_test(test = "//test/syscalls/linux:sysret_test")
 
 syscall_test(
     size = "medium",
+    shard_count = 50,
     test = "//test/syscalls/linux:tcp_socket_test",
 )
 
@@ -468,6 +489,7 @@ syscall_test(test = "//test/syscalls/linux:udp_bind_test")
 
 syscall_test(
     size = "medium",
+    shard_count = 50,
     test = "//test/syscalls/linux:udp_socket_test",
 )
 
@@ -499,17 +521,12 @@ syscall_test(
 
 syscall_test(test = "//test/syscalls/linux:write_test")
 
-go_test(
-    name = "syscall_test",
-    srcs = ["syscall_test.go"],
+go_binary(
+    name = "syscall_test_runner",
+    srcs = ["syscall_test_runner.go"],
     data = [
         "//runsc",
     ],
-    # Running this test by itself does not make sense. It should only be run
-    # via the syscall_test macro.
-    tags = [
-        "manual",
-    ],
     deps = [
         "//pkg/log",
         "//runsc/boot",
diff --git a/test/syscalls/build_defs.bzl b/test/syscalls/build_defs.bzl
index 31b311f63..d7feeb9e1 100644
--- a/test/syscalls/build_defs.bzl
+++ b/test/syscalls/build_defs.bzl
@@ -2,12 +2,12 @@
 
 # syscall_test is a macro that will create targets to run the given test target
 # on the host (native) and runsc.
-def syscall_test(test, size = "small"):
-    _syscall_test(test, size, "native")
-    _syscall_test(test, size, "kvm")
-    _syscall_test(test, size, "ptrace")
+def syscall_test(test, shard_count = 1, size = "small"):
+    _syscall_test(test, shard_count, size, "native")
+    _syscall_test(test, shard_count, size, "kvm")
+    _syscall_test(test, shard_count, size, "ptrace")
 
-def _syscall_test(test, size, platform):
+def _syscall_test(test, shard_count, size, platform):
     test_name = test.split(":")[1]
 
     # Prepend "runsc" to non-native platform names.
@@ -30,13 +30,13 @@ def _syscall_test(test, size, platform):
         srcs = ["syscall_test_runner.sh"],
         name = test_name + "_" + full_platform,
         data = [
-            ":syscall_test",
+            ":syscall_test_runner",
             test,
         ],
         args = [
-            # First argument is location to syscall_test binary.
-            "$(location :syscall_test)",
-            # Rest of arguments are passed directly to syscall_test binary.
+            # First argument is location to syscall_test_runner go binary.
+            "$(location :syscall_test_runner)",
+            # Rest of arguments are passed directly to syscall_test_runner binary.
             "--test-name=" + test_name,
             "--platform=" + platform,
             "--debug=false",
@@ -45,6 +45,7 @@ def _syscall_test(test, size, platform):
         ],
         size = size,
         tags = tags,
+        shard_count = shard_count,
     )
 
 def sh_test(**kwargs):
diff --git a/test/syscalls/syscall_test.go b/test/syscalls/syscall_test_runner.go
index 8463289fe..9ee0361ee 100644
--- a/test/syscalls/syscall_test.go
+++ b/test/syscalls/syscall_test_runner.go
@@ -12,17 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package syscall_test runs the syscall test suites in gVisor containers. It
-// is meant to be run with "go test", and will panic if run on its own.
-package syscall_test
+// Binary syscall_test_runner runs the syscall test suites in gVisor
+// containers and on the host platform.
+package main
 
 import (
 	"flag"
 	"fmt"
 	"io/ioutil"
+	"math"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"syscall"
 	"testing"
@@ -47,62 +49,6 @@ var (
 	parallel = flag.Bool("parallel", false, "run tests in parallel")
 )
 
-func TestSyscalls(t *testing.T) {
-	if *testName == "" {
-		t.Fatalf("test-name flag must be provided")
-	}
-
-	// Get path to test binary.
-	fullTestName := filepath.Join(testDir, *testName)
-	testBin, err := testutil.FindFile(fullTestName)
-	if err != nil {
-		t.Fatalf("FindFile(%q) failed: %v", fullTestName, err)
-	}
-
-	// Get all test cases in each binary.
-	testCases, err := gtest.ParseTestCases(testBin)
-	if err != nil {
-		t.Fatalf("ParseTestCases(%q) failed: %v", testBin, err)
-	}
-
-	// Make sure stdout and stderr are opened with O_APPEND, otherwise logs
-	// from outside the sandbox can (and will) stomp on logs from inside
-	// the sandbox.
-	for _, f := range []*os.File{os.Stdout, os.Stderr} {
-		flags, err := unix.FcntlInt(f.Fd(), unix.F_GETFL, 0)
-		if err != nil {
-			t.Fatalf("error getting file flags for %v: %v", f, err)
-		}
-		if flags&unix.O_APPEND == 0 {
-			flags |= unix.O_APPEND
-			if _, err := unix.FcntlInt(f.Fd(), unix.F_SETFL, flags); err != nil {
-				t.Fatalf("error setting file flags for %v: %v", f, err)
-			}
-		}
-	}
-
-	for _, tc := range testCases {
-		// Capture tc.
-		tc := tc
-
-		testName := fmt.Sprintf("%s_%s", tc.Suite, tc.Name)
-		t.Run(testName, func(t *testing.T) {
-			if *parallel {
-				t.Parallel()
-			}
-
-			if *platform == "native" {
-				// Run the test case on host.
-				runTestCaseNative(testBin, tc, t)
-				return
-			}
-
-			// Run the test case in runsc.
-			runTestCaseRunsc(testBin, tc, t)
-		})
-	}
-}
-
 // runTestCaseNative runs the test case directly on the host machine.
 func runTestCaseNative(testBin string, tc gtest.TestCase, t *testing.T) {
 	// These tests might be running in parallel, so make sure they have a
@@ -128,9 +74,14 @@ func runTestCaseNative(testBin string, tc gtest.TestCase, t *testing.T) {
 	if !found {
 		env = append(env, newEnvVar)
 	}
-	// Remove the TEST_PREMATURE_EXIT_FILE variable and XML_OUTPUT_FILE
-	// from the environment.
-	env = filterEnv(env, []string{"TEST_PREMATURE_EXIT_FILE", "XML_OUTPUT_FILE"})
+	// Remove env variables that cause the gunit binary to write output
+	// files, since they will stomp on eachother, and on the output files
+	// from this go test.
+	env = filterEnv(env, []string{"GUNIT_OUTPUT", "TEST_PREMATURE_EXIT_FILE", "XML_OUTPUT_FILE"})
+
+	// Remove shard env variables so that the gunit binary does not try to
+	// intepret them.
+	env = filterEnv(env, []string{"TEST_SHARD_INDEX", "TEST_TOTAL_SHARDS"})
 
 	cmd := exec.Command(testBin, gtest.FilterTestFlag+"="+tc.FullName())
 	cmd.Env = env
@@ -173,9 +124,14 @@ func runTestCaseRunsc(testBin string, tc gtest.TestCase, t *testing.T) {
 	platformVar := "TEST_ON_GVISOR"
 	env := append(os.Environ(), platformVar+"="+*platform)
 
-	// Remove the TEST_PREMATURE_EXIT_FILE variable and XML_OUTPUT_FILE
-	// from the environment.
-	env = filterEnv(env, []string{"TEST_PREMATURE_EXIT_FILE", "XML_OUTPUT_FILE"})
+	// Remove env variables that cause the gunit binary to write output
+	// files, since they will stomp on eachother, and on the output files
+	// from this go test.
+	env = filterEnv(env, []string{"GUNIT_OUTPUT", "TEST_PREMATURE_EXIT_FILE", "XML_OUTPUT_FILE"})
+
+	// Remove shard env variables so that the gunit binary does not try to
+	// intepret them.
+	env = filterEnv(env, []string{"TEST_SHARD_INDEX", "TEST_TOTAL_SHARDS"})
 
 	// Set TEST_TMPDIR to /tmp, as some of the syscall tests require it to
 	// be backed by tmpfs.
@@ -224,22 +180,110 @@ func filterEnv(env, blacklist []string) []string {
 	return out
 }
 
-func TestMain(m *testing.M) {
+func fatalf(s string, args ...interface{}) {
+	fmt.Fprintf(os.Stderr, s+"\n", args...)
+	os.Exit(1)
+}
+
+func matchString(a, b string) (bool, error) {
+	return a == b, nil
+}
+
+func main() {
 	flag.Parse()
+	if *testName == "" {
+		fatalf("test-name flag must be provided")
+	}
 
 	log.SetLevel(log.Warning)
 	if *debug {
 		log.SetLevel(log.Debug)
 	}
-	if err := testutil.ConfigureExePath(); err != nil {
-		panic(err.Error())
-	}
 
 	if *platform != "native" {
+		if err := testutil.ConfigureExePath(); err != nil {
+			panic(err.Error())
+		}
 		// The native tests don't expect to be running as root, but
 		// runsc requires it.
 		testutil.RunAsRoot()
 	}
 
-	os.Exit(m.Run())
+	// Make sure stdout and stderr are opened with O_APPEND, otherwise logs
+	// from outside the sandbox can (and will) stomp on logs from inside
+	// the sandbox.
+	for _, f := range []*os.File{os.Stdout, os.Stderr} {
+		flags, err := unix.FcntlInt(f.Fd(), unix.F_GETFL, 0)
+		if err != nil {
+			fatalf("error getting file flags for %v: %v", f, err)
+		}
+		if flags&unix.O_APPEND == 0 {
+			flags |= unix.O_APPEND
+			if _, err := unix.FcntlInt(f.Fd(), unix.F_SETFL, flags); err != nil {
+				fatalf("error setting file flags for %v: %v", f, err)
+			}
+		}
+	}
+
+	// Get path to test binary.
+	fullTestName := filepath.Join(testDir, *testName)
+	testBin, err := testutil.FindFile(fullTestName)
+	if err != nil {
+		fatalf("FindFile(%q) failed: %v", fullTestName, err)
+	}
+
+	// Get all test cases in each binary.
+	testCases, err := gtest.ParseTestCases(testBin)
+	if err != nil {
+		fatalf("ParseTestCases(%q) failed: %v", testBin, err)
+	}
+
+	// If sharding, then get the subset of tests to run based on the shard index.
+	if indexStr, totalStr := os.Getenv("TEST_SHARD_INDEX"), os.Getenv("TEST_TOTAL_SHARDS"); indexStr != "" && totalStr != "" {
+		// Parse index and total to ints.
+		index, err := strconv.Atoi(indexStr)
+		if err != nil {
+			fatalf("invalid TEST_SHARD_INDEX %q: %v", indexStr, err)
+		}
+		total, err := strconv.Atoi(totalStr)
+		if err != nil {
+			fatalf("invalid TEST_TOTAL_SHARDS %q: %v", totalStr, err)
+		}
+		// Calculate subslice of tests to run.
+		shardSize := int(math.Ceil(float64(len(testCases)) / float64(total)))
+		begin := index * shardSize
+		end := ((index + 1) * shardSize) - 1
+		if begin > len(testCases) {
+			// Nothing to run.
+			return
+		}
+		if end > len(testCases) {
+			end = len(testCases)
+		}
+		testCases = testCases[begin:end]
+	}
+
+	var tests []testing.InternalTest
+	for _, tc := range testCases {
+		// Capture tc.
+		tc := tc
+		testName := fmt.Sprintf("%s_%s", tc.Suite, tc.Name)
+		tests = append(tests, testing.InternalTest{
+			Name: testName,
+			F: func(t *testing.T) {
+				if *parallel {
+					t.Parallel()
+				}
+				if *platform == "native" {
+					// Run the test case on host.
+					runTestCaseNative(testBin, tc, t)
+				} else {
+					// Run the test case in runsc.
+					runTestCaseRunsc(testBin, tc, t)
+				}
+			},
+		})
+	}
+
+	testing.Main(matchString, tests, nil, nil)
 }