summaryrefslogtreecommitdiffhomepage
path: root/test/runtimes/runner
diff options
context:
space:
mode:
Diffstat (limited to 'test/runtimes/runner')
-rw-r--r--test/runtimes/runner/BUILD5
-rw-r--r--test/runtimes/runner/exclude_test.go (renamed from test/runtimes/runner/blacklist_test.go)12
-rw-r--r--test/runtimes/runner/main.go90
3 files changed, 62 insertions, 45 deletions
diff --git a/test/runtimes/runner/BUILD b/test/runtimes/runner/BUILD
index 63924b9c5..dc0d5d5b4 100644
--- a/test/runtimes/runner/BUILD
+++ b/test/runtimes/runner/BUILD
@@ -8,14 +8,15 @@ go_binary(
srcs = ["main.go"],
visibility = ["//test/runtimes:__pkg__"],
deps = [
+ "//pkg/log",
"//pkg/test/dockerutil",
"//pkg/test/testutil",
],
)
go_test(
- name = "blacklist_test",
+ name = "exclude_test",
size = "small",
- srcs = ["blacklist_test.go"],
+ srcs = ["exclude_test.go"],
library = ":runner",
)
diff --git a/test/runtimes/runner/blacklist_test.go b/test/runtimes/runner/exclude_test.go
index 0ff69ab18..67c2170c8 100644
--- a/test/runtimes/runner/blacklist_test.go
+++ b/test/runtimes/runner/exclude_test.go
@@ -25,13 +25,13 @@ func TestMain(m *testing.M) {
os.Exit(m.Run())
}
-// Test that the blacklist parses without error.
-func TestBlacklists(t *testing.T) {
- bl, err := getBlacklist()
+// Test that the exclude file parses without error.
+func TestExcludelist(t *testing.T) {
+ ex, err := getExcludes()
if err != nil {
- t.Fatalf("error parsing blacklist: %v", err)
+ t.Fatalf("error parsing exclude file: %v", err)
}
- if *blacklistFile != "" && len(bl) == 0 {
- t.Errorf("got empty blacklist for file %q", *blacklistFile)
+ if *excludeFile != "" && len(ex) == 0 {
+ t.Errorf("got empty excludes for file %q", *excludeFile)
}
}
diff --git a/test/runtimes/runner/main.go b/test/runtimes/runner/main.go
index 57540e00e..948e7cf9c 100644
--- a/test/runtimes/runner/main.go
+++ b/test/runtimes/runner/main.go
@@ -16,6 +16,7 @@
package main
import (
+ "context"
"encoding/csv"
"flag"
"fmt"
@@ -26,18 +27,20 @@ import (
"testing"
"time"
+ "gvisor.dev/gvisor/pkg/log"
"gvisor.dev/gvisor/pkg/test/dockerutil"
"gvisor.dev/gvisor/pkg/test/testutil"
)
var (
- lang = flag.String("lang", "", "language runtime to test")
- image = flag.String("image", "", "docker image with runtime tests")
- blacklistFile = flag.String("blacklist_file", "", "file containing blacklist of tests to exclude, in CSV format with fields: test name, bug id, comment")
+ lang = flag.String("lang", "", "language runtime to test")
+ image = flag.String("image", "", "docker image with runtime tests")
+ excludeFile = flag.String("exclude_file", "", "file containing list of tests to exclude, in CSV format with fields: test name, bug id, comment")
+ batchSize = flag.Int("batch", 50, "number of test cases run in one command")
)
// Wait time for each test to run.
-const timeout = 5 * time.Minute
+const timeout = 90 * time.Minute
func main() {
flag.Parse()
@@ -52,21 +55,27 @@ func main() {
// defered functions before exiting. It returns an exit code that should be
// passed to os.Exit.
func runTests() int {
- // Get tests to blacklist.
- blacklist, err := getBlacklist()
+ // Get tests to exclude..
+ excludes, err := getExcludes()
if err != nil {
- fmt.Fprintf(os.Stderr, "Error getting blacklist: %s\n", err.Error())
+ fmt.Fprintf(os.Stderr, "Error getting exclude list: %s\n", err.Error())
return 1
}
// Construct the shared docker instance.
- d := dockerutil.MakeDocker(testutil.DefaultLogger(*lang))
- defer d.CleanUp()
+ ctx := context.Background()
+ d := dockerutil.MakeContainer(ctx, testutil.DefaultLogger(*lang))
+ defer d.CleanUp(ctx)
+
+ if err := testutil.TouchShardStatusFile(); err != nil {
+ fmt.Fprintf(os.Stderr, "error touching status shard file: %v\n", err)
+ return 1
+ }
// Get a slice of tests to run. This will also start a single Docker
// container that will be used to run each test. The final test will
// stop the Docker container.
- tests, err := getTests(d, blacklist)
+ tests, err := getTests(ctx, d, excludes)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err.Error())
return 1
@@ -77,17 +86,18 @@ func runTests() int {
}
// getTests executes all tests as table tests.
-func getTests(d *dockerutil.Docker, blacklist map[string]struct{}) ([]testing.InternalTest, error) {
+func getTests(ctx context.Context, d *dockerutil.Container, excludes map[string]struct{}) ([]testing.InternalTest, error) {
// Start the container.
- d.CopyFiles("/proctor", "test/runtimes/proctor/proctor")
- if err := d.Spawn(dockerutil.RunOpts{
+ opts := dockerutil.RunOpts{
Image: fmt.Sprintf("runtimes/%s", *image),
- }, "/proctor/proctor", "--pause"); err != nil {
+ }
+ d.CopyFiles(&opts, "/proctor", "test/runtimes/proctor/proctor")
+ if err := d.Spawn(ctx, opts, "/proctor/proctor", "--pause"); err != nil {
return nil, fmt.Errorf("docker run failed: %v", err)
}
// Get a list of all tests in the image.
- list, err := d.Exec(dockerutil.RunOpts{}, "/proctor/proctor", "--runtime", *lang, "--list")
+ list, err := d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", *lang, "--list")
if err != nil {
return nil, fmt.Errorf("docker exec failed: %v", err)
}
@@ -102,17 +112,23 @@ func getTests(d *dockerutil.Docker, blacklist map[string]struct{}) ([]testing.In
}
var itests []testing.InternalTest
- for _, tci := range indices {
- // Capture tc in this scope.
- tc := tests[tci]
+ for i := 0; i < len(indices); i += *batchSize {
+ var tcs []string
+ end := i + *batchSize
+ if end > len(indices) {
+ end = len(indices)
+ }
+ for _, tc := range indices[i:end] {
+ // Add test if not excluded.
+ if _, ok := excludes[tests[tc]]; ok {
+ log.Infof("Skipping test case %s\n", tests[tc])
+ continue
+ }
+ tcs = append(tcs, tests[tc])
+ }
itests = append(itests, testing.InternalTest{
- Name: tc,
+ Name: strings.Join(tcs, ", "),
F: func(t *testing.T) {
- // Is the test blacklisted?
- if _, ok := blacklist[tc]; ok {
- t.Skipf("SKIP: blacklisted test %q", tc)
- }
-
var (
now = time.Now()
done = make(chan struct{})
@@ -121,20 +137,20 @@ func getTests(d *dockerutil.Docker, blacklist map[string]struct{}) ([]testing.In
)
go func() {
- fmt.Printf("RUNNING %s...\n", tc)
- output, err = d.Exec(dockerutil.RunOpts{}, "/proctor/proctor", "--runtime", *lang, "--test", tc)
+ fmt.Printf("RUNNING the following in a batch\n%s\n", strings.Join(tcs, "\n"))
+ output, err = d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", *lang, "--tests", strings.Join(tcs, ","))
close(done)
}()
select {
case <-done:
if err == nil {
- fmt.Printf("PASS: %s (%v)\n\n", tc, time.Since(now))
+ fmt.Printf("PASS: (%v)\n\n", time.Since(now))
return
}
- t.Errorf("FAIL: %s (%v):\n%s\n", tc, time.Since(now), output)
+ t.Errorf("FAIL: (%v):\n%s\n", time.Since(now), output)
case <-time.After(timeout):
- t.Errorf("TIMEOUT: %s (%v):\n%s\n", tc, time.Since(now), output)
+ t.Errorf("TIMEOUT: (%v):\n%s\n", time.Since(now), output)
}
},
})
@@ -143,14 +159,14 @@ func getTests(d *dockerutil.Docker, blacklist map[string]struct{}) ([]testing.In
return itests, nil
}
-// getBlacklist reads the blacklist file and returns a set of test names to
+// getBlacklist reads the exclude file and returns a set of test names to
// exclude.
-func getBlacklist() (map[string]struct{}, error) {
- blacklist := make(map[string]struct{})
- if *blacklistFile == "" {
- return blacklist, nil
+func getExcludes() (map[string]struct{}, error) {
+ excludes := make(map[string]struct{})
+ if *excludeFile == "" {
+ return excludes, nil
}
- f, err := os.Open(*blacklistFile)
+ f, err := os.Open(*excludeFile)
if err != nil {
return nil, err
}
@@ -171,9 +187,9 @@ func getBlacklist() (map[string]struct{}, error) {
if err != nil {
return nil, err
}
- blacklist[record[0]] = struct{}{}
+ excludes[record[0]] = struct{}{}
}
- return blacklist, nil
+ return excludes, nil
}
// testDeps implements testing.testDeps (an unexported interface), and is
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-25 09:17:53 +0000