diff options
| -rw-r--r-- | test/runtimes/proctor/BUILD | 25 | ||||
| -rw-r--r-- | test/runtimes/proctor/lib/BUILD | 24 | ||||
| -rw-r--r-- | test/runtimes/proctor/lib/go.go (renamed from test/runtimes/proctor/go.go) | 4 | ||||
| -rw-r--r-- | test/runtimes/proctor/lib/java.go (renamed from test/runtimes/proctor/java.go) | 2 | ||||
| -rw-r--r-- | test/runtimes/proctor/lib/lib.go (renamed from test/runtimes/proctor/proctor.go) | 78 | ||||
| -rw-r--r-- | test/runtimes/proctor/lib/lib_test.go (renamed from test/runtimes/proctor/proctor_test.go) | 6 | ||||
| -rw-r--r-- | test/runtimes/proctor/lib/nodejs.go (renamed from test/runtimes/proctor/nodejs.go) | 4 | ||||
| -rw-r--r-- | test/runtimes/proctor/lib/php.go (renamed from test/runtimes/proctor/php.go) | 4 | ||||
| -rw-r--r-- | test/runtimes/proctor/lib/python.go (renamed from test/runtimes/proctor/python.go) | 2 | ||||
| -rw-r--r-- | test/runtimes/proctor/main.go | 85 | ||||
| -rw-r--r-- | test/runtimes/runner/BUILD | 15 | ||||
| -rw-r--r-- | test/runtimes/runner/lib/BUILD | 22 | ||||
| -rw-r--r-- | test/runtimes/runner/lib/exclude_test.go (renamed from test/runtimes/runner/exclude_test.go) | 6 | ||||
| -rw-r--r-- | test/runtimes/runner/lib/lib.go | 185 | ||||
| -rw-r--r-- | test/runtimes/runner/main.go | 169 |
15 files changed, 347 insertions, 284 deletions
diff --git a/test/runtimes/proctor/BUILD b/test/runtimes/proctor/BUILD index d1935cbe8..f66371265 100644 --- a/test/runtimes/proctor/BUILD +++ b/test/runtimes/proctor/BUILD @@ -1,29 +1,10 @@ -load("//tools:defs.bzl", "go_binary", "go_test") +load("//tools:defs.bzl", "go_binary") package(licenses = ["notice"]) go_binary( name = "proctor", - srcs = [ - "go.go", - "java.go", - "nodejs.go", - "php.go", - "proctor.go", - "python.go", - ], - pure = True, + srcs = ["main.go"], visibility = ["//test/runtimes:__pkg__"], -) - -go_test( - name = "proctor_test", - size = "small", - srcs = ["proctor_test.go"], - library = ":proctor", - nogo = False, # FIXME(gvisor.dev/issue/3374): Not working with all build systems. - pure = True, - deps = [ - "//pkg/test/testutil", - ], + deps = ["//test/runtimes/proctor/lib"], ) diff --git a/test/runtimes/proctor/lib/BUILD b/test/runtimes/proctor/lib/BUILD new file mode 100644 index 000000000..0c8367dfe --- /dev/null +++ b/test/runtimes/proctor/lib/BUILD @@ -0,0 +1,24 @@ +load("//tools:defs.bzl", "go_library", "go_test") + +package(licenses = ["notice"]) + +go_library( + name = "lib", + srcs = [ + "go.go", + "java.go", + "lib.go", + "nodejs.go", + "php.go", + "python.go", + ], + visibility = ["//test/runtimes/proctor:__pkg__"], +) + +go_test( + name = "lib_test", + size = "small", + srcs = ["lib_test.go"], + library = ":lib", + deps = ["//pkg/test/testutil"], +) diff --git a/test/runtimes/proctor/go.go b/test/runtimes/proctor/lib/go.go index d0ae844e6..5c48fb60b 100644 --- a/test/runtimes/proctor/go.go +++ b/test/runtimes/proctor/lib/go.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package lib import ( "fmt" @@ -59,7 +59,7 @@ func (goRunner) ListTests() ([]string, error) { } // Go tests on disk. - diskSlice, err := search(goTestDir, goTestRegEx) + diskSlice, err := Search(goTestDir, goTestRegEx) if err != nil { return nil, err } diff --git a/test/runtimes/proctor/java.go b/test/runtimes/proctor/lib/java.go index d456fa681..3105011ff 100644 --- a/test/runtimes/proctor/java.go +++ b/test/runtimes/proctor/lib/java.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package lib import ( "fmt" diff --git a/test/runtimes/proctor/proctor.go b/test/runtimes/proctor/lib/lib.go index 9e0642424..f2ba82498 100644 --- a/test/runtimes/proctor/proctor.go +++ b/test/runtimes/proctor/lib/lib.go @@ -12,20 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Binary proctor runs the test for a particular runtime. It is meant to be -// included in Docker images for all runtime tests. -package main +// Package lib contains proctor functions. +package lib import ( - "flag" "fmt" - "log" "os" "os/exec" "os/signal" "path/filepath" "regexp" - "strings" "syscall" ) @@ -42,66 +38,8 @@ type TestRunner interface { TestCmds(tests []string) []*exec.Cmd } -var ( - runtime = flag.String("runtime", "", "name of runtime") - list = flag.Bool("list", false, "list all available tests") - testNames = flag.String("tests", "", "run a subset of the available tests") - pause = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children") -) - -func main() { - flag.Parse() - - if *pause { - pauseAndReap() - panic("pauseAndReap should never return") - } - - if *runtime == "" { - log.Fatalf("runtime flag must be provided") - } - - tr, err := testRunnerForRuntime(*runtime) - if err != nil { - log.Fatalf("%v", err) - } - - // List tests. - if *list { - tests, err := tr.ListTests() - if err != nil { - log.Fatalf("failed to list tests: %v", err) - } - for _, test := range tests { - fmt.Println(test) - } - return - } - - var tests []string - if *testNames == "" { - // Run every test. - tests, err = tr.ListTests() - if err != nil { - log.Fatalf("failed to get all tests: %v", err) - } - } else { - // Run subset of test. - tests = strings.Split(*testNames, ",") - } - - // Run tests. - cmds := tr.TestCmds(tests) - for _, cmd := range cmds { - cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr - if err := cmd.Run(); err != nil { - log.Fatalf("FAIL: %v", err) - } - } -} - -// testRunnerForRuntime returns a new TestRunner for the given runtime. -func testRunnerForRuntime(runtime string) (TestRunner, error) { +// TestRunnerForRuntime returns a new TestRunner for the given runtime. +func TestRunnerForRuntime(runtime string) (TestRunner, error) { switch runtime { case "go": return goRunner{}, nil @@ -117,8 +55,8 @@ func testRunnerForRuntime(runtime string) (TestRunner, error) { return nil, fmt.Errorf("invalid runtime %q", runtime) } -// pauseAndReap is like init. It runs forever and reaps any children. -func pauseAndReap() { +// PauseAndReap is like init. It runs forever and reaps any children. +func PauseAndReap() { // Get notified of any new children. ch := make(chan os.Signal, 1) signal.Notify(ch, syscall.SIGCHLD) @@ -138,9 +76,9 @@ func pauseAndReap() { } } -// search is a helper function to find tests in the given directory that match +// Search is a helper function to find tests in the given directory that match // the regex. -func search(root string, testFilter *regexp.Regexp) ([]string, error) { +func Search(root string, testFilter *regexp.Regexp) ([]string, error) { var testSlice []string err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { diff --git a/test/runtimes/proctor/proctor_test.go b/test/runtimes/proctor/lib/lib_test.go index 6ef2de085..1193d2e28 100644 --- a/test/runtimes/proctor/proctor_test.go +++ b/test/runtimes/proctor/lib/lib_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package lib import ( "io/ioutil" @@ -47,7 +47,7 @@ func TestSearchEmptyDir(t *testing.T) { var want []string testFilter := regexp.MustCompile(`^test-[^-].+\.tc$`) - got, err := search(td, testFilter) + got, err := Search(td, testFilter) if err != nil { t.Errorf("search error: %v", err) } @@ -116,7 +116,7 @@ func TestSearch(t *testing.T) { } testFilter := regexp.MustCompile(`^test-[^-].+\.tc$`) - got, err := search(td, testFilter) + got, err := Search(td, testFilter) if err != nil { t.Errorf("search error: %v", err) } diff --git a/test/runtimes/proctor/nodejs.go b/test/runtimes/proctor/lib/nodejs.go index dead5af4f..320597aa5 100644 --- a/test/runtimes/proctor/nodejs.go +++ b/test/runtimes/proctor/lib/nodejs.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package lib import ( "os/exec" @@ -32,7 +32,7 @@ var _ TestRunner = nodejsRunner{} // ListTests implements TestRunner.ListTests. func (nodejsRunner) ListTests() ([]string, error) { - testSlice, err := search(nodejsTestDir, nodejsTestRegEx) + testSlice, err := Search(nodejsTestDir, nodejsTestRegEx) if err != nil { return nil, err } diff --git a/test/runtimes/proctor/php.go b/test/runtimes/proctor/lib/php.go index 6a83d64e3..b67a60a97 100644 --- a/test/runtimes/proctor/php.go +++ b/test/runtimes/proctor/lib/php.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package lib import ( "os/exec" @@ -29,7 +29,7 @@ var _ TestRunner = phpRunner{} // ListTests implements TestRunner.ListTests. func (phpRunner) ListTests() ([]string, error) { - testSlice, err := search(".", phpTestRegEx) + testSlice, err := Search(".", phpTestRegEx) if err != nil { return nil, err } diff --git a/test/runtimes/proctor/python.go b/test/runtimes/proctor/lib/python.go index 7c598801b..429bfd850 100644 --- a/test/runtimes/proctor/python.go +++ b/test/runtimes/proctor/lib/python.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package lib import ( "fmt" diff --git a/test/runtimes/proctor/main.go b/test/runtimes/proctor/main.go new file mode 100644 index 000000000..e5607ac92 --- /dev/null +++ b/test/runtimes/proctor/main.go @@ -0,0 +1,85 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Binary proctor runs the test for a particular runtime. It is meant to be +// included in Docker images for all runtime tests. +package main + +import ( + "flag" + "fmt" + "log" + "os" + "strings" + + "gvisor.dev/gvisor/test/runtimes/proctor/lib" +) + +var ( + runtime = flag.String("runtime", "", "name of runtime") + list = flag.Bool("list", false, "list all available tests") + testNames = flag.String("tests", "", "run a subset of the available tests") + pause = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children") +) + +func main() { + flag.Parse() + + if *pause { + lib.PauseAndReap() + panic("pauseAndReap should never return") + } + + if *runtime == "" { + log.Fatalf("runtime flag must be provided") + } + + tr, err := lib.TestRunnerForRuntime(*runtime) + if err != nil { + log.Fatalf("%v", err) + } + + // List tests. + if *list { + tests, err := tr.ListTests() + if err != nil { + log.Fatalf("failed to list tests: %v", err) + } + for _, test := range tests { + fmt.Println(test) + } + return + } + + var tests []string + if *testNames == "" { + // Run every test. + tests, err = tr.ListTests() + if err != nil { + log.Fatalf("failed to get all tests: %v", err) + } + } else { + // Run subset of test. + tests = strings.Split(*testNames, ",") + } + + // Run tests. + cmds := tr.TestCmds(tests) + for _, cmd := range cmds { + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + if err := cmd.Run(); err != nil { + log.Fatalf("FAIL: %v", err) + } + } +} diff --git a/test/runtimes/runner/BUILD b/test/runtimes/runner/BUILD index dc0d5d5b4..70cc01594 100644 --- a/test/runtimes/runner/BUILD +++ b/test/runtimes/runner/BUILD @@ -1,4 +1,4 @@ -load("//tools:defs.bzl", "go_binary", "go_test") +load("//tools:defs.bzl", "go_binary") package(licenses = ["notice"]) @@ -7,16 +7,5 @@ go_binary( testonly = 1, srcs = ["main.go"], visibility = ["//test/runtimes:__pkg__"], - deps = [ - "//pkg/log", - "//pkg/test/dockerutil", - "//pkg/test/testutil", - ], -) - -go_test( - name = "exclude_test", - size = "small", - srcs = ["exclude_test.go"], - library = ":runner", + deps = ["//test/runtimes/runner/lib"], ) diff --git a/test/runtimes/runner/lib/BUILD b/test/runtimes/runner/lib/BUILD new file mode 100644 index 000000000..d308f41b0 --- /dev/null +++ b/test/runtimes/runner/lib/BUILD @@ -0,0 +1,22 @@ +load("//tools:defs.bzl", "go_library", "go_test") + +package(licenses = ["notice"]) + +go_library( + name = "lib", + testonly = 1, + srcs = ["lib.go"], + visibility = ["//test/runtimes/runner:__pkg__"], + deps = [ + "//pkg/log", + "//pkg/test/dockerutil", + "//pkg/test/testutil", + ], +) + +go_test( + name = "lib_test", + size = "small", + srcs = ["exclude_test.go"], + library = ":lib", +) diff --git a/test/runtimes/runner/exclude_test.go b/test/runtimes/runner/lib/exclude_test.go index 67c2170c8..f996e895b 100644 --- a/test/runtimes/runner/exclude_test.go +++ b/test/runtimes/runner/lib/exclude_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package lib import ( "flag" @@ -20,6 +20,8 @@ import ( "testing" ) +var excludeFile = flag.String("exclude_file", "", "file to test (standard format)") + func TestMain(m *testing.M) { flag.Parse() os.Exit(m.Run()) @@ -27,7 +29,7 @@ func TestMain(m *testing.M) { // Test that the exclude file parses without error. func TestExcludelist(t *testing.T) { - ex, err := getExcludes() + ex, err := getExcludes(*excludeFile) if err != nil { t.Fatalf("error parsing exclude file: %v", err) } diff --git a/test/runtimes/runner/lib/lib.go b/test/runtimes/runner/lib/lib.go new file mode 100644 index 000000000..78285cb0e --- /dev/null +++ b/test/runtimes/runner/lib/lib.go @@ -0,0 +1,185 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package lib provides utilities for runner. +package lib + +import ( + "context" + "encoding/csv" + "fmt" + "io" + "os" + "sort" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/test/dockerutil" + "gvisor.dev/gvisor/pkg/test/testutil" +) + +// RunTests is a helper that is called by main. It exists so that we can run +// defered functions before exiting. It returns an exit code that should be +// passed to os.Exit. +func RunTests(lang, image, excludeFile string, batchSize int, timeout time.Duration) int { + // Get tests to exclude.. + excludes, err := getExcludes(excludeFile) + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting exclude list: %s\n", err.Error()) + return 1 + } + + // Construct the shared docker instance. + ctx := context.Background() + d := dockerutil.MakeContainer(ctx, testutil.DefaultLogger(lang)) + defer d.CleanUp(ctx) + + if err := testutil.TouchShardStatusFile(); err != nil { + fmt.Fprintf(os.Stderr, "error touching status shard file: %v\n", err) + return 1 + } + + // Get a slice of tests to run. This will also start a single Docker + // container that will be used to run each test. The final test will + // stop the Docker container. + tests, err := getTests(ctx, d, lang, image, batchSize, timeout, excludes) + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err.Error()) + return 1 + } + + m := testing.MainStart(testDeps{}, tests, nil, nil) + return m.Run() +} + +// getTests executes all tests as table tests. +func getTests(ctx context.Context, d *dockerutil.Container, lang, image string, batchSize int, timeout time.Duration, excludes map[string]struct{}) ([]testing.InternalTest, error) { + // Start the container. + opts := dockerutil.RunOpts{ + Image: fmt.Sprintf("runtimes/%s", image), + } + d.CopyFiles(&opts, "/proctor", "test/runtimes/proctor/proctor") + if err := d.Spawn(ctx, opts, "/proctor/proctor", "--pause"); err != nil { + return nil, fmt.Errorf("docker run failed: %v", err) + } + + // Get a list of all tests in the image. + list, err := d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", lang, "--list") + if err != nil { + return nil, fmt.Errorf("docker exec failed: %v", err) + } + + // Calculate a subset of tests to run corresponding to the current + // shard. + tests := strings.Fields(list) + sort.Strings(tests) + indices, err := testutil.TestIndicesForShard(len(tests)) + if err != nil { + return nil, fmt.Errorf("TestsForShard() failed: %v", err) + } + + var itests []testing.InternalTest + for i := 0; i < len(indices); i += batchSize { + var tcs []string + end := i + batchSize + if end > len(indices) { + end = len(indices) + } + for _, tc := range indices[i:end] { + // Add test if not excluded. + if _, ok := excludes[tests[tc]]; ok { + log.Infof("Skipping test case %s\n", tests[tc]) + continue + } + tcs = append(tcs, tests[tc]) + } + itests = append(itests, testing.InternalTest{ + Name: strings.Join(tcs, ", "), + F: func(t *testing.T) { + var ( + now = time.Now() + done = make(chan struct{}) + output string + err error + ) + + go func() { + fmt.Printf("RUNNING the following in a batch\n%s\n", strings.Join(tcs, "\n")) + output, err = d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", lang, "--tests", strings.Join(tcs, ",")) + close(done) + }() + + select { + case <-done: + if err == nil { + fmt.Printf("PASS: (%v)\n\n", time.Since(now)) + return + } + t.Errorf("FAIL: (%v):\n%s\n", time.Since(now), output) + case <-time.After(timeout): + t.Errorf("TIMEOUT: (%v):\n%s\n", time.Since(now), output) + } + }, + }) + } + + return itests, nil +} + +// getBlacklist reads the exclude file and returns a set of test names to +// exclude. +func getExcludes(excludeFile string) (map[string]struct{}, error) { + excludes := make(map[string]struct{}) + if excludeFile == "" { + return excludes, nil + } + f, err := os.Open(excludeFile) + if err != nil { + return nil, err + } + defer f.Close() + + r := csv.NewReader(f) + + // First line is header. Skip it. + if _, err := r.Read(); err != nil { + return nil, err + } + + for { + record, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + excludes[record[0]] = struct{}{} + } + return excludes, nil +} + +// testDeps implements testing.testDeps (an unexported interface), and is +// required to use testing.MainStart. +type testDeps struct{} + +func (f testDeps) MatchString(a, b string) (bool, error) { return a == b, nil } +func (f testDeps) StartCPUProfile(io.Writer) error { return nil } +func (f testDeps) StopCPUProfile() {} +func (f testDeps) WriteProfileTo(string, io.Writer, int) error { return nil } +func (f testDeps) ImportPath() string { return "" } +func (f testDeps) StartTestLog(io.Writer) {} +func (f testDeps) StopTestLog() error { return nil } diff --git a/test/runtimes/runner/main.go b/test/runtimes/runner/main.go index 948e7cf9c..ec79a22c2 100644 --- a/test/runtimes/runner/main.go +++ b/test/runtimes/runner/main.go @@ -16,20 +16,12 @@ package main import ( - "context" - "encoding/csv" "flag" "fmt" - "io" "os" - "sort" - "strings" - "testing" "time" - "gvisor.dev/gvisor/pkg/log" - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/pkg/test/testutil" + "gvisor.dev/gvisor/test/runtimes/runner/lib" ) var ( @@ -37,169 +29,14 @@ var ( image = flag.String("image", "", "docker image with runtime tests") excludeFile = flag.String("exclude_file", "", "file containing list of tests to exclude, in CSV format with fields: test name, bug id, comment") batchSize = flag.Int("batch", 50, "number of test cases run in one command") + timeout = flag.Duration("timeout", 90*time.Minute, "batch timeout") ) -// Wait time for each test to run. -const timeout = 90 * time.Minute - func main() { flag.Parse() if *lang == "" || *image == "" { fmt.Fprintf(os.Stderr, "lang and image flags must not be empty\n") os.Exit(1) } - os.Exit(runTests()) -} - -// runTests is a helper that is called by main. It exists so that we can run -// defered functions before exiting. It returns an exit code that should be -// passed to os.Exit. -func runTests() int { - // Get tests to exclude.. - excludes, err := getExcludes() - if err != nil { - fmt.Fprintf(os.Stderr, "Error getting exclude list: %s\n", err.Error()) - return 1 - } - - // Construct the shared docker instance. - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, testutil.DefaultLogger(*lang)) - defer d.CleanUp(ctx) - - if err := testutil.TouchShardStatusFile(); err != nil { - fmt.Fprintf(os.Stderr, "error touching status shard file: %v\n", err) - return 1 - } - - // Get a slice of tests to run. This will also start a single Docker - // container that will be used to run each test. The final test will - // stop the Docker container. - tests, err := getTests(ctx, d, excludes) - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err.Error()) - return 1 - } - - m := testing.MainStart(testDeps{}, tests, nil, nil) - return m.Run() -} - -// getTests executes all tests as table tests. -func getTests(ctx context.Context, d *dockerutil.Container, excludes map[string]struct{}) ([]testing.InternalTest, error) { - // Start the container. - opts := dockerutil.RunOpts{ - Image: fmt.Sprintf("runtimes/%s", *image), - } - d.CopyFiles(&opts, "/proctor", "test/runtimes/proctor/proctor") - if err := d.Spawn(ctx, opts, "/proctor/proctor", "--pause"); err != nil { - return nil, fmt.Errorf("docker run failed: %v", err) - } - - // Get a list of all tests in the image. - list, err := d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", *lang, "--list") - if err != nil { - return nil, fmt.Errorf("docker exec failed: %v", err) - } - - // Calculate a subset of tests to run corresponding to the current - // shard. - tests := strings.Fields(list) - sort.Strings(tests) - indices, err := testutil.TestIndicesForShard(len(tests)) - if err != nil { - return nil, fmt.Errorf("TestsForShard() failed: %v", err) - } - - var itests []testing.InternalTest - for i := 0; i < len(indices); i += *batchSize { - var tcs []string - end := i + *batchSize - if end > len(indices) { - end = len(indices) - } - for _, tc := range indices[i:end] { - // Add test if not excluded. - if _, ok := excludes[tests[tc]]; ok { - log.Infof("Skipping test case %s\n", tests[tc]) - continue - } - tcs = append(tcs, tests[tc]) - } - itests = append(itests, testing.InternalTest{ - Name: strings.Join(tcs, ", "), - F: func(t *testing.T) { - var ( - now = time.Now() - done = make(chan struct{}) - output string - err error - ) - - go func() { - fmt.Printf("RUNNING the following in a batch\n%s\n", strings.Join(tcs, "\n")) - output, err = d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", *lang, "--tests", strings.Join(tcs, ",")) - close(done) - }() - - select { - case <-done: - if err == nil { - fmt.Printf("PASS: (%v)\n\n", time.Since(now)) - return - } - t.Errorf("FAIL: (%v):\n%s\n", time.Since(now), output) - case <-time.After(timeout): - t.Errorf("TIMEOUT: (%v):\n%s\n", time.Since(now), output) - } - }, - }) - } - - return itests, nil + os.Exit(lib.RunTests(*lang, *image, *excludeFile, *batchSize, *timeout)) } - -// getBlacklist reads the exclude file and returns a set of test names to -// exclude. -func getExcludes() (map[string]struct{}, error) { - excludes := make(map[string]struct{}) - if *excludeFile == "" { - return excludes, nil - } - f, err := os.Open(*excludeFile) - if err != nil { - return nil, err - } - defer f.Close() - - r := csv.NewReader(f) - - // First line is header. Skip it. - if _, err := r.Read(); err != nil { - return nil, err - } - - for { - record, err := r.Read() - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - excludes[record[0]] = struct{}{} - } - return excludes, nil -} - -// testDeps implements testing.testDeps (an unexported interface), and is -// required to use testing.MainStart. -type testDeps struct{} - -func (f testDeps) MatchString(a, b string) (bool, error) { return a == b, nil } -func (f testDeps) StartCPUProfile(io.Writer) error { return nil } -func (f testDeps) StopCPUProfile() {} -func (f testDeps) WriteProfileTo(string, io.Writer, int) error { return nil } -func (f testDeps) ImportPath() string { return "" } -func (f testDeps) StartTestLog(io.Writer) {} -func (f testDeps) StopTestLog() error { return nil } |