summaryrefslogtreecommitdiffhomepage
path: root/test/runtimes
diff options
context:
space:
mode:
Diffstat (limited to 'test/runtimes')
-rw-r--r--test/runtimes/exclude/php7.3.6.csv3
-rw-r--r--test/runtimes/exclude/python3.7.3.csv1
-rw-r--r--test/runtimes/runner/lib/lib.go39
-rw-r--r--test/runtimes/runner/main.go14
4 files changed, 41 insertions, 16 deletions
diff --git a/test/runtimes/exclude/php7.3.6.csv b/test/runtimes/exclude/php7.3.6.csv
index f984a579a..c051fe571 100644
--- a/test/runtimes/exclude/php7.3.6.csv
+++ b/test/runtimes/exclude/php7.3.6.csv
@@ -8,6 +8,7 @@ ext/mbstring/tests/bug77165.phpt,,
ext/mbstring/tests/bug77454.phpt,,
ext/mbstring/tests/mb_convert_encoding_leak.phpt,,
ext/mbstring/tests/mb_strrpos_encoding_3rd_param.phpt,,
+ext/pcre/tests/cache_limit.phpt,,Broken test - Flaky
ext/session/tests/session_module_name_variation4.phpt,,Flaky
ext/session/tests/session_set_save_handler_class_018.phpt,,
ext/session/tests/session_set_save_handler_iface_003.phpt,,
@@ -29,7 +30,7 @@ ext/standard/tests/file/php_fd_wrapper_04.phpt,,
ext/standard/tests/file/realpath_bug77484.phpt,b/162894969,VFS1 only failure
ext/standard/tests/file/rename_variation.phpt,b/68717309,
ext/standard/tests/file/symlink_link_linkinfo_is_link_variation4.phpt,b/162895341,
-ext/standard/tests/file/symlink_link_linkinfo_is_link_variation8.phpt,b/162896223,
+ext/standard/tests/file/symlink_link_linkinfo_is_link_variation8.phpt,b/162896223,VFS1 only failure
ext/standard/tests/general_functions/escapeshellarg_bug71270.phpt,,
ext/standard/tests/general_functions/escapeshellcmd_bug71270.phpt,,
ext/standard/tests/streams/proc_open_bug60120.phpt,,Flaky until php-src 3852a35fdbcb
diff --git a/test/runtimes/exclude/python3.7.3.csv b/test/runtimes/exclude/python3.7.3.csv
index 911f22855..e9fef03b7 100644
--- a/test/runtimes/exclude/python3.7.3.csv
+++ b/test/runtimes/exclude/python3.7.3.csv
@@ -4,7 +4,6 @@ test_asyncore,b/162973328,
test_epoll,b/162983393,
test_fcntl,b/162978767,fcntl invalid argument -- artificial test to make sure something works in 64 bit mode.
test_httplib,b/163000009,OSError: [Errno 98] Address already in use
-test_imaplib,b/162979661,
test_logging,b/162980079,
test_multiprocessing_fork,,Flaky. Sometimes times out.
test_multiprocessing_forkserver,,Flaky. Sometimes times out.
diff --git a/test/runtimes/runner/lib/lib.go b/test/runtimes/runner/lib/lib.go
index 78285cb0e..64e6e14db 100644
--- a/test/runtimes/runner/lib/lib.go
+++ b/test/runtimes/runner/lib/lib.go
@@ -34,8 +34,16 @@ import (
// RunTests is a helper that is called by main. It exists so that we can run
// defered functions before exiting. It returns an exit code that should be
// passed to os.Exit.
-func RunTests(lang, image, excludeFile string, batchSize int, timeout time.Duration) int {
- // Get tests to exclude..
+func RunTests(lang, image, excludeFile string, partitionNum, totalPartitions, batchSize int, timeout time.Duration) int {
+ if partitionNum <= 0 || totalPartitions <= 0 || partitionNum > totalPartitions {
+ fmt.Fprintf(os.Stderr, "invalid partition %d of %d", partitionNum, totalPartitions)
+ return 1
+ }
+
+ // TODO(gvisor.dev/issue/1624): Remove those tests from all exclude lists
+ // that only fail with VFS1.
+
+ // Get tests to exclude.
excludes, err := getExcludes(excludeFile)
if err != nil {
fmt.Fprintf(os.Stderr, "Error getting exclude list: %s\n", err.Error())
@@ -55,7 +63,7 @@ func RunTests(lang, image, excludeFile string, batchSize int, timeout time.Durat
// Get a slice of tests to run. This will also start a single Docker
// container that will be used to run each test. The final test will
// stop the Docker container.
- tests, err := getTests(ctx, d, lang, image, batchSize, timeout, excludes)
+ tests, err := getTests(ctx, d, lang, image, partitionNum, totalPartitions, batchSize, timeout, excludes)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err.Error())
return 1
@@ -66,7 +74,7 @@ func RunTests(lang, image, excludeFile string, batchSize int, timeout time.Durat
}
// getTests executes all tests as table tests.
-func getTests(ctx context.Context, d *dockerutil.Container, lang, image string, batchSize int, timeout time.Duration, excludes map[string]struct{}) ([]testing.InternalTest, error) {
+func getTests(ctx context.Context, d *dockerutil.Container, lang, image string, partitionNum, totalPartitions, batchSize int, timeout time.Duration, excludes map[string]struct{}) ([]testing.InternalTest, error) {
// Start the container.
opts := dockerutil.RunOpts{
Image: fmt.Sprintf("runtimes/%s", image),
@@ -86,6 +94,14 @@ func getTests(ctx context.Context, d *dockerutil.Container, lang, image string,
// shard.
tests := strings.Fields(list)
sort.Strings(tests)
+
+ partitionSize := len(tests) / totalPartitions
+ if partitionNum == totalPartitions {
+ tests = tests[(partitionNum-1)*partitionSize:]
+ } else {
+ tests = tests[(partitionNum-1)*partitionSize : partitionNum*partitionSize]
+ }
+
indices, err := testutil.TestIndicesForShard(len(tests))
if err != nil {
return nil, fmt.Errorf("TestsForShard() failed: %v", err)
@@ -116,8 +132,15 @@ func getTests(ctx context.Context, d *dockerutil.Container, lang, image string,
err error
)
+ state, err := d.Status(ctx)
+ if err != nil {
+ t.Fatalf("Could not find container status: %v", err)
+ }
+ if !state.Running {
+ t.Fatalf("container is not running: state = %s", state.Status)
+ }
+
go func() {
- fmt.Printf("RUNNING the following in a batch\n%s\n", strings.Join(tcs, "\n"))
output, err = d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", lang, "--tests", strings.Join(tcs, ","))
close(done)
}()
@@ -125,12 +148,12 @@ func getTests(ctx context.Context, d *dockerutil.Container, lang, image string,
select {
case <-done:
if err == nil {
- fmt.Printf("PASS: (%v)\n\n", time.Since(now))
+ fmt.Printf("PASS: (%v) %d tests passed\n", time.Since(now), len(tcs))
return
}
- t.Errorf("FAIL: (%v):\n%s\n", time.Since(now), output)
+ t.Errorf("FAIL: (%v):\nBatch:\n%s\nOutput:\n%s\n", time.Since(now), strings.Join(tcs, "\n"), output)
case <-time.After(timeout):
- t.Errorf("TIMEOUT: (%v):\n%s\n", time.Since(now), output)
+ t.Errorf("TIMEOUT: (%v):\nBatch:\n%s\nOutput:\n%s\n", time.Since(now), strings.Join(tcs, "\n"), output)
}
},
})
diff --git a/test/runtimes/runner/main.go b/test/runtimes/runner/main.go
index ec79a22c2..5b3443e36 100644
--- a/test/runtimes/runner/main.go
+++ b/test/runtimes/runner/main.go
@@ -25,11 +25,13 @@ import (
)
var (
- lang = flag.String("lang", "", "language runtime to test")
- image = flag.String("image", "", "docker image with runtime tests")
- excludeFile = flag.String("exclude_file", "", "file containing list of tests to exclude, in CSV format with fields: test name, bug id, comment")
- batchSize = flag.Int("batch", 50, "number of test cases run in one command")
- timeout = flag.Duration("timeout", 90*time.Minute, "batch timeout")
+ lang = flag.String("lang", "", "language runtime to test")
+ image = flag.String("image", "", "docker image with runtime tests")
+ excludeFile = flag.String("exclude_file", "", "file containing list of tests to exclude, in CSV format with fields: test name, bug id, comment")
+ partition = flag.Int("partition", 1, "partition number, this is 1-indexed")
+ totalPartitions = flag.Int("total_partitions", 1, "total number of partitions")
+ batchSize = flag.Int("batch", 50, "number of test cases run in one command")
+ timeout = flag.Duration("timeout", 90*time.Minute, "batch timeout")
)
func main() {
@@ -38,5 +40,5 @@ func main() {
fmt.Fprintf(os.Stderr, "lang and image flags must not be empty\n")
os.Exit(1)
}
- os.Exit(lib.RunTests(*lang, *image, *excludeFile, *batchSize, *timeout))
+ os.Exit(lib.RunTests(*lang, *image, *excludeFile, *partition, *totalPartitions, *batchSize, *timeout))
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-02-18 11:43:00 +0000