diff options
Diffstat (limited to 'test/runtimes')
| -rw-r--r-- | test/runtimes/BUILD | 24 | ||||
| -rw-r--r-- | test/runtimes/blacklist_go1.12.csv | 16 | ||||
| -rw-r--r-- | test/runtimes/blacklist_java11.csv | 126 | ||||
| -rw-r--r-- | test/runtimes/blacklist_nodejs12.4.0.csv | 47 | ||||
| -rw-r--r-- | test/runtimes/blacklist_php7.3.6.csv | 29 | ||||
| -rw-r--r-- | test/runtimes/blacklist_python3.7.3.csv | 27 | ||||
| -rw-r--r-- | test/runtimes/blacklist_test.go | 37 | ||||
| -rw-r--r-- | test/runtimes/build_defs.bzl | 69 | ||||
| -rw-r--r-- | test/runtimes/runner.go | 60 |
9 files changed, 410 insertions, 25 deletions
diff --git a/test/runtimes/BUILD b/test/runtimes/BUILD index dfb4e2a97..367295206 100644 --- a/test/runtimes/BUILD +++ b/test/runtimes/BUILD @@ -1,6 +1,6 @@ # These packages are used to run language runtime tests inside gVisor sandboxes. -load("@io_bazel_rules_go//go:def.bzl", "go_binary") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_test") load("//test/runtimes:build_defs.bzl", "runtime_test") package(licenses = ["notice"]) @@ -16,26 +16,38 @@ go_binary( ) runtime_test( - image = "gcr.io/gvisor-presubmit/go1.12", + name = "go1.12", + blacklist_file = "blacklist_go1.12.csv", lang = "go", ) runtime_test( - image = "gcr.io/gvisor-presubmit/java11", + name = "java11", + blacklist_file = "blacklist_java11.csv", lang = "java", ) runtime_test( - image = "gcr.io/gvisor-presubmit/nodejs12.4.0", + name = "nodejs12.4.0", + blacklist_file = "blacklist_nodejs12.4.0.csv", lang = "nodejs", ) runtime_test( - image = "gcr.io/gvisor-presubmit/php7.3.6", + name = "php7.3.6", + blacklist_file = "blacklist_php7.3.6.csv", lang = "php", ) runtime_test( - image = "gcr.io/gvisor-presubmit/python3.7.3", + name = "python3.7.3", + blacklist_file = "blacklist_python3.7.3.csv", lang = "python", ) + +go_test( + name = "blacklist_test", + size = "small", + srcs = ["blacklist_test.go"], + embed = [":runner"], +) diff --git a/test/runtimes/blacklist_go1.12.csv b/test/runtimes/blacklist_go1.12.csv new file mode 100644 index 000000000..8c8ae0c5d --- /dev/null +++ b/test/runtimes/blacklist_go1.12.csv @@ -0,0 +1,16 @@ +test name,bug id,comment +cgo_errors,,FLAKY +cgo_test,,FLAKY +go_test:cmd/go,,FLAKY +go_test:cmd/vendor/golang.org/x/sys/unix,b/118783622,/dev devices missing +go_test:net,b/118784196,socket: invalid argument. Works as intended: see bug. +go_test:os,b/118780122,we have a pollable filesystem but that's a surprise +go_test:os/signal,b/118780860,/dev/pts not properly supported +go_test:runtime,b/118782341,sigtrap not reported or caught or something +go_test:syscall,b/118781998,bad bytes -- bad mem addr +race,b/118782931,thread sanitizer. Works as intended: b/62219744. +runtime:cpu124,b/118778254,segmentation fault +test:0_1,,FLAKY +testasan,, +testcarchive,b/118782924,no sigpipe +testshared,,FLAKY diff --git a/test/runtimes/blacklist_java11.csv b/test/runtimes/blacklist_java11.csv new file mode 100644 index 000000000..c012e5a56 --- /dev/null +++ b/test/runtimes/blacklist_java11.csv @@ -0,0 +1,126 @@ +test name,bug id,comment +com/sun/crypto/provider/Cipher/PBE/PKCS12Cipher.java,,Fails in Docker +com/sun/jdi/NashornPopFrameTest.java,, +com/sun/jdi/ProcessAttachTest.java,, +com/sun/management/HotSpotDiagnosticMXBean/CheckOrigin.java,,Fails in Docker +com/sun/management/OperatingSystemMXBean/GetCommittedVirtualMemorySize.java,, +com/sun/management/UnixOperatingSystemMXBean/GetMaxFileDescriptorCount.sh,, +com/sun/tools/attach/AttachSelf.java,, +com/sun/tools/attach/BasicTests.java,, +com/sun/tools/attach/PermissionTest.java,, +com/sun/tools/attach/StartManagementAgent.java,, +com/sun/tools/attach/TempDirTest.java,, +com/sun/tools/attach/modules/Driver.java,, +java/lang/Character/CheckScript.java,,Fails in Docker +java/lang/Character/CheckUnicode.java,,Fails in Docker +java/lang/Class/GetPackageBootLoaderChildLayer.java,, +java/lang/ClassLoader/nativeLibrary/NativeLibraryTest.java,,Fails in Docker +java/lang/String/nativeEncoding/StringPlatformChars.java,, +java/net/DatagramSocket/ReuseAddressTest.java,, +java/net/DatagramSocket/SendDatagramToBadAddress.java,b/78473345, +java/net/Inet4Address/PingThis.java,, +java/net/InterfaceAddress/NetworkPrefixLength.java,b/78507103, +java/net/MulticastSocket/MulticastTTL.java,, +java/net/MulticastSocket/Promiscuous.java,, +java/net/MulticastSocket/SetLoopbackMode.java,, +java/net/MulticastSocket/SetTTLAndGetTTL.java,, +java/net/MulticastSocket/Test.java,, +java/net/MulticastSocket/TestDefaults.java,, +java/net/MulticastSocket/TimeToLive.java,, +java/net/NetworkInterface/NetworkInterfaceStreamTest.java,, +java/net/Socket/SetSoLinger.java,b/78527327,SO_LINGER is not yet supported +java/net/Socket/TrafficClass.java,b/78527818,Not supported on gVisor +java/net/Socket/UrgentDataTest.java,b/111515323, +java/net/Socket/setReuseAddress/Basic.java,b/78519214,SO_REUSEADDR enabled by default +java/net/SocketOption/OptionsTest.java,,Fails in Docker +java/net/SocketOption/TcpKeepAliveTest.java,, +java/net/SocketPermission/SocketPermissionTest.java,, +java/net/URLConnection/6212146/TestDriver.java,,Fails in Docker +java/net/httpclient/RequestBuilderTest.java,,Fails in Docker +java/net/httpclient/ShortResponseBody.java,, +java/net/httpclient/ShortResponseBodyWithRetry.java,, +java/nio/channels/AsyncCloseAndInterrupt.java,, +java/nio/channels/AsynchronousServerSocketChannel/Basic.java,, +java/nio/channels/AsynchronousSocketChannel/Basic.java,b/77921528,SO_KEEPALIVE is not settable +java/nio/channels/DatagramChannel/BasicMulticastTests.java,, +java/nio/channels/DatagramChannel/SocketOptionTests.java,,Fails in Docker +java/nio/channels/DatagramChannel/UseDGWithIPv6.java,, +java/nio/channels/FileChannel/directio/DirectIOTest.java,,Fails in Docker +java/nio/channels/Selector/OutOfBand.java,, +java/nio/channels/Selector/SelectWithConsumer.java,,Flaky +java/nio/channels/ServerSocketChannel/SocketOptionTests.java,, +java/nio/channels/SocketChannel/LingerOnClose.java,, +java/nio/channels/SocketChannel/SocketOptionTests.java,b/77965901, +java/nio/channels/spi/SelectorProvider/inheritedChannel/InheritedChannelTest.java,,Fails in Docker +java/rmi/activation/Activatable/extLoadedImpl/ext.sh,, +java/rmi/transport/checkLeaseInfoLeak/CheckLeaseLeak.java,, +java/text/Format/NumberFormat/CurrencyFormat.java,,Fails in Docker +java/text/Format/NumberFormat/CurrencyFormat.java,,Fails in Docker +java/util/Calendar/JapaneseEraNameTest.java,, +java/util/Currency/CurrencyTest.java,,Fails in Docker +java/util/Currency/ValidateISO4217.java,,Fails in Docker +java/util/Locale/LSRDataTest.java,, +java/util/concurrent/locks/Lock/TimedAcquireLeak.java,, +java/util/jar/JarFile/mrjar/MultiReleaseJarAPI.java,,Fails in Docker +java/util/logging/LogManager/Configuration/updateConfiguration/SimpleUpdateConfigWithInputStreamTest.java,, +java/util/logging/TestLoggerWeakRefLeak.java,, +javax/imageio/AppletResourceTest.java,, +javax/management/security/HashedPasswordFileTest.java,, +javax/net/ssl/SSLSession/JSSERenegotiate.java,,Fails in Docker +javax/sound/sampled/AudioInputStream/FrameLengthAfterConversion.java,, +jdk/jfr/event/runtime/TestNetworkUtilizationEvent.java,, +jdk/jfr/event/runtime/TestThreadParkEvent.java,, +jdk/jfr/event/sampling/TestNative.java,, +jdk/jfr/jcmd/TestJcmdChangeLogLevel.java,, +jdk/jfr/jcmd/TestJcmdConfigure.java,, +jdk/jfr/jcmd/TestJcmdDump.java,, +jdk/jfr/jcmd/TestJcmdDumpGeneratedFilename.java,, +jdk/jfr/jcmd/TestJcmdDumpLimited.java,, +jdk/jfr/jcmd/TestJcmdDumpPathToGCRoots.java,, +jdk/jfr/jcmd/TestJcmdLegacy.java,, +jdk/jfr/jcmd/TestJcmdSaveToFile.java,, +jdk/jfr/jcmd/TestJcmdStartDirNotExist.java,, +jdk/jfr/jcmd/TestJcmdStartInvaldFile.java,, +jdk/jfr/jcmd/TestJcmdStartPathToGCRoots.java,, +jdk/jfr/jcmd/TestJcmdStartStopDefault.java,, +jdk/jfr/jcmd/TestJcmdStartWithOptions.java,, +jdk/jfr/jcmd/TestJcmdStartWithSettings.java,, +jdk/jfr/jcmd/TestJcmdStopInvalidFile.java,, +jdk/jfr/jvm/TestJfrJavaBase.java,, +jdk/jfr/startupargs/TestStartRecording.java,, +jdk/modules/incubator/ImageModules.java,, +jdk/net/Sockets/ExtOptionTest.java,, +jdk/net/Sockets/QuickAckTest.java,, +lib/security/cacerts/VerifyCACerts.java,, +sun/management/jmxremote/bootstrap/CustomLauncherTest.java,, +sun/management/jmxremote/bootstrap/JvmstatCountersTest.java,, +sun/management/jmxremote/bootstrap/LocalManagementTest.java,, +sun/management/jmxremote/bootstrap/RmiRegistrySslTest.java,, +sun/management/jmxremote/bootstrap/RmiSslBootstrapTest.sh,, +sun/management/jmxremote/startstop/JMXStartStopTest.java,, +sun/management/jmxremote/startstop/JMXStatusPerfCountersTest.java,, +sun/management/jmxremote/startstop/JMXStatusTest.java,, +sun/text/resources/LocaleDataTest.java,, +sun/tools/jcmd/TestJcmdSanity.java,, +sun/tools/jhsdb/AlternateHashingTest.java,, +sun/tools/jhsdb/BasicLauncherTest.java,, +sun/tools/jhsdb/HeapDumpTest.java,, +sun/tools/jhsdb/heapconfig/JMapHeapConfigTest.java,, +sun/tools/jinfo/BasicJInfoTest.java,, +sun/tools/jinfo/JInfoTest.java,, +sun/tools/jmap/BasicJMapTest.java,, +sun/tools/jstack/BasicJStackTest.java,, +sun/tools/jstack/DeadlockDetectionTest.java,, +sun/tools/jstatd/TestJstatdExternalRegistry.java,, +sun/tools/jstatd/TestJstatdPort.java,,Flaky +sun/tools/jstatd/TestJstatdPortAndServer.java,,Flaky +sun/util/calendar/zi/TestZoneInfo310.java,, +tools/jar/modularJar/Basic.java,, +tools/jar/multiRelease/Basic.java,, +tools/jimage/JImageExtractTest.java,, +tools/jimage/JImageTest.java,, +tools/jlink/JLinkTest.java,, +tools/jlink/plugins/IncludeLocalesPluginTest.java,, +tools/jmod/hashes/HashesTest.java,, +tools/launcher/BigJar.java,b/111611473, +tools/launcher/modules/patch/systemmodules/PatchSystemModules.java,, diff --git a/test/runtimes/blacklist_nodejs12.4.0.csv b/test/runtimes/blacklist_nodejs12.4.0.csv new file mode 100644 index 000000000..4ab4e2927 --- /dev/null +++ b/test/runtimes/blacklist_nodejs12.4.0.csv @@ -0,0 +1,47 @@ +test name,bug id,comment +benchmark/test-benchmark-fs.js,, +benchmark/test-benchmark-module.js,, +benchmark/test-benchmark-napi.js,, +doctool/test-make-doc.js,b/68848110,Expected to fail. +fixtures/test-error-first-line-offset.js,, +fixtures/test-fs-readfile-error.js,, +fixtures/test-fs-stat-sync-overflow.js,, +internet/test-dgram-broadcast-multi-process.js,, +internet/test-dgram-multicast-multi-process.js,, +internet/test-dgram-multicast-set-interface-lo.js,, +parallel/test-cluster-dgram-reuse.js,b/64024294, +parallel/test-dgram-bind-fd.js,b/132447356, +parallel/test-dgram-create-socket-handle-fd.js,b/132447238, +parallel/test-dgram-createSocket-type.js,b/68847739, +parallel/test-dgram-socket-buffer-size.js,b/68847921, +parallel/test-fs-access.js,, +parallel/test-fs-write-stream-double-close.js,, +parallel/test-fs-write-stream-throw-type-error.js,b/110226209, +parallel/test-fs-write-stream.js,, +parallel/test-http2-respond-file-error-pipe-offset.js,, +parallel/test-os.js,, +parallel/test-process-uid-gid.js,, +pseudo-tty/test-assert-colors.js,, +pseudo-tty/test-assert-no-color.js,, +pseudo-tty/test-assert-position-indicator.js,, +pseudo-tty/test-async-wrap-getasyncid-tty.js,, +pseudo-tty/test-fatal-error.js,, +pseudo-tty/test-handle-wrap-isrefed-tty.js,, +pseudo-tty/test-readable-tty-keepalive.js,, +pseudo-tty/test-set-raw-mode-reset-process-exit.js,, +pseudo-tty/test-set-raw-mode-reset-signal.js,, +pseudo-tty/test-set-raw-mode-reset.js,, +pseudo-tty/test-stderr-stdout-handle-sigwinch.js,, +pseudo-tty/test-stdout-read.js,, +pseudo-tty/test-tty-color-support.js,, +pseudo-tty/test-tty-isatty.js,, +pseudo-tty/test-tty-stdin-call-end.js,, +pseudo-tty/test-tty-stdin-end.js,, +pseudo-tty/test-stdin-write.js,, +pseudo-tty/test-tty-stdout-end.js,, +pseudo-tty/test-tty-stdout-resize.js,, +pseudo-tty/test-tty-stream-constructors.js,, +pseudo-tty/test-tty-window-size.js,, +pseudo-tty/test-tty-wrap.js,, +pummel/test-net-pingpong.js,, +pummel/test-vm-memleak.js,, diff --git a/test/runtimes/blacklist_php7.3.6.csv b/test/runtimes/blacklist_php7.3.6.csv new file mode 100644 index 000000000..456bf7487 --- /dev/null +++ b/test/runtimes/blacklist_php7.3.6.csv @@ -0,0 +1,29 @@ +test name,bug id,comment +ext/intl/tests/bug77895.phpt,, +ext/intl/tests/dateformat_bug65683_2.phpt,, +ext/mbstring/tests/bug76319.phpt,, +ext/mbstring/tests/bug76958.phpt,, +ext/mbstring/tests/bug77025.phpt,, +ext/mbstring/tests/bug77165.phpt,, +ext/mbstring/tests/bug77454.phpt,, +ext/mbstring/tests/mb_convert_encoding_leak.phpt,, +ext/mbstring/tests/mb_strrpos_encoding_3rd_param.phpt,, +ext/standard/tests/file/filetype_variation.phpt,, +ext/standard/tests/file/fopen_variation19.phpt,, +ext/standard/tests/file/php_fd_wrapper_01.phpt,, +ext/standard/tests/file/php_fd_wrapper_02.phpt,, +ext/standard/tests/file/php_fd_wrapper_03.phpt,, +ext/standard/tests/file/php_fd_wrapper_04.phpt,, +ext/standard/tests/file/realpath_bug77484.phpt,, +ext/standard/tests/file/rename_variation.phpt,b/68717309, +ext/standard/tests/file/symlink_link_linkinfo_is_link_variation4.phpt,, +ext/standard/tests/file/symlink_link_linkinfo_is_link_variation8.phpt,, +ext/standard/tests/general_functions/escapeshellarg_bug71270.phpt,, +ext/standard/tests/general_functions/escapeshellcmd_bug71270.phpt,, +ext/standard/tests/network/bug20134.phpt,, +tests/output/stream_isatty_err.phpt,b/68720279, +tests/output/stream_isatty_in-err.phpt,b/68720282, +tests/output/stream_isatty_in-out-err.phpt,, +tests/output/stream_isatty_in-out.phpt,b/68720299, +tests/output/stream_isatty_out-err.phpt,b/68720311, +tests/output/stream_isatty_out.phpt,b/68720325, diff --git a/test/runtimes/blacklist_python3.7.3.csv b/test/runtimes/blacklist_python3.7.3.csv new file mode 100644 index 000000000..2b9947212 --- /dev/null +++ b/test/runtimes/blacklist_python3.7.3.csv @@ -0,0 +1,27 @@ +test name,bug id,comment +test_asynchat,b/76031995,SO_REUSEADDR +test_asyncio,,Fails on Docker. +test_asyncore,b/76031995,SO_REUSEADDR +test_epoll,, +test_fcntl,,fcntl invalid argument -- artificial test to make sure something works in 64 bit mode. +test_ftplib,,Fails in Docker +test_httplib,b/76031995,SO_REUSEADDR +test_imaplib,, +test_logging,, +test_multiprocessing_fork,,Flaky. Sometimes times out. +test_multiprocessing_forkserver,,Flaky. Sometimes times out. +test_multiprocessing_main_handling,,Flaky. Sometimes times out. +test_multiprocessing_spawn,,Flaky. Sometimes times out. +test_nntplib,b/76031995,tests should not set SO_REUSEADDR +test_poplib,,Fails on Docker +test_posix,b/76174079,posix.sched_get_priority_min not implemented + posix.sched_rr_get_interval not permitted +test_pty,b/76157709,out of pty devices +test_readline,b/76157709,out of pty devices +test_resource,b/76174079, +test_selectors,b/76116849,OSError not raised with epoll +test_smtplib,b/76031995,SO_REUSEADDR and unclosed sockets +test_socket,b/75983380, +test_ssl,b/76031995,SO_REUSEADDR +test_subprocess,, +test_support,b/76031995,SO_REUSEADDR +test_telnetlib,b/76031995,SO_REUSEADDR diff --git a/test/runtimes/blacklist_test.go b/test/runtimes/blacklist_test.go new file mode 100644 index 000000000..52f49b984 --- /dev/null +++ b/test/runtimes/blacklist_test.go @@ -0,0 +1,37 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "os" + "testing" +) + +func TestMain(m *testing.M) { + flag.Parse() + os.Exit(m.Run()) +} + +// Test that the blacklist parses without error. +func TestBlacklists(t *testing.T) { + bl, err := getBlacklist() + if err != nil { + t.Fatalf("error parsing blacklist: %v", err) + } + if *blacklistFile != "" && len(bl) == 0 { + t.Errorf("got empty blacklist for file %q", blacklistFile) + } +} diff --git a/test/runtimes/build_defs.bzl b/test/runtimes/build_defs.bzl index 5e3065342..6f84ca852 100644 --- a/test/runtimes/build_defs.bzl +++ b/test/runtimes/build_defs.bzl @@ -1,33 +1,72 @@ -"""Defines a rule for runsc test targets.""" +"""Defines a rule for runtime test targets.""" + +load("@io_bazel_rules_go//go:def.bzl", "go_test") -# runtime_test is a macro that will create targets to run the given test target -# with different runtime options. def runtime_test( + name, lang, - image, + image_repo = "gcr.io/gvisor-presubmit", + image_name = None, + blacklist_file = None, shard_count = 50, size = "enormous"): + """Generates sh_test and blacklist test targets for a given runtime. + + Args: + name: The name of the runtime being tested. Typically, the lang + version. + This is used in the names of the generated test targets. + lang: The language being tested. + image_repo: The docker repository containing the proctor image to run. + i.e., the prefix to the fully qualified docker image id. + image_name: The name of the image in the image_repo. + Defaults to the test name. + blacklist_file: A test blacklist to pass to the runtime test's runner. + shard_count: See Bazel common test attributes. + size: See Bazel common test attributes. + """ + if image_name == None: + image_name = name + args = [ + "--lang", + lang, + "--image", + "/".join([image_repo, image_name]), + ] + data = [ + ":runner", + ] + if blacklist_file: + args += ["--blacklist_file", "test/runtimes/" + blacklist_file] + data += [blacklist_file] + + # Add a test that the blacklist parses correctly. + blacklist_test(name, blacklist_file) + sh_test( - name = lang + "_test", + name = name + "_test", srcs = ["runner.sh"], - args = [ - "--lang", - lang, - "--image", - image, - ], - data = [ - ":runner", - ], + args = args, + data = data, size = size, shard_count = shard_count, tags = [ # Requires docker and runsc to be configured before the test runs. - "manual", "local", + # Don't include test target in wildcard target patterns. + "manual", ], ) +def blacklist_test(name, blacklist_file): + """Test that a blacklist parses correctly.""" + go_test( + name = name + "_blacklist_test", + embed = [":runner"], + srcs = ["blacklist_test.go"], + args = ["--blacklist_file", "test/runtimes/" + blacklist_file], + data = [blacklist_file], + ) + def sh_test(**kwargs): """Wraps the standard sh_test.""" native.sh_test( diff --git a/test/runtimes/runner.go b/test/runtimes/runner.go index 3a15f59a7..bec37c69d 100644 --- a/test/runtimes/runner.go +++ b/test/runtimes/runner.go @@ -16,6 +16,7 @@ package main import ( + "encoding/csv" "flag" "fmt" "io" @@ -31,8 +32,9 @@ import ( ) var ( - lang = flag.String("lang", "", "language runtime to test") - image = flag.String("image", "", "docker image with runtime tests") + lang = flag.String("lang", "", "language runtime to test") + image = flag.String("image", "", "docker image with runtime tests") + blacklistFile = flag.String("blacklist_file", "", "file containing blacklist of tests to exclude, in CSV format with fields: test name, bug id, comment") ) // Wait time for each test to run. @@ -52,6 +54,13 @@ func main() { // defered functions before exiting. It returns an exit code that should be // passed to os.Exit. func runTests() int { + // Get tests to blacklist. + blacklist, err := getBlacklist() + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting blacklist: %s\n", err.Error()) + return 1 + } + // Create a single docker container that will be used for all tests. d := dockerutil.MakeDocker("gvisor-" + *lang) defer d.CleanUp() @@ -59,7 +68,7 @@ func runTests() int { // Get a slice of tests to run. This will also start a single Docker // container that will be used to run each test. The final test will // stop the Docker container. - tests, err := getTests(d) + tests, err := getTests(d, blacklist) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", err.Error()) return 1 @@ -71,7 +80,7 @@ func runTests() int { // getTests returns a slice of tests to run, subject to the shard size and // index. -func getTests(d dockerutil.Docker) ([]testing.InternalTest, error) { +func getTests(d dockerutil.Docker, blacklist map[string]struct{}) ([]testing.InternalTest, error) { // Pull the image. if err := dockerutil.Pull(*image); err != nil { return nil, fmt.Errorf("docker pull %q failed: %v", *image, err) @@ -106,12 +115,18 @@ func getTests(d dockerutil.Docker) ([]testing.InternalTest, error) { itests = append(itests, testing.InternalTest{ Name: tc, F: func(t *testing.T) { + // Is the test blacklisted? + if _, ok := blacklist[tc]; ok { + t.Skip("SKIP: blacklisted test %q", tc) + } + var ( now = time.Now() done = make(chan struct{}) output string err error ) + go func() { fmt.Printf("RUNNING %s...\n", tc) output, err = d.Exec("/proctor", "--runtime", *lang, "--test", tc) @@ -134,6 +149,43 @@ func getTests(d dockerutil.Docker) ([]testing.InternalTest, error) { return itests, nil } +// getBlacklist reads the blacklist file and returns a set of test names to +// exclude. +func getBlacklist() (map[string]struct{}, error) { + blacklist := make(map[string]struct{}) + if *blacklistFile == "" { + return blacklist, nil + } + file, err := testutil.FindFile(*blacklistFile) + if err != nil { + return nil, err + } + f, err := os.Open(file) + if err != nil { + return nil, err + } + defer f.Close() + + r := csv.NewReader(f) + + // First line is header. Skip it. + if _, err := r.Read(); err != nil { + return nil, err + } + + for { + record, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + blacklist[record[0]] = struct{}{} + } + return blacklist, nil +} + // testDeps implements testing.testDeps (an unexported interface), and is // required to use testing.MainStart. type testDeps struct{} |