diff options
Diffstat (limited to 'test/runtimes')
23 files changed, 0 insertions, 1551 deletions
diff --git a/test/runtimes/BUILD b/test/runtimes/BUILD deleted file mode 100644 index 22b526f59..000000000 --- a/test/runtimes/BUILD +++ /dev/null @@ -1,46 +0,0 @@ -load("//tools:defs.bzl", "bzl_library") -load("//test/runtimes:defs.bzl", "runtime_test") - -package(licenses = ["notice"]) - -runtime_test( - name = "go1.12", - exclude_file = "exclude/go1.12.csv", - lang = "go", - shard_count = 8, -) - -runtime_test( - name = "java11", - batch = 100, - exclude_file = "exclude/java11.csv", - lang = "java", - shard_count = 16, -) - -runtime_test( - name = "nodejs12.4.0", - exclude_file = "exclude/nodejs12.4.0.csv", - lang = "nodejs", - shard_count = 8, -) - -runtime_test( - name = "php7.3.6", - exclude_file = "exclude/php7.3.6.csv", - lang = "php", - shard_count = 8, -) - -runtime_test( - name = "python3.7.3", - exclude_file = "exclude/python3.7.3.csv", - lang = "python", - shard_count = 8, -) - -bzl_library( - name = "defs_bzl", - srcs = ["defs.bzl"], - visibility = ["//visibility:private"], -) diff --git a/test/runtimes/README.md b/test/runtimes/README.md deleted file mode 100644 index 9dda1a728..000000000 --- a/test/runtimes/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# gVisor Runtime Tests - -App Engine uses gvisor to sandbox application containers. The runtime tests aim -to test `runsc` compatibility with these -[standard runtimes](https://cloud.google.com/appengine/docs/standard/runtimes). -The test itself runs the language-defined tests inside the sandboxed standard -runtime container. - -Note: [Ruby runtime](https://cloud.google.com/appengine/docs/standard/ruby) is -currently in beta mode and so we do not run tests for it yet. - -### Testing Locally - -To run runtime tests individually from a given runtime, use the following table. - -Language | Version | Download Image | Run Test(s) --------- | ------- | ------------------------------------------- | ----------- -Go | 1.12 | `make -C images load-runtimes_go1.12` | If the test name ends with `.go`, it is an on-disk test: <br> `docker run --runtime=runsc -it gvisor.dev/images/runtimes/go1.12 ( cd /usr/local/go/test ; go run run.go -v -- <TEST_NAME>... )` <br> Otherwise it is a tool test: <br> `docker run --runtime=runsc -it gvisor.dev/images/runtimes/go1.12 go tool dist test -v -no-rebuild ^TEST1$\|^TEST2$...` -Java | 11 | `make -C images load-runtimes_java11` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/java11 jtreg -agentvm -dir:/root/test/jdk -noreport -timeoutFactor:20 -verbose:summary <TEST_NAME>...` -NodeJS | 12.4.0 | `make -C images load-runtimes_nodejs12.4.0` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/nodejs12.4.0 python tools/test.py --timeout=180 <TEST_NAME>...` -Php | 7.3.6 | `make -C images load-runtimes_php7.3.6` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/php7.3.6 make test "TESTS=<TEST_NAME>..."` -Python | 3.7.3 | `make -C images load-runtimes_python3.7.3` | `docker run --runtime=runsc -it gvisor.dev/images/runtimes/python3.7.3 ./python -m test <TEST_NAME>...` - -To run an entire runtime test locally, use the following table. - -Note: java runtime test take 1+ hours with 16 cores. - -Language | Version | Running the test suite --------- | ------- | ---------------------------------------- -Go | 1.12 | `make go1.12-runtime-tests{_vfs2}` -Java | 11 | `make java11-runtime-tests{_vfs2}` -NodeJS | 12.4.0 | `make nodejs12.4.0-runtime-tests{_vfs2}` -Php | 7.3.6 | `make php7.3.6-runtime-tests{_vfs2}` -Python | 3.7.3 | `make python3.7.3-runtime-tests{_vfs2}` - -#### Clean Up - -Sometimes when runtime tests fail or when the testing container itself crashes -unexpectedly, the containers are not removed or sometimes do not even exit. This -can cause some docker commands like `docker system prune` to hang forever. - -Here are some helpful commands (should be executed in order): - -```bash -docker ps -a # Lists all docker processes; useful when investigating hanging containers. -docker kill $(docker ps -a -q) # Kills all running containers. -docker rm $(docker ps -a -q) # Removes all exited containers. -docker system prune # Remove unused data. -``` - -### Testing Infrastructure - -There are 3 components to this tests infrastructure: - -- [`runner`](runner) - This is the test entrypoint. This is the binary is - invoked by `bazel test`. The runner spawns the target runtime container - using `runsc` and then copies over the `proctor` binary into the container. -- [`proctor`](proctor) - This binary acts as our agent inside the container - which communicates with the runner and actually executes tests. -- [`exclude`](exclude) - Holds a CSV file for each language runtime containing - the full path of tests that should be excluded from running along with a - reason for exclusion. diff --git a/test/runtimes/defs.bzl b/test/runtimes/defs.bzl deleted file mode 100644 index 702522d86..000000000 --- a/test/runtimes/defs.bzl +++ /dev/null @@ -1,90 +0,0 @@ -"""Defines a rule for runtime test targets.""" - -load("//tools:defs.bzl", "go_test") - -def _runtime_test_impl(ctx): - # Construct arguments. - args = [ - "--lang", - ctx.attr.lang, - "--image", - ctx.attr.image, - "--batch", - str(ctx.attr.batch), - ] - if ctx.attr.exclude_file: - args += [ - "--exclude_file", - ctx.files.exclude_file[0].short_path, - ] - - # Build a runner. - runner = ctx.actions.declare_file("%s-executer" % ctx.label.name) - runner_content = "\n".join([ - "#!/bin/bash", - "%s %s $@\n" % (ctx.files._runner[0].short_path, " ".join(args)), - ]) - ctx.actions.write(runner, runner_content, is_executable = True) - - # Return the runner. - return [DefaultInfo( - executable = runner, - runfiles = ctx.runfiles( - files = ctx.files._runner + ctx.files.exclude_file + ctx.files._proctor, - collect_default = True, - collect_data = True, - ), - )] - -_runtime_test = rule( - implementation = _runtime_test_impl, - attrs = { - "image": attr.string( - mandatory = False, - ), - "lang": attr.string( - mandatory = True, - ), - "exclude_file": attr.label( - mandatory = False, - allow_single_file = True, - ), - "batch": attr.int( - default = 50, - mandatory = False, - ), - "_runner": attr.label( - default = "//test/runtimes/runner:runner", - executable = True, - cfg = "target", - ), - "_proctor": attr.label( - default = "//test/runtimes/proctor:proctor", - executable = True, - cfg = "target", - ), - }, - test = True, -) - -def runtime_test(name, **kwargs): - _runtime_test( - name = name, - image = name, # Resolved as images/runtimes/%s. - tags = [ - "local", - "manual", - ], - size = "enormous", - **kwargs - ) - -def exclude_test(name, exclude_file): - """Test that a exclude file parses correctly.""" - go_test( - name = name + "_exclude_test", - library = ":runner", - srcs = ["exclude_test.go"], - args = ["--exclude_file", "test/runtimes/" + exclude_file], - data = [exclude_file], - ) diff --git a/test/runtimes/exclude/go1.12.csv b/test/runtimes/exclude/go1.12.csv deleted file mode 100644 index 81e02cf64..000000000 --- a/test/runtimes/exclude/go1.12.csv +++ /dev/null @@ -1,13 +0,0 @@ -test name,bug id,comment -cgo_errors,,FLAKY -cgo_test,,FLAKY -go_test:cmd/go,,FLAKY -go_test:net,b/162473575,setsockopt: protocol not available. -go_test:os,b/118780122,we have a pollable filesystem but that's a surprise -go_test:os/signal,b/118780860,/dev/pts not properly supported. Also being tracked in b/29356795. -go_test:runtime,b/118782341,sigtrap not reported or caught or something. Also being tracked in b/33003106. -go_test:syscall,b/118781998,bad bytes -- bad mem addr; FcntlFlock(F_GETLK) not supported. -runtime:cpu124,b/118778254,segmentation fault -test:0_1,,FLAKY -testcarchive,b/118782924,no sigpipe -testshared,,FLAKY diff --git a/test/runtimes/exclude/java11.csv b/test/runtimes/exclude/java11.csv deleted file mode 100644 index e41441374..000000000 --- a/test/runtimes/exclude/java11.csv +++ /dev/null @@ -1,212 +0,0 @@ -test name,bug id,comment -com/sun/crypto/provider/Cipher/PBE/PKCS12Cipher.java,,Fails in Docker -com/sun/jdi/InvokeHangTest.java,https://bugs.openjdk.java.net/browse/JDK-8218463, -com/sun/jdi/NashornPopFrameTest.java,, -com/sun/jdi/ProcessAttachTest.java,, -com/sun/management/HotSpotDiagnosticMXBean/CheckOrigin.java,,Fails in Docker -com/sun/management/OperatingSystemMXBean/GetCommittedVirtualMemorySize.java,, -com/sun/management/ThreadMXBean/ThreadCpuTimeArray.java,,Test assumes high CPU clock precision -com/sun/management/UnixOperatingSystemMXBean/GetMaxFileDescriptorCount.sh,, -com/sun/tools/attach/AttachSelf.java,, -com/sun/tools/attach/BasicTests.java,, -com/sun/tools/attach/PermissionTest.java,, -com/sun/tools/attach/StartManagementAgent.java,, -com/sun/tools/attach/TempDirTest.java,, -com/sun/tools/attach/modules/Driver.java,, -java/lang/Character/CheckScript.java,,Fails in Docker -java/lang/Character/CheckUnicode.java,,Fails in Docker -java/lang/Class/GetPackageBootLoaderChildLayer.java,, -java/lang/ClassLoader/nativeLibrary/NativeLibraryTest.java,,Fails in Docker -java/lang/module/ModuleDescriptorTest.java,, -java/lang/String/nativeEncoding/StringPlatformChars.java,, -java/net/CookieHandler/B6791927.java,,java.lang.RuntimeException: Expiration date shouldn't be 0 -java/net/ipv6tests/TcpTest.java,,java.net.ConnectException: Connection timed out (Connection timed out) -java/net/ipv6tests/UdpTest.java,,Times out -java/net/Inet6Address/B6558853.java,,Times out -java/net/InetAddress/CheckJNI.java,,java.net.ConnectException: Connection timed out (Connection timed out) -java/net/InterfaceAddress/NetworkPrefixLength.java,b/78507103, -java/net/MulticastSocket/B6425815.java,,java.net.SocketException: Protocol not available (Error getting socket option) -java/net/MulticastSocket/B6427403.java,,java.net.SocketException: Protocol not available -java/net/MulticastSocket/MulticastTTL.java,, -java/net/MulticastSocket/NetworkInterfaceEmptyGetInetAddressesTest.java,,java.net.SocketException: Protocol not available (Error getting socket option) -java/net/MulticastSocket/NoLoopbackPackets.java,,java.net.SocketException: Protocol not available -java/net/MulticastSocket/Promiscuous.java,, -java/net/MulticastSocket/SetLoopbackMode.java,, -java/net/MulticastSocket/SetTTLAndGetTTL.java,, -java/net/MulticastSocket/Test.java,, -java/net/MulticastSocket/TestDefaults.java,, -java/net/MulticastSocket/TimeToLive.java,, -java/net/NetworkInterface/NetworkInterfaceStreamTest.java,, -java/net/Socket/LinkLocal.java,,java.net.SocketTimeoutException: Receive timed out -java/net/Socket/SetSoLinger.java,b/78527327,SO_LINGER is not yet supported -java/net/Socket/UrgentDataTest.java,b/111515323, -java/net/SocketOption/OptionsTest.java,,Fails in Docker -java/net/SocketPermission/SocketPermissionTest.java,, -java/net/URLConnection/6212146/TestDriver.java,,Fails in Docker -java/net/httpclient/RequestBuilderTest.java,,Fails in Docker -java/nio/channels/DatagramChannel/BasicMulticastTests.java,, -java/nio/channels/DatagramChannel/SocketOptionTests.java,,java.net.SocketException: Invalid argument -java/nio/channels/DatagramChannel/UseDGWithIPv6.java,, -java/nio/channels/FileChannel/directio/DirectIOTest.java,,Fails in Docker -java/nio/channels/FileChannel/directio/PwriteDirect.java,,java.io.IOException: Invalid argument -java/nio/channels/Selector/OutOfBand.java,, -java/nio/channels/Selector/SelectWithConsumer.java,,Flaky -java/nio/channels/ServerSocketChannel/SocketOptionTests.java,, -java/nio/channels/SocketChannel/LingerOnClose.java,, -java/nio/channels/SocketChannel/SocketOptionTests.java,b/77965901, -java/nio/channels/spi/SelectorProvider/inheritedChannel/InheritedChannelTest.java,,Fails in Docker -java/rmi/activation/Activatable/extLoadedImpl/ext.sh,, -java/rmi/transport/checkLeaseInfoLeak/CheckLeaseLeak.java,, -java/security/cert/PolicyNode/GetPolicyQualifiers.java,b/170263154,Kokoro executor cert expired -java/text/Format/NumberFormat/CurrencyFormat.java,,Fails in Docker -java/text/Format/NumberFormat/CurrencyFormat.java,,Fails in Docker -java/util/Calendar/JapaneseEraNameTest.java,, -java/util/Currency/CurrencyTest.java,,Fails in Docker -java/util/Currency/ValidateISO4217.java,,Fails in Docker -java/util/EnumSet/BogusEnumSet.java,,"java.io.InvalidClassException: java.util.EnumSet; local class incompatible: stream classdesc serialVersionUID = -2409567991088730183, local class serialVersionUID = 1009687484059888093" -java/util/Locale/Bug8040211.java,,java.lang.RuntimeException: Failed. -java/util/Locale/LSRDataTest.java,, -java/util/Properties/CompatibilityTest.java,,"java.lang.RuntimeException: jdk.internal.org.xml.sax.SAXParseException; Internal DTD subset is not allowed. The Properties XML document must have the following DOCTYPE declaration: <!DOCTYPE properties SYSTEM ""http://java.sun.com/dtd/properties.dtd"">" -java/util/ResourceBundle/Control/XMLResourceBundleTest.java,,java.util.MissingResourceException: Can't find bundle for base name XmlRB locale -java/util/ResourceBundle/modules/xmlformat/xmlformat.sh,,Timeout reached: 60000. Process is not alive! -java/util/TimeZone/TimeZoneTest.java,,Uncaught exception thrown in test method TestShortZoneIDs -java/util/concurrent/locks/Lock/TimedAcquireLeak.java,, -java/util/jar/JarFile/mrjar/MultiReleaseJarAPI.java,,Fails in Docker -java/util/logging/LogManager/Configuration/updateConfiguration/SimpleUpdateConfigWithInputStreamTest.java,, -java/util/logging/TestLoggerWeakRefLeak.java,, -java/util/spi/ResourceBundleControlProvider/UserDefaultControlTest.java,,java.util.MissingResourceException: Can't find bundle for base name com.foo.XmlRB locale -javax/imageio/AppletResourceTest.java,, -javax/imageio/plugins/jpeg/JPEGsNotAcceleratedTest.java,,java.awt.HeadlessException: No X11 DISPLAY variable was set but this program performed an operation which requires it. -javax/management/security/HashedPasswordFileTest.java,, -javax/net/ssl/DTLS/DTLSBufferOverflowUnderflowTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSDataExchangeTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSEnginesClosureTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSHandshakeTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSHandshakeWithReplicatedPacketsTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSIncorrectAppDataTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSMFLNTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSNotEnabledRC4Test.java,,Compilation failed -javax/net/ssl/DTLS/DTLSRehandshakeTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSRehandshakeWithCipherChangeTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSRehandshakeWithDataExTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSSequenceNumberTest.java,,Compilation failed -javax/net/ssl/DTLS/DTLSUnsupportedCiphersTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10BufferOverflowUnderflowTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10DataExchangeTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10EnginesClosureTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10HandshakeTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10HandshakeWithReplicatedPacketsTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10IncorrectAppDataTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10MFLNTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10NotEnabledRC4Test.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10RehandshakeTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10RehandshakeWithCipherChangeTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10RehandshakeWithDataExTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10SequenceNumberTest.java,,Compilation failed -javax/net/ssl/DTLSv10/DTLSv10UnsupportedCiphersTest.java,,Compilation failed -javax/net/ssl/SSLSession/JSSERenegotiate.java,,Fails in Docker -javax/net/ssl/TLS/TLSDataExchangeTest.java,,Compilation failed -javax/net/ssl/TLS/TLSEnginesClosureTest.java,,Compilation failed -javax/net/ssl/TLS/TLSHandshakeTest.java,,Compilation failed -javax/net/ssl/TLS/TLSMFLNTest.java,,Compilation failed -javax/net/ssl/TLS/TLSNotEnabledRC4Test.java,,Compilation failed -javax/net/ssl/TLS/TLSRehandshakeTest.java,,Compilation failed -javax/net/ssl/TLS/TLSRehandshakeWithCipherChangeTest.java,,Compilation failed -javax/net/ssl/TLS/TLSRehandshakeWithDataExTest.java,,Compilation failed -javax/net/ssl/TLS/TLSUnsupportedCiphersTest.java,,Compilation failed -javax/net/ssl/TLSv1/TLSDataExchangeTest.java,,Compilation failed -javax/net/ssl/TLSv1/TLSEnginesClosureTest.java,,Compilation failed -javax/net/ssl/TLSv1/TLSHandshakeTest.java,,Compilation failed -javax/net/ssl/TLSv1/TLSMFLNTest.java,,Compilation failed -javax/net/ssl/TLSv1/TLSNotEnabledRC4Test.java,,Compilation failed -javax/net/ssl/TLSv1/TLSRehandshakeTest.java,,Compilation failed -javax/net/ssl/TLSv1/TLSRehandshakeWithCipherChangeTest.java,,Compilation failed -javax/net/ssl/TLSv1/TLSRehandshakeWithDataExTest.java,,Compilation failed -javax/net/ssl/TLSv1/TLSUnsupportedCiphersTest.java,,Compilation failed -javax/net/ssl/TLSv11/TLSDataExchangeTest.java,,Compilation failed -javax/net/ssl/TLSv11/TLSEnginesClosureTest.java,,Compilation failed -javax/net/ssl/TLSv11/TLSHandshakeTest.java,,Compilation failed -javax/net/ssl/TLSv11/TLSMFLNTest.java,,Compilation failed -javax/net/ssl/TLSv11/TLSNotEnabledRC4Test.java,,Compilation failed -javax/net/ssl/TLSv11/TLSRehandshakeTest.java,,Compilation failed -javax/net/ssl/TLSv11/TLSRehandshakeWithCipherChangeTest.java,,Compilation failed -javax/net/ssl/TLSv11/TLSRehandshakeWithDataExTest.java,,Compilation failed -javax/net/ssl/TLSv11/TLSUnsupportedCiphersTest.java,,Compilation failed -javax/net/ssl/TLSv12/TLSEnginesClosureTest.java,,Compilation failed -javax/sound/sampled/AudioInputStream/FrameLengthAfterConversion.java,, -jdk/jfr/cmd/TestHelp.java,,java.lang.RuntimeException: 'Available commands are:' missing from stdout/stderr -jdk/jfr/cmd/TestPrint.java,,Missing file' missing from stdout/stderr -jdk/jfr/cmd/TestPrintDefault.java,,java.lang.RuntimeException: 'JVMInformation' missing from stdout/stderr -jdk/jfr/cmd/TestPrintJSON.java,,javax.script.ScriptException: <eval>:1:17 Expected an operand but found eof var jsonObject = ^ in <eval> at line number 1 at column number 17 -jdk/jfr/cmd/TestPrintXML.java,,org.xml.sax.SAXParseException; lineNumber: 1; columnNumber: 1; Premature end of file. -jdk/jfr/cmd/TestReconstruct.java,,java.lang.RuntimeException: 'Too few arguments' missing from stdout/stderr -jdk/jfr/cmd/TestSplit.java,,java.lang.RuntimeException: 'Missing file' missing from stdout/stderr -jdk/jfr/cmd/TestSummary.java,,java.lang.RuntimeException: 'Missing file' missing from stdout/stderr -jdk/jfr/event/compiler/TestCompilerStats.java,,java.lang.RuntimeException: Field nmetodsSize not in event -jdk/jfr/event/metadata/TestDefaultConfigurations.java,,Setting 'threshold' in event 'jdk.SecurityPropertyModification' was not configured in the configuration 'default' -jdk/jfr/event/oldobject/TestLargeRootSet.java,,Flaky - `main' threw exception: java.lang.RuntimeException: Could not find root object -jdk/jfr/event/runtime/TestActiveSettingEvent.java,,java.lang.Exception: Could not find setting with name jdk.X509Validation#threshold -jdk/jfr/event/runtime/TestModuleEvents.java,,java.lang.RuntimeException: assertEquals: expected jdk.proxy1 to equal java.base -jdk/jfr/event/runtime/TestNetworkUtilizationEvent.java,, -jdk/jfr/event/runtime/TestThreadParkEvent.java,, -jdk/jfr/event/sampling/TestNative.java,, -jdk/jfr/jcmd/TestJcmdChangeLogLevel.java,, -jdk/jfr/jcmd/TestJcmdConfigure.java,, -jdk/jfr/jcmd/TestJcmdDump.java,, -jdk/jfr/jcmd/TestJcmdDumpGeneratedFilename.java,, -jdk/jfr/jcmd/TestJcmdDumpLimited.java,, -jdk/jfr/jcmd/TestJcmdDumpPathToGCRoots.java,, -jdk/jfr/jcmd/TestJcmdLegacy.java,, -jdk/jfr/jcmd/TestJcmdSaveToFile.java,, -jdk/jfr/jcmd/TestJcmdStartDirNotExist.java,, -jdk/jfr/jcmd/TestJcmdStartInvaldFile.java,, -jdk/jfr/jcmd/TestJcmdStartPathToGCRoots.java,, -jdk/jfr/jcmd/TestJcmdStartStopDefault.java,, -jdk/jfr/jcmd/TestJcmdStartWithOptions.java,, -jdk/jfr/jcmd/TestJcmdStartWithSettings.java,, -jdk/jfr/jcmd/TestJcmdStopInvalidFile.java,, -jdk/jfr/jvm/TestGetAllEventClasses.java,,Compilation failed -jdk/jfr/jvm/TestJfrJavaBase.java,, -jdk/jfr/startupargs/TestStartRecording.java,, -jdk/modules/incubator/ImageModules.java,, -jdk/net/Sockets/ExtOptionTest.java,, -jdk/net/Sockets/QuickAckTest.java,, -lib/security/cacerts/VerifyCACerts.java,, -sun/management/jmxremote/bootstrap/CustomLauncherTest.java,, -sun/management/jmxremote/bootstrap/JvmstatCountersTest.java,, -sun/management/jmxremote/bootstrap/LocalManagementTest.java,, -sun/management/jmxremote/bootstrap/RmiRegistrySslTest.java,, -sun/management/jmxremote/bootstrap/RmiSslBootstrapTest.sh,, -sun/management/jmxremote/startstop/JMXStartStopTest.java,, -sun/management/jmxremote/startstop/JMXStatusPerfCountersTest.java,, -sun/management/jmxremote/startstop/JMXStatusTest.java,, -sun/management/jdp/JdpDefaultsTest.java,, -sun/management/jdp/JdpJmxRemoteDynamicPortTest.java,, -sun/management/jdp/JdpOffTest.java,, -sun/management/jdp/JdpSpecificAddressTest.java,, -sun/text/resources/LocaleDataTest.java,, -sun/tools/jcmd/TestJcmdSanity.java,, -sun/tools/jhsdb/AlternateHashingTest.java,, -sun/tools/jhsdb/BasicLauncherTest.java,, -sun/tools/jhsdb/HeapDumpTest.java,, -sun/tools/jhsdb/heapconfig/JMapHeapConfigTest.java,, -sun/tools/jinfo/BasicJInfoTest.java,, -sun/tools/jinfo/JInfoTest.java,, -sun/tools/jmap/BasicJMapTest.java,, -sun/tools/jstack/BasicJStackTest.java,, -sun/tools/jstack/DeadlockDetectionTest.java,, -sun/tools/jstatd/TestJstatdExternalRegistry.java,, -sun/tools/jstatd/TestJstatdPort.java,,Flaky -sun/tools/jstatd/TestJstatdPortAndServer.java,,Flaky -sun/util/calendar/zi/TestZoneInfo310.java,, -tools/jar/modularJar/Basic.java,, -tools/jar/multiRelease/Basic.java,, -tools/jimage/JImageExtractTest.java,, -tools/jimage/JImageTest.java,, -tools/jlink/JLinkTest.java,, -tools/jlink/plugins/IncludeLocalesPluginTest.java,, -tools/jmod/hashes/HashesTest.java,, -tools/launcher/BigJar.java,b/111611473, -tools/launcher/HelpFlagsTest.java,,java.lang.AssertionError: HelpFlagsTest failed: Tool jfr not covered by this test. Add specification to jdkTools array! -tools/launcher/VersionCheck.java,,java.lang.AssertionError: VersionCheck failed: testToolVersion: [jfr]; -tools/launcher/modules/patch/systemmodules/PatchSystemModules.java,, diff --git a/test/runtimes/exclude/nodejs12.4.0.csv b/test/runtimes/exclude/nodejs12.4.0.csv deleted file mode 100644 index c4e7917ec..000000000 --- a/test/runtimes/exclude/nodejs12.4.0.csv +++ /dev/null @@ -1,45 +0,0 @@ -test name,bug id,comment -async-hooks/test-statwatcher.js,https://github.com/nodejs/node/issues/21425,Check for fix inclusion in nodejs releases after 2020-03-29 -benchmark/test-benchmark-fs.js,,Broken test -benchmark/test-benchmark-napi.js,,Broken test -doctool/test-make-doc.js,b/68848110,Expected to fail. -internet/test-dgram-multicast-set-interface-lo.js,b/162798882, -internet/test-doctool-versions.js,,Broken test -internet/test-uv-threadpool-schedule.js,,Broken test -parallel/test-dgram-bind-fd.js,b/132447356, -parallel/test-dgram-socket-buffer-size.js,b/68847921, -parallel/test-dns-channel-timeout.js,b/161893056, -parallel/test-fs-access.js,,Broken test -parallel/test-fs-watchfile.js,b/166819807,Flaky - VFS1 only -parallel/test-fs-write-stream.js,b/166819807,Flaky - VFS1 only -parallel/test-fs-write-stream-double-close.js,b/166819807,Flaky - VFS1 only -parallel/test-fs-write-stream-throw-type-error.js,b/166819807,Flaky - VFS1 only -parallel/test-http-writable-true-after-close.js,b/171301436,Flaky - Mismatched <anonymous> function calls. Expected exactly 1 actual 2 -parallel/test-os.js,b/63997097, -parallel/test-process-uid-gid.js,,Does not work inside Docker with gid nobody -pseudo-tty/test-assert-colors.js,b/162801321, -pseudo-tty/test-assert-no-color.js,b/162801321, -pseudo-tty/test-assert-position-indicator.js,b/162801321, -pseudo-tty/test-async-wrap-getasyncid-tty.js,b/162801321, -pseudo-tty/test-fatal-error.js,b/162801321, -pseudo-tty/test-handle-wrap-isrefed-tty.js,b/162801321, -pseudo-tty/test-readable-tty-keepalive.js,b/162801321, -pseudo-tty/test-set-raw-mode-reset-process-exit.js,b/162801321, -pseudo-tty/test-set-raw-mode-reset-signal.js,b/162801321, -pseudo-tty/test-set-raw-mode-reset.js,b/162801321, -pseudo-tty/test-stderr-stdout-handle-sigwinch.js,b/162801321, -pseudo-tty/test-stdout-read.js,b/162801321, -pseudo-tty/test-tty-color-support.js,b/162801321, -pseudo-tty/test-tty-isatty.js,b/162801321, -pseudo-tty/test-tty-stdin-call-end.js,b/162801321, -pseudo-tty/test-tty-stdin-end.js,b/162801321, -pseudo-tty/test-stdin-write.js,b/162801321, -pseudo-tty/test-tty-stdout-end.js,b/162801321, -pseudo-tty/test-tty-stdout-resize.js,b/162801321, -pseudo-tty/test-tty-stream-constructors.js,b/162801321, -pseudo-tty/test-tty-window-size.js,b/162801321, -pseudo-tty/test-tty-wrap.js,b/162801321, -pummel/test-net-pingpong.js,,Broken test -pummel/test-vm-memleak.js,b/162799436, -pummel/test-watch-file.js,,Flaky - VFS1 only -tick-processor/test-tick-processor-builtin.js,,Broken test diff --git a/test/runtimes/exclude/php7.3.6.csv b/test/runtimes/exclude/php7.3.6.csv deleted file mode 100644 index 0e8fc2c98..000000000 --- a/test/runtimes/exclude/php7.3.6.csv +++ /dev/null @@ -1,48 +0,0 @@ -test name,bug id,comment -ext/intl/tests/bug77895.phpt,, -ext/intl/tests/dateformat_bug65683_2.phpt,, -ext/mbstring/tests/bug76319.phpt,, -ext/mbstring/tests/bug76958.phpt,, -ext/mbstring/tests/bug77025.phpt,, -ext/mbstring/tests/bug77165.phpt,, -ext/mbstring/tests/bug77454.phpt,, -ext/mbstring/tests/mb_convert_encoding_leak.phpt,, -ext/mbstring/tests/mb_strrpos_encoding_3rd_param.phpt,, -ext/pcre/tests/cache_limit.phpt,,Broken test - Flaky -ext/session/tests/session_module_name_variation4.phpt,,Flaky -ext/session/tests/session_set_save_handler_class_018.phpt,, -ext/session/tests/session_set_save_handler_iface_003.phpt,, -ext/session/tests/session_set_save_handler_sid_001.phpt,, -ext/session/tests/session_set_save_handler_variation4.phpt,, -ext/standard/tests/file/disk.phpt,https://bugs.php.net/bug.php?id=80018, -ext/standard/tests/file/disk_free_space_basic.phpt,https://bugs.php.net/bug.php?id=80018, -ext/standard/tests/file/disk_free_space_error.phpt,https://bugs.php.net/bug.php?id=80018, -ext/standard/tests/file/disk_free_space_variation.phpt,https://bugs.php.net/bug.php?id=80018, -ext/standard/tests/file/disk_total_space_basic.phpt,https://bugs.php.net/bug.php?id=80018, -ext/standard/tests/file/disk_total_space_error.phpt,https://bugs.php.net/bug.php?id=80018, -ext/standard/tests/file/disk_total_space_variation.phpt,https://bugs.php.net/bug.php?id=80018, -ext/standard/tests/file/fopen_variation19.phpt,b/162894964, -ext/standard/tests/file/lstat_stat_variation14.phpt,,Flaky -ext/standard/tests/file/php_fd_wrapper_01.phpt,, -ext/standard/tests/file/php_fd_wrapper_02.phpt,, -ext/standard/tests/file/php_fd_wrapper_03.phpt,, -ext/standard/tests/file/php_fd_wrapper_04.phpt,, -ext/standard/tests/file/realpath_bug77484.phpt,b/162894969,VFS1 only failure -ext/standard/tests/file/rename_variation.phpt,b/68717309, -ext/standard/tests/file/symlink_link_linkinfo_is_link_variation4.phpt,b/162895341, -ext/standard/tests/file/symlink_link_linkinfo_is_link_variation8.phpt,b/162896223, -ext/standard/tests/general_functions/escapeshellarg_bug71270.phpt,, -ext/standard/tests/general_functions/escapeshellcmd_bug71270.phpt,, -ext/standard/tests/streams/proc_open_bug60120.phpt,,Flaky until php-src 3852a35fdbcb -ext/standard/tests/streams/proc_open_bug64438.phpt,,Flaky -ext/standard/tests/streams/proc_open_bug69900.phpt,,Flaky -ext/standard/tests/streams/stream_socket_sendto.phpt,, -ext/standard/tests/strings/007.phpt,, -sapi/cli/tests/upload_2G.phpt,, -tests/output/stream_isatty_err.phpt,b/68720279, -tests/output/stream_isatty_in-err.phpt,b/68720282, -tests/output/stream_isatty_in-out-err.phpt,, -tests/output/stream_isatty_in-out.phpt,b/68720299, -tests/output/stream_isatty_out-err.phpt,b/68720311, -tests/output/stream_isatty_out.phpt,b/68720325, -Zend/tests/concat_003.phpt,b/162896021, diff --git a/test/runtimes/exclude/python3.7.3.csv b/test/runtimes/exclude/python3.7.3.csv deleted file mode 100644 index e9fef03b7..000000000 --- a/test/runtimes/exclude/python3.7.3.csv +++ /dev/null @@ -1,19 +0,0 @@ -test name,bug id,comment -test_asyncio,,Fails on Docker. -test_asyncore,b/162973328, -test_epoll,b/162983393, -test_fcntl,b/162978767,fcntl invalid argument -- artificial test to make sure something works in 64 bit mode. -test_httplib,b/163000009,OSError: [Errno 98] Address already in use -test_logging,b/162980079, -test_multiprocessing_fork,,Flaky. Sometimes times out. -test_multiprocessing_forkserver,,Flaky. Sometimes times out. -test_multiprocessing_main_handling,,Flaky. Sometimes times out. -test_multiprocessing_spawn,,Flaky. Sometimes times out. -test_posix,b/76174079,posix.sched_get_priority_min not implemented + posix.sched_rr_get_interval not permitted -test_pty,b/162979921, -test_readline,b/162980389,TestReadline hangs forever -test_resource,b/76174079, -test_selectors,b/76116849,OSError not raised with epoll -test_smtplib,b/162980434,unclosed sockets -test_signal,,Flaky - signal: alarm clock -test_socket,b/75983380, diff --git a/test/runtimes/proctor/BUILD b/test/runtimes/proctor/BUILD deleted file mode 100644 index fdc6d3173..000000000 --- a/test/runtimes/proctor/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -load("//tools:defs.bzl", "go_binary") - -package(licenses = ["notice"]) - -go_binary( - name = "proctor", - srcs = ["main.go"], - pure = True, - visibility = ["//test/runtimes:__pkg__"], - deps = ["//test/runtimes/proctor/lib"], -) diff --git a/test/runtimes/proctor/lib/BUILD b/test/runtimes/proctor/lib/BUILD deleted file mode 100644 index 0c8367dfe..000000000 --- a/test/runtimes/proctor/lib/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "lib", - srcs = [ - "go.go", - "java.go", - "lib.go", - "nodejs.go", - "php.go", - "python.go", - ], - visibility = ["//test/runtimes/proctor:__pkg__"], -) - -go_test( - name = "lib_test", - size = "small", - srcs = ["lib_test.go"], - library = ":lib", - deps = ["//pkg/test/testutil"], -) diff --git a/test/runtimes/proctor/lib/go.go b/test/runtimes/proctor/lib/go.go deleted file mode 100644 index 5c48fb60b..000000000 --- a/test/runtimes/proctor/lib/go.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package lib - -import ( - "fmt" - "os" - "os/exec" - "regexp" - "strings" -) - -var ( - goTestRegEx = regexp.MustCompile(`^.+\.go$`) - - // Directories with .dir contain helper files for tests. - // Exclude benchmarks and stress tests. - goDirFilter = regexp.MustCompile(`^(bench|stress)\/.+$|^.+\.dir.+$`) -) - -// Location of Go tests on disk. -const goTestDir = "/usr/local/go/test" - -// goRunner implements TestRunner for Go. -// -// There are two types of Go tests: "Go tool tests" and "Go tests on disk". -// "Go tool tests" are found and executed using `go tool dist test`. "Go tests -// on disk" are found in the /usr/local/go/test directory and are executed -// using `go run run.go`. -type goRunner struct{} - -var _ TestRunner = goRunner{} - -// ListTests implements TestRunner.ListTests. -func (goRunner) ListTests() ([]string, error) { - // Go tool dist test tests. - args := []string{"tool", "dist", "test", "-list"} - cmd := exec.Command("go", args...) - cmd.Stderr = os.Stderr - out, err := cmd.Output() - if err != nil { - return nil, fmt.Errorf("failed to list: %v", err) - } - var toolSlice []string - for _, test := range strings.Split(string(out), "\n") { - toolSlice = append(toolSlice, test) - } - - // Go tests on disk. - diskSlice, err := Search(goTestDir, goTestRegEx) - if err != nil { - return nil, err - } - // Remove items from /bench/, /stress/ and .dir files - diskFiltered := diskSlice[:0] - for _, file := range diskSlice { - if !goDirFilter.MatchString(file) { - diskFiltered = append(diskFiltered, file) - } - } - - return append(toolSlice, diskFiltered...), nil -} - -// TestCmds implements TestRunner.TestCmds. -func (goRunner) TestCmds(tests []string) []*exec.Cmd { - var toolTests, onDiskTests []string - for _, test := range tests { - if strings.HasSuffix(test, ".go") { - onDiskTests = append(onDiskTests, test) - } else { - toolTests = append(toolTests, "^"+test+"$") - } - } - - var cmds []*exec.Cmd - if len(toolTests) > 0 { - cmds = append(cmds, exec.Command("go", "tool", "dist", "test", "-v", "-no-rebuild", "-run", strings.Join(toolTests, "\\|"))) - } - if len(onDiskTests) > 0 { - cmd := exec.Command("go", append([]string{"run", "run.go", "-v", "--"}, onDiskTests...)...) - cmd.Dir = goTestDir - cmds = append(cmds, cmd) - } - - return cmds -} diff --git a/test/runtimes/proctor/lib/java.go b/test/runtimes/proctor/lib/java.go deleted file mode 100644 index 3105011ff..000000000 --- a/test/runtimes/proctor/lib/java.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package lib - -import ( - "fmt" - "os" - "os/exec" - "regexp" - "strings" -) - -// Directories to exclude from tests. -var javaExclDirs = regexp.MustCompile(`(^(sun\/security)|(java\/util\/stream)|(java\/time)| )`) - -// Location of java tests. -const javaTestDir = "/root/test/jdk" - -// javaRunner implements TestRunner for Java. -type javaRunner struct{} - -var _ TestRunner = javaRunner{} - -// ListTests implements TestRunner.ListTests. -func (javaRunner) ListTests() ([]string, error) { - args := []string{ - "-dir:" + javaTestDir, - "-ignore:quiet", - "-a", - "-listtests", - ":jdk_core", - ":jdk_svc", - ":jdk_sound", - ":jdk_imageio", - } - cmd := exec.Command("jtreg", args...) - cmd.Stderr = os.Stderr - out, err := cmd.Output() - if err != nil { - return nil, fmt.Errorf("jtreg -listtests : %v", err) - } - var testSlice []string - for _, test := range strings.Split(string(out), "\n") { - if !javaExclDirs.MatchString(test) { - testSlice = append(testSlice, test) - } - } - return testSlice, nil -} - -// TestCmds implements TestRunner.TestCmds. -func (javaRunner) TestCmds(tests []string) []*exec.Cmd { - args := append( - []string{ - "-agentvm", // Execute each action using a pool of reusable JVMs. - "-dir:" + javaTestDir, // Base directory for test files and directories. - "-noreport", // Do not generate a final report. - "-timeoutFactor:20", // Extend the default timeout (2 min) of all tests by this factor. - "-verbose:nopass", // Verbose output but supress it for tests that passed. - }, - tests..., - ) - return []*exec.Cmd{exec.Command("jtreg", args...)} -} diff --git a/test/runtimes/proctor/lib/lib.go b/test/runtimes/proctor/lib/lib.go deleted file mode 100644 index f2ba82498..000000000 --- a/test/runtimes/proctor/lib/lib.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package lib contains proctor functions. -package lib - -import ( - "fmt" - "os" - "os/exec" - "os/signal" - "path/filepath" - "regexp" - "syscall" -) - -// TestRunner is an interface that must be implemented for each runtime -// integrated with proctor. -type TestRunner interface { - // ListTests returns a string slice of tests available to run. - ListTests() ([]string, error) - - // TestCmds returns a slice of *exec.Cmd that will run the given tests. - // There is no correlation between the number of exec.Cmds returned and the - // number of tests. It could return one command to run all tests or a few - // commands that collectively run all. - TestCmds(tests []string) []*exec.Cmd -} - -// TestRunnerForRuntime returns a new TestRunner for the given runtime. -func TestRunnerForRuntime(runtime string) (TestRunner, error) { - switch runtime { - case "go": - return goRunner{}, nil - case "java": - return javaRunner{}, nil - case "nodejs": - return nodejsRunner{}, nil - case "php": - return phpRunner{}, nil - case "python": - return pythonRunner{}, nil - } - return nil, fmt.Errorf("invalid runtime %q", runtime) -} - -// PauseAndReap is like init. It runs forever and reaps any children. -func PauseAndReap() { - // Get notified of any new children. - ch := make(chan os.Signal, 1) - signal.Notify(ch, syscall.SIGCHLD) - - for { - if _, ok := <-ch; !ok { - // Channel closed. This should not happen. - panic("signal channel closed") - } - - // Reap the child. - for { - if cpid, _ := syscall.Wait4(-1, nil, syscall.WNOHANG, nil); cpid < 1 { - break - } - } - } -} - -// Search is a helper function to find tests in the given directory that match -// the regex. -func Search(root string, testFilter *regexp.Regexp) ([]string, error) { - var testSlice []string - - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - name := filepath.Base(path) - - if info.IsDir() || !testFilter.MatchString(name) { - return nil - } - - relPath, err := filepath.Rel(root, path) - if err != nil { - return err - } - testSlice = append(testSlice, relPath) - return nil - }) - if err != nil { - return nil, fmt.Errorf("walking %q: %v", root, err) - } - - return testSlice, nil -} diff --git a/test/runtimes/proctor/lib/lib_test.go b/test/runtimes/proctor/lib/lib_test.go deleted file mode 100644 index 1193d2e28..000000000 --- a/test/runtimes/proctor/lib/lib_test.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package lib - -import ( - "io/ioutil" - "os" - "path/filepath" - "reflect" - "regexp" - "strings" - "testing" - - "gvisor.dev/gvisor/pkg/test/testutil" -) - -func touch(t *testing.T, name string) { - t.Helper() - f, err := os.Create(name) - if err != nil { - t.Fatalf("error creating file %q: %v", name, err) - } - if err := f.Close(); err != nil { - t.Fatalf("error closing file %q: %v", name, err) - } -} - -func TestSearchEmptyDir(t *testing.T) { - td, err := ioutil.TempDir(testutil.TmpDir(), "searchtest") - if err != nil { - t.Fatalf("error creating searchtest: %v", err) - } - defer os.RemoveAll(td) - - var want []string - - testFilter := regexp.MustCompile(`^test-[^-].+\.tc$`) - got, err := Search(td, testFilter) - if err != nil { - t.Errorf("search error: %v", err) - } - - if !reflect.DeepEqual(got, want) { - t.Errorf("Found %#v; want %#v", got, want) - } -} - -func TestSearch(t *testing.T) { - td, err := ioutil.TempDir(testutil.TmpDir(), "searchtest") - if err != nil { - t.Fatalf("error creating searchtest: %v", err) - } - defer os.RemoveAll(td) - - // Creating various files similar to the test filter regex. - files := []string{ - "emp/", - "tee/", - "test-foo.tc", - "test-foo.tc", - "test-bar.tc", - "test-sam.tc", - "Test-que.tc", - "test-brett", - "test--abc.tc", - "test---xyz.tc", - "test-bool.TC", - "--test-gvs.tc", - " test-pew.tc", - "dir/test_baz.tc", - "dir/testsnap.tc", - "dir/test-luk.tc", - "dir/nest/test-ok.tc", - "dir/dip/diz/goog/test-pack.tc", - "dir/dip/diz/wobble/thud/test-cas.e", - "dir/dip/diz/wobble/thud/test-cas.tc", - } - want := []string{ - "dir/dip/diz/goog/test-pack.tc", - "dir/dip/diz/wobble/thud/test-cas.tc", - "dir/nest/test-ok.tc", - "dir/test-luk.tc", - "test-bar.tc", - "test-foo.tc", - "test-sam.tc", - } - - for _, item := range files { - if strings.HasSuffix(item, "/") { - // This item is a directory, create it. - if err := os.MkdirAll(filepath.Join(td, item), 0755); err != nil { - t.Fatalf("error making directory: %v", err) - } - } else { - // This item is a file, create the directory and touch file. - // Create directory in which file should be created - fullDirPath := filepath.Join(td, filepath.Dir(item)) - if err := os.MkdirAll(fullDirPath, 0755); err != nil { - t.Fatalf("error making directory: %v", err) - } - // Create file with full path to file. - touch(t, filepath.Join(td, item)) - } - } - - testFilter := regexp.MustCompile(`^test-[^-].+\.tc$`) - got, err := Search(td, testFilter) - if err != nil { - t.Errorf("search error: %v", err) - } - - if !reflect.DeepEqual(got, want) { - t.Errorf("Found %#v; want %#v", got, want) - } -} diff --git a/test/runtimes/proctor/lib/nodejs.go b/test/runtimes/proctor/lib/nodejs.go deleted file mode 100644 index 320597aa5..000000000 --- a/test/runtimes/proctor/lib/nodejs.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package lib - -import ( - "os/exec" - "path/filepath" - "regexp" -) - -var nodejsTestRegEx = regexp.MustCompile(`^test-[^-].+\.js$`) - -// Location of nodejs tests relative to working dir. -const nodejsTestDir = "test" - -// nodejsRunner implements TestRunner for NodeJS. -type nodejsRunner struct{} - -var _ TestRunner = nodejsRunner{} - -// ListTests implements TestRunner.ListTests. -func (nodejsRunner) ListTests() ([]string, error) { - testSlice, err := Search(nodejsTestDir, nodejsTestRegEx) - if err != nil { - return nil, err - } - return testSlice, nil -} - -// TestCmds implements TestRunner.TestCmds. -func (nodejsRunner) TestCmds(tests []string) []*exec.Cmd { - args := append([]string{filepath.Join("tools", "test.py"), "--timeout=180"}, tests...) - return []*exec.Cmd{exec.Command("/usr/bin/python", args...)} -} diff --git a/test/runtimes/proctor/lib/php.go b/test/runtimes/proctor/lib/php.go deleted file mode 100644 index b67a60a97..000000000 --- a/test/runtimes/proctor/lib/php.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package lib - -import ( - "os/exec" - "regexp" - "strings" -) - -var phpTestRegEx = regexp.MustCompile(`^.+\.phpt$`) - -// phpRunner implements TestRunner for PHP. -type phpRunner struct{} - -var _ TestRunner = phpRunner{} - -// ListTests implements TestRunner.ListTests. -func (phpRunner) ListTests() ([]string, error) { - testSlice, err := Search(".", phpTestRegEx) - if err != nil { - return nil, err - } - return testSlice, nil -} - -// TestCmds implements TestRunner.TestCmds. -func (phpRunner) TestCmds(tests []string) []*exec.Cmd { - args := []string{"test", "TESTS=" + strings.Join(tests, " ")} - return []*exec.Cmd{exec.Command("make", args...)} -} diff --git a/test/runtimes/proctor/lib/python.go b/test/runtimes/proctor/lib/python.go deleted file mode 100644 index 429bfd850..000000000 --- a/test/runtimes/proctor/lib/python.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package lib - -import ( - "fmt" - "os" - "os/exec" - "strings" -) - -// pythonRunner implements TestRunner for Python. -type pythonRunner struct{} - -var _ TestRunner = pythonRunner{} - -// ListTests implements TestRunner.ListTests. -func (pythonRunner) ListTests() ([]string, error) { - args := []string{"-m", "test", "--list-tests"} - cmd := exec.Command("./python", args...) - cmd.Stderr = os.Stderr - out, err := cmd.Output() - if err != nil { - return nil, fmt.Errorf("failed to list: %v", err) - } - var toolSlice []string - for _, test := range strings.Split(string(out), "\n") { - toolSlice = append(toolSlice, test) - } - return toolSlice, nil -} - -// TestCmds implements TestRunner.TestCmds. -func (pythonRunner) TestCmds(tests []string) []*exec.Cmd { - args := append([]string{"-m", "test"}, tests...) - return []*exec.Cmd{exec.Command("./python", args...)} -} diff --git a/test/runtimes/proctor/main.go b/test/runtimes/proctor/main.go deleted file mode 100644 index 81cb68381..000000000 --- a/test/runtimes/proctor/main.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Binary proctor runs the test for a particular runtime. It is meant to be -// included in Docker images for all runtime tests. -package main - -import ( - "flag" - "fmt" - "log" - "os" - "strings" - "syscall" - - "gvisor.dev/gvisor/test/runtimes/proctor/lib" -) - -var ( - runtime = flag.String("runtime", "", "name of runtime") - list = flag.Bool("list", false, "list all available tests") - testNames = flag.String("tests", "", "run a subset of the available tests") - pause = flag.Bool("pause", false, "cause container to pause indefinitely, reaping any zombie children") -) - -// setNumFilesLimit changes the NOFILE soft rlimit if it is too high. -func setNumFilesLimit() error { - // In docker containers, the default value of the NOFILE limit is - // 1048576. A few runtime tests (e.g. python:test_subprocess) - // enumerates all possible file descriptors and these tests can fail by - // timeout if the NOFILE limit is too high. On gVisor, syscalls are - // slower so these tests will need even more time to pass. - const nofile = 32768 - rLimit := syscall.Rlimit{} - err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit) - if err != nil { - return fmt.Errorf("failed to get RLIMIT_NOFILE: %v", err) - } - if rLimit.Cur > nofile { - rLimit.Cur = nofile - err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit) - if err != nil { - return fmt.Errorf("failed to set RLIMIT_NOFILE: %v", err) - } - } - return nil -} - -func main() { - flag.Parse() - - if *pause { - lib.PauseAndReap() - panic("pauseAndReap should never return") - } - - if *runtime == "" { - log.Fatalf("runtime flag must be provided") - } - - tr, err := lib.TestRunnerForRuntime(*runtime) - if err != nil { - log.Fatalf("%v", err) - } - - // List tests. - if *list { - tests, err := tr.ListTests() - if err != nil { - log.Fatalf("failed to list tests: %v", err) - } - for _, test := range tests { - fmt.Println(test) - } - return - } - - var tests []string - if *testNames == "" { - // Run every test. - tests, err = tr.ListTests() - if err != nil { - log.Fatalf("failed to get all tests: %v", err) - } - } else { - // Run subset of test. - tests = strings.Split(*testNames, ",") - } - - if err := setNumFilesLimit(); err != nil { - log.Fatalf("%v", err) - } - - // Run tests. - cmds := tr.TestCmds(tests) - for _, cmd := range cmds { - cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr - if err := cmd.Run(); err != nil { - log.Fatalf("FAIL: %v", err) - } - } -} diff --git a/test/runtimes/runner/BUILD b/test/runtimes/runner/BUILD deleted file mode 100644 index 70cc01594..000000000 --- a/test/runtimes/runner/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -load("//tools:defs.bzl", "go_binary") - -package(licenses = ["notice"]) - -go_binary( - name = "runner", - testonly = 1, - srcs = ["main.go"], - visibility = ["//test/runtimes:__pkg__"], - deps = ["//test/runtimes/runner/lib"], -) diff --git a/test/runtimes/runner/lib/BUILD b/test/runtimes/runner/lib/BUILD deleted file mode 100644 index d308f41b0..000000000 --- a/test/runtimes/runner/lib/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "lib", - testonly = 1, - srcs = ["lib.go"], - visibility = ["//test/runtimes/runner:__pkg__"], - deps = [ - "//pkg/log", - "//pkg/test/dockerutil", - "//pkg/test/testutil", - ], -) - -go_test( - name = "lib_test", - size = "small", - srcs = ["exclude_test.go"], - library = ":lib", -) diff --git a/test/runtimes/runner/lib/exclude_test.go b/test/runtimes/runner/lib/exclude_test.go deleted file mode 100644 index f996e895b..000000000 --- a/test/runtimes/runner/lib/exclude_test.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package lib - -import ( - "flag" - "os" - "testing" -) - -var excludeFile = flag.String("exclude_file", "", "file to test (standard format)") - -func TestMain(m *testing.M) { - flag.Parse() - os.Exit(m.Run()) -} - -// Test that the exclude file parses without error. -func TestExcludelist(t *testing.T) { - ex, err := getExcludes(*excludeFile) - if err != nil { - t.Fatalf("error parsing exclude file: %v", err) - } - if *excludeFile != "" && len(ex) == 0 { - t.Errorf("got empty excludes for file %q", *excludeFile) - } -} diff --git a/test/runtimes/runner/lib/lib.go b/test/runtimes/runner/lib/lib.go deleted file mode 100644 index 41616e5e0..000000000 --- a/test/runtimes/runner/lib/lib.go +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package lib provides utilities for runner. -package lib - -import ( - "context" - "encoding/csv" - "fmt" - "io" - "os" - "sort" - "strings" - "testing" - "time" - - "gvisor.dev/gvisor/pkg/log" - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/pkg/test/testutil" -) - -// RunTests is a helper that is called by main. It exists so that we can run -// defered functions before exiting. It returns an exit code that should be -// passed to os.Exit. -func RunTests(lang, image, excludeFile string, partitionNum, totalPartitions, batchSize int, timeout time.Duration) int { - if partitionNum <= 0 || totalPartitions <= 0 || partitionNum > totalPartitions { - fmt.Fprintf(os.Stderr, "invalid partition %d of %d", partitionNum, totalPartitions) - return 1 - } - - // Get tests to exclude.. - excludes, err := getExcludes(excludeFile) - if err != nil { - fmt.Fprintf(os.Stderr, "Error getting exclude list: %s\n", err.Error()) - return 1 - } - - // Construct the shared docker instance. - ctx := context.Background() - d := dockerutil.MakeContainer(ctx, testutil.DefaultLogger(lang)) - defer d.CleanUp(ctx) - - if err := testutil.TouchShardStatusFile(); err != nil { - fmt.Fprintf(os.Stderr, "error touching status shard file: %v\n", err) - return 1 - } - - // Get a slice of tests to run. This will also start a single Docker - // container that will be used to run each test. The final test will - // stop the Docker container. - tests, err := getTests(ctx, d, lang, image, partitionNum, totalPartitions, batchSize, timeout, excludes) - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err.Error()) - return 1 - } - - m := testing.MainStart(testDeps{}, tests, nil, nil) - return m.Run() -} - -// getTests executes all tests as table tests. -func getTests(ctx context.Context, d *dockerutil.Container, lang, image string, partitionNum, totalPartitions, batchSize int, timeout time.Duration, excludes map[string]struct{}) ([]testing.InternalTest, error) { - // Start the container. - opts := dockerutil.RunOpts{ - Image: fmt.Sprintf("runtimes/%s", image), - } - d.CopyFiles(&opts, "/proctor", "test/runtimes/proctor/proctor") - if err := d.Spawn(ctx, opts, "/proctor/proctor", "--pause"); err != nil { - return nil, fmt.Errorf("docker run failed: %v", err) - } - - // Get a list of all tests in the image. - list, err := d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", lang, "--list") - if err != nil { - return nil, fmt.Errorf("docker exec failed: %v", err) - } - - // Calculate a subset of tests to run corresponding to the current - // shard. - tests := strings.Fields(list) - sort.Strings(tests) - - partitionSize := len(tests) / totalPartitions - if partitionNum == totalPartitions { - tests = tests[(partitionNum-1)*partitionSize:] - } else { - tests = tests[(partitionNum-1)*partitionSize : partitionNum*partitionSize] - } - - indices, err := testutil.TestIndicesForShard(len(tests)) - if err != nil { - return nil, fmt.Errorf("TestsForShard() failed: %v", err) - } - - var itests []testing.InternalTest - for i := 0; i < len(indices); i += batchSize { - var tcs []string - end := i + batchSize - if end > len(indices) { - end = len(indices) - } - for _, tc := range indices[i:end] { - // Add test if not excluded. - if _, ok := excludes[tests[tc]]; ok { - log.Infof("Skipping test case %s\n", tests[tc]) - continue - } - tcs = append(tcs, tests[tc]) - } - itests = append(itests, testing.InternalTest{ - Name: strings.Join(tcs, ", "), - F: func(t *testing.T) { - var ( - now = time.Now() - done = make(chan struct{}) - output string - err error - ) - - state, err := d.Status(ctx) - if err != nil { - t.Fatalf("Could not find container status: %v", err) - } - if !state.Running { - t.Fatalf("container is not running: state = %s", state.Status) - } - - go func() { - output, err = d.Exec(ctx, dockerutil.ExecOpts{}, "/proctor/proctor", "--runtime", lang, "--tests", strings.Join(tcs, ",")) - close(done) - }() - - select { - case <-done: - if err == nil { - fmt.Printf("PASS: (%v) %d tests passed\n", time.Since(now), len(tcs)) - return - } - t.Errorf("FAIL: (%v):\nBatch:\n%s\nOutput:\n%s\n", time.Since(now), strings.Join(tcs, "\n"), output) - case <-time.After(timeout): - t.Errorf("TIMEOUT: (%v):\nBatch:\n%s\nOutput:\n%s\n", time.Since(now), strings.Join(tcs, "\n"), output) - } - }, - }) - } - - return itests, nil -} - -// getBlacklist reads the exclude file and returns a set of test names to -// exclude. -func getExcludes(excludeFile string) (map[string]struct{}, error) { - excludes := make(map[string]struct{}) - if excludeFile == "" { - return excludes, nil - } - f, err := os.Open(excludeFile) - if err != nil { - return nil, err - } - defer f.Close() - - r := csv.NewReader(f) - - // First line is header. Skip it. - if _, err := r.Read(); err != nil { - return nil, err - } - - for { - record, err := r.Read() - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - excludes[record[0]] = struct{}{} - } - return excludes, nil -} - -// testDeps implements testing.testDeps (an unexported interface), and is -// required to use testing.MainStart. -type testDeps struct{} - -func (f testDeps) MatchString(a, b string) (bool, error) { return a == b, nil } -func (f testDeps) StartCPUProfile(io.Writer) error { return nil } -func (f testDeps) StopCPUProfile() {} -func (f testDeps) WriteProfileTo(string, io.Writer, int) error { return nil } -func (f testDeps) ImportPath() string { return "" } -func (f testDeps) StartTestLog(io.Writer) {} -func (f testDeps) StopTestLog() error { return nil } diff --git a/test/runtimes/runner/main.go b/test/runtimes/runner/main.go deleted file mode 100644 index 5b3443e36..000000000 --- a/test/runtimes/runner/main.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Binary runner runs the runtime tests in a Docker container. -package main - -import ( - "flag" - "fmt" - "os" - "time" - - "gvisor.dev/gvisor/test/runtimes/runner/lib" -) - -var ( - lang = flag.String("lang", "", "language runtime to test") - image = flag.String("image", "", "docker image with runtime tests") - excludeFile = flag.String("exclude_file", "", "file containing list of tests to exclude, in CSV format with fields: test name, bug id, comment") - partition = flag.Int("partition", 1, "partition number, this is 1-indexed") - totalPartitions = flag.Int("total_partitions", 1, "total number of partitions") - batchSize = flag.Int("batch", 50, "number of test cases run in one command") - timeout = flag.Duration("timeout", 90*time.Minute, "batch timeout") -) - -func main() { - flag.Parse() - if *lang == "" || *image == "" { - fmt.Fprintf(os.Stderr, "lang and image flags must not be empty\n") - os.Exit(1) - } - os.Exit(lib.RunTests(*lang, *image, *excludeFile, *partition, *totalPartitions, *batchSize, *timeout)) -} |