fuse: add benchmarking support for FUSE

This change adds the following:
-  Add support for containerizing syscall tests for FUSE
-  Mount tmpfs in the container so we can run benchmarks against it
-  Run the server in a background process
-  benchmarks for fuse syscall

Co-authored-by: Ridwan Sharif <ridwanmsharif@google.com>

13 files changed, 618 insertions, 14 deletions

diff --git a/test/e2e/integration_test.go b/test/e2e/integration_test.go
index 809244bab..0c82e98d4 100644
--- a/test/e2e/integration_test.go
+++ b/test/e2e/integration_test.go
@@ -168,10 +168,10 @@ func TestCheckpointRestore(t *testing.T) {
 	}
 
 	// TODO(gvisor.dev/issue/3373): Remove after implementing.
-	if usingVFS2, err := dockerutil.UsingVFS2(); usingVFS2 {
-		t.Skip("CheckpointRestore not implemented in VFS2.")
-	} else if err != nil {
+	if usingVFS2, err := dockerutil.UsingVFS2(); err != nil {
 		t.Fatalf("failed to read config for runtime %s: %v", dockerutil.Runtime(), err)
+	} else if usingVFS2 {
+		t.Skip("CheckpointRestore not implemented in VFS2.")
 	}
 
 	ctx := context.Background()
diff --git a/test/fuse/BUILD b/test/fuse/BUILD
index a1b29aa33..02498b3a1 100644
--- a/test/fuse/BUILD
+++ b/test/fuse/BUILD
@@ -51,3 +51,49 @@ syscall_test(
     fuse = "True",
     test = "//test/fuse/linux:readdir_test",
 )
+
+
+syscall_test(
+    size = "large",
+    add_overlay = True,
+    debug = False,
+    setup_command = "'./server-bin mountpoint'",
+    test = "//test/fuse/benchmark:stat_benchmark",
+    use_image = "basic/fuse",
+)
+
+syscall_test(
+    size = "large",
+    add_overlay = True,
+    debug = False,
+    setup_command = "'./server-bin mountpoint'",
+    test = "//test/fuse/benchmark:open_benchmark",
+    use_image = "basic/fuse",
+)
+
+syscall_test(
+    size = "large",
+    add_overlay = True,
+    debug = False,
+    setup_command = "'./server-bin mountpoint'",
+    test = "//test/fuse/benchmark:read_benchmark",
+    use_image = "basic/fuse",
+)
+
+syscall_test(
+    size = "large",
+    add_overlay = True,
+    debug = False,
+    setup_command = "'./server-bin mountpoint'",
+    test = "//test/fuse/benchmark:symlink_benchmark",
+    use_image = "basic/fuse",
+)
+
+syscall_test(
+    size = "large",
+    add_overlay = True,
+    debug = False,
+    setup_command = "'./server-bin mountpoint'",
+    test = "//test/fuse/benchmark:mkdir_benchmark",
+    use_image = "basic/fuse",
+)
diff --git a/test/fuse/benchmark/BUILD b/test/fuse/benchmark/BUILD
new file mode 100644
index 000000000..16369d99b
--- /dev/null
+++ b/test/fuse/benchmark/BUILD
@@ -0,0 +1,91 @@
+load("//tools:defs.bzl", "cc_binary", "gbenchmark", "gtest")
+
+package(
+    default_visibility = ["//:sandbox"],
+    licenses = ["notice"],
+)
+
+cc_binary(
+    name = "stat_benchmark",
+    testonly = 1,
+    srcs = [
+        "stat_benchmark.cc",
+    ],
+    deps = [
+        gbenchmark,
+        gtest,
+        "//test/util:fs_util",
+        "//test/util:temp_path",
+        "//test/util:test_main",
+        "//test/util:test_util",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_binary(
+    name = "open_benchmark",
+    testonly = 1,
+    srcs = [
+        "open_benchmark.cc",
+    ],
+    deps = [
+        gbenchmark,
+        gtest,
+        "//test/util:fs_util",
+        "//test/util:temp_path",
+        "//test/util:test_main",
+        "//test/util:test_util",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_binary(
+    name = "read_benchmark",
+    testonly = 1,
+    srcs = [
+        "read_benchmark.cc",
+    ],
+    deps = [
+        gbenchmark,
+        gtest,
+        "//test/util:fs_util",
+        "//test/util:temp_path",
+        "//test/util:test_main",
+        "//test/util:test_util",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_binary(
+    name = "symlink_benchmark",
+    testonly = 1,
+    srcs = [
+        "symlink_benchmark.cc",
+    ],
+    deps = [
+        gbenchmark,
+        gtest,
+        "//test/util:fs_util",
+        "//test/util:temp_path",
+        "//test/util:test_main",
+        "//test/util:test_util",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_binary(
+    name = "mkdir_benchmark",
+    testonly = 1,
+    srcs = [
+        "mkdir_benchmark.cc",
+    ],
+    deps = [
+        gbenchmark,
+        gtest,
+        "//test/util:fs_util",
+        "//test/util:temp_path",
+        "//test/util:test_main",
+        "//test/util:test_util",
+        "@com_google_absl//absl/strings",
+    ],
+)
diff --git a/test/fuse/benchmark/mkdir_benchmark.cc b/test/fuse/benchmark/mkdir_benchmark.cc
new file mode 100644
index 000000000..30759603e
--- /dev/null
+++ b/test/fuse/benchmark/mkdir_benchmark.cc
@@ -0,0 +1,51 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "absl/strings/str_cat.h"
+#include "benchmark/benchmark.h"
+#include "gtest/gtest.h"
+#include "test/util/fs_util.h"
+#include "test/util/temp_path.h"
+#include "test/util/test_util.h"
+
+namespace gvisor {
+namespace testing {
+
+namespace {
+
+void BM_Mkdir(benchmark::State& state) {
+  const char* fuse_prefix = getenv("TEST_FUSEPRE");
+  ASSERT_NE(fuse_prefix, nullptr);
+
+  const TempPath top_dir = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir());
+  std::string dir_path = top_dir.path();
+
+  int index = 0;
+  for (auto t : state) {
+    const std::string new_dir_path = absl::StrCat(dir_path, index);
+    ASSERT_THAT(mkdir(new_dir_path.c_str(), 0777), SyscallSucceeds());
+    index++;
+  }
+}
+
+BENCHMARK(BM_Mkdir)->Range(1, 128)->UseRealTime();
+
+}  // namespace
+
+}  // namespace testing
+}  // namespace gvisor
diff --git a/test/fuse/benchmark/open_benchmark.cc b/test/fuse/benchmark/open_benchmark.cc
new file mode 100644
index 000000000..11c1c1c80
--- /dev/null
+++ b/test/fuse/benchmark/open_benchmark.cc
@@ -0,0 +1,60 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fcntl.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "benchmark/benchmark.h"
+#include "gtest/gtest.h"
+#include "test/util/fs_util.h"
+#include "test/util/logging.h"
+#include "test/util/temp_path.h"
+
+namespace gvisor {
+namespace testing {
+
+namespace {
+
+void BM_Open(benchmark::State& state) {
+  const char* fuse_prefix = getenv("TEST_FUSEPRE");
+  ASSERT_NE(fuse_prefix, nullptr);
+
+  const int size = state.range(0);
+  std::vector<TempPath> cache;
+  for (int i = 0; i < size; i++) {
+    auto path = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateFile());
+    cache.emplace_back(std::move(path));
+  }
+
+  unsigned int seed = 1;
+  for (auto _ : state) {
+    const int chosen = rand_r(&seed) % size;
+    const std::string file_path = JoinPath(fuse_prefix, cache[chosen].path());
+    int fd = open(file_path.c_str(), O_RDONLY);
+    TEST_CHECK(fd != -1);
+    close(fd);
+  }
+}
+
+BENCHMARK(BM_Open)->Range(1, 128)->UseRealTime();
+
+}  // namespace
+
+}  // namespace testing
+}  // namespace gvisor
diff --git a/test/fuse/benchmark/read_benchmark.cc b/test/fuse/benchmark/read_benchmark.cc
new file mode 100644
index 000000000..2106b7d5a
--- /dev/null
+++ b/test/fuse/benchmark/read_benchmark.cc
@@ -0,0 +1,57 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fcntl.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "benchmark/benchmark.h"
+#include "gtest/gtest.h"
+#include "test/util/fs_util.h"
+#include "test/util/logging.h"
+#include "test/util/temp_path.h"
+#include "test/util/test_util.h"
+
+namespace gvisor {
+namespace testing {
+
+namespace {
+
+void BM_Read(benchmark::State& state) {
+  const char* fuse_prefix = getenv("TEST_FUSEPRE");
+  ASSERT_NE(fuse_prefix, nullptr);
+
+  const int size = state.range(0);
+  const std::string contents(size, 0);
+  auto path = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateFileWith(
+      GetAbsoluteTestTmpdir(), contents, TempPath::kDefaultFileMode));
+  FileDescriptor fd = ASSERT_NO_ERRNO_AND_VALUE(
+      Open(JoinPath(fuse_prefix, path.path()), O_RDONLY));
+
+  std::vector<char> buf(size);
+  for (auto _ : state) {
+    TEST_CHECK(PreadFd(fd.get(), buf.data(), buf.size(), 0) == size);
+  }
+
+  state.SetBytesProcessed(static_cast<int64_t>(size) *
+                          static_cast<int64_t>(state.iterations()));
+}
+
+BENCHMARK(BM_Read)->Range(1, 1 << 26)->UseRealTime();
+
+}  // namespace
+
+}  // namespace testing
+}  // namespace gvisor
diff --git a/test/fuse/benchmark/stat_benchmark.cc b/test/fuse/benchmark/stat_benchmark.cc
new file mode 100644
index 000000000..d2ab6a706
--- /dev/null
+++ b/test/fuse/benchmark/stat_benchmark.cc
@@ -0,0 +1,65 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "gtest/gtest.h"
+#include "absl/strings/str_cat.h"
+#include "benchmark/benchmark.h"
+#include "test/util/fs_util.h"
+#include "test/util/temp_path.h"
+#include "test/util/test_util.h"
+
+namespace gvisor {
+namespace testing {
+
+namespace {
+
+// Creates a file in a nested directory hierarchy at least `depth` directories
+// deep, and stats that file multiple times.
+void BM_Stat(benchmark::State& state) {
+  const char* fuse_prefix = getenv("TEST_FUSEPRE");
+  ASSERT_NE(fuse_prefix, nullptr);
+
+  // Create nested directories with given depth.
+  int depth = state.range(0);
+  const TempPath top_dir = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir());
+  std::string dir_path = top_dir.path();
+
+  while (depth-- > 0) {
+    // Don't use TempPath because it will make paths too long to use.
+    //
+    // The top_dir destructor will clean up this whole tree.
+    dir_path = JoinPath(dir_path, absl::StrCat(depth));
+    ASSERT_NO_ERRNO(Mkdir(dir_path, 0755));
+  }
+
+  // Create the file that will be stat'd.
+  const TempPath file =
+      ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateFileIn(dir_path));
+  std::string file_path = JoinPath(fuse_prefix, file.path());
+  struct stat st;
+  for (auto _ : state) {
+    ASSERT_THAT(stat(file_path.c_str(), &st), SyscallSucceeds());
+  }
+}
+
+BENCHMARK(BM_Stat)->Range(1, 100)->UseRealTime();
+
+}  // namespace
+
+}  // namespace testing
+}  // namespace gvisor
diff --git a/test/fuse/benchmark/symlink_benchmark.cc b/test/fuse/benchmark/symlink_benchmark.cc
new file mode 100644
index 000000000..363b9a976
--- /dev/null
+++ b/test/fuse/benchmark/symlink_benchmark.cc
@@ -0,0 +1,60 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "absl/strings/str_cat.h"
+#include "benchmark/benchmark.h"
+#include "gtest/gtest.h"
+#include "test/util/fs_util.h"
+#include "test/util/temp_path.h"
+#include "test/util/test_util.h"
+
+namespace gvisor {
+namespace testing {
+
+namespace {
+
+void BM_Symlink(benchmark::State& state) {
+  char* fuse_prefix = getenv("TEST_FUSEPRE");
+  ASSERT_NE(fuse_prefix, nullptr);
+  const TempPath top_dir = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir());
+  std::string dir_path = top_dir.path();
+
+  const int size = state.range(0);
+  std::vector<TempPath> cache;
+  for (int i = 0; i < size; i++) {
+    auto path = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateFile());
+    cache.emplace_back(std::move(path));
+  }
+
+  int index = 0;
+  unsigned int seed = 1;
+  for (auto t : state) {
+    const int chosen = rand_r(&seed) % size;
+    const std::string symlink_path = absl::StrCat(fuse_prefix, dir_path, index);
+    ASSERT_THAT(symlink(cache[chosen].path().c_str(), symlink_path.c_str()),
+                SyscallSucceeds());
+    index++;
+  }
+}
+
+BENCHMARK(BM_Symlink)->Range(1, 128)->UseRealTime();
+
+}  // namespace
+
+}  // namespace testing
+}  // namespace gvisor
diff --git a/test/image/BUILD b/test/image/BUILD
index e749e47d4..e270c52ac 100644
--- a/test/image/BUILD
+++ b/test/image/BUILD
@@ -24,6 +24,7 @@ go_test(
     deps = [
         "//pkg/test/dockerutil",
         "//pkg/test/testutil",
+        "@com_github_docker_docker//api/types/mount:go_default_library",
     ],
 )
 
diff --git a/test/image/image_test.go b/test/image/image_test.go
index ac6186688..6b5928ef0 100644
--- a/test/image/image_test.go
+++ b/test/image/image_test.go
@@ -33,6 +33,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/docker/docker/api/types/mount"
 	"gvisor.dev/gvisor/pkg/test/dockerutil"
 	"gvisor.dev/gvisor/pkg/test/testutil"
 )
@@ -63,6 +64,57 @@ func TestHelloWorld(t *testing.T) {
 	}
 }
 
+// Test that the FUSE container is set up and being used properly.
+func TestFUSEInContainer(t *testing.T) {
+	if usingFUSE, err := dockerutil.UsingFUSE(); err != nil {
+		t.Fatalf("failed to read config for runtime %s: %v", dockerutil.Runtime(), err)
+	} else if !usingFUSE {
+		t.Skip("FUSE not being used.")
+	}
+
+	ctx := context.Background()
+	d := dockerutil.MakeContainer(ctx, t)
+	defer d.CleanUp(ctx)
+
+	tmpDir := "/tmpDir/"
+	// Run the basic container.
+	err := d.Spawn(ctx, dockerutil.RunOpts{
+		Image:      "basic/fuse",
+		Privileged: true,
+		CapAdd:     []string{"CAP_SYS_ADMIN"},
+
+		// Mount a tmpfs directory for benchmark.
+		Mounts: []mount.Mount{
+			{
+				Type:     mount.TypeTmpfs,
+				Target:   tmpDir,
+				ReadOnly: false,
+			},
+		},
+	}, "sleep", "1000")
+	if err != nil {
+		t.Fatalf("docker spawn failed: %v", err)
+	}
+
+	out, err := d.Exec(ctx, dockerutil.ExecOpts{
+		Privileged: true,
+	}, "/bin/sh", "-c", "ls")
+	if err != nil {
+		t.Fatalf("docker exec failed: %v, message %s", err, out)
+	}
+	if !strings.Contains(out, "server-bin") {
+		t.Fatalf("docker didn't find server binary: got %s", out)
+	}
+
+	// Run the server.
+	out, err = d.Exec(ctx, dockerutil.ExecOpts{
+		Privileged: true,
+	}, "/bin/sh", "-c", "./server-bin mountpoint")
+	if err != nil {
+		t.Fatalf("docker exec failed: %v, message %s", err, out)
+	}
+}
+
 func runHTTPRequest(port int) error {
 	url := fmt.Sprintf("http://localhost:%d/not-found", port)
 	resp, err := http.Get(url)
diff --git a/test/runner/BUILD b/test/runner/BUILD
index 582d2946d..049c26081 100644
--- a/test/runner/BUILD
+++ b/test/runner/BUILD
@@ -11,11 +11,14 @@ go_binary(
     ],
     visibility = ["//:sandbox"],
     deps = [
+        "//pkg/context",
         "//pkg/log",
+        "//pkg/test/dockerutil",
         "//pkg/test/testutil",
         "//runsc/specutils",
         "//test/runner/gtest",
         "//test/uds",
+        "@com_github_docker_docker//api/types/mount:go_default_library",
         "@com_github_opencontainers_runtime_spec//specs-go:go_default_library",
         "@com_github_syndtr_gocapability//capability:go_default_library",
         "@org_golang_x_sys//unix:go_default_library",
diff --git a/test/runner/defs.bzl b/test/runner/defs.bzl
index 032ebd04e..9dc955c77 100644
--- a/test/runner/defs.bzl
+++ b/test/runner/defs.bzl
@@ -57,6 +57,8 @@ def _syscall_test(
         platform,
         use_tmpfs,
         tags,
+        use_image = "",
+        setup_command = "",
         network = "none",
         file_access = "exclusive",
         overlay = False,
@@ -79,6 +81,8 @@ def _syscall_test(
             name += "_fuse"
     if network != "none":
         name += "_" + network + "net"
+    if use_image != "":
+        name += "_container"
 
     # Apply all tags.
     if tags == None:
@@ -107,6 +111,8 @@ def _syscall_test(
         "--platform=" + platform,
         "--network=" + network,
         "--use-tmpfs=" + str(use_tmpfs),
+        "--use-image=" + use_image,
+        "--setup-command=" + setup_command,
         "--file-access=" + file_access,
         "--overlay=" + str(overlay),
         "--add-uds-tree=" + str(add_uds_tree),
@@ -132,6 +138,8 @@ def syscall_test(
         shard_count = 5,
         size = "small",
         use_tmpfs = False,
+        use_image = "",
+        setup_command = "",
         add_overlay = False,
         add_uds_tree = False,
         add_hostinet = False,
@@ -146,6 +154,8 @@ def syscall_test(
       shard_count: shards for defined tests.
       size: the defined test size.
       use_tmpfs: use tmpfs in the defined tests.
+      use_image: use specified docker image in the defined tests.
+      setup_command: command to set up the docker container. Should be used when ise_image is.
       add_overlay: add an overlay test.
       add_uds_tree: add a UDS test.
       add_hostinet: add a hostinet test.
@@ -178,8 +188,26 @@ def syscall_test(
         vfs2 = True,
         fuse = fuse,
     )
+
+    if use_image != "":
+        # Run the test in the container specified.
+        _syscall_test(
+            test = test,
+            shard_count = shard_count,
+            size = size,
+            platform = default_platform,
+            use_tmpfs = use_tmpfs,
+            use_image = use_image,
+            setup_command = setup_command,
+            add_uds_tree = add_uds_tree,
+            tags = platforms[default_platform] + vfs2_tags,
+            vfs2 = True,
+            fuse = True,
+        )
+
     if fuse:
         # Only generate *_vfs2_fuse target if fuse parameter is enabled.
+        # The rest of the targets don't support FUSE as of yet.
         return
 
     _syscall_test(
diff --git a/test/runner/runner.go b/test/runner/runner.go
index 22d535f8d..fe501c4c7 100644
--- a/test/runner/runner.go
+++ b/test/runner/runner.go
@@ -23,16 +23,20 @@ import (
 	"os"
 	"os/exec"
 	"os/signal"
+	"path"
 	"path/filepath"
 	"strings"
 	"syscall"
 	"testing"
 	"time"
 
+	"github.com/docker/docker/api/types/mount"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/syndtr/gocapability/capability"
 	"golang.org/x/sys/unix"
+	"gvisor.dev/gvisor/pkg/context"
 	"gvisor.dev/gvisor/pkg/log"
+	"gvisor.dev/gvisor/pkg/test/dockerutil"
 	"gvisor.dev/gvisor/pkg/test/testutil"
 	"gvisor.dev/gvisor/runsc/specutils"
 	"gvisor.dev/gvisor/test/runner/gtest"
@@ -40,17 +44,19 @@ import (
 )
 
 var (
-	debug      = flag.Bool("debug", false, "enable debug logs")
-	strace     = flag.Bool("strace", false, "enable strace logs")
-	platform   = flag.String("platform", "ptrace", "platform to run on")
-	network    = flag.String("network", "none", "network stack to run on (sandbox, host, none)")
-	useTmpfs   = flag.Bool("use-tmpfs", false, "mounts tmpfs for /tmp")
-	fileAccess = flag.String("file-access", "exclusive", "mounts root in exclusive or shared mode")
-	overlay    = flag.Bool("overlay", false, "wrap filesystem mounts with writable tmpfs overlay")
-	vfs2       = flag.Bool("vfs2", false, "enable VFS2")
-	fuse       = flag.Bool("fuse", false, "enable FUSE")
-	parallel   = flag.Bool("parallel", false, "run tests in parallel")
-	runscPath  = flag.String("runsc", "", "path to runsc binary")
+	debug        = flag.Bool("debug", false, "enable debug logs")
+	strace       = flag.Bool("strace", false, "enable strace logs")
+	platform     = flag.String("platform", "ptrace", "platform to run on")
+	network      = flag.String("network", "none", "network stack to run on (sandbox, host, none)")
+	useTmpfs     = flag.Bool("use-tmpfs", false, "mounts tmpfs for /tmp")
+	useImage     = flag.String("use-image", "", "container image to use for test. Path relative to //images")
+	setupCommand = flag.String("setup-command", "", "command to run before running the test to set up container environment")
+	fileAccess   = flag.String("file-access", "exclusive", "mounts root in exclusive or shared mode")
+	overlay      = flag.Bool("overlay", false, "wrap filesystem mounts with writable tmpfs overlay")
+	vfs2         = flag.Bool("vfs2", false, "enable VFS2")
+	fuse         = flag.Bool("fuse", false, "enable FUSE")
+	parallel     = flag.Bool("parallel", false, "run tests in parallel")
+	runscPath    = flag.String("runsc", "", "path to runsc binary")
 
 	addUDSTree = flag.Bool("add-uds-tree", false, "expose a tree of UDS utilities for use in tests")
 )
@@ -313,8 +319,92 @@ func setupUDSTree(spec *specs.Spec) (cleanup func(), err error) {
 	return cleanup, nil
 }
 
+func runTestCaseInContainer(testBin string, tc gtest.TestCase, image string, t *testing.T) {
+	if usingFUSE, err := dockerutil.UsingFUSE(); err != nil {
+		t.Fatalf("failed to read config for runtime %s: %v", dockerutil.Runtime(), err)
+	} else if !usingFUSE {
+		t.Skip("FUSE not being used.")
+	}
+
+	ctx := context.Background()
+	d := dockerutil.MakeContainer(ctx, t)
+	defer d.CleanUp(ctx)
+
+	// Run the basic container.
+	tmpDir := "/tmpDir"
+	testBinDir := "/testDir"
+	opts := dockerutil.RunOpts{
+		Image:      image,
+		Privileged: true,
+		CapAdd:     []string{"CAP_SYS_ADMIN"},
+
+		// Mount a tmpfs directory to use when benchmarking.
+		Mounts: []mount.Mount{
+			{
+				Type:     mount.TypeTmpfs,
+				Target:   tmpDir,
+				ReadOnly: false,
+			},
+		},
+		Env: []string{
+			fmt.Sprintf("TEST_TMPDIR=%s", tmpDir),
+			fmt.Sprintf("TEST_FUSEPRE=%s", "/fus/mountpoint"),
+		},
+	}
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("Getwd run failed: %v", err)
+	}
+
+	wdPathToTestBin := strings.TrimPrefix(testBin, wd)
+	containerTestBin := path.Join(testBinDir, path.Base(wdPathToTestBin))
+	d.CopyFiles(&opts, testBinDir, wdPathToTestBin)
+	if err := d.CopyErr(); err != nil {
+		t.Fatalf("Copy failed %v", err)
+	}
+
+	err = d.Spawn(ctx, opts, "sleep", "1000")
+	if err != nil {
+		t.Fatalf("docker run failed: %v", err)
+	}
+
+	// Run the server setup command.
+	if *setupCommand != "" {
+		out, err := d.Exec(ctx, dockerutil.ExecOpts{
+			Privileged: true,
+		}, "/bin/sh", "-c", *setupCommand)
+		if err != nil {
+			t.Fatalf("docker exec failed: %v with output %v", err, out)
+		}
+	}
+
+	cmd := "chmod +x " + containerTestBin
+	out, err := d.Exec(ctx, dockerutil.ExecOpts{
+		Privileged: true,
+	}, "/bin/sh", "-c", cmd)
+	if err != nil {
+		t.Fatalf("docker exec failed: %v with output %v", err, out)
+	}
+
+	cmd = containerTestBin + " " + strings.Join(tc.Args(), " ")
+	out, err = d.Exec(ctx, dockerutil.ExecOpts{
+		Privileged: true,
+	}, "/bin/sh", "-c", cmd)
+	if err != nil {
+		t.Fatalf("docker exec failed: %v with output %v", err, out)
+	}
+
+	fmt.Print(out)
+	return
+}
+
 // runsTestCaseRunsc runs the test case in runsc.
 func runTestCaseRunsc(testBin string, tc gtest.TestCase, t *testing.T) {
+	if *useImage != "" {
+		runTestCaseInContainer(testBin, tc, *useImage, t)
+		return
+	}
+
 	// Run a new container with the test executable and filter for the
 	// given test suite and name.
 	spec := testutil.NewSpecWithArgs(append([]string{testBin}, tc.Args()...)...)