diff options
| -rw-r--r-- | Makefile | 4 | ||||
| -rw-r--r-- | images/basic/fuse/Dockerfile | 15 | ||||
| -rw-r--r-- | pkg/sentry/fs/g3doc/fuse.md | 47 | ||||
| -rw-r--r-- | pkg/test/dockerutil/container.go | 10 | ||||
| -rw-r--r-- | pkg/test/dockerutil/dockerutil.go | 20 | ||||
| -rwxr-xr-x | scripts/common_build.sh | 5 | ||||
| -rw-r--r-- | test/e2e/integration_test.go | 6 | ||||
| -rw-r--r-- | test/fuse/BUILD | 46 | ||||
| -rw-r--r-- | test/fuse/benchmark/BUILD | 91 | ||||
| -rw-r--r-- | test/fuse/benchmark/mkdir_benchmark.cc | 51 | ||||
| -rw-r--r-- | test/fuse/benchmark/open_benchmark.cc | 60 | ||||
| -rw-r--r-- | test/fuse/benchmark/read_benchmark.cc | 57 | ||||
| -rw-r--r-- | test/fuse/benchmark/stat_benchmark.cc | 65 | ||||
| -rw-r--r-- | test/fuse/benchmark/symlink_benchmark.cc | 60 | ||||
| -rw-r--r-- | test/image/BUILD | 1 | ||||
| -rw-r--r-- | test/image/image_test.go | 52 | ||||
| -rw-r--r-- | test/runner/BUILD | 3 | ||||
| -rw-r--r-- | test/runner/defs.bzl | 28 | ||||
| -rw-r--r-- | test/runner/runner.go | 112 |
19 files changed, 717 insertions, 16 deletions
@@ -341,6 +341,10 @@ RUNTIME_BIN := $(RUNTIME_DIR)/runsc RUNTIME_LOG_DIR := $(RUNTIME_DIR)/logs RUNTIME_LOGS := $(RUNTIME_LOG_DIR)/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND% +ifeq (,$(RUNTIME_NAME)) +RUNTIME_NAME := $(RUNTIME) +endif + dev: ## Installs a set of local runtimes. Requires sudo. @$(call submake,refresh ARGS="--net-raw") @$(call submake,configure RUNTIME_NAME="$(RUNTIME)" ARGS="--net-raw") diff --git a/images/basic/fuse/Dockerfile b/images/basic/fuse/Dockerfile new file mode 100644 index 000000000..9e88aa2c5 --- /dev/null +++ b/images/basic/fuse/Dockerfile @@ -0,0 +1,15 @@ +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update +RUN apt-get install -y build-essential git pkg-config fuse3 libfuse3-3 libfuse3-dev strace + +WORKDIR /fus + +RUN mkdir -pv mountpoint +RUN git clone https://github.com/libfuse/libfuse + +RUN gcc -Wall ./libfuse/example/passthrough.c `pkg-config fuse3 --cflags --libs` -o server-bin + +CMD ["bash"] diff --git a/pkg/sentry/fs/g3doc/fuse.md b/pkg/sentry/fs/g3doc/fuse.md index 2ca84dd74..496e339ce 100644 --- a/pkg/sentry/fs/g3doc/fuse.md +++ b/pkg/sentry/fs/g3doc/fuse.md @@ -254,6 +254,53 @@ I/O syscalls like `read(2)`, `write(2)` and `mmap(2)`. - `FUSE_BMAP`: Old address space API for block defrag. Probably not needed. - `FUSE_NOTIFY_REPLY`: [TODO: what does this do?] +## Benchmark FUSE + +FUSE benchmark makes FUSE syscall inside docker container to make sure required +environment conditions are met - such as having the right libraries to start a +FUSE server. + +### Setup + +To run benchmark: + +1. Make sure you have `Docker` installed. +2. Download all docker images `make load-all-images`. +3. Config `runsc` docker runtime to have VFS2 and FUSE supported. +(e.g. `make configure RUNTIME=runsc ARGS="--vfs2 --fuse ..." ...`) + +You should now have a runtime with the following options configured in +`/etc/docker/daemon.json` +``` +"runsc": { + "path": "path/to/your/runsc", + "runtimeArgs": [ + "--vfs2", + "--fuse" + ... + ] + } +``` + +### Running benchmarks +With above setup, benchmark can be run with following command +``` +bazel test --test_output=all --cache_test_results=no --test_arg=-test.bench= //path/to:target +``` +For example: if you want to run stat test +``` +bazel test --test_output=all --cache_test_results=no --test_arg=-test.bench= //test/fuse:open_benchmark_runsc_ptrace_vfs2_fuse_container +``` + +Note: +- test target need to have `vfs2_fuse_container` to run in container with `vfs2` and `fuse` enabled +- `test_output` set to `all` to view the result in terminal +- `--cache_test_results` set to `no` to avoid cached benchmark + +### Use your fuse server + +To use your own FUSE server, change the `images/basic/fuse/Dockerfile` to compile your FUSE server into the container and name it `server-bin`. + # References - [fuse(4) Linux manual page](https://www.man7.org/linux/man-pages/man4/fuse.4.html) diff --git a/pkg/test/dockerutil/container.go b/pkg/test/dockerutil/container.go index 64d17f661..727be26b2 100644 --- a/pkg/test/dockerutil/container.go +++ b/pkg/test/dockerutil/container.go @@ -136,6 +136,11 @@ func MakeNativeContainer(ctx context.Context, logger testutil.Logger) *Container } } +// Runtime returns the runtime of the container. +func (c *Container) Runtime() string { + return c.runtime +} + // AddProfile adds a profile to this container. func (c *Container) AddProfile(p Profile) { c.profiles = append(c.profiles, p) @@ -541,3 +546,8 @@ func (c *Container) CleanUp(ctx context.Context) { // Forget all mounts. c.mounts = nil } + +// CopyErr returns the error that happened during copy. +func (c *Container) CopyErr() error { + return c.copyErr +} diff --git a/pkg/test/dockerutil/dockerutil.go b/pkg/test/dockerutil/dockerutil.go index 7027df1a5..a2d7e8c85 100644 --- a/pkg/test/dockerutil/dockerutil.go +++ b/pkg/test/dockerutil/dockerutil.go @@ -121,6 +121,26 @@ func UsingVFS2() (bool, error) { return false, nil } +// UsingFUSE returns true if the 'runtime' has the fuse flag set. +func UsingFUSE() (bool, error) { + rMap, err := runtimeMap() + if err != nil { + return false, err + } + + list, ok := rMap["runtimeArgs"].([]interface{}) + if !ok { + return false, fmt.Errorf("unexpected format: %v", rMap) + } + + for _, element := range list { + if element == "--fuse" { + return true, nil + } + } + return false, nil +} + func runtimeMap() (map[string]interface{}, error) { // Read the configuration data; the file must exist. configBytes, err := ioutil.ReadFile(*config) diff --git a/scripts/common_build.sh b/scripts/common_build.sh index d4a6c4908..6874e56f9 100755 --- a/scripts/common_build.sh +++ b/scripts/common_build.sh @@ -109,8 +109,9 @@ function collect_logs() { } function find_branch_name() { - git branch --show-current \ + (git branch --show-current \ || git rev-parse HEAD \ || bazel info workspace \ - | xargs basename + | xargs basename) \ + | tr '/' '-' } diff --git a/test/e2e/integration_test.go b/test/e2e/integration_test.go index 809244bab..0c82e98d4 100644 --- a/test/e2e/integration_test.go +++ b/test/e2e/integration_test.go @@ -168,10 +168,10 @@ func TestCheckpointRestore(t *testing.T) { } // TODO(gvisor.dev/issue/3373): Remove after implementing. - if usingVFS2, err := dockerutil.UsingVFS2(); usingVFS2 { - t.Skip("CheckpointRestore not implemented in VFS2.") - } else if err != nil { + if usingVFS2, err := dockerutil.UsingVFS2(); err != nil { t.Fatalf("failed to read config for runtime %s: %v", dockerutil.Runtime(), err) + } else if usingVFS2 { + t.Skip("CheckpointRestore not implemented in VFS2.") } ctx := context.Background() diff --git a/test/fuse/BUILD b/test/fuse/BUILD index a1b29aa33..02498b3a1 100644 --- a/test/fuse/BUILD +++ b/test/fuse/BUILD @@ -51,3 +51,49 @@ syscall_test( fuse = "True", test = "//test/fuse/linux:readdir_test", ) + + +syscall_test( + size = "large", + add_overlay = True, + debug = False, + setup_command = "'./server-bin mountpoint'", + test = "//test/fuse/benchmark:stat_benchmark", + use_image = "basic/fuse", +) + +syscall_test( + size = "large", + add_overlay = True, + debug = False, + setup_command = "'./server-bin mountpoint'", + test = "//test/fuse/benchmark:open_benchmark", + use_image = "basic/fuse", +) + +syscall_test( + size = "large", + add_overlay = True, + debug = False, + setup_command = "'./server-bin mountpoint'", + test = "//test/fuse/benchmark:read_benchmark", + use_image = "basic/fuse", +) + +syscall_test( + size = "large", + add_overlay = True, + debug = False, + setup_command = "'./server-bin mountpoint'", + test = "//test/fuse/benchmark:symlink_benchmark", + use_image = "basic/fuse", +) + +syscall_test( + size = "large", + add_overlay = True, + debug = False, + setup_command = "'./server-bin mountpoint'", + test = "//test/fuse/benchmark:mkdir_benchmark", + use_image = "basic/fuse", +) diff --git a/test/fuse/benchmark/BUILD b/test/fuse/benchmark/BUILD new file mode 100644 index 000000000..16369d99b --- /dev/null +++ b/test/fuse/benchmark/BUILD @@ -0,0 +1,91 @@ +load("//tools:defs.bzl", "cc_binary", "gbenchmark", "gtest") + +package( + default_visibility = ["//:sandbox"], + licenses = ["notice"], +) + +cc_binary( + name = "stat_benchmark", + testonly = 1, + srcs = [ + "stat_benchmark.cc", + ], + deps = [ + gbenchmark, + gtest, + "//test/util:fs_util", + "//test/util:temp_path", + "//test/util:test_main", + "//test/util:test_util", + "@com_google_absl//absl/strings", + ], +) + +cc_binary( + name = "open_benchmark", + testonly = 1, + srcs = [ + "open_benchmark.cc", + ], + deps = [ + gbenchmark, + gtest, + "//test/util:fs_util", + "//test/util:temp_path", + "//test/util:test_main", + "//test/util:test_util", + "@com_google_absl//absl/strings", + ], +) + +cc_binary( + name = "read_benchmark", + testonly = 1, + srcs = [ + "read_benchmark.cc", + ], + deps = [ + gbenchmark, + gtest, + "//test/util:fs_util", + "//test/util:temp_path", + "//test/util:test_main", + "//test/util:test_util", + "@com_google_absl//absl/strings", + ], +) + +cc_binary( + name = "symlink_benchmark", + testonly = 1, + srcs = [ + "symlink_benchmark.cc", + ], + deps = [ + gbenchmark, + gtest, + "//test/util:fs_util", + "//test/util:temp_path", + "//test/util:test_main", + "//test/util:test_util", + "@com_google_absl//absl/strings", + ], +) + +cc_binary( + name = "mkdir_benchmark", + testonly = 1, + srcs = [ + "mkdir_benchmark.cc", + ], + deps = [ + gbenchmark, + gtest, + "//test/util:fs_util", + "//test/util:temp_path", + "//test/util:test_main", + "//test/util:test_util", + "@com_google_absl//absl/strings", + ], +) diff --git a/test/fuse/benchmark/mkdir_benchmark.cc b/test/fuse/benchmark/mkdir_benchmark.cc new file mode 100644 index 000000000..30759603e --- /dev/null +++ b/test/fuse/benchmark/mkdir_benchmark.cc @@ -0,0 +1,51 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "absl/strings/str_cat.h" +#include "benchmark/benchmark.h" +#include "gtest/gtest.h" +#include "test/util/fs_util.h" +#include "test/util/temp_path.h" +#include "test/util/test_util.h" + +namespace gvisor { +namespace testing { + +namespace { + +void BM_Mkdir(benchmark::State& state) { + const char* fuse_prefix = getenv("TEST_FUSEPRE"); + ASSERT_NE(fuse_prefix, nullptr); + + const TempPath top_dir = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir()); + std::string dir_path = top_dir.path(); + + int index = 0; + for (auto t : state) { + const std::string new_dir_path = absl::StrCat(dir_path, index); + ASSERT_THAT(mkdir(new_dir_path.c_str(), 0777), SyscallSucceeds()); + index++; + } +} + +BENCHMARK(BM_Mkdir)->Range(1, 128)->UseRealTime(); + +} // namespace + +} // namespace testing +} // namespace gvisor diff --git a/test/fuse/benchmark/open_benchmark.cc b/test/fuse/benchmark/open_benchmark.cc new file mode 100644 index 000000000..11c1c1c80 --- /dev/null +++ b/test/fuse/benchmark/open_benchmark.cc @@ -0,0 +1,60 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <fcntl.h> +#include <stdlib.h> +#include <unistd.h> + +#include <memory> +#include <string> +#include <vector> + +#include "benchmark/benchmark.h" +#include "gtest/gtest.h" +#include "test/util/fs_util.h" +#include "test/util/logging.h" +#include "test/util/temp_path.h" + +namespace gvisor { +namespace testing { + +namespace { + +void BM_Open(benchmark::State& state) { + const char* fuse_prefix = getenv("TEST_FUSEPRE"); + ASSERT_NE(fuse_prefix, nullptr); + + const int size = state.range(0); + std::vector<TempPath> cache; + for (int i = 0; i < size; i++) { + auto path = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateFile()); + cache.emplace_back(std::move(path)); + } + + unsigned int seed = 1; + for (auto _ : state) { + const int chosen = rand_r(&seed) % size; + const std::string file_path = JoinPath(fuse_prefix, cache[chosen].path()); + int fd = open(file_path.c_str(), O_RDONLY); + TEST_CHECK(fd != -1); + close(fd); + } +} + +BENCHMARK(BM_Open)->Range(1, 128)->UseRealTime(); + +} // namespace + +} // namespace testing +} // namespace gvisor diff --git a/test/fuse/benchmark/read_benchmark.cc b/test/fuse/benchmark/read_benchmark.cc new file mode 100644 index 000000000..2106b7d5a --- /dev/null +++ b/test/fuse/benchmark/read_benchmark.cc @@ -0,0 +1,57 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <fcntl.h> +#include <stdlib.h> +#include <sys/stat.h> +#include <unistd.h> + +#include "benchmark/benchmark.h" +#include "gtest/gtest.h" +#include "test/util/fs_util.h" +#include "test/util/logging.h" +#include "test/util/temp_path.h" +#include "test/util/test_util.h" + +namespace gvisor { +namespace testing { + +namespace { + +void BM_Read(benchmark::State& state) { + const char* fuse_prefix = getenv("TEST_FUSEPRE"); + ASSERT_NE(fuse_prefix, nullptr); + + const int size = state.range(0); + const std::string contents(size, 0); + auto path = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateFileWith( + GetAbsoluteTestTmpdir(), contents, TempPath::kDefaultFileMode)); + FileDescriptor fd = ASSERT_NO_ERRNO_AND_VALUE( + Open(JoinPath(fuse_prefix, path.path()), O_RDONLY)); + + std::vector<char> buf(size); + for (auto _ : state) { + TEST_CHECK(PreadFd(fd.get(), buf.data(), buf.size(), 0) == size); + } + + state.SetBytesProcessed(static_cast<int64_t>(size) * + static_cast<int64_t>(state.iterations())); +} + +BENCHMARK(BM_Read)->Range(1, 1 << 26)->UseRealTime(); + +} // namespace + +} // namespace testing +} // namespace gvisor diff --git a/test/fuse/benchmark/stat_benchmark.cc b/test/fuse/benchmark/stat_benchmark.cc new file mode 100644 index 000000000..d2ab6a706 --- /dev/null +++ b/test/fuse/benchmark/stat_benchmark.cc @@ -0,0 +1,65 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "gtest/gtest.h" +#include "absl/strings/str_cat.h" +#include "benchmark/benchmark.h" +#include "test/util/fs_util.h" +#include "test/util/temp_path.h" +#include "test/util/test_util.h" + +namespace gvisor { +namespace testing { + +namespace { + +// Creates a file in a nested directory hierarchy at least `depth` directories +// deep, and stats that file multiple times. +void BM_Stat(benchmark::State& state) { + const char* fuse_prefix = getenv("TEST_FUSEPRE"); + ASSERT_NE(fuse_prefix, nullptr); + + // Create nested directories with given depth. + int depth = state.range(0); + const TempPath top_dir = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir()); + std::string dir_path = top_dir.path(); + + while (depth-- > 0) { + // Don't use TempPath because it will make paths too long to use. + // + // The top_dir destructor will clean up this whole tree. + dir_path = JoinPath(dir_path, absl::StrCat(depth)); + ASSERT_NO_ERRNO(Mkdir(dir_path, 0755)); + } + + // Create the file that will be stat'd. + const TempPath file = + ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateFileIn(dir_path)); + std::string file_path = JoinPath(fuse_prefix, file.path()); + struct stat st; + for (auto _ : state) { + ASSERT_THAT(stat(file_path.c_str(), &st), SyscallSucceeds()); + } +} + +BENCHMARK(BM_Stat)->Range(1, 100)->UseRealTime(); + +} // namespace + +} // namespace testing +} // namespace gvisor diff --git a/test/fuse/benchmark/symlink_benchmark.cc b/test/fuse/benchmark/symlink_benchmark.cc new file mode 100644 index 000000000..363b9a976 --- /dev/null +++ b/test/fuse/benchmark/symlink_benchmark.cc @@ -0,0 +1,60 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "absl/strings/str_cat.h" +#include "benchmark/benchmark.h" +#include "gtest/gtest.h" +#include "test/util/fs_util.h" +#include "test/util/temp_path.h" +#include "test/util/test_util.h" + +namespace gvisor { +namespace testing { + +namespace { + +void BM_Symlink(benchmark::State& state) { + char* fuse_prefix = getenv("TEST_FUSEPRE"); + ASSERT_NE(fuse_prefix, nullptr); + const TempPath top_dir = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateDir()); + std::string dir_path = top_dir.path(); + + const int size = state.range(0); + std::vector<TempPath> cache; + for (int i = 0; i < size; i++) { + auto path = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateFile()); + cache.emplace_back(std::move(path)); + } + + int index = 0; + unsigned int seed = 1; + for (auto t : state) { + const int chosen = rand_r(&seed) % size; + const std::string symlink_path = absl::StrCat(fuse_prefix, dir_path, index); + ASSERT_THAT(symlink(cache[chosen].path().c_str(), symlink_path.c_str()), + SyscallSucceeds()); + index++; + } +} + +BENCHMARK(BM_Symlink)->Range(1, 128)->UseRealTime(); + +} // namespace + +} // namespace testing +} // namespace gvisor diff --git a/test/image/BUILD b/test/image/BUILD index e749e47d4..e270c52ac 100644 --- a/test/image/BUILD +++ b/test/image/BUILD @@ -24,6 +24,7 @@ go_test( deps = [ "//pkg/test/dockerutil", "//pkg/test/testutil", + "@com_github_docker_docker//api/types/mount:go_default_library", ], ) diff --git a/test/image/image_test.go b/test/image/image_test.go index ac6186688..6b5928ef0 100644 --- a/test/image/image_test.go +++ b/test/image/image_test.go @@ -33,6 +33,7 @@ import ( "testing" "time" + "github.com/docker/docker/api/types/mount" "gvisor.dev/gvisor/pkg/test/dockerutil" "gvisor.dev/gvisor/pkg/test/testutil" ) @@ -63,6 +64,57 @@ func TestHelloWorld(t *testing.T) { } } +// Test that the FUSE container is set up and being used properly. +func TestFUSEInContainer(t *testing.T) { + if usingFUSE, err := dockerutil.UsingFUSE(); err != nil { + t.Fatalf("failed to read config for runtime %s: %v", dockerutil.Runtime(), err) + } else if !usingFUSE { + t.Skip("FUSE not being used.") + } + + ctx := context.Background() + d := dockerutil.MakeContainer(ctx, t) + defer d.CleanUp(ctx) + + tmpDir := "/tmpDir/" + // Run the basic container. + err := d.Spawn(ctx, dockerutil.RunOpts{ + Image: "basic/fuse", + Privileged: true, + CapAdd: []string{"CAP_SYS_ADMIN"}, + + // Mount a tmpfs directory for benchmark. + Mounts: []mount.Mount{ + { + Type: mount.TypeTmpfs, + Target: tmpDir, + ReadOnly: false, + }, + }, + }, "sleep", "1000") + if err != nil { + t.Fatalf("docker spawn failed: %v", err) + } + + out, err := d.Exec(ctx, dockerutil.ExecOpts{ + Privileged: true, + }, "/bin/sh", "-c", "ls") + if err != nil { + t.Fatalf("docker exec failed: %v, message %s", err, out) + } + if !strings.Contains(out, "server-bin") { + t.Fatalf("docker didn't find server binary: got %s", out) + } + + // Run the server. + out, err = d.Exec(ctx, dockerutil.ExecOpts{ + Privileged: true, + }, "/bin/sh", "-c", "./server-bin mountpoint") + if err != nil { + t.Fatalf("docker exec failed: %v, message %s", err, out) + } +} + func runHTTPRequest(port int) error { url := fmt.Sprintf("http://localhost:%d/not-found", port) resp, err := http.Get(url) diff --git a/test/runner/BUILD b/test/runner/BUILD index 582d2946d..049c26081 100644 --- a/test/runner/BUILD +++ b/test/runner/BUILD @@ -11,11 +11,14 @@ go_binary( ], visibility = ["//:sandbox"], deps = [ + "//pkg/context", "//pkg/log", + "//pkg/test/dockerutil", "//pkg/test/testutil", "//runsc/specutils", "//test/runner/gtest", "//test/uds", + "@com_github_docker_docker//api/types/mount:go_default_library", "@com_github_opencontainers_runtime_spec//specs-go:go_default_library", "@com_github_syndtr_gocapability//capability:go_default_library", "@org_golang_x_sys//unix:go_default_library", diff --git a/test/runner/defs.bzl b/test/runner/defs.bzl index 032ebd04e..9dc955c77 100644 --- a/test/runner/defs.bzl +++ b/test/runner/defs.bzl @@ -57,6 +57,8 @@ def _syscall_test( platform, use_tmpfs, tags, + use_image = "", + setup_command = "", network = "none", file_access = "exclusive", overlay = False, @@ -79,6 +81,8 @@ def _syscall_test( name += "_fuse" if network != "none": name += "_" + network + "net" + if use_image != "": + name += "_container" # Apply all tags. if tags == None: @@ -107,6 +111,8 @@ def _syscall_test( "--platform=" + platform, "--network=" + network, "--use-tmpfs=" + str(use_tmpfs), + "--use-image=" + use_image, + "--setup-command=" + setup_command, "--file-access=" + file_access, "--overlay=" + str(overlay), "--add-uds-tree=" + str(add_uds_tree), @@ -132,6 +138,8 @@ def syscall_test( shard_count = 5, size = "small", use_tmpfs = False, + use_image = "", + setup_command = "", add_overlay = False, add_uds_tree = False, add_hostinet = False, @@ -146,6 +154,8 @@ def syscall_test( shard_count: shards for defined tests. size: the defined test size. use_tmpfs: use tmpfs in the defined tests. + use_image: use specified docker image in the defined tests. + setup_command: command to set up the docker container. Should be used when ise_image is. add_overlay: add an overlay test. add_uds_tree: add a UDS test. add_hostinet: add a hostinet test. @@ -178,8 +188,26 @@ def syscall_test( vfs2 = True, fuse = fuse, ) + + if use_image != "": + # Run the test in the container specified. + _syscall_test( + test = test, + shard_count = shard_count, + size = size, + platform = default_platform, + use_tmpfs = use_tmpfs, + use_image = use_image, + setup_command = setup_command, + add_uds_tree = add_uds_tree, + tags = platforms[default_platform] + vfs2_tags, + vfs2 = True, + fuse = True, + ) + if fuse: # Only generate *_vfs2_fuse target if fuse parameter is enabled. + # The rest of the targets don't support FUSE as of yet. return _syscall_test( diff --git a/test/runner/runner.go b/test/runner/runner.go index 22d535f8d..fe501c4c7 100644 --- a/test/runner/runner.go +++ b/test/runner/runner.go @@ -23,16 +23,20 @@ import ( "os" "os/exec" "os/signal" + "path" "path/filepath" "strings" "syscall" "testing" "time" + "github.com/docker/docker/api/types/mount" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/syndtr/gocapability/capability" "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/context" "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/test/dockerutil" "gvisor.dev/gvisor/pkg/test/testutil" "gvisor.dev/gvisor/runsc/specutils" "gvisor.dev/gvisor/test/runner/gtest" @@ -40,17 +44,19 @@ import ( ) var ( - debug = flag.Bool("debug", false, "enable debug logs") - strace = flag.Bool("strace", false, "enable strace logs") - platform = flag.String("platform", "ptrace", "platform to run on") - network = flag.String("network", "none", "network stack to run on (sandbox, host, none)") - useTmpfs = flag.Bool("use-tmpfs", false, "mounts tmpfs for /tmp") - fileAccess = flag.String("file-access", "exclusive", "mounts root in exclusive or shared mode") - overlay = flag.Bool("overlay", false, "wrap filesystem mounts with writable tmpfs overlay") - vfs2 = flag.Bool("vfs2", false, "enable VFS2") - fuse = flag.Bool("fuse", false, "enable FUSE") - parallel = flag.Bool("parallel", false, "run tests in parallel") - runscPath = flag.String("runsc", "", "path to runsc binary") + debug = flag.Bool("debug", false, "enable debug logs") + strace = flag.Bool("strace", false, "enable strace logs") + platform = flag.String("platform", "ptrace", "platform to run on") + network = flag.String("network", "none", "network stack to run on (sandbox, host, none)") + useTmpfs = flag.Bool("use-tmpfs", false, "mounts tmpfs for /tmp") + useImage = flag.String("use-image", "", "container image to use for test. Path relative to //images") + setupCommand = flag.String("setup-command", "", "command to run before running the test to set up container environment") + fileAccess = flag.String("file-access", "exclusive", "mounts root in exclusive or shared mode") + overlay = flag.Bool("overlay", false, "wrap filesystem mounts with writable tmpfs overlay") + vfs2 = flag.Bool("vfs2", false, "enable VFS2") + fuse = flag.Bool("fuse", false, "enable FUSE") + parallel = flag.Bool("parallel", false, "run tests in parallel") + runscPath = flag.String("runsc", "", "path to runsc binary") addUDSTree = flag.Bool("add-uds-tree", false, "expose a tree of UDS utilities for use in tests") ) @@ -313,8 +319,92 @@ func setupUDSTree(spec *specs.Spec) (cleanup func(), err error) { return cleanup, nil } +func runTestCaseInContainer(testBin string, tc gtest.TestCase, image string, t *testing.T) { + if usingFUSE, err := dockerutil.UsingFUSE(); err != nil { + t.Fatalf("failed to read config for runtime %s: %v", dockerutil.Runtime(), err) + } else if !usingFUSE { + t.Skip("FUSE not being used.") + } + + ctx := context.Background() + d := dockerutil.MakeContainer(ctx, t) + defer d.CleanUp(ctx) + + // Run the basic container. + tmpDir := "/tmpDir" + testBinDir := "/testDir" + opts := dockerutil.RunOpts{ + Image: image, + Privileged: true, + CapAdd: []string{"CAP_SYS_ADMIN"}, + + // Mount a tmpfs directory to use when benchmarking. + Mounts: []mount.Mount{ + { + Type: mount.TypeTmpfs, + Target: tmpDir, + ReadOnly: false, + }, + }, + Env: []string{ + fmt.Sprintf("TEST_TMPDIR=%s", tmpDir), + fmt.Sprintf("TEST_FUSEPRE=%s", "/fus/mountpoint"), + }, + } + wd, err := os.Getwd() + if err != nil { + t.Fatalf("Getwd run failed: %v", err) + } + + wdPathToTestBin := strings.TrimPrefix(testBin, wd) + containerTestBin := path.Join(testBinDir, path.Base(wdPathToTestBin)) + d.CopyFiles(&opts, testBinDir, wdPathToTestBin) + if err := d.CopyErr(); err != nil { + t.Fatalf("Copy failed %v", err) + } + + err = d.Spawn(ctx, opts, "sleep", "1000") + if err != nil { + t.Fatalf("docker run failed: %v", err) + } + + // Run the server setup command. + if *setupCommand != "" { + out, err := d.Exec(ctx, dockerutil.ExecOpts{ + Privileged: true, + }, "/bin/sh", "-c", *setupCommand) + if err != nil { + t.Fatalf("docker exec failed: %v with output %v", err, out) + } + } + + cmd := "chmod +x " + containerTestBin + out, err := d.Exec(ctx, dockerutil.ExecOpts{ + Privileged: true, + }, "/bin/sh", "-c", cmd) + if err != nil { + t.Fatalf("docker exec failed: %v with output %v", err, out) + } + + cmd = containerTestBin + " " + strings.Join(tc.Args(), " ") + out, err = d.Exec(ctx, dockerutil.ExecOpts{ + Privileged: true, + }, "/bin/sh", "-c", cmd) + if err != nil { + t.Fatalf("docker exec failed: %v with output %v", err, out) + } + + fmt.Print(out) + return +} + // runsTestCaseRunsc runs the test case in runsc. func runTestCaseRunsc(testBin string, tc gtest.TestCase, t *testing.T) { + if *useImage != "" { + runTestCaseInContainer(testBin, tc, *useImage, t) + return + } + // Run a new container with the test executable and filter for the // given test suite and name. spec := testutil.NewSpecWithArgs(append([]string{testBin}, tc.Args()...)...) |