diff options
Diffstat (limited to 'test/root')
-rw-r--r-- | test/root/BUILD | 44 | ||||
-rw-r--r-- | test/root/cgroup_test.go | 232 | ||||
-rw-r--r-- | test/root/chroot_test.go | 142 | ||||
-rw-r--r-- | test/root/crictl_test.go | 295 | ||||
-rw-r--r-- | test/root/main_test.go | 49 | ||||
-rw-r--r-- | test/root/oom_score_adj_test.go | 376 | ||||
-rw-r--r-- | test/root/root.go | 21 | ||||
-rw-r--r-- | test/root/testdata/BUILD | 19 | ||||
-rw-r--r-- | test/root/testdata/busybox.go | 32 | ||||
-rw-r--r-- | test/root/testdata/containerd_config.go | 39 | ||||
-rw-r--r-- | test/root/testdata/httpd.go | 32 | ||||
-rw-r--r-- | test/root/testdata/httpd_mount_paths.go | 53 | ||||
-rw-r--r-- | test/root/testdata/sandbox.go | 30 | ||||
-rw-r--r-- | test/root/testdata/simple.go | 41 |
14 files changed, 1405 insertions, 0 deletions
diff --git a/test/root/BUILD b/test/root/BUILD new file mode 100644 index 000000000..d5dd9bca2 --- /dev/null +++ b/test/root/BUILD @@ -0,0 +1,44 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +package(licenses = ["notice"]) + +go_library( + name = "root", + srcs = ["root.go"], + importpath = "gvisor.dev/gvisor/test/root", +) + +go_test( + name = "root_test", + size = "small", + srcs = [ + "cgroup_test.go", + "chroot_test.go", + "crictl_test.go", + "main_test.go", + "oom_score_adj_test.go", + ], + data = [ + "//runsc", + ], + embed = [":root"], + tags = [ + # Requires docker and runsc to be configured before the test runs. + # Also test only runs as root. + "manual", + "local", + ], + visibility = ["//:sandbox"], + deps = [ + "//runsc/boot", + "//runsc/cgroup", + "//runsc/container", + "//runsc/criutil", + "//runsc/dockerutil", + "//runsc/specutils", + "//runsc/testutil", + "//test/root/testdata", + "@com_github_opencontainers_runtime-spec//specs-go:go_default_library", + "@com_github_syndtr_gocapability//capability:go_default_library", + ], +) diff --git a/test/root/cgroup_test.go b/test/root/cgroup_test.go new file mode 100644 index 000000000..76f1e4f2a --- /dev/null +++ b/test/root/cgroup_test.go @@ -0,0 +1,232 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package root + +import ( + "bufio" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + + "gvisor.dev/gvisor/runsc/cgroup" + "gvisor.dev/gvisor/runsc/dockerutil" + "gvisor.dev/gvisor/runsc/testutil" +) + +func verifyPid(pid int, path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + + var gots []int + scanner := bufio.NewScanner(f) + for scanner.Scan() { + got, err := strconv.Atoi(scanner.Text()) + if err != nil { + return err + } + if got == pid { + return nil + } + gots = append(gots, got) + } + if scanner.Err() != nil { + return scanner.Err() + } + return fmt.Errorf("got: %s, want: %d", gots, pid) +} + +// TestCgroup sets cgroup options and checks that cgroup was properly configured. +func TestCgroup(t *testing.T) { + if err := dockerutil.Pull("alpine"); err != nil { + t.Fatal("docker pull failed:", err) + } + d := dockerutil.MakeDocker("cgroup-test") + + // This is not a comprehensive list of attributes. + // + // Note that we are specifically missing cpusets, which fail if specified. + // In any case, it's unclear if cpusets can be reliably tested here: these + // are often run on a single core virtual machine, and there is only a single + // CPU available in our current set, and every container's set. + attrs := []struct { + arg string + ctrl string + file string + want string + skipIfNotFound bool + }{ + { + arg: "--cpu-shares=1000", + ctrl: "cpu", + file: "cpu.shares", + want: "1000", + }, + { + arg: "--cpu-period=2000", + ctrl: "cpu", + file: "cpu.cfs_period_us", + want: "2000", + }, + { + arg: "--cpu-quota=3000", + ctrl: "cpu", + file: "cpu.cfs_quota_us", + want: "3000", + }, + { + arg: "--kernel-memory=100MB", + ctrl: "memory", + file: "memory.kmem.limit_in_bytes", + want: "104857600", + }, + { + arg: "--memory=1GB", + ctrl: "memory", + file: "memory.limit_in_bytes", + want: "1073741824", + }, + { + arg: "--memory-reservation=500MB", + ctrl: "memory", + file: "memory.soft_limit_in_bytes", + want: "524288000", + }, + { + arg: "--memory-swap=2GB", + ctrl: "memory", + file: "memory.memsw.limit_in_bytes", + want: "2147483648", + skipIfNotFound: true, // swap may be disabled on the machine. + }, + { + arg: "--memory-swappiness=5", + ctrl: "memory", + file: "memory.swappiness", + want: "5", + }, + { + arg: "--blkio-weight=750", + ctrl: "blkio", + file: "blkio.weight", + want: "750", + }, + } + + args := make([]string, 0, len(attrs)) + for _, attr := range attrs { + args = append(args, attr.arg) + } + + args = append(args, "alpine", "sleep", "10000") + if err := d.Run(args...); err != nil { + t.Fatal("docker create failed:", err) + } + defer d.CleanUp() + + gid, err := d.ID() + if err != nil { + t.Fatalf("Docker.ID() failed: %v", err) + } + t.Logf("cgroup ID: %s", gid) + + // Check list of attributes defined above. + for _, attr := range attrs { + path := filepath.Join("/sys/fs/cgroup", attr.ctrl, "docker", gid, attr.file) + out, err := ioutil.ReadFile(path) + if err != nil { + if os.IsNotExist(err) && attr.skipIfNotFound { + t.Logf("skipped %s/%s", attr.ctrl, attr.file) + continue + } + t.Fatalf("failed to read %q: %v", path, err) + } + if got := strings.TrimSpace(string(out)); got != attr.want { + t.Errorf("arg: %q, cgroup attribute %s/%s, got: %q, want: %q", attr.arg, attr.ctrl, attr.file, got, attr.want) + } + } + + // Check that sandbox is inside cgroup. + controllers := []string{ + "blkio", + "cpu", + "cpuset", + "memory", + "net_cls", + "net_prio", + "devices", + "freezer", + "perf_event", + "pids", + "systemd", + } + pid, err := d.SandboxPid() + if err != nil { + t.Fatalf("SandboxPid: %v", err) + } + for _, ctrl := range controllers { + path := filepath.Join("/sys/fs/cgroup", ctrl, "docker", gid, "cgroup.procs") + if err := verifyPid(pid, path); err != nil { + t.Errorf("cgroup control %q processes: %v", ctrl, err) + } + } +} + +func TestCgroupParent(t *testing.T) { + if err := dockerutil.Pull("alpine"); err != nil { + t.Fatal("docker pull failed:", err) + } + d := dockerutil.MakeDocker("cgroup-test") + + parent := testutil.RandomName("runsc") + if err := d.Run("--cgroup-parent", parent, "alpine", "sleep", "10000"); err != nil { + t.Fatal("docker create failed:", err) + } + defer d.CleanUp() + gid, err := d.ID() + if err != nil { + t.Fatalf("Docker.ID() failed: %v", err) + } + t.Logf("cgroup ID: %s", gid) + + // Check that sandbox is inside cgroup. + pid, err := d.SandboxPid() + if err != nil { + t.Fatalf("SandboxPid: %v", err) + } + + // Finds cgroup for the sandbox's parent process to check that cgroup is + // created in the right location relative to the parent. + cmd := fmt.Sprintf("grep PPid: /proc/%d/status | sed 's/PPid:\\s//'", pid) + ppid, err := exec.Command("bash", "-c", cmd).CombinedOutput() + if err != nil { + t.Fatalf("Executing %q: %v", cmd, err) + } + cgroups, err := cgroup.LoadPaths(strings.TrimSpace(string(ppid))) + if err != nil { + t.Fatalf("cgroup.LoadPath(%s): %v", ppid, err) + } + path := filepath.Join("/sys/fs/cgroup/memory", cgroups["memory"], parent, gid, "cgroup.procs") + if err := verifyPid(pid, path); err != nil { + t.Errorf("cgroup control %q processes: %v", "memory", err) + } +} diff --git a/test/root/chroot_test.go b/test/root/chroot_test.go new file mode 100644 index 000000000..be0f63d18 --- /dev/null +++ b/test/root/chroot_test.go @@ -0,0 +1,142 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package root is used for tests that requires sysadmin privileges run. +package root + +import ( + "fmt" + "io/ioutil" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + + "gvisor.dev/gvisor/runsc/dockerutil" +) + +// TestChroot verifies that the sandbox is chroot'd and that mounts are cleaned +// up after the sandbox is destroyed. +func TestChroot(t *testing.T) { + d := dockerutil.MakeDocker("chroot-test") + if err := d.Run("alpine", "sleep", "10000"); err != nil { + t.Fatalf("docker run failed: %v", err) + } + defer d.CleanUp() + + pid, err := d.SandboxPid() + if err != nil { + t.Fatalf("Docker.SandboxPid(): %v", err) + } + + // Check that sandbox is chroot'ed. + procRoot := filepath.Join("/proc", strconv.Itoa(pid), "root") + chroot, err := filepath.EvalSymlinks(procRoot) + if err != nil { + t.Fatalf("error resolving /proc/<pid>/root symlink: %v", err) + } + if chroot != "/" { + t.Errorf("sandbox is not chroot'd, it should be inside: /, got: %q", chroot) + } + + path, err := filepath.EvalSymlinks(filepath.Join("/proc", strconv.Itoa(pid), "cwd")) + if err != nil { + t.Fatalf("error resolving /proc/<pid>/cwd symlink: %v", err) + } + if chroot != path { + t.Errorf("sandbox current dir is wrong, want: %q, got: %q", chroot, path) + } + + fi, err := ioutil.ReadDir(procRoot) + if err != nil { + t.Fatalf("error listing %q: %v", chroot, err) + } + if want, got := 1, len(fi); want != got { + t.Fatalf("chroot dir got %d entries, want %d", got, want) + } + + // chroot dir is prepared by runsc and should contains only /proc. + if fi[0].Name() != "proc" { + t.Errorf("chroot got children %v, want %v", fi[0].Name(), "proc") + } + + d.CleanUp() +} + +func TestChrootGofer(t *testing.T) { + d := dockerutil.MakeDocker("chroot-test") + if err := d.Run("alpine", "sleep", "10000"); err != nil { + t.Fatalf("docker run failed: %v", err) + } + defer d.CleanUp() + + // It's tricky to find gofers. Get sandbox PID first, then find parent. From + // parent get all immediate children, remove the sandbox, and everything else + // are gofers. + sandPID, err := d.SandboxPid() + if err != nil { + t.Fatalf("Docker.SandboxPid(): %v", err) + } + + // Find sandbox's parent PID. + cmd := fmt.Sprintf("grep PPid /proc/%d/status | awk '{print $2}'", sandPID) + parent, err := exec.Command("sh", "-c", cmd).CombinedOutput() + if err != nil { + t.Fatalf("failed to fetch runsc (%d) parent PID: %v, out:\n%s", sandPID, err, string(parent)) + } + parentPID, err := strconv.Atoi(strings.TrimSpace(string(parent))) + if err != nil { + t.Fatalf("failed to parse PPID %q: %v", string(parent), err) + } + + // Get all children from parent. + childrenOut, err := exec.Command("/usr/bin/pgrep", "-P", strconv.Itoa(parentPID)).CombinedOutput() + if err != nil { + t.Fatalf("failed to fetch containerd-shim children: %v", err) + } + children := strings.Split(strings.TrimSpace(string(childrenOut)), "\n") + + // This where the root directory is mapped on the host and that's where the + // gofer must have chroot'd to. + root := "/root" + + for _, child := range children { + childPID, err := strconv.Atoi(child) + if err != nil { + t.Fatalf("failed to parse child PID %q: %v", child, err) + } + if childPID == sandPID { + // Skip the sandbox, all other immediate children are gofers. + continue + } + + // Check that gofer is chroot'ed. + chroot, err := filepath.EvalSymlinks(filepath.Join("/proc", child, "root")) + if err != nil { + t.Fatalf("error resolving /proc/<pid>/root symlink: %v", err) + } + if root != chroot { + t.Errorf("gofer chroot is wrong, want: %q, got: %q", root, chroot) + } + + path, err := filepath.EvalSymlinks(filepath.Join("/proc", child, "cwd")) + if err != nil { + t.Fatalf("error resolving /proc/<pid>/cwd symlink: %v", err) + } + if root != path { + t.Errorf("gofer current dir is wrong, want: %q, got: %q", root, path) + } + } +} diff --git a/test/root/crictl_test.go b/test/root/crictl_test.go new file mode 100644 index 000000000..3f90c4c6a --- /dev/null +++ b/test/root/crictl_test.go @@ -0,0 +1,295 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package root + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "testing" + "time" + + "gvisor.dev/gvisor/runsc/criutil" + "gvisor.dev/gvisor/runsc/dockerutil" + "gvisor.dev/gvisor/runsc/specutils" + "gvisor.dev/gvisor/runsc/testutil" + "gvisor.dev/gvisor/test/root/testdata" +) + +// Tests for crictl have to be run as root (rather than in a user namespace) +// because crictl creates named network namespaces in /var/run/netns/. + +// TestCrictlSanity refers to b/112433158. +func TestCrictlSanity(t *testing.T) { + // Setup containerd and crictl. + crictl, cleanup, err := setup(t) + if err != nil { + t.Fatalf("failed to setup crictl: %v", err) + } + defer cleanup() + podID, contID, err := crictl.StartPodAndContainer("httpd", testdata.Sandbox, testdata.Httpd) + if err != nil { + t.Fatal(err) + } + + // Look for the httpd page. + if err = httpGet(crictl, podID, "index.html"); err != nil { + t.Fatalf("failed to get page: %v", err) + } + + // Stop everything. + if err := crictl.StopPodAndContainer(podID, contID); err != nil { + t.Fatal(err) + } +} + +// TestMountPaths refers to b/117635704. +func TestMountPaths(t *testing.T) { + // Setup containerd and crictl. + crictl, cleanup, err := setup(t) + if err != nil { + t.Fatalf("failed to setup crictl: %v", err) + } + defer cleanup() + podID, contID, err := crictl.StartPodAndContainer("httpd", testdata.Sandbox, testdata.HttpdMountPaths) + if err != nil { + t.Fatal(err) + } + + // Look for the directory available at /test. + if err = httpGet(crictl, podID, "test"); err != nil { + t.Fatalf("failed to get page: %v", err) + } + + // Stop everything. + if err := crictl.StopPodAndContainer(podID, contID); err != nil { + t.Fatal(err) + } +} + +// TestMountPaths refers to b/118728671. +func TestMountOverSymlinks(t *testing.T) { + // Setup containerd and crictl. + crictl, cleanup, err := setup(t) + if err != nil { + t.Fatalf("failed to setup crictl: %v", err) + } + defer cleanup() + podID, contID, err := crictl.StartPodAndContainer("k8s.gcr.io/busybox", testdata.Sandbox, testdata.MountOverSymlink) + if err != nil { + t.Fatal(err) + } + + out, err := crictl.Exec(contID, "readlink", "/etc/resolv.conf") + if err != nil { + t.Fatal(err) + } + if want := "/tmp/resolv.conf"; !strings.Contains(string(out), want) { + t.Fatalf("/etc/resolv.conf is not pointing to %q: %q", want, string(out)) + } + + etc, err := crictl.Exec(contID, "cat", "/etc/resolv.conf") + if err != nil { + t.Fatal(err) + } + tmp, err := crictl.Exec(contID, "cat", "/tmp/resolv.conf") + if err != nil { + t.Fatal(err) + } + if tmp != etc { + t.Fatalf("file content doesn't match:\n\t/etc/resolv.conf: %s\n\t/tmp/resolv.conf: %s", string(etc), string(tmp)) + } + + // Stop everything. + if err := crictl.StopPodAndContainer(podID, contID); err != nil { + t.Fatal(err) + } +} + +// TestHomeDir tests that the HOME environment variable is set for +// multi-containers. +func TestHomeDir(t *testing.T) { + // Setup containerd and crictl. + crictl, cleanup, err := setup(t) + if err != nil { + t.Fatalf("failed to setup crictl: %v", err) + } + defer cleanup() + contSpec := testdata.SimpleSpec("root", "k8s.gcr.io/busybox", []string{"sleep", "1000"}) + podID, contID, err := crictl.StartPodAndContainer("k8s.gcr.io/busybox", testdata.Sandbox, contSpec) + if err != nil { + t.Fatal(err) + } + + t.Run("root container", func(t *testing.T) { + out, err := crictl.Exec(contID, "sh", "-c", "echo $HOME") + if err != nil { + t.Fatal(err) + } + if got, want := strings.TrimSpace(string(out)), "/root"; got != want { + t.Fatalf("Home directory invalid. Got %q, Want : %q", got, want) + } + }) + + t.Run("sub-container", func(t *testing.T) { + // Create a sub container in the same pod. + subContSpec := testdata.SimpleSpec("subcontainer", "k8s.gcr.io/busybox", []string{"sleep", "1000"}) + subContID, err := crictl.StartContainer(podID, "k8s.gcr.io/busybox", testdata.Sandbox, subContSpec) + if err != nil { + t.Fatal(err) + } + + out, err := crictl.Exec(subContID, "sh", "-c", "echo $HOME") + if err != nil { + t.Fatal(err) + } + if got, want := strings.TrimSpace(string(out)), "/root"; got != want { + t.Fatalf("Home directory invalid. Got %q, Want: %q", got, want) + } + + if err := crictl.StopContainer(subContID); err != nil { + t.Fatal(err) + } + }) + + // Stop everything. + if err := crictl.StopPodAndContainer(podID, contID); err != nil { + t.Fatal(err) + } + +} + +// setup sets up before a test. Specifically it: +// * Creates directories and a socket for containerd to utilize. +// * Runs containerd and waits for it to reach a "ready" state for testing. +// * Returns a cleanup function that should be called at the end of the test. +func setup(t *testing.T) (*criutil.Crictl, func(), error) { + var cleanups []func() + cleanupFunc := func() { + for i := len(cleanups) - 1; i >= 0; i-- { + cleanups[i]() + } + } + cleanup := specutils.MakeCleanup(cleanupFunc) + defer cleanup.Clean() + + // Create temporary containerd root and state directories, and a socket + // via which crictl and containerd communicate. + containerdRoot, err := ioutil.TempDir(testutil.TmpDir(), "containerd-root") + if err != nil { + t.Fatalf("failed to create containerd root: %v", err) + } + cleanups = append(cleanups, func() { os.RemoveAll(containerdRoot) }) + containerdState, err := ioutil.TempDir(testutil.TmpDir(), "containerd-state") + if err != nil { + t.Fatalf("failed to create containerd state: %v", err) + } + cleanups = append(cleanups, func() { os.RemoveAll(containerdState) }) + sockAddr := filepath.Join(testutil.TmpDir(), "containerd-test.sock") + + // We rewrite a configuration. This is based on the current docker + // configuration for the runtime under test. + runtime, err := dockerutil.RuntimePath() + if err != nil { + t.Fatalf("error discovering runtime path: %v", err) + } + config, err := testutil.WriteTmpFile("containerd-config", testdata.ContainerdConfig(runtime)) + if err != nil { + t.Fatalf("failed to write containerd config") + } + cleanups = append(cleanups, func() { os.RemoveAll(config) }) + + // Start containerd. + containerd := exec.Command(getContainerd(), + "--config", config, + "--log-level", "debug", + "--root", containerdRoot, + "--state", containerdState, + "--address", sockAddr) + cleanups = append(cleanups, func() { + if err := testutil.KillCommand(containerd); err != nil { + log.Printf("error killing containerd: %v", err) + } + }) + containerdStderr, err := containerd.StderrPipe() + if err != nil { + t.Fatalf("failed to get containerd stderr: %v", err) + } + containerdStdout, err := containerd.StdoutPipe() + if err != nil { + t.Fatalf("failed to get containerd stdout: %v", err) + } + if err := containerd.Start(); err != nil { + t.Fatalf("failed running containerd: %v", err) + } + + // Wait for containerd to boot. Then put all containerd output into a + // buffer to be logged at the end of the test. + testutil.WaitUntilRead(containerdStderr, "Start streaming server", nil, 10*time.Second) + stdoutBuf := &bytes.Buffer{} + stderrBuf := &bytes.Buffer{} + go func() { io.Copy(stdoutBuf, containerdStdout) }() + go func() { io.Copy(stderrBuf, containerdStderr) }() + cleanups = append(cleanups, func() { + t.Logf("containerd stdout: %s", string(stdoutBuf.Bytes())) + t.Logf("containerd stderr: %s", string(stderrBuf.Bytes())) + }) + + cleanup.Release() + return criutil.NewCrictl(20*time.Second, sockAddr), cleanupFunc, nil +} + +// httpGet GETs the contents of a file served from a pod on port 80. +func httpGet(crictl *criutil.Crictl, podID, filePath string) error { + // Get the IP of the httpd server. + ip, err := crictl.PodIP(podID) + if err != nil { + return fmt.Errorf("failed to get IP from pod %q: %v", podID, err) + } + + // GET the page. We may be waiting for the server to start, so retry + // with a timeout. + var resp *http.Response + cb := func() error { + r, err := http.Get(fmt.Sprintf("http://%s", path.Join(ip, filePath))) + resp = r + return err + } + if err := testutil.Poll(cb, 20*time.Second); err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fmt.Errorf("bad status returned: %d", resp.StatusCode) + } + return nil +} + +func getContainerd() string { + // Use the local path if it exists, otherwise, use the system one. + if _, err := os.Stat("/usr/local/bin/containerd"); err == nil { + return "/usr/local/bin/containerd" + } + return "/usr/bin/containerd" +} diff --git a/test/root/main_test.go b/test/root/main_test.go new file mode 100644 index 000000000..d74dec85f --- /dev/null +++ b/test/root/main_test.go @@ -0,0 +1,49 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package root + +import ( + "flag" + "fmt" + "os" + "testing" + + "github.com/syndtr/gocapability/capability" + "gvisor.dev/gvisor/runsc/dockerutil" + "gvisor.dev/gvisor/runsc/specutils" +) + +// TestMain is the main function for root tests. This function checks the +// supported docker version, required capabilities, and configures the executable +// path for runsc. +func TestMain(m *testing.M) { + flag.Parse() + + if !specutils.HasCapabilities(capability.CAP_SYS_ADMIN, capability.CAP_DAC_OVERRIDE) { + fmt.Println("Test requires sysadmin privileges to run. Try again with sudo.") + os.Exit(1) + } + + dockerutil.EnsureSupportedDockerVersion() + + // Configure exe for tests. + path, err := dockerutil.RuntimePath() + if err != nil { + panic(err.Error()) + } + specutils.ExePath = path + + os.Exit(m.Run()) +} diff --git a/test/root/oom_score_adj_test.go b/test/root/oom_score_adj_test.go new file mode 100644 index 000000000..6cd378a1b --- /dev/null +++ b/test/root/oom_score_adj_test.go @@ -0,0 +1,376 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package root + +import ( + "fmt" + "os" + "testing" + + specs "github.com/opencontainers/runtime-spec/specs-go" + "gvisor.dev/gvisor/runsc/boot" + "gvisor.dev/gvisor/runsc/container" + "gvisor.dev/gvisor/runsc/specutils" + "gvisor.dev/gvisor/runsc/testutil" +) + +var ( + maxOOMScoreAdj = 1000 + highOOMScoreAdj = 500 + lowOOMScoreAdj = -500 + minOOMScoreAdj = -1000 +) + +// Tests for oom_score_adj have to be run as root (rather than in a user +// namespace) because we need to adjust oom_score_adj for PIDs other than our +// own and test values below 0. + +// TestOOMScoreAdjSingle tests that oom_score_adj is set properly in a +// single container sandbox. +func TestOOMScoreAdjSingle(t *testing.T) { + ppid, err := specutils.GetParentPid(os.Getpid()) + if err != nil { + t.Fatalf("getting parent pid: %v", err) + } + parentOOMScoreAdj, err := specutils.GetOOMScoreAdj(ppid) + if err != nil { + t.Fatalf("getting parent oom_score_adj: %v", err) + } + + testCases := []struct { + Name string + + // OOMScoreAdj is the oom_score_adj set to the OCI spec. If nil then + // no value is set. + OOMScoreAdj *int + }{ + { + Name: "max", + OOMScoreAdj: &maxOOMScoreAdj, + }, + { + Name: "high", + OOMScoreAdj: &highOOMScoreAdj, + }, + { + Name: "low", + OOMScoreAdj: &lowOOMScoreAdj, + }, + { + Name: "min", + OOMScoreAdj: &minOOMScoreAdj, + }, + { + Name: "nil", + OOMScoreAdj: &parentOOMScoreAdj, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.Name, func(t *testing.T) { + id := testutil.UniqueContainerID() + s := testutil.NewSpecWithArgs("sleep", "1000") + s.Process.OOMScoreAdj = testCase.OOMScoreAdj + + conf := testutil.TestConfig() + containers, cleanup, err := startContainers(conf, []*specs.Spec{s}, []string{id}) + if err != nil { + t.Fatalf("error starting containers: %v", err) + } + defer cleanup() + + c := containers[0] + + // Verify the gofer's oom_score_adj + if testCase.OOMScoreAdj != nil { + goferScore, err := specutils.GetOOMScoreAdj(c.GoferPid) + if err != nil { + t.Fatalf("error reading gofer oom_score_adj: %v", err) + } + if goferScore != *testCase.OOMScoreAdj { + t.Errorf("gofer oom_score_adj got: %d, want: %d", goferScore, *testCase.OOMScoreAdj) + } + + // Verify the sandbox's oom_score_adj. + // + // The sandbox should be the same for all containers so just use + // the first one. + sandboxPid := c.Sandbox.Pid + sandboxScore, err := specutils.GetOOMScoreAdj(sandboxPid) + if err != nil { + t.Fatalf("error reading sandbox oom_score_adj: %v", err) + } + if sandboxScore != *testCase.OOMScoreAdj { + t.Errorf("sandbox oom_score_adj got: %d, want: %d", sandboxScore, *testCase.OOMScoreAdj) + } + } + }) + } +} + +// TestOOMScoreAdjMulti tests that oom_score_adj is set properly in a +// multi-container sandbox. +func TestOOMScoreAdjMulti(t *testing.T) { + ppid, err := specutils.GetParentPid(os.Getpid()) + if err != nil { + t.Fatalf("getting parent pid: %v", err) + } + parentOOMScoreAdj, err := specutils.GetOOMScoreAdj(ppid) + if err != nil { + t.Fatalf("getting parent oom_score_adj: %v", err) + } + + testCases := []struct { + Name string + + // OOMScoreAdj is the oom_score_adj set to the OCI spec. If nil then + // no value is set. One value for each container. The first value is the + // root container. + OOMScoreAdj []*int + + // Expected is the expected oom_score_adj of the sandbox. If nil, then + // this value is ignored. + Expected *int + + // Remove is a set of container indexes to remove from the sandbox. + Remove []int + + // ExpectedAfterRemove is the expected oom_score_adj of the sandbox + // after containers are removed. Ignored if nil. + ExpectedAfterRemove *int + }{ + // A single container CRI test case. This should not happen in + // practice as there should be at least one container besides the pause + // container. However, we include a test case to ensure sane behavior. + { + Name: "single", + OOMScoreAdj: []*int{&highOOMScoreAdj}, + Expected: &parentOOMScoreAdj, + }, + { + Name: "multi_no_value", + OOMScoreAdj: []*int{nil, nil, nil}, + Expected: &parentOOMScoreAdj, + }, + { + Name: "multi_non_nil_root", + OOMScoreAdj: []*int{&minOOMScoreAdj, nil, nil}, + Expected: &parentOOMScoreAdj, + }, + { + Name: "multi_value", + OOMScoreAdj: []*int{&minOOMScoreAdj, &highOOMScoreAdj, &lowOOMScoreAdj}, + // The lowest value excluding the root container is expected. + Expected: &lowOOMScoreAdj, + }, + { + Name: "multi_min_value", + OOMScoreAdj: []*int{&minOOMScoreAdj, &lowOOMScoreAdj}, + // The lowest value excluding the root container is expected. + Expected: &lowOOMScoreAdj, + }, + { + Name: "multi_max_value", + OOMScoreAdj: []*int{&minOOMScoreAdj, &maxOOMScoreAdj, &highOOMScoreAdj}, + // The lowest value excluding the root container is expected. + Expected: &highOOMScoreAdj, + }, + { + Name: "remove_adjusted", + OOMScoreAdj: []*int{&minOOMScoreAdj, &maxOOMScoreAdj, &highOOMScoreAdj}, + // The lowest value excluding the root container is expected. + Expected: &highOOMScoreAdj, + // Remove highOOMScoreAdj container. + Remove: []int{2}, + ExpectedAfterRemove: &maxOOMScoreAdj, + }, + { + // This test removes all non-root sandboxes with a specified oomScoreAdj. + Name: "remove_to_nil", + OOMScoreAdj: []*int{&minOOMScoreAdj, nil, &lowOOMScoreAdj}, + Expected: &lowOOMScoreAdj, + // Remove lowOOMScoreAdj container. + Remove: []int{2}, + // The oom_score_adj expected after remove is that of the parent process. + ExpectedAfterRemove: &parentOOMScoreAdj, + }, + { + Name: "remove_no_effect", + OOMScoreAdj: []*int{&minOOMScoreAdj, &maxOOMScoreAdj, &highOOMScoreAdj}, + // The lowest value excluding the root container is expected. + Expected: &highOOMScoreAdj, + // Remove the maxOOMScoreAdj container. + Remove: []int{1}, + ExpectedAfterRemove: &highOOMScoreAdj, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.Name, func(t *testing.T) { + var cmds [][]string + var oomScoreAdj []*int + var toRemove []string + + for _, oomScore := range testCase.OOMScoreAdj { + oomScoreAdj = append(oomScoreAdj, oomScore) + cmds = append(cmds, []string{"sleep", "100"}) + } + + specs, ids := createSpecs(cmds...) + for i, spec := range specs { + // Ensure the correct value is set, including no value. + spec.Process.OOMScoreAdj = oomScoreAdj[i] + + for _, j := range testCase.Remove { + if i == j { + toRemove = append(toRemove, ids[i]) + } + } + } + + conf := testutil.TestConfig() + containers, cleanup, err := startContainers(conf, specs, ids) + if err != nil { + t.Fatalf("error starting containers: %v", err) + } + defer cleanup() + + for i, c := range containers { + if oomScoreAdj[i] != nil { + // Verify the gofer's oom_score_adj + score, err := specutils.GetOOMScoreAdj(c.GoferPid) + if err != nil { + t.Fatalf("error reading gofer oom_score_adj: %v", err) + } + if score != *oomScoreAdj[i] { + t.Errorf("gofer oom_score_adj got: %d, want: %d", score, *oomScoreAdj[i]) + } + } + } + + // Verify the sandbox's oom_score_adj. + // + // The sandbox should be the same for all containers so just use + // the first one. + sandboxPid := containers[0].Sandbox.Pid + if testCase.Expected != nil { + score, err := specutils.GetOOMScoreAdj(sandboxPid) + if err != nil { + t.Fatalf("error reading sandbox oom_score_adj: %v", err) + } + if score != *testCase.Expected { + t.Errorf("sandbox oom_score_adj got: %d, want: %d", score, *testCase.Expected) + } + } + + if len(toRemove) == 0 { + return + } + + // Remove containers. + for _, removeID := range toRemove { + for _, c := range containers { + if c.ID == removeID { + c.Destroy() + } + } + } + + // Check the new adjusted oom_score_adj. + if testCase.ExpectedAfterRemove != nil { + scoreAfterRemove, err := specutils.GetOOMScoreAdj(sandboxPid) + if err != nil { + t.Fatalf("error reading sandbox oom_score_adj: %v", err) + } + if scoreAfterRemove != *testCase.ExpectedAfterRemove { + t.Errorf("sandbox oom_score_adj got: %d, want: %d", scoreAfterRemove, *testCase.ExpectedAfterRemove) + } + } + }) + } +} + +func createSpecs(cmds ...[]string) ([]*specs.Spec, []string) { + var specs []*specs.Spec + var ids []string + rootID := testutil.UniqueContainerID() + + for i, cmd := range cmds { + spec := testutil.NewSpecWithArgs(cmd...) + if i == 0 { + spec.Annotations = map[string]string{ + specutils.ContainerdContainerTypeAnnotation: specutils.ContainerdContainerTypeSandbox, + } + ids = append(ids, rootID) + } else { + spec.Annotations = map[string]string{ + specutils.ContainerdContainerTypeAnnotation: specutils.ContainerdContainerTypeContainer, + specutils.ContainerdSandboxIDAnnotation: rootID, + } + ids = append(ids, testutil.UniqueContainerID()) + } + specs = append(specs, spec) + } + return specs, ids +} + +func startContainers(conf *boot.Config, specs []*specs.Spec, ids []string) ([]*container.Container, func(), error) { + // Setup root dir if one hasn't been provided. + if len(conf.RootDir) == 0 { + rootDir, err := testutil.SetupRootDir() + if err != nil { + return nil, nil, fmt.Errorf("error creating root dir: %v", err) + } + conf.RootDir = rootDir + } + + var containers []*container.Container + var bundles []string + cleanup := func() { + for _, c := range containers { + c.Destroy() + } + for _, b := range bundles { + os.RemoveAll(b) + } + os.RemoveAll(conf.RootDir) + } + for i, spec := range specs { + bundleDir, err := testutil.SetupBundleDir(spec) + if err != nil { + cleanup() + return nil, nil, fmt.Errorf("error setting up container: %v", err) + } + bundles = append(bundles, bundleDir) + + args := container.Args{ + ID: ids[i], + Spec: spec, + BundleDir: bundleDir, + } + cont, err := container.New(conf, args) + if err != nil { + cleanup() + return nil, nil, fmt.Errorf("error creating container: %v", err) + } + containers = append(containers, cont) + + if err := cont.Start(conf); err != nil { + cleanup() + return nil, nil, fmt.Errorf("error starting container: %v", err) + } + } + return containers, cleanup, nil +} diff --git a/test/root/root.go b/test/root/root.go new file mode 100644 index 000000000..0f1d29faf --- /dev/null +++ b/test/root/root.go @@ -0,0 +1,21 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package root is used for tests that requires sysadmin privileges run. First, +// follow the setup instruction in runsc/test/README.md. You should also have +// docker, containerd, and crictl installed. To run these tests from the +// project root directory: +// +// ./scripts/root_tests.sh +package root diff --git a/test/root/testdata/BUILD b/test/root/testdata/BUILD new file mode 100644 index 000000000..125633680 --- /dev/null +++ b/test/root/testdata/BUILD @@ -0,0 +1,19 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +package(licenses = ["notice"]) + +go_library( + name = "testdata", + srcs = [ + "busybox.go", + "containerd_config.go", + "httpd.go", + "httpd_mount_paths.go", + "sandbox.go", + "simple.go", + ], + importpath = "gvisor.dev/gvisor/test/root/testdata", + visibility = [ + "//visibility:public", + ], +) diff --git a/test/root/testdata/busybox.go b/test/root/testdata/busybox.go new file mode 100644 index 000000000..e4dbd2843 --- /dev/null +++ b/test/root/testdata/busybox.go @@ -0,0 +1,32 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testdata + +// MountOverSymlink is a JSON config for a container that /etc/resolv.conf is a +// symlink to /tmp/resolv.conf. +var MountOverSymlink = ` +{ + "metadata": { + "name": "busybox" + }, + "image": { + "image": "k8s.gcr.io/busybox" + }, + "command": [ + "sleep", + "1000" + ] +} +` diff --git a/test/root/testdata/containerd_config.go b/test/root/testdata/containerd_config.go new file mode 100644 index 000000000..e12f1ec88 --- /dev/null +++ b/test/root/testdata/containerd_config.go @@ -0,0 +1,39 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package testdata contains data required for root tests. +package testdata + +import "fmt" + +// containerdConfigTemplate is a .toml config for containerd. It contains a +// formatting verb so the runtime field can be set via fmt.Sprintf. +const containerdConfigTemplate = ` +disabled_plugins = ["restart"] +[plugins.linux] + runtime = "%s" + runtime_root = "/tmp/test-containerd/runsc" + shim = "/usr/local/bin/gvisor-containerd-shim" + shim_debug = true + +[plugins.cri.containerd.runtimes.runsc] + runtime_type = "io.containerd.runtime.v1.linux" + runtime_engine = "%s" +` + +// ContainerdConfig returns a containerd config file with the specified +// runtime. +func ContainerdConfig(runtime string) string { + return fmt.Sprintf(containerdConfigTemplate, runtime, runtime) +} diff --git a/test/root/testdata/httpd.go b/test/root/testdata/httpd.go new file mode 100644 index 000000000..45d5e33d4 --- /dev/null +++ b/test/root/testdata/httpd.go @@ -0,0 +1,32 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testdata + +// Httpd is a JSON config for an httpd container. +const Httpd = ` +{ + "metadata": { + "name": "httpd" + }, + "image":{ + "image": "httpd" + }, + "mounts": [ + ], + "linux": { + }, + "log_path": "httpd.log" +} +` diff --git a/test/root/testdata/httpd_mount_paths.go b/test/root/testdata/httpd_mount_paths.go new file mode 100644 index 000000000..ac3f4446a --- /dev/null +++ b/test/root/testdata/httpd_mount_paths.go @@ -0,0 +1,53 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testdata + +// HttpdMountPaths is a JSON config for an httpd container with additional +// mounts. +const HttpdMountPaths = ` +{ + "metadata": { + "name": "httpd" + }, + "image":{ + "image": "httpd" + }, + "mounts": [ + { + "container_path": "/var/run/secrets/kubernetes.io/serviceaccount", + "host_path": "/var/lib/kubelet/pods/82bae206-cdf5-11e8-b245-8cdcd43ac064/volumes/kubernetes.io~secret/default-token-2rpfx", + "readonly": true + }, + { + "container_path": "/etc/hosts", + "host_path": "/var/lib/kubelet/pods/82bae206-cdf5-11e8-b245-8cdcd43ac064/etc-hosts", + "readonly": false + }, + { + "container_path": "/dev/termination-log", + "host_path": "/var/lib/kubelet/pods/82bae206-cdf5-11e8-b245-8cdcd43ac064/containers/httpd/d1709580", + "readonly": false + }, + { + "container_path": "/usr/local/apache2/htdocs/test", + "host_path": "/var/lib/kubelet/pods/82bae206-cdf5-11e8-b245-8cdcd43ac064", + "readonly": true + } + ], + "linux": { + }, + "log_path": "httpd.log" +} +` diff --git a/test/root/testdata/sandbox.go b/test/root/testdata/sandbox.go new file mode 100644 index 000000000..0db210370 --- /dev/null +++ b/test/root/testdata/sandbox.go @@ -0,0 +1,30 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testdata + +// Sandbox is a default JSON config for a sandbox. +const Sandbox = ` +{ + "metadata": { + "name": "default-sandbox", + "namespace": "default", + "attempt": 1, + "uid": "hdishd83djaidwnduwk28bcsb" + }, + "linux": { + }, + "log_directory": "/tmp" +} +` diff --git a/test/root/testdata/simple.go b/test/root/testdata/simple.go new file mode 100644 index 000000000..1cca53f0c --- /dev/null +++ b/test/root/testdata/simple.go @@ -0,0 +1,41 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testdata + +import ( + "encoding/json" + "fmt" +) + +// SimpleSpec returns a JSON config for a simple container that runs the +// specified command in the specified image. +func SimpleSpec(name, image string, cmd []string) string { + cmds, err := json.Marshal(cmd) + if err != nil { + // This shouldn't happen. + panic(err) + } + return fmt.Sprintf(` +{ + "metadata": { + "name": %q + }, + "image": { + "image": %q + }, + "command": %s + } +`, name, image, cmds) +} |