diff options
Diffstat (limited to 'pkg/test/dockerutil')
| -rw-r--r-- | pkg/test/dockerutil/BUILD | 42 | ||||
| -rw-r--r-- | pkg/test/dockerutil/README.md | 86 | ||||
| -rw-r--r-- | pkg/test/dockerutil/container.go | 547 | ||||
| -rw-r--r-- | pkg/test/dockerutil/dockerutil.go | 176 | ||||
| -rw-r--r-- | pkg/test/dockerutil/exec.go | 193 | ||||
| -rw-r--r-- | pkg/test/dockerutil/network.go | 113 | ||||
| -rw-r--r-- | pkg/test/dockerutil/profile.go | 147 | ||||
| -rw-r--r-- | pkg/test/dockerutil/profile_test.go | 116 |
8 files changed, 0 insertions, 1420 deletions
diff --git a/pkg/test/dockerutil/BUILD b/pkg/test/dockerutil/BUILD deleted file mode 100644 index a5e84658a..000000000 --- a/pkg/test/dockerutil/BUILD +++ /dev/null @@ -1,42 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "dockerutil", - testonly = 1, - srcs = [ - "container.go", - "dockerutil.go", - "exec.go", - "network.go", - "profile.go", - ], - visibility = ["//:sandbox"], - deps = [ - "//pkg/test/testutil", - "@com_github_docker_docker//api/types:go_default_library", - "@com_github_docker_docker//api/types/container:go_default_library", - "@com_github_docker_docker//api/types/mount:go_default_library", - "@com_github_docker_docker//api/types/network:go_default_library", - "@com_github_docker_docker//client:go_default_library", - "@com_github_docker_docker//pkg/stdcopy:go_default_library", - "@com_github_docker_go_connections//nat:go_default_library", - ], -) - -go_test( - name = "profile_test", - size = "large", - srcs = [ - "profile_test.go", - ], - library = ":dockerutil", - tags = [ - # Requires docker and runsc to be configured before test runs. - # Also requires the test to be run as root. - "manual", - "local", - ], - visibility = ["//:sandbox"], -) diff --git a/pkg/test/dockerutil/README.md b/pkg/test/dockerutil/README.md deleted file mode 100644 index 870292096..000000000 --- a/pkg/test/dockerutil/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# dockerutil - -This package is for creating and controlling docker containers for testing -runsc, gVisor's docker/kubernetes binary. A simple test may look like: - -``` - func TestSuperCool(t *testing.T) { - ctx := context.Background() - c := dockerutil.MakeContainer(ctx, t) - got, err := c.Run(ctx, dockerutil.RunOpts{ - Image: "basic/alpine" - }, "echo", "super cool") - if err != nil { - t.Fatalf("err was not nil: %v", err) - } - want := "super cool" - if !strings.Contains(got, want){ - t.Fatalf("want: %s, got: %s", want, got) - } - } -``` - -For further examples, see many of our end to end tests elsewhere in the repo, -such as those in //test/e2e or benchmarks at //test/benchmarks. - -dockerutil uses the "official" docker golang api, which is -[very powerful](https://godoc.org/github.com/docker/docker/client). dockerutil -is a thin wrapper around this API, allowing desired new use cases to be easily -implemented. - -## Profiling - -dockerutil is capable of generating profiles. Currently, the only option is to -use pprof profiles generated by `runsc debug`. The profiler will generate Block, -CPU, Heap, Goroutine, and Mutex profiles. To generate profiles: - -* Install runsc with the `--profile` flag: `make configure RUNTIME=myrunsc - ARGS="--profile"` Also add other flags with ARGS like `--platform=kvm` or - `--vfs2`. -* Restart docker: `sudo service docker restart` - -To run and generate CPU profiles run: - -``` -make sudo TARGETS=//path/to:target \ - ARGS="--runtime=myrunsc -test.v -test.bench=. --pprof-cpu" OPTIONS="-c opt" -``` - -Profiles would be at: `/tmp/profile/myrunsc/CONTAINERNAME/cpu.pprof` - -Container name in most tests and benchmarks in gVisor is usually the test name -and some random characters like so: -`BenchmarkABSL-CleanCache-JF2J2ZYF3U7SL47QAA727CSJI3C4ZAW2` - -Profiling requires root as runsc debug inspects running containers in /var/run -among other things. - -### Writing for Profiling - -The below shows an example of using profiles with dockerutil. - -``` -func TestSuperCool(t *testing.T){ - ctx := context.Background() - // profiled and using runtime from dockerutil.runtime flag - profiled := MakeContainer() - - // not profiled and using runtime runc - native := MakeNativeContainer() - - err := profiled.Spawn(ctx, RunOpts{ - Image: "some/image", - }, "sleep", "100000") - // profiling has begun here - ... - expensive setup that I don't want to profile. - ... - profiled.RestartProfiles() - // profiled activity -} -``` - -In the above example, `profiled` would be profiled and `native` would not. The -call to `RestartProfiles()` restarts the clock on profiling. This is useful if -the main activity being tested is done with `docker exec` or `container.Spawn()` -followed by one or more `container.Exec()` calls. diff --git a/pkg/test/dockerutil/container.go b/pkg/test/dockerutil/container.go deleted file mode 100644 index 2bf0a22ff..000000000 --- a/pkg/test/dockerutil/container.go +++ /dev/null @@ -1,547 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dockerutil - -import ( - "bytes" - "context" - "errors" - "fmt" - "io/ioutil" - "net" - "os" - "path" - "path/filepath" - "regexp" - "strconv" - "strings" - "time" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/mount" - "github.com/docker/docker/api/types/network" - "github.com/docker/docker/client" - "github.com/docker/docker/pkg/stdcopy" - "github.com/docker/go-connections/nat" - "gvisor.dev/gvisor/pkg/test/testutil" -) - -// Container represents a Docker Container allowing -// user to configure and control as one would with the 'docker' -// client. Container is backed by the offical golang docker API. -// See: https://pkg.go.dev/github.com/docker/docker. -type Container struct { - Name string - runtime string - - logger testutil.Logger - client *client.Client - id string - mounts []mount.Mount - links []string - copyErr error - cleanups []func() - - // Profiles are profiles added to this container. They contain methods - // that are run after Creation, Start, and Cleanup of this Container, along - // a handle to restart the profile. Generally, tests/benchmarks using - // profiles need to run as root. - profiles []Profile -} - -// RunOpts are options for running a container. -type RunOpts struct { - // Image is the image relative to images/. This will be mangled - // appropriately, to ensure that only first-party images are used. - Image string - - // Memory is the memory limit in bytes. - Memory int - - // Cpus in which to allow execution. ("0", "1", "0-2"). - CpusetCpus string - - // Ports are the ports to be allocated. - Ports []int - - // WorkDir sets the working directory. - WorkDir string - - // ReadOnly sets the read-only flag. - ReadOnly bool - - // Env are additional environment variables. - Env []string - - // User is the user to use. - User string - - // Privileged enables privileged mode. - Privileged bool - - // CapAdd are the extra set of capabilities to add. - CapAdd []string - - // CapDrop are the extra set of capabilities to drop. - CapDrop []string - - // Mounts is the list of directories/files to be mounted inside the container. - Mounts []mount.Mount - - // Links is the list of containers to be connected to the container. - Links []string -} - -// MakeContainer sets up the struct for a Docker container. -// -// Names of containers will be unique. -// Containers will check flags for profiling requests. -func MakeContainer(ctx context.Context, logger testutil.Logger) *Container { - c := MakeNativeContainer(ctx, logger) - c.runtime = *runtime - if p := MakePprofFromFlags(c); p != nil { - c.AddProfile(p) - } - return c -} - -// MakeNativeContainer sets up the struct for a DockerContainer using runc. Native -// containers aren't profiled. -func MakeNativeContainer(ctx context.Context, logger testutil.Logger) *Container { - // Slashes are not allowed in container names. - name := testutil.RandomID(logger.Name()) - name = strings.ReplaceAll(name, "/", "-") - client, err := client.NewClientWithOpts(client.FromEnv) - if err != nil { - return nil - } - client.NegotiateAPIVersion(ctx) - return &Container{ - logger: logger, - Name: name, - runtime: "", - client: client, - } -} - -// AddProfile adds a profile to this container. -func (c *Container) AddProfile(p Profile) { - c.profiles = append(c.profiles, p) -} - -// RestartProfiles calls Restart on all profiles for this container. -func (c *Container) RestartProfiles() error { - for _, profile := range c.profiles { - if err := profile.Restart(c); err != nil { - return err - } - } - return nil -} - -// Spawn is analogous to 'docker run -d'. -func (c *Container) Spawn(ctx context.Context, r RunOpts, args ...string) error { - if err := c.create(ctx, c.config(r, args), c.hostConfig(r), nil); err != nil { - return err - } - return c.Start(ctx) -} - -// SpawnProcess is analogous to 'docker run -it'. It returns a process -// which represents the root process. -func (c *Container) SpawnProcess(ctx context.Context, r RunOpts, args ...string) (Process, error) { - config, hostconf, netconf := c.ConfigsFrom(r, args...) - config.Tty = true - config.OpenStdin = true - - if err := c.CreateFrom(ctx, config, hostconf, netconf); err != nil { - return Process{}, err - } - - // Open a connection to the container for parsing logs and for TTY. - stream, err := c.client.ContainerAttach(ctx, c.id, - types.ContainerAttachOptions{ - Stream: true, - Stdin: true, - Stdout: true, - Stderr: true, - }) - if err != nil { - return Process{}, fmt.Errorf("connect failed container id %s: %v", c.id, err) - } - - c.cleanups = append(c.cleanups, func() { stream.Close() }) - - if err := c.Start(ctx); err != nil { - return Process{}, err - } - - return Process{container: c, conn: stream}, nil -} - -// Run is analogous to 'docker run'. -func (c *Container) Run(ctx context.Context, r RunOpts, args ...string) (string, error) { - if err := c.create(ctx, c.config(r, args), c.hostConfig(r), nil); err != nil { - return "", err - } - - if err := c.Start(ctx); err != nil { - return "", err - } - - if err := c.Wait(ctx); err != nil { - return "", err - } - - return c.Logs(ctx) -} - -// ConfigsFrom returns container configs from RunOpts and args. The caller should call 'CreateFrom' -// and Start. -func (c *Container) ConfigsFrom(r RunOpts, args ...string) (*container.Config, *container.HostConfig, *network.NetworkingConfig) { - return c.config(r, args), c.hostConfig(r), &network.NetworkingConfig{} -} - -// MakeLink formats a link to add to a RunOpts. -func (c *Container) MakeLink(target string) string { - return fmt.Sprintf("%s:%s", c.Name, target) -} - -// CreateFrom creates a container from the given configs. -func (c *Container) CreateFrom(ctx context.Context, conf *container.Config, hostconf *container.HostConfig, netconf *network.NetworkingConfig) error { - return c.create(ctx, conf, hostconf, netconf) -} - -// Create is analogous to 'docker create'. -func (c *Container) Create(ctx context.Context, r RunOpts, args ...string) error { - return c.create(ctx, c.config(r, args), c.hostConfig(r), nil) -} - -func (c *Container) create(ctx context.Context, conf *container.Config, hostconf *container.HostConfig, netconf *network.NetworkingConfig) error { - cont, err := c.client.ContainerCreate(ctx, conf, hostconf, nil, c.Name) - if err != nil { - return err - } - c.id = cont.ID - for _, profile := range c.profiles { - if err := profile.OnCreate(c); err != nil { - return fmt.Errorf("OnCreate method failed with: %v", err) - } - } - return nil -} - -func (c *Container) config(r RunOpts, args []string) *container.Config { - ports := nat.PortSet{} - for _, p := range r.Ports { - port := nat.Port(fmt.Sprintf("%d", p)) - ports[port] = struct{}{} - } - env := append(r.Env, fmt.Sprintf("RUNSC_TEST_NAME=%s", c.Name)) - - return &container.Config{ - Image: testutil.ImageByName(r.Image), - Cmd: args, - ExposedPorts: ports, - Env: env, - WorkingDir: r.WorkDir, - User: r.User, - } -} - -func (c *Container) hostConfig(r RunOpts) *container.HostConfig { - c.mounts = append(c.mounts, r.Mounts...) - - return &container.HostConfig{ - Runtime: c.runtime, - Mounts: c.mounts, - PublishAllPorts: true, - Links: r.Links, - CapAdd: r.CapAdd, - CapDrop: r.CapDrop, - Privileged: r.Privileged, - ReadonlyRootfs: r.ReadOnly, - Resources: container.Resources{ - Memory: int64(r.Memory), // In bytes. - CpusetCpus: r.CpusetCpus, - }, - } -} - -// Start is analogous to 'docker start'. -func (c *Container) Start(ctx context.Context) error { - if err := c.client.ContainerStart(ctx, c.id, types.ContainerStartOptions{}); err != nil { - return fmt.Errorf("ContainerStart failed: %v", err) - } - for _, profile := range c.profiles { - if err := profile.OnStart(c); err != nil { - return fmt.Errorf("OnStart method failed: %v", err) - } - } - return nil -} - -// Stop is analogous to 'docker stop'. -func (c *Container) Stop(ctx context.Context) error { - return c.client.ContainerStop(ctx, c.id, nil) -} - -// Pause is analogous to'docker pause'. -func (c *Container) Pause(ctx context.Context) error { - return c.client.ContainerPause(ctx, c.id) -} - -// Unpause is analogous to 'docker unpause'. -func (c *Container) Unpause(ctx context.Context) error { - return c.client.ContainerUnpause(ctx, c.id) -} - -// Checkpoint is analogous to 'docker checkpoint'. -func (c *Container) Checkpoint(ctx context.Context, name string) error { - return c.client.CheckpointCreate(ctx, c.Name, types.CheckpointCreateOptions{CheckpointID: name, Exit: true}) -} - -// Restore is analogous to 'docker start --checkname [name]'. -func (c *Container) Restore(ctx context.Context, name string) error { - return c.client.ContainerStart(ctx, c.id, types.ContainerStartOptions{CheckpointID: name}) -} - -// Logs is analogous 'docker logs'. -func (c *Container) Logs(ctx context.Context) (string, error) { - var out bytes.Buffer - err := c.logs(ctx, &out, &out) - return out.String(), err -} - -func (c *Container) logs(ctx context.Context, stdout, stderr *bytes.Buffer) error { - opts := types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true} - writer, err := c.client.ContainerLogs(ctx, c.id, opts) - if err != nil { - return err - } - defer writer.Close() - _, err = stdcopy.StdCopy(stdout, stderr, writer) - - return err -} - -// ID returns the container id. -func (c *Container) ID() string { - return c.id -} - -// SandboxPid returns the container's pid. -func (c *Container) SandboxPid(ctx context.Context) (int, error) { - resp, err := c.client.ContainerInspect(ctx, c.id) - if err != nil { - return -1, err - } - return resp.ContainerJSONBase.State.Pid, nil -} - -// ErrNoIP indicates that no IP address is available. -var ErrNoIP = errors.New("no IP available") - -// FindIP returns the IP address of the container. -func (c *Container) FindIP(ctx context.Context, ipv6 bool) (net.IP, error) { - resp, err := c.client.ContainerInspect(ctx, c.id) - if err != nil { - return nil, err - } - - var ip net.IP - if ipv6 { - ip = net.ParseIP(resp.NetworkSettings.DefaultNetworkSettings.GlobalIPv6Address) - } else { - ip = net.ParseIP(resp.NetworkSettings.DefaultNetworkSettings.IPAddress) - } - if ip == nil { - return net.IP{}, ErrNoIP - } - return ip, nil -} - -// FindPort returns the host port that is mapped to 'sandboxPort'. -func (c *Container) FindPort(ctx context.Context, sandboxPort int) (int, error) { - desc, err := c.client.ContainerInspect(ctx, c.id) - if err != nil { - return -1, fmt.Errorf("error retrieving port: %v", err) - } - - format := fmt.Sprintf("%d/tcp", sandboxPort) - ports, ok := desc.NetworkSettings.Ports[nat.Port(format)] - if !ok { - return -1, fmt.Errorf("error retrieving port: %v", err) - - } - - port, err := strconv.Atoi(ports[0].HostPort) - if err != nil { - return -1, fmt.Errorf("error parsing port %q: %v", port, err) - } - return port, nil -} - -// CopyFiles copies in and mounts the given files. They are always ReadOnly. -func (c *Container) CopyFiles(opts *RunOpts, target string, sources ...string) { - dir, err := ioutil.TempDir("", c.Name) - if err != nil { - c.copyErr = fmt.Errorf("ioutil.TempDir failed: %v", err) - return - } - c.cleanups = append(c.cleanups, func() { os.RemoveAll(dir) }) - if err := os.Chmod(dir, 0755); err != nil { - c.copyErr = fmt.Errorf("os.Chmod(%q, 0755) failed: %v", dir, err) - return - } - for _, name := range sources { - src := name - if !filepath.IsAbs(src) { - src, err = testutil.FindFile(name) - if err != nil { - c.copyErr = fmt.Errorf("testutil.FindFile(%q) failed: %w", name, err) - return - } - } - dst := path.Join(dir, path.Base(name)) - if err := testutil.Copy(src, dst); err != nil { - c.copyErr = fmt.Errorf("testutil.Copy(%q, %q) failed: %v", src, dst, err) - return - } - c.logger.Logf("copy: %s -> %s", src, dst) - } - opts.Mounts = append(opts.Mounts, mount.Mount{ - Type: mount.TypeBind, - Source: dir, - Target: target, - ReadOnly: false, - }) -} - -// Status inspects the container returns its status. -func (c *Container) Status(ctx context.Context) (types.ContainerState, error) { - resp, err := c.client.ContainerInspect(ctx, c.id) - if err != nil { - return types.ContainerState{}, err - } - return *resp.State, err -} - -// Wait waits for the container to exit. -func (c *Container) Wait(ctx context.Context) error { - statusChan, errChan := c.client.ContainerWait(ctx, c.id, container.WaitConditionNotRunning) - select { - case err := <-errChan: - return err - case <-statusChan: - return nil - } -} - -// WaitTimeout waits for the container to exit with a timeout. -func (c *Container) WaitTimeout(ctx context.Context, timeout time.Duration) error { - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - statusChan, errChan := c.client.ContainerWait(ctx, c.id, container.WaitConditionNotRunning) - select { - case <-ctx.Done(): - if ctx.Err() == context.DeadlineExceeded { - return fmt.Errorf("container %s timed out after %v seconds", c.Name, timeout.Seconds()) - } - return nil - case err := <-errChan: - return err - case <-statusChan: - return nil - } -} - -// WaitForOutput searches container logs for pattern and returns or timesout. -func (c *Container) WaitForOutput(ctx context.Context, pattern string, timeout time.Duration) (string, error) { - matches, err := c.WaitForOutputSubmatch(ctx, pattern, timeout) - if err != nil { - return "", err - } - if len(matches) == 0 { - return "", fmt.Errorf("didn't find pattern %s logs", pattern) - } - return matches[0], nil -} - -// WaitForOutputSubmatch searches container logs for the given -// pattern or times out. It returns any regexp submatches as well. -func (c *Container) WaitForOutputSubmatch(ctx context.Context, pattern string, timeout time.Duration) ([]string, error) { - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - re := regexp.MustCompile(pattern) - for { - logs, err := c.Logs(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get logs: %v logs: %s", err, logs) - } - if matches := re.FindStringSubmatch(logs); matches != nil { - return matches, nil - } - time.Sleep(50 * time.Millisecond) - } -} - -// Kill kills the container. -func (c *Container) Kill(ctx context.Context) error { - return c.client.ContainerKill(ctx, c.id, "") -} - -// Remove is analogous to 'docker rm'. -func (c *Container) Remove(ctx context.Context) error { - // Remove the image. - remove := types.ContainerRemoveOptions{ - RemoveVolumes: c.mounts != nil, - RemoveLinks: c.links != nil, - Force: true, - } - return c.client.ContainerRemove(ctx, c.Name, remove) -} - -// CleanUp kills and deletes the container (best effort). -func (c *Container) CleanUp(ctx context.Context) { - // Execute profile cleanups before the container goes down. - for _, profile := range c.profiles { - profile.OnCleanUp(c) - } - - // Forget profiles. - c.profiles = nil - - // Execute all cleanups. We execute cleanups here to close any - // open connections to the container before closing. Open connections - // can cause Kill and Remove to hang. - for _, c := range c.cleanups { - c() - } - c.cleanups = nil - - // Kill the container. - if err := c.Kill(ctx); err != nil && !strings.Contains(err.Error(), "is not running") { - // Just log; can't do anything here. - c.logger.Logf("error killing container %q: %v", c.Name, err) - } - // Remove the image. - if err := c.Remove(ctx); err != nil { - c.logger.Logf("error removing container %q: %v", c.Name, err) - } - // Forget all mounts. - c.mounts = nil -} diff --git a/pkg/test/dockerutil/dockerutil.go b/pkg/test/dockerutil/dockerutil.go deleted file mode 100644 index 7027df1a5..000000000 --- a/pkg/test/dockerutil/dockerutil.go +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package dockerutil is a collection of utility functions. -package dockerutil - -import ( - "encoding/json" - "flag" - "fmt" - "io" - "io/ioutil" - "log" - "os/exec" - "regexp" - "strconv" - "time" - - "gvisor.dev/gvisor/pkg/test/testutil" -) - -var ( - // runtime is the runtime to use for tests. This will be applied to all - // containers. Note that the default here ("runsc") corresponds to the - // default used by the installations. This is important, because the - // default installer for vm_tests (in tools/installers:head, invoked - // via tools/vm:defs.bzl) will install with this name. So without - // changing anything, tests should have a runsc runtime available to - // them. Otherwise installers should update the existing runtime - // instead of installing a new one. - runtime = flag.String("runtime", "runsc", "specify which runtime to use") - - // config is the default Docker daemon configuration path. - config = flag.String("config_path", "/etc/docker/daemon.json", "configuration file for reading paths") - - // The following flags are for the "pprof" profiler tool. - - // pprofBaseDir allows the user to change the directory to which profiles are - // written. By default, profiles will appear under: - // /tmp/profile/RUNTIME/CONTAINER_NAME/*.pprof. - pprofBaseDir = flag.String("pprof-dir", "/tmp/profile", "base directory in: BASEDIR/RUNTIME/CONTINER_NAME/FILENAME (e.g. /tmp/profile/runtime/mycontainer/cpu.pprof)") - - // duration is the max duration `runsc debug` will run and capture profiles. - // If the container's clean up method is called prior to duration, the - // profiling process will be killed. - duration = flag.Duration("pprof-duration", 10*time.Second, "duration to run the profile in seconds") - - // The below flags enable each type of profile. Multiple profiles can be - // enabled for each run. - pprofBlock = flag.Bool("pprof-block", false, "enables block profiling with runsc debug") - pprofCPU = flag.Bool("pprof-cpu", false, "enables CPU profiling with runsc debug") - pprofHeap = flag.Bool("pprof-heap", false, "enables heap profiling with runsc debug") - pprofMutex = flag.Bool("pprof-mutex", false, "enables mutex profiling with runsc debug") -) - -// EnsureSupportedDockerVersion checks if correct docker is installed. -// -// This logs directly to stderr, as it is typically called from a Main wrapper. -func EnsureSupportedDockerVersion() { - cmd := exec.Command("docker", "version") - out, err := cmd.CombinedOutput() - if err != nil { - log.Fatalf("error running %q: %v", "docker version", err) - } - re := regexp.MustCompile(`Version:\s+(\d+)\.(\d+)\.\d.*`) - matches := re.FindStringSubmatch(string(out)) - if len(matches) != 3 { - log.Fatalf("Invalid docker output: %s", out) - } - major, _ := strconv.Atoi(matches[1]) - minor, _ := strconv.Atoi(matches[2]) - if major < 17 || (major == 17 && minor < 9) { - log.Fatalf("Docker version 17.09.0 or greater is required, found: %02d.%02d", major, minor) - } -} - -// RuntimePath returns the binary path for the current runtime. -func RuntimePath() (string, error) { - rs, err := runtimeMap() - if err != nil { - return "", err - } - - p, ok := rs["path"].(string) - if !ok { - // The runtime does not declare a path. - return "", fmt.Errorf("runtime does not declare a path: %v", rs) - } - return p, nil -} - -// UsingVFS2 returns true if the 'runtime' has the vfs2 flag set. -// TODO(gvisor.dev/issue/1624): Remove. -func UsingVFS2() (bool, error) { - rMap, err := runtimeMap() - if err != nil { - return false, err - } - - list, ok := rMap["runtimeArgs"].([]interface{}) - if !ok { - return false, fmt.Errorf("unexpected format: %v", rMap) - } - - for _, element := range list { - if element == "--vfs2" { - return true, nil - } - } - return false, nil -} - -func runtimeMap() (map[string]interface{}, error) { - // Read the configuration data; the file must exist. - configBytes, err := ioutil.ReadFile(*config) - if err != nil { - return nil, err - } - - // Unmarshal the configuration. - c := make(map[string]interface{}) - if err := json.Unmarshal(configBytes, &c); err != nil { - return nil, err - } - - // Decode the expected configuration. - r, ok := c["runtimes"] - if !ok { - return nil, fmt.Errorf("no runtimes declared: %v", c) - } - rs, ok := r.(map[string]interface{}) - if !ok { - // The runtimes are not a map. - return nil, fmt.Errorf("unexpected format: %v", rs) - } - r, ok = rs[*runtime] - if !ok { - // The expected runtime is not declared. - return nil, fmt.Errorf("runtime %q not found: %v", *runtime, rs) - } - rs, ok = r.(map[string]interface{}) - if !ok { - // The runtime is not a map. - return nil, fmt.Errorf("unexpected format: %v", r) - } - return rs, nil -} - -// Save exports a container image to the given Writer. -// -// Note that the writer should be actively consuming the output, otherwise it -// is not guaranteed that the Save will make any progress and the call may -// stall indefinitely. -// -// This is called by criutil in order to import imports. -func Save(logger testutil.Logger, image string, w io.Writer) error { - cmd := testutil.Command(logger, "docker", "save", testutil.ImageByName(image)) - cmd.Stdout = w // Send directly to the writer. - return cmd.Run() -} - -// Runtime returns the value of the flag runtime. -func Runtime() string { - return *runtime -} diff --git a/pkg/test/dockerutil/exec.go b/pkg/test/dockerutil/exec.go deleted file mode 100644 index 4c739c9e9..000000000 --- a/pkg/test/dockerutil/exec.go +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dockerutil - -import ( - "bytes" - "context" - "fmt" - "time" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/pkg/stdcopy" -) - -// ExecOpts holds arguments for Exec calls. -type ExecOpts struct { - // Env are additional environment variables. - Env []string - - // Privileged enables privileged mode. - Privileged bool - - // User is the user to use. - User string - - // Enables Tty and stdin for the created process. - UseTTY bool - - // WorkDir is the working directory of the process. - WorkDir string -} - -// Exec creates a process inside the container. -func (c *Container) Exec(ctx context.Context, opts ExecOpts, args ...string) (string, error) { - p, err := c.doExec(ctx, opts, args) - if err != nil { - return "", err - } - - if exitStatus, err := p.WaitExitStatus(ctx); err != nil { - return "", err - } else if exitStatus != 0 { - out, _ := p.Logs() - return out, fmt.Errorf("process terminated with status: %d", exitStatus) - } - - return p.Logs() -} - -// ExecProcess creates a process inside the container and returns a process struct -// for the caller to use. -func (c *Container) ExecProcess(ctx context.Context, opts ExecOpts, args ...string) (Process, error) { - return c.doExec(ctx, opts, args) -} - -func (c *Container) doExec(ctx context.Context, r ExecOpts, args []string) (Process, error) { - config := c.execConfig(r, args) - resp, err := c.client.ContainerExecCreate(ctx, c.id, config) - if err != nil { - return Process{}, fmt.Errorf("exec create failed with err: %v", err) - } - - hijack, err := c.client.ContainerExecAttach(ctx, resp.ID, types.ExecStartCheck{}) - if err != nil { - return Process{}, fmt.Errorf("exec attach failed with err: %v", err) - } - - if err := c.client.ContainerExecStart(ctx, resp.ID, types.ExecStartCheck{}); err != nil { - hijack.Close() - return Process{}, fmt.Errorf("exec start failed with err: %v", err) - } - - return Process{ - container: c, - execid: resp.ID, - conn: hijack, - }, nil -} - -func (c *Container) execConfig(r ExecOpts, cmd []string) types.ExecConfig { - env := append(r.Env, fmt.Sprintf("RUNSC_TEST_NAME=%s", c.Name)) - return types.ExecConfig{ - AttachStdin: r.UseTTY, - AttachStderr: true, - AttachStdout: true, - Cmd: cmd, - Privileged: r.Privileged, - WorkingDir: r.WorkDir, - Env: env, - Tty: r.UseTTY, - User: r.User, - } - -} - -// Process represents a containerized process. -type Process struct { - container *Container - execid string - conn types.HijackedResponse -} - -// Write writes buf to the process's stdin. -func (p *Process) Write(timeout time.Duration, buf []byte) (int, error) { - p.conn.Conn.SetDeadline(time.Now().Add(timeout)) - return p.conn.Conn.Write(buf) -} - -// Read returns process's stdout and stderr. -func (p *Process) Read() (string, string, error) { - var stdout, stderr bytes.Buffer - if err := p.read(&stdout, &stderr); err != nil { - return "", "", err - } - return stdout.String(), stderr.String(), nil -} - -// Logs returns combined stdout/stderr from the process. -func (p *Process) Logs() (string, error) { - var out bytes.Buffer - if err := p.read(&out, &out); err != nil { - return "", err - } - return out.String(), nil -} - -func (p *Process) read(stdout, stderr *bytes.Buffer) error { - _, err := stdcopy.StdCopy(stdout, stderr, p.conn.Reader) - return err -} - -// ExitCode returns the process's exit code. -func (p *Process) ExitCode(ctx context.Context) (int, error) { - _, exitCode, err := p.runningExitCode(ctx) - return exitCode, err -} - -// IsRunning checks if the process is running. -func (p *Process) IsRunning(ctx context.Context) (bool, error) { - running, _, err := p.runningExitCode(ctx) - return running, err -} - -// WaitExitStatus until process completes and returns exit status. -func (p *Process) WaitExitStatus(ctx context.Context) (int, error) { - waitChan := make(chan (int)) - errChan := make(chan (error)) - - go func() { - for { - running, exitcode, err := p.runningExitCode(ctx) - if err != nil { - errChan <- fmt.Errorf("error waiting process %s: container %v", p.execid, p.container.Name) - } - if !running { - waitChan <- exitcode - } - time.Sleep(time.Millisecond * 500) - } - }() - - select { - case ws := <-waitChan: - return ws, nil - case err := <-errChan: - return -1, err - } -} - -// runningExitCode collects if the process is running and the exit code. -// The exit code is only valid if the process has exited. -func (p *Process) runningExitCode(ctx context.Context) (bool, int, error) { - // If execid is not empty, this is a execed process. - if p.execid != "" { - status, err := p.container.client.ContainerExecInspect(ctx, p.execid) - return status.Running, status.ExitCode, err - } - // else this is the root process. - status, err := p.container.Status(ctx) - return status.Running, status.ExitCode, err -} diff --git a/pkg/test/dockerutil/network.go b/pkg/test/dockerutil/network.go deleted file mode 100644 index 047091e75..000000000 --- a/pkg/test/dockerutil/network.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dockerutil - -import ( - "context" - "net" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/network" - "github.com/docker/docker/client" - "gvisor.dev/gvisor/pkg/test/testutil" -) - -// Network is a docker network. -type Network struct { - client *client.Client - id string - logger testutil.Logger - Name string - containers []*Container - Subnet *net.IPNet -} - -// NewNetwork sets up the struct for a Docker network. Names of networks -// will be unique. -func NewNetwork(ctx context.Context, logger testutil.Logger) *Network { - client, err := client.NewClientWithOpts(client.FromEnv) - if err != nil { - logger.Logf("create client failed with: %v", err) - return nil - } - client.NegotiateAPIVersion(ctx) - - return &Network{ - logger: logger, - Name: testutil.RandomID(logger.Name()), - client: client, - } -} - -func (n *Network) networkCreate() types.NetworkCreate { - - var subnet string - if n.Subnet != nil { - subnet = n.Subnet.String() - } - - ipam := network.IPAM{ - Config: []network.IPAMConfig{{ - Subnet: subnet, - }}, - } - - return types.NetworkCreate{ - CheckDuplicate: true, - IPAM: &ipam, - } -} - -// Create is analogous to 'docker network create'. -func (n *Network) Create(ctx context.Context) error { - - opts := n.networkCreate() - resp, err := n.client.NetworkCreate(ctx, n.Name, opts) - if err != nil { - return err - } - n.id = resp.ID - return nil -} - -// Connect is analogous to 'docker network connect' with the arguments provided. -func (n *Network) Connect(ctx context.Context, container *Container, ipv4, ipv6 string) error { - settings := network.EndpointSettings{ - IPAMConfig: &network.EndpointIPAMConfig{ - IPv4Address: ipv4, - IPv6Address: ipv6, - }, - } - err := n.client.NetworkConnect(ctx, n.id, container.id, &settings) - if err == nil { - n.containers = append(n.containers, container) - } - return err -} - -// Inspect returns this network's info. -func (n *Network) Inspect(ctx context.Context) (types.NetworkResource, error) { - return n.client.NetworkInspect(ctx, n.id, types.NetworkInspectOptions{Verbose: true}) -} - -// Cleanup cleans up the docker network and all the containers attached to it. -func (n *Network) Cleanup(ctx context.Context) error { - for _, c := range n.containers { - c.CleanUp(ctx) - } - n.containers = nil - - return n.client.NetworkRemove(ctx, n.id) -} diff --git a/pkg/test/dockerutil/profile.go b/pkg/test/dockerutil/profile.go deleted file mode 100644 index 55f9496cd..000000000 --- a/pkg/test/dockerutil/profile.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dockerutil - -import ( - "context" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "time" -) - -// Profile represents profile-like operations on a container, -// such as running perf or pprof. It is meant to be added to containers -// such that the container type calls the Profile during its lifecycle. -type Profile interface { - // OnCreate is called just after the container is created when the container - // has a valid ID (e.g. c.ID()). - OnCreate(c *Container) error - - // OnStart is called just after the container is started when the container - // has a valid Pid (e.g. c.SandboxPid()). - OnStart(c *Container) error - - // Restart restarts the Profile on request. - Restart(c *Container) error - - // OnCleanUp is called during the container's cleanup method. - // Cleanups should just log errors if they have them. - OnCleanUp(c *Container) error -} - -// Pprof is for running profiles with 'runsc debug'. Pprof workloads -// should be run as root and ONLY against runsc sandboxes. The runtime -// should have --profile set as an option in /etc/docker/daemon.json in -// order for profiling to work with Pprof. -type Pprof struct { - BasePath string // path to put profiles - BlockProfile bool - CPUProfile bool - HeapProfile bool - MutexProfile bool - Duration time.Duration // duration to run profiler e.g. '10s' or '1m'. - shouldRun bool - cmd *exec.Cmd - stdout io.ReadCloser - stderr io.ReadCloser -} - -// MakePprofFromFlags makes a Pprof profile from flags. -func MakePprofFromFlags(c *Container) *Pprof { - if !(*pprofBlock || *pprofCPU || *pprofHeap || *pprofMutex) { - return nil - } - return &Pprof{ - BasePath: filepath.Join(*pprofBaseDir, c.runtime, c.Name), - BlockProfile: *pprofBlock, - CPUProfile: *pprofCPU, - HeapProfile: *pprofHeap, - MutexProfile: *pprofMutex, - Duration: *duration, - } -} - -// OnCreate implements Profile.OnCreate. -func (p *Pprof) OnCreate(c *Container) error { - return os.MkdirAll(p.BasePath, 0755) -} - -// OnStart implements Profile.OnStart. -func (p *Pprof) OnStart(c *Container) error { - path, err := RuntimePath() - if err != nil { - return fmt.Errorf("failed to get runtime path: %v", err) - } - - // The root directory of this container's runtime. - root := fmt.Sprintf("--root=/var/run/docker/runtime-%s/moby", c.runtime) - // Format is `runsc --root=rootdir debug --profile-*=file --duration=* containerID`. - args := []string{root, "debug"} - args = append(args, p.makeProfileArgs(c)...) - args = append(args, c.ID()) - - // Best effort wait until container is running. - for now := time.Now(); time.Since(now) < 5*time.Second; { - if status, err := c.Status(context.Background()); err != nil { - return fmt.Errorf("failed to get status with: %v", err) - - } else if status.Running { - break - } - time.Sleep(500 * time.Millisecond) - } - p.cmd = exec.Command(path, args...) - if err := p.cmd.Start(); err != nil { - return fmt.Errorf("process failed: %v", err) - } - return nil -} - -// Restart implements Profile.Restart. -func (p *Pprof) Restart(c *Container) error { - p.OnCleanUp(c) - return p.OnStart(c) -} - -// OnCleanUp implements Profile.OnCleanup -func (p *Pprof) OnCleanUp(c *Container) error { - defer func() { p.cmd = nil }() - if p.cmd != nil && p.cmd.Process != nil && p.cmd.ProcessState != nil && !p.cmd.ProcessState.Exited() { - return p.cmd.Process.Kill() - } - return nil -} - -// makeProfileArgs turns Pprof fields into runsc debug flags. -func (p *Pprof) makeProfileArgs(c *Container) []string { - var ret []string - if p.BlockProfile { - ret = append(ret, fmt.Sprintf("--profile-block=%s", filepath.Join(p.BasePath, "block.pprof"))) - } - if p.CPUProfile { - ret = append(ret, fmt.Sprintf("--profile-cpu=%s", filepath.Join(p.BasePath, "cpu.pprof"))) - } - if p.HeapProfile { - ret = append(ret, fmt.Sprintf("--profile-heap=%s", filepath.Join(p.BasePath, "heap.pprof"))) - } - if p.MutexProfile { - ret = append(ret, fmt.Sprintf("--profile-mutex=%s", filepath.Join(p.BasePath, "mutex.pprof"))) - } - ret = append(ret, fmt.Sprintf("--duration=%s", p.Duration)) - return ret -} diff --git a/pkg/test/dockerutil/profile_test.go b/pkg/test/dockerutil/profile_test.go deleted file mode 100644 index 8c4ffe483..000000000 --- a/pkg/test/dockerutil/profile_test.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dockerutil - -import ( - "context" - "fmt" - "os" - "path/filepath" - "testing" - "time" -) - -type testCase struct { - name string - pprof Pprof - expectedFiles []string -} - -func TestPprof(t *testing.T) { - // Basepath and expected file names for each type of profile. - basePath := "/tmp/test/profile" - block := "block.pprof" - cpu := "cpu.pprof" - goprofle := "go.pprof" - heap := "heap.pprof" - mutex := "mutex.pprof" - - testCases := []testCase{ - { - name: "Cpu", - pprof: Pprof{ - BasePath: basePath, - CPUProfile: true, - Duration: 2 * time.Second, - }, - expectedFiles: []string{cpu}, - }, - { - name: "All", - pprof: Pprof{ - BasePath: basePath, - BlockProfile: true, - CPUProfile: true, - HeapProfile: true, - MutexProfile: true, - Duration: 2 * time.Second, - }, - expectedFiles: []string{block, cpu, goprofle, heap, mutex}, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - ctx := context.Background() - c := MakeContainer(ctx, t) - // Set basepath to include the container name so there are no conflicts. - tc.pprof.BasePath = filepath.Join(tc.pprof.BasePath, c.Name) - c.AddProfile(&tc.pprof) - - func() { - defer c.CleanUp(ctx) - // Start a container. - if err := c.Spawn(ctx, RunOpts{ - Image: "basic/alpine", - }, "sleep", "1000"); err != nil { - t.Fatalf("run failed with: %v", err) - } - - if status, err := c.Status(context.Background()); !status.Running { - t.Fatalf("container is not yet running: %+v err: %v", status, err) - } - - // End early if the expected files exist and have data. - for start := time.Now(); time.Since(start) < tc.pprof.Duration; time.Sleep(500 * time.Millisecond) { - if err := checkFiles(tc); err == nil { - break - } - } - }() - - // Check all expected files exist and have data. - if err := checkFiles(tc); err != nil { - t.Fatalf(err.Error()) - } - }) - } -} - -func checkFiles(tc testCase) error { - for _, file := range tc.expectedFiles { - stat, err := os.Stat(filepath.Join(tc.pprof.BasePath, file)) - if err != nil { - return fmt.Errorf("stat failed with: %v", err) - } else if stat.Size() < 1 { - return fmt.Errorf("file not written to: %+v", stat) - } - } - return nil -} - -func TestMain(m *testing.M) { - EnsureSupportedDockerVersion() - os.Exit(m.Run()) -} |