diff options
Diffstat (limited to 'pkg/test')
| -rw-r--r-- | pkg/test/criutil/BUILD | 14 | ||||
| -rw-r--r-- | pkg/test/criutil/criutil.go | 372 | ||||
| -rw-r--r-- | pkg/test/dockerutil/BUILD | 43 | ||||
| -rw-r--r-- | pkg/test/dockerutil/README.md | 86 | ||||
| -rw-r--r-- | pkg/test/dockerutil/container.go | 548 | ||||
| -rw-r--r-- | pkg/test/dockerutil/dockerutil.go | 172 | ||||
| -rw-r--r-- | pkg/test/dockerutil/exec.go | 188 | ||||
| -rw-r--r-- | pkg/test/dockerutil/network.go | 110 | ||||
| -rw-r--r-- | pkg/test/dockerutil/profile.go | 150 | ||||
| -rw-r--r-- | pkg/test/dockerutil/profile_test.go | 125 | ||||
| -rw-r--r-- | pkg/test/testutil/BUILD | 23 | ||||
| -rw-r--r-- | pkg/test/testutil/sh.go | 515 | ||||
| -rw-r--r-- | pkg/test/testutil/testutil.go | 597 | ||||
| -rw-r--r-- | pkg/test/testutil/testutil_runfiles.go | 75 |
14 files changed, 0 insertions, 3018 deletions
diff --git a/pkg/test/criutil/BUILD b/pkg/test/criutil/BUILD deleted file mode 100644 index a7b082cee..000000000 --- a/pkg/test/criutil/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -package(licenses = ["notice"]) - -go_library( - name = "criutil", - testonly = 1, - srcs = ["criutil.go"], - visibility = ["//:sandbox"], - deps = [ - "//pkg/test/dockerutil", - "//pkg/test/testutil", - ], -) diff --git a/pkg/test/criutil/criutil.go b/pkg/test/criutil/criutil.go deleted file mode 100644 index 3b41a2824..000000000 --- a/pkg/test/criutil/criutil.go +++ /dev/null @@ -1,372 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package criutil contains utility functions for interacting with the -// Container Runtime Interface (CRI), principally via the crictl command line -// tool. This requires critools to be installed on the local system. -package criutil - -import ( - "encoding/json" - "fmt" - "os" - "os/exec" - "path" - "regexp" - "strconv" - "strings" - "time" - - "gvisor.dev/gvisor/pkg/test/dockerutil" - "gvisor.dev/gvisor/pkg/test/testutil" -) - -// Crictl contains information required to run the crictl utility. -type Crictl struct { - logger testutil.Logger - endpoint string - cleanup []func() -} - -// ResolvePath attempts to find binary paths. It may set the path to invalid, -// which will cause the execution to fail with a sensible error. -func ResolvePath(executable string) string { - runtime, err := dockerutil.RuntimePath() - if err == nil { - // Check first the directory of the runtime itself. - if dir := path.Dir(runtime); dir != "" && dir != "." { - guess := path.Join(dir, executable) - if fi, err := os.Stat(guess); err == nil && (fi.Mode()&0111) != 0 { - return guess - } - } - } - - // Favor /usr/local/bin, if it exists. - localBin := fmt.Sprintf("/usr/local/bin/%s", executable) - if _, err := os.Stat(localBin); err == nil { - return localBin - } - - // Try to find via the path. - guess, _ := exec.LookPath(executable) - if err == nil { - return guess - } - - // Return a bare path; this generates a suitable error. - return executable -} - -// NewCrictl returns a Crictl configured with a timeout and an endpoint over -// which it will talk to containerd. -func NewCrictl(logger testutil.Logger, endpoint string) *Crictl { - // Attempt to find the executable, but don't bother propagating the - // error at this point. The first command executed will return with a - // binary not found error. - return &Crictl{ - logger: logger, - endpoint: endpoint, - } -} - -// CleanUp executes cleanup functions. -func (cc *Crictl) CleanUp() { - for _, c := range cc.cleanup { - c() - } - cc.cleanup = nil -} - -// RunPod creates a sandbox. It corresponds to `crictl runp`. -func (cc *Crictl) RunPod(runtime, sbSpecFile string) (string, error) { - podID, err := cc.run("runp", "--runtime", runtime, sbSpecFile) - if err != nil { - return "", fmt.Errorf("runp failed: %v", err) - } - // Strip the trailing newline from crictl output. - return strings.TrimSpace(podID), nil -} - -// Create creates a container within a sandbox. It corresponds to `crictl -// create`. -func (cc *Crictl) Create(podID, contSpecFile, sbSpecFile string) (string, error) { - // In version 1.16.0, crictl annoying starting attempting to pull the - // container, even if it was already available locally. We therefore - // need to parse the version and add an appropriate --no-pull argument - // since the image has already been loaded locally. - out, err := cc.run("-v") - if err != nil { - return "", err - } - r := regexp.MustCompile("crictl version ([0-9]+)\\.([0-9]+)\\.([0-9+])") - vs := r.FindStringSubmatch(out) - if len(vs) != 4 { - return "", fmt.Errorf("crictl -v had unexpected output: %s", out) - } - major, err := strconv.ParseUint(vs[1], 10, 64) - if err != nil { - return "", fmt.Errorf("crictl had invalid version: %v (%s)", err, out) - } - minor, err := strconv.ParseUint(vs[2], 10, 64) - if err != nil { - return "", fmt.Errorf("crictl had invalid version: %v (%s)", err, out) - } - - args := []string{"create"} - if (major == 1 && minor >= 16) || major > 1 { - args = append(args, "--no-pull") - } - args = append(args, podID) - args = append(args, contSpecFile) - args = append(args, sbSpecFile) - - podID, err = cc.run(args...) - if err != nil { - time.Sleep(10 * time.Minute) // XXX - return "", fmt.Errorf("create failed: %v", err) - } - - // Strip the trailing newline from crictl output. - return strings.TrimSpace(podID), nil -} - -// Start starts a container. It corresponds to `crictl start`. -func (cc *Crictl) Start(contID string) (string, error) { - output, err := cc.run("start", contID) - if err != nil { - return "", fmt.Errorf("start failed: %v", err) - } - return output, nil -} - -// Stop stops a container. It corresponds to `crictl stop`. -func (cc *Crictl) Stop(contID string) error { - _, err := cc.run("stop", contID) - return err -} - -// Exec execs a program inside a container. It corresponds to `crictl exec`. -func (cc *Crictl) Exec(contID string, args ...string) (string, error) { - a := []string{"exec", contID} - a = append(a, args...) - output, err := cc.run(a...) - if err != nil { - return "", fmt.Errorf("exec failed: %v", err) - } - return output, nil -} - -// Logs retrieves the container logs. It corresponds to `crictl logs`. -func (cc *Crictl) Logs(contID string, args ...string) (string, error) { - a := []string{"logs", contID} - a = append(a, args...) - output, err := cc.run(a...) - if err != nil { - return "", fmt.Errorf("logs failed: %v", err) - } - return output, nil -} - -// Rm removes a container. It corresponds to `crictl rm`. -func (cc *Crictl) Rm(contID string) error { - _, err := cc.run("rm", contID) - return err -} - -// StopPod stops a pod. It corresponds to `crictl stopp`. -func (cc *Crictl) StopPod(podID string) error { - _, err := cc.run("stopp", podID) - return err -} - -// containsConfig is a minimal copy of -// https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/apis/cri/runtime/v1alpha2/api.proto -// It only contains fields needed for testing. -type containerConfig struct { - Status containerStatus -} - -type containerStatus struct { - Network containerNetwork -} - -type containerNetwork struct { - IP string -} - -// PodIP returns a pod's IP address. -func (cc *Crictl) PodIP(podID string) (string, error) { - output, err := cc.run("inspectp", podID) - if err != nil { - return "", err - } - conf := &containerConfig{} - if err := json.Unmarshal([]byte(output), conf); err != nil { - return "", fmt.Errorf("failed to unmarshal JSON: %v, %s", err, output) - } - if conf.Status.Network.IP == "" { - return "", fmt.Errorf("no IP found in config: %s", output) - } - return conf.Status.Network.IP, nil -} - -// RmPod removes a container. It corresponds to `crictl rmp`. -func (cc *Crictl) RmPod(podID string) error { - _, err := cc.run("rmp", podID) - return err -} - -// Import imports the given container from the local Docker instance. -func (cc *Crictl) Import(image string) error { - // Note that we provide a 10 minute timeout after connect because we may - // be pushing a lot of bytes in order to import the image. The connect - // timeout stays the same and is inherited from the Crictl instance. - cmd := testutil.Command(cc.logger, - ResolvePath("ctr"), - fmt.Sprintf("--connect-timeout=%s", 30*time.Second), - fmt.Sprintf("--address=%s", cc.endpoint), - "-n", "k8s.io", "images", "import", "-") - cmd.Stderr = os.Stderr // Pass through errors. - - // Create a pipe and start the program. - w, err := cmd.StdinPipe() - if err != nil { - return err - } - if err := cmd.Start(); err != nil { - return err - } - - // Save the image on the other end. - if err := dockerutil.Save(cc.logger, image, w); err != nil { - cmd.Wait() - return err - } - - // Close our pipe reference & see if it was loaded. - if err := w.Close(); err != nil { - return w.Close() - } - - return cmd.Wait() -} - -// StartContainer pulls the given image ands starts the container in the -// sandbox with the given podID. -// -// Note that the image will always be imported from the local docker daemon. -func (cc *Crictl) StartContainer(podID, image, sbSpec, contSpec string) (string, error) { - if err := cc.Import(image); err != nil { - return "", err - } - - // Write the specs to files that can be read by crictl. - sbSpecFile, cleanup, err := testutil.WriteTmpFile("sbSpec", sbSpec) - if err != nil { - return "", fmt.Errorf("failed to write sandbox spec: %v", err) - } - cc.cleanup = append(cc.cleanup, cleanup) - contSpecFile, cleanup, err := testutil.WriteTmpFile("contSpec", contSpec) - if err != nil { - return "", fmt.Errorf("failed to write container spec: %v", err) - } - cc.cleanup = append(cc.cleanup, cleanup) - - return cc.startContainer(podID, image, sbSpecFile, contSpecFile) -} - -func (cc *Crictl) startContainer(podID, image, sbSpecFile, contSpecFile string) (string, error) { - contID, err := cc.Create(podID, contSpecFile, sbSpecFile) - if err != nil { - return "", fmt.Errorf("failed to create container in pod %q: %v", podID, err) - } - - if _, err := cc.Start(contID); err != nil { - return "", fmt.Errorf("failed to start container %q in pod %q: %v", contID, podID, err) - } - - return contID, nil -} - -// StopContainer stops and deletes the container with the given container ID. -func (cc *Crictl) StopContainer(contID string) error { - if err := cc.Stop(contID); err != nil { - return fmt.Errorf("failed to stop container %q: %v", contID, err) - } - - if err := cc.Rm(contID); err != nil { - return fmt.Errorf("failed to remove container %q: %v", contID, err) - } - - return nil -} - -// StartPodAndContainer starts a sandbox and container in that sandbox. It -// returns the pod ID and container ID. -func (cc *Crictl) StartPodAndContainer(runtime, image, sbSpec, contSpec string) (string, string, error) { - if err := cc.Import(image); err != nil { - return "", "", err - } - - // Write the specs to files that can be read by crictl. - sbSpecFile, cleanup, err := testutil.WriteTmpFile("sbSpec", sbSpec) - if err != nil { - return "", "", fmt.Errorf("failed to write sandbox spec: %v", err) - } - cc.cleanup = append(cc.cleanup, cleanup) - contSpecFile, cleanup, err := testutil.WriteTmpFile("contSpec", contSpec) - if err != nil { - return "", "", fmt.Errorf("failed to write container spec: %v", err) - } - cc.cleanup = append(cc.cleanup, cleanup) - - podID, err := cc.RunPod(runtime, sbSpecFile) - if err != nil { - return "", "", err - } - - contID, err := cc.startContainer(podID, image, sbSpecFile, contSpecFile) - - return podID, contID, err -} - -// StopPodAndContainer stops a container and pod. -func (cc *Crictl) StopPodAndContainer(podID, contID string) error { - if err := cc.StopContainer(contID); err != nil { - return fmt.Errorf("failed to stop container %q in pod %q: %v", contID, podID, err) - } - - if err := cc.StopPod(podID); err != nil { - return fmt.Errorf("failed to stop pod %q: %v", podID, err) - } - - if err := cc.RmPod(podID); err != nil { - return fmt.Errorf("failed to remove pod %q: %v", podID, err) - } - - return nil -} - -// run runs crictl with the given args. -func (cc *Crictl) run(args ...string) (string, error) { - defaultArgs := []string{ - ResolvePath("crictl"), - "--image-endpoint", fmt.Sprintf("unix://%s", cc.endpoint), - "--runtime-endpoint", fmt.Sprintf("unix://%s", cc.endpoint), - } - fullArgs := append(defaultArgs, args...) - out, err := testutil.Command(cc.logger, fullArgs...).CombinedOutput() - return string(out), err -} diff --git a/pkg/test/dockerutil/BUILD b/pkg/test/dockerutil/BUILD deleted file mode 100644 index 366f068e3..000000000 --- a/pkg/test/dockerutil/BUILD +++ /dev/null @@ -1,43 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "dockerutil", - testonly = 1, - srcs = [ - "container.go", - "dockerutil.go", - "exec.go", - "network.go", - "profile.go", - ], - visibility = ["//:sandbox"], - deps = [ - "//pkg/test/testutil", - "@com_github_docker_docker//api/types:go_default_library", - "@com_github_docker_docker//api/types/container:go_default_library", - "@com_github_docker_docker//api/types/mount:go_default_library", - "@com_github_docker_docker//api/types/network:go_default_library", - "@com_github_docker_docker//client:go_default_library", - "@com_github_docker_docker//pkg/stdcopy:go_default_library", - "@com_github_docker_go_connections//nat:go_default_library", - "@org_golang_x_sys//unix:go_default_library", - ], -) - -go_test( - name = "profile_test", - size = "large", - srcs = [ - "profile_test.go", - ], - library = ":dockerutil", - tags = [ - # Requires docker and runsc to be configured before test runs. - # Also requires the test to be run as root. - "local", - "manual", - ], - visibility = ["//:sandbox"], -) diff --git a/pkg/test/dockerutil/README.md b/pkg/test/dockerutil/README.md deleted file mode 100644 index 870292096..000000000 --- a/pkg/test/dockerutil/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# dockerutil - -This package is for creating and controlling docker containers for testing -runsc, gVisor's docker/kubernetes binary. A simple test may look like: - -``` - func TestSuperCool(t *testing.T) { - ctx := context.Background() - c := dockerutil.MakeContainer(ctx, t) - got, err := c.Run(ctx, dockerutil.RunOpts{ - Image: "basic/alpine" - }, "echo", "super cool") - if err != nil { - t.Fatalf("err was not nil: %v", err) - } - want := "super cool" - if !strings.Contains(got, want){ - t.Fatalf("want: %s, got: %s", want, got) - } - } -``` - -For further examples, see many of our end to end tests elsewhere in the repo, -such as those in //test/e2e or benchmarks at //test/benchmarks. - -dockerutil uses the "official" docker golang api, which is -[very powerful](https://godoc.org/github.com/docker/docker/client). dockerutil -is a thin wrapper around this API, allowing desired new use cases to be easily -implemented. - -## Profiling - -dockerutil is capable of generating profiles. Currently, the only option is to -use pprof profiles generated by `runsc debug`. The profiler will generate Block, -CPU, Heap, Goroutine, and Mutex profiles. To generate profiles: - -* Install runsc with the `--profile` flag: `make configure RUNTIME=myrunsc - ARGS="--profile"` Also add other flags with ARGS like `--platform=kvm` or - `--vfs2`. -* Restart docker: `sudo service docker restart` - -To run and generate CPU profiles run: - -``` -make sudo TARGETS=//path/to:target \ - ARGS="--runtime=myrunsc -test.v -test.bench=. --pprof-cpu" OPTIONS="-c opt" -``` - -Profiles would be at: `/tmp/profile/myrunsc/CONTAINERNAME/cpu.pprof` - -Container name in most tests and benchmarks in gVisor is usually the test name -and some random characters like so: -`BenchmarkABSL-CleanCache-JF2J2ZYF3U7SL47QAA727CSJI3C4ZAW2` - -Profiling requires root as runsc debug inspects running containers in /var/run -among other things. - -### Writing for Profiling - -The below shows an example of using profiles with dockerutil. - -``` -func TestSuperCool(t *testing.T){ - ctx := context.Background() - // profiled and using runtime from dockerutil.runtime flag - profiled := MakeContainer() - - // not profiled and using runtime runc - native := MakeNativeContainer() - - err := profiled.Spawn(ctx, RunOpts{ - Image: "some/image", - }, "sleep", "100000") - // profiling has begun here - ... - expensive setup that I don't want to profile. - ... - profiled.RestartProfiles() - // profiled activity -} -``` - -In the above example, `profiled` would be profiled and `native` would not. The -call to `RestartProfiles()` restarts the clock on profiling. This is useful if -the main activity being tested is done with `docker exec` or `container.Spawn()` -followed by one or more `container.Exec()` calls. diff --git a/pkg/test/dockerutil/container.go b/pkg/test/dockerutil/container.go deleted file mode 100644 index 06152a444..000000000 --- a/pkg/test/dockerutil/container.go +++ /dev/null @@ -1,548 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dockerutil - -import ( - "bytes" - "context" - "errors" - "fmt" - "io/ioutil" - "net" - "os" - "path" - "path/filepath" - "regexp" - "strconv" - "strings" - "time" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/mount" - "github.com/docker/docker/api/types/network" - "github.com/docker/docker/client" - "github.com/docker/docker/pkg/stdcopy" - "github.com/docker/go-connections/nat" - "gvisor.dev/gvisor/pkg/test/testutil" -) - -// Container represents a Docker Container allowing -// user to configure and control as one would with the 'docker' -// client. Container is backed by the offical golang docker API. -// See: https://pkg.go.dev/github.com/docker/docker. -type Container struct { - Name string - runtime string - - logger testutil.Logger - client *client.Client - id string - mounts []mount.Mount - links []string - copyErr error - cleanups []func() - - // profile is the profiling hook associated with this container. - profile *profile -} - -// RunOpts are options for running a container. -type RunOpts struct { - // Image is the image relative to images/. This will be mangled - // appropriately, to ensure that only first-party images are used. - Image string - - // Memory is the memory limit in bytes. - Memory int - - // Cpus in which to allow execution. ("0", "1", "0-2"). - CpusetCpus string - - // Ports are the ports to be allocated. - Ports []int - - // WorkDir sets the working directory. - WorkDir string - - // ReadOnly sets the read-only flag. - ReadOnly bool - - // Env are additional environment variables. - Env []string - - // User is the user to use. - User string - - // Privileged enables privileged mode. - Privileged bool - - // CapAdd are the extra set of capabilities to add. - CapAdd []string - - // CapDrop are the extra set of capabilities to drop. - CapDrop []string - - // Mounts is the list of directories/files to be mounted inside the container. - Mounts []mount.Mount - - // Links is the list of containers to be connected to the container. - Links []string -} - -func makeContainer(ctx context.Context, logger testutil.Logger, runtime string) *Container { - // Slashes are not allowed in container names. - name := testutil.RandomID(logger.Name()) - name = strings.ReplaceAll(name, "/", "-") - client, err := client.NewClientWithOpts(client.FromEnv) - if err != nil { - return nil - } - client.NegotiateAPIVersion(ctx) - return &Container{ - logger: logger, - Name: name, - runtime: runtime, - client: client, - } -} - -// MakeContainer constructs a suitable Container object. -// -// The runtime used is determined by the runtime flag. -// -// Containers will check flags for profiling requests. -func MakeContainer(ctx context.Context, logger testutil.Logger) *Container { - return makeContainer(ctx, logger, *runtime) -} - -// MakeNativeContainer constructs a suitable Container object. -// -// The runtime used will be the system default. -// -// Native containers aren't profiled. -func MakeNativeContainer(ctx context.Context, logger testutil.Logger) *Container { - return makeContainer(ctx, logger, "" /*runtime*/) -} - -// Spawn is analogous to 'docker run -d'. -func (c *Container) Spawn(ctx context.Context, r RunOpts, args ...string) error { - if err := c.create(ctx, r.Image, c.config(r, args), c.hostConfig(r), nil); err != nil { - return err - } - return c.Start(ctx) -} - -// SpawnProcess is analogous to 'docker run -it'. It returns a process -// which represents the root process. -func (c *Container) SpawnProcess(ctx context.Context, r RunOpts, args ...string) (Process, error) { - config, hostconf, netconf := c.ConfigsFrom(r, args...) - config.Tty = true - config.OpenStdin = true - - if err := c.CreateFrom(ctx, r.Image, config, hostconf, netconf); err != nil { - return Process{}, err - } - - // Open a connection to the container for parsing logs and for TTY. - stream, err := c.client.ContainerAttach(ctx, c.id, - types.ContainerAttachOptions{ - Stream: true, - Stdin: true, - Stdout: true, - Stderr: true, - }) - if err != nil { - return Process{}, fmt.Errorf("connect failed container id %s: %v", c.id, err) - } - - c.cleanups = append(c.cleanups, func() { stream.Close() }) - - if err := c.Start(ctx); err != nil { - return Process{}, err - } - - return Process{container: c, conn: stream}, nil -} - -// Run is analogous to 'docker run'. -func (c *Container) Run(ctx context.Context, r RunOpts, args ...string) (string, error) { - if err := c.create(ctx, r.Image, c.config(r, args), c.hostConfig(r), nil); err != nil { - return "", err - } - - if err := c.Start(ctx); err != nil { - return "", err - } - - if err := c.Wait(ctx); err != nil { - return "", err - } - - return c.Logs(ctx) -} - -// ConfigsFrom returns container configs from RunOpts and args. The caller should call 'CreateFrom' -// and Start. -func (c *Container) ConfigsFrom(r RunOpts, args ...string) (*container.Config, *container.HostConfig, *network.NetworkingConfig) { - return c.config(r, args), c.hostConfig(r), &network.NetworkingConfig{} -} - -// MakeLink formats a link to add to a RunOpts. -func (c *Container) MakeLink(target string) string { - return fmt.Sprintf("%s:%s", c.Name, target) -} - -// CreateFrom creates a container from the given configs. -func (c *Container) CreateFrom(ctx context.Context, profileImage string, conf *container.Config, hostconf *container.HostConfig, netconf *network.NetworkingConfig) error { - return c.create(ctx, profileImage, conf, hostconf, netconf) -} - -// Create is analogous to 'docker create'. -func (c *Container) Create(ctx context.Context, r RunOpts, args ...string) error { - return c.create(ctx, r.Image, c.config(r, args), c.hostConfig(r), nil) -} - -func (c *Container) create(ctx context.Context, profileImage string, conf *container.Config, hostconf *container.HostConfig, netconf *network.NetworkingConfig) error { - if c.runtime != "" && c.runtime != "runc" { - // Use the image name as provided here; which normally represents the - // unmodified "basic/alpine" image name. This should be easy to grok. - c.profileInit(profileImage) - } - cont, err := c.client.ContainerCreate(ctx, conf, hostconf, nil, c.Name) - if err != nil { - return err - } - c.id = cont.ID - return nil -} - -func (c *Container) config(r RunOpts, args []string) *container.Config { - ports := nat.PortSet{} - for _, p := range r.Ports { - port := nat.Port(fmt.Sprintf("%d", p)) - ports[port] = struct{}{} - } - env := append(r.Env, fmt.Sprintf("RUNSC_TEST_NAME=%s", c.Name)) - - return &container.Config{ - Image: testutil.ImageByName(r.Image), - Cmd: args, - ExposedPorts: ports, - Env: env, - WorkingDir: r.WorkDir, - User: r.User, - } -} - -func (c *Container) hostConfig(r RunOpts) *container.HostConfig { - c.mounts = append(c.mounts, r.Mounts...) - - return &container.HostConfig{ - Runtime: c.runtime, - Mounts: c.mounts, - PublishAllPorts: true, - Links: r.Links, - CapAdd: r.CapAdd, - CapDrop: r.CapDrop, - Privileged: r.Privileged, - ReadonlyRootfs: r.ReadOnly, - Resources: container.Resources{ - Memory: int64(r.Memory), // In bytes. - CpusetCpus: r.CpusetCpus, - }, - } -} - -// Start is analogous to 'docker start'. -func (c *Container) Start(ctx context.Context) error { - if err := c.client.ContainerStart(ctx, c.id, types.ContainerStartOptions{}); err != nil { - return fmt.Errorf("ContainerStart failed: %v", err) - } - - if c.profile != nil { - if err := c.profile.Start(c); err != nil { - c.logger.Logf("profile.Start failed: %v", err) - } - } - - return nil -} - -// Stop is analogous to 'docker stop'. -func (c *Container) Stop(ctx context.Context) error { - return c.client.ContainerStop(ctx, c.id, nil) -} - -// Pause is analogous to'docker pause'. -func (c *Container) Pause(ctx context.Context) error { - return c.client.ContainerPause(ctx, c.id) -} - -// Unpause is analogous to 'docker unpause'. -func (c *Container) Unpause(ctx context.Context) error { - return c.client.ContainerUnpause(ctx, c.id) -} - -// Checkpoint is analogous to 'docker checkpoint'. -func (c *Container) Checkpoint(ctx context.Context, name string) error { - return c.client.CheckpointCreate(ctx, c.Name, types.CheckpointCreateOptions{CheckpointID: name, Exit: true}) -} - -// Restore is analogous to 'docker start --checkname [name]'. -func (c *Container) Restore(ctx context.Context, name string) error { - return c.client.ContainerStart(ctx, c.id, types.ContainerStartOptions{CheckpointID: name}) -} - -// Logs is analogous 'docker logs'. -func (c *Container) Logs(ctx context.Context) (string, error) { - var out bytes.Buffer - err := c.logs(ctx, &out, &out) - return out.String(), err -} - -func (c *Container) logs(ctx context.Context, stdout, stderr *bytes.Buffer) error { - opts := types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true} - writer, err := c.client.ContainerLogs(ctx, c.id, opts) - if err != nil { - return err - } - defer writer.Close() - _, err = stdcopy.StdCopy(stdout, stderr, writer) - - return err -} - -// ID returns the container id. -func (c *Container) ID() string { - return c.id -} - -// SandboxPid returns the container's pid. -func (c *Container) SandboxPid(ctx context.Context) (int, error) { - resp, err := c.client.ContainerInspect(ctx, c.id) - if err != nil { - return -1, err - } - return resp.ContainerJSONBase.State.Pid, nil -} - -// ErrNoIP indicates that no IP address is available. -var ErrNoIP = errors.New("no IP available") - -// FindIP returns the IP address of the container. -func (c *Container) FindIP(ctx context.Context, ipv6 bool) (net.IP, error) { - resp, err := c.client.ContainerInspect(ctx, c.id) - if err != nil { - return nil, err - } - - var ip net.IP - if ipv6 { - ip = net.ParseIP(resp.NetworkSettings.DefaultNetworkSettings.GlobalIPv6Address) - } else { - ip = net.ParseIP(resp.NetworkSettings.DefaultNetworkSettings.IPAddress) - } - if ip == nil { - return net.IP{}, ErrNoIP - } - return ip, nil -} - -// FindPort returns the host port that is mapped to 'sandboxPort'. -func (c *Container) FindPort(ctx context.Context, sandboxPort int) (int, error) { - desc, err := c.client.ContainerInspect(ctx, c.id) - if err != nil { - return -1, fmt.Errorf("error retrieving port: %v", err) - } - - format := fmt.Sprintf("%d/tcp", sandboxPort) - ports, ok := desc.NetworkSettings.Ports[nat.Port(format)] - if !ok { - return -1, fmt.Errorf("error retrieving port: %v", err) - - } - - port, err := strconv.Atoi(ports[0].HostPort) - if err != nil { - return -1, fmt.Errorf("error parsing port %q: %v", port, err) - } - return port, nil -} - -// CopyFiles copies in and mounts the given files. They are always ReadOnly. -func (c *Container) CopyFiles(opts *RunOpts, target string, sources ...string) { - dir, err := ioutil.TempDir("", c.Name) - if err != nil { - c.copyErr = fmt.Errorf("ioutil.TempDir failed: %v", err) - return - } - c.cleanups = append(c.cleanups, func() { os.RemoveAll(dir) }) - if err := os.Chmod(dir, 0755); err != nil { - c.copyErr = fmt.Errorf("os.Chmod(%q, 0755) failed: %v", dir, err) - return - } - for _, name := range sources { - src := name - if !filepath.IsAbs(src) { - src, err = testutil.FindFile(name) - if err != nil { - c.copyErr = fmt.Errorf("testutil.FindFile(%q) failed: %w", name, err) - return - } - } - dst := path.Join(dir, path.Base(name)) - if err := testutil.Copy(src, dst); err != nil { - c.copyErr = fmt.Errorf("testutil.Copy(%q, %q) failed: %v", src, dst, err) - return - } - c.logger.Logf("copy: %s -> %s", src, dst) - } - opts.Mounts = append(opts.Mounts, mount.Mount{ - Type: mount.TypeBind, - Source: dir, - Target: target, - ReadOnly: false, - }) -} - -// Status inspects the container returns its status. -func (c *Container) Status(ctx context.Context) (types.ContainerState, error) { - resp, err := c.client.ContainerInspect(ctx, c.id) - if err != nil { - return types.ContainerState{}, err - } - return *resp.State, err -} - -// Wait waits for the container to exit. -func (c *Container) Wait(ctx context.Context) error { - defer c.stopProfiling() - statusChan, errChan := c.client.ContainerWait(ctx, c.id, container.WaitConditionNotRunning) - select { - case err := <-errChan: - return err - case res := <-statusChan: - if res.StatusCode != 0 { - var msg string - if res.Error != nil { - msg = res.Error.Message - } - return fmt.Errorf("container returned non-zero status: %d, msg: %q", res.StatusCode, msg) - } - return nil - } -} - -// WaitTimeout waits for the container to exit with a timeout. -func (c *Container) WaitTimeout(ctx context.Context, timeout time.Duration) error { - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - statusChan, errChan := c.client.ContainerWait(ctx, c.id, container.WaitConditionNotRunning) - select { - case <-ctx.Done(): - if ctx.Err() == context.DeadlineExceeded { - return fmt.Errorf("container %s timed out after %v seconds", c.Name, timeout.Seconds()) - } - return nil - case err := <-errChan: - return err - case <-statusChan: - return nil - } -} - -// WaitForOutput searches container logs for pattern and returns or timesout. -func (c *Container) WaitForOutput(ctx context.Context, pattern string, timeout time.Duration) (string, error) { - matches, err := c.WaitForOutputSubmatch(ctx, pattern, timeout) - if err != nil { - return "", err - } - if len(matches) == 0 { - return "", fmt.Errorf("didn't find pattern %s logs", pattern) - } - return matches[0], nil -} - -// WaitForOutputSubmatch searches container logs for the given -// pattern or times out. It returns any regexp submatches as well. -func (c *Container) WaitForOutputSubmatch(ctx context.Context, pattern string, timeout time.Duration) ([]string, error) { - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - re := regexp.MustCompile(pattern) - for { - logs, err := c.Logs(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get logs: %v logs: %s", err, logs) - } - if matches := re.FindStringSubmatch(logs); matches != nil { - return matches, nil - } - time.Sleep(50 * time.Millisecond) - } -} - -// stopProfiling stops profiling. -func (c *Container) stopProfiling() { - if c.profile != nil { - if err := c.profile.Stop(c); err != nil { - // This most likely means that the runtime for the container - // was too short to connect and actually get a profile. - c.logger.Logf("warning: profile.Stop failed: %v", err) - } - } -} - -// Kill kills the container. -func (c *Container) Kill(ctx context.Context) error { - defer c.stopProfiling() - return c.client.ContainerKill(ctx, c.id, "") -} - -// Remove is analogous to 'docker rm'. -func (c *Container) Remove(ctx context.Context) error { - // Remove the image. - remove := types.ContainerRemoveOptions{ - RemoveVolumes: c.mounts != nil, - RemoveLinks: c.links != nil, - Force: true, - } - return c.client.ContainerRemove(ctx, c.Name, remove) -} - -// CleanUp kills and deletes the container (best effort). -func (c *Container) CleanUp(ctx context.Context) { - // Execute all cleanups. We execute cleanups here to close any - // open connections to the container before closing. Open connections - // can cause Kill and Remove to hang. - for _, c := range c.cleanups { - c() - } - c.cleanups = nil - - // Kill the container. - if err := c.Kill(ctx); err != nil && !strings.Contains(err.Error(), "is not running") { - // Just log; can't do anything here. - c.logger.Logf("error killing container %q: %v", c.Name, err) - } - - // Remove the image. - if err := c.Remove(ctx); err != nil { - c.logger.Logf("error removing container %q: %v", c.Name, err) - } - - // Forget all mounts. - c.mounts = nil -} diff --git a/pkg/test/dockerutil/dockerutil.go b/pkg/test/dockerutil/dockerutil.go deleted file mode 100644 index a40005799..000000000 --- a/pkg/test/dockerutil/dockerutil.go +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package dockerutil is a collection of utility functions. -package dockerutil - -import ( - "encoding/json" - "flag" - "fmt" - "io" - "io/ioutil" - "log" - "os/exec" - "regexp" - "strconv" - "time" - - "gvisor.dev/gvisor/pkg/test/testutil" -) - -var ( - // runtime is the runtime to use for tests. This will be applied to all - // containers. Note that the default here ("runsc") corresponds to the - // default used by the installations. This is important, because the - // default installer for vm_tests (in tools/installers:head, invoked - // via tools/vm:defs.bzl) will install with this name. So without - // changing anything, tests should have a runsc runtime available to - // them. Otherwise installers should update the existing runtime - // instead of installing a new one. - runtime = flag.String("runtime", "runsc", "specify which runtime to use") - - // config is the default Docker daemon configuration path. - config = flag.String("config_path", "/etc/docker/daemon.json", "configuration file for reading paths") - - // The following flags are for the "pprof" profiler tool. - - // pprofBaseDir allows the user to change the directory to which profiles are - // written. By default, profiles will appear under: - // /tmp/profile/RUNTIME/CONTAINER_NAME/*.pprof. - pprofBaseDir = flag.String("pprof-dir", "/tmp/profile", "base directory in: BASEDIR/RUNTIME/CONTINER_NAME/FILENAME (e.g. /tmp/profile/runtime/mycontainer/cpu.pprof)") - pprofDuration = flag.Duration("pprof-duration", time.Hour, "profiling duration (automatically stopped at container exit)") - - // The below flags enable each type of profile. Multiple profiles can be - // enabled for each run. The profile will be collected from the start. - pprofBlock = flag.Bool("pprof-block", false, "enables block profiling with runsc debug") - pprofCPU = flag.Bool("pprof-cpu", false, "enables CPU profiling with runsc debug") - pprofHeap = flag.Bool("pprof-heap", false, "enables heap profiling with runsc debug") - pprofMutex = flag.Bool("pprof-mutex", false, "enables mutex profiling with runsc debug") -) - -// EnsureSupportedDockerVersion checks if correct docker is installed. -// -// This logs directly to stderr, as it is typically called from a Main wrapper. -func EnsureSupportedDockerVersion() { - cmd := exec.Command("docker", "version") - out, err := cmd.CombinedOutput() - if err != nil { - log.Fatalf("error running %q: %v", "docker version", err) - } - re := regexp.MustCompile(`Version:\s+(\d+)\.(\d+)\.\d.*`) - matches := re.FindStringSubmatch(string(out)) - if len(matches) != 3 { - log.Fatalf("Invalid docker output: %s", out) - } - major, _ := strconv.Atoi(matches[1]) - minor, _ := strconv.Atoi(matches[2]) - if major < 17 || (major == 17 && minor < 9) { - log.Fatalf("Docker version 17.09.0 or greater is required, found: %02d.%02d", major, minor) - } -} - -// RuntimePath returns the binary path for the current runtime. -func RuntimePath() (string, error) { - rs, err := runtimeMap() - if err != nil { - return "", err - } - - p, ok := rs["path"].(string) - if !ok { - // The runtime does not declare a path. - return "", fmt.Errorf("runtime does not declare a path: %v", rs) - } - return p, nil -} - -// UsingVFS2 returns true if the 'runtime' has the vfs2 flag set. -// TODO(gvisor.dev/issue/1624): Remove. -func UsingVFS2() (bool, error) { - rMap, err := runtimeMap() - if err != nil { - return false, err - } - - list, ok := rMap["runtimeArgs"].([]interface{}) - if !ok { - return false, fmt.Errorf("unexpected format: %v", rMap) - } - - for _, element := range list { - if element == "--vfs2" { - return true, nil - } - } - return false, nil -} - -func runtimeMap() (map[string]interface{}, error) { - // Read the configuration data; the file must exist. - configBytes, err := ioutil.ReadFile(*config) - if err != nil { - return nil, err - } - - // Unmarshal the configuration. - c := make(map[string]interface{}) - if err := json.Unmarshal(configBytes, &c); err != nil { - return nil, err - } - - // Decode the expected configuration. - r, ok := c["runtimes"] - if !ok { - return nil, fmt.Errorf("no runtimes declared: %v", c) - } - rs, ok := r.(map[string]interface{}) - if !ok { - // The runtimes are not a map. - return nil, fmt.Errorf("unexpected format: %v", rs) - } - r, ok = rs[*runtime] - if !ok { - // The expected runtime is not declared. - return nil, fmt.Errorf("runtime %q not found: %v", *runtime, rs) - } - rs, ok = r.(map[string]interface{}) - if !ok { - // The runtime is not a map. - return nil, fmt.Errorf("unexpected format: %v", r) - } - return rs, nil -} - -// Save exports a container image to the given Writer. -// -// Note that the writer should be actively consuming the output, otherwise it -// is not guaranteed that the Save will make any progress and the call may -// stall indefinitely. -// -// This is called by criutil in order to import imports. -func Save(logger testutil.Logger, image string, w io.Writer) error { - cmd := testutil.Command(logger, "docker", "save", testutil.ImageByName(image)) - cmd.Stdout = w // Send directly to the writer. - return cmd.Run() -} - -// Runtime returns the value of the flag runtime. -func Runtime() string { - return *runtime -} diff --git a/pkg/test/dockerutil/exec.go b/pkg/test/dockerutil/exec.go deleted file mode 100644 index bf968acec..000000000 --- a/pkg/test/dockerutil/exec.go +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dockerutil - -import ( - "bytes" - "context" - "fmt" - "time" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/pkg/stdcopy" -) - -// ExecOpts holds arguments for Exec calls. -type ExecOpts struct { - // Env are additional environment variables. - Env []string - - // Privileged enables privileged mode. - Privileged bool - - // User is the user to use. - User string - - // Enables Tty and stdin for the created process. - UseTTY bool - - // WorkDir is the working directory of the process. - WorkDir string -} - -// Exec creates a process inside the container. -func (c *Container) Exec(ctx context.Context, opts ExecOpts, args ...string) (string, error) { - p, err := c.doExec(ctx, opts, args) - if err != nil { - return "", err - } - - if exitStatus, err := p.WaitExitStatus(ctx); err != nil { - return "", err - } else if exitStatus != 0 { - out, _ := p.Logs() - return out, fmt.Errorf("process terminated with status: %d", exitStatus) - } - - return p.Logs() -} - -// ExecProcess creates a process inside the container and returns a process struct -// for the caller to use. -func (c *Container) ExecProcess(ctx context.Context, opts ExecOpts, args ...string) (Process, error) { - return c.doExec(ctx, opts, args) -} - -func (c *Container) doExec(ctx context.Context, r ExecOpts, args []string) (Process, error) { - config := c.execConfig(r, args) - resp, err := c.client.ContainerExecCreate(ctx, c.id, config) - if err != nil { - return Process{}, fmt.Errorf("exec create failed with err: %v", err) - } - - hijack, err := c.client.ContainerExecAttach(ctx, resp.ID, types.ExecStartCheck{}) - if err != nil { - return Process{}, fmt.Errorf("exec attach failed with err: %v", err) - } - - return Process{ - container: c, - execid: resp.ID, - conn: hijack, - }, nil -} - -func (c *Container) execConfig(r ExecOpts, cmd []string) types.ExecConfig { - env := append(r.Env, fmt.Sprintf("RUNSC_TEST_NAME=%s", c.Name)) - return types.ExecConfig{ - AttachStdin: r.UseTTY, - AttachStderr: true, - AttachStdout: true, - Cmd: cmd, - Privileged: r.Privileged, - WorkingDir: r.WorkDir, - Env: env, - Tty: r.UseTTY, - User: r.User, - } - -} - -// Process represents a containerized process. -type Process struct { - container *Container - execid string - conn types.HijackedResponse -} - -// Write writes buf to the process's stdin. -func (p *Process) Write(timeout time.Duration, buf []byte) (int, error) { - p.conn.Conn.SetDeadline(time.Now().Add(timeout)) - return p.conn.Conn.Write(buf) -} - -// Read returns process's stdout and stderr. -func (p *Process) Read() (string, string, error) { - var stdout, stderr bytes.Buffer - if err := p.read(&stdout, &stderr); err != nil { - return "", "", err - } - return stdout.String(), stderr.String(), nil -} - -// Logs returns combined stdout/stderr from the process. -func (p *Process) Logs() (string, error) { - var out bytes.Buffer - if err := p.read(&out, &out); err != nil { - return "", err - } - return out.String(), nil -} - -func (p *Process) read(stdout, stderr *bytes.Buffer) error { - _, err := stdcopy.StdCopy(stdout, stderr, p.conn.Reader) - return err -} - -// ExitCode returns the process's exit code. -func (p *Process) ExitCode(ctx context.Context) (int, error) { - _, exitCode, err := p.runningExitCode(ctx) - return exitCode, err -} - -// IsRunning checks if the process is running. -func (p *Process) IsRunning(ctx context.Context) (bool, error) { - running, _, err := p.runningExitCode(ctx) - return running, err -} - -// WaitExitStatus until process completes and returns exit status. -func (p *Process) WaitExitStatus(ctx context.Context) (int, error) { - waitChan := make(chan (int)) - errChan := make(chan (error)) - - go func() { - for { - running, exitcode, err := p.runningExitCode(ctx) - if err != nil { - errChan <- fmt.Errorf("error waiting process %s: container %v", p.execid, p.container.Name) - } - if !running { - waitChan <- exitcode - } - time.Sleep(time.Millisecond * 500) - } - }() - - select { - case ws := <-waitChan: - return ws, nil - case err := <-errChan: - return -1, err - } -} - -// runningExitCode collects if the process is running and the exit code. -// The exit code is only valid if the process has exited. -func (p *Process) runningExitCode(ctx context.Context) (bool, int, error) { - // If execid is not empty, this is a execed process. - if p.execid != "" { - status, err := p.container.client.ContainerExecInspect(ctx, p.execid) - return status.Running, status.ExitCode, err - } - // else this is the root process. - status, err := p.container.Status(ctx) - return status.Running, status.ExitCode, err -} diff --git a/pkg/test/dockerutil/network.go b/pkg/test/dockerutil/network.go deleted file mode 100644 index dbe17fa5e..000000000 --- a/pkg/test/dockerutil/network.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dockerutil - -import ( - "context" - "net" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/network" - "github.com/docker/docker/client" - "gvisor.dev/gvisor/pkg/test/testutil" -) - -// Network is a docker network. -type Network struct { - client *client.Client - id string - logger testutil.Logger - Name string - containers []*Container - Subnet *net.IPNet -} - -// NewNetwork sets up the struct for a Docker network. Names of networks -// will be unique. -func NewNetwork(ctx context.Context, logger testutil.Logger) *Network { - client, err := client.NewClientWithOpts(client.FromEnv) - if err != nil { - logger.Logf("create client failed with: %v", err) - return nil - } - client.NegotiateAPIVersion(ctx) - - return &Network{ - logger: logger, - Name: testutil.RandomID(logger.Name()), - client: client, - } -} - -func (n *Network) networkCreate() types.NetworkCreate { - - var subnet string - if n.Subnet != nil { - subnet = n.Subnet.String() - } - - ipam := network.IPAM{ - Config: []network.IPAMConfig{{ - Subnet: subnet, - }}, - } - - return types.NetworkCreate{ - CheckDuplicate: true, - IPAM: &ipam, - } -} - -// Create is analogous to 'docker network create'. -func (n *Network) Create(ctx context.Context) error { - - opts := n.networkCreate() - resp, err := n.client.NetworkCreate(ctx, n.Name, opts) - if err != nil { - return err - } - n.id = resp.ID - return nil -} - -// Connect is analogous to 'docker network connect' with the arguments provided. -func (n *Network) Connect(ctx context.Context, container *Container, ipv4, ipv6 string) error { - settings := network.EndpointSettings{ - IPAMConfig: &network.EndpointIPAMConfig{ - IPv4Address: ipv4, - IPv6Address: ipv6, - }, - } - err := n.client.NetworkConnect(ctx, n.id, container.id, &settings) - if err == nil { - n.containers = append(n.containers, container) - } - return err -} - -// Inspect returns this network's info. -func (n *Network) Inspect(ctx context.Context) (types.NetworkResource, error) { - return n.client.NetworkInspect(ctx, n.id, types.NetworkInspectOptions{Verbose: true}) -} - -// Cleanup cleans up the docker network. -func (n *Network) Cleanup(ctx context.Context) error { - n.containers = nil - - return n.client.NetworkRemove(ctx, n.id) -} diff --git a/pkg/test/dockerutil/profile.go b/pkg/test/dockerutil/profile.go deleted file mode 100644 index 12fe98b16..000000000 --- a/pkg/test/dockerutil/profile.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dockerutil - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" - "time" - - "golang.org/x/sys/unix" -) - -// profile represents profile-like operations on a container. -// -// It is meant to be added to containers such that the container type calls -// the profile during its lifecycle. Standard implementations are below. - -// profile is for running profiles with 'runsc debug'. -type profile struct { - BasePath string - Types []string - Duration time.Duration - cmd *exec.Cmd -} - -// profileInit initializes a profile object, if required. -// -// N.B. The profiling filename initialized here will use the *image* -// name, and not the unique container name. This is intentional. Most -// of the time, profiling will be used for benchmarks. Benchmarks will -// be run iteratively until a sufficiently large N is reached. It is -// useful in this context to overwrite previous runs, and generate a -// single profile result for the final test. -func (c *Container) profileInit(image string) { - if !*pprofBlock && !*pprofCPU && !*pprofMutex && !*pprofHeap { - return // Nothing to do. - } - c.profile = &profile{ - BasePath: filepath.Join(*pprofBaseDir, c.runtime, c.logger.Name(), image), - Duration: *pprofDuration, - } - if *pprofCPU { - c.profile.Types = append(c.profile.Types, "cpu") - } - if *pprofHeap { - c.profile.Types = append(c.profile.Types, "heap") - } - if *pprofMutex { - c.profile.Types = append(c.profile.Types, "mutex") - } - if *pprofBlock { - c.profile.Types = append(c.profile.Types, "block") - } -} - -// createProcess creates the collection process. -func (p *profile) createProcess(c *Container) error { - // Ensure our directory exists. - if err := os.MkdirAll(p.BasePath, 0755); err != nil { - return err - } - - // Find the runtime to invoke. - path, err := RuntimePath() - if err != nil { - return fmt.Errorf("failed to get runtime path: %v", err) - } - - // The root directory of this container's runtime. - rootDir := fmt.Sprintf("/var/run/docker/runtime-%s/moby", c.runtime) - if _, err := os.Stat(rootDir); os.IsNotExist(err) { - // In docker v20+, due to https://github.com/moby/moby/issues/42345 the - // rootDir seems to always be the following. - rootDir = "/var/run/docker/runtime-runc/moby" - } - - // Format is `runsc --root=rootDir debug --profile-*=file --duration=24h containerID`. - args := []string{fmt.Sprintf("--root=%s", rootDir), "debug"} - for _, profileArg := range p.Types { - outputPath := filepath.Join(p.BasePath, fmt.Sprintf("%s.pprof", profileArg)) - args = append(args, fmt.Sprintf("--profile-%s=%s", profileArg, outputPath)) - } - args = append(args, fmt.Sprintf("--duration=%s", p.Duration)) // Or until container exits. - args = append(args, fmt.Sprintf("--delay=%s", p.Duration)) // Ditto. - args = append(args, c.ID()) - - // Best effort wait until container is running. - for now := time.Now(); time.Since(now) < 5*time.Second; { - if status, err := c.Status(context.Background()); err != nil { - return fmt.Errorf("failed to get status with: %v", err) - } else if status.Running { - break - } - time.Sleep(100 * time.Millisecond) - } - p.cmd = exec.Command(path, args...) - p.cmd.Stderr = os.Stderr // Pass through errors. - if err := p.cmd.Start(); err != nil { - return fmt.Errorf("start process failed: %v", err) - } - - return nil -} - -// killProcess kills the process, if running. -func (p *profile) killProcess() error { - if p.cmd != nil && p.cmd.Process != nil { - return p.cmd.Process.Signal(unix.SIGTERM) - } - return nil -} - -// waitProcess waits for the process, if running. -func (p *profile) waitProcess() error { - defer func() { p.cmd = nil }() - if p.cmd != nil { - return p.cmd.Wait() - } - return nil -} - -// Start is called when profiling is started. -func (p *profile) Start(c *Container) error { - return p.createProcess(c) -} - -// Stop is called when profiling is started. -func (p *profile) Stop(c *Container) error { - killErr := p.killProcess() - waitErr := p.waitProcess() - if waitErr != nil && killErr != nil { - return killErr - } - return waitErr // Ignore okay wait, err kill. -} diff --git a/pkg/test/dockerutil/profile_test.go b/pkg/test/dockerutil/profile_test.go deleted file mode 100644 index 4fe9ce15c..000000000 --- a/pkg/test/dockerutil/profile_test.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dockerutil - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "testing" - "time" -) - -type testCase struct { - name string - profile profile - expectedFiles []string -} - -func TestProfile(t *testing.T) { - // Basepath and expected file names for each type of profile. - tmpDir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("unable to create temporary directory: %v", err) - } - defer os.RemoveAll(tmpDir) - - // All expected names. - basePath := tmpDir - block := "block.pprof" - cpu := "cpu.pprof" - heap := "heap.pprof" - mutex := "mutex.pprof" - - testCases := []testCase{ - { - name: "One", - profile: profile{ - BasePath: basePath, - Types: []string{"cpu"}, - Duration: 2 * time.Second, - }, - expectedFiles: []string{cpu}, - }, - { - name: "All", - profile: profile{ - BasePath: basePath, - Types: []string{"block", "cpu", "heap", "mutex"}, - Duration: 2 * time.Second, - }, - expectedFiles: []string{block, cpu, heap, mutex}, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - ctx := context.Background() - c := MakeContainer(ctx, t) - - // Set basepath to include the container name so there are no conflicts. - localProfile := tc.profile // Copy it. - localProfile.BasePath = filepath.Join(localProfile.BasePath, tc.name) - - // Set directly on the container, to avoid flags. - c.profile = &localProfile - - func() { - defer c.CleanUp(ctx) - - // Start a container. - if err := c.Spawn(ctx, RunOpts{ - Image: "basic/alpine", - }, "sleep", "1000"); err != nil { - t.Fatalf("run failed with: %v", err) - } - - if status, err := c.Status(context.Background()); !status.Running { - t.Fatalf("container is not yet running: %+v err: %v", status, err) - } - - // End early if the expected files exist and have data. - for start := time.Now(); time.Since(start) < localProfile.Duration; time.Sleep(100 * time.Millisecond) { - if err := checkFiles(localProfile.BasePath, tc.expectedFiles); err == nil { - break - } - } - }() - - // Check all expected files exist and have data. - if err := checkFiles(localProfile.BasePath, tc.expectedFiles); err != nil { - t.Fatalf(err.Error()) - } - }) - } -} - -func checkFiles(basePath string, expectedFiles []string) error { - for _, file := range expectedFiles { - stat, err := os.Stat(filepath.Join(basePath, file)) - if err != nil { - return fmt.Errorf("stat failed with: %v", err) - } else if stat.Size() < 1 { - return fmt.Errorf("file not written to: %+v", stat) - } - } - return nil -} - -func TestMain(m *testing.M) { - EnsureSupportedDockerVersion() - os.Exit(m.Run()) -} diff --git a/pkg/test/testutil/BUILD b/pkg/test/testutil/BUILD deleted file mode 100644 index a789c246e..000000000 --- a/pkg/test/testutil/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -package(licenses = ["notice"]) - -go_library( - name = "testutil", - testonly = 1, - srcs = [ - "sh.go", - "testutil.go", - "testutil_runfiles.go", - ], - visibility = ["//:sandbox"], - deps = [ - "//pkg/sync", - "//runsc/config", - "//runsc/specutils", - "@com_github_cenkalti_backoff//:go_default_library", - "@com_github_kr_pty//:go_default_library", - "@com_github_opencontainers_runtime_spec//specs-go:go_default_library", - "@org_golang_x_sys//unix:go_default_library", - ], -) diff --git a/pkg/test/testutil/sh.go b/pkg/test/testutil/sh.go deleted file mode 100644 index cd5b0557a..000000000 --- a/pkg/test/testutil/sh.go +++ /dev/null @@ -1,515 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package testutil - -import ( - "bytes" - "context" - "fmt" - "io" - "os" - "os/exec" - "strings" - "time" - - "github.com/kr/pty" - "golang.org/x/sys/unix" -) - -// Prompt is used as shell prompt. -// It is meant to be unique enough to not be seen in command outputs. -const Prompt = "PROMPT> " - -// Simplistic shell string escape. -func shellEscape(s string) string { - // specialChars is used to determine whether s needs quoting at all. - const specialChars = "\\'\"`${[|&;<>()*?! \t\n" - // If s needs quoting, escapedChars is the set of characters that are - // escaped with a backslash. - const escapedChars = "\\\"$`" - if len(s) == 0 { - return "''" - } - if !strings.ContainsAny(s, specialChars) { - return s - } - var b bytes.Buffer - b.WriteString("\"") - for _, c := range s { - if strings.ContainsAny(string(c), escapedChars) { - b.WriteString("\\") - } - b.WriteRune(c) - } - b.WriteString("\"") - return b.String() -} - -type byteOrError struct { - b byte - err error -} - -// Shell manages a /bin/sh invocation with convenience functions to handle I/O. -// The shell is run in its own interactive TTY and should present its prompt. -type Shell struct { - // cmd is a reference to the underlying sh process. - cmd *exec.Cmd - // cmdFinished is closed when cmd exits. - cmdFinished chan struct{} - - // echo is whether the shell will echo input back to us. - // This helps setting expectations of getting feedback of written bytes. - echo bool - // Control characters we expect to see in the shell. - controlCharIntr string - controlCharEOF string - - // ptyMaster and ptyReplica are the TTY pair associated with the shell. - ptyMaster *os.File - ptyReplica *os.File - // readCh is a channel where everything read from ptyMaster is written. - readCh chan byteOrError - - // logger is used for logging. It may be nil. - logger Logger -} - -// cleanup kills the shell process and closes the TTY. -// Users of this library get a reference to this function with NewShell. -func (s *Shell) cleanup() { - s.logf("cleanup", "Shell cleanup started.") - if s.cmd.ProcessState == nil { - if err := s.cmd.Process.Kill(); err != nil { - s.logf("cleanup", "cannot kill shell process: %v", err) - } - // We don't log the error returned by Wait because the monitorExit - // goroutine will already do so. - s.cmd.Wait() - } - s.ptyReplica.Close() - s.ptyMaster.Close() - // Wait for monitorExit goroutine to write exit status to the debug log. - <-s.cmdFinished - // Empty out everything in the readCh, but don't wait too long for it. - var extraBytes bytes.Buffer - unreadTimeout := time.After(100 * time.Millisecond) -unreadLoop: - for { - select { - case r, ok := <-s.readCh: - if !ok { - break unreadLoop - } else if r.err == nil { - extraBytes.WriteByte(r.b) - } - case <-unreadTimeout: - break unreadLoop - } - } - if extraBytes.Len() > 0 { - s.logIO("unread", extraBytes.Bytes(), nil) - } - s.logf("cleanup", "Shell cleanup complete.") -} - -// logIO logs byte I/O to both standard logging and the test log, if provided. -func (s *Shell) logIO(prefix string, b []byte, err error) { - var sb strings.Builder - if len(b) > 0 { - sb.WriteString(fmt.Sprintf("%q", b)) - } else { - sb.WriteString("(nothing)") - } - if err != nil { - sb.WriteString(fmt.Sprintf(" [error: %v]", err)) - } - s.logf(prefix, "%s", sb.String()) -} - -// logf logs something to both standard logging and the test log, if provided. -func (s *Shell) logf(prefix, format string, values ...interface{}) { - if s.logger != nil { - s.logger.Logf("[%s] %s", prefix, fmt.Sprintf(format, values...)) - } -} - -// monitorExit waits for the shell process to exit and logs the exit result. -func (s *Shell) monitorExit() { - if err := s.cmd.Wait(); err != nil { - s.logf("cmd", "shell process terminated: %v", err) - } else { - s.logf("cmd", "shell process terminated successfully") - } - close(s.cmdFinished) -} - -// reader continuously reads the shell output and populates readCh. -func (s *Shell) reader(ctx context.Context) { - b := make([]byte, 4096) - defer close(s.readCh) - for { - select { - case <-s.cmdFinished: - // Shell process terminated; stop trying to read. - return - case <-ctx.Done(): - // Shell process will also have terminated in this case; - // stop trying to read. - // We don't print an error here because doing so would print this in the - // normal case where the context passed to NewShell is canceled at the - // end of a successful test. - return - default: - // Shell still running, try reading. - } - if got, err := s.ptyMaster.Read(b); err != nil { - s.readCh <- byteOrError{err: err} - if err == io.EOF { - return - } - } else { - for i := 0; i < got; i++ { - s.readCh <- byteOrError{b: b[i]} - } - } - } -} - -// readByte reads a single byte, respecting the context. -func (s *Shell) readByte(ctx context.Context) (byte, error) { - select { - case <-ctx.Done(): - return 0, ctx.Err() - case r := <-s.readCh: - return r.b, r.err - } -} - -// readLoop reads as many bytes as possible until the context expires, b is -// full, or a short time passes. It returns how many bytes it has successfully -// read. -func (s *Shell) readLoop(ctx context.Context, b []byte) (int, error) { - soonCtx, soonCancel := context.WithTimeout(ctx, 5*time.Second) - defer soonCancel() - var i int - for i = 0; i < len(b) && soonCtx.Err() == nil; i++ { - next, err := s.readByte(soonCtx) - if err != nil { - if i > 0 { - s.logIO("read", b[:i-1], err) - } else { - s.logIO("read", nil, err) - } - return i, err - } - b[i] = next - } - s.logIO("read", b[:i], soonCtx.Err()) - return i, soonCtx.Err() -} - -// readLine reads a single line. Strips out all \r characters for convenience. -// Upon error, it will still return what it has read so far. -// It will also exit quickly if the line content it has read so far (without a -// line break) matches `prompt`. -func (s *Shell) readLine(ctx context.Context, prompt string) ([]byte, error) { - soonCtx, soonCancel := context.WithTimeout(ctx, 5*time.Second) - defer soonCancel() - var lineData bytes.Buffer - var b byte - var err error - for soonCtx.Err() == nil && b != '\n' { - b, err = s.readByte(soonCtx) - if err != nil { - data := lineData.Bytes() - s.logIO("read", data, err) - return data, err - } - if b != '\r' { - lineData.WriteByte(b) - } - if bytes.Equal(lineData.Bytes(), []byte(prompt)) { - // Assume that there will not be any further output if we get the prompt. - // This avoids waiting for the read deadline just to read the prompt. - break - } - } - data := lineData.Bytes() - s.logIO("read", data, soonCtx.Err()) - return data, soonCtx.Err() -} - -// Expect verifies that the next `len(want)` bytes we read match `want`. -func (s *Shell) Expect(ctx context.Context, want []byte) error { - errPrefix := fmt.Sprintf("want(%q)", want) - b := make([]byte, len(want)) - got, err := s.readLoop(ctx, b) - if err != nil { - if ctx.Err() != nil { - return fmt.Errorf("%s: context done (%w), got: %q", errPrefix, err, b[:got]) - } - return fmt.Errorf("%s: %w", errPrefix, err) - } - if got < len(want) { - return fmt.Errorf("%s: short read (read %d bytes, expected %d): %q", errPrefix, got, len(want), b[:got]) - } - if !bytes.Equal(b, want) { - return fmt.Errorf("got %q want %q", b, want) - } - return nil -} - -// ExpectString verifies that the next `len(want)` bytes we read match `want`. -func (s *Shell) ExpectString(ctx context.Context, want string) error { - return s.Expect(ctx, []byte(want)) -} - -// ExpectPrompt verifies that the next few bytes we read are the shell prompt. -func (s *Shell) ExpectPrompt(ctx context.Context) error { - return s.ExpectString(ctx, Prompt) -} - -// ExpectEmptyLine verifies that the next few bytes we read are an empty line, -// as defined by any number of carriage or line break characters. -func (s *Shell) ExpectEmptyLine(ctx context.Context) error { - line, err := s.readLine(ctx, Prompt) - if err != nil { - return fmt.Errorf("cannot read line: %w", err) - } - if strings.Trim(string(line), "\r\n") != "" { - return fmt.Errorf("line was not empty: %q", line) - } - return nil -} - -// ExpectLine verifies that the next `len(want)` bytes we read match `want`, -// followed by carriage returns or newline characters. -func (s *Shell) ExpectLine(ctx context.Context, want string) error { - if err := s.ExpectString(ctx, want); err != nil { - return err - } - if err := s.ExpectEmptyLine(ctx); err != nil { - return fmt.Errorf("ExpectLine(%q): no line break: %w", want, err) - } - return nil -} - -// Write writes `b` to the shell and verifies that all of them get written. -func (s *Shell) Write(b []byte) error { - written, err := s.ptyMaster.Write(b) - s.logIO("write", b[:written], err) - if err != nil { - return fmt.Errorf("write(%q): %w", b, err) - } - if written != len(b) { - return fmt.Errorf("write(%q): wrote %d of %d bytes (%q)", b, written, len(b), b[:written]) - } - return nil -} - -// WriteLine writes `line` (to which \n will be appended) to the shell. -// If the shell is in `echo` mode, it will also check that we got these bytes -// back to read. -func (s *Shell) WriteLine(ctx context.Context, line string) error { - if err := s.Write([]byte(line + "\n")); err != nil { - return err - } - if s.echo { - // We expect to see everything we've typed. - if err := s.ExpectLine(ctx, line); err != nil { - return fmt.Errorf("echo: %w", err) - } - } - return nil -} - -// StartCommand is a convenience wrapper for WriteLine that mimics entering a -// command line and pressing Enter. It does some basic shell argument escaping. -func (s *Shell) StartCommand(ctx context.Context, cmd ...string) error { - escaped := make([]string, len(cmd)) - for i, arg := range cmd { - escaped[i] = shellEscape(arg) - } - return s.WriteLine(ctx, strings.Join(escaped, " ")) -} - -// GetCommandOutput gets all following bytes until the prompt is encountered. -// This is useful for matching the output of a command. -// All \r are removed for ease of matching. -func (s *Shell) GetCommandOutput(ctx context.Context) ([]byte, error) { - return s.ReadUntil(ctx, Prompt) -} - -// ReadUntil gets all following bytes until a certain line is encountered. -// This final line is not returned as part of the output, but everything before -// it (including the \n) is included. -// This is useful for matching the output of a command. -// All \r are removed for ease of matching. -func (s *Shell) ReadUntil(ctx context.Context, finalLine string) ([]byte, error) { - var output bytes.Buffer - for ctx.Err() == nil { - line, err := s.readLine(ctx, finalLine) - if err != nil { - return nil, err - } - if bytes.Equal(line, []byte(finalLine)) { - break - } - // readLine ensures that `line` either matches `finalLine` or contains \n. - // Thus we can be confident that `line` has a \n here. - output.Write(line) - } - return output.Bytes(), ctx.Err() -} - -// RunCommand is a convenience wrapper for StartCommand + GetCommandOutput. -func (s *Shell) RunCommand(ctx context.Context, cmd ...string) ([]byte, error) { - if err := s.StartCommand(ctx, cmd...); err != nil { - return nil, err - } - return s.GetCommandOutput(ctx) -} - -// RefreshSTTY interprets output from `stty -a` to check whether we are in echo -// mode and other settings. -// It will assume that any line matching `expectPrompt` means the end of -// the `stty -a` output. -// Why do this rather than using `tcgets`? Because this function can be used in -// conjunction with sub-shell processes that can allocate their own TTYs. -func (s *Shell) RefreshSTTY(ctx context.Context, expectPrompt string) error { - // Temporarily assume we will not get any output. - // If echo is actually on, we'll get the "stty -a" line as if it was command - // output. This is OK because we parse the output generously. - s.echo = false - if err := s.WriteLine(ctx, "stty -a"); err != nil { - return fmt.Errorf("could not run `stty -a`: %w", err) - } - sttyOutput, err := s.ReadUntil(ctx, expectPrompt) - if err != nil { - return fmt.Errorf("cannot get `stty -a` output: %w", err) - } - - // Set default control characters in case we can't see them in the output. - s.controlCharIntr = "^C" - s.controlCharEOF = "^D" - // stty output has two general notations: - // `a = b;` (for control characters), and `option` vs `-option` (for boolean - // options). We parse both kinds here. - // For `a = b;`, `controlChar` contains `a`, and `previousToken` is used to - // set `controlChar` to `previousToken` when we see an "=" token. - var previousToken, controlChar string - for _, token := range strings.Fields(string(sttyOutput)) { - if controlChar != "" { - value := strings.TrimSuffix(token, ";") - switch controlChar { - case "intr": - s.controlCharIntr = value - case "eof": - s.controlCharEOF = value - } - controlChar = "" - } else { - switch token { - case "=": - controlChar = previousToken - case "-echo": - s.echo = false - case "echo": - s.echo = true - } - } - previousToken = token - } - s.logf("stty", "refreshed settings: echo=%v, intr=%q, eof=%q", s.echo, s.controlCharIntr, s.controlCharEOF) - return nil -} - -// sendControlCode sends `code` to the shell and expects to see `repr`. -// If `expectLinebreak` is true, it also expects to see a linebreak. -func (s *Shell) sendControlCode(ctx context.Context, code byte, repr string, expectLinebreak bool) error { - if err := s.Write([]byte{code}); err != nil { - return fmt.Errorf("cannot send %q: %w", code, err) - } - if err := s.ExpectString(ctx, repr); err != nil { - return fmt.Errorf("did not see %s: %w", repr, err) - } - if expectLinebreak { - if err := s.ExpectEmptyLine(ctx); err != nil { - return fmt.Errorf("linebreak after %s: %v", repr, err) - } - } - return nil -} - -// SendInterrupt sends the \x03 (Ctrl+C) control character to the shell. -func (s *Shell) SendInterrupt(ctx context.Context, expectLinebreak bool) error { - return s.sendControlCode(ctx, 0x03, s.controlCharIntr, expectLinebreak) -} - -// SendEOF sends the \x04 (Ctrl+D) control character to the shell. -func (s *Shell) SendEOF(ctx context.Context, expectLinebreak bool) error { - return s.sendControlCode(ctx, 0x04, s.controlCharEOF, expectLinebreak) -} - -// NewShell returns a new managed sh process along with a cleanup function. -// The caller is expected to call this function once it no longer needs the -// shell. -// The optional passed-in logger will be used for logging. -func NewShell(ctx context.Context, logger Logger) (*Shell, func(), error) { - ptyMaster, ptyReplica, err := pty.Open() - if err != nil { - return nil, nil, fmt.Errorf("cannot create PTY: %w", err) - } - cmd := exec.CommandContext(ctx, "/bin/sh", "--noprofile", "--norc", "-i") - cmd.Stdin = ptyReplica - cmd.Stdout = ptyReplica - cmd.Stderr = ptyReplica - cmd.SysProcAttr = &unix.SysProcAttr{ - Setsid: true, - Setctty: true, - Ctty: 0, - } - cmd.Env = append(cmd.Env, fmt.Sprintf("PS1=%s", Prompt)) - if err := cmd.Start(); err != nil { - return nil, nil, fmt.Errorf("cannot start shell: %w", err) - } - s := &Shell{ - cmd: cmd, - cmdFinished: make(chan struct{}), - ptyMaster: ptyMaster, - ptyReplica: ptyReplica, - readCh: make(chan byteOrError, 1<<20), - logger: logger, - } - s.logf("creation", "Shell spawned.") - go s.monitorExit() - go s.reader(ctx) - setupCtx, setupCancel := context.WithTimeout(ctx, 5*time.Second) - defer setupCancel() - // We expect to see the prompt immediately on startup, - // since the shell is started in interactive mode. - if err := s.ExpectPrompt(setupCtx); err != nil { - s.cleanup() - return nil, nil, fmt.Errorf("did not get initial prompt: %w", err) - } - s.logf("creation", "Initial prompt observed.") - // Get initial TTY settings. - if err := s.RefreshSTTY(setupCtx, Prompt); err != nil { - s.cleanup() - return nil, nil, fmt.Errorf("cannot get initial STTY settings: %w", err) - } - return s, s.cleanup, nil -} diff --git a/pkg/test/testutil/testutil.go b/pkg/test/testutil/testutil.go deleted file mode 100644 index 663c83679..000000000 --- a/pkg/test/testutil/testutil.go +++ /dev/null @@ -1,597 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package testutil contains utility functions for runsc tests. -package testutil - -import ( - "bufio" - "context" - "debug/elf" - "encoding/base32" - "encoding/json" - "flag" - "fmt" - "io" - "io/ioutil" - "log" - "math" - "math/rand" - "net/http" - "os" - "os/exec" - "os/signal" - "path" - "path/filepath" - "strconv" - "strings" - "testing" - "time" - - "github.com/cenkalti/backoff" - specs "github.com/opencontainers/runtime-spec/specs-go" - "golang.org/x/sys/unix" - "gvisor.dev/gvisor/pkg/sync" - "gvisor.dev/gvisor/runsc/config" - "gvisor.dev/gvisor/runsc/specutils" -) - -var ( - checkpoint = flag.Bool("checkpoint", true, "control checkpoint/restore support") - partition = flag.Int("partition", 1, "partition number, this is 1-indexed") - totalPartitions = flag.Int("total_partitions", 1, "total number of partitions") - isRunningWithHostNet = flag.Bool("hostnet", false, "whether test is running with hostnet") -) - -// IsCheckpointSupported returns the relevant command line flag. -func IsCheckpointSupported() bool { - return *checkpoint -} - -// IsRunningWithHostNet returns the relevant command line flag. -func IsRunningWithHostNet() bool { - return *isRunningWithHostNet -} - -// ImageByName mangles the image name used locally. This depends on the image -// build infrastructure in images/ and tools/vm. -func ImageByName(name string) string { - return fmt.Sprintf("gvisor.dev/images/%s", name) -} - -// ConfigureExePath configures the executable for runsc in the test environment. -func ConfigureExePath() error { - path, err := FindFile("runsc/runsc") - if err != nil { - return err - } - specutils.ExePath = path - return nil -} - -// TmpDir returns the absolute path to a writable directory that can be used as -// scratch by the test. -func TmpDir() string { - if dir, ok := os.LookupEnv("TEST_TMPDIR"); ok { - return dir - } - return "/tmp" -} - -// Logger is a simple logging wrapper. -// -// This is designed to be implemented by *testing.T. -type Logger interface { - Name() string - Logf(fmt string, args ...interface{}) -} - -// DefaultLogger logs using the log package. -type DefaultLogger string - -// Name implements Logger.Name. -func (d DefaultLogger) Name() string { - return string(d) -} - -// Logf implements Logger.Logf. -func (d DefaultLogger) Logf(fmt string, args ...interface{}) { - log.Printf(fmt, args...) -} - -// multiLogger logs to multiple Loggers. -type multiLogger []Logger - -// Name implements Logger.Name. -func (m multiLogger) Name() string { - names := make([]string, len(m)) - for i, l := range m { - names[i] = l.Name() - } - return strings.Join(names, "+") -} - -// Logf implements Logger.Logf. -func (m multiLogger) Logf(fmt string, args ...interface{}) { - for _, l := range m { - l.Logf(fmt, args...) - } -} - -// NewMultiLogger returns a new Logger that logs on multiple Loggers. -func NewMultiLogger(loggers ...Logger) Logger { - return multiLogger(loggers) -} - -// Cmd is a simple wrapper. -type Cmd struct { - logger Logger - *exec.Cmd -} - -// CombinedOutput returns the output and logs. -func (c *Cmd) CombinedOutput() ([]byte, error) { - out, err := c.Cmd.CombinedOutput() - if len(out) > 0 { - c.logger.Logf("output: %s", string(out)) - } - if err != nil { - c.logger.Logf("error: %v", err) - } - return out, err -} - -// Command is a simple wrapper around exec.Command, that logs. -func Command(logger Logger, args ...string) *Cmd { - logger.Logf("command: %s", strings.Join(args, " ")) - return &Cmd{ - logger: logger, - Cmd: exec.Command(args[0], args[1:]...), - } -} - -// TestConfig returns the default configuration to use in tests. Note that -// 'RootDir' must be set by caller if required. -func TestConfig(t *testing.T) *config.Config { - logDir := os.TempDir() - if dir, ok := os.LookupEnv("TEST_UNDECLARED_OUTPUTS_DIR"); ok { - logDir = dir + "/" - } - - // Only register flags if config is being used. Otherwise anyone that uses - // testutil will get flags registered and they may conflict. - config.RegisterFlags() - - conf, err := config.NewFromFlags() - if err != nil { - panic(err) - } - // Change test defaults. - conf.Debug = true - conf.DebugLog = path.Join(logDir, "runsc.log."+t.Name()+".%TIMESTAMP%.%COMMAND%") - conf.LogPackets = true - conf.Network = config.NetworkNone - conf.Strace = true - conf.TestOnlyAllowRunAsCurrentUserWithoutChroot = true - return conf -} - -// NewSpecWithArgs creates a simple spec with the given args suitable for use -// in tests. -func NewSpecWithArgs(args ...string) *specs.Spec { - return &specs.Spec{ - // The host filesystem root is the container root. - Root: &specs.Root{ - Path: "/", - Readonly: true, - }, - Process: &specs.Process{ - Args: args, - Env: []string{ - "PATH=" + os.Getenv("PATH"), - }, - Capabilities: specutils.AllCapabilities(), - }, - Mounts: []specs.Mount{ - // Hide the host /etc to avoid any side-effects. - // For example, bash reads /etc/passwd and if it is - // very big, tests can fail by timeout. - { - Type: "tmpfs", - Destination: "/etc", - }, - // Root is readonly, but many tests want to write to tmpdir. - // This creates a writable mount inside the root. Also, when tmpdir points - // to "/tmp", it makes the the actual /tmp to be mounted and not a tmpfs - // inside the sentry. - { - Type: "bind", - Destination: TmpDir(), - Source: TmpDir(), - }, - }, - Hostname: "runsc-test-hostname", - } -} - -// SetupRootDir creates a root directory for containers. -func SetupRootDir() (string, func(), error) { - rootDir, err := ioutil.TempDir(TmpDir(), "containers") - if err != nil { - return "", nil, fmt.Errorf("error creating root dir: %v", err) - } - return rootDir, func() { os.RemoveAll(rootDir) }, nil -} - -// SetupContainer creates a bundle and root dir for the container, generates a -// test config, and writes the spec to config.json in the bundle dir. -func SetupContainer(spec *specs.Spec, conf *config.Config) (rootDir, bundleDir string, cleanup func(), err error) { - rootDir, rootCleanup, err := SetupRootDir() - if err != nil { - return "", "", nil, err - } - conf.RootDir = rootDir - bundleDir, bundleCleanup, err := SetupBundleDir(spec) - if err != nil { - rootCleanup() - return "", "", nil, err - } - return rootDir, bundleDir, func() { - bundleCleanup() - rootCleanup() - }, err -} - -// SetupBundleDir creates a bundle dir and writes the spec to config.json. -func SetupBundleDir(spec *specs.Spec) (string, func(), error) { - bundleDir, err := ioutil.TempDir(TmpDir(), "bundle") - if err != nil { - return "", nil, fmt.Errorf("error creating bundle dir: %v", err) - } - cleanup := func() { os.RemoveAll(bundleDir) } - if err := writeSpec(bundleDir, spec); err != nil { - cleanup() - return "", nil, fmt.Errorf("error writing spec: %v", err) - } - return bundleDir, cleanup, nil -} - -// writeSpec writes the spec to disk in the given directory. -func writeSpec(dir string, spec *specs.Spec) error { - b, err := json.Marshal(spec) - if err != nil { - return err - } - return ioutil.WriteFile(filepath.Join(dir, "config.json"), b, 0755) -} - -// idRandomSrc is a pseudo random generator used to in RandomID. -var idRandomSrc = rand.New(rand.NewSource(time.Now().UnixNano())) - -// idRandomSrcMtx is the mutex protecting idRandomSrc.Read from being used -// concurrently in differnt goroutines. -var idRandomSrcMtx sync.Mutex - -// RandomID returns 20 random bytes following the given prefix. -func RandomID(prefix string) string { - // Read 20 random bytes. - b := make([]byte, 20) - // Rand.Read is not safe for concurrent use. Packetimpact tests can be run in - // parallel now, so we have to protect the Read with a mutex. Otherwise we'll - // run into name conflicts. - // https://golang.org/pkg/math/rand/#Rand.Read - idRandomSrcMtx.Lock() - // "[Read] always returns len(p) and a nil error." --godoc - if _, err := idRandomSrc.Read(b); err != nil { - idRandomSrcMtx.Unlock() - panic("rand.Read failed: " + err.Error()) - } - idRandomSrcMtx.Unlock() - if prefix != "" { - prefix = prefix + "-" - } - return fmt.Sprintf("%s%s", prefix, base32.StdEncoding.EncodeToString(b)) -} - -// RandomContainerID generates a random container id for each test. -// -// The container id is used to create an abstract unix domain socket, which -// must be unique. While the container forbids creating two containers with the -// same name, sometimes between test runs the socket does not get cleaned up -// quickly enough, causing container creation to fail. -func RandomContainerID() string { - return RandomID("test-container") -} - -// Copy copies file from src to dst. -func Copy(src, dst string) error { - in, err := os.Open(src) - if err != nil { - return err - } - defer in.Close() - - st, err := in.Stat() - if err != nil { - return err - } - - out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, st.Mode().Perm()) - if err != nil { - return err - } - defer out.Close() - - // Mirror the local user's permissions across all users. This is - // because as we inject things into the container, the UID/GID will - // change. Also, the build system may generate artifacts with different - // modes. At the top-level (volume mapping) we have a big read-only - // knob that can be applied to prevent modifications. - // - // Note that this must be done via a separate Chmod call, otherwise the - // current process's umask will get in the way. - var mode os.FileMode - if st.Mode()&0100 != 0 { - mode |= 0111 - } - if st.Mode()&0200 != 0 { - mode |= 0222 - } - if st.Mode()&0400 != 0 { - mode |= 0444 - } - if err := os.Chmod(dst, mode); err != nil { - return err - } - - _, err = io.Copy(out, in) - return err -} - -// Poll is a shorthand function to poll for something with given timeout. -func Poll(cb func() error, timeout time.Duration) error { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return PollContext(ctx, cb) -} - -// PollContext is like Poll, but takes a context instead of a timeout. -func PollContext(ctx context.Context, cb func() error) error { - b := backoff.WithContext(backoff.NewConstantBackOff(100*time.Millisecond), ctx) - return backoff.Retry(cb, b) -} - -// WaitForHTTP tries GET requests on a port until the call succeeds or timeout. -func WaitForHTTP(ip string, port int, timeout time.Duration) error { - cb := func() error { - c := &http.Client{ - // Calculate timeout to be able to do minimum 5 attempts. - Timeout: timeout / 5, - } - url := fmt.Sprintf("http://%s:%d/", ip, port) - resp, err := c.Get(url) - if err != nil { - log.Printf("Waiting %s: %v", url, err) - return err - } - resp.Body.Close() - return nil - } - return Poll(cb, timeout) -} - -// Reaper reaps child processes. -type Reaper struct { - // mu protects ch, which will be nil if the reaper is not running. - mu sync.Mutex - ch chan os.Signal -} - -// Start starts reaping child processes. -func (r *Reaper) Start() { - r.mu.Lock() - defer r.mu.Unlock() - - if r.ch != nil { - panic("reaper.Start called on a running reaper") - } - - r.ch = make(chan os.Signal, 1) - signal.Notify(r.ch, unix.SIGCHLD) - - go func() { - for { - r.mu.Lock() - ch := r.ch - r.mu.Unlock() - if ch == nil { - return - } - - _, ok := <-ch - if !ok { - // Channel closed. - return - } - for { - cpid, _ := unix.Wait4(-1, nil, unix.WNOHANG, nil) - if cpid < 1 { - break - } - } - } - }() -} - -// Stop stops reaping child processes. -func (r *Reaper) Stop() { - r.mu.Lock() - defer r.mu.Unlock() - - if r.ch == nil { - panic("reaper.Stop called on a stopped reaper") - } - - signal.Stop(r.ch) - close(r.ch) - r.ch = nil -} - -// StartReaper is a helper that starts a new Reaper and returns a function to -// stop it. -func StartReaper() func() { - r := &Reaper{} - r.Start() - return r.Stop -} - -// WaitUntilRead reads from the given reader until the wanted string is found -// or until timeout. -func WaitUntilRead(r io.Reader, want string, timeout time.Duration) error { - sc := bufio.NewScanner(r) - // done must be accessed atomically. A value greater than 0 indicates - // that the read loop can exit. - doneCh := make(chan bool) - defer close(doneCh) - go func() { - for sc.Scan() { - t := sc.Text() - if strings.Contains(t, want) { - doneCh <- true - return - } - select { - case <-doneCh: - return - default: - } - } - doneCh <- false - }() - - select { - case <-time.After(timeout): - return fmt.Errorf("timeout waiting to read %q", want) - case res := <-doneCh: - if !res { - return fmt.Errorf("reader closed while waiting to read %q", want) - } - return nil - } -} - -// KillCommand kills the process running cmd unless it hasn't been started. It -// returns an error if it cannot kill the process unless the reason is that the -// process has already exited. -// -// KillCommand will also reap the process. -func KillCommand(cmd *exec.Cmd) error { - if cmd.Process == nil { - return nil - } - if err := cmd.Process.Kill(); err != nil { - if !strings.Contains(err.Error(), "process already finished") { - return fmt.Errorf("failed to kill process %v: %v", cmd, err) - } - } - return cmd.Wait() -} - -// WriteTmpFile writes text to a temporary file, closes the file, and returns -// the name of the file. A cleanup function is also returned. -func WriteTmpFile(pattern, text string) (string, func(), error) { - file, err := ioutil.TempFile(TmpDir(), pattern) - if err != nil { - return "", nil, err - } - defer file.Close() - if _, err := file.Write([]byte(text)); err != nil { - return "", nil, err - } - return file.Name(), func() { os.RemoveAll(file.Name()) }, nil -} - -// IsStatic returns true iff the given file is a static binary. -func IsStatic(filename string) (bool, error) { - f, err := elf.Open(filename) - if err != nil { - return false, err - } - for _, prog := range f.Progs { - if prog.Type == elf.PT_INTERP { - return false, nil // Has interpreter. - } - } - return true, nil -} - -// TouchShardStatusFile indicates to Bazel that the test runner supports -// sharding by creating or updating the last modified date of the file -// specified by TEST_SHARD_STATUS_FILE. -// -// See https://docs.bazel.build/versions/master/test-encyclopedia.html#role-of-the-test-runner. -func TouchShardStatusFile() error { - if statusFile, ok := os.LookupEnv("TEST_SHARD_STATUS_FILE"); ok { - cmd := exec.Command("touch", statusFile) - if b, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("touch %q failed:\n output: %s\n error: %s", statusFile, string(b), err.Error()) - } - } - return nil -} - -// TestIndicesForShard returns indices for this test shard based on the -// TEST_SHARD_INDEX and TEST_TOTAL_SHARDS environment vars, as well as -// the passed partition flags. -// -// If either of the env vars are not present, then the function will return all -// tests. If there are more shards than there are tests, then the returned list -// may be empty. -func TestIndicesForShard(numTests int) ([]int, error) { - var ( - shardIndex = 0 - shardTotal = 1 - ) - - indexStr, indexOk := os.LookupEnv("TEST_SHARD_INDEX") - totalStr, totalOk := os.LookupEnv("TEST_TOTAL_SHARDS") - if indexOk && totalOk { - // Parse index and total to ints. - var err error - shardIndex, err = strconv.Atoi(indexStr) - if err != nil { - return nil, fmt.Errorf("invalid TEST_SHARD_INDEX %q: %v", indexStr, err) - } - shardTotal, err = strconv.Atoi(totalStr) - if err != nil { - return nil, fmt.Errorf("invalid TEST_TOTAL_SHARDS %q: %v", totalStr, err) - } - } - - // Combine with the partitions. - partitionSize := shardTotal - shardTotal = (*totalPartitions) * shardTotal - shardIndex = partitionSize*(*partition-1) + shardIndex - - // Calculate! - var indices []int - numBlocks := int(math.Ceil(float64(numTests) / float64(shardTotal))) - for i := 0; i < numBlocks; i++ { - pick := i*shardTotal + shardIndex - if pick < numTests { - indices = append(indices, pick) - } - } - return indices, nil -} diff --git a/pkg/test/testutil/testutil_runfiles.go b/pkg/test/testutil/testutil_runfiles.go deleted file mode 100644 index ece9ea9a1..000000000 --- a/pkg/test/testutil/testutil_runfiles.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package testutil - -import ( - "fmt" - "os" - "path/filepath" -) - -// FindFile searchs for a file inside the test run environment. It returns the -// full path to the file. It fails if none or more than one file is found. -func FindFile(path string) (string, error) { - wd, err := os.Getwd() - if err != nil { - return "", err - } - - // The test root is demarcated by a path element called "__main__". Search for - // it backwards from the working directory. - root := wd - for { - dir, name := filepath.Split(root) - if name == "__main__" { - break - } - if len(dir) == 0 { - return "", fmt.Errorf("directory __main__ not found in %q", wd) - } - // Remove ending slash to loop around. - root = dir[:len(dir)-1] - } - - // Annoyingly, bazel adds the build type to the directory path for go - // binaries, but not for c++ binaries. We use two different patterns to - // to find our file. - patterns := []string{ - // Try the obvious path first. - filepath.Join(root, path), - // If it was a go binary, use a wildcard to match the build - // type. The pattern is: /test-path/__main__/directories/*/file. - filepath.Join(root, filepath.Dir(path), "*", filepath.Base(path)), - } - - for _, p := range patterns { - matches, err := filepath.Glob(p) - if err != nil { - // "The only possible returned error is ErrBadPattern, - // when pattern is malformed." -godoc - return "", fmt.Errorf("error globbing %q: %v", p, err) - } - switch len(matches) { - case 0: - // Try the next pattern. - case 1: - // We found it. - return matches[0], nil - default: - return "", fmt.Errorf("more than one match found for %q: %s", path, matches) - } - } - return "", fmt.Errorf("file %q not found", path) -} |