First crictl integration tests.

More tests will come, but it's worth getting what's done so far reviewed.

PiperOrigin-RevId: 219734531
Change-Id: If15ca6e6855e3d1cc28c83b5f9c3a72cb65b2e59

12 files changed, 643 insertions, 7 deletions

diff --git a/kokoro/run_tests.sh b/kokoro/run_tests.sh
index b4952cedd..bfdb3fe09 100755
--- a/kokoro/run_tests.sh
+++ b/kokoro/run_tests.sh
@@ -80,7 +80,7 @@ installCrictl() (
   chmod +x ${shim_path}
   sudo -n -E mv ${shim_path} /usr/local/bin
 
-  # Configure containerd.
+  # Configure containerd-shim.
   local shim_config_path=/etc/containerd
   local shim_config_tmp_path=/tmp/gvisor-containerd-shim.toml
   sudo -n -E mkdir -p ${shim_config_path}
@@ -89,11 +89,14 @@ installCrictl() (
 
     [runsc_config]
       debug = "true"
-      debug-log = "/tmp/runsc-log/"
+      debug-log = "/tmp/runsc-logs/"
       strace = "true"
       file-access = "shared"
 EOF
   sudo mv ${shim_config_tmp_path} ${shim_config_path}
+
+  # Configure CNI.
+  sudo -n -E env PATH=${PATH} ${GOPATH}/src/github.com/containerd/containerd/script/setup/install-cni
 )
 
 # Install containerd and crictl.
@@ -128,7 +131,7 @@ if [[ ${exit_code} -eq 0 ]]; then
     echo "root_test executable not found"
     exit 1
   fi
-  sudo -n -E RUNSC_RUNTIME=${runtime} ${root_test}
+  sudo -n -E RUNSC_RUNTIME=${runtime} RUNSC_EXEC=/tmp/${runtime}/runsc ${root_test}
   exit_code=${?}
 fi
 
diff --git a/runsc/test/root/BUILD b/runsc/test/root/BUILD
index c2567ef23..77dcbd79e 100644
--- a/runsc/test/root/BUILD
+++ b/runsc/test/root/BUILD
@@ -14,6 +14,7 @@ go_test(
     srcs = [
         "cgroup_test.go",
         "chroot_test.go",
+        "crictl_test.go",
     ],
     embed = [":root"],
     tags = [
@@ -24,6 +25,7 @@ go_test(
     ],
     deps = [
         "//runsc/specutils",
+        "//runsc/test/root/testdata",
         "//runsc/test/testutil",
         "@com_github_syndtr_gocapability//capability:go_default_library",
     ],
diff --git a/runsc/test/root/chroot_test.go b/runsc/test/root/chroot_test.go
index 0ffaaf87b..9f705c860 100644
--- a/runsc/test/root/chroot_test.go
+++ b/runsc/test/root/chroot_test.go
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 // Package root is used for tests that requires sysadmin privileges run. First,
-// follow the setup instruction in runsc/test/README.md. To run these test:
+// follow the setup instruction in runsc/test/README.md. To run these tests:
 //
 //     bazel build //runsc/test/root:root_test
 //     root_test=$(find -L ./bazel-bin/ -executable -type f -name root_test | grep __main__)
diff --git a/runsc/test/root/crictl_test.go b/runsc/test/root/crictl_test.go
new file mode 100644
index 000000000..88e24782a
--- /dev/null
+++ b/runsc/test/root/crictl_test.go
@@ -0,0 +1,201 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package root
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"gvisor.googlesource.com/gvisor/runsc/specutils"
+	"gvisor.googlesource.com/gvisor/runsc/test/root/testdata"
+	"gvisor.googlesource.com/gvisor/runsc/test/testutil"
+)
+
+// Tests for crictl have to be run as root (rather than in a user namespace)
+// because crictl creates named network namespaces in /var/run/netns/.
+func TestCrictlSanity(t *testing.T) {
+	// Setup containerd and crictl.
+	crictl, cleanup, err := setup(t)
+	if err != nil {
+		t.Fatalf("failed to setup crictl: %v", err)
+	}
+	defer cleanup()
+	podID, contID, err := crictl.StartPodAndContainer("httpd", testdata.Sandbox, testdata.Httpd)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Look for the httpd page.
+	if err = httpGet(crictl, podID, "index.html"); err != nil {
+		t.Fatalf("failed to get page: %v", err)
+	}
+
+	// Stop everything.
+	if err := crictl.StopPodAndContainer(podID, contID); err != nil {
+		t.Fatal(err)
+	}
+}
+func TestMountPaths(t *testing.T) {
+	// Setup containerd and crictl.
+	crictl, cleanup, err := setup(t)
+	if err != nil {
+		t.Fatalf("failed to setup crictl: %v", err)
+	}
+	defer cleanup()
+	podID, contID, err := crictl.StartPodAndContainer("httpd", testdata.Sandbox, testdata.HttpdMountPaths)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Look for the directory available at /test.
+	if err = httpGet(crictl, podID, "test"); err != nil {
+		t.Fatalf("failed to get page: %v", err)
+	}
+
+	// Stop everything.
+	if err := crictl.StopPodAndContainer(podID, contID); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// setup sets up before a test. Specifically it:
+// * Creates directories and a socket for containerd to utilize.
+// * Runs containerd and waits for it to reach a "ready" state for testing.
+// * Returns a cleanup function that should be called at the end of the test.
+func setup(t *testing.T) (*testutil.Crictl, func(), error) {
+	var cleanups []func()
+	cleanupFunc := func() {
+		for i := len(cleanups) - 1; i >= 0; i-- {
+			cleanups[i]()
+		}
+	}
+	cleanup := specutils.MakeCleanup(cleanupFunc)
+	defer cleanup.Clean()
+
+	// Create temporary containerd root and state directories, and a socket
+	// via which crictl and containerd communicate.
+	containerdRoot, err := ioutil.TempDir(testutil.TmpDir(), "containerd-root")
+	if err != nil {
+		t.Fatalf("failed to create containerd root: %v", err)
+	}
+	cleanups = append(cleanups, func() { os.RemoveAll(containerdRoot) })
+	containerdState, err := ioutil.TempDir(testutil.TmpDir(), "containerd-state")
+	if err != nil {
+		t.Fatalf("failed to create containerd state: %v", err)
+	}
+	cleanups = append(cleanups, func() { os.RemoveAll(containerdState) })
+	sockAddr := filepath.Join(testutil.TmpDir(), "containerd-test.sock")
+
+	// Start containerd.
+	config, err := testutil.WriteTmpFile("containerd-config", testdata.ContainerdConfig(getRunsc()))
+	if err != nil {
+		t.Fatalf("failed to write containerd config")
+	}
+	cleanups = append(cleanups, func() { os.RemoveAll(config) })
+	containerd := exec.Command(getContainerd(),
+		"--config", config,
+		"--log-level", "debug",
+		"--root", containerdRoot,
+		"--state", containerdState,
+		"--address", sockAddr)
+	cleanups = append(cleanups, func() {
+		if err := testutil.KillCommand(containerd); err != nil {
+			log.Printf("error killing containerd: %v", err)
+		}
+	})
+	containerdStderr, err := containerd.StderrPipe()
+	if err != nil {
+		t.Fatalf("failed to get containerd stderr: %v", err)
+	}
+	containerdStdout, err := containerd.StdoutPipe()
+	if err != nil {
+		t.Fatalf("failed to get containerd stdout: %v", err)
+	}
+	if err := containerd.Start(); err != nil {
+		t.Fatalf("failed running containerd: %v", err)
+	}
+
+	// Wait for containerd to boot. Then put all containerd output into a
+	// buffer to be logged at the end of the test.
+	testutil.WaitUntilRead(containerdStderr, "Start streaming server", nil, 10*time.Second)
+	stdoutBuf := &bytes.Buffer{}
+	stderrBuf := &bytes.Buffer{}
+	go func() { io.Copy(stdoutBuf, containerdStdout) }()
+	go func() { io.Copy(stderrBuf, containerdStderr) }()
+	cleanups = append(cleanups, func() {
+		t.Logf("containerd stdout: %s", string(stdoutBuf.Bytes()))
+		t.Logf("containerd stderr: %s", string(stderrBuf.Bytes()))
+	})
+
+	cleanup.Release()
+	return testutil.NewCrictl(20*time.Second, sockAddr), cleanupFunc, nil
+}
+
+// httpGet GETs the contents of a file served from a pod on port 80.
+func httpGet(crictl *testutil.Crictl, podID, filePath string) error {
+	// Get the IP of the httpd server.
+	ip, err := crictl.PodIP(podID)
+	if err != nil {
+		return fmt.Errorf("failed to get IP from pod %q: %v", podID, err)
+	}
+
+	// GET the page. We may be waiting for the server to start, so retry
+	// with a timeout.
+	var resp *http.Response
+	cb := func() error {
+		r, err := http.Get(fmt.Sprintf("http://%s", path.Join(ip, filePath)))
+		resp = r
+		return err
+	}
+	if err := testutil.Poll(cb, 20*time.Second); err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("bad status returned: %d", resp.StatusCode)
+	}
+	return nil
+}
+
+func getContainerd() string {
+	// Bazel doesn't pass PATH through, assume the location of containerd
+	// unless specified by environment variable.
+	c := os.Getenv("CONTAINERD_PATH")
+	if c == "" {
+		return "/usr/local/bin/containerd"
+	}
+	return c
+}
+
+func getRunsc() string {
+	// Bazel doesn't pass PATH through, assume the location of runsc unless
+	// specified by environment variable.
+	c := os.Getenv("RUNSC_EXEC")
+	if c == "" {
+		return "/tmp/runsc-test/runsc"
+	}
+	return c
+}
diff --git a/runsc/test/root/testdata/BUILD b/runsc/test/root/testdata/BUILD
new file mode 100644
index 000000000..a22635129
--- /dev/null
+++ b/runsc/test/root/testdata/BUILD
@@ -0,0 +1,17 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+package(licenses = ["notice"])  # Apache 2.0
+
+go_library(
+    name = "testdata",
+    srcs = [
+        "containerd_config.go",
+        "httpd.go",
+        "httpd_mount_paths.go",
+        "sandbox.go",
+    ],
+    importpath = "gvisor.googlesource.com/gvisor/runsc/test/root/testdata",
+    visibility = [
+        "//visibility:public",
+    ],
+)
diff --git a/runsc/test/root/testdata/containerd_config.go b/runsc/test/root/testdata/containerd_config.go
new file mode 100644
index 000000000..949354987
--- /dev/null
+++ b/runsc/test/root/testdata/containerd_config.go
@@ -0,0 +1,39 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package testdata contains data required for root tests.
+package testdata
+
+import "fmt"
+
+// containerdConfigTemplate is a .toml config for containerd. It contains a
+// formatting verb so the runtime field can be set via fmt.Sprintf.
+const containerdConfigTemplate = `
+disabled_plugins = ["restart"]
+[plugins.linux]
+  runtime = "%s"
+  runtime_root = "/tmp/test-containerd/runsc"
+  shim = "/usr/local/bin/gvisor-containerd-shim"
+  shim_debug = true
+
+[plugins.cri.containerd.runtimes.runsc]
+  runtime_type = "io.containerd.runtime.v1.linux"
+  runtime_engine = "%s"
+`
+
+// ContainerdConfig returns a containerd config file with the specified
+// runtime.
+func ContainerdConfig(runtime string) string {
+	return fmt.Sprintf(containerdConfigTemplate, runtime, runtime)
+}
diff --git a/runsc/test/root/testdata/httpd.go b/runsc/test/root/testdata/httpd.go
new file mode 100644
index 000000000..f65b1da5d
--- /dev/null
+++ b/runsc/test/root/testdata/httpd.go
@@ -0,0 +1,32 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testdata
+
+// Httpd is a JSON config for an httpd container.
+const Httpd = `
+{
+  "metadata": {
+    "name": "httpd"
+  },
+  "image":{
+    "image": "httpd"
+  },
+  "mounts": [
+  ],
+  "linux": {
+  },
+  "log_path": "httpd.log"
+}
+`
diff --git a/runsc/test/root/testdata/httpd_mount_paths.go b/runsc/test/root/testdata/httpd_mount_paths.go
new file mode 100644
index 000000000..5ca14340e
--- /dev/null
+++ b/runsc/test/root/testdata/httpd_mount_paths.go
@@ -0,0 +1,53 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testdata
+
+// HttpdMountPaths is a JSON config for an httpd container with additional
+// mounts.
+const HttpdMountPaths = `
+{
+  "metadata": {
+    "name": "httpd"
+  },
+  "image":{
+    "image": "httpd"
+  },
+  "mounts": [
+      {
+        "container_path": "/var/run/secrets/kubernetes.io/serviceaccount",
+        "host_path": "/var/lib/kubelet/pods/82bae206-cdf5-11e8-b245-8cdcd43ac064/volumes/kubernetes.io~secret/default-token-2rpfx",
+        "readonly": true
+      },
+      {
+        "container_path": "/etc/hosts",
+        "host_path": "/var/lib/kubelet/pods/82bae206-cdf5-11e8-b245-8cdcd43ac064/etc-hosts",
+        "readonly": false
+      },
+      {
+        "container_path": "/dev/termination-log",
+        "host_path": "/var/lib/kubelet/pods/82bae206-cdf5-11e8-b245-8cdcd43ac064/containers/httpd/d1709580",
+        "readonly": false
+      },
+      {
+        "container_path": "/usr/local/apache2/htdocs/test",
+        "host_path": "/var/lib/kubelet/pods/82bae206-cdf5-11e8-b245-8cdcd43ac064",
+        "readonly": true
+      }
+  ],
+  "linux": {
+  },
+  "log_path": "httpd.log"
+}
+`
diff --git a/runsc/test/root/testdata/sandbox.go b/runsc/test/root/testdata/sandbox.go
new file mode 100644
index 000000000..194242a27
--- /dev/null
+++ b/runsc/test/root/testdata/sandbox.go
@@ -0,0 +1,30 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testdata
+
+// Sandbox is a default JSON config for a sandbox.
+const Sandbox = `
+{
+    "metadata": {
+        "name": "default-sandbox",
+        "namespace": "default",
+        "attempt": 1,
+        "uid": "hdishd83djaidwnduwk28bcsb"
+    },
+    "linux": {
+    },
+    "log_directory": "/tmp"
+}
+`
diff --git a/runsc/test/testutil/BUILD b/runsc/test/testutil/BUILD
index 128bd80fb..3ed235393 100644
--- a/runsc/test/testutil/BUILD
+++ b/runsc/test/testutil/BUILD
@@ -5,6 +5,7 @@ package(licenses = ["notice"])  # Apache 2.0
 go_library(
     name = "testutil",
     srcs = [
+        "crictl.go",
         "docker.go",
         "testutil.go",
         "testutil_race.go",
diff --git a/runsc/test/testutil/crictl.go b/runsc/test/testutil/crictl.go
new file mode 100644
index 000000000..9740ea6b5
--- /dev/null
+++ b/runsc/test/testutil/crictl.go
@@ -0,0 +1,229 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testutil
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+const endpointPrefix = "unix://"
+
+// Crictl contains information required to run the crictl utility.
+type Crictl struct {
+	executable      string
+	timeout         time.Duration
+	imageEndpoint   string
+	runtimeEndpoint string
+}
+
+// NewCrictl returns a Crictl configured with a timeout and an endpoint over
+// which it will talk to containerd.
+func NewCrictl(timeout time.Duration, endpoint string) *Crictl {
+	// Bazel doesn't pass PATH through, assume the location of crictl
+	// unless specified by environment variable.
+	executable := os.Getenv("CRICTL_PATH")
+	if executable == "" {
+		executable = "/usr/local/bin/crictl"
+	}
+	return &Crictl{
+		executable:      executable,
+		timeout:         timeout,
+		imageEndpoint:   endpointPrefix + endpoint,
+		runtimeEndpoint: endpointPrefix + endpoint,
+	}
+}
+
+// Pull pulls an container image. It corresponds to `crictl pull`.
+func (cc *Crictl) Pull(imageName string) error {
+	_, err := cc.run("pull", imageName)
+	return err
+}
+
+// RunPod creates a sandbox. It corresponds to `crictl runp`.
+func (cc *Crictl) RunPod(sbSpecFile string) (string, error) {
+	podID, err := cc.run("runp", sbSpecFile)
+	if err != nil {
+		return "", fmt.Errorf("runp failed: %v", err)
+	}
+	// Strip the trailing newline from crictl output.
+	return strings.TrimSpace(podID), nil
+}
+
+// Create creates a container within a sandbox. It corresponds to `crictl
+// create`.
+func (cc *Crictl) Create(podID, contSpecFile, sbSpecFile string) (string, error) {
+	podID, err := cc.run("create", podID, contSpecFile, sbSpecFile)
+	if err != nil {
+		return "", fmt.Errorf("create failed: %v", err)
+	}
+	// Strip the trailing newline from crictl output.
+	return strings.TrimSpace(podID), nil
+}
+
+// Start starts a container. It corresponds to `crictl start`.
+func (cc *Crictl) Start(contID string) (string, error) {
+	output, err := cc.run("start", contID)
+	if err != nil {
+		return "", fmt.Errorf("start failed: %v", err)
+	}
+	return output, nil
+}
+
+// Stop stops a container. It corresponds to `crictl stop`.
+func (cc *Crictl) Stop(contID string) error {
+	_, err := cc.run("stop", contID)
+	return err
+}
+
+// Rm removes a container. It corresponds to `crictl rm`.
+func (cc *Crictl) Rm(contID string) error {
+	_, err := cc.run("rm", contID)
+	return err
+}
+
+// StopPod stops a pod. It corresponds to `crictl stopp`.
+func (cc *Crictl) StopPod(podID string) error {
+	_, err := cc.run("stopp", podID)
+	return err
+}
+
+// containsConfig is a minimal copy of
+// https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/apis/cri/runtime/v1alpha2/api.proto
+// It only contains fields needed for testing.
+type containerConfig struct {
+	Status containerStatus
+}
+
+type containerStatus struct {
+	Network containerNetwork
+}
+
+type containerNetwork struct {
+	IP string
+}
+
+// PodIP returns a pod's IP address.
+func (cc *Crictl) PodIP(podID string) (string, error) {
+	output, err := cc.run("inspectp", podID)
+	if err != nil {
+		return "", err
+	}
+	conf := &containerConfig{}
+	if err := json.Unmarshal([]byte(output), conf); err != nil {
+		return "", fmt.Errorf("failed to unmarshal JSON: %v, %s", err, output)
+	}
+	if conf.Status.Network.IP == "" {
+		return "", fmt.Errorf("no IP found in config: %s", output)
+	}
+	return conf.Status.Network.IP, nil
+}
+
+// RmPod removes a container. It corresponds to `crictl rmp`.
+func (cc *Crictl) RmPod(podID string) error {
+	_, err := cc.run("rmp", podID)
+	return err
+}
+
+// StartPodAndContainer pulls an image, then starts a sandbox and container in
+// that sandbox. It returns the pod ID and container ID.
+func (cc *Crictl) StartPodAndContainer(image, sbSpec, contSpec string) (string, string, error) {
+	if err := cc.Pull(image); err != nil {
+		return "", "", fmt.Errorf("failed to pull %s: %v", image, err)
+	}
+
+	// Write the specs to files that can be read by crictl.
+	sbSpecFile, err := WriteTmpFile("sbSpec", sbSpec)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to write sandbox spec: %v", err)
+	}
+	contSpecFile, err := WriteTmpFile("contSpec", contSpec)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to write container spec: %v", err)
+	}
+
+	podID, err := cc.RunPod(sbSpecFile)
+	if err != nil {
+		return "", "", err
+	}
+
+	contID, err := cc.Create(podID, contSpecFile, sbSpecFile)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to create container in pod %q: %v", podID, err)
+	}
+
+	if _, err := cc.Start(contID); err != nil {
+		return "", "", fmt.Errorf("failed to start container %q in pod %q: %v", contID, podID, err)
+	}
+
+	return podID, contID, nil
+}
+
+// StopPodAndContainer stops a container and pod.
+func (cc *Crictl) StopPodAndContainer(podID, contID string) error {
+	if err := cc.Stop(contID); err != nil {
+		return fmt.Errorf("failed to stop container %q in pod %q: %v", contID, podID, err)
+	}
+
+	if err := cc.Rm(contID); err != nil {
+		return fmt.Errorf("failed to remove container %q in pod %q: %v", contID, podID, err)
+	}
+
+	if err := cc.StopPod(podID); err != nil {
+		return fmt.Errorf("failed to stop pod %q: %v", podID, err)
+	}
+
+	if err := cc.RmPod(podID); err != nil {
+		return fmt.Errorf("failed to remove pod %q: %v", podID, err)
+	}
+
+	return nil
+}
+
+// run runs crictl with the given args and returns an error if it takes longer
+// than cc.Timeout to run.
+func (cc *Crictl) run(args ...string) (string, error) {
+	defaultArgs := []string{
+		"--image-endpoint", cc.imageEndpoint,
+		"--runtime-endpoint", cc.runtimeEndpoint,
+	}
+	cmd := exec.Command(cc.executable, append(defaultArgs, args...)...)
+
+	// Run the command with a timeout.
+	done := make(chan string)
+	errCh := make(chan error)
+	go func() {
+		output, err := cmd.CombinedOutput()
+		if err != nil {
+			errCh <- fmt.Errorf("error: \"%v\", output: %s", err, string(output))
+		}
+		done <- string(output)
+	}()
+	select {
+	case output := <-done:
+		return output, nil
+	case err := <-errCh:
+		return "", err
+	case <-time.After(cc.timeout):
+		if err := KillCommand(cmd); err != nil {
+			return "", fmt.Errorf("timed out, then couldn't kill process %+v: %v", cmd, err)
+		}
+		return "", fmt.Errorf("timed out: %+v", cmd)
+	}
+}
diff --git a/runsc/test/testutil/testutil.go b/runsc/test/testutil/testutil.go
index fd558e2d5..59dc55887 100644
--- a/runsc/test/testutil/testutil.go
+++ b/runsc/test/testutil/testutil.go
@@ -72,7 +72,7 @@ func FindFile(path string) (string, error) {
 	}
 
 	// The test root is demarcated by a path element called "__main__". Search for
-	// it backwards from the in the working directory.
+	// it backwards from the working directory.
 	root := wd
 	for {
 		dir, name := filepath.Split(root)
@@ -242,7 +242,7 @@ func WaitForHTTP(port int, timeout time.Duration) error {
 
 // RunAsRoot ensures the test runs with CAP_SYS_ADMIN and CAP_SYS_CHROOT. If
 // needed it will create a new user namespace and re-execute the test as root
-// inside of the namespace. This functionr returns when it's running as root. If
+// inside of the namespace. This function returns when it's running as root. If
 // it needs to create another process, it will exit from there and not return.
 func RunAsRoot() {
 	if specutils.HasCapabilities(capability.CAP_SYS_ADMIN, capability.CAP_SYS_CHROOT) {
@@ -288,7 +288,7 @@ func RunAsRoot() {
 	os.Exit(0)
 }
 
-// StartReaper starts a gorouting that will reap all children processes created
+// StartReaper starts a goroutine that will reap all children processes created
 // by the tests. Caller must call the returned function to stop it.
 func StartReaper() func() {
 	ch := make(chan os.Signal, 1)
@@ -356,3 +356,32 @@ func WaitUntilRead(r io.Reader, want string, split bufio.SplitFunc, timeout time
 		return nil
 	}
 }
+
+// KillCommand kills the process running cmd unless it hasn't been started. It
+// returns an error if it cannot kill the process unless the reason is that the
+// process has already exited.
+func KillCommand(cmd *exec.Cmd) error {
+	if cmd.Process == nil {
+		return nil
+	}
+	if err := cmd.Process.Kill(); err != nil {
+		if !strings.Contains(err.Error(), "process already finished") {
+			return fmt.Errorf("failed to kill process %v: %v", cmd, err)
+		}
+	}
+	return nil
+}
+
+// WriteTmpFile writes text to a temporary file, closes the file, and returns
+// the name of the file.
+func WriteTmpFile(pattern, text string) (string, error) {
+	file, err := ioutil.TempFile(TmpDir(), pattern)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+	if _, err := file.Write([]byte(text)); err != nil {
+		return "", err
+	}
+	return file.Name(), nil
+}