Fix build break in test

integration_test runs manually and breakage wasn't detected. Added test to
kokoro to ensure breakages are detected in the future.

PiperOrigin-RevId: 207772835
Change-Id: Iada81b579b558477d4db3516b38366ef6a2e933d

7 files changed, 155 insertions, 95 deletions

diff --git a/kokoro/run_tests.sh b/kokoro/run_tests.sh
index 0069aa0c4..665d63390 100755
--- a/kokoro/run_tests.sh
+++ b/kokoro/run_tests.sh
@@ -44,10 +44,14 @@ bazel test --test_output=errors //...
 exit_code=${?}
 
 if [[ ${exit_code} -eq 0 ]]; then
+  # These names are used to exclude tests not supported in certain
+  # configuration, e.g. save/restore not supported with hostnet.
   declare -a variations=("" "-kvm" "-hostnet" "-overlay")
   for v in "${variations[@]}"; do
-    # image_test is tagged manual
-    bazel test --test_output=errors --test_env=RUNSC_RUNTIME=${runtime}${v} //runsc/test/image:image_test
+    # Run runsc tests with docker that are tagged manual.
+    bazel test --test_output=errors --test_env=RUNSC_RUNTIME=${runtime}${v} \
+      //runsc/test/image:image_test \
+      //runsc/test/integration:integration_test
     exit_code=${?}
     if [[ ${exit_code} -ne 0 ]]; then
       break
diff --git a/runsc/test/image/image_test.go b/runsc/test/image/image_test.go
index 248934484..962c31b24 100644
--- a/runsc/test/image/image_test.go
+++ b/runsc/test/image/image_test.go
@@ -39,8 +39,8 @@ import (
 
 func TestHelloWorld(t *testing.T) {
 	d := testutil.MakeDocker("hello-test")
-	if out, err := d.Run("hello-world"); err != nil {
-		t.Fatalf("docker run failed: %v\nout: %s", err, out)
+	if _, err := d.Run("hello-world"); err != nil {
+		t.Fatalf("docker run failed: %v", err)
 	}
 	defer d.CleanUp()
 
@@ -82,8 +82,8 @@ func testHTTPServer(port int) error {
 }
 
 func TestHttpd(t *testing.T) {
-	if out, err := testutil.Pull("httpd"); err != nil {
-		t.Fatalf("docker pull failed: %v\nout: %s", err, out)
+	if err := testutil.Pull("httpd"); err != nil {
+		t.Fatalf("docker pull failed: %v", err)
 	}
 	d := testutil.MakeDocker("http-test")
 
@@ -93,8 +93,8 @@ func TestHttpd(t *testing.T) {
 	}
 
 	// Start the container.
-	if out, err := d.Run("-p", "80", "-v", testutil.MountArg(dir, "/usr/local/apache2/htdocs:ro"), "httpd"); err != nil {
-		t.Fatalf("docker run failed: %v\nout: %s", err, out)
+	if _, err := d.Run("-p", "80", "-v", testutil.MountArg(dir, "/usr/local/apache2/htdocs:ro"), "httpd"); err != nil {
+		t.Fatalf("docker run failed: %v", err)
 	}
 	defer d.CleanUp()
 
@@ -105,8 +105,8 @@ func TestHttpd(t *testing.T) {
 	}
 
 	// Wait until it's up and running.
-	if err := d.WaitForHTTP(port, 5*time.Second); err != nil {
-		t.Fatalf("docker.WaitForHTTP() timeout: %v", err)
+	if err := testutil.WaitForHTTP(port, 5*time.Second); err != nil {
+		t.Fatalf("WaitForHTTP() timeout: %v", err)
 	}
 
 	if err := testHTTPServer(port); err != nil {
@@ -115,8 +115,8 @@ func TestHttpd(t *testing.T) {
 }
 
 func TestNginx(t *testing.T) {
-	if out, err := testutil.Pull("nginx"); err != nil {
-		t.Fatalf("docker pull failed: %v\nout: %s", err, out)
+	if err := testutil.Pull("nginx"); err != nil {
+		t.Fatalf("docker pull failed: %v", err)
 	}
 	d := testutil.MakeDocker("net-test")
 
@@ -126,8 +126,8 @@ func TestNginx(t *testing.T) {
 	}
 
 	// Start the container.
-	if out, err := d.Run("-p", "80", "-v", testutil.MountArg(dir, "/usr/share/nginx/html:ro"), "nginx"); err != nil {
-		t.Fatalf("docker run failed: %v\nout: %s", err, out)
+	if _, err := d.Run("-p", "80", "-v", testutil.MountArg(dir, "/usr/share/nginx/html:ro"), "nginx"); err != nil {
+		t.Fatalf("docker run failed: %v", err)
 	}
 	defer d.CleanUp()
 
@@ -138,8 +138,8 @@ func TestNginx(t *testing.T) {
 	}
 
 	// Wait until it's up and running.
-	if err := d.WaitForHTTP(port, 5*time.Second); err != nil {
-		t.Fatalf("docker.WaitForHTTP() timeout: %v", err)
+	if err := testutil.WaitForHTTP(port, 5*time.Second); err != nil {
+		t.Fatalf("WaitForHTTP() timeout: %v", err)
 	}
 
 	if err := testHTTPServer(port); err != nil {
@@ -148,14 +148,14 @@ func TestNginx(t *testing.T) {
 }
 
 func TestMysql(t *testing.T) {
-	if out, err := testutil.Pull("mysql"); err != nil {
-		t.Fatalf("docker pull failed: %v\nout: %s", err, out)
+	if err := testutil.Pull("mysql"); err != nil {
+		t.Fatalf("docker pull failed: %v", err)
 	}
 	d := testutil.MakeDocker("mysql-test")
 
 	// Start the container.
-	if out, err := d.Run("-e", "MYSQL_ROOT_PASSWORD=foobar123", "mysql"); err != nil {
-		t.Fatalf("docker run failed: %v\nout: %s", err, out)
+	if _, err := d.Run("-e", "MYSQL_ROOT_PASSWORD=foobar123", "mysql"); err != nil {
+		t.Fatalf("docker run failed: %v", err)
 	}
 	defer d.CleanUp()
 
@@ -178,8 +178,8 @@ func TestMysql(t *testing.T) {
 		"mysql",
 		"mysql", "-hmysql", "-uroot", "-pfoobar123", "-v", "-e", "source /sql/mysql.sql",
 	}
-	if out, err := client.Run(args...); err != nil {
-		t.Fatalf("docker run failed: %v\nout: %s", err, out)
+	if _, err := client.Run(args...); err != nil {
+		t.Fatalf("docker run failed: %v", err)
 	}
 	defer client.CleanUp()
 
diff --git a/runsc/test/image/python_test.go b/runsc/test/image/python_test.go
index b77a6ec87..a8d28e080 100644
--- a/runsc/test/image/python_test.go
+++ b/runsc/test/image/python_test.go
@@ -24,12 +24,12 @@ import (
 )
 
 func TestPythonHello(t *testing.T) {
-	if out, err := testutil.Pull("google/python-hello"); err != nil {
-		t.Fatalf("docker pull failed: %v\nout: %s", err, out)
+	if err := testutil.Pull("google/python-hello"); err != nil {
+		t.Fatalf("docker pull failed: %v", err)
 	}
 	d := testutil.MakeDocker("python-hello-test")
-	if out, err := d.Run("-p", "8080", "google/python-hello"); err != nil {
-		t.Fatalf("docker run failed: %v\nout: %s", err, out)
+	if _, err := d.Run("-p", "8080", "google/python-hello"); err != nil {
+		t.Fatalf("docker run failed: %v", err)
 	}
 	defer d.CleanUp()
 
@@ -40,8 +40,8 @@ func TestPythonHello(t *testing.T) {
 	}
 
 	// Wait until it's up and running.
-	if err := d.WaitForHTTP(port, 10*time.Second); err != nil {
-		t.Fatalf("docker.WaitForHTTP() timeout: %v", err)
+	if err := testutil.WaitForHTTP(port, 10*time.Second); err != nil {
+		t.Fatalf("WaitForHTTP() timeout: %v", err)
 	}
 
 	// Ensure that content is being served.
diff --git a/runsc/test/image/tomcat_test.go b/runsc/test/image/tomcat_test.go
index dd47ab6da..97cf95834 100644
--- a/runsc/test/image/tomcat_test.go
+++ b/runsc/test/image/tomcat_test.go
@@ -24,12 +24,12 @@ import (
 )
 
 func TestTomcat(t *testing.T) {
-	if out, err := testutil.Pull("tomcat:8.0"); err != nil {
-		t.Fatalf("docker pull failed: %v\nout: %s", err, out)
+	if err := testutil.Pull("tomcat:8.0"); err != nil {
+		t.Fatalf("docker pull failed: %v", err)
 	}
 	d := testutil.MakeDocker("tomcat-test")
-	if out, err := d.Run("-p", "8080", "tomcat:8.0"); err != nil {
-		t.Fatalf("docker run failed: %v\nout: %s", err, out)
+	if _, err := d.Run("-p", "8080", "tomcat:8.0"); err != nil {
+		t.Fatalf("docker run failed: %v", err)
 	}
 	defer d.CleanUp()
 
@@ -40,8 +40,8 @@ func TestTomcat(t *testing.T) {
 	}
 
 	// Wait until it's up and running.
-	if err := d.WaitForHTTP(port, 10*time.Second); err != nil {
-		t.Fatalf("docker.WaitForHTTP() timeout: %v", err)
+	if err := testutil.WaitForHTTP(port, 10*time.Second); err != nil {
+		t.Fatalf("WaitForHTTP() timeout: %v", err)
 	}
 
 	// Ensure that content is being served.
diff --git a/runsc/test/integration/integration_test.go b/runsc/test/integration/integration_test.go
index 09d845bfc..67b58523d 100644
--- a/runsc/test/integration/integration_test.go
+++ b/runsc/test/integration/integration_test.go
@@ -37,11 +37,9 @@ import (
 	"gvisor.googlesource.com/gvisor/runsc/test/testutil"
 )
 
-// This container is a docker image for the Flask microframework hello world application.
-const container = "python-hello-test"
-
 // httpRequestSucceeds sends a request to a given url and checks that the status is OK.
-func httpRequestSucceeds(client http.Client, url string) error {
+func httpRequestSucceeds(client http.Client, server string, port int) error {
+	url := fmt.Sprintf("http://%s:%d", server, port)
 	// Ensure that content is being served.
 	resp, err := client.Get(url)
 	if err != nil {
@@ -55,33 +53,50 @@ func httpRequestSucceeds(client http.Client, url string) error {
 
 // TestLifeCycle tests a basic Create/Start/Stop docker container life cycle.
 func TestLifeCycle(t *testing.T) {
-	d := testutil.MakeDocker(container)
-
-	// Test docker create.
-	if out, err := d.Do("create", "--runtime", d.Runtime, "--name", d.Name, "-p", "8080", "google/python-hello"); err != nil {
-		t.Fatalf("docker create failed: %v\nout: %s", err, out)
+	if err := testutil.Pull("nginx"); err != nil {
+		t.Fatalf("docker pull failed: %v", err)
 	}
-
-	// Test docker start.
-	if out, err := d.Do("start", d.Name); err != nil {
+	d := testutil.MakeDocker("lifecycle-test")
+	if err := d.Create("-p", "80", "nginx"); err != nil {
+		t.Fatalf("docker create failed: %v", err)
+	}
+	if err := d.Start(); err != nil {
 		d.CleanUp()
-		t.Fatalf("docker start failed: %v\nout: %s", err, out)
+		t.Fatalf("docker start failed: %v", err)
 	}
 
-	// Test docker stop.
-	if out, err := d.Do("stop", d.Name); err != nil {
-		d.CleanUp()
-		t.Fatalf("docker stop failed: %v\nout: %s", err, out)
+	// Test that container is working
+	port, err := d.FindPort(80)
+	if err != nil {
+		t.Fatalf("docker.FindPort(80) failed: %v", err)
+	}
+	if err := testutil.WaitForHTTP(port, 5*time.Second); err != nil {
+		t.Fatalf("WaitForHTTP() timeout: %v", err)
+	}
+	client := http.Client{Timeout: time.Duration(2 * time.Second)}
+	if err := httpRequestSucceeds(client, "localhost", port); err != nil {
+		t.Errorf("http request failed: %v", err)
 	}
 
-	// Test removing the container.
-	if out, err := d.Do("rm", d.Name); err != nil {
-		t.Fatalf("docker rm failed: %v\nout: %s", err, out)
+	if err := d.Stop(); err != nil {
+		d.CleanUp()
+		t.Fatalf("docker stop failed: %v", err)
+	}
+	if err := d.Remove(); err != nil {
+		t.Fatalf("docker rm failed: %v", err)
 	}
 }
 
 func TestPauseResume(t *testing.T) {
-	d := testutil.MakeDocker(container)
+	if !testutil.IsPauseResumeSupported() {
+		t.Log("Pause/resume is not supported, skipping test.")
+		return
+	}
+
+	if err := testutil.Pull("google/python-hello"); err != nil {
+		t.Fatalf("docker pull failed: %v", err)
+	}
+	d := testutil.MakeDocker("pause-resume-test")
 	if out, err := d.Run("-p", "8080", "google/python-hello"); err != nil {
 		t.Fatalf("docker run failed: %v\nout: %s", err, out)
 	}
@@ -94,28 +109,22 @@ func TestPauseResume(t *testing.T) {
 	}
 
 	// Wait until it's up and running.
-	if err := d.WaitForHTTP(port, 5*time.Second); err != nil {
-		t.Fatalf("docker.WaitForHTTP() timeout: %v", err)
-	}
-
-	timeout := time.Duration(2 * time.Second)
-	client := http.Client{
-		Timeout: timeout,
+	if err := testutil.WaitForHTTP(port, 20*time.Second); err != nil {
+		t.Fatalf("WaitForHTTP() timeout: %v", err)
 	}
 
-	url := fmt.Sprintf("http://localhost:%d", port)
 	// Check that container is working.
-	if err := httpRequestSucceeds(client, url); err != nil {
+	client := http.Client{Timeout: time.Duration(2 * time.Second)}
+	if err := httpRequestSucceeds(client, "localhost", port); err != nil {
 		t.Errorf("http request failed: %v", err)
 	}
 
-	// Pause container.
-	if out, err := d.Do("pause", d.Name); err != nil {
-		t.Fatalf("docker pause failed: %v\nout: %s", err, out)
+	if err := d.Pause(); err != nil {
+		t.Fatalf("docker pause failed: %v", err)
 	}
 
 	// Check if container is paused.
-	switch _, err := client.Get(url); v := err.(type) {
+	switch _, err := client.Get(fmt.Sprintf("http://localhost:%d", port)); v := err.(type) {
 	case nil:
 		t.Errorf("http req expected to fail but it succeeded")
 	case net.Error:
@@ -126,18 +135,17 @@ func TestPauseResume(t *testing.T) {
 		t.Errorf("http req got unexpected error %v", v)
 	}
 
-	// Resume container.
-	if out, err := d.Do("unpause", d.Name); err != nil {
-		t.Fatalf("docker unpause failed: %v\nout: %s", err, out)
+	if err := d.Unpause(); err != nil {
+		t.Fatalf("docker unpause failed: %v", err)
 	}
 
 	// Wait until it's up and running.
-	if err := d.WaitForHTTP(port, 5*time.Second); err != nil {
-		t.Fatalf("docker.WaitForHTTP() timeout: %v", err)
+	if err := testutil.WaitForHTTP(port, 20*time.Second); err != nil {
+		t.Fatalf("WaitForHTTP() timeout: %v", err)
 	}
 
 	// Check if container is working again.
-	if err := httpRequestSucceeds(client, url); err != nil {
+	if err := httpRequestSucceeds(client, "localhost", port); err != nil {
 		t.Errorf("http request failed: %v", err)
 	}
 }
diff --git a/runsc/test/testutil/docker.go b/runsc/test/testutil/docker.go
index ec5ff850b..6825ed9ec 100644
--- a/runsc/test/testutil/docker.go
+++ b/runsc/test/testutil/docker.go
@@ -19,7 +19,6 @@ import (
 	"io/ioutil"
 	"log"
 	"math/rand"
-	"net/http"
 	"os"
 	"os/exec"
 	"path"
@@ -41,6 +40,12 @@ func runtime() string {
 	return r
 }
 
+// IsPauseResumeSupported returns true if Pause/Resume is supported by runtime.
+func IsPauseResumeSupported() bool {
+	// Native host network stack can't be saved.
+	return !strings.Contains(runtime(), "hostnet")
+}
+
 // EnsureSupportedDockerVersion checks if correct docker is installed.
 func EnsureSupportedDockerVersion() {
 	cmd := exec.Command("docker", "version")
@@ -100,7 +105,7 @@ func do(args ...string) (string, error) {
 	cmd := exec.Command("docker", args...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return "", fmt.Errorf("error executing docker %s: %v", args, err)
+		return "", fmt.Errorf("error executing docker %s: %v\nout: %s", args, err, out)
 	}
 	return string(out), nil
 }
@@ -108,8 +113,9 @@ func do(args ...string) (string, error) {
 // Pull pulls a docker image. This is used in tests to isolate the
 // time to pull the image off the network from the time to actually
 // start the container, to avoid timeouts over slow networks.
-func Pull(image string) (string, error) {
-	return do("pull", image)
+func Pull(image string) error {
+	_, err := do("pull", image)
+	return err
 }
 
 // Docker contains the name and the runtime of a docker container.
@@ -125,6 +131,30 @@ func MakeDocker(namePrefix string) Docker {
 	return Docker{Name: namePrefix + suffix, Runtime: runtime()}
 }
 
+// Create calls 'docker create' with the arguments provided.
+func (d *Docker) Create(args ...string) error {
+	a := []string{"create", "--runtime", d.Runtime, "--name", d.Name}
+	a = append(a, args...)
+	_, err := do(a...)
+	return err
+}
+
+// Start calls 'docker start'.
+func (d *Docker) Start() error {
+	if _, err := do("start", d.Name); err != nil {
+		return fmt.Errorf("error starting container %q: %v", d.Name, err)
+	}
+	return nil
+}
+
+// Stop calls 'docker stop'.
+func (d *Docker) Stop() error {
+	if _, err := do("stop", d.Name); err != nil {
+		return fmt.Errorf("error stopping container %q: %v", d.Name, err)
+	}
+	return nil
+}
+
 // Run calls 'docker run' with the arguments provided.
 func (d *Docker) Run(args ...string) (string, error) {
 	a := []string{"run", "--runtime", d.Runtime, "--name", d.Name, "-d"}
@@ -132,17 +162,38 @@ func (d *Docker) Run(args ...string) (string, error) {
 	return do(a...)
 }
 
-// CleanUp kills and deletes the container.
-func (d *Docker) CleanUp() error {
-	if _, err := do("kill", d.Name); err != nil {
-		return fmt.Errorf("error killing container %q: %v", d.Name, err)
+// Pause calls 'docker pause'.
+func (d *Docker) Pause() error {
+	if _, err := do("pause", d.Name); err != nil {
+		return fmt.Errorf("error pausing container %q: %v", d.Name, err)
 	}
+	return nil
+}
+
+// Unpause calls 'docker pause'.
+func (d *Docker) Unpause() error {
+	if _, err := do("unpause", d.Name); err != nil {
+		return fmt.Errorf("error unpausing container %q: %v", d.Name, err)
+	}
+	return nil
+}
+
+// Remove calls 'docker rm'.
+func (d *Docker) Remove() error {
 	if _, err := do("rm", d.Name); err != nil {
 		return fmt.Errorf("error deleting container %q: %v", d.Name, err)
 	}
 	return nil
 }
 
+// CleanUp kills and deletes the container.
+func (d *Docker) CleanUp() error {
+	if _, err := do("kill", d.Name); err != nil {
+		return fmt.Errorf("error killing container %q: %v", d.Name, err)
+	}
+	return d.Remove()
+}
+
 // FindPort returns the host port that is mapped to 'sandboxPort'. This calls
 // docker to allocate a free port in the host and prevent conflicts.
 func (d *Docker) FindPort(sandboxPort int) (int, error) {
@@ -177,16 +228,3 @@ func (d *Docker) WaitForOutput(pattern string, timeout time.Duration) error {
 	}
 	return fmt.Errorf("timeout waiting for output %q: %s", re.String(), out)
 }
-
-// WaitForHTTP tries GET requests on a port until the call succeeds or a timeout.
-func (d *Docker) WaitForHTTP(port int, timeout time.Duration) error {
-	for exp := time.Now().Add(timeout); time.Now().Before(exp); {
-		url := fmt.Sprintf("http://localhost:%d/", port)
-		if _, err := http.Get(url); err == nil {
-			// Success!
-			return nil
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	return fmt.Errorf("timeout waiting for HTTP server on port %d", port)
-}
diff --git a/runsc/test/testutil/testutil.go b/runsc/test/testutil/testutil.go
index 721478353..4e7ab3760 100644
--- a/runsc/test/testutil/testutil.go
+++ b/runsc/test/testutil/testutil.go
@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path/filepath"
 	"time"
@@ -182,3 +183,12 @@ func Poll(cb func() error, timeout time.Duration) error {
 	b := backoff.WithContext(backoff.NewConstantBackOff(100*time.Millisecond), ctx)
 	return backoff.Retry(cb, b)
 }
+
+// WaitForHTTP tries GET requests on a port until the call succeeds or timeout.
+func WaitForHTTP(port int, timeout time.Duration) error {
+	cb := func() error {
+		_, err := http.Get(fmt.Sprintf("http://localhost:%d/", port))
+		return err
+	}
+	return Poll(cb, timeout)
+}