Return error when child exits early

PiperOrigin-RevId: 195365050
Change-Id: I8754dc7a3fc2975d422cae453762a455478a8e6a

5 files changed, 193 insertions, 54 deletions

diff --git a/runsc/cmd/exec.go b/runsc/cmd/exec.go
index 576031b5b..052e00316 100644
--- a/runsc/cmd/exec.go
+++ b/runsc/cmd/exec.go
@@ -123,48 +123,7 @@ func (ex *Exec) Execute(_ context.Context, f *flag.FlagSet, args ...interface{})
 	// write the child's PID to the pid file. So when the container returns, the
 	// child process will also return and signal containerd.
 	if ex.detach {
-		binPath, err := specutils.BinPath()
-		if err != nil {
-			Fatalf("error getting bin path: %v", err)
-		}
-		var args []string
-		for _, a := range os.Args[1:] {
-			if !strings.Contains(a, "detach") {
-				args = append(args, a)
-			}
-		}
-		cmd := exec.Command(binPath, args...)
-		cmd.Stdin = os.Stdin
-		cmd.Stdout = os.Stdout
-		cmd.Stderr = os.Stderr
-		if err := cmd.Start(); err != nil {
-			Fatalf("failure to start child exec process, err: %v", err)
-		}
-
-		log.Infof("Started child (PID: %d) to exec and wait: %s %s", cmd.Process.Pid, binPath, args)
-
-		// Wait for PID file to ensure that child process has started. Otherwise,
-		// '--process' file is deleted as soon as this process returns and the child
-		// may fail to read it.
-		sleepTime := 10 * time.Millisecond
-		for start := time.Now(); time.Now().Sub(start) < 10*time.Second; {
-			_, err := os.Stat(ex.pidFile)
-			if err == nil {
-				break
-			}
-			if pe, ok := err.(*os.PathError); !ok || pe.Err != syscall.ENOENT {
-				Fatalf("unexpected error waiting for PID file, err: %v", err)
-			}
-
-			log.Infof("Waiting for PID file to be created...")
-			time.Sleep(sleepTime)
-			sleepTime *= sleepTime * 2
-			if sleepTime > 1*time.Second {
-				sleepTime = 1 * time.Second
-			}
-		}
-		*waitStatus = 0
-		return subcommands.ExitSuccess
+		return ex.execAndWait(waitStatus)
 	}
 
 	if ex.pidFile != "" {
@@ -191,6 +150,50 @@ func (ex *Exec) Execute(_ context.Context, f *flag.FlagSet, args ...interface{})
 	return subcommands.ExitSuccess
 }
 
+func (ex *Exec) execAndWait(waitStatus *syscall.WaitStatus) subcommands.ExitStatus {
+	binPath, err := specutils.BinPath()
+	if err != nil {
+		Fatalf("error getting bin path: %v", err)
+	}
+	var args []string
+	for _, a := range os.Args[1:] {
+		if !strings.Contains(a, "detach") {
+			args = append(args, a)
+		}
+	}
+	cmd := exec.Command(binPath, args...)
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Start(); err != nil {
+		Fatalf("failure to start child exec process, err: %v", err)
+	}
+
+	log.Infof("Started child (PID: %d) to exec and wait: %s %s", cmd.Process.Pid, binPath, args)
+
+	// Wait for PID file to ensure that child process has started. Otherwise,
+	// '--process' file is deleted as soon as this process returns and the child
+	// may fail to read it.
+	ready := func() (bool, error) {
+		_, err := os.Stat(ex.pidFile)
+		if err == nil {
+			// File appeared, we're done!
+			return true, nil
+		}
+		if pe, ok := err.(*os.PathError); !ok || pe.Err != syscall.ENOENT {
+			return false, err
+		}
+		// No file yet, continue to wait...
+		return false, nil
+	}
+	if err := specutils.WaitForReady(cmd.Process.Pid, 10*time.Second, ready); err != nil {
+		Fatalf("unexpected error waiting for PID file, err: %v", err)
+	}
+
+	*waitStatus = 0
+	return subcommands.ExitSuccess
+}
+
 // parseArgs parses exec information from the command line or a JSON file
 // depending on whether the --process flag was used. Returns an ExecArgs and
 // the ID of the sandbox to be used.
diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go
index 954824ada..13bf5d800 100644
--- a/runsc/sandbox/sandbox.go
+++ b/runsc/sandbox/sandbox.go
@@ -560,19 +560,20 @@ func (s *Sandbox) createSandboxProcess(conf *boot.Config, binPath string, common
 // running, at which point the sandbox is in Created state.
 func (s *Sandbox) waitForCreated(timeout time.Duration) error {
 	log.Debugf("Waiting for sandbox %q creation", s.ID)
-	tchan := time.After(timeout)
-	for {
-		select {
-		case <-tchan:
-			return fmt.Errorf("timed out waiting for sandbox control server")
-		default:
-			if c, err := client.ConnectTo(boot.ControlSocketAddr(s.ID)); err == nil {
-				// It's alive!
-				c.Close()
-				return nil
-			}
+
+	ready := func() (bool, error) {
+		c, err := client.ConnectTo(boot.ControlSocketAddr(s.ID))
+		if err != nil {
+			return false, nil
 		}
+		// It's alive!
+		c.Close()
+		return true, nil
 	}
+	if err := specutils.WaitForReady(s.Pid, timeout, ready); err != nil {
+		return fmt.Errorf("unexpected error waiting for sandbox %q, err: %v", s.ID, err)
+	}
+	return nil
 }
 
 // Wait waits for the containerized process to exit, and returns its WaitStatus.
diff --git a/runsc/specutils/BUILD b/runsc/specutils/BUILD
index ae89260d2..1b6d265bc 100644
--- a/runsc/specutils/BUILD
+++ b/runsc/specutils/BUILD
@@ -1,6 +1,6 @@
 package(licenses = ["notice"])  # Apache 2.0
 
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "specutils",
@@ -16,3 +16,10 @@ go_library(
         "@com_github_opencontainers_runtime-spec//specs-go:go_default_library",
     ],
 )
+
+go_test(
+    name = "specutils_test",
+    size = "small",
+    srcs = ["specutils_test.go"],
+    embed = [":specutils"],
+)
diff --git a/runsc/specutils/specutils.go b/runsc/specutils/specutils.go
index bed0f75eb..04ecb6ae3 100644
--- a/runsc/specutils/specutils.go
+++ b/runsc/specutils/specutils.go
@@ -23,6 +23,8 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
+	"time"
 
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"gvisor.googlesource.com/gvisor/pkg/abi/linux"
@@ -181,3 +183,33 @@ func BinPath() (string, error) {
 	}
 	return binPath, nil
 }
+
+// WaitForReady waits for a process to become ready. The process is ready when
+// the 'ready' function returns true. It continues to wait if 'ready' returns
+// false. It returns error on timeout, if the process stops or if 'ready' fails.
+func WaitForReady(pid int, timeout time.Duration, ready func() (bool, error)) error {
+	backoff := 1 * time.Millisecond
+	for start := time.Now(); time.Now().Sub(start) < timeout; {
+		if ok, err := ready(); err != nil {
+			return err
+		} else if ok {
+			return nil
+		}
+
+		// Check if the process is still running.
+		var ws syscall.WaitStatus
+		var ru syscall.Rusage
+		child, err := syscall.Wait4(pid, &ws, syscall.WNOHANG, &ru)
+		if err != nil || child == pid {
+			return fmt.Errorf("process (%d) is not running, err: %v", pid, err)
+		}
+
+		// Process continues to run, backoff and retry.
+		time.Sleep(backoff)
+		backoff *= 2
+		if backoff > 1*time.Second {
+			backoff = 1 * time.Second
+		}
+	}
+	return fmt.Errorf("timed out waiting for process (%d)", pid)
+}
diff --git a/runsc/specutils/specutils_test.go b/runsc/specutils/specutils_test.go
new file mode 100644
index 000000000..ef293e608
--- /dev/null
+++ b/runsc/specutils/specutils_test.go
@@ -0,0 +1,96 @@
+// Copyright 2018 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package specutils
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestWaitForReadyHappy(t *testing.T) {
+	cmd := exec.Command("/bin/sleep", "1000")
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("cmd.Start() failed, err: %v", err)
+	}
+	defer cmd.Wait()
+
+	var count int
+	err := WaitForReady(cmd.Process.Pid, 5*time.Second, func() (bool, error) {
+		if count < 3 {
+			count++
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Errorf("ProcessWaitReady got: %v, expected: nil", err)
+	}
+	cmd.Process.Kill()
+}
+
+func TestWaitForReadyFail(t *testing.T) {
+	cmd := exec.Command("/bin/sleep", "1000")
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("cmd.Start() failed, err: %v", err)
+	}
+	defer cmd.Wait()
+
+	var count int
+	err := WaitForReady(cmd.Process.Pid, 5*time.Second, func() (bool, error) {
+		if count < 3 {
+			count++
+			return false, nil
+		}
+		return false, fmt.Errorf("Fake error")
+	})
+	if err == nil {
+		t.Errorf("ProcessWaitReady got: nil, expected: error")
+	}
+	cmd.Process.Kill()
+}
+
+func TestWaitForReadyNotRunning(t *testing.T) {
+	cmd := exec.Command("/bin/true")
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("cmd.Start() failed, err: %v", err)
+	}
+	defer cmd.Wait()
+
+	err := WaitForReady(cmd.Process.Pid, 5*time.Second, func() (bool, error) {
+		return false, nil
+	})
+	if !strings.Contains(err.Error(), "not running") {
+		t.Errorf("ProcessWaitReady got: %v, expected: not running", err)
+	}
+}
+
+func TestWaitForReadyTimeout(t *testing.T) {
+	cmd := exec.Command("/bin/sleep", "1000")
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("cmd.Start() failed, err: %v", err)
+	}
+	defer cmd.Wait()
+
+	err := WaitForReady(cmd.Process.Pid, 50*time.Millisecond, func() (bool, error) {
+		return false, nil
+	})
+	if !strings.Contains(err.Error(), "timed out") {
+		t.Errorf("ProcessWaitReady got: %v, expected: timed out", err)
+	}
+	cmd.Process.Kill()
+}