Implement 'runsc kill --all'

In order to implement kill --all correctly, the Sentry needs
to track all tasks that belong to a given container. This change
introduces ContainerID to the task, that gets inherited by all
children. 'kill --all' then iterates over all tasks comparing the
ContainerID field to find all processes that need to be signalled.

PiperOrigin-RevId: 214841768
Change-Id: I693b2374be8692d88cc441ef13a0ae34abf73ac6

5 files changed, 271 insertions, 37 deletions

diff --git a/runsc/container/BUILD b/runsc/container/BUILD
index d72d05c13..e68fb1e8e 100644
--- a/runsc/container/BUILD
+++ b/runsc/container/BUILD
@@ -53,6 +53,7 @@ go_test(
         "//runsc/boot",
         "//runsc/specutils",
         "//runsc/test/testutil",
+        "@com_github_cenkalti_backoff//:go_default_library",
         "@com_github_opencontainers_runtime-spec//specs-go:go_default_library",
         "@org_golang_x_sys//unix:go_default_library",
     ],
@@ -61,5 +62,8 @@ go_test(
 go_binary(
     name = "test_app",
     srcs = ["test_app.go"],
-    deps = ["@com_github_google_subcommands//:go_default_library"],
+    deps = [
+        "//runsc/test/testutil",
+        "@com_github_google_subcommands//:go_default_library",
+    ],
 )
diff --git a/runsc/container/container.go b/runsc/container/container.go
index a1b31d861..44b7dad8a 100644
--- a/runsc/container/container.go
+++ b/runsc/container/container.go
@@ -159,7 +159,7 @@ func Load(rootDir, id string) (*Container, error) {
 		} else if c.Status == Running {
 			// Container state should reflect the actual state of the application, so
 			// we don't consider gofer process here.
-			if err := c.Signal(syscall.Signal(0)); err != nil {
+			if err := c.Signal(syscall.Signal(0), false); err != nil {
 				c.changeStatus(Stopped)
 			}
 		}
@@ -398,7 +398,8 @@ func (c *Container) Execute(args *control.ExecArgs) (int32, error) {
 	if err := c.requireStatus("execute in", Created, Running); err != nil {
 		return 0, err
 	}
-	return c.Sandbox.Execute(c.ID, args)
+	args.ContainerID = c.ID
+	return c.Sandbox.Execute(args)
 }
 
 // Event returns events for the container.
@@ -453,13 +454,13 @@ func (c *Container) WaitPID(pid int32, clearStatus bool) (syscall.WaitStatus, er
 // Signal sends the signal to the container.
 // Signal returns an error if the container is already stopped.
 // TODO: Distinguish different error types.
-func (c *Container) Signal(sig syscall.Signal) error {
+func (c *Container) Signal(sig syscall.Signal, all bool) error {
 	log.Debugf("Signal container %q: %v", c.ID, sig)
 	if err := c.requireStatus("signal", Running); err != nil {
 		return err
 	}
 	// TODO: Query the container for its state, then save it.
-	return c.Sandbox.Signal(c.ID, sig)
+	return c.Sandbox.Signal(c.ID, sig, all)
 }
 
 // Checkpoint sends the checkpoint call to the container.
@@ -612,7 +613,7 @@ func (c *Container) waitForStopped() error {
 	b := backoff.WithContext(backoff.NewConstantBackOff(100*time.Millisecond), ctx)
 	op := func() error {
 		if c.isSandboxRunning() {
-			if err := c.Signal(syscall.Signal(0)); err == nil {
+			if err := c.Signal(syscall.Signal(0), false); err == nil {
 				return fmt.Errorf("container is still running")
 			}
 		}
diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go
index efa598202..de1e50a3f 100644
--- a/runsc/container/container_test.go
+++ b/runsc/container/container_test.go
@@ -30,6 +30,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cenkalti/backoff"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/sys/unix"
 	"gvisor.googlesource.com/gvisor/pkg/abi/linux"
@@ -49,21 +50,34 @@ func init() {
 }
 
 // waitForProcessList waits for the given process list to show up in the container.
-func waitForProcessList(cont *Container, expected []*control.Process) error {
-	var got []*control.Process
-	for start := time.Now(); time.Now().Sub(start) < 10*time.Second; {
-		var err error
-		got, err = cont.Processes()
+func waitForProcessList(cont *Container, want []*control.Process) error {
+	cb := func() error {
+		got, err := cont.Processes()
 		if err != nil {
-			return fmt.Errorf("error getting process data from container: %v", err)
+			err = fmt.Errorf("error getting process data from container: %v", err)
+			return &backoff.PermanentError{Err: err}
 		}
-		if procListsEqual(got, expected) {
-			return nil
+		if !procListsEqual(got, want) {
+			return fmt.Errorf("container got process list: %s, want: %s", procListToString(got), procListToString(want))
 		}
-		// Process might not have started, try again...
-		time.Sleep(10 * time.Millisecond)
+		return nil
+	}
+	return testutil.Poll(cb, 5*time.Second)
+}
+
+func waitForProcessCount(cont *Container, want int) error {
+	cb := func() error {
+		pss, err := cont.Processes()
+		if err != nil {
+			err = fmt.Errorf("error getting process data from container: %v", err)
+			return &backoff.PermanentError{Err: err}
+		}
+		if got := len(pss); got != want {
+			return fmt.Errorf("wrong process count, got: %d, want: %d", got, want)
+		}
+		return nil
 	}
-	return fmt.Errorf("container got process list: %s, want: %s", procListToString(got), procListToString(expected))
+	return testutil.Poll(cb, 5*time.Second)
 }
 
 // procListsEqual is used to check whether 2 Process lists are equal for all
@@ -345,7 +359,7 @@ func TestLifecycle(t *testing.T) {
 		<-ch
 		time.Sleep(100 * time.Millisecond)
 		// Send the container a SIGTERM which will cause it to stop.
-		if err := c.Signal(syscall.SIGTERM); err != nil {
+		if err := c.Signal(syscall.SIGTERM, false); err != nil {
 			t.Fatalf("error sending signal %v to container: %v", syscall.SIGTERM, err)
 		}
 		// Wait for it to die.
diff --git a/runsc/container/multi_container_test.go b/runsc/container/multi_container_test.go
index 2867aa3b9..dc938066b 100644
--- a/runsc/container/multi_container_test.go
+++ b/runsc/container/multi_container_test.go
@@ -16,6 +16,7 @@ package container
 
 import (
 	"io/ioutil"
+	"math"
 	"os"
 	"path"
 	"path/filepath"
@@ -91,11 +92,16 @@ func TestMultiContainerSanity(t *testing.T) {
 		// Check via ps that multiple processes are running.
 		expectedPL := []*control.Process{
 			{PID: 1, Cmd: "sleep"},
-			{PID: 2, Cmd: "sleep"},
 		}
 		if err := waitForProcessList(containers[0], expectedPL); err != nil {
 			t.Errorf("failed to wait for sleep to start: %v", err)
 		}
+		expectedPL = []*control.Process{
+			{PID: 2, Cmd: "sleep"},
+		}
+		if err := waitForProcessList(containers[1], expectedPL); err != nil {
+			t.Errorf("failed to wait for sleep to start: %v", err)
+		}
 	}
 }
 
@@ -134,10 +140,9 @@ func TestMultiContainerWait(t *testing.T) {
 
 	// Check via ps that multiple processes are running.
 	expectedPL := []*control.Process{
-		{PID: 1, Cmd: "sleep"},
 		{PID: 2, Cmd: "sleep"},
 	}
-	if err := waitForProcessList(containers[0], expectedPL); err != nil {
+	if err := waitForProcessList(containers[1], expectedPL); err != nil {
 		t.Errorf("failed to wait for sleep to start: %v", err)
 	}
 
@@ -179,7 +184,10 @@ func TestMultiContainerWait(t *testing.T) {
 
 	// After Wait returns, ensure that the root container is running and
 	// the child has finished.
-	if err := waitForProcessList(containers[0], expectedPL[:1]); err != nil {
+	expectedPL = []*control.Process{
+		{PID: 1, Cmd: "sleep"},
+	}
+	if err := waitForProcessList(containers[0], expectedPL); err != nil {
 		t.Errorf("failed to wait for %q to start: %v", strings.Join(containers[0].Spec.Process.Args, " "), err)
 	}
 }
@@ -219,17 +227,16 @@ func TestExecWait(t *testing.T) {
 		containers = append(containers, cont)
 	}
 
-	// Check via ps that multiple processes are running.
+	// Check via ps that process is running.
 	expectedPL := []*control.Process{
-		{PID: 1, Cmd: "sleep"},
 		{PID: 2, Cmd: "sleep"},
 	}
-	if err := waitForProcessList(containers[0], expectedPL); err != nil {
+	if err := waitForProcessList(containers[1], expectedPL); err != nil {
 		t.Fatalf("failed to wait for sleep to start: %v", err)
 	}
 
 	// Wait for the second container to finish.
-	if err := waitForProcessList(containers[0], expectedPL[:1]); err != nil {
+	if err := waitForProcessCount(containers[1], 0); err != nil {
 		t.Fatalf("failed to wait for second container to stop: %v", err)
 	}
 
@@ -256,7 +263,10 @@ func TestExecWait(t *testing.T) {
 	}
 
 	// Wait for the exec'd process to exit.
-	if err := waitForProcessList(containers[0], expectedPL[:1]); err != nil {
+	expectedPL = []*control.Process{
+		{PID: 1, Cmd: "sleep"},
+	}
+	if err := waitForProcessList(containers[0], expectedPL); err != nil {
 		t.Fatalf("failed to wait for second container to stop: %v", err)
 	}
 
@@ -360,23 +370,25 @@ func TestMultiContainerSignal(t *testing.T) {
 			containers = append(containers, cont)
 		}
 
-		// Check via ps that multiple processes are running.
+		// Check via ps that container 1 process is running.
 		expectedPL := []*control.Process{
-			{PID: 1, Cmd: "sleep"},
 			{PID: 2, Cmd: "sleep"},
 		}
 
-		if err := waitForProcessList(containers[0], expectedPL); err != nil {
+		if err := waitForProcessList(containers[1], expectedPL); err != nil {
 			t.Errorf("failed to wait for sleep to start: %v", err)
 		}
 
 		// Kill process 2.
-		if err := containers[1].Signal(syscall.SIGKILL); err != nil {
+		if err := containers[1].Signal(syscall.SIGKILL, false); err != nil {
 			t.Errorf("failed to kill process 2: %v", err)
 		}
 
 		// Make sure process 1 is still running.
-		if err := waitForProcessList(containers[0], expectedPL[:1]); err != nil {
+		expectedPL = []*control.Process{
+			{PID: 1, Cmd: "sleep"},
+		}
+		if err := waitForProcessList(containers[0], expectedPL); err != nil {
 			t.Errorf("failed to wait for sleep to start: %v", err)
 		}
 
@@ -395,18 +407,18 @@ func TestMultiContainerSignal(t *testing.T) {
 			t.Errorf("error waiting for gofer to exit: %v", err)
 		}
 		// Make sure process 1 is still running.
-		if err := waitForProcessList(containers[0], expectedPL[:1]); err != nil {
+		if err := waitForProcessList(containers[0], expectedPL); err != nil {
 			t.Errorf("failed to wait for sleep to start: %v", err)
 		}
 
 		// Now that process 2 is gone, ensure we get an error trying to
 		// signal it again.
-		if err := containers[1].Signal(syscall.SIGKILL); err == nil {
+		if err := containers[1].Signal(syscall.SIGKILL, false); err == nil {
 			t.Errorf("container %q shouldn't exist, but we were able to signal it", containers[1].ID)
 		}
 
 		// Kill process 1.
-		if err := containers[0].Signal(syscall.SIGKILL); err != nil {
+		if err := containers[0].Signal(syscall.SIGKILL, false); err != nil {
 			t.Errorf("failed to kill process 1: %v", err)
 		}
 
@@ -428,7 +440,7 @@ func TestMultiContainerSignal(t *testing.T) {
 		}
 
 		// The sentry should be gone, so signaling should yield an error.
-		if err := containers[0].Signal(syscall.SIGKILL); err == nil {
+		if err := containers[0].Signal(syscall.SIGKILL, false); err == nil {
 			t.Errorf("sandbox %q shouldn't exist, but we were able to signal it", containers[0].Sandbox.ID)
 		}
 	}
@@ -453,7 +465,6 @@ func TestMultiContainerDestroy(t *testing.T) {
 		// Setup the containers.
 		var containers []*Container
 		for i, spec := range specs {
-			conf := testutil.TestConfig()
 			bundleDir, err := testutil.SetupContainerInRoot(rootDir, spec, conf)
 			if err != nil {
 				t.Fatalf("error setting up container: %v", err)
@@ -501,3 +512,144 @@ func TestMultiContainerDestroy(t *testing.T) {
 		}
 	}
 }
+
+func TestMultiContainerProcesses(t *testing.T) {
+	// Note: use 'while true' to keep 'sh' process around. Otherwise, shell will
+	// just execve into 'sleep' and both containers will look the same.
+	specs, ids := createSpecs(
+		[]string{"sleep", "100"},
+		[]string{"sh", "-c", "while true; do sleep 100; done"})
+
+	rootDir, err := testutil.SetupRootDir()
+	if err != nil {
+		t.Fatalf("error creating root dir: %v", err)
+	}
+	defer os.RemoveAll(rootDir)
+
+	var containers []*Container
+	for i, spec := range specs {
+		conf := testutil.TestConfig()
+		bundleDir, err := testutil.SetupContainerInRoot(rootDir, spec, conf)
+		if err != nil {
+			t.Fatalf("error setting up container: %v", err)
+		}
+		defer os.RemoveAll(bundleDir)
+		cont, err := Create(ids[i], spec, conf, bundleDir, "", "")
+		if err != nil {
+			t.Fatalf("error creating container: %v", err)
+		}
+		defer cont.Destroy()
+		if err := cont.Start(conf); err != nil {
+			t.Fatalf("error starting container: %v", err)
+		}
+		containers = append(containers, cont)
+	}
+
+	// Check root's container process list doesn't include other containers.
+	expectedPL0 := []*control.Process{
+		{PID: 1, Cmd: "sleep"},
+	}
+	if err := waitForProcessList(containers[0], expectedPL0); err != nil {
+		t.Errorf("failed to wait for process to start: %v", err)
+	}
+
+	// Same for the other container.
+	expectedPL1 := []*control.Process{
+		{PID: 2, Cmd: "sh"},
+		{PID: 3, PPID: 2, Cmd: "sleep"},
+	}
+	if err := waitForProcessList(containers[1], expectedPL1); err != nil {
+		t.Errorf("failed to wait for process to start: %v", err)
+	}
+
+	// Now exec into the second container and verify it shows up in the container.
+	args := &control.ExecArgs{
+		Filename: "/bin/sleep",
+		Argv:     []string{"/bin/sleep", "100"},
+	}
+	if _, err := containers[1].Execute(args); err != nil {
+		t.Fatalf("error exec'ing: %v", err)
+	}
+	expectedPL1 = append(expectedPL1, &control.Process{PID: 4, Cmd: "sleep"})
+	if err := waitForProcessList(containers[1], expectedPL1); err != nil {
+		t.Errorf("failed to wait for process to start: %v", err)
+	}
+	// Root container should remain unchanged.
+	if err := waitForProcessList(containers[0], expectedPL0); err != nil {
+		t.Errorf("failed to wait for process to start: %v", err)
+	}
+}
+
+// TestMultiContainerKillAll checks that all process that belong to a container
+// are killed when SIGKILL is sent to *all* processes in that container.
+func TestMultiContainerKillAll(t *testing.T) {
+	app, err := testutil.FindFile("runsc/container/test_app")
+	if err != nil {
+		t.Fatal("error finding test_app:", err)
+	}
+
+	// First container will remain intact while the second container is killed.
+	specs, ids := createSpecs(
+		[]string{app, "task-tree", "--depth=2", "--width=2"},
+		[]string{app, "task-tree", "--depth=4", "--width=2"})
+
+	rootDir, err := testutil.SetupRootDir()
+	if err != nil {
+		t.Fatalf("error creating root dir: %v", err)
+	}
+	defer os.RemoveAll(rootDir)
+
+	var containers []*Container
+	for i, spec := range specs {
+		conf := testutil.TestConfig()
+		bundleDir, err := testutil.SetupContainerInRoot(rootDir, spec, conf)
+		if err != nil {
+			t.Fatalf("error setting up container: %v", err)
+		}
+		defer os.RemoveAll(bundleDir)
+		cont, err := Create(ids[i], spec, conf, bundleDir, "", "")
+		if err != nil {
+			t.Fatalf("error creating container: %v", err)
+		}
+		defer cont.Destroy()
+		if err := cont.Start(conf); err != nil {
+			t.Fatalf("error starting container: %v", err)
+		}
+		containers = append(containers, cont)
+	}
+
+	// Wait until all processes are created.
+	rootProcCount := int(math.Pow(2, 3) - 1)
+	if err := waitForProcessCount(containers[0], rootProcCount); err != nil {
+		t.Fatal(err)
+	}
+	procCount := int(math.Pow(2, 5) - 1)
+	if err := waitForProcessCount(containers[1], procCount); err != nil {
+		t.Fatal(err)
+	}
+
+	// Exec more processes to ensure signal works for exec'd processes too.
+	args := &control.ExecArgs{
+		Filename: app,
+		Argv:     []string{app, "task-tree", "--depth=2", "--width=2"},
+	}
+	if _, err := containers[1].Execute(args); err != nil {
+		t.Fatalf("error exec'ing: %v", err)
+	}
+	procCount += 3
+	if err := waitForProcessCount(containers[1], procCount); err != nil {
+		t.Fatal(err)
+	}
+
+	// Kill'Em All
+	containers[1].Signal(syscall.SIGKILL, true)
+
+	// Check that all processes are gone.
+	if err := waitForProcessCount(containers[1], 0); err != nil {
+		t.Fatal(err)
+	}
+	// Check that root container was not affected.
+	if err := waitForProcessCount(containers[0], rootProcCount); err != nil {
+		t.Fatal(err)
+	}
+}
diff --git a/runsc/container/test_app.go b/runsc/container/test_app.go
index 768293cf9..a99eb97c4 100644
--- a/runsc/container/test_app.go
+++ b/runsc/container/test_app.go
@@ -22,17 +22,20 @@ import (
 	"log"
 	"net"
 	"os"
+	"os/exec"
 	"strconv"
 	"time"
 
 	"flag"
 	"github.com/google/subcommands"
+	"gvisor.googlesource.com/gvisor/runsc/test/testutil"
 )
 
 func main() {
 	subcommands.Register(subcommands.HelpCommand(), "")
 	subcommands.Register(subcommands.FlagsCommand(), "")
 	subcommands.Register(new(uds), "")
+	subcommands.Register(new(taskTree), "")
 
 	flag.Parse()
 
@@ -114,3 +117,63 @@ func server(listener net.Listener, out *os.File) {
 		fmt.Fprint(out, string(data)+"\n")
 	}
 }
+
+type taskTree struct {
+	depth int
+	width int
+}
+
+// Name implements subcommands.Command.
+func (*taskTree) Name() string {
+	return "task-tree"
+}
+
+// Synopsis implements subcommands.Command.
+func (*taskTree) Synopsis() string {
+	return "creates a tree of tasks"
+}
+
+// Usage implements subcommands.Command.
+func (*taskTree) Usage() string {
+	return "task-tree <flags>"
+}
+
+// SetFlags implements subcommands.Command.
+func (c *taskTree) SetFlags(f *flag.FlagSet) {
+	f.IntVar(&c.depth, "depth", 1, "number of levels to create")
+	f.IntVar(&c.width, "width", 1, "number of tasks at each level")
+}
+
+// Execute implements subcommands.Command.
+func (c *taskTree) Execute(ctx context.Context, f *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {
+	stop := testutil.StartReaper()
+	defer stop()
+
+	if c.depth == 0 {
+		log.Printf("Child sleeping, PID: %d\n", os.Getpid())
+		for {
+			time.Sleep(24 * time.Hour)
+		}
+	}
+	log.Printf("Parent %d sleeping, PID: %d\n", c.depth, os.Getpid())
+
+	var cmds []*exec.Cmd
+	for i := 0; i < c.width; i++ {
+		cmd := exec.Command(
+			"/proc/self/exe", c.Name(),
+			"--depth", strconv.Itoa(c.depth-1),
+			"--width", strconv.Itoa(c.width))
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+
+		if err := cmd.Start(); err != nil {
+			log.Fatal("failed to call self:", err)
+		}
+		cmds = append(cmds, cmd)
+	}
+
+	for _, c := range cmds {
+		c.Wait()
+	}
+	return subcommands.ExitSuccess
+}