Put fsgofer inside chroot

Now each container gets its own dedicated gofer that is chroot'd to the
rootfs path. This is done to add an extra layer of security in case the
gofer gets compromised.

PiperOrigin-RevId: 210396476
Change-Id: Iba21360a59dfe90875d61000db103f8609157ca0

3 files changed, 59 insertions, 31 deletions

diff --git a/runsc/cmd/BUILD b/runsc/cmd/BUILD
index b9ef4022f..5dee26a5c 100644
--- a/runsc/cmd/BUILD
+++ b/runsc/cmd/BUILD
@@ -36,6 +36,7 @@ go_library(
         "//pkg/p9",
         "//pkg/sentry/control",
         "//pkg/sentry/kernel/auth",
+        "//pkg/unet",
         "//pkg/urpc",
         "//runsc/boot",
         "//runsc/console",
diff --git a/runsc/cmd/gofer.go b/runsc/cmd/gofer.go
index e23f64d12..ab76734fc 100644
--- a/runsc/cmd/gofer.go
+++ b/runsc/cmd/gofer.go
@@ -16,6 +16,8 @@ package cmd
 
 import (
 	"os"
+	"path"
+	"sync"
 	"syscall"
 
 	"context"
@@ -24,6 +26,7 @@ import (
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"gvisor.googlesource.com/gvisor/pkg/log"
 	"gvisor.googlesource.com/gvisor/pkg/p9"
+	"gvisor.googlesource.com/gvisor/pkg/unet"
 	"gvisor.googlesource.com/gvisor/runsc/fsgofer"
 	"gvisor.googlesource.com/gvisor/runsc/specutils"
 )
@@ -35,10 +38,6 @@ type Gofer struct {
 	ioFDs     intFlags
 	applyCaps bool
 
-	// controllerFD is the file descriptor of a stream socket for the
-	// control server that is donated to this process.
-	controllerFD int
-
 	panicOnWrite bool
 }
 
@@ -62,26 +61,16 @@ func (g *Gofer) SetFlags(f *flag.FlagSet) {
 	f.StringVar(&g.bundleDir, "bundle", "", "path to the root of the bundle directory, defaults to the current directory")
 	f.Var(&g.ioFDs, "io-fds", "list of FDs to connect 9P servers. They must follow this order: root first, then mounts as defined in the spec")
 	f.BoolVar(&g.applyCaps, "apply-caps", true, "if true, apply capabilities to restrict what the Gofer process can do")
-	f.IntVar(&g.controllerFD, "controller-fd", -1, "required FD of a stream socket for the control server that must be donated to this process")
 	f.BoolVar(&g.panicOnWrite, "panic-on-write", false, "if true, panics on attempts to write to RO mounts. RW mounts are unnaffected")
 }
 
 // Execute implements subcommands.Command.
 func (g *Gofer) Execute(_ context.Context, f *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {
-	if g.bundleDir == "" || len(g.ioFDs) < 1 || g.controllerFD == -1 {
+	if g.bundleDir == "" || len(g.ioFDs) < 1 {
 		f.Usage()
 		return subcommands.ExitUsageError
 	}
 
-	// fsgofer should run with a umask of 0, because we want to preserve file
-	// modes exactly as sent by the sandbox, which will have applied its own umask.
-	syscall.Umask(0)
-
-	spec, err := specutils.ReadSpec(g.bundleDir)
-	if err != nil {
-		Fatalf("error reading spec: %v", err)
-	}
-
 	if g.applyCaps {
 		// Minimal set of capabilities needed by the Gofer to operate on files.
 		caps := []string{
@@ -107,49 +96,84 @@ func (g *Gofer) Execute(_ context.Context, f *flag.FlagSet, args ...interface{})
 		panic("unreachable")
 	}
 
+	spec, err := specutils.ReadSpec(g.bundleDir)
+	if err != nil {
+		Fatalf("error reading spec: %v", err)
+	}
 	specutils.LogSpec(spec)
 
-	// Start with root mount, then add any other addition mount as needed.
+	// fsgofer should run with a umask of 0, because we want to preserve file
+	// modes exactly as sent by the sandbox, which will have applied its own umask.
+	syscall.Umask(0)
+
+	// Find what path is going to be served by this gofer.
+	root := absPath(g.bundleDir, spec.Root.Path)
+	if err := syscall.Chroot(root); err != nil {
+		Fatalf("failed to chroot to %q: %v", root, err)
+	}
+	if err := syscall.Chdir("/"); err != nil {
+		Fatalf("failed to change working dir: %v", err)
+	}
+	log.Infof("Process chroot'd to %q", root)
+
+	// Start with root mount, then add any other additional mount as needed.
 	ats := make([]p9.Attacher, 0, len(spec.Mounts)+1)
-	p := absPath(g.bundleDir, spec.Root.Path)
-	ats = append(ats, fsgofer.NewAttachPoint(p, fsgofer.Config{
+	ats = append(ats, fsgofer.NewAttachPoint("/", fsgofer.Config{
 		ROMount:      spec.Root.Readonly,
 		PanicOnWrite: g.panicOnWrite,
 		// Docker uses overlay2 by default for the root mount, and overlay2 does a copy-up when
 		// each file is opened as writable. Thus, we open files lazily to avoid copy-up.
 		LazyOpenForWrite: true,
 	}))
-	log.Infof("Serving %q mapped to %q on FD %d", "/", p, g.ioFDs[0])
+	log.Infof("Serving %q mapped to %q on FD %d (ro: %t)", "/", root, g.ioFDs[0], spec.Root.Readonly)
 
 	mountIdx := 1 // first one is the root
 	for _, m := range spec.Mounts {
 		if specutils.Is9PMount(m) {
-			p = absPath(g.bundleDir, m.Source)
-			ats = append(ats, fsgofer.NewAttachPoint(p, fsgofer.Config{
+			if !path.IsAbs(m.Destination) {
+				Fatalf("destination must be absolute path: %v", m.Destination)
+			}
+			cfg := fsgofer.Config{
 				ROMount:          isReadonlyMount(m.Options),
 				PanicOnWrite:     g.panicOnWrite,
 				LazyOpenForWrite: false,
-			}))
+			}
+			ats = append(ats, fsgofer.NewAttachPoint(m.Destination, cfg))
 
 			if mountIdx >= len(g.ioFDs) {
 				Fatalf("No FD found for mount. Did you forget --io-fd? mount: %d, %v", len(g.ioFDs), m)
 			}
-			log.Infof("Serving %q mapped to %q on FD %d", m.Destination, p, g.ioFDs[mountIdx])
+			log.Infof("Serving %q mapped on FD %d (ro: %t)", m.Destination, g.ioFDs[mountIdx], cfg.ROMount)
 			mountIdx++
 		}
 	}
 	if mountIdx != len(g.ioFDs) {
-		Fatalf("Too many FDs passed for mounts. mounts: %d, FDs: %d", mountIdx, len(g.ioFDs))
+		Fatalf("too many FDs passed for mounts. mounts: %d, FDs: %d", mountIdx, len(g.ioFDs))
 	}
 
-	ctrl, err := fsgofer.NewController(g.controllerFD, g.bundleDir)
+	runServers(ats, g.ioFDs)
+	return subcommands.ExitSuccess
+}
 
-	if err := ctrl.Serve(ats, g.ioFDs); err != nil {
-		Fatalf("Failed to serve via P9: %v", err)
+func runServers(ats []p9.Attacher, ioFDs []int) {
+	// Run the loops and wait for all to exit.
+	var wg sync.WaitGroup
+	for i, ioFD := range ioFDs {
+		wg.Add(1)
+		go func(ioFD int, at p9.Attacher) {
+			socket, err := unet.NewSocket(ioFD)
+			if err != nil {
+				Fatalf("err creating server on FD %d: %v", ioFD, err)
+			}
+			s := p9.NewServer(at)
+			if err := s.Handle(socket); err != nil {
+				Fatalf("P9 server returned error. Gofer is shutting down. FD: %d, err: %v", ioFD, err)
+			}
+			wg.Done()
+		}(ioFD, ats[i])
 	}
-	ctrl.Wait()
-
-	return subcommands.ExitSuccess
+	wg.Wait()
+	log.Infof("All 9P servers exited.")
 }
 
 func isReadonlyMount(opts []string) bool {
diff --git a/runsc/cmd/state.go b/runsc/cmd/state.go
index 28752d95e..265014e1b 100644
--- a/runsc/cmd/state.go
+++ b/runsc/cmd/state.go
@@ -63,8 +63,11 @@ func (*State) Execute(_ context.Context, f *flag.FlagSet, args ...interface{}) s
 	}
 	log.Debugf("Returning state for container %+v", c)
 
+	state := c.State()
+	log.Debugf("State: %+v", state)
+
 	// Write json-encoded state directly to stdout.
-	b, err := json.MarshalIndent(c.State(), "", "  ")
+	b, err := json.MarshalIndent(state, "", "  ")
 	if err != nil {
 		Fatalf("error marshaling container state: %v", err)
 	}