runsc fsgofer: Support dynamic serving of filesystems.

When multiple containers run inside a sentry, each container has its own root
filesystem and set of mounts. Containers are also added after sentry boot rather
than all configured and known at boot time.

The fsgofer needs to be able to serve the root filesystem of each container.
Thus, it must be possible to add filesystems after the fsgofer has already
started.

This change:
* Creates a URPC endpoint within the gofer process that listens for requests to
  serve new content.
* Enables the sentry, when starting a new container, to add the new container's
  filesystem.
* Mounts those new filesystems at separate roots within the sentry.

PiperOrigin-RevId: 208903248
Change-Id: Ifa91ec9c8caf5f2f0a9eead83c4a57090ce92068

3 files changed, 212 insertions, 0 deletions

diff --git a/runsc/fsgofer/BUILD b/runsc/fsgofer/BUILD
index 24e172f48..0bc682b5f 100644
--- a/runsc/fsgofer/BUILD
+++ b/runsc/fsgofer/BUILD
@@ -5,6 +5,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "fsgofer",
     srcs = [
+        "control.go",
         "fsgofer.go",
         "fsgofer_unsafe.go",
     ],
@@ -14,9 +15,12 @@ go_library(
     ],
     deps = [
         "//pkg/abi/linux",
+        "//pkg/control/server",
         "//pkg/fd",
         "//pkg/log",
         "//pkg/p9",
+        "//pkg/unet",
+        "//pkg/urpc",
         "@org_golang_x_sys//unix:go_default_library",
     ],
 )
diff --git a/runsc/fsgofer/control.go b/runsc/fsgofer/control.go
new file mode 100644
index 000000000..8ce8ee8a0
--- /dev/null
+++ b/runsc/fsgofer/control.go
@@ -0,0 +1,203 @@
+// Copyright 2018 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fsgofer
+
+import (
+	"fmt"
+	"path/filepath"
+	"sync"
+
+	"gvisor.googlesource.com/gvisor/pkg/control/server"
+	"gvisor.googlesource.com/gvisor/pkg/log"
+	"gvisor.googlesource.com/gvisor/pkg/p9"
+	"gvisor.googlesource.com/gvisor/pkg/unet"
+	"gvisor.googlesource.com/gvisor/pkg/urpc"
+)
+
+// Controller manages the fsgofer's control server.
+type Controller struct {
+	// api holds the control server's URPC endpoints.
+	api api
+
+	// srv is the control server.
+	srv *server.Server
+}
+
+// NewController creates a new Controller and starts it listenting
+func NewController(fd int, rootBundleDir string) (*Controller, error) {
+	if !filepath.IsAbs(rootBundleDir) {
+		return nil, fmt.Errorf("NewController should receive an absolute bundle dir path, but got %q", rootBundleDir)
+	}
+
+	srv, err := server.CreateFromFD(fd)
+	if err != nil {
+		return nil, err
+	}
+
+	cr := &Controller{srv: srv}
+	cr.api.rootBundleDir = rootBundleDir
+	cr.api.bundleDirs = make(map[string]string)
+	srv.Register(&cr.api)
+
+	if err := srv.StartServing(); err != nil {
+		return nil, err
+	}
+
+	return cr, nil
+}
+
+// Wait waits for all the p9 servers to finish, then shuts down the control
+// server.
+func (cr *Controller) Wait() {
+	cr.api.p9wg.Wait()
+	cr.srv.Stop()
+	log.Infof("All 9P servers exited.")
+}
+
+// Serve starts serving each Attacher in ats via its corresponding file
+// descriptor in ioFDs.
+func (cr *Controller) Serve(ats []p9.Attacher, ioFDs []int) error {
+	if len(ats) != len(ioFDs) {
+		return fmt.Errorf("number of attach points does not match the number of IO FDs (%d and %d)", len(ats), len(ioFDs))
+	}
+	for i, _ := range ats {
+		cr.api.serve(ats[i], ioFDs[i])
+	}
+	return nil
+}
+
+// api URPC methods.
+const (
+	// AddBundleDirs readies the gofer to serve from a new bundle
+	// directory. It should be called during runsc create.
+	AddBundleDirs = "api.AddBundleDirs"
+
+	// ServeDirectory serves a new directory via the fsgofer. It should be
+	// called during runsc start.
+	ServeDirectory = "api.ServeDirectory"
+)
+
+// API defines and implements the URPC endpoints for the gofer.
+type api struct {
+	// p9wg waits for all the goroutines serving the sentry via p9. When its
+	// counter is 0, the gofer is out of work and exits.
+	p9wg sync.WaitGroup
+
+	// bundleDirs maps from container ID to bundle directory for each
+	// container.
+	bundleDirs map[string]string
+
+	// rootBundleDir is the bundle directory of the root container.
+	rootBundleDir string
+}
+
+// AddBundleDirsRequest is the URPC argument to AddBundleDirs.
+type AddBundleDirsRequest struct {
+	// BundleDirs is a map of container IDs to bundle directories to add to
+	// the gofer.
+	BundleDirs map[string]string
+}
+
+// AddBundleDirsRequest adds bundle directories that for the gofer to serve.
+func (api *api) AddBundleDirs(req *AddBundleDirsRequest, _ *struct{}) error {
+	log.Debugf("fsgofer.AddBundleDirs")
+	for cid, bd := range req.BundleDirs {
+		if _, ok := api.bundleDirs[cid]; ok {
+			return fmt.Errorf("fsgofer already has a bundleDir for container %q", cid)
+		}
+		api.bundleDirs[cid] = bd
+	}
+	return nil
+}
+
+// ServeDirectoryRequest is the URPC argument to ServeDirectory.
+type ServeDirectoryRequest struct {
+	// Dir is the absolute path to a directory to be served to the sentry.
+	Dir string
+
+	// IsReadOnly specifies whether the directory should be served in
+	// read-only mode.
+	IsReadOnly bool
+
+	// CID is the container ID of the container that needs to serve a
+	// directory.
+	CID string
+
+	// FilePayload contains the socket over which the sentry will request
+	// files from Dir.
+	urpc.FilePayload
+}
+
+// ServeDirectory begins serving a directory via a file descriptor for the
+// sentry. Directories must be added via AddBundleDirsRequest before
+// ServeDirectory is called.
+func (api *api) ServeDirectory(req *ServeDirectoryRequest, _ *struct{}) error {
+	log.Debugf("fsgofer.ServeDirectory: %+v", req)
+
+	if req.Dir == "" {
+		return fmt.Errorf("ServeDirectory should receive a directory argument, but was empty")
+	}
+	if req.CID == "" {
+		return fmt.Errorf("ServeDirectory should receive a CID argument, but was empty")
+	}
+	// Prevent CIDs containing ".." from confusing the sentry when creating
+	// /containers/<cid> directory.
+	// TODO: Once we have multiple independant roots, this
+	// check won't be necessary.
+	if filepath.Clean(req.CID) != req.CID {
+		return fmt.Errorf("container ID shouldn't contain directory traversals such as \"..\": %q", req.CID)
+	}
+	if nFiles := len(req.FilePayload.Files); nFiles != 1 {
+		return fmt.Errorf("ServeDirectory should receive 1 file descriptor, but got %d", nFiles)
+	}
+
+	bd, ok := api.bundleDirs[req.CID]
+	if !ok {
+		// If there's no entry in bundleDirs for the container ID, this
+		// is the root container.
+		bd = api.rootBundleDir
+	}
+
+	// Relative paths are served relative to the bundle directory.
+	absDir := req.Dir
+	if !filepath.IsAbs(absDir) {
+		absDir = filepath.Join(bd, req.Dir)
+	}
+
+	// Create the attach point and start serving.
+	at := NewAttachPoint(absDir, Config{
+		ROMount:          req.IsReadOnly,
+		LazyOpenForWrite: true,
+	})
+	api.serve(at, int(req.FilePayload.Files[0].Fd()))
+
+	return nil
+}
+
+// serve begins serving a directory via a file descriptor.
+func (api *api) serve(at p9.Attacher, ioFD int) {
+	api.p9wg.Add(1)
+	go func(ioFD int, at p9.Attacher) {
+		socket, err := unet.NewSocket(ioFD)
+		if err != nil {
+			panic(fmt.Sprintf("err creating server on FD %d: %v", ioFD, err))
+		}
+		s := p9.NewServer(at)
+		if err := s.Handle(socket); err != nil {
+			panic(fmt.Sprintf("P9 server returned error. Gofer is shutting down. FD: %d, err: %v", ioFD, err))
+		}
+		api.p9wg.Done()
+	}(ioFD, at)
+}
diff --git a/runsc/fsgofer/fsgofer.go b/runsc/fsgofer/fsgofer.go
index 52cdc91a2..38263896a 100644
--- a/runsc/fsgofer/fsgofer.go
+++ b/runsc/fsgofer/fsgofer.go
@@ -66,6 +66,11 @@ func (f fileType) String() string {
 	return "unknown"
 }
 
+// ControlSocketAddr generates an abstract unix socket name for the given id.
+func ControlSocketAddr(id string) string {
+	return fmt.Sprintf("\x00runsc-gofer.%s", id)
+}
+
 // Config sets configuration options for each attach point.
 type Config struct {
 	// ROMount is set to true if this is a readonly mount.