runsc: Change cache policy for root fs and volume mounts.

Previously, gofer filesystems were configured with the default "fscache"
policy, which caches filesystem metadata and contents aggressively.  While this
setting is best for performance, it means that changes from inside the sandbox
may not be immediately propagated outside the sandbox, and vice-versa.

This CL changes volumes and the root fs configuration to use a new
"remote-revalidate" cache policy which tries to retain as much caching as
possible while still making fs changes visible across the sandbox boundary.

This cache policy is enabled by default for the root filesystem. The default
value for the "--file-access" flag is still "proxy", but the behavior is
changed to use the new cache policy.

A new value for the "--file-access" flag is added, called "proxy-exclusive",
which turns on the previous aggressive caching behavior. As the name implies,
this flag should be used when the sandbox has "exclusive" access to the
filesystem.

All volume mounts are configured to use the new cache policy, since it is
safest and most likely to be correct. There is not currently a way to change
this behavior, but it's possible to add such a mechanism in the future. The
configurability is a smaller issue for volumes, since most of the expensive
application fs operations (walking + stating files) will likely served by the
root fs.

PiperOrigin-RevId: 208735037
Change-Id: Ife048fab1948205f6665df8563434dbc6ca8cfc9

8 files changed, 261 insertions, 75 deletions

diff --git a/pkg/sentry/fs/gofer/cache_policy.go b/pkg/sentry/fs/gofer/cache_policy.go
index 52d97b54f..fa8abf51c 100644
--- a/pkg/sentry/fs/gofer/cache_policy.go
+++ b/pkg/sentry/fs/gofer/cache_policy.go
@@ -35,10 +35,9 @@ const (
 	// fs agent immediately.
 	cacheAllWritethrough
 
-	// Use virtual file system cache for everything, but reload dirents
-	// from the remote filesystem on each lookup. Thus, if the remote
-	// filesystem has changed, the returned dirent will have the updated
-	// state.
+	// Use the (host) page cache for reads/writes, but don't cache anything
+	// else. This allows the sandbox filesystem to stay in sync with any
+	// changes to the remote filesystem.
 	//
 	// This policy should *only* be used with remote filesystems that
 	// donate their host FDs to the sandbox and thus use the host page
diff --git a/runsc/boot/config.go b/runsc/boot/config.go
index 074cd6a63..6c69a7c38 100644
--- a/runsc/boot/config.go
+++ b/runsc/boot/config.go
@@ -64,6 +64,11 @@ const (
 	// requests and forwards them to the host.
 	FileAccessProxy FileAccessType = iota
 
+	// FileAccessProxyExclusive is the same as FileAccessProxy, but enables
+	// extra caching for improved performance. It should only be used if
+	// the sandbox has exclusive access to the filesystem.
+	FileAccessProxyExclusive
+
 	// FileAccessDirect connects the sandbox directly to the host filesystem.
 	FileAccessDirect
 )
@@ -73,6 +78,8 @@ func MakeFileAccessType(s string) (FileAccessType, error) {
 	switch s {
 	case "proxy":
 		return FileAccessProxy, nil
+	case "proxy-exclusive":
+		return FileAccessProxyExclusive, nil
 	case "direct":
 		return FileAccessDirect, nil
 	default:
@@ -84,6 +91,8 @@ func (f FileAccessType) String() string {
 	switch f {
 	case FileAccessProxy:
 		return "proxy"
+	case FileAccessProxyExclusive:
+		return "proxy-exclusive"
 	case FileAccessDirect:
 		return "direct"
 	default:
diff --git a/runsc/boot/fs.go b/runsc/boot/fs.go
index e596c739f..eea2ec1f5 100644
--- a/runsc/boot/fs.go
+++ b/runsc/boot/fs.go
@@ -17,6 +17,7 @@ package boot
 import (
 	"fmt"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	// Include filesystem types that OCI spec might mount.
@@ -54,6 +55,9 @@ type fdDispenser struct {
 }
 
 func (f *fdDispenser) remove() int {
+	if f.empty() {
+		panic("fdDispenser out of fds")
+	}
 	rv := f.fds[0]
 	f.fds = f.fds[1:]
 	return rv
@@ -160,8 +164,6 @@ func compileMounts(spec *specs.Spec) []specs.Mount {
 // setMounts iterates over mounts and mounts them in the specified
 // mount namespace.
 func setMounts(ctx context.Context, conf *Config, mns *fs.MountNamespace, fds *fdDispenser, mounts []specs.Mount) error {
-
-	// Mount all submounts from mounts.
 	for _, m := range mounts {
 		if err := mountSubmount(ctx, conf, mns, fds, m, mounts); err != nil {
 			return err
@@ -181,11 +183,12 @@ func createRootMount(ctx context.Context, spec *specs.Spec, conf *Config, fds *f
 	)
 
 	switch conf.FileAccess {
-	case FileAccessProxy:
+	case FileAccessProxy, FileAccessProxyExclusive:
 		fd := fds.remove()
 		log.Infof("Mounting root over 9P, ioFD: %d", fd)
 		hostFS := mustFindFilesystem("9p")
-		rootInode, err = hostFS.Mount(ctx, rootDevice, mf, fmt.Sprintf("trans=fd,rfdno=%d,wfdno=%d,privateunixsocket=true", fd, fd))
+		opts := p9MountOptions(conf, fd)
+		rootInode, err = hostFS.Mount(ctx, rootDevice, mf, strings.Join(opts, ","))
 		if err != nil {
 			return nil, fmt.Errorf("failed to generate root mount point: %v", err)
 		}
@@ -242,13 +245,16 @@ func addOverlay(ctx context.Context, conf *Config, lower *fs.Inode, name string,
 	return fs.NewOverlayRoot(ctx, upper, lower, lowerFlags)
 }
 
-// getMountNameAndOptions retrieves the fsName, data, and useOverlay values
+// getMountNameAndOptions retrieves the fsName, opts, and useOverlay values
 // used for mounts.
 func getMountNameAndOptions(conf *Config, m specs.Mount, fds *fdDispenser) (string, []string, bool, error) {
-	var fsName string
-	var data []string
-	var useOverlay bool
-	var err error
+	var (
+		fsName     string
+		opts       []string
+		useOverlay bool
+		err        error
+	)
+
 	switch m.Type {
 	case "devpts", "devtmpfs", "proc", "sysfs":
 		fsName = m.Type
@@ -258,17 +264,17 @@ func getMountNameAndOptions(conf *Config, m specs.Mount, fds *fdDispenser) (stri
 		fsName = m.Type
 
 		// tmpfs has some extra supported options that we must pass through.
-		data, err = parseAndFilterOptions(m.Options, "mode", "uid", "gid")
+		opts, err = parseAndFilterOptions(m.Options, "mode", "uid", "gid")
 
 	case "bind":
 		switch conf.FileAccess {
-		case FileAccessProxy:
+		case FileAccessProxy, FileAccessProxyExclusive:
 			fd := fds.remove()
 			fsName = "9p"
-			data = []string{"trans=fd", fmt.Sprintf("rfdno=%d", fd), fmt.Sprintf("wfdno=%d", fd), "privateunixsocket=true"}
+			opts = p9MountOptions(conf, fd)
 		case FileAccessDirect:
 			fsName = "whitelistfs"
-			data = []string{"root=" + m.Source, "dont_translate_ownership=true"}
+			opts = []string{"root=" + m.Source, "dont_translate_ownership=true"}
 		default:
 			err = fmt.Errorf("invalid file access type: %v", conf.FileAccess)
 		}
@@ -282,13 +288,13 @@ func getMountNameAndOptions(conf *Config, m specs.Mount, fds *fdDispenser) (stri
 		// we do not support.
 		log.Warningf("ignoring unknown filesystem type %q", m.Type)
 	}
-	return fsName, data, useOverlay, err
+	return fsName, opts, useOverlay, err
 }
 
 func mountSubmount(ctx context.Context, conf *Config, mns *fs.MountNamespace, fds *fdDispenser, m specs.Mount, mounts []specs.Mount) error {
 	// Map mount type to filesystem name, and parse out the options that we are
 	// capable of dealing with.
-	fsName, data, useOverlay, err := getMountNameAndOptions(conf, m, fds)
+	fsName, opts, useOverlay, err := getMountNameAndOptions(conf, m, fds)
 
 	// Return the error or nil that corresponds to the default case in getMountNameAndOptions.
 	if err != nil {
@@ -307,7 +313,7 @@ func mountSubmount(ctx context.Context, conf *Config, mns *fs.MountNamespace, fd
 		mf.ReadOnly = true
 	}
 
-	inode, err := filesystem.Mount(ctx, mountDevice(m), mf, strings.Join(data, ","))
+	inode, err := filesystem.Mount(ctx, mountDevice(m), mf, strings.Join(opts, ","))
 	if err != nil {
 		return fmt.Errorf("failed to create mount with source %q: %v", m.Source, err)
 	}
@@ -387,6 +393,20 @@ func mkdirAll(ctx context.Context, mns *fs.MountNamespace, path string) error {
 	return nil
 }
 
+// p9MountOptions creates a slice of options for a p9 mount.
+func p9MountOptions(conf *Config, fd int) []string {
+	opts := []string{
+		"trans=fd",
+		"rfdno=" + strconv.Itoa(fd),
+		"wfdno=" + strconv.Itoa(fd),
+		"privateunixsocket=true",
+	}
+	if conf.FileAccess == FileAccessProxy {
+		opts = append(opts, "cache=remote_revalidating")
+	}
+	return opts
+}
+
 // parseAndFilterOptions parses a MountOptions slice and filters by the allowed
 // keys.
 func parseAndFilterOptions(opts []string, allowedKeys ...string) ([]string, error) {
@@ -436,8 +456,7 @@ func mountDevice(m specs.Mount) string {
 // addRestoreMount adds a mount to the MountSources map used for restoring a
 // checkpointed container.
 func addRestoreMount(conf *Config, renv *fs.RestoreEnvironment, m specs.Mount, fds *fdDispenser) error {
-	fsName, data, _, err := getMountNameAndOptions(conf, m, fds)
-	dataString := strings.Join(data, ",")
+	fsName, opts, _, err := getMountNameAndOptions(conf, m, fds)
 
 	// Return the error or nil that corresponds to the default case in getMountNameAndOptions.
 	if err != nil {
@@ -452,7 +471,7 @@ func addRestoreMount(conf *Config, renv *fs.RestoreEnvironment, m specs.Mount, f
 	newMount := fs.MountArgs{
 		Dev:   mountDevice(m),
 		Flags: mountFlags(m.Options),
-		Data:  dataString,
+		Data:  strings.Join(opts, ","),
 	}
 	renv.MountSources[fsName] = append(renv.MountSources[fsName], newMount)
 	log.Infof("Added mount at %q: %+v", fsName, newMount)
@@ -473,7 +492,8 @@ func createRestoreEnvironment(spec *specs.Spec, conf *Config, fds *fdDispenser)
 
 	// Add root mount.
 	fd := fds.remove()
-	dataString := strings.Join([]string{"trans=fd", fmt.Sprintf("rfdno=%d", fd), fmt.Sprintf("wfdno=%d", fd), "privateunixsocket=true"}, ",")
+	opts := p9MountOptions(conf, fd)
+
 	mf := fs.MountSourceFlags{}
 	if spec.Root.Readonly {
 		mf.ReadOnly = true
@@ -482,7 +502,7 @@ func createRestoreEnvironment(spec *specs.Spec, conf *Config, fds *fdDispenser)
 	rootMount := fs.MountArgs{
 		Dev:   rootDevice,
 		Flags: mf,
-		Data:  dataString,
+		Data:  strings.Join(opts, ","),
 	}
 	renv.MountSources[rootFsName] = append(renv.MountSources[rootFsName], rootMount)
 
diff --git a/runsc/boot/loader_test.go b/runsc/boot/loader_test.go
index 7ea2e1ee5..f2f690b5d 100644
--- a/runsc/boot/loader_test.go
+++ b/runsc/boot/loader_test.go
@@ -398,7 +398,7 @@ func TestRestoreEnvironment(t *testing.T) {
 						{
 							Dev:   "9pfs-/",
 							Flags: fs.MountSourceFlags{ReadOnly: true},
-							Data:  "trans=fd,rfdno=0,wfdno=0,privateunixsocket=true",
+							Data:  "trans=fd,rfdno=0,wfdno=0,privateunixsocket=true,cache=remote_revalidating",
 						},
 					},
 					"tmpfs": {
@@ -458,11 +458,11 @@ func TestRestoreEnvironment(t *testing.T) {
 						{
 							Dev:   "9pfs-/",
 							Flags: fs.MountSourceFlags{ReadOnly: true},
-							Data:  "trans=fd,rfdno=0,wfdno=0,privateunixsocket=true",
+							Data:  "trans=fd,rfdno=0,wfdno=0,privateunixsocket=true,cache=remote_revalidating",
 						},
 						{
 							Dev:  "9pfs-/dev/fd-foo",
-							Data: "trans=fd,rfdno=1,wfdno=1,privateunixsocket=true",
+							Data: "trans=fd,rfdno=1,wfdno=1,privateunixsocket=true,cache=remote_revalidating",
 						},
 					},
 					"tmpfs": {
@@ -522,7 +522,7 @@ func TestRestoreEnvironment(t *testing.T) {
 						{
 							Dev:   "9pfs-/",
 							Flags: fs.MountSourceFlags{ReadOnly: true},
-							Data:  "trans=fd,rfdno=0,wfdno=0,privateunixsocket=true",
+							Data:  "trans=fd,rfdno=0,wfdno=0,privateunixsocket=true,cache=remote_revalidating",
 						},
 					},
 					"tmpfs": {
@@ -606,21 +606,21 @@ func TestRestoreEnvironment(t *testing.T) {
 			errorExpected: true,
 		},
 	}
-
 	for _, tc := range testCases {
-		fds := &fdDispenser{fds: tc.ioFDs}
-
-		actualRenv, err := createRestoreEnvironment(tc.spec, tc.conf, fds)
-		if !tc.errorExpected && err != nil {
-			t.Fatalf("could not create restore environment for test:%s", tc.name)
-		} else if tc.errorExpected {
-			if err == nil {
-				t.Fatalf("expected an error, but no error occurred.")
+		t.Run(tc.name, func(t *testing.T) {
+			fds := &fdDispenser{fds: tc.ioFDs}
+			actualRenv, err := createRestoreEnvironment(tc.spec, tc.conf, fds)
+			if !tc.errorExpected && err != nil {
+				t.Fatalf("could not create restore environment for test:%s", tc.name)
+			} else if tc.errorExpected {
+				if err == nil {
+					t.Errorf("expected an error, but no error occurred.")
+				}
+			} else {
+				if !reflect.DeepEqual(*actualRenv, tc.expectedRenv) {
+					t.Errorf("restore environments did not match for test:%s\ngot:%+v\nwant:%+v\n", tc.name, *actualRenv, tc.expectedRenv)
+				}
 			}
-		} else {
-			if !reflect.DeepEqual(*actualRenv, tc.expectedRenv) {
-				t.Fatalf("restore environments did not match for test:%s\ngot:%+v\nwant:%+v\n", tc.name, *actualRenv, tc.expectedRenv)
-			}
-		}
+		})
 	}
 }
diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go
index 11edcd615..33c53e189 100644
--- a/runsc/container/container_test.go
+++ b/runsc/container/container_test.go
@@ -227,33 +227,43 @@ func findUDSApp() (string, error) {
 	return matches[0], nil
 }
 
-type configOptions int
+type configOption int
 
 const (
-	overlay configOptions = 1 << iota
+	overlay configOption = iota
 	kvm
+	nonExclusiveFS
 )
-const all = overlay | kvm
+
+var all = []configOption{overlay, kvm, nonExclusiveFS}
 
 // configs generates different configurations to run tests.
-func configs(opts configOptions) []*boot.Config {
-	cs := []*boot.Config{testutil.TestConfig(), testutil.TestConfig()}
-	return cs
+func configs(opts ...configOption) []*boot.Config {
+	// Always load the default config.
+	cs := []*boot.Config{testutil.TestConfig()}
 
-	if opts&overlay != 0 {
+	for _, o := range opts {
 		c := testutil.TestConfig()
-		c.Overlay = true
+		switch o {
+		case overlay:
+			c.Overlay = true
+		case kvm:
+			// TODO: KVM tests are flaky. Disable until fixed.
+			continue
+
+			// TODO: KVM doesn't work with --race.
+			if testutil.RaceEnabled {
+				continue
+			}
+			c.Platform = boot.PlatformKVM
+		case nonExclusiveFS:
+			c.FileAccess = boot.FileAccessProxy
+		default:
+			panic(fmt.Sprintf("unknown config option %v", o))
+
+		}
 		cs = append(cs, c)
 	}
-
-	// TODO: KVM tests are flaky. Disable until fixed.
-	// // TODO: KVM doesn't work with --race.
-	// if !testutil.RaceEnabled && opts&kvm != 0 {
-	// 	c := testutil.TestConfig()
-	// 	c.Platform = boot.PlatformKVM
-	// 	cs = append(cs, c)
-	// }
-
 	return cs
 }
 
@@ -261,7 +271,7 @@ func configs(opts configOptions) []*boot.Config {
 // It verifies after each step that the container can be loaded from disk, and
 // has the correct status.
 func TestLifecycle(t *testing.T) {
-	for _, conf := range configs(all) {
+	for _, conf := range configs(all...) {
 		t.Logf("Running test with conf: %+v", conf)
 		// The container will just sleep for a long time.  We will kill it before
 		// it finishes sleeping.
@@ -1049,10 +1059,11 @@ func TestPauseResumeStatus(t *testing.T) {
 // - Running exec as non-root with CAP_DAC_OVERRIDE succeeds because it skips
 //   this check.
 func TestCapabilities(t *testing.T) {
-	const uid = 343
-	const gid = 2401
+	// Pick uid/gid different than ours.
+	uid := auth.KUID(os.Getuid() + 1)
+	gid := auth.KGID(os.Getgid() + 1)
 
-	for _, conf := range configs(all) {
+	for _, conf := range configs(all...) {
 		t.Logf("Running test with conf: %+v", conf)
 
 		spec := testutil.NewSpecWithArgs("sleep", "100")
@@ -1142,7 +1153,7 @@ func TestCapabilities(t *testing.T) {
 
 // Test that an tty FD is sent over the console socket if one is provided.
 func TestConsoleSocket(t *testing.T) {
-	for _, conf := range configs(all) {
+	for _, conf := range configs(all...) {
 		t.Logf("Running test with conf: %+v", conf)
 		spec := testutil.NewSpecWithArgs("true")
 		rootDir, bundleDir, err := testutil.SetupContainer(spec, conf)
@@ -1303,8 +1314,6 @@ func TestReadonlyRoot(t *testing.T) {
 		defer os.RemoveAll(rootDir)
 		defer os.RemoveAll(bundleDir)
 
-		conf.Overlay = true
-
 		// Create, start and wait for the container.
 		s, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "")
 		if err != nil {
@@ -1348,8 +1357,6 @@ func TestReadonlyMount(t *testing.T) {
 		defer os.RemoveAll(rootDir)
 		defer os.RemoveAll(bundleDir)
 
-		conf.Overlay = true
-
 		// Create, start and wait for the container.
 		s, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "")
 		if err != nil {
@@ -1430,7 +1437,7 @@ func TestAbbreviatedIDs(t *testing.T) {
 // TestMultiContainerSanity checks that it is possible to run 2 dead-simple
 // containers in the same sandbox.
 func TestMultiContainerSanity(t *testing.T) {
-	for _, conf := range configs(all) {
+	for _, conf := range configs(all...) {
 		t.Logf("Running test with conf: %+v", conf)
 
 		containerIDs := []string{
@@ -1619,3 +1626,149 @@ func TestMultiContainerWait(t *testing.T) {
 
 	wg.Wait()
 }
+
+// Check that modifications to a volume mount are propigated into and out of
+// the sandbox.
+func TestContainerVolumeContentsShared(t *testing.T) {
+	// Only run this test with shared proxy, since that is the only
+	// behavior it is testing.
+	conf := testutil.TestConfig()
+	conf.FileAccess = boot.FileAccessProxy
+	t.Logf("Running test with conf: %+v", conf)
+
+	// Main process just sleeps. We will use "exec" to probe the state of
+	// the filesystem.
+	spec := testutil.NewSpecWithArgs("sleep", "1000")
+
+	// Mount host temp dir inside the sandbox at '/tmp2'.
+	hostTmpDir, err := ioutil.TempDir("", "root-fs-test")
+	sandboxTmpDir := "/tmp2"
+	if err != nil {
+		t.Fatalf("TempDir failed: %v", err)
+	}
+	spec.Mounts = append(spec.Mounts, specs.Mount{
+		Type:        "bind",
+		Destination: sandboxTmpDir,
+		Source:      hostTmpDir,
+	})
+
+	rootDir, bundleDir, err := testutil.SetupContainer(spec, conf)
+	if err != nil {
+		t.Fatalf("error setting up container: %v", err)
+	}
+	defer os.RemoveAll(rootDir)
+	defer os.RemoveAll(bundleDir)
+
+	// Create and start the container.
+	c, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "")
+	if err != nil {
+		t.Fatalf("error creating container: %v", err)
+	}
+	defer c.Destroy()
+	if err := c.Start(conf); err != nil {
+		t.Fatalf("error starting container: %v", err)
+	}
+
+	// File that will be used to check consistency inside/outside sandbox.
+	hostFilename := filepath.Join(hostTmpDir, "file")
+	sandboxFilename := filepath.Join(sandboxTmpDir, "file")
+
+	// File does not exist yet. Reading from the sandbox should fail.
+	execArgsTestFile := control.ExecArgs{
+		Filename: "/usr/bin/test",
+		Argv:     []string{"test", "-f", sandboxFilename},
+	}
+	if ws, err := c.Execute(&execArgsTestFile); err != nil {
+		t.Fatalf("unexpected error testing file %q: %v", sandboxFilename, err)
+	} else if ws.ExitStatus() == 0 {
+		t.Errorf("test %q exited with code %v, wanted not zero", ws.ExitStatus(), err)
+	}
+
+	// Create the file from outside of the sandbox.
+	if err := ioutil.WriteFile(hostFilename, []byte("foobar"), 0777); err != nil {
+		t.Fatalf("error writing to file %q: %v", hostFilename, err)
+	}
+
+	// Now we should be able to test the file from within the sandbox.
+	if ws, err := c.Execute(&execArgsTestFile); err != nil {
+		t.Fatalf("unexpected error testing file %q: %v", sandboxFilename, err)
+	} else if ws.ExitStatus() != 0 {
+		t.Errorf("test %q exited with code %v, wanted zero", sandboxFilename, ws.ExitStatus())
+	}
+
+	// Rename the file from outside of the sandbox.
+	newHostFilename := filepath.Join(hostTmpDir, "newfile")
+	newSandboxFilename := filepath.Join(sandboxTmpDir, "newfile")
+	if err := os.Rename(hostFilename, newHostFilename); err != nil {
+		t.Fatalf("os.Rename(%q, %q) failed: %v", hostFilename, newHostFilename, err)
+	}
+
+	// File should no longer exist at the old path within the sandbox.
+	if ws, err := c.Execute(&execArgsTestFile); err != nil {
+		t.Fatalf("unexpected error testing file %q: %v", sandboxFilename, err)
+	} else if ws.ExitStatus() == 0 {
+		t.Errorf("test %q exited with code %v, wanted not zero", sandboxFilename, ws.ExitStatus())
+	}
+
+	// We should be able to test the new filename from within the sandbox.
+	execArgsTestNewFile := control.ExecArgs{
+		Filename: "/usr/bin/test",
+		Argv:     []string{"test", "-f", newSandboxFilename},
+	}
+	if ws, err := c.Execute(&execArgsTestNewFile); err != nil {
+		t.Fatalf("unexpected error testing file %q: %v", newSandboxFilename, err)
+	} else if ws.ExitStatus() != 0 {
+		t.Errorf("test %q exited with code %v, wanted zero", newSandboxFilename, ws.ExitStatus())
+	}
+
+	// Delete the renamed file from outside of the sandbox.
+	if err := os.Remove(newHostFilename); err != nil {
+		t.Fatalf("error removing file %q: %v", hostFilename, err)
+	}
+
+	// Renamed file should no longer exist at the old path within the sandbox.
+	if ws, err := c.Execute(&execArgsTestNewFile); err != nil {
+		t.Fatalf("unexpected error testing file %q: %v", newSandboxFilename, err)
+	} else if ws.ExitStatus() == 0 {
+		t.Errorf("test %q exited with code %v, wanted not zero", newSandboxFilename, ws.ExitStatus())
+	}
+
+	// Now create the file from WITHIN the sandbox.
+	execArgsTouch := control.ExecArgs{
+		Filename: "/usr/bin/touch",
+		Argv:     []string{"touch", sandboxFilename},
+		KUID:     auth.KUID(os.Getuid()),
+		KGID:     auth.KGID(os.Getgid()),
+	}
+	if ws, err := c.Execute(&execArgsTouch); err != nil {
+		t.Fatalf("unexpected error touching file %q: %v", sandboxFilename, err)
+	} else if ws.ExitStatus() != 0 {
+		t.Errorf("touch %q exited with code %v, wanted zero", sandboxFilename, ws.ExitStatus())
+	}
+
+	// File should exist outside the sandbox.
+	if _, err := os.Stat(hostFilename); err != nil {
+		t.Errorf("stat %q got error %v, wanted nil", hostFilename, err)
+	}
+
+	// File should exist outside the sandbox.
+	if _, err := os.Stat(hostFilename); err != nil {
+		t.Errorf("stat %q got error %v, wanted nil", hostFilename, err)
+	}
+
+	// Delete the file from within the sandbox.
+	execArgsRemove := control.ExecArgs{
+		Filename: "/bin/rm",
+		Argv:     []string{"rm", sandboxFilename},
+	}
+	if ws, err := c.Execute(&execArgsRemove); err != nil {
+		t.Fatalf("unexpected error removing file %q: %v", sandboxFilename, err)
+	} else if ws.ExitStatus() != 0 {
+		t.Errorf("remove %q exited with code %v, wanted zero", sandboxFilename, ws.ExitStatus())
+	}
+
+	// File should not exist outside the sandbox.
+	if _, err := os.Stat(hostFilename); !os.IsNotExist(err) {
+		t.Errorf("stat %q got error %v, wanted ErrNotExist", hostFilename, err)
+	}
+}
diff --git a/runsc/main.go b/runsc/main.go
index 10ae44b5e..b36100cca 100644
--- a/runsc/main.go
+++ b/runsc/main.go
@@ -57,7 +57,7 @@ var (
 	// Flags that control sandbox runtime behavior.
 	platform       = flag.String("platform", "ptrace", "specifies which platform to use: ptrace (default), kvm")
 	network        = flag.String("network", "sandbox", "specifies which network to use: sandbox (default), host, none. Using network inside the sandbox is more secure because it's isolated from the host network.")
-	fileAccess     = flag.String("file-access", "proxy", "specifies which filesystem to use: proxy (default), direct. Using a proxy is more secure because it disallows the sandbox from opennig files directly in the host.")
+	fileAccess     = flag.String("file-access", "proxy-exclusive", "specifies which filesystem to use: proxy-exclusive (default), proxy-shared, or direct. Using a proxy is more secure because it disallows the sandbox from opening files directly in the host. Setting 'proxy-shared' will disable caches and should be used if external modifications to the filesystem are expected.")
 	overlay        = flag.Bool("overlay", false, "wrap filesystem mounts with writable overlay. All modifications are stored in memory inside the sandbox.")
 	multiContainer = flag.Bool("multi-container", false, "enable *experimental* multi-container support.")
 	watchdogAction = flag.String("watchdog-action", "log", "sets what action the watchdog takes when triggered: log (default), panic.")
@@ -119,6 +119,10 @@ func main() {
 		cmd.Fatalf("%v", err)
 	}
 
+	if *fileAccess == "proxy" && *overlay {
+		cmd.Fatalf("overlay flag is incompatible with file-access=proxy")
+	}
+
 	// Create a new Config from the flags.
 	conf := &boot.Config{
 		RootDir:        *rootDir,
diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go
index 196949f11..2b043d412 100644
--- a/runsc/sandbox/sandbox.go
+++ b/runsc/sandbox/sandbox.go
@@ -233,7 +233,7 @@ func (s *Sandbox) connError(err error) error {
 }
 
 func (s *Sandbox) createGoferProcess(spec *specs.Spec, conf *boot.Config, bundleDir, binPath string) ([]*os.File, error) {
-	if conf.FileAccess != boot.FileAccessProxy {
+	if conf.FileAccess == boot.FileAccessDirect {
 		// Don't start a gofer. The sandbox will access host FS directly.
 		return nil, nil
 	}
@@ -369,11 +369,11 @@ func (s *Sandbox) createSandboxProcess(spec *specs.Spec, conf *boot.Config, bund
 		nss = append(nss, specs.LinuxNamespace{Type: specs.PIDNamespace})
 	}
 
-	if conf.FileAccess == boot.FileAccessProxy {
+	if conf.FileAccess == boot.FileAccessDirect {
+		log.Infof("Sandbox will be started in the current mount namespace")
+	} else {
 		log.Infof("Sandbox will be started in new mount namespace")
 		nss = append(nss, specs.LinuxNamespace{Type: specs.MountNamespace})
-	} else {
-		log.Infof("Sandbox will be started in the current mount namespace")
 	}
 
 	// Joins the network namespace if network is enabled. the sandbox talks
diff --git a/runsc/test/testutil/testutil.go b/runsc/test/testutil/testutil.go
index 4e7ab3760..d2b39b58c 100644
--- a/runsc/test/testutil/testutil.go
+++ b/runsc/test/testutil/testutil.go
@@ -81,6 +81,7 @@ func TestConfig() *boot.Config {
 		Network:        boot.NetworkNone,
 		Strace:         true,
 		MultiContainer: true,
+		FileAccess:     boot.FileAccessProxyExclusive,
 	}
 }