Moved restore code out of create and made to be called after create.

Docker expects containers to be created before they are restored.
However, gVisor restoring requires specificactions regarding the kernel
and the file system. These actions were originally in booting the sandbox.

Now setting up the file system is deferred until a call to a call to
runsc start. In the restore case, the kernel is destroyed and a new kernel
is created in the same process, as we need the same process for Docker.

These changes required careful execution of concurrent processes which
required the use of a channel.

Full docker integration still needs the ability to restore into the same
container.

PiperOrigin-RevId: 205161441
Change-Id: Ie1d2304ead7e06855319d5dc310678f701bd099f

13 files changed, 358 insertions, 210 deletions

diff --git a/runsc/boot/controller.go b/runsc/boot/controller.go
index ff75a382e..c6e934e66 100644
--- a/runsc/boot/controller.go
+++ b/runsc/boot/controller.go
@@ -23,9 +23,13 @@ import (
 	"gvisor.googlesource.com/gvisor/pkg/log"
 	"gvisor.googlesource.com/gvisor/pkg/sentry/arch"
 	"gvisor.googlesource.com/gvisor/pkg/sentry/control"
+	"gvisor.googlesource.com/gvisor/pkg/sentry/fs"
 	"gvisor.googlesource.com/gvisor/pkg/sentry/kernel"
 	"gvisor.googlesource.com/gvisor/pkg/sentry/socket/epsocket"
+	"gvisor.googlesource.com/gvisor/pkg/sentry/state"
+	"gvisor.googlesource.com/gvisor/pkg/sentry/time"
 	"gvisor.googlesource.com/gvisor/pkg/sentry/watchdog"
+	"gvisor.googlesource.com/gvisor/pkg/urpc"
 )
 
 const (
@@ -47,9 +51,15 @@ const (
 	// processes running in a container.
 	ContainerProcesses = "containerManager.Processes"
 
+	// ContainerRestore restores a container from a statefile.
+	ContainerRestore = "containerManager.Restore"
+
 	// ContainerResume unpauses the paused container.
 	ContainerResume = "containerManager.Resume"
 
+	// ContainerWaitForLoader blocks until the container's loader has been created.
+	ContainerWaitForLoader = "containerManager.WaitForLoader"
+
 	// ContainerSignal is used to send a signal to a container.
 	ContainerSignal = "containerManager.Signal"
 
@@ -85,7 +95,7 @@ func ControlSocketAddr(id string) string {
 // controller holds the control server, and is used for communication into the
 // sandbox.
 type controller struct {
-	// srv is the contorl server.
+	// srv is the control server.
 	srv *server.Server
 
 	// manager holds the containerManager methods.
@@ -100,10 +110,9 @@ func newController(fd int, k *kernel.Kernel, w *watchdog.Watchdog) (*controller,
 	}
 
 	manager := &containerManager{
-		startChan:       make(chan struct{}),
-		startResultChan: make(chan error),
-		k:               k,
-		watchdog:        w,
+		startChan:         make(chan struct{}),
+		startResultChan:   make(chan error),
+		loaderCreatedChan: make(chan struct{}),
 	}
 	srv.Register(manager)
 
@@ -137,15 +146,13 @@ type containerManager struct {
 	// channel. A nil value indicates success.
 	startResultChan chan error
 
-	// k is the emulated linux kernel on which the sandboxed
-	// containers run.
-	k *kernel.Kernel
-
-	// watchdog is the kernel watchdog.
-	watchdog *watchdog.Watchdog
-
 	// l is the loader that creates containers and sandboxes.
 	l *Loader
+
+	// loaderCreatedChan is used to signal when the loader has been created.
+	// After a loader is created, a notify method is called that writes to
+	// this channel.
+	loaderCreatedChan chan struct{}
 }
 
 // StartRoot will start the root container process.
@@ -160,7 +167,7 @@ func (cm *containerManager) StartRoot(cid *string, _ *struct{}) error {
 // Processes retrieves information about processes running in the sandbox.
 func (cm *containerManager) Processes(_, out *[]*control.Process) error {
 	log.Debugf("containerManager.Processes")
-	return control.Processes(cm.k, out)
+	return control.Processes(cm.l.k, out)
 }
 
 // StartArgs contains arguments to the Start method.
@@ -194,7 +201,7 @@ func (cm *containerManager) Start(args *StartArgs, _ *struct{}) error {
 		return errors.New("start argument missing container ID")
 	}
 
-	tgid, err := cm.l.startContainer(args, cm.k)
+	tgid, err := cm.l.startContainer(args, cm.l.k)
 	if err != nil {
 		return err
 	}
@@ -206,7 +213,7 @@ func (cm *containerManager) Start(args *StartArgs, _ *struct{}) error {
 // Execute runs a command on a created or running sandbox.
 func (cm *containerManager) Execute(e *control.ExecArgs, waitStatus *uint32) error {
 	log.Debugf("containerManager.Execute")
-	proc := control.Proc{Kernel: cm.k}
+	proc := control.Proc{Kernel: cm.l.k}
 	if err := proc.Exec(e, waitStatus); err != nil {
 		return fmt.Errorf("error executing: %+v: %v", e, err)
 	}
@@ -217,21 +224,105 @@ func (cm *containerManager) Execute(e *control.ExecArgs, waitStatus *uint32) err
 func (cm *containerManager) Checkpoint(o *control.SaveOpts, _ *struct{}) error {
 	log.Debugf("containerManager.Checkpoint")
 	state := control.State{
-		Kernel:   cm.k,
-		Watchdog: cm.watchdog,
+		Kernel:   cm.l.k,
+		Watchdog: cm.l.watchdog,
 	}
 	return state.Save(o, nil)
 }
 
 // Pause suspends a container.
 func (cm *containerManager) Pause(_, _ *struct{}) error {
-	cm.k.Pause()
+	cm.l.k.Pause()
 	return nil
 }
 
+// WaitForLoader blocks until the container's loader has been created.
+func (cm *containerManager) WaitForLoader(_, _ *struct{}) error {
+	log.Debugf("containerManager.WaitForLoader")
+	<-cm.loaderCreatedChan
+	return nil
+}
+
+// RestoreOpts contains options related to restoring a container's file system.
+type RestoreOpts struct {
+	// FilePayload contains the state file to be restored.
+	urpc.FilePayload
+
+	// SandboxID contains the ID of the sandbox.
+	SandboxID string
+}
+
+// Restore loads a container from a statefile.
+// The container's current kernel is destroyed, a restore environment is created,
+// and the kernel is recreated with the restore state file. The container then sends the
+// signal to start.
+func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error {
+	log.Debugf("containerManager.Restore")
+	if len(o.FilePayload.Files) != 1 {
+		return fmt.Errorf("exactly one file must be provided")
+	}
+	defer o.FilePayload.Files[0].Close()
+
+	// Destroy the old kernel and create a new kernel.
+	cm.l.k.Pause()
+	cm.l.k.Destroy()
+
+	p, err := createPlatform(cm.l.conf)
+	if err != nil {
+		return fmt.Errorf("error creating platform: %v", err)
+	}
+	k := &kernel.Kernel{
+		Platform: p,
+	}
+	cm.l.k = k
+
+	// Set up the restore environment.
+	fds := &fdDispenser{fds: cm.l.ioFDs}
+	renv, err := createRestoreEnvironment(cm.l.spec, cm.l.conf, fds)
+	if err != nil {
+		return fmt.Errorf("error creating RestoreEnvironment: %v", err)
+	}
+	fs.SetRestoreEnvironment(*renv)
+
+	// Prepare to load from the state file.
+	networkStack := newEmptyNetworkStack(cm.l.conf, k)
+	info, err := o.FilePayload.Files[0].Stat()
+	if err != nil {
+		return err
+	}
+	if info.Size() == 0 {
+		return fmt.Errorf("error file was empty")
+	}
+
+	// Load the state.
+	loadOpts := state.LoadOpts{
+		Source: o.FilePayload.Files[0],
+	}
+	if err := loadOpts.Load(k, p, networkStack); err != nil {
+		return err
+	}
+
+	// Set timekeeper.
+	k.Timekeeper().SetClocks(time.NewCalibratedClocks())
+
+	// Since we have a new kernel we also must make a new watchdog.
+	watchdog := watchdog.New(k, watchdog.DefaultTimeout, cm.l.conf.WatchdogAction)
+
+	// Change the loader fields to reflect the changes made when restoring.
+	cm.l.k = k
+	cm.l.watchdog = watchdog
+	cm.l.rootProcArgs = kernel.CreateProcessArgs{}
+	cm.l.setRootContainerID(o.SandboxID)
+	cm.l.restore = true
+
+	// Tell the root container to start and wait for the result.
+	cm.startChan <- struct{}{}
+	return <-cm.startResultChan
+}
+
 // Resume unpauses a container.
 func (cm *containerManager) Resume(_, _ *struct{}) error {
-	cm.k.Unpause()
+	cm.l.k.Unpause()
 	return nil
 }
 
@@ -272,7 +363,7 @@ func (cm *containerManager) Signal(args *SignalArgs, _ *struct{}) error {
 	// process in theat container. Currently we just signal PID 1 in the
 	// sandbox.
 	si := arch.SignalInfo{Signo: args.Signo}
-	t := cm.k.TaskSet().Root.TaskWithID(1)
+	t := cm.l.k.TaskSet().Root.TaskWithID(1)
 	if t == nil {
 		return fmt.Errorf("cannot signal: no task with id 1")
 	}
diff --git a/runsc/boot/events.go b/runsc/boot/events.go
index 0eb75c14c..832339cf4 100644
--- a/runsc/boot/events.go
+++ b/runsc/boot/events.go
@@ -62,8 +62,8 @@ type Memory struct {
 // Event gets the events from the container.
 func (cm *containerManager) Event(_ *struct{}, out *Event) error {
 	stats := &Stats{}
-	stats.populateMemory(cm.k)
-	stats.populatePIDs(cm.k)
+	stats.populateMemory(cm.l.k)
+	stats.populatePIDs(cm.l.k)
 	*out = Event{Type: "stats", Data: stats}
 	return nil
 }
diff --git a/runsc/boot/fs.go b/runsc/boot/fs.go
index 51c8d620d..e596c739f 100644
--- a/runsc/boot/fs.go
+++ b/runsc/boot/fs.go
@@ -27,6 +27,9 @@ import (
 	_ "gvisor.googlesource.com/gvisor/pkg/sentry/fs/sys"
 	_ "gvisor.googlesource.com/gvisor/pkg/sentry/fs/tmpfs"
 	_ "gvisor.googlesource.com/gvisor/pkg/sentry/fs/tty"
+	"gvisor.googlesource.com/gvisor/pkg/sentry/kernel"
+	"gvisor.googlesource.com/gvisor/pkg/sentry/kernel/auth"
+	"gvisor.googlesource.com/gvisor/pkg/sentry/limits"
 
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"gvisor.googlesource.com/gvisor/pkg/abi/linux"
@@ -563,3 +566,45 @@ func subtargets(root string, mnts []specs.Mount) []string {
 	}
 	return targets
 }
+
+// setFileSystemForProcess is used to set up the file system and amend the procArgs accordingly.
+// procArgs are passed by reference and the FDMap field is modified.
+func setFileSystemForProcess(procArgs *kernel.CreateProcessArgs, spec *specs.Spec, conf *Config, ioFDs []int, console bool, creds *auth.Credentials, ls *limits.LimitSet, k *kernel.Kernel) error {
+	ctx := procArgs.NewContext(k)
+
+	// Create the FD map, which will set stdin, stdout, and stderr.  If
+	// console is true, then ioctl calls will be passed through to the host
+	// fd.
+	fdm, err := createFDMap(ctx, k, ls, console)
+	if err != nil {
+		return fmt.Errorf("error importing fds: %v", err)
+	}
+
+	// CreateProcess takes a reference on FDMap if successful. We
+	// won't need ours either way.
+	procArgs.FDMap = fdm
+
+	// If this is the root container, we also need to setup the root mount
+	// namespace.
+	if k.RootMountNamespace() == nil {
+		// Use root user to configure mounts. The current user might not have
+		// permission to do so.
+		rootProcArgs := kernel.CreateProcessArgs{
+			WorkingDirectory:     "/",
+			Credentials:          auth.NewRootCredentials(creds.UserNamespace),
+			Umask:                0022,
+			MaxSymlinkTraversals: linux.MaxSymlinkTraversals,
+		}
+		rootCtx := rootProcArgs.NewContext(k)
+
+		// Create the virtual filesystem.
+		mns, err := createMountNamespace(ctx, rootCtx, spec, conf, ioFDs)
+		if err != nil {
+			return fmt.Errorf("error creating mounts: %v", err)
+		}
+
+		k.SetRootMountNamespace(mns)
+	}
+
+	return nil
+}
diff --git a/runsc/boot/loader.go b/runsc/boot/loader.go
index 706910d8a..66394cdf8 100644
--- a/runsc/boot/loader.go
+++ b/runsc/boot/loader.go
@@ -19,7 +19,6 @@ import (
 	"errors"
 	"fmt"
 	"math/rand"
-	"os"
 	"runtime"
 	"sync"
 	"sync/atomic"
@@ -29,7 +28,6 @@ import (
 	"gvisor.googlesource.com/gvisor/pkg/abi/linux"
 	"gvisor.googlesource.com/gvisor/pkg/cpuid"
 	"gvisor.googlesource.com/gvisor/pkg/log"
-	"gvisor.googlesource.com/gvisor/pkg/sentry/fs"
 	"gvisor.googlesource.com/gvisor/pkg/sentry/inet"
 	"gvisor.googlesource.com/gvisor/pkg/sentry/kernel"
 	"gvisor.googlesource.com/gvisor/pkg/sentry/kernel/auth"
@@ -38,7 +36,6 @@ import (
 	"gvisor.googlesource.com/gvisor/pkg/sentry/platform/kvm"
 	"gvisor.googlesource.com/gvisor/pkg/sentry/platform/ptrace"
 	"gvisor.googlesource.com/gvisor/pkg/sentry/sighandling"
-	"gvisor.googlesource.com/gvisor/pkg/sentry/state"
 	slinux "gvisor.googlesource.com/gvisor/pkg/sentry/syscalls/linux"
 	"gvisor.googlesource.com/gvisor/pkg/sentry/time"
 	"gvisor.googlesource.com/gvisor/pkg/sentry/watchdog"
@@ -77,6 +74,12 @@ type Loader struct {
 
 	watchdog *watchdog.Watchdog
 
+	// ioFDs are the FDs that attach the sandbox to the gofers.
+	ioFDs []int
+
+	// spec is the base configuration for the root container.
+	spec *specs.Spec
+
 	// stopSignalForwarding disables forwarding of signals to the sandboxed
 	// container. It should be called when a sandbox is destroyed.
 	stopSignalForwarding func()
@@ -111,16 +114,7 @@ func init() {
 
 // New initializes a new kernel loader configured by spec.
 // New also handles setting up a kernel for restoring a container.
-func New(spec *specs.Spec, conf *Config, controllerFD, restoreFD int, ioFDs []int, console bool) (*Loader, error) {
-	var (
-		tk          *kernel.Timekeeper
-		creds       *auth.Credentials
-		vdso        *loader.VDSO
-		utsns       *kernel.UTSNamespace
-		ipcns       *kernel.IPCNamespace
-		restoreFile *os.File
-		procArgs    kernel.CreateProcessArgs
-	)
+func New(spec *specs.Spec, conf *Config, controllerFD int, ioFDs []int, console bool) (*Loader, error) {
 	// Create kernel and platform.
 	p, err := createPlatform(conf)
 	if err != nil {
@@ -130,60 +124,47 @@ func New(spec *specs.Spec, conf *Config, controllerFD, restoreFD int, ioFDs []in
 		Platform: p,
 	}
 
-	if restoreFD == -1 {
-		// Create VDSO.
-		//
-		// Pass k as the platform since it is savable, unlike the actual platform.
-		vdso, err = loader.PrepareVDSO(k)
-		if err != nil {
-			return nil, fmt.Errorf("error creating vdso: %v", err)
-		}
+	// Create VDSO.
+	//
+	// Pass k as the platform since it is savable, unlike the actual platform.
+	vdso, err := loader.PrepareVDSO(k)
+	if err != nil {
+		return nil, fmt.Errorf("error creating vdso: %v", err)
+	}
 
-		// Create timekeeper.
-		tk, err = kernel.NewTimekeeper(k, vdso.ParamPage.FileRange())
-		if err != nil {
-			return nil, fmt.Errorf("error creating timekeeper: %v", err)
-		}
-		tk.SetClocks(time.NewCalibratedClocks())
+	// Create timekeeper.
+	tk, err := kernel.NewTimekeeper(k, vdso.ParamPage.FileRange())
+	if err != nil {
+		return nil, fmt.Errorf("error creating timekeeper: %v", err)
+	}
+	tk.SetClocks(time.NewCalibratedClocks())
 
-		// Create capabilities.
-		caps, err := specutils.Capabilities(spec.Process.Capabilities)
-		if err != nil {
-			return nil, fmt.Errorf("error creating capabilities: %v", err)
-		}
+	// Create capabilities.
+	caps, err := specutils.Capabilities(spec.Process.Capabilities)
+	if err != nil {
+		return nil, fmt.Errorf("error creating capabilities: %v", err)
+	}
 
-		// Convert the spec's additional GIDs to KGIDs.
-		extraKGIDs := make([]auth.KGID, 0, len(spec.Process.User.AdditionalGids))
-		for _, GID := range spec.Process.User.AdditionalGids {
-			extraKGIDs = append(extraKGIDs, auth.KGID(GID))
-		}
+	// Convert the spec's additional GIDs to KGIDs.
+	extraKGIDs := make([]auth.KGID, 0, len(spec.Process.User.AdditionalGids))
+	for _, GID := range spec.Process.User.AdditionalGids {
+		extraKGIDs = append(extraKGIDs, auth.KGID(GID))
+	}
 
-		// Create credentials.
-		creds = auth.NewUserCredentials(
-			auth.KUID(spec.Process.User.UID),
-			auth.KGID(spec.Process.User.GID),
-			extraKGIDs,
-			caps,
-			auth.NewRootUserNamespace())
+	// Create credentials.
+	creds := auth.NewUserCredentials(
+		auth.KUID(spec.Process.User.UID),
+		auth.KGID(spec.Process.User.GID),
+		extraKGIDs,
+		caps,
+		auth.NewRootUserNamespace())
 
-		// Create user namespace.
-		// TODO: Not clear what domain name should be here.  It is
-		// not configurable from runtime spec.
-		utsns = kernel.NewUTSNamespace(spec.Hostname, "", creds.UserNamespace)
+	// Create user namespace.
+	// TODO: Not clear what domain name should be here.  It is
+	// not configurable from runtime spec.
+	utsns := kernel.NewUTSNamespace(spec.Hostname, "", creds.UserNamespace)
 
-		ipcns = kernel.NewIPCNamespace(creds.UserNamespace)
-	} else {
-		// Create and set RestoreEnvironment
-		fds := &fdDispenser{fds: ioFDs}
-		renv, err := createRestoreEnvironment(spec, conf, fds)
-		if err != nil {
-			return nil, fmt.Errorf("error creating RestoreEnvironment: %v", err)
-		}
-		fs.SetRestoreEnvironment(*renv)
-
-		restoreFile = os.NewFile(uintptr(restoreFD), "restore_file")
-		defer restoreFile.Close()
-	}
+	ipcns := kernel.NewIPCNamespace(creds.UserNamespace)
 
 	if err := enableStrace(conf); err != nil {
 		return nil, fmt.Errorf("failed to enable strace: %v", err)
@@ -195,33 +176,20 @@ func New(spec *specs.Spec, conf *Config, controllerFD, restoreFD int, ioFDs []in
 	// Run().
 	networkStack := newEmptyNetworkStack(conf, k)
 
-	if restoreFile == nil {
-		// Initiate the Kernel object, which is required by the Context passed
-		// to createVFS in order to mount (among other things) procfs.
-		if err = k.Init(kernel.InitKernelArgs{
-			FeatureSet:        cpuid.HostFeatureSet(),
-			Timekeeper:        tk,
-			RootUserNamespace: creds.UserNamespace,
-			NetworkStack:      networkStack,
-			// TODO: use number of logical processors from cgroups.
-			ApplicationCores: uint(runtime.NumCPU()),
-			Vdso:             vdso,
-			RootUTSNamespace: utsns,
-			RootIPCNamespace: ipcns,
-		}); err != nil {
-			return nil, fmt.Errorf("error initializing kernel: %v", err)
-		}
-	} else {
-		// Load the state.
-		loadOpts := state.LoadOpts{
-			Source: restoreFile,
-		}
-		if err := loadOpts.Load(k, p, networkStack); err != nil {
-			return nil, err
-		}
-
-		// Set timekeeper.
-		k.Timekeeper().SetClocks(time.NewCalibratedClocks())
+	// Initiate the Kernel object, which is required by the Context passed
+	// to createVFS in order to mount (among other things) procfs.
+	if err = k.Init(kernel.InitKernelArgs{
+		FeatureSet:        cpuid.HostFeatureSet(),
+		Timekeeper:        tk,
+		RootUserNamespace: creds.UserNamespace,
+		NetworkStack:      networkStack,
+		// TODO: use number of logical processors from cgroups.
+		ApplicationCores: uint(runtime.NumCPU()),
+		Vdso:             vdso,
+		RootUTSNamespace: utsns,
+		RootIPCNamespace: ipcns,
+	}); err != nil {
+		return nil, fmt.Errorf("error initializing kernel: %v", err)
 	}
 
 	// Turn on packet logging if enabled.
@@ -258,11 +226,9 @@ func New(spec *specs.Spec, conf *Config, controllerFD, restoreFD int, ioFDs []in
 	// Ensure that signals received are forwarded to the emulated kernel.
 	stopSignalForwarding := sighandling.PrepareForwarding(k, false)()
 
-	if restoreFile == nil {
-		procArgs, err = newProcess(spec, conf, ioFDs, console, creds, utsns, ipcns, k)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create root process: %v", err)
-		}
+	procArgs, err := newProcess(spec, conf, ioFDs, console, creds, utsns, ipcns, k)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create root process: %v", err)
 	}
 
 	l := &Loader{
@@ -271,9 +237,10 @@ func New(spec *specs.Spec, conf *Config, controllerFD, restoreFD int, ioFDs []in
 		conf:                 conf,
 		console:              console,
 		watchdog:             watchdog,
+		ioFDs:                ioFDs,
+		spec:                 spec,
 		stopSignalForwarding: stopSignalForwarding,
 		rootProcArgs:         procArgs,
-		restore:              restoreFile != nil,
 	}
 	ctrl.manager.l = l
 	return l, nil
@@ -307,41 +274,6 @@ func newProcess(spec *specs.Spec, conf *Config, ioFDs []int, console bool, creds
 		UTSNamespace:         utsns,
 		IPCNamespace:         ipcns,
 	}
-	ctx := procArgs.NewContext(k)
-
-	// Create the FD map, which will set stdin, stdout, and stderr.  If
-	// console is true, then ioctl calls will be passed through to the host
-	// fd.
-	fdm, err := createFDMap(ctx, k, ls, console)
-	if err != nil {
-		return kernel.CreateProcessArgs{}, fmt.Errorf("error importing fds: %v", err)
-	}
-
-	// CreateProcess takes a reference on FDMap if successful. We
-	// won't need ours either way.
-	procArgs.FDMap = fdm
-
-	// If this is the root container, we also need to setup the root mount
-	// namespace.
-	if k.RootMountNamespace() == nil {
-		// Use root user to configure mounts. The current user might not have
-		// permission to do so.
-		rootProcArgs := kernel.CreateProcessArgs{
-			WorkingDirectory:     "/",
-			Credentials:          auth.NewRootCredentials(creds.UserNamespace),
-			Umask:                0022,
-			MaxSymlinkTraversals: linux.MaxSymlinkTraversals,
-		}
-		rootCtx := rootProcArgs.NewContext(k)
-
-		// Create the virtual filesystem.
-		mns, err := createMountNamespace(ctx, rootCtx, spec, conf, ioFDs)
-		if err != nil {
-			return kernel.CreateProcessArgs{}, fmt.Errorf("error creating mounts: %v", err)
-		}
-
-		k.SetRootMountNamespace(mns)
-	}
 
 	return procArgs, nil
 }
@@ -411,7 +343,20 @@ func (l *Loader) run() error {
 	}
 
 	// If we are restoring, we do not want to create a process.
+	// l.restore is set by the container manager when a restore call is made.
 	if !l.restore {
+		err := setFileSystemForProcess(
+			&l.rootProcArgs,
+			l.spec,
+			l.conf,
+			l.ioFDs,
+			l.console,
+			l.rootProcArgs.Credentials,
+			l.rootProcArgs.Limits,
+			l.k)
+		if err != nil {
+			return err
+		}
 		// Create the root container init task.
 		if _, err := l.k.CreateProcess(l.rootProcArgs); err != nil {
 			return fmt.Errorf("failed to create init process: %v", err)
@@ -421,6 +366,7 @@ func (l *Loader) run() error {
 		l.rootProcArgs.FDMap.DecRef()
 	}
 
+	log.Infof("Process should have started...")
 	l.watchdog.Start()
 	return l.k.Start()
 }
@@ -468,6 +414,18 @@ func (l *Loader) startContainer(args *StartArgs, k *kernel.Kernel) (kernel.Threa
 	if err != nil {
 		return 0, fmt.Errorf("failed to create new process: %v", err)
 	}
+	err = setFileSystemForProcess(
+		&procArgs,
+		args.Spec,
+		args.Conf,
+		nil,
+		false,
+		creds,
+		procArgs.Limits,
+		k)
+	if err != nil {
+		return 0, fmt.Errorf("failed to create new process: %v", err)
+	}
 
 	tg, err := l.k.CreateProcess(procArgs)
 	if err != nil {
@@ -553,6 +511,12 @@ func (l *Loader) WaitForStartSignal() {
 	<-l.ctrl.manager.startChan
 }
 
+// NotifyLoaderCreated sends a signal to the container manager that this
+// loader has been created.
+func (l *Loader) NotifyLoaderCreated() {
+	l.ctrl.manager.loaderCreatedChan <- struct{}{}
+}
+
 // WaitExit waits for the root container to exit, and returns its exit status.
 func (l *Loader) WaitExit() kernel.ExitStatus {
 	// Wait for container.
diff --git a/runsc/boot/loader_test.go b/runsc/boot/loader_test.go
index 30ec236e4..7ea2e1ee5 100644
--- a/runsc/boot/loader_test.go
+++ b/runsc/boot/loader_test.go
@@ -61,7 +61,8 @@ func createLoader() (*Loader, error) {
 		FileAccess:     FileAccessDirect,
 		DisableSeccomp: true,
 	}
-	return New(testSpec(), conf, fd, -1, nil, false)
+	spec := testSpec()
+	return New(spec, conf, fd, nil, false)
 }
 
 // TestRun runs a simple application in a sandbox and checks that it succeeds.
diff --git a/runsc/cmd/boot.go b/runsc/cmd/boot.go
index 70c4616b4..4e08dafc8 100644
--- a/runsc/cmd/boot.go
+++ b/runsc/cmd/boot.go
@@ -49,9 +49,6 @@ type Boot struct {
 	// applyCaps determines if capabilities defined in the spec should be applied
 	// to the process.
 	applyCaps bool
-
-	// restoreFD is the file descriptor to the state file to be restored.
-	restoreFD int
 }
 
 // Name implements subcommands.Command.Name.
@@ -76,7 +73,6 @@ func (b *Boot) SetFlags(f *flag.FlagSet) {
 	f.Var(&b.ioFDs, "io-fds", "list of FDs to connect 9P clients. They must follow this order: root first, then mounts as defined in the spec")
 	f.BoolVar(&b.console, "console", false, "set to true if the sandbox should allow terminal ioctl(2) syscalls")
 	f.BoolVar(&b.applyCaps, "apply-caps", false, "if true, apply capabilities defined in the spec to the process")
-	f.IntVar(&b.restoreFD, "restore-fd", -1, "FD of the state file to be restored")
 }
 
 // Execute implements subcommands.Command.Execute.  It starts a sandbox in a
@@ -142,11 +138,14 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...interface{})
 	}
 
 	// Create the loader.
-	l, err := boot.New(spec, conf, b.controllerFD, b.restoreFD, b.ioFDs.GetArray(), b.console)
+	l, err := boot.New(spec, conf, b.controllerFD, b.ioFDs.GetArray(), b.console)
 	if err != nil {
 		Fatalf("error creating loader: %v", err)
 	}
 
+	// Notify other processes the loader has been created.
+	l.NotifyLoaderCreated()
+
 	// Wait for the start signal from runsc.
 	l.WaitForStartSignal()
 
diff --git a/runsc/cmd/checkpoint.go b/runsc/cmd/checkpoint.go
index 94efc3517..05014ba3d 100644
--- a/runsc/cmd/checkpoint.go
+++ b/runsc/cmd/checkpoint.go
@@ -133,12 +133,12 @@ func (c *Checkpoint) Execute(_ context.Context, f *flag.FlagSet, args ...interfa
 		Fatalf("error destroying container: %v", err)
 	}
 
-	cont, err = container.Create(id, spec, conf, bundleDir, "", "", fullImagePath)
+	cont, err = container.Create(id, spec, conf, bundleDir, "", "")
 	if err != nil {
 		Fatalf("error restoring container: %v", err)
 	}
 
-	if err := cont.Start(conf); err != nil {
+	if err := cont.Restore(spec, conf, fullImagePath); err != nil {
 		Fatalf("error starting container: %v", err)
 	}
 
diff --git a/runsc/cmd/create.go b/runsc/cmd/create.go
index 5a887c73c..94a889077 100644
--- a/runsc/cmd/create.go
+++ b/runsc/cmd/create.go
@@ -87,7 +87,7 @@ func (c *Create) Execute(_ context.Context, f *flag.FlagSet, args ...interface{}
 	// Create the container. A new sandbox will be created for the
 	// container unless the metadata specifies that it should be run in an
 	// existing container.
-	if _, err := container.Create(id, spec, conf, bundleDir, c.consoleSocket, c.pidFile, ""); err != nil {
+	if _, err := container.Create(id, spec, conf, bundleDir, c.consoleSocket, c.pidFile); err != nil {
 		Fatalf("error creating container: %v", err)
 	}
 	return subcommands.ExitSuccess
diff --git a/runsc/cmd/restore.go b/runsc/cmd/restore.go
index 69cdb35c1..6dc044672 100644
--- a/runsc/cmd/restore.go
+++ b/runsc/cmd/restore.go
@@ -94,16 +94,15 @@ func (r *Restore) Execute(_ context.Context, f *flag.FlagSet, args ...interface{
 
 	restoreFile := filepath.Join(r.imagePath, checkpointFileName)
 
-	cont, err := container.Create(id, spec, conf, bundleDir, r.consoleSocket, r.pidFile, restoreFile)
+	c, err := container.Load(conf.RootDir, id)
 	if err != nil {
-		Fatalf("error restoring container: %v", err)
+		Fatalf("error loading container: %v", err)
 	}
-
-	if err := cont.Start(conf); err != nil {
-		Fatalf("error starting container: %v", err)
+	if err := c.Restore(spec, conf, restoreFile); err != nil {
+		Fatalf("error restoring container: %v", err)
 	}
 
-	ws, err := cont.Wait()
+	ws, err := c.Wait()
 	if err != nil {
 		Fatalf("error running container: %v", err)
 	}
diff --git a/runsc/container/container.go b/runsc/container/container.go
index c4e5bf9f6..574075b00 100644
--- a/runsc/container/container.go
+++ b/runsc/container/container.go
@@ -190,7 +190,7 @@ func List(rootDir string) ([]string, error) {
 
 // Create creates the container in a new Sandbox process, unless the metadata
 // indicates that an existing Sandbox should be used.
-func Create(id string, spec *specs.Spec, conf *boot.Config, bundleDir, consoleSocket, pidFile string, restoreFile string) (*Container, error) {
+func Create(id string, spec *specs.Spec, conf *boot.Config, bundleDir, consoleSocket, pidFile string) (*Container, error) {
 	log.Debugf("Create container %q in root dir: %s", id, conf.RootDir)
 	if err := validateID(id); err != nil {
 		return nil, err
@@ -221,7 +221,7 @@ func Create(id string, spec *specs.Spec, conf *boot.Config, bundleDir, consoleSo
 		log.Debugf("Creating new sandbox for container %q", id)
 		// Start a new sandbox for this container. Any errors after this point
 		// must destroy the container.
-		s, err := sandbox.Create(id, spec, conf, bundleDir, consoleSocket, restoreFile)
+		s, err := sandbox.Create(id, spec, conf, bundleDir, consoleSocket)
 		if err != nil {
 			c.Destroy()
 			return nil, err
@@ -309,10 +309,26 @@ func (c *Container) Start(conf *boot.Config) error {
 	return c.save()
 }
 
+// Restore takes a container and replaces its kernel and file system
+// to restore a container from its state file.
+func (c *Container) Restore(spec *specs.Spec, conf *boot.Config, restoreFile string) error {
+	log.Debugf("Restore container %q", c.ID)
+
+	if c.Status != Created {
+		return fmt.Errorf("cannot restore container in state %s", c.Status)
+	}
+
+	if err := c.Sandbox.Restore(c.ID, spec, conf, restoreFile); err != nil {
+		return err
+	}
+	c.Status = Running
+	return c.save()
+}
+
 // Run is a helper that calls Create + Start + Wait.
 func Run(id string, spec *specs.Spec, conf *boot.Config, bundleDir, consoleSocket, pidFile string) (syscall.WaitStatus, error) {
 	log.Debugf("Run container %q in root dir: %s", id, conf.RootDir)
-	c, err := Create(id, spec, conf, bundleDir, consoleSocket, pidFile, "")
+	c, err := Create(id, spec, conf, bundleDir, consoleSocket, pidFile)
 	if err != nil {
 		return 0, fmt.Errorf("error creating container: %v", err)
 	}
diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go
index fc441e353..62a681ac2 100644
--- a/runsc/container/container_test.go
+++ b/runsc/container/container_test.go
@@ -168,7 +168,7 @@ func run(spec *specs.Spec) error {
 	defer os.RemoveAll(bundleDir)
 
 	// Create, start and wait for the container.
-	s, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "", "")
+	s, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "")
 	if err != nil {
 		return fmt.Errorf("error creating container: %v", err)
 	}
@@ -213,7 +213,7 @@ func TestLifecycle(t *testing.T) {
 	}
 	// Create the container.
 	id := testutil.UniqueContainerID()
-	if _, err := container.Create(id, spec, conf, bundleDir, "", "", ""); err != nil {
+	if _, err := container.Create(id, spec, conf, bundleDir, "", ""); err != nil {
 		t.Fatalf("error creating container: %v", err)
 	}
 
@@ -411,7 +411,7 @@ func TestExec(t *testing.T) {
 	defer os.RemoveAll(bundleDir)
 
 	// Create and start the container.
-	s, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "", "")
+	s, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "")
 	if err != nil {
 		t.Fatalf("error creating container: %v", err)
 	}
@@ -514,7 +514,7 @@ func TestCheckpointRestore(t *testing.T) {
 	defer os.RemoveAll(bundleDir)
 
 	// Create and start the container.
-	cont, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "", "")
+	cont, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "")
 	if err != nil {
 		t.Fatalf("error creating container: %v", err)
 	}
@@ -557,13 +557,14 @@ func TestCheckpointRestore(t *testing.T) {
 	defer outputFile2.Close()
 
 	// Restore into a new container.
-	cont2, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "", imagePath)
+	cont2, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "")
 	if err != nil {
 		t.Fatalf("error creating container: %v", err)
 	}
 	defer cont2.Destroy()
-	if err := cont2.Start(conf); err != nil {
-		t.Fatalf("error starting container: %v", err)
+
+	if err := cont2.Restore(spec, conf, imagePath); err != nil {
+		t.Fatalf("error restoring container: %v", err)
 	}
 
 	firstNum, err := readOutputNum(outputFile2, true)
@@ -588,13 +589,14 @@ func TestCheckpointRestore(t *testing.T) {
 	defer outputFile3.Close()
 
 	// Restore into a new container.
-	cont3, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "", imagePath)
+	cont3, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "")
 	if err != nil {
 		t.Fatalf("error creating container: %v", err)
 	}
 	defer cont3.Destroy()
-	if err := cont3.Start(conf); err != nil {
-		t.Fatalf("error starting container: %v", err)
+
+	if err := cont3.Restore(spec, conf, imagePath); err != nil {
+		t.Fatalf("error restoring container: %v", err)
 	}
 
 	firstNum2, err := readOutputNum(outputFile3, true)
@@ -604,7 +606,7 @@ func TestCheckpointRestore(t *testing.T) {
 
 	// Check that lastNum is one less than firstNum and that the container picks up from where it left off.
 	if lastNum+1 != firstNum2 {
-		t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum)
+		t.Errorf("error numbers not in order, previous: %d, next: %d", lastNum, firstNum2)
 	}
 
 }
@@ -626,7 +628,7 @@ func TestPauseResume(t *testing.T) {
 	defer os.RemoveAll(bundleDir)
 
 	// Create and start the container.
-	cont, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "", "")
+	cont, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "")
 	if err != nil {
 		t.Fatalf("error creating container: %v", err)
 	}
@@ -728,7 +730,7 @@ func TestPauseResumeStatus(t *testing.T) {
 	defer os.RemoveAll(bundleDir)
 
 	// Create and start the container.
-	cont, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "", "")
+	cont, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "")
 	if err != nil {
 		t.Fatalf("error creating container: %v", err)
 	}
@@ -795,7 +797,7 @@ func TestCapabilities(t *testing.T) {
 	defer os.RemoveAll(bundleDir)
 
 	// Create and start the container.
-	s, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "", "")
+	s, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "")
 	if err != nil {
 		t.Fatalf("error creating container: %v", err)
 	}
@@ -894,7 +896,7 @@ func TestConsoleSocket(t *testing.T) {
 
 	// Create the container and pass the socket name.
 	id := testutil.UniqueContainerID()
-	s, err := container.Create(id, spec, conf, bundleDir, socketRelPath, "", "")
+	s, err := container.Create(id, spec, conf, bundleDir, socketRelPath, "")
 	if err != nil {
 		t.Fatalf("error creating container: %v", err)
 	}
@@ -1014,7 +1016,7 @@ func TestReadonlyRoot(t *testing.T) {
 	conf.Overlay = true
 
 	// Create, start and wait for the container.
-	s, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "", "")
+	s, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "")
 	if err != nil {
 		t.Fatalf("error creating container: %v", err)
 	}
@@ -1055,7 +1057,7 @@ func TestReadonlyMount(t *testing.T) {
 	conf.Overlay = true
 
 	// Create, start and wait for the container.
-	s, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "", "")
+	s, err := container.Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "")
 	if err != nil {
 		t.Fatalf("error creating container: %v", err)
 	}
@@ -1095,7 +1097,7 @@ func TestAbbreviatedIDs(t *testing.T) {
 		defer os.RemoveAll(bundleDir)
 
 		// Create and start the container.
-		cont, err := container.Create(cid, spec, conf, bundleDir, "", "", "")
+		cont, err := container.Create(cid, spec, conf, bundleDir, "", "")
 		if err != nil {
 			t.Fatalf("error creating container: %v", err)
 		}
@@ -1165,7 +1167,7 @@ func TestMultiContainerSanity(t *testing.T) {
 			t.Fatalf("error setting up container: %v", err)
 		}
 		defer os.RemoveAll(bundleDir)
-		cont, err := container.Create(containerIDs[i], spec, conf, bundleDir, "", "", "")
+		cont, err := container.Create(containerIDs[i], spec, conf, bundleDir, "", "")
 		if err != nil {
 			t.Fatalf("error creating container: %v", err)
 		}
@@ -1241,7 +1243,7 @@ func TestMultiContainerWait(t *testing.T) {
 			t.Fatalf("error setting up container: %v", err)
 		}
 		defer os.RemoveAll(bundleDir)
-		cont, err := container.Create(containerIDs[i], spec, conf, bundleDir, "", "", "")
+		cont, err := container.Create(containerIDs[i], spec, conf, bundleDir, "", "")
 		if err != nil {
 			t.Fatalf("error creating container: %v", err)
 		}
diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go
index 9200fbee9..1f2cd6018 100644
--- a/runsc/sandbox/sandbox.go
+++ b/runsc/sandbox/sandbox.go
@@ -56,7 +56,7 @@ type Sandbox struct {
 // Create creates the sandbox process.
 //
 // If restoreFile is not empty, the sandbox will be restored from file.
-func Create(id string, spec *specs.Spec, conf *boot.Config, bundleDir, consoleSocket string, restoreFile string) (*Sandbox, error) {
+func Create(id string, spec *specs.Spec, conf *boot.Config, bundleDir, consoleSocket string) (*Sandbox, error) {
 	s := &Sandbox{ID: id}
 
 	binPath, err := specutils.BinPath()
@@ -71,7 +71,7 @@ func Create(id string, spec *specs.Spec, conf *boot.Config, bundleDir, consoleSo
 	}
 
 	// Create the sandbox process.
-	if err := s.createSandboxProcess(spec, conf, bundleDir, consoleSocket, binPath, ioFiles, restoreFile); err != nil {
+	if err := s.createSandboxProcess(spec, conf, bundleDir, consoleSocket, binPath, ioFiles); err != nil {
 		return nil, err
 	}
 
@@ -127,6 +127,42 @@ func (s *Sandbox) Start(spec *specs.Spec, conf *boot.Config, cid string) error {
 	return nil
 }
 
+// Restore sends the restore call for a container in the sandbox.
+func (s *Sandbox) Restore(cid string, spec *specs.Spec, conf *boot.Config, f string) error {
+	log.Debugf("Restore sandbox %q", s.ID)
+
+	rf, err := os.Open(f)
+	if err != nil {
+		return fmt.Errorf("os.Open(%q) failed: %v", f, err)
+	}
+	defer rf.Close()
+
+	opt := boot.RestoreOpts{
+		FilePayload: urpc.FilePayload{
+			Files: []*os.File{rf},
+		},
+		SandboxID: s.ID,
+	}
+
+	conn, err := s.connect()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	// Configure the network.
+	if err := setupNetwork(conn, s.Pid, spec, conf); err != nil {
+		return fmt.Errorf("error setting up network: %v", err)
+	}
+
+	// Restore the container and start the root container.
+	if err := conn.Call(boot.ContainerRestore, &opt, nil); err != nil {
+		return fmt.Errorf("error restoring container %q: %v", cid, err)
+	}
+
+	return nil
+}
+
 // Processes retrieves the list of processes and associated metadata for a
 // given container in this sandbox.
 func (s *Sandbox) Processes(cid string) ([]*control.Process, error) {
@@ -254,7 +290,7 @@ func (s *Sandbox) createGoferProcess(spec *specs.Spec, conf *boot.Config, bundle
 
 // createSandboxProcess starts the sandbox as a subprocess by running the "boot"
 // command, passing in the bundle dir.
-func (s *Sandbox) createSandboxProcess(spec *specs.Spec, conf *boot.Config, bundleDir, consoleSocket, binPath string, ioFiles []*os.File, restoreFile string) error {
+func (s *Sandbox) createSandboxProcess(spec *specs.Spec, conf *boot.Config, bundleDir, consoleSocket, binPath string, ioFiles []*os.File) error {
 	// nextFD is used to get unused FDs that we can pass to the sandbox.  It
 	// starts at 3 because 0, 1, and 2 are taken by stdin/out/err.
 	nextFD := 3
@@ -276,27 +312,12 @@ func (s *Sandbox) createSandboxProcess(spec *specs.Spec, conf *boot.Config, bund
 		"--bundle", bundleDir,
 		"--controller-fd="+strconv.Itoa(nextFD),
 		fmt.Sprintf("--console=%t", consoleEnabled))
+	nextFD++
 
 	controllerFile := os.NewFile(uintptr(fd), "control_server_socket")
 	defer controllerFile.Close()
 	cmd.ExtraFiles = append(cmd.ExtraFiles, controllerFile)
 
-	// If a restore filename was given, open the file and append its FD to Args
-	// and the file to ExtraFiles.
-	if restoreFile != "" {
-		// Create the image file and open for reading.
-		rF, err := os.Open(restoreFile)
-		if err != nil {
-			return fmt.Errorf("os.Open(%q) failed: %v", restoreFile, err)
-		}
-		defer rF.Close()
-
-		nextFD++
-		cmd.Args = append(cmd.Args, "--restore-fd="+strconv.Itoa(nextFD))
-		cmd.ExtraFiles = append(cmd.ExtraFiles, rF)
-	}
-	nextFD++
-
 	// If there is a gofer, sends all socket ends to the sandbox.
 	for _, f := range ioFiles {
 		defer f.Close()
@@ -402,7 +423,8 @@ func (s *Sandbox) createSandboxProcess(spec *specs.Spec, conf *boot.Config, bund
 }
 
 // waitForCreated waits for the sandbox subprocess control server to be
-// running, at which point the sandbox is in Created state.
+// running and for the loader to have been created, at which point the sandbox
+// is in Created state.
 func (s *Sandbox) waitForCreated(timeout time.Duration) error {
 	log.Debugf("Waiting for sandbox %q creation", s.ID)
 
@@ -418,6 +440,15 @@ func (s *Sandbox) waitForCreated(timeout time.Duration) error {
 	if err := specutils.WaitForReady(s.Pid, timeout, ready); err != nil {
 		return fmt.Errorf("unexpected error waiting for sandbox %q, err: %v", s.ID, err)
 	}
+	conn, err := s.connect()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	if err := conn.Call(boot.ContainerWaitForLoader, nil, nil); err != nil {
+		return fmt.Errorf("err waiting on loader on sandbox %q, err: %v", s.ID, err)
+	}
 	return nil
 }
 
diff --git a/runsc/sandbox/sandbox_test.go b/runsc/sandbox/sandbox_test.go
index 9db90ef07..fee2de283 100644
--- a/runsc/sandbox/sandbox_test.go
+++ b/runsc/sandbox/sandbox_test.go
@@ -39,7 +39,7 @@ func TestGoferExits(t *testing.T) {
 	defer os.RemoveAll(bundleDir)
 
 	// Create, start and wait for the container.
-	s, err := Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "", "")
+	s, err := Create(testutil.UniqueContainerID(), spec, conf, bundleDir, "")
 	if err != nil {
 		t.Fatalf("error creating container: %v", err)
 	}