diff options
author | Nicolas Lacasse <nlacasse@google.com> | 2018-05-15 10:17:19 -0700 |
---|---|---|
committer | Shentubot <shentubot@google.com> | 2018-05-15 10:18:03 -0700 |
commit | 205f1027e6beb84101439172b3c776c2671b5be8 (patch) | |
tree | 10294e667ee529e140c474c475e7309cb72ea1d8 /runsc/sandbox/sandbox.go | |
parent | ed02ac4f668ec41063cd51cbbd451baba9e9a6e7 (diff) |
Refactor the Sandbox package into Sandbox + Container.
This is a necessary prerequisite for supporting multiple containers in a single
sandbox.
All the commands (in cmd package) now call operations on Containers (container
package). When a Container first starts, it will create a Sandbox with the same
ID.
The Sandbox class is now simpler, as it only knows how to create boot/gofer
processes, and how to forward commands into the running boot process.
There are TODOs sprinkled around for additional support for multiple
containers. Most notably, we need to detect when a container is intended to run
in an existing sandbox (by reading the metadata), and then have some way to
signal to the sandbox to start a new container. Other urpc calls into the
sandbox need to pass the container ID, so the sandbox can run the operation on
the given container. These are only half-plummed through right now.
PiperOrigin-RevId: 196688269
Change-Id: I1ecf4abbb9dd8987a53ae509df19341aaf42b5b0
Diffstat (limited to 'runsc/sandbox/sandbox.go')
-rw-r--r-- | runsc/sandbox/sandbox.go | 430 |
1 files changed, 94 insertions, 336 deletions
diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 34bd6ea67..5dfa4cf0b 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -16,13 +16,9 @@ package sandbox import ( - "encoding/json" "fmt" - "io/ioutil" "os" "os/exec" - "path/filepath" - "regexp" "strconv" "syscall" "time" @@ -38,308 +34,110 @@ import ( "gvisor.googlesource.com/gvisor/runsc/specutils" ) -// metadataFilename is the name of the metadata file relative to sandboxRoot -// that holds sandbox metadata. -const metadataFilename = "meta.json" - -// See libcontainer/factory_linux.go -var idRegex = regexp.MustCompile(`^[\w+-\.]+$`) - -// validateID validates the sandbox id. -func validateID(id string) error { - if !idRegex.MatchString(id) { - return fmt.Errorf("invalid sandbox id: %v", id) - } - return nil -} - -func validateSpec(spec *specs.Spec) error { - if spec.Process.SelinuxLabel != "" { - return fmt.Errorf("SELinux is not supported: %s", spec.Process.SelinuxLabel) - } - - // Docker uses AppArmor by default, so just log that it's being ignored. - if spec.Process.ApparmorProfile != "" { - log.Warningf("AppArmor profile %q is being ignored", spec.Process.ApparmorProfile) - } - // TODO: Apply seccomp to application inside sandbox. - if spec.Linux != nil && spec.Linux.Seccomp != nil { - log.Warningf("Seccomp spec is being ignored") - } - return nil -} - -// Sandbox wraps a child sandbox process, and is responsible for saving and -// loading sandbox metadata to disk. -// -// Within a root directory, we maintain subdirectories for each sandbox named -// with the sandbox id. The sandbox metadata is is stored as json within the -// sandbox directory in a file named "meta.json". This metadata format is -// defined by us, and is not part of the OCI spec. -// -// Sandboxes must write this metadata file after any change to their internal -// state. The entire sandbox directory is deleted when the sandbox is -// destroyed. +// Sandbox wraps a sandbox process. // -// TODO: Protect against concurrent changes to the sandbox metadata -// file. +// It is used to start/stop sandbox process (and associated processes like +// gofers), as well as for running and manipulating containers inside a running +// sandbox. type Sandbox struct { - // ID is the sandbox ID. + // ID is the id of the sandbox. By convention, this is the same ID as + // the first container run in the sandbox. ID string `json:"id"` - // Spec is the OCI runtime spec that configures this sandbox. - Spec *specs.Spec `json:"spec"` - - // BundleDir is the directory containing the sandbox bundle. - BundleDir string `json:"bundleDir"` - - // SandboxRoot is the directory containing the sandbox metadata file. - SandboxRoot string `json:"sandboxRoot"` - - // CreatedAt is the time the sandbox was created. - CreatedAt time.Time `json:"createdAt"` - - // Owner is the sandbox owner. - Owner string `json:"owner"` - - // ConsoleSocket is the path to a unix domain socket that will receive - // the console FD. It is only used during create, so we don't need to - // store it in the metadata. - ConsoleSocket string `json:"-"` - - // Pid is the pid of the running sandbox. Only valid if Status is - // Created or Running. + // Pid is the pid of the running sandbox. May be 0 is the sandbox is + // not running. Pid int `json:"pid"` - // GoferPid is the pid of the gofer running along side the sandbox. May be 0 - // if the gofer has been killed or it's not being used. + // GoferPid is the pid of the gofer running along side the sandbox. May + // be 0 if the gofer has been killed or it's not being used. GoferPid int `json:"goferPid"` - - // Status is the current sandbox Status. - Status Status `json:"status"` } -// Create creates the sandbox subprocess and writes the metadata file. Args -// are additional arguments that will be passed to the sandbox process. -func Create(id string, spec *specs.Spec, conf *boot.Config, bundleDir, consoleSocket, pidFile string, args []string) (*Sandbox, error) { - log.Debugf("Create sandbox %q in root dir: %s", id, conf.RootDir) - if err := validateID(id); err != nil { - return nil, err - } - if err := validateSpec(spec); err != nil { - return nil, err - } - - sandboxRoot := filepath.Join(conf.RootDir, id) - if exists(sandboxRoot) { - return nil, fmt.Errorf("sandbox with id %q already exists: %q ", id, sandboxRoot) - } +// Create creates the sandbox process. +func Create(id string, spec *specs.Spec, conf *boot.Config, bundleDir, consoleSocket string) (*Sandbox, error) { + s := &Sandbox{ID: id} - s := &Sandbox{ - ID: id, - Spec: spec, - ConsoleSocket: consoleSocket, - BundleDir: bundleDir, - SandboxRoot: sandboxRoot, - Status: Creating, - Owner: os.Getenv("USER"), - } - - // Create sandbox process. If anything errors between now and the end of this - // function, we MUST clean up all sandbox resources. - if err := s.createProcesses(conf, args); err != nil { - s.Destroy() + binPath, err := specutils.BinPath() + if err != nil { return nil, err } - // Wait for the control server to come up (or timeout). The sandbox is - // not "created" until that happens. - if err := s.waitForCreated(10 * time.Second); err != nil { - s.Destroy() + // Create the gofer process. + ioFiles, err := s.createGoferProcess(spec, conf, bundleDir, binPath) + if err != nil { return nil, err } - s.Status = Created - s.CreatedAt = time.Now() - - // Save the metadata file. - if err := s.save(); err != nil { - s.Destroy() + // Create the sandbox process. + if err := s.createSandboxProcess(spec, conf, bundleDir, consoleSocket, binPath, ioFiles); err != nil { return nil, err } - // Write the pid file. Containerd consideres the create complete after - // this file is created, so it must be the last thing we do. - if pidFile != "" { - if err := ioutil.WriteFile(pidFile, []byte(strconv.Itoa(s.Pid)), 0644); err != nil { - s.Destroy() - return nil, fmt.Errorf("error writing pid file: %v", err) - } - } - - return s, nil -} - -// Run is a helper that calls Create + Start + Wait. -func Run(id string, spec *specs.Spec, conf *boot.Config, bundleDir, consoleSocket, pidFile string, args []string) (syscall.WaitStatus, error) { - s, err := Create(id, spec, conf, bundleDir, consoleSocket, pidFile, args) - if err != nil { - return 0, fmt.Errorf("error creating sandbox: %v", err) - } - if err := s.Start(conf); err != nil { - return 0, fmt.Errorf("error starting sandbox: %v", err) - } - return s.Wait() -} - -// Load loads a sandbox from with the given id from a metadata file. -func Load(rootDir, id string) (*Sandbox, error) { - log.Debugf("Load sandbox %q %q", rootDir, id) - if err := validateID(id); err != nil { + // Wait for the control server to come up (or timeout). + if err := s.waitForCreated(10 * time.Second); err != nil { return nil, err } - sandboxRoot := filepath.Join(rootDir, id) - if !exists(sandboxRoot) { - return nil, fmt.Errorf("sandbox with id %q does not exist", id) - } - metaFile := filepath.Join(sandboxRoot, metadataFilename) - if !exists(metaFile) { - return nil, fmt.Errorf("sandbox with id %q does not have metadata file %q", id, metaFile) - } - metaBytes, err := ioutil.ReadFile(metaFile) - if err != nil { - return nil, fmt.Errorf("error reading sandbox metadata file %q: %v", metaFile, err) - } - var s Sandbox - if err := json.Unmarshal(metaBytes, &s); err != nil { - return nil, fmt.Errorf("error unmarshaling sandbox metadata from %q: %v", metaFile, err) - } - - // If the status is "Running" or "Created", check that the process - // still exists, and set it to Stopped if it does not. - // - // This is inherently racey. - if s.Status == Running || s.Status == Created { - // Send signal 0 to check if process exists. - if err := s.Signal(0); err != nil { - // Process no longer exists. - s.Status = Stopped - s.Pid = 0 - } - } - return &s, nil -} - -// List returns all sandbox ids in the given root directory. -func List(rootDir string) ([]string, error) { - log.Debugf("List sandboxes %q", rootDir) - fs, err := ioutil.ReadDir(rootDir) - if err != nil { - return nil, fmt.Errorf("ReadDir(%s) failed: %v", rootDir, err) - } - var out []string - for _, f := range fs { - out = append(out, f.Name()) - } - return out, nil -} - -// State returns the metadata of the sandbox. -func (s *Sandbox) State() specs.State { - return specs.State{ - Version: specs.Version, - ID: s.ID, - Status: s.Status.String(), - Pid: s.Pid, - Bundle: s.BundleDir, - } + return s, nil } // Start starts running the containerized process inside the sandbox. -func (s *Sandbox) Start(conf *boot.Config) error { +func (s *Sandbox) Start(cid string, spec *specs.Spec, conf *boot.Config) error { log.Debugf("Start sandbox %q, pid: %d", s.ID, s.Pid) - if s.Status != Created { - return fmt.Errorf("cannot start container in state %s", s.Status) - } - - // "If any prestart hook fails, the runtime MUST generate an error, - // stop and destroy the container". - if s.Spec.Hooks != nil { - if err := executeHooks(s.Spec.Hooks.Prestart, s.State()); err != nil { - s.Destroy() - return err - } - } - - c, err := s.connect() + conn, err := s.connect() if err != nil { - s.Destroy() return err } - defer c.Close() + defer conn.Close() // Configure the network. - if err := setupNetwork(c, s.Pid, s.Spec, conf); err != nil { - s.Destroy() + if err := setupNetwork(conn, s.Pid, spec, conf); err != nil { return fmt.Errorf("error setting up network: %v", err) } // Send a message to the sandbox control server to start the // application. - if err := c.Call(boot.ApplicationStart, nil, nil); err != nil { - s.Destroy() - return fmt.Errorf("error starting application %v: %v", s.Spec.Process.Args, err) - } - - // "If any poststart hook fails, the runtime MUST log a warning, but - // the remaining hooks and lifecycle continue as if the hook had - // succeeded". - if s.Spec.Hooks != nil { - executeHooksBestEffort(s.Spec.Hooks.Poststart, s.State()) + // + // TODO: Pass in the container id (cid) here. The sandbox + // should start only that container. + if err := conn.Call(boot.ApplicationStart, nil, nil); err != nil { + return fmt.Errorf("error starting application %v: %v", spec.Process.Args, err) } - s.Status = Running - return s.save() + return nil } -// Processes retrieves the list of processes and associated metadata inside a -// sandbox. -func (s *Sandbox) Processes() ([]*control.Process, error) { - if s.Status != Running { - return nil, fmt.Errorf("cannot get processes of container %q because it isn't running. It is in state %v", s.ID, s.Status) - } - - c, err := s.connect() +// Processes retrieves the list of processes and associated metadata for a +// given container in this sandbox. +func (s *Sandbox) Processes(cid string) ([]*control.Process, error) { + conn, err := s.connect() if err != nil { return nil, err } - defer c.Close() + defer conn.Close() var pl []*control.Process - if err := c.Call(boot.ApplicationProcesses, nil, &pl); err != nil { + // TODO: Pass in the container id (cid) here. The sandbox + // should return process info for only that container. + if err := conn.Call(boot.ApplicationProcesses, nil, &pl); err != nil { return nil, fmt.Errorf("error retrieving process data from sandbox: %v", err) } return pl, nil } -// Execute runs the specified command in the sandbox. -func (s *Sandbox) Execute(e *control.ExecArgs) (syscall.WaitStatus, error) { - log.Debugf("Execute in sandbox %q, pid: %d, args: %+v", s.ID, s.Pid, e) - if s.Status != Created && s.Status != Running { - return 0, fmt.Errorf("cannot exec in container in state %s", s.Status) - } - - log.Debugf("Connecting to sandbox...") - c, err := s.connect() +// Execute runs the specified command in the container. +func (s *Sandbox) Execute(cid string, e *control.ExecArgs) (syscall.WaitStatus, error) { + conn, err := s.connect() if err != nil { return 0, fmt.Errorf("error connecting to control server at pid %d: %v", s.Pid, err) } - defer c.Close() + defer conn.Close() // Send a message to the sandbox control server to start the application. var waitStatus uint32 - if err := c.Call(boot.ApplicationExecute, e, &waitStatus); err != nil { + // TODO: Pass in the container id (cid) here. The sandbox + // should execute in the context of that container. + if err := conn.Call(boot.ApplicationExecute, e, &waitStatus); err != nil { return 0, fmt.Errorf("error executing in sandbox: %v", err) } @@ -347,60 +145,45 @@ func (s *Sandbox) Execute(e *control.ExecArgs) (syscall.WaitStatus, error) { } // Event retrieves stats about the sandbox such as memory and CPU utilization. -func (s *Sandbox) Event() (*boot.Event, error) { - if s.Status != Running && s.Status != Created { - return nil, fmt.Errorf("cannot get events for container in state: %s", s.Status) - } - - c, err := s.connect() +func (s *Sandbox) Event(cid string) (*boot.Event, error) { + conn, err := s.connect() if err != nil { return nil, err } - defer c.Close() + defer conn.Close() var e boot.Event - if err := c.Call(boot.ApplicationEvent, nil, &e); err != nil { + // TODO: Pass in the container id (cid) here. The sandbox + // should return events only for that container. + if err := conn.Call(boot.ApplicationEvent, nil, &e); err != nil { return nil, fmt.Errorf("error retrieving event data from sandbox: %v", err) } - e.ID = s.ID + e.ID = cid return &e, nil } func (s *Sandbox) connect() (*urpc.Client, error) { log.Debugf("Connecting to sandbox...") - c, err := client.ConnectTo(boot.ControlSocketAddr(s.ID)) + conn, err := client.ConnectTo(boot.ControlSocketAddr(s.ID)) if err != nil { return nil, fmt.Errorf("error connecting to control server at pid %d: %v", s.Pid, err) } - return c, nil -} - -func (s *Sandbox) createProcesses(conf *boot.Config, args []string) error { - binPath, err := specutils.BinPath() - if err != nil { - return err - } - - ioFiles, err := s.createGoferProcess(conf, binPath, args) - if err != nil { - return err - } - return s.createSandboxProcess(conf, binPath, args, ioFiles) + return conn, nil } -func (s *Sandbox) createGoferProcess(conf *boot.Config, binPath string, commonArgs []string) ([]*os.File, error) { +func (s *Sandbox) createGoferProcess(spec *specs.Spec, conf *boot.Config, bundleDir, binPath string) ([]*os.File, error) { if conf.FileAccess != boot.FileAccessProxy { // Don't start a gofer. The sandbox will access host FS directly. return nil, nil } - var args []string - args = append(args, commonArgs...) - args = append(args, "gofer", "--bundle", s.BundleDir) + // Start with the general config flags. + args := conf.ToFlags() + args = append(args, "gofer", "--bundle", bundleDir) - // Start with root mount and then add any other additional mount. + // Add root mount and then add any other additional mounts. mountCount := 1 - for _, m := range s.Spec.Mounts { + for _, m := range spec.Mounts { if specutils.Is9PMount(m) { mountCount++ } @@ -429,8 +212,8 @@ func (s *Sandbox) createGoferProcess(conf *boot.Config, binPath string, commonAr // Setup any uid/gid mappings, and create or join the configured user // namespace so the gofer's view of the filesystem aligns with the // users in the sandbox. - setUIDGIDMappings(cmd, s.Spec) - nss := filterNS([]specs.LinuxNamespaceType{specs.UserNamespace}, s.Spec) + setUIDGIDMappings(cmd, spec) + nss := filterNS([]specs.LinuxNamespaceType{specs.UserNamespace}, spec) // Start the gofer in the given namespace. log.Debugf("Starting gofer: %s %v", binPath, args) @@ -444,7 +227,7 @@ func (s *Sandbox) createGoferProcess(conf *boot.Config, binPath string, commonAr // createSandboxProcess starts the sandbox as a subprocess by running the "boot" // command, passing in the bundle dir. -func (s *Sandbox) createSandboxProcess(conf *boot.Config, binPath string, commonArgs []string, ioFiles []*os.File) error { +func (s *Sandbox) createSandboxProcess(spec *specs.Spec, conf *boot.Config, bundleDir, consoleSocket, binPath string, ioFiles []*os.File) error { // nextFD is used to get unused FDs that we can pass to the sandbox. It // starts at 3 because 0, 1, and 2 are taken by stdin/out/err. nextFD := 3 @@ -457,13 +240,13 @@ func (s *Sandbox) createSandboxProcess(conf *boot.Config, binPath string, common return fmt.Errorf("error creating control server socket for sandbox %q: %v", s.ID, err) } - consoleEnabled := s.ConsoleSocket != "" + consoleEnabled := consoleSocket != "" - cmd := exec.Command(binPath, commonArgs...) + cmd := exec.Command(binPath, conf.ToFlags()...) cmd.SysProcAttr = &syscall.SysProcAttr{} cmd.Args = append(cmd.Args, "boot", - "--bundle", s.BundleDir, + "--bundle", bundleDir, "--controller-fd="+strconv.Itoa(nextFD), fmt.Sprintf("--console=%t", consoleEnabled)) nextFD++ @@ -485,9 +268,9 @@ func (s *Sandbox) createSandboxProcess(conf *boot.Config, binPath string, common if consoleEnabled { // setupConsole will send the master on the socket, and return // the slave. - tty, err := setupConsole(s.ConsoleSocket) + tty, err := setupConsole(consoleSocket) if err != nil { - return fmt.Errorf("error setting up control socket %q: %v", s.ConsoleSocket, err) + return fmt.Errorf("error setting up control socket %q: %v", consoleSocket, err) } defer tty.Close() @@ -535,7 +318,7 @@ func (s *Sandbox) createSandboxProcess(conf *boot.Config, binPath string, common // Joins the network namespace if network is enabled. the sandbox talks // directly to the host network, which may have been configured in the // namespace. - if ns, ok := getNS(specs.NetworkNamespace, s.Spec); ok && conf.Network != boot.NetworkNone { + if ns, ok := getNS(specs.NetworkNamespace, spec); ok && conf.Network != boot.NetworkNone { log.Infof("Sandbox will be started in the container's network namespace: %+v", ns) nss = append(nss, ns) } else { @@ -549,10 +332,10 @@ func (s *Sandbox) createSandboxProcess(conf *boot.Config, binPath string, common // - Gofer: when using a Gofer, the sandbox process can run isolated in an // empty namespace. if conf.Network == boot.NetworkHost || conf.FileAccess == boot.FileAccessDirect { - if userns, ok := getNS(specs.UserNamespace, s.Spec); ok { + if userns, ok := getNS(specs.UserNamespace, spec); ok { log.Infof("Sandbox will be started in container's user namespace: %+v", userns) nss = append(nss, userns) - setUIDGIDMappings(cmd, s.Spec) + setUIDGIDMappings(cmd, spec) } else { log.Infof("Sandbox will be started in the current user namespace") } @@ -596,8 +379,10 @@ func (s *Sandbox) waitForCreated(timeout time.Duration) error { } // Wait waits for the containerized process to exit, and returns its WaitStatus. -func (s *Sandbox) Wait() (syscall.WaitStatus, error) { - log.Debugf("Wait on sandbox %q with pid %d", s.ID, s.Pid) +func (s *Sandbox) Wait(cid string) (syscall.WaitStatus, error) { + // TODO: This waits on the sandbox process. We need a way + // to wait on an individual container in the sandbox. + p, err := os.FindProcess(s.Pid) if err != nil { // "On Unix systems, FindProcess always succeeds and returns a @@ -611,6 +396,13 @@ func (s *Sandbox) Wait() (syscall.WaitStatus, error) { return ps.Sys().(syscall.WaitStatus), nil } +// Stop stops the container in the sandbox. +func (s *Sandbox) Stop(cid string) error { + // TODO: This should stop the container with the given ID + // in the sandbox. + return nil +} + // Destroy frees all resources associated with the sandbox. func (s *Sandbox) Destroy() error { log.Debugf("Destroy sandbox %q", s.ID) @@ -625,60 +417,26 @@ func (s *Sandbox) Destroy() error { sendSignal(s.GoferPid, unix.SIGKILL) s.GoferPid = 0 } - if err := os.RemoveAll(s.SandboxRoot); err != nil { - log.Warningf("Failed to delete sandbox root directory %q, err: %v", s.SandboxRoot, err) - } - - // "If any poststop hook fails, the runtime MUST log a warning, but the - // remaining hooks and lifecycle continue as if the hook had succeeded". - if s.Spec.Hooks != nil && (s.Status == Created || s.Status == Running) { - executeHooksBestEffort(s.Spec.Hooks.Poststop, s.State()) - } - s.Status = Stopped return nil } -// Signal sends the signal to the sandbox. -func (s *Sandbox) Signal(sig syscall.Signal) error { +// Signal sends the signal to a container in the sandbox. +func (s *Sandbox) Signal(cid string, sig syscall.Signal) error { log.Debugf("Signal sandbox %q", s.ID) - if s.Status == Stopped { - log.Warningf("sandbox %q not running, not sending signal %v to pid %d", s.ID, sig, s.Pid) - return nil - } + + // TODO: This sends a signal to the sandbox process, which + // will be forwarded to the first process in the sandbox. We need a way + // to send a signal to any container in the sandbox. + // to wait on an individual container in the sandbox. + return sendSignal(s.Pid, sig) } +// sendSignal sends a signal to the sandbox process. func sendSignal(pid int, sig syscall.Signal) error { if err := syscall.Kill(pid, sig); err != nil { return fmt.Errorf("error sending signal %d to pid %d: %v", sig, pid, err) } return nil } - -// save saves the sandbox metadata to a file. -func (s *Sandbox) save() error { - log.Debugf("Save sandbox %q", s.ID) - if err := os.MkdirAll(s.SandboxRoot, 0711); err != nil { - return fmt.Errorf("error creating sandbox root directory %q: %v", s.SandboxRoot, err) - } - meta, err := json.Marshal(s) - if err != nil { - return fmt.Errorf("error marshaling sandbox metadata: %v", err) - } - metaFile := filepath.Join(s.SandboxRoot, metadataFilename) - if err := ioutil.WriteFile(metaFile, meta, 0640); err != nil { - return fmt.Errorf("error writing sandbox metadata: %v", err) - } - return nil -} - -// exists returns true if the given file exists. -func exists(f string) bool { - if _, err := os.Stat(f); err == nil { - return true - } else if !os.IsNotExist(err) { - log.Warningf("error checking for file %q: %v", f, err) - } - return false -} |