diff options
| author | Nicolas Lacasse <nlacasse@google.com> | 2018-10-17 09:01:42 -0700 |
|---|---|---|
| committer | Shentubot <shentubot@google.com> | 2018-10-17 09:03:01 -0700 |
| commit | 4fae756645cf11a9f8bae87dd845d82dce5e428e (patch) | |
| tree | 4a60ef15548daf2c93b0cb9c231ca151521fa29e | |
| parent | fb4629277839495ef42ad6af13d1823d79a3e6c0 (diff) | |
Make removing cgroups retry up to 5 seconds.
Sometimes if we try to remove the cgroup directory too soon after killing the
sandbox we EBUSY. This CL adds a retry (up to 5 seconds) for removing.
Deflakes ChrootTest.
PiperOrigin-RevId: 217526909
Change-Id: I749bb172117e2298c9888ecad094072393b94810
| -rw-r--r-- | runsc/cgroup/BUILD | 1 | ||||
| -rw-r--r-- | runsc/cgroup/cgroup.go | 18 |
2 files changed, 17 insertions, 2 deletions
diff --git a/runsc/cgroup/BUILD b/runsc/cgroup/BUILD index 4a535d230..10a8e5feb 100644 --- a/runsc/cgroup/BUILD +++ b/runsc/cgroup/BUILD @@ -12,6 +12,7 @@ go_library( deps = [ "//pkg/log", "//runsc/specutils", + "@com_github_cenkalti_backoff//:go_default_library", "@com_github_opencontainers_runtime-spec//specs-go:go_default_library", ], ) diff --git a/runsc/cgroup/cgroup.go b/runsc/cgroup/cgroup.go index af0252bb3..7a75a189a 100644 --- a/runsc/cgroup/cgroup.go +++ b/runsc/cgroup/cgroup.go @@ -17,6 +17,7 @@ package cgroup import ( + "context" "fmt" "io/ioutil" "os" @@ -24,7 +25,9 @@ import ( "strconv" "strings" "syscall" + "time" + "github.com/cenkalti/backoff" specs "github.com/opencontainers/runtime-spec/specs-go" "gvisor.googlesource.com/gvisor/pkg/log" "gvisor.googlesource.com/gvisor/runsc/specutils" @@ -214,8 +217,19 @@ func (c *Cgroup) Uninstall() error { } log.Debugf("Deleting cgroup %q", c.Name) for key := range controllers { - if err := syscall.Rmdir(c.makePath(key)); err != nil && !os.IsNotExist(err) { - return err + path := c.makePath(key) + log.Debugf("Removing cgroup controller for key=%q path=%q", key, path) + + // If we try to remove the cgroup too soon after killing the + // sandbox we might get EBUSY, so we retry for a few seconds + // until it succeeds. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + b := backoff.WithContext(backoff.NewConstantBackOff(100*time.Millisecond), ctx) + if err := backoff.Retry(func() error { + return syscall.Rmdir(path) + }, b); err != nil { + return fmt.Errorf("error removing cgroup path %q: %v", path, err) } } return nil |