Make sandbox join the pod cgroup in K8s

cgroups in K8s are setup with the following hierarchy: `.../pod/container`.
The sandbox is created with the first container and consequently uses the
the pause container cgroup. This change removes the container cgroup from
the path to make the sandbox use the pod cgroup instead. Otherwise limits
set to the pause container will apply to the entire sandbox.

PiperOrigin-RevId: 374273277

4 files changed, 42 insertions, 60 deletions

diff --git a/pkg/shim/utils/annotations.go b/pkg/shim/utils/annotations.go
index 1e9d3f365..c744800bb 100644
--- a/pkg/shim/utils/annotations.go
+++ b/pkg/shim/utils/annotations.go
@@ -19,7 +19,9 @@ package utils
 // These are vendor due to import conflicts.
 const (
 	sandboxLogDirAnnotation = "io.kubernetes.cri.sandbox-log-directory"
-	containerTypeAnnotation = "io.kubernetes.cri.container-type"
+	// ContainerTypeAnnotation is they key that defines sandbox or container.
+	ContainerTypeAnnotation = "io.kubernetes.cri.container-type"
 	containerTypeSandbox    = "sandbox"
-	containerTypeContainer  = "container"
+	// ContainerTypeContainer is the value for container.
+	ContainerTypeContainer = "container"
 )
diff --git a/pkg/shim/utils/utils.go b/pkg/shim/utils/utils.go
index 7b1cd983e..f183b1bbc 100644
--- a/pkg/shim/utils/utils.go
+++ b/pkg/shim/utils/utils.go
@@ -18,19 +18,16 @@ package utils
 import (
 	"encoding/json"
 	"io/ioutil"
-	"os"
 	"path/filepath"
 
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
+const configFilename = "config.json"
+
 // ReadSpec reads OCI spec from the bundle directory.
 func ReadSpec(bundle string) (*specs.Spec, error) {
-	f, err := os.Open(filepath.Join(bundle, "config.json"))
-	if err != nil {
-		return nil, err
-	}
-	b, err := ioutil.ReadAll(f)
+	b, err := ioutil.ReadFile(filepath.Join(bundle, configFilename))
 	if err != nil {
 		return nil, err
 	}
@@ -41,9 +38,18 @@ func ReadSpec(bundle string) (*specs.Spec, error) {
 	return &spec, nil
 }
 
+// WriteSpec writes OCI spec to the bundle directory.
+func WriteSpec(bundle string, spec *specs.Spec) error {
+	b, err := json.Marshal(spec)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(filepath.Join(bundle, configFilename), b, 0666)
+}
+
 // IsSandbox checks whether a container is a sandbox container.
 func IsSandbox(spec *specs.Spec) bool {
-	t, ok := spec.Annotations[containerTypeAnnotation]
+	t, ok := spec.Annotations[ContainerTypeAnnotation]
 	return !ok || t == containerTypeSandbox
 }
 
diff --git a/pkg/shim/utils/volumes.go b/pkg/shim/utils/volumes.go
index cdcb88229..6bc75139d 100644
--- a/pkg/shim/utils/volumes.go
+++ b/pkg/shim/utils/volumes.go
@@ -15,9 +15,7 @@
 package utils
 
 import (
-	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"path/filepath"
 	"strings"
 
@@ -89,8 +87,8 @@ func isVolumePath(volume, path string) (bool, error) {
 }
 
 // UpdateVolumeAnnotations add necessary OCI annotations for gvisor
-// volume optimization.
-func UpdateVolumeAnnotations(bundle string, s *specs.Spec) error {
+// volume optimization. Returns true if the spec was modified.
+func UpdateVolumeAnnotations(s *specs.Spec) (bool, error) {
 	var uid string
 	if IsSandbox(s) {
 		var err error
@@ -98,7 +96,7 @@ func UpdateVolumeAnnotations(bundle string, s *specs.Spec) error {
 		if err != nil {
 			// Skip if we can't get pod UID, because this doesn't work
 			// for containerd 1.1.
-			return nil
+			return false, nil
 		}
 	}
 	var updated bool
@@ -114,7 +112,7 @@ func UpdateVolumeAnnotations(bundle string, s *specs.Spec) error {
 			// This is a sandbox.
 			path, err := volumePath(volume, uid)
 			if err != nil {
-				return fmt.Errorf("get volume path for %q: %w", volume, err)
+				return false, fmt.Errorf("get volume path for %q: %w", volume, err)
 			}
 			s.Annotations[volumeSourceKey(volume)] = path
 			updated = true
@@ -138,15 +136,7 @@ func UpdateVolumeAnnotations(bundle string, s *specs.Spec) error {
 			}
 		}
 	}
-	if !updated {
-		return nil
-	}
-	// Update bundle.
-	b, err := json.Marshal(s)
-	if err != nil {
-		return err
-	}
-	return ioutil.WriteFile(filepath.Join(bundle, "config.json"), b, 0666)
+	return updated, nil
 }
 
 func changeMountType(m *specs.Mount, newType string) {
diff --git a/pkg/shim/utils/volumes_test.go b/pkg/shim/utils/volumes_test.go
index b25c53c73..5db43cdf1 100644
--- a/pkg/shim/utils/volumes_test.go
+++ b/pkg/shim/utils/volumes_test.go
@@ -15,11 +15,9 @@
 package utils
 
 import (
-	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
-	"path/filepath"
 	"reflect"
 	"testing"
 
@@ -58,7 +56,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 			spec: &specs.Spec{
 				Annotations: map[string]string{
 					sandboxLogDirAnnotation:                       testLogDirPath,
-					containerTypeAnnotation:                       containerTypeSandbox,
+					ContainerTypeAnnotation:                       containerTypeSandbox,
 					volumeKeyPrefix + testVolumeName + ".share":   "pod",
 					volumeKeyPrefix + testVolumeName + ".type":    "tmpfs",
 					volumeKeyPrefix + testVolumeName + ".options": "ro",
@@ -67,7 +65,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 			expected: &specs.Spec{
 				Annotations: map[string]string{
 					sandboxLogDirAnnotation:                       testLogDirPath,
-					containerTypeAnnotation:                       containerTypeSandbox,
+					ContainerTypeAnnotation:                       containerTypeSandbox,
 					volumeKeyPrefix + testVolumeName + ".share":   "pod",
 					volumeKeyPrefix + testVolumeName + ".type":    "tmpfs",
 					volumeKeyPrefix + testVolumeName + ".options": "ro",
@@ -81,7 +79,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 			spec: &specs.Spec{
 				Annotations: map[string]string{
 					sandboxLogDirAnnotation:                       testLegacyLogDirPath,
-					containerTypeAnnotation:                       containerTypeSandbox,
+					ContainerTypeAnnotation:                       containerTypeSandbox,
 					volumeKeyPrefix + testVolumeName + ".share":   "pod",
 					volumeKeyPrefix + testVolumeName + ".type":    "tmpfs",
 					volumeKeyPrefix + testVolumeName + ".options": "ro",
@@ -90,7 +88,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 			expected: &specs.Spec{
 				Annotations: map[string]string{
 					sandboxLogDirAnnotation:                       testLegacyLogDirPath,
-					containerTypeAnnotation:                       containerTypeSandbox,
+					ContainerTypeAnnotation:                       containerTypeSandbox,
 					volumeKeyPrefix + testVolumeName + ".share":   "pod",
 					volumeKeyPrefix + testVolumeName + ".type":    "tmpfs",
 					volumeKeyPrefix + testVolumeName + ".options": "ro",
@@ -117,7 +115,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 					},
 				},
 				Annotations: map[string]string{
-					containerTypeAnnotation:                       containerTypeContainer,
+					ContainerTypeAnnotation:                       ContainerTypeContainer,
 					volumeKeyPrefix + testVolumeName + ".share":   "pod",
 					volumeKeyPrefix + testVolumeName + ".type":    "tmpfs",
 					volumeKeyPrefix + testVolumeName + ".options": "ro",
@@ -139,7 +137,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 					},
 				},
 				Annotations: map[string]string{
-					containerTypeAnnotation:                       containerTypeContainer,
+					ContainerTypeAnnotation:                       ContainerTypeContainer,
 					volumeKeyPrefix + testVolumeName + ".share":   "pod",
 					volumeKeyPrefix + testVolumeName + ".type":    "tmpfs",
 					volumeKeyPrefix + testVolumeName + ".options": "ro",
@@ -159,7 +157,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 					},
 				},
 				Annotations: map[string]string{
-					containerTypeAnnotation:                       containerTypeContainer,
+					ContainerTypeAnnotation:                       ContainerTypeContainer,
 					volumeKeyPrefix + testVolumeName + ".share":   "container",
 					volumeKeyPrefix + testVolumeName + ".type":    "bind",
 					volumeKeyPrefix + testVolumeName + ".options": "ro",
@@ -175,7 +173,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 					},
 				},
 				Annotations: map[string]string{
-					containerTypeAnnotation:                       containerTypeContainer,
+					ContainerTypeAnnotation:                       ContainerTypeContainer,
 					volumeKeyPrefix + testVolumeName + ".share":   "container",
 					volumeKeyPrefix + testVolumeName + ".type":    "bind",
 					volumeKeyPrefix + testVolumeName + ".options": "ro",
@@ -187,7 +185,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 			name: "should not return error without pod log directory",
 			spec: &specs.Spec{
 				Annotations: map[string]string{
-					containerTypeAnnotation:                       containerTypeSandbox,
+					ContainerTypeAnnotation:                       containerTypeSandbox,
 					volumeKeyPrefix + testVolumeName + ".share":   "pod",
 					volumeKeyPrefix + testVolumeName + ".type":    "tmpfs",
 					volumeKeyPrefix + testVolumeName + ".options": "ro",
@@ -195,7 +193,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 			},
 			expected: &specs.Spec{
 				Annotations: map[string]string{
-					containerTypeAnnotation:                       containerTypeSandbox,
+					ContainerTypeAnnotation:                       containerTypeSandbox,
 					volumeKeyPrefix + testVolumeName + ".share":   "pod",
 					volumeKeyPrefix + testVolumeName + ".type":    "tmpfs",
 					volumeKeyPrefix + testVolumeName + ".options": "ro",
@@ -207,7 +205,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 			spec: &specs.Spec{
 				Annotations: map[string]string{
 					sandboxLogDirAnnotation:              testLogDirPath,
-					containerTypeAnnotation:              containerTypeSandbox,
+					ContainerTypeAnnotation:              containerTypeSandbox,
 					volumeKeyPrefix + "notexist.share":   "pod",
 					volumeKeyPrefix + "notexist.type":    "tmpfs",
 					volumeKeyPrefix + "notexist.options": "ro",
@@ -220,13 +218,13 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 			spec: &specs.Spec{
 				Annotations: map[string]string{
 					sandboxLogDirAnnotation: testLogDirPath,
-					containerTypeAnnotation: containerTypeSandbox,
+					ContainerTypeAnnotation: containerTypeSandbox,
 				},
 			},
 			expected: &specs.Spec{
 				Annotations: map[string]string{
 					sandboxLogDirAnnotation: testLogDirPath,
-					containerTypeAnnotation: containerTypeSandbox,
+					ContainerTypeAnnotation: containerTypeSandbox,
 				},
 			},
 		},
@@ -248,7 +246,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 					},
 				},
 				Annotations: map[string]string{
-					containerTypeAnnotation: containerTypeContainer,
+					ContainerTypeAnnotation: ContainerTypeContainer,
 				},
 			},
 			expected: &specs.Spec{
@@ -267,7 +265,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 					},
 				},
 				Annotations: map[string]string{
-					containerTypeAnnotation: containerTypeContainer,
+					ContainerTypeAnnotation: ContainerTypeContainer,
 				},
 			},
 		},
@@ -275,7 +273,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 			name: "bind options removed",
 			spec: &specs.Spec{
 				Annotations: map[string]string{
-					containerTypeAnnotation:                       containerTypeContainer,
+					ContainerTypeAnnotation:                       ContainerTypeContainer,
 					volumeKeyPrefix + testVolumeName + ".share":   "pod",
 					volumeKeyPrefix + testVolumeName + ".type":    "tmpfs",
 					volumeKeyPrefix + testVolumeName + ".options": "ro",
@@ -292,7 +290,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 			},
 			expected: &specs.Spec{
 				Annotations: map[string]string{
-					containerTypeAnnotation:                       containerTypeContainer,
+					ContainerTypeAnnotation:                       ContainerTypeContainer,
 					volumeKeyPrefix + testVolumeName + ".share":   "pod",
 					volumeKeyPrefix + testVolumeName + ".type":    "tmpfs",
 					volumeKeyPrefix + testVolumeName + ".options": "ro",
@@ -311,11 +309,7 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 		},
 	} {
 		t.Run(test.name, func(t *testing.T) {
-			bundle, err := ioutil.TempDir(dir, "test-bundle")
-			if err != nil {
-				t.Fatalf("Create test bundle: %v", err)
-			}
-			err = UpdateVolumeAnnotations(bundle, test.spec)
+			updated, err := UpdateVolumeAnnotations(test.spec)
 			if test.expectErr {
 				if err == nil {
 					t.Fatal("Expected error, but got nil")
@@ -328,18 +322,8 @@ func TestUpdateVolumeAnnotations(t *testing.T) {
 			if !reflect.DeepEqual(test.expected, test.spec) {
 				t.Fatalf("Expected %+v, got %+v", test.expected, test.spec)
 			}
-			if test.expectUpdate {
-				b, err := ioutil.ReadFile(filepath.Join(bundle, "config.json"))
-				if err != nil {
-					t.Fatalf("Read spec from bundle: %v", err)
-				}
-				var spec specs.Spec
-				if err := json.Unmarshal(b, &spec); err != nil {
-					t.Fatalf("Unmarshal spec: %v", err)
-				}
-				if !reflect.DeepEqual(test.expected, &spec) {
-					t.Fatalf("Expected %+v, got %+v", test.expected, &spec)
-				}
+			if test.expectUpdate != updated {
+				t.Errorf("Expected %v, got %v", test.expected, updated)
 			}
 		})
 	}