diff options
Diffstat (limited to 'shim')
| -rw-r--r-- | shim/BUILD | 15 | ||||
| -rw-r--r-- | shim/README.md | 21 | ||||
| -rw-r--r-- | shim/configure-containerd-shim-runsc-v1.md | 72 | ||||
| -rw-r--r-- | shim/configure-gvisor-containerd-shim.md | 42 | ||||
| -rw-r--r-- | shim/runsc.toml | 6 | ||||
| -rw-r--r-- | shim/runtime-handler-quickstart.md | 251 | ||||
| -rw-r--r-- | shim/untrusted-workload-quickstart.md | 212 | ||||
| -rw-r--r-- | shim/v1/BUILD | 43 | ||||
| -rw-r--r-- | shim/v1/README.md | 50 | ||||
| -rw-r--r-- | shim/v1/config.go (renamed from shim/v2/config.go) | 0 | ||||
| -rw-r--r-- | shim/v1/main.go | 247 | ||||
| -rw-r--r-- | shim/v2/BUILD | 32 | ||||
| -rw-r--r-- | shim/v2/README.md | 90 | ||||
| -rw-r--r-- | shim/v2/main.go | 298 | ||||
| -rw-r--r-- | shim/v2/runtime-handler-shim-v2-quickstart.md (renamed from shim/runtime-handler-shim-v2-quickstart.md) | 0 |
15 files changed, 491 insertions, 888 deletions
diff --git a/shim/BUILD b/shim/BUILD new file mode 100644 index 000000000..e581618b2 --- /dev/null +++ b/shim/BUILD @@ -0,0 +1,15 @@ +load("//tools:defs.bzl", "pkg_tar") + +package(licenses = ["notice"]) + +pkg_tar( + name = "config", + srcs = [ + "runsc.toml", + ], + mode = "0644", + package_dir = "/etc/containerd", + visibility = [ + "//runsc:__pkg__", + ], +) diff --git a/shim/README.md b/shim/README.md index e446ec970..c6824ebdc 100644 --- a/shim/README.md +++ b/shim/README.md @@ -1,16 +1,11 @@ -# gvisor-containerd-shim +# gVisor Containerd shims -gvisor-containerd-shim is a containerd shim. It implements the containerd v1 -shim API. It can be used as a drop-in replacement for -[containerd-shim][containerd-shim] -(though containerd-shim must still be installed). It allows the use of both -gVisor (runsc) and normal containers in the same containerd installation by -deferring to the runc shim if the desired runtime engine is not runsc. +There are various shims supported for differt versions of +[containerd][containerd]. -- [Untrusted Workload Quick Start (containerd >=1.1)](docs/untrusted-workload-quickstart.md) -- [Runtime Handler/RuntimeClass Quick Start (containerd >=1.2)](docs/runtime-handler-quickstart.md) -- [Runtime Handler/RuntimeClass Quick Start (shim v2) (containerd >=1.2)](docs/runtime-handler-shim-v2-quickstart.md) -- [Configure containerd-shim-runsc-v1 (shim v2) (containerd >= 1.3)](docs/configure-containerd-shim-runsc-v1.md) -- [Configure gvisor-containerd-shim (shim v1) (containerd <= 1.2)](docs/configure-gvisor-containerd-shim.md) +- [Configure gvisor-containerd-shim (shim v1) (containerd ≤ 1.2)](v1/configure-gvisor-containerd-shim.md) +- [Runtime Handler/RuntimeClass Quick Start (containerd >= 1.2)](v2/runtime-handler-quickstart.md) +- [Runtime Handler/RuntimeClass Quick Start (shim v2) (containerd >= 1.2)](v2/runtime-handler-shim-v2-quickstart.md) +- [Configure containerd-shim-runsc-v1 (shim v2) (containerd >= 1.3)](v2/configure-containerd-shim-runsc-v1.md) -[containerd-shim]: https://github.com/containerd/containerd/tree/master/cmd/containerd-shim +[containerd]: https://github.com/containerd/containerd diff --git a/shim/configure-containerd-shim-runsc-v1.md b/shim/configure-containerd-shim-runsc-v1.md deleted file mode 100644 index 977ceacbd..000000000 --- a/shim/configure-containerd-shim-runsc-v1.md +++ /dev/null @@ -1,72 +0,0 @@ -# Configure containerd-shim-runsc-v1 (Shim V2) - -This document describes how to configure runtime options for -`containerd-shim-runsc-v1`. This is follows on to the instructions of -[Runtime Handler Quick Start (shim v2) (containerd >=1.2)](runtime-handler-shim-v2-quickstart.md) -and requires containerd 1.3 or later. - -### Update `/etc/containerd/config.toml` to point to a configuration file for `containerd-shim-runsc-v1`. - -`containerd-shim-runsc-v1` supports a few different configuration options based -on the version of containerd that is used. For versions >= 1.3, it supports a -configurable config path in the containerd runtime configuration. - -```shell -{ # Step 1: Update runtime options for runsc in containerd config.toml -cat <<EOF | sudo tee /etc/containerd/config.toml -disabled_plugins = ["restart"] -[plugins.linux] - shim_debug = true -[plugins.cri.containerd.runtimes.runsc] - runtime_type = "io.containerd.runsc.v1" -[plugins.cri.containerd.runtimes.runsc.options] - TypeUrl = "io.containerd.runsc.v1.options" - ConfigPath = "/etc/containerd/runsc.toml" -EOF -} -``` - -### Configure `/etc/containerd/runsc.toml` - -The set of options that can be configured can be found in -[options.go](../pkg/v2/options/options.go). - -#### Example: Enable the KVM platform - -gVisor enables the use of a number of platforms. This example shows how to -configure `containerd-shim-runsc-v1` to use gvisor with the KVM platform. - -Find out more about platform in the -(gVisor documentation)[https://gvisor.dev/docs/user_guide/platforms/]. - -```shell -cat <<EOF | sudo tee /etc/containerd/runsc.toml -[runsc_config] -platform = "kvm" -EOF -``` - -### Example: Enable gVisor debug logging - -gVisor debug logging can be enabled by setting the `debug` and `debug-log` -flag. The shim will replace "%ID%" with the container ID in the path of the -`debug-log` flag. - -Find out more about debugging in the -(gVisor documentation)[https://gvisor.dev/docs/user_guide/debugging/]. - -```shell -cat <<EOF | sudo tee /etc/containerd/runsc.toml -[runsc_config] - debug=true - debug-log=/var/log/%ID%/gvisor.log -EOF -``` - -## Restart `containerd` - -When you are done restart containerd to pick up the new configuration files. - -```shell -sudo systemctl restart containerd -``` diff --git a/shim/configure-gvisor-containerd-shim.md b/shim/configure-gvisor-containerd-shim.md deleted file mode 100644 index 40151da56..000000000 --- a/shim/configure-gvisor-containerd-shim.md +++ /dev/null @@ -1,42 +0,0 @@ -# Configure gvisor-containerd-shim (Shim V1) - -This document describes how to configure runtime options for `gvisor-containerd-shim`. - -The shim configuration is stored in `/etc/containerd/gvisor-containerd-shim.toml`. The configuration file supports two values. - -`runc_shim`: The path to the runc shim. This is used by the gvisor-containerd-shim to run normal containers. -`runsc_config`: This is a set of key/value pairs that are converted into `runsc` command line flags. You can learn more about which flags are available by running `runsc flags`. - -## Example: Enable the KVM platform - -gVisor enables the use of a number of platforms. This configuration enables the -KVM platform. - -Find out more about platform in the -(gVisor documentation)[https://gvisor.dev/docs/user_guide/platforms/]. - -```shell -cat <<EOF | sudo tee /etc/containerd/gvisor-containerd-shim.toml -[runsc_config] -platform = "kvm" -EOF -``` - -## Example: Enable gVisor debug logging - -gVisor debug logging can be enabled by setting the `debug` and `debug-log` -flag. The shim will replace "%ID%" with the container ID in the path of the -`debug-log` flag. - -Find out more about debugging in the -(gVisor documentation)[https://gvisor.dev/docs/user_guide/debugging/]. - -```shell -cat <<EOF | sudo tee /etc/containerd/gvisor-containerd-shim.toml -# This is the path to the default runc containerd-shim. -runc_shim = "/usr/local/bin/containerd-shim" -[runsc_config] - debug=true - debug-log=/var/log/%ID%/gvisor.log -EOF -``` diff --git a/shim/runsc.toml b/shim/runsc.toml new file mode 100644 index 000000000..e1c7de1bb --- /dev/null +++ b/shim/runsc.toml @@ -0,0 +1,6 @@ +# This is an example configuration file for runsc. +# +# By default, it will be parsed from /etc/containerd/runsc.toml, but see the +# static path configured in v1/main.go. Note that the configuration mechanism +# for newer container shim versions is different: see the documentation in v2. +[runsc_config] diff --git a/shim/runtime-handler-quickstart.md b/shim/runtime-handler-quickstart.md deleted file mode 100644 index 684390b55..000000000 --- a/shim/runtime-handler-quickstart.md +++ /dev/null @@ -1,251 +0,0 @@ -# Runtime Handler Quickstart - -This document describes how to install and run the `gvisor-containerd-shim` -using the containerd runtime handler support. This requires containerd 1.2 or -later. - -## Requirements - -- **runsc**: See the [gVisor documentation](https://github.com/google/gvisor) for information on how to install runsc. -- **containerd**: See the [containerd website](https://containerd.io/) for information on how to install containerd. - -## Install - -### Install gvisor-containerd-shim - -1. Download the latest release of the `gvisor-containerd-shim`. See the - [releases page](https://github.com/google/gvisor-containerd-shim/releases) - -[embedmd]:# (../test/e2e/shim-install.sh shell /{ # Step 1\(release\)/ /^}/) -```shell -{ # Step 1(release): Install gvisor-containerd-shim -LATEST_RELEASE=$(wget -qO - https://api.github.com/repos/google/gvisor-containerd-shim/releases | grep -oP '(?<="browser_download_url": ")https://[^"]*gvisor-containerd-shim.linux-amd64' | head -1) -wget -O gvisor-containerd-shim ${LATEST_RELEASE} -chmod +x gvisor-containerd-shim -sudo mv gvisor-containerd-shim /usr/local/bin/gvisor-containerd-shim -} -``` - -2. Create the configuration for the gvisor shim in - `/etc/containerd/gvisor-containerd-shim.toml`: - -[embedmd]:# (../test/e2e/shim-install.sh shell /{ # Step 2/ /^}/) -```shell -{ # Step 2: Create the gvisor-containerd-shim.toml -cat <<EOF | sudo tee /etc/containerd/gvisor-containerd-shim.toml -# This is the path to the default runc containerd-shim. -runc_shim = "/usr/local/bin/containerd-shim" -EOF -} -``` - -### Configure containerd - -1. Update `/etc/containerd/config.toml`. Be sure to update the path to - `gvisor-containerd-shim` and `runsc` if necessary: - -[embedmd]:# (../test/e2e/runtime-handler/install.sh shell /{ # Step 1/ /^}/) -```shell -{ # Step 1: Create containerd config.toml -cat <<EOF | sudo tee /etc/containerd/config.toml -disabled_plugins = ["restart"] -[plugins.linux] - shim = "/usr/local/bin/gvisor-containerd-shim" - shim_debug = true -[plugins.cri.containerd.runtimes.runsc] - runtime_type = "io.containerd.runtime.v1.linux" - runtime_engine = "/usr/local/bin/runsc" - runtime_root = "/run/containerd/runsc" -EOF -} -``` - -2. Restart `containerd` - -```shell -sudo systemctl restart containerd -``` - -## Usage - -You can run containers in gVisor via containerd's CRI. - -### Install crictl - -1. Download and install the crictl binary: - -[embedmd]:# (../test/e2e/crictl-install.sh shell /{ # Step 1/ /^}/) -```shell -{ # Step 1: Download crictl -wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.13.0/crictl-v1.13.0-linux-amd64.tar.gz -tar xf crictl-v1.13.0-linux-amd64.tar.gz -sudo mv crictl /usr/local/bin -} -``` - -2. Write the crictl configuration file - -[embedmd]:# (../test/e2e/crictl-install.sh shell /{ # Step 2/ /^}/) -```shell -{ # Step 2: Configure crictl -cat <<EOF | sudo tee /etc/crictl.yaml -runtime-endpoint: unix:///run/containerd/containerd.sock -EOF -} -``` - -### Create the nginx Sandbox in gVisor - -1. Pull the nginx image - -[embedmd]:# (../test/e2e/runtime-handler/usage.sh shell /{ # Step 1/ /^}/) -```shell -{ # Step 1: Pull the nginx image -sudo crictl pull nginx -} -``` - -2. Create the sandbox creation request - -[embedmd]:# (../test/e2e/runtime-handler/usage.sh shell /{ # Step 2/ /^EOF\n}/) -```shell -{ # Step 2: Create sandbox.json -cat <<EOF | tee sandbox.json -{ - "metadata": { - "name": "nginx-sandbox", - "namespace": "default", - "attempt": 1, - "uid": "hdishd83djaidwnduwk28bcsb" - }, - "linux": { - }, - "log_directory": "/tmp" -} -EOF -} -``` - -3. Create the pod in gVisor - -[embedmd]:# (../test/e2e/runtime-handler/usage.sh shell /{ # Step 3/ /^}/) -```shell -{ # Step 3: Create the sandbox -SANDBOX_ID=$(sudo crictl runp --runtime runsc sandbox.json) -} -``` - -### Run the nginx Container in the Sandbox - -1. Create the nginx container creation request - -[embedmd]:# (../test/e2e/run-container.sh shell /{ # Step 1/ /^EOF\n}/) -```shell -{ # Step 1: Create nginx container config -cat <<EOF | tee container.json -{ - "metadata": { - "name": "nginx" - }, - "image":{ - "image": "nginx" - }, - "log_path":"nginx.0.log", - "linux": { - } -} -EOF -} -``` - -2. Create the nginx container - -[embedmd]:# (../test/e2e/run-container.sh shell /{ # Step 2/ /^}/) -```shell -{ # Step 2: Create nginx container -CONTAINER_ID=$(sudo crictl create ${SANDBOX_ID} container.json sandbox.json) -} -``` - -3. Start the nginx container - -[embedmd]:# (../test/e2e/run-container.sh shell /{ # Step 3/ /^}/) -```shell -{ # Step 3: Start nginx container -sudo crictl start ${CONTAINER_ID} -} -``` - -### Validate the container - -1. Inspect the created pod - -[embedmd]:# (../test/e2e/validate.sh shell /{ # Step 1/ /^}/) -```shell -{ # Step 1: Inspect the pod -sudo crictl inspectp ${SANDBOX_ID} -} -``` - -2. Inspect the nginx container - -[embedmd]:# (../test/e2e/validate.sh shell /{ # Step 2/ /^}/) -```shell -{ # Step 2: Inspect the container -sudo crictl inspect ${CONTAINER_ID} -} -``` - -3. Verify that nginx is running in gVisor - -[embedmd]:# (../test/e2e/validate.sh shell /{ # Step 3/ /^}/) -```shell -{ # Step 3: Check dmesg -sudo crictl exec ${CONTAINER_ID} dmesg | grep -i gvisor -} -``` - -### Set up the Kubernetes Runtime Class - -1. Install the Runtime Class for gVisor - -[embedmd]:# (../test/e2e/runtimeclass-install.sh shell /{ # Step 1/ /^}/) -```shell -{ # Step 1: Install a RuntimeClass -cat <<EOF | kubectl apply -f - -apiVersion: node.k8s.io/v1beta1 -kind: RuntimeClass -metadata: - name: gvisor -handler: runsc -EOF -} -``` - -2. Create a Pod with the gVisor Runtime Class - -[embedmd]:# (../test/e2e/runtimeclass-install.sh shell /{ # Step 2/ /^}/) -```shell -{ # Step 2: Create a pod -cat <<EOF | kubectl apply -f - -apiVersion: v1 -kind: Pod -metadata: - name: nginx-gvisor -spec: - runtimeClassName: gvisor - containers: - - name: nginx - image: nginx -EOF -} -``` - -3. Verify that the Pod is running - -[embedmd]:# (../test/e2e/runtimeclass-install.sh shell /{ # Step 3/ /^}/) -```shell -{ # Step 3: Get the pod -kubectl get pod nginx-gvisor -o wide -} -``` diff --git a/shim/untrusted-workload-quickstart.md b/shim/untrusted-workload-quickstart.md deleted file mode 100644 index fb4441845..000000000 --- a/shim/untrusted-workload-quickstart.md +++ /dev/null @@ -1,212 +0,0 @@ -# Untrusted Workload Quickstart - -This document describes how to install and run the `gvisor-containerd-shim` -using the untrusted workload CRI extension. This requires containerd 1.1 or -later. - -*Note: The untrusted workload CRI extension is deprecated by containerd. If you -are using containerd 1.2, please consider using runtime handler.* - -## Requirements - -- **runsc**: See the [gVisor documentation](https://github.com/google/gvisor) for information on how to install runsc. -- **containerd**: See the [containerd website](https://containerd.io/) for information on how to install containerd. - -## Install - -### Install gvisor-containerd-shim - -1. Download the latest release of the `gvisor-containerd-shim`. See the - [releases page](https://github.com/google/gvisor-containerd-shim/releases) - -[embedmd]:# (../test/e2e/shim-install.sh shell /{ # Step 1/ /^}/) -```shell -{ # Step 1(release): Install gvisor-containerd-shim -LATEST_RELEASE=$(wget -qO - https://api.github.com/repos/google/gvisor-containerd-shim/releases | grep -oP '(?<="browser_download_url": ")https://[^"]*gvisor-containerd-shim.linux-amd64' | head -1) -wget -O gvisor-containerd-shim ${LATEST_RELEASE} -chmod +x gvisor-containerd-shim -sudo mv gvisor-containerd-shim /usr/local/bin/gvisor-containerd-shim -} -``` - -2. Create the configuration for the gvisor shim in - `/etc/containerd/gvisor-containerd-shim.toml`: - -[embedmd]:# (../test/e2e/shim-install.sh shell /{ # Step 2/ /^}/) -```shell -{ # Step 2: Create the gvisor-containerd-shim.toml -cat <<EOF | sudo tee /etc/containerd/gvisor-containerd-shim.toml -# This is the path to the default runc containerd-shim. -runc_shim = "/usr/local/bin/containerd-shim" -EOF -} -``` - -### Configure containerd - -1. Update `/etc/containerd/config.toml`. Be sure to update the path to - `gvisor-containerd-shim` and `runsc` if necessary: - -[embedmd]:# (../test/e2e/untrusted-workload/install.sh shell /{ # Step 1/ /^}/) -```shell -{ # Step 1: Create containerd config.toml -cat <<EOF | sudo tee /etc/containerd/config.toml -disabled_plugins = ["restart"] -[plugins.linux] - shim = "/usr/local/bin/gvisor-containerd-shim" - shim_debug = true -[plugins.cri.containerd.untrusted_workload_runtime] - runtime_type = "io.containerd.runtime.v1.linux" - runtime_engine = "/usr/local/bin/runsc" - runtime_root = "/run/containerd/runsc" -EOF -} -``` - -2. Restart `containerd` - -```shell -sudo systemctl restart containerd -``` - -## Usage - -You can run containers in gVisor via containerd's CRI. - -### Install crictl - -1. Download and install the crictl binary: - -[embedmd]:# (../test/e2e/crictl-install.sh shell /{ # Step 1/ /^}/) -```shell -{ # Step 1: Download crictl -wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.13.0/crictl-v1.13.0-linux-amd64.tar.gz -tar xf crictl-v1.13.0-linux-amd64.tar.gz -sudo mv crictl /usr/local/bin -} -``` - -2. Write the crictl configuration file - -[embedmd]:# (../test/e2e/crictl-install.sh shell /{ # Step 2/ /^}/) -```shell -{ # Step 2: Configure crictl -cat <<EOF | sudo tee /etc/crictl.yaml -runtime-endpoint: unix:///run/containerd/containerd.sock -EOF -} -``` - -### Create the nginx Sandbox in gVisor - -1. Pull the nginx image - -[embedmd]:# (../test/e2e/untrusted-workload/usage.sh shell /{ # Step 1/ /^}/) -```shell -{ # Step 1: Pull the nginx image -sudo crictl pull nginx -} -``` - -2. Create the sandbox creation request - -[embedmd]:# (../test/e2e/untrusted-workload/usage.sh shell /{ # Step 2/ /^EOF\n}/) -```shell -{ # Step 2: Create sandbox.json -cat <<EOF | tee sandbox.json -{ - "metadata": { - "name": "nginx-sandbox", - "namespace": "default", - "attempt": 1, - "uid": "hdishd83djaidwnduwk28bcsb" - }, - "annotations": { - "io.kubernetes.cri.untrusted-workload": "true" - }, - "linux": { - }, - "log_directory": "/tmp" -} -EOF -} -``` - -3. Create the pod in gVisor - -[embedmd]:# (../test/e2e/untrusted-workload/usage.sh shell /{ # Step 3/ /^}/) -```shell -{ # Step 3: Create the sandbox -SANDBOX_ID=$(sudo crictl runp sandbox.json) -} -``` - -### Run the nginx Container in the Sandbox - -1. Create the nginx container creation request - -[embedmd]:# (../test/e2e/run-container.sh shell /{ # Step 1/ /^EOF\n}/) -```shell -{ # Step 1: Create nginx container config -cat <<EOF | tee container.json -{ - "metadata": { - "name": "nginx" - }, - "image":{ - "image": "nginx" - }, - "log_path":"nginx.0.log", - "linux": { - } -} -EOF -} -``` - -2. Create the nginx container - -[embedmd]:# (../test/e2e/run-container.sh shell /{ # Step 2/ /^}/) -```shell -{ # Step 2: Create nginx container -CONTAINER_ID=$(sudo crictl create ${SANDBOX_ID} container.json sandbox.json) -} -``` - -3. Start the nginx container - -[embedmd]:# (../test/e2e/run-container.sh shell /{ # Step 3/ /^}/) -```shell -{ # Step 3: Start nginx container -sudo crictl start ${CONTAINER_ID} -} -``` - -### Validate the container - -1. Inspect the created pod - -[embedmd]:# (../test/e2e/validate.sh shell /{ # Step 1/ /^}/) -```shell -{ # Step 1: Inspect the pod -sudo crictl inspectp ${SANDBOX_ID} -} -``` - -2. Inspect the nginx container - -[embedmd]:# (../test/e2e/validate.sh shell /{ # Step 2/ /^}/) -```shell -{ # Step 2: Inspect the container -sudo crictl inspect ${CONTAINER_ID} -} -``` - -3. Verify that nginx is running in gVisor - -[embedmd]:# (../test/e2e/validate.sh shell /{ # Step 3/ /^}/) -```shell -{ # Step 3: Check dmesg -sudo crictl exec ${CONTAINER_ID} dmesg | grep -i gvisor -} -``` diff --git a/shim/v1/BUILD b/shim/v1/BUILD new file mode 100644 index 000000000..3a863ecbb --- /dev/null +++ b/shim/v1/BUILD @@ -0,0 +1,43 @@ +load("//tools:defs.bzl", "go_binary") +load("//website:defs.bzl", "doc") + +package(licenses = ["notice"]) + +go_binary( + name = "gvisor-containerd-shim", + srcs = [ + "config.go", + "main.go", + ], + pure = True, + visibility = [ + "//visibility:public", + ], + deps = [ + "//pkg/shim/runsc", + "//pkg/shim/v1/shim", + "//pkg/shim/v2", + "@com_github_burntsushi_toml//:go_default_library", + "@com_github_containerd_containerd//events:go_default_library", + "@com_github_containerd_containerd//namespaces:go_default_library", + "@com_github_containerd_containerd//runtime/v1/linux/proc:go_default_library", + "@com_github_containerd_containerd//runtime/v1/shim:go_default_library", + "@com_github_containerd_containerd//runtime/v1/shim/v1:go_default_library", + "@com_github_containerd_containerd//runtime/v2/shim:go_default_library", + "@com_github_containerd_ttrpc//:go_default_library", + "@com_github_containerd_typeurl//:go_default_library", + "@com_github_gogo_protobuf//types:go_default_library", + "@com_github_opencontainers_runc//libcontainer/system:go_default_library", + "@org_golang_x_sys//unix:go_default_library", + ], +) + +doc( + name = "doc", + src = "README.md", + category = "User Guide", + permalink = "/docs/user_guide/gvisor-containerd-shim/", + subcategory = "Advanced", + visibility = ["//website:__pkg__"], + weight = "93", +) diff --git a/shim/v1/README.md b/shim/v1/README.md new file mode 100644 index 000000000..fcdf3ad77 --- /dev/null +++ b/shim/v1/README.md @@ -0,0 +1,50 @@ +# gvisor-containerd-shim + +> Note: This shim version is supported only for containerd versions less than +> 1.2. If you are using a containerd version greater than or equal to 1.2, then +> please use `containerd-shim-runsc-v1` (Shim API v1). +> +> This containerd shim is supported only in a best-effort capacity. + +This document describes how to configure and use `gvisor-containerd-shim`. + +## Containerd Configuration + +To use this shim, you must configure `/etc/containerd/config.toml` as follows: + +``` +[plugins.linux] + shim = "/usr/bin/gvisor-containerd-shim" +[plugins.cri.containerd.runtimes.gvisor] + runtime_type = "io.containerd.runtime.v1.linux" + runtime_engine = "/usr/bin/runsc" + runtime_root = "/run/containerd/runsc" +``` + +In order to pick-up the new configuration, you may need to restart containerd: + +```shell +sudo systemctl restart containerd +``` + +## Shim Confguration + +The shim configuration is stored in `/etc/containerd/runsc.toml`. The +configuration file supports two values. + +* `runc_shim`: The path to the runc shim. This is used by + `gvisor-containerd-shim` to run standard containers. + +* `runsc_config`: This is a set of key/value pairs that are converted into + `runsc` command line flags. You can learn more about which flags are available + by running `runsc flags`. + +For example, a configuration might look as follows: + +``` +runc_shim = "/usr/local/bin/containerd-shim" +[runsc_config] +platform = "kvm" +debug = true +debug-log = /var/log/%ID%/gvisor.log +``` diff --git a/shim/v2/config.go b/shim/v1/config.go index a72cc7754..a72cc7754 100644 --- a/shim/v2/config.go +++ b/shim/v1/config.go diff --git a/shim/v1/main.go b/shim/v1/main.go index 41c77394a..43deee858 100644 --- a/shim/v1/main.go +++ b/shim/v1/main.go @@ -16,11 +16,252 @@ package main import ( - "github.com/containerd/containerd/runtime/v2/shim" + "bytes" + "context" + "flag" + "fmt" + "log" + "net" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strings" + "sync" + "syscall" - runsc "gvisor.dev/gvisor/pkg/shim/v2" + "github.com/containerd/containerd/events" + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/runtime/v1/linux/proc" + containerdshim "github.com/containerd/containerd/runtime/v1/shim" + shimapi "github.com/containerd/containerd/runtime/v1/shim/v1" + "github.com/containerd/ttrpc" + "github.com/containerd/typeurl" + ptypes "github.com/gogo/protobuf/types" + "github.com/opencontainers/runc/libcontainer/system" + "golang.org/x/sys/unix" + + "gvisor.dev/gvisor/pkg/shim/runsc" + "gvisor.dev/gvisor/pkg/shim/v1/shim" +) + +var ( + debugFlag bool + namespaceFlag string + socketFlag string + addressFlag string + workdirFlag string + runtimeRootFlag string + containerdBinaryFlag string + shimConfigFlag string ) +func init() { + flag.BoolVar(&debugFlag, "debug", false, "enable debug output in logs") + flag.StringVar(&namespaceFlag, "namespace", "", "namespace that owns the shim") + flag.StringVar(&socketFlag, "socket", "", "abstract socket path to serve") + flag.StringVar(&addressFlag, "address", "", "grpc address back to main containerd") + flag.StringVar(&workdirFlag, "workdir", "", "path used to storge large temporary data") + + // Containerd default to runc, unless another runtime is explicitly + // specified. We keep the same default to make the default behavior + // consistent. + flag.StringVar(&runtimeRootFlag, "runtime-root", proc.RuncRoot, "root directory for the runtime") + + // Currently, the `containerd publish` utility is embedded in the + // daemon binary. The daemon invokes `containerd-shim + // -containerd-binary ...` with its own os.Executable() path. + flag.StringVar(&containerdBinaryFlag, "containerd-binary", "containerd", "path to containerd binary (used for `containerd publish`)") + flag.StringVar(&shimConfigFlag, "config", "/etc/containerd/runsc.toml", "path to the shim configuration file") +} + func main() { - shim.Run("io.containerd.runsc.v1", runsc.New) + flag.Parse() + + // This is a hack. Exec current process to run standard containerd-shim + // if runtime root is not `runsc`. We don't need this for shim v2 api. + if filepath.Base(runtimeRootFlag) != "runsc" { + if err := executeRuncShim(); err != nil { + fmt.Fprintf(os.Stderr, "gvisor-containerd-shim: %s\n", err) + os.Exit(1) + } + } + + // Run regular shim if needed. + if err := executeShim(); err != nil { + fmt.Fprintf(os.Stderr, "gvisor-containerd-shim: %s\n", err) + os.Exit(1) + } +} + +// executeRuncShim execs current process to a containerd-shim process and +// retains all flags and envs. +func executeRuncShim() error { + c, err := loadConfig(shimConfigFlag) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to load shim config: %w", err) + } + shimPath := c.RuncShim + if shimPath == "" { + shimPath, err = exec.LookPath("containerd-shim") + if err != nil { + return fmt.Errorf("lookup containerd-shim failed: %w", err) + } + } + + args := append([]string{shimPath}, os.Args[1:]...) + if err := syscall.Exec(shimPath, args, os.Environ()); err != nil { + return fmt.Errorf("exec containerd-shim @ %q failed: %w", shimPath, err) + } + return nil +} + +func executeShim() error { + // start handling signals as soon as possible so that things are + // properly reaped or if runtime exits before we hit the handler. + signals, err := setupSignals() + if err != nil { + return err + } + path, err := os.Getwd() + if err != nil { + return err + } + server, err := ttrpc.NewServer(ttrpc.WithServerHandshaker(ttrpc.UnixSocketRequireSameUser())) + if err != nil { + return fmt.Errorf("failed creating server: %w", err) + } + c, err := loadConfig(shimConfigFlag) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to load shim config: %w", err) + } + sv, err := shim.NewService( + shim.Config{ + Path: path, + Namespace: namespaceFlag, + WorkDir: workdirFlag, + RuntimeRoot: runtimeRootFlag, + RunscConfig: c.RunscConfig, + }, + &remoteEventsPublisher{address: addressFlag}, + ) + if err != nil { + return err + } + shimapi.RegisterShimService(server, sv) + if err := serve(server, socketFlag); err != nil { + return err + } + return handleSignals(signals, server, sv) +} + +// serve serves the ttrpc API over a unix socket at the provided path this +// function does not block. +func serve(server *ttrpc.Server, path string) error { + var ( + l net.Listener + err error + ) + if path == "" { + l, err = net.FileListener(os.NewFile(3, "socket")) + path = "[inherited from parent]" + } else { + if len(path) > 106 { + return fmt.Errorf("%q: unix socket path too long (> 106)", path) + } + l, err = net.Listen("unix", "\x00"+path) + } + if err != nil { + return err + } + go func() { + defer l.Close() + err := server.Serve(context.Background(), l) + if err != nil && !strings.Contains(err.Error(), "use of closed network connection") { + log.Fatalf("ttrpc server failure: %v", err) + } + }() + return nil +} + +// setupSignals creates a new signal handler for all signals and sets the shim +// as a sub-reaper so that the container processes are reparented. +func setupSignals() (chan os.Signal, error) { + signals := make(chan os.Signal, 32) + signal.Notify(signals, unix.SIGTERM, unix.SIGINT, unix.SIGCHLD, unix.SIGPIPE) + // make sure runc is setup to use the monitor for waiting on processes. + // TODO(random-liu): Move shim/reaper.go to a separate package. + runsc.Monitor = containerdshim.Default + // Set the shim as the subreaper for all orphaned processes created by + // the container. + if err := system.SetSubreaper(1); err != nil { + return nil, err + } + return signals, nil +} + +func handleSignals(signals chan os.Signal, server *ttrpc.Server, sv *shim.Service) error { + var ( + termOnce sync.Once + done = make(chan struct{}) + ) + + for { + select { + case <-done: + return nil + case s := <-signals: + switch s { + case unix.SIGCHLD: + if err := containerdshim.Reap(); err != nil { + log.Printf("reap exit status: %v") + } + case unix.SIGTERM, unix.SIGINT: + go termOnce.Do(func() { + ctx := context.TODO() + if err := server.Shutdown(ctx); err != nil { + log.Printf("failed to shutdown server: %v") + } + // Ensure our child is dead if any. + sv.Kill(ctx, &shimapi.KillRequest{ + Signal: uint32(syscall.SIGKILL), + All: true, + }) + sv.Delete(context.Background(), &ptypes.Empty{}) + close(done) + }) + case unix.SIGPIPE: + } + } + } +} + +type remoteEventsPublisher struct { + address string +} + +func (l *remoteEventsPublisher) Publish(ctx context.Context, topic string, event events.Event) error { + ns, _ := namespaces.Namespace(ctx) + encoded, err := typeurl.MarshalAny(event) + if err != nil { + return err + } + data, err := encoded.Marshal() + if err != nil { + return err + } + cmd := exec.CommandContext(ctx, containerdBinaryFlag, "--address", l.address, "publish", "--topic", topic, "--namespace", ns) + cmd.Stdin = bytes.NewReader(data) + c, err := containerdshim.Default.Start(cmd) + if err != nil { + return err + } + status, err := containerdshim.Default.Wait(cmd, c) + if err != nil { + return fmt.Errorf("failed to publish event: %w", err) + } + if status != 0 { + return fmt.Errorf("failed to publish event: status %d", status) + } + return nil } diff --git a/shim/v2/BUILD b/shim/v2/BUILD new file mode 100644 index 000000000..1e1947dab --- /dev/null +++ b/shim/v2/BUILD @@ -0,0 +1,32 @@ +load("//tools:defs.bzl", "go_binary", "pkg_tar") +load("//website:defs.bzl", "doc") + +package(licenses = ["notice"]) + +go_binary( + name = "containerd-shim-runsc-v1", + srcs = [ + "main.go", + ], + pure = True, + visibility = [ + "//visibility:public", + ], + deps = [ + "//pkg/shim/runsc", + "//pkg/shim/v1/shim", + "//pkg/shim/v2", + "@com_github_burntsushi_toml//:go_default_library", + "@com_github_containerd_containerd//runtime/v2/shim:go_default_library", + ], +) + +doc( + name = "doc", + src = "README.md", + category = "User Guide", + permalink = "/docs/user_guide/containerd-shim-runsc-v1/", + subcategory = "Advanced", + visibility = ["//website:__pkg__"], + weight = "92", +) diff --git a/shim/v2/README.md b/shim/v2/README.md new file mode 100644 index 000000000..2fd625415 --- /dev/null +++ b/shim/v2/README.md @@ -0,0 +1,90 @@ +# containerd-shim-runsc-v1 + +> Note: This shim version is the recommended shim for containerd versions +> greater than or equal to 1.2. For older versions of containerd, use +> `gvisor-containerd-shim`. + +This document describes how to configure and use `containerd-shim-runsc-v1`. + +## Configuring Containerd 1.2 + +To configure containerd 1.2 to use this shim, add the runtime to +`/etc/containerd/config.toml` as follows: + +``` +[plugins.cri.containerd.runtimes.runsc] + runtime_type = "io.containerd.runsc.v1" + runtime_root = "/run/containerd/runsc" +[plugins.cri.containerd.runtimes.runsc.options] + TypeUrl = "io.containerd.runsc.v1.options" +``` + +The configuration will optionally loaded from a file named `config.toml` in the +`runtime_root` configured above. + +In order to pick up the new configuration, you may need to restart containerd: + +```shell +sudo systemctl restart containerd +``` + +## Configuring Containerd 1.3 and above + +To configure containerd 1.3 to use this shim, add the runtime to +`/etc/containerd/config.toml` as follows: + +``` +[plugins.cri.containerd.runtimes.runsc] + runtime_type = "io.containerd.runsc.v1" +[plugins.cri.containerd.runtimes.runsc.options] + TypeUrl = "io.containerd.runsc.v1.options" + ConfigPath = "/etc/containerd/runsc.toml" +``` + +The `ConfigPath` above will be used to provide a pointer to the configuration +file to be loaded. + +> Note that there will be configuration file loaded if `ConfigPath` is not set. + +In order to pick up the new configuration, you may need to restart containerd: + +```shell +sudo systemctl restart containerd +``` +## Shim Confguration + +The shim configuration may carry the following options: + +* `shim_cgroup`: The cgroup to use for the shim itself. +* `io_uid`: The UID to use for pipes. +* `ui_gid`: The GID to use for pipes. +* `binary_name`: The runtime binary name (defaults to `runsc`). +* `root`: The root directory for the runtime. +* `runsc_config`: A dictionary of key-value pairs that will be passed to the + runtime as arguments. + +### Example: Enable the KVM platform + +gVisor enables the use of a number of platforms. This example shows how to +configure `containerd-shim-runsc-v1` to use gVisor with the KVM platform: + +```shell +cat <<EOF | sudo tee /etc/containerd/runsc.toml +[runsc_config] +platform = "kvm" +EOF +``` + +### Example: Enable gVisor debug logging + +gVisor debug logging can be enabled by setting the `debug` and `debug-log` flag. +The shim will replace "%ID%" with the container ID in the path of the +`debug-log` flag. + +```shell +cat <<EOF | sudo tee /etc/containerd/runsc.toml +[runsc_config] +debug = true +debug-log = /var/log/%ID%/gvisor.log +EOF +``` diff --git a/shim/v2/main.go b/shim/v2/main.go index 118b6f6e5..41c77394a 100644 --- a/shim/v2/main.go +++ b/shim/v2/main.go @@ -16,303 +16,11 @@ package main import ( - "bytes" - "context" - "flag" - "fmt" - "net" - "os" - "os/exec" - "os/signal" - "path/filepath" - "runtime" - "runtime/debug" - "strings" - "sync" - "syscall" - "time" + "github.com/containerd/containerd/runtime/v2/shim" - "github.com/containerd/containerd/events" - "github.com/containerd/containerd/namespaces" - "github.com/containerd/containerd/runtime/v1/linux/proc" - containerdshim "github.com/containerd/containerd/runtime/v1/shim" - shimapi "github.com/containerd/containerd/runtime/v1/shim/v1" - "github.com/containerd/ttrpc" - "github.com/containerd/typeurl" - ptypes "github.com/gogo/protobuf/types" - "github.com/opencontainers/runc/libcontainer/system" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" - - "gvisor.dev/gvisor/pkg/shim/runsc" - "gvisor.dev/gvisor/pkg/shim/v1/shim" -) - -var ( - debugFlag bool - namespaceFlag string - socketFlag string - addressFlag string - workdirFlag string - runtimeRootFlag string - containerdBinaryFlag string - shimConfigFlag string + runsc "gvisor.dev/gvisor/pkg/shim/v2" ) -// ShimConfigPath is the default shim config file path. -const ShimConfigPath = "/etc/containerd/gvisor-containerd-shim.toml" - -func init() { - flag.BoolVar(&debugFlag, "debug", false, "enable debug output in logs") - flag.StringVar(&namespaceFlag, "namespace", "", "namespace that owns the shim") - flag.StringVar(&socketFlag, "socket", "", "abstract socket path to serve") - flag.StringVar(&addressFlag, "address", "", "grpc address back to main containerd") - flag.StringVar(&workdirFlag, "workdir", "", "path used to storge large temporary data") - // Containerd default to runc, unless another runtime is explicitly specified. - // We keep the same default to make the default behavior consistent. - flag.StringVar(&runtimeRootFlag, "runtime-root", proc.RuncRoot, "root directory for the runtime") - // currently, the `containerd publish` utility is embedded in the daemon binary. - // The daemon invokes `containerd-shim -containerd-binary ...` with its own os.Executable() path. - flag.StringVar(&containerdBinaryFlag, "containerd-binary", "containerd", "path to containerd binary (used for `containerd publish`)") - flag.StringVar(&shimConfigFlag, "config", ShimConfigPath, "path to the shim configuration file") - flag.Parse() -} - func main() { - // This is a hack. Exec current process to run standard containerd-shim - // if runtime root is not `runsc`. We don't need this for shim v2 api. - if filepath.Base(runtimeRootFlag) != "runsc" { - if err := executeRuncShim(); err != nil { - fmt.Fprintf(os.Stderr, "gvisor-containerd-shim: %s\n", err) - os.Exit(1) - } - } - - debug.SetGCPercent(40) - go func() { - for range time.Tick(30 * time.Second) { - debug.FreeOSMemory() - } - }() - - if debugFlag { - logrus.SetLevel(logrus.DebugLevel) - } - - if os.Getenv("GOMAXPROCS") == "" { - // If GOMAXPROCS hasn't been set, we default to a value of 2 to reduce - // the number of Go stacks present in the shim. - runtime.GOMAXPROCS(2) - } - - // Run regular shim if needed. - if err := executeShim(); err != nil { - fmt.Fprintf(os.Stderr, "gvisor-containerd-shim: %s\n", err) - os.Exit(1) - } -} - -// executeRuncShim execs current process to a containerd-shim process and -// retains all flags and envs. -func executeRuncShim() error { - c, err := loadConfig(shimConfigFlag) - if err != nil && !os.IsNotExist(err) { - return errors.Wrap(err, "failed to load shim config") - } - shimPath := c.RuncShim - if shimPath == "" { - shimPath, err = exec.LookPath("containerd-shim") - if err != nil { - return errors.Wrapf(err, "lookup containerd-shim") - } - } - - args := append([]string{shimPath}, os.Args[1:]...) - if err := syscall.Exec(shimPath, args, os.Environ()); err != nil { - return errors.Wrapf(err, "exec containerd-shim %q", shimPath) - } - return nil -} - -func executeShim() error { - // start handling signals as soon as possible so that things are properly reaped - // or if runtime exits before we hit the handler - signals, err := setupSignals() - if err != nil { - return err - } - dump := make(chan os.Signal, 32) - signal.Notify(dump, syscall.SIGUSR1) - - path, err := os.Getwd() - if err != nil { - return err - } - server, err := ttrpc.NewServer(ttrpc.WithServerHandshaker(ttrpc.UnixSocketRequireSameUser())) - if err != nil { - return errors.Wrap(err, "failed creating server") - } - c, err := loadConfig(shimConfigFlag) - if err != nil && !os.IsNotExist(err) { - return errors.Wrap(err, "failed to load shim config") - } - sv, err := shim.NewService( - shim.Config{ - Path: path, - Namespace: namespaceFlag, - WorkDir: workdirFlag, - RuntimeRoot: runtimeRootFlag, - RunscConfig: c.RunscConfig, - }, - &remoteEventsPublisher{address: addressFlag}, - ) - if err != nil { - return err - } - logrus.Debug("registering ttrpc server") - shimapi.RegisterShimService(server, sv) - - socket := socketFlag - if err := serve(server, socket); err != nil { - return err - } - logger := logrus.WithFields(logrus.Fields{ - "pid": os.Getpid(), - "path": path, - "namespace": namespaceFlag, - }) - go func() { - for range dump { - dumpStacks(logger) - } - }() - return handleSignals(logger, signals, server, sv) -} - -// serve serves the ttrpc API over a unix socket at the provided path -// this function does not block -func serve(server *ttrpc.Server, path string) error { - var ( - l net.Listener - err error - ) - if path == "" { - l, err = net.FileListener(os.NewFile(3, "socket")) - path = "[inherited from parent]" - } else { - if len(path) > 106 { - return errors.Errorf("%q: unix socket path too long (> 106)", path) - } - l, err = net.Listen("unix", "\x00"+path) - } - if err != nil { - return err - } - logrus.WithField("socket", path).Debug("serving api on unix socket") - go func() { - defer l.Close() - if err := server.Serve(context.Background(), l); err != nil && - !strings.Contains(err.Error(), "use of closed network connection") { - logrus.WithError(err).Fatal("gvisor-containerd-shim: ttrpc server failure") - } - }() - return nil -} - -// setupSignals creates a new signal handler for all signals and sets the shim as a -// sub-reaper so that the container processes are reparented -func setupSignals() (chan os.Signal, error) { - signals := make(chan os.Signal, 32) - signal.Notify(signals, unix.SIGTERM, unix.SIGINT, unix.SIGCHLD, unix.SIGPIPE) - // make sure runc is setup to use the monitor - // for waiting on processes - // TODO(random-liu): Move shim/reaper.go to a separate package. - runsc.Monitor = containerdshim.Default - // set the shim as the subreaper for all orphaned processes created by the container - if err := system.SetSubreaper(1); err != nil { - return nil, err - } - return signals, nil -} - -func handleSignals(logger *logrus.Entry, signals chan os.Signal, server *ttrpc.Server, sv *shim.Service) error { - var ( - termOnce sync.Once - done = make(chan struct{}) - ) - - for { - select { - case <-done: - return nil - case s := <-signals: - switch s { - case unix.SIGCHLD: - if err := containerdshim.Reap(); err != nil { - logger.WithError(err).Error("reap exit status") - } - case unix.SIGTERM, unix.SIGINT: - go termOnce.Do(func() { - ctx := context.TODO() - if err := server.Shutdown(ctx); err != nil { - logger.WithError(err).Error("failed to shutdown server") - } - // Ensure our child is dead if any - sv.Kill(ctx, &shimapi.KillRequest{ - Signal: uint32(syscall.SIGKILL), - All: true, - }) - sv.Delete(context.Background(), &ptypes.Empty{}) - close(done) - }) - case unix.SIGPIPE: - } - } - } -} - -func dumpStacks(logger *logrus.Entry) { - var ( - buf []byte - stackSize int - ) - bufferLen := 16384 - for stackSize == len(buf) { - buf = make([]byte, bufferLen) - stackSize = runtime.Stack(buf, true) - bufferLen *= 2 - } - buf = buf[:stackSize] - logger.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf) -} - -type remoteEventsPublisher struct { - address string -} - -func (l *remoteEventsPublisher) Publish(ctx context.Context, topic string, event events.Event) error { - ns, _ := namespaces.Namespace(ctx) - encoded, err := typeurl.MarshalAny(event) - if err != nil { - return err - } - data, err := encoded.Marshal() - if err != nil { - return err - } - cmd := exec.CommandContext(ctx, containerdBinaryFlag, "--address", l.address, "publish", "--topic", topic, "--namespace", ns) - cmd.Stdin = bytes.NewReader(data) - c, err := containerdshim.Default.Start(cmd) - if err != nil { - return err - } - status, err := containerdshim.Default.Wait(cmd, c) - if err != nil { - return err - } - if status != 0 { - return errors.New("failed to publish event") - } - return nil + shim.Run("io.containerd.runsc.v1", runsc.New) } diff --git a/shim/runtime-handler-shim-v2-quickstart.md b/shim/v2/runtime-handler-shim-v2-quickstart.md index ca8336089..ca8336089 100644 --- a/shim/runtime-handler-shim-v2-quickstart.md +++ b/shim/v2/runtime-handler-shim-v2-quickstart.md |