diff options
Diffstat (limited to 'shim')
| -rw-r--r-- | shim/README.md | 16 | ||||
| -rw-r--r-- | shim/configure-containerd-shim-runsc-v1.md | 72 | ||||
| -rw-r--r-- | shim/configure-gvisor-containerd-shim.md | 42 | ||||
| -rw-r--r-- | shim/runtime-handler-quickstart.md | 251 | ||||
| -rw-r--r-- | shim/runtime-handler-shim-v2-quickstart.md | 232 | ||||
| -rw-r--r-- | shim/untrusted-workload-quickstart.md | 212 | ||||
| -rw-r--r-- | shim/v1/main.go | 26 | ||||
| -rw-r--r-- | shim/v2/config.go | 40 | ||||
| -rw-r--r-- | shim/v2/main.go | 318 |
9 files changed, 1209 insertions, 0 deletions
diff --git a/shim/README.md b/shim/README.md new file mode 100644 index 000000000..e446ec970 --- /dev/null +++ b/shim/README.md @@ -0,0 +1,16 @@ +# gvisor-containerd-shim + +gvisor-containerd-shim is a containerd shim. It implements the containerd v1 +shim API. It can be used as a drop-in replacement for +[containerd-shim][containerd-shim] +(though containerd-shim must still be installed). It allows the use of both +gVisor (runsc) and normal containers in the same containerd installation by +deferring to the runc shim if the desired runtime engine is not runsc. + +- [Untrusted Workload Quick Start (containerd >=1.1)](docs/untrusted-workload-quickstart.md) +- [Runtime Handler/RuntimeClass Quick Start (containerd >=1.2)](docs/runtime-handler-quickstart.md) +- [Runtime Handler/RuntimeClass Quick Start (shim v2) (containerd >=1.2)](docs/runtime-handler-shim-v2-quickstart.md) +- [Configure containerd-shim-runsc-v1 (shim v2) (containerd >= 1.3)](docs/configure-containerd-shim-runsc-v1.md) +- [Configure gvisor-containerd-shim (shim v1) (containerd <= 1.2)](docs/configure-gvisor-containerd-shim.md) + +[containerd-shim]: https://github.com/containerd/containerd/tree/master/cmd/containerd-shim diff --git a/shim/configure-containerd-shim-runsc-v1.md b/shim/configure-containerd-shim-runsc-v1.md new file mode 100644 index 000000000..977ceacbd --- /dev/null +++ b/shim/configure-containerd-shim-runsc-v1.md @@ -0,0 +1,72 @@ +# Configure containerd-shim-runsc-v1 (Shim V2) + +This document describes how to configure runtime options for +`containerd-shim-runsc-v1`. This is follows on to the instructions of +[Runtime Handler Quick Start (shim v2) (containerd >=1.2)](runtime-handler-shim-v2-quickstart.md) +and requires containerd 1.3 or later. + +### Update `/etc/containerd/config.toml` to point to a configuration file for `containerd-shim-runsc-v1`. + +`containerd-shim-runsc-v1` supports a few different configuration options based +on the version of containerd that is used. For versions >= 1.3, it supports a +configurable config path in the containerd runtime configuration. + +```shell +{ # Step 1: Update runtime options for runsc in containerd config.toml +cat <<EOF | sudo tee /etc/containerd/config.toml +disabled_plugins = ["restart"] +[plugins.linux] + shim_debug = true +[plugins.cri.containerd.runtimes.runsc] + runtime_type = "io.containerd.runsc.v1" +[plugins.cri.containerd.runtimes.runsc.options] + TypeUrl = "io.containerd.runsc.v1.options" + ConfigPath = "/etc/containerd/runsc.toml" +EOF +} +``` + +### Configure `/etc/containerd/runsc.toml` + +The set of options that can be configured can be found in +[options.go](../pkg/v2/options/options.go). + +#### Example: Enable the KVM platform + +gVisor enables the use of a number of platforms. This example shows how to +configure `containerd-shim-runsc-v1` to use gvisor with the KVM platform. + +Find out more about platform in the +(gVisor documentation)[https://gvisor.dev/docs/user_guide/platforms/]. + +```shell +cat <<EOF | sudo tee /etc/containerd/runsc.toml +[runsc_config] +platform = "kvm" +EOF +``` + +### Example: Enable gVisor debug logging + +gVisor debug logging can be enabled by setting the `debug` and `debug-log` +flag. The shim will replace "%ID%" with the container ID in the path of the +`debug-log` flag. + +Find out more about debugging in the +(gVisor documentation)[https://gvisor.dev/docs/user_guide/debugging/]. + +```shell +cat <<EOF | sudo tee /etc/containerd/runsc.toml +[runsc_config] + debug=true + debug-log=/var/log/%ID%/gvisor.log +EOF +``` + +## Restart `containerd` + +When you are done restart containerd to pick up the new configuration files. + +```shell +sudo systemctl restart containerd +``` diff --git a/shim/configure-gvisor-containerd-shim.md b/shim/configure-gvisor-containerd-shim.md new file mode 100644 index 000000000..40151da56 --- /dev/null +++ b/shim/configure-gvisor-containerd-shim.md @@ -0,0 +1,42 @@ +# Configure gvisor-containerd-shim (Shim V1) + +This document describes how to configure runtime options for `gvisor-containerd-shim`. + +The shim configuration is stored in `/etc/containerd/gvisor-containerd-shim.toml`. The configuration file supports two values. + +`runc_shim`: The path to the runc shim. This is used by the gvisor-containerd-shim to run normal containers. +`runsc_config`: This is a set of key/value pairs that are converted into `runsc` command line flags. You can learn more about which flags are available by running `runsc flags`. + +## Example: Enable the KVM platform + +gVisor enables the use of a number of platforms. This configuration enables the +KVM platform. + +Find out more about platform in the +(gVisor documentation)[https://gvisor.dev/docs/user_guide/platforms/]. + +```shell +cat <<EOF | sudo tee /etc/containerd/gvisor-containerd-shim.toml +[runsc_config] +platform = "kvm" +EOF +``` + +## Example: Enable gVisor debug logging + +gVisor debug logging can be enabled by setting the `debug` and `debug-log` +flag. The shim will replace "%ID%" with the container ID in the path of the +`debug-log` flag. + +Find out more about debugging in the +(gVisor documentation)[https://gvisor.dev/docs/user_guide/debugging/]. + +```shell +cat <<EOF | sudo tee /etc/containerd/gvisor-containerd-shim.toml +# This is the path to the default runc containerd-shim. +runc_shim = "/usr/local/bin/containerd-shim" +[runsc_config] + debug=true + debug-log=/var/log/%ID%/gvisor.log +EOF +``` diff --git a/shim/runtime-handler-quickstart.md b/shim/runtime-handler-quickstart.md new file mode 100644 index 000000000..684390b55 --- /dev/null +++ b/shim/runtime-handler-quickstart.md @@ -0,0 +1,251 @@ +# Runtime Handler Quickstart + +This document describes how to install and run the `gvisor-containerd-shim` +using the containerd runtime handler support. This requires containerd 1.2 or +later. + +## Requirements + +- **runsc**: See the [gVisor documentation](https://github.com/google/gvisor) for information on how to install runsc. +- **containerd**: See the [containerd website](https://containerd.io/) for information on how to install containerd. + +## Install + +### Install gvisor-containerd-shim + +1. Download the latest release of the `gvisor-containerd-shim`. See the + [releases page](https://github.com/google/gvisor-containerd-shim/releases) + +[embedmd]:# (../test/e2e/shim-install.sh shell /{ # Step 1\(release\)/ /^}/) +```shell +{ # Step 1(release): Install gvisor-containerd-shim +LATEST_RELEASE=$(wget -qO - https://api.github.com/repos/google/gvisor-containerd-shim/releases | grep -oP '(?<="browser_download_url": ")https://[^"]*gvisor-containerd-shim.linux-amd64' | head -1) +wget -O gvisor-containerd-shim ${LATEST_RELEASE} +chmod +x gvisor-containerd-shim +sudo mv gvisor-containerd-shim /usr/local/bin/gvisor-containerd-shim +} +``` + +2. Create the configuration for the gvisor shim in + `/etc/containerd/gvisor-containerd-shim.toml`: + +[embedmd]:# (../test/e2e/shim-install.sh shell /{ # Step 2/ /^}/) +```shell +{ # Step 2: Create the gvisor-containerd-shim.toml +cat <<EOF | sudo tee /etc/containerd/gvisor-containerd-shim.toml +# This is the path to the default runc containerd-shim. +runc_shim = "/usr/local/bin/containerd-shim" +EOF +} +``` + +### Configure containerd + +1. Update `/etc/containerd/config.toml`. Be sure to update the path to + `gvisor-containerd-shim` and `runsc` if necessary: + +[embedmd]:# (../test/e2e/runtime-handler/install.sh shell /{ # Step 1/ /^}/) +```shell +{ # Step 1: Create containerd config.toml +cat <<EOF | sudo tee /etc/containerd/config.toml +disabled_plugins = ["restart"] +[plugins.linux] + shim = "/usr/local/bin/gvisor-containerd-shim" + shim_debug = true +[plugins.cri.containerd.runtimes.runsc] + runtime_type = "io.containerd.runtime.v1.linux" + runtime_engine = "/usr/local/bin/runsc" + runtime_root = "/run/containerd/runsc" +EOF +} +``` + +2. Restart `containerd` + +```shell +sudo systemctl restart containerd +``` + +## Usage + +You can run containers in gVisor via containerd's CRI. + +### Install crictl + +1. Download and install the crictl binary: + +[embedmd]:# (../test/e2e/crictl-install.sh shell /{ # Step 1/ /^}/) +```shell +{ # Step 1: Download crictl +wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.13.0/crictl-v1.13.0-linux-amd64.tar.gz +tar xf crictl-v1.13.0-linux-amd64.tar.gz +sudo mv crictl /usr/local/bin +} +``` + +2. Write the crictl configuration file + +[embedmd]:# (../test/e2e/crictl-install.sh shell /{ # Step 2/ /^}/) +```shell +{ # Step 2: Configure crictl +cat <<EOF | sudo tee /etc/crictl.yaml +runtime-endpoint: unix:///run/containerd/containerd.sock +EOF +} +``` + +### Create the nginx Sandbox in gVisor + +1. Pull the nginx image + +[embedmd]:# (../test/e2e/runtime-handler/usage.sh shell /{ # Step 1/ /^}/) +```shell +{ # Step 1: Pull the nginx image +sudo crictl pull nginx +} +``` + +2. Create the sandbox creation request + +[embedmd]:# (../test/e2e/runtime-handler/usage.sh shell /{ # Step 2/ /^EOF\n}/) +```shell +{ # Step 2: Create sandbox.json +cat <<EOF | tee sandbox.json +{ + "metadata": { + "name": "nginx-sandbox", + "namespace": "default", + "attempt": 1, + "uid": "hdishd83djaidwnduwk28bcsb" + }, + "linux": { + }, + "log_directory": "/tmp" +} +EOF +} +``` + +3. Create the pod in gVisor + +[embedmd]:# (../test/e2e/runtime-handler/usage.sh shell /{ # Step 3/ /^}/) +```shell +{ # Step 3: Create the sandbox +SANDBOX_ID=$(sudo crictl runp --runtime runsc sandbox.json) +} +``` + +### Run the nginx Container in the Sandbox + +1. Create the nginx container creation request + +[embedmd]:# (../test/e2e/run-container.sh shell /{ # Step 1/ /^EOF\n}/) +```shell +{ # Step 1: Create nginx container config +cat <<EOF | tee container.json +{ + "metadata": { + "name": "nginx" + }, + "image":{ + "image": "nginx" + }, + "log_path":"nginx.0.log", + "linux": { + } +} +EOF +} +``` + +2. Create the nginx container + +[embedmd]:# (../test/e2e/run-container.sh shell /{ # Step 2/ /^}/) +```shell +{ # Step 2: Create nginx container +CONTAINER_ID=$(sudo crictl create ${SANDBOX_ID} container.json sandbox.json) +} +``` + +3. Start the nginx container + +[embedmd]:# (../test/e2e/run-container.sh shell /{ # Step 3/ /^}/) +```shell +{ # Step 3: Start nginx container +sudo crictl start ${CONTAINER_ID} +} +``` + +### Validate the container + +1. Inspect the created pod + +[embedmd]:# (../test/e2e/validate.sh shell /{ # Step 1/ /^}/) +```shell +{ # Step 1: Inspect the pod +sudo crictl inspectp ${SANDBOX_ID} +} +``` + +2. Inspect the nginx container + +[embedmd]:# (../test/e2e/validate.sh shell /{ # Step 2/ /^}/) +```shell +{ # Step 2: Inspect the container +sudo crictl inspect ${CONTAINER_ID} +} +``` + +3. Verify that nginx is running in gVisor + +[embedmd]:# (../test/e2e/validate.sh shell /{ # Step 3/ /^}/) +```shell +{ # Step 3: Check dmesg +sudo crictl exec ${CONTAINER_ID} dmesg | grep -i gvisor +} +``` + +### Set up the Kubernetes Runtime Class + +1. Install the Runtime Class for gVisor + +[embedmd]:# (../test/e2e/runtimeclass-install.sh shell /{ # Step 1/ /^}/) +```shell +{ # Step 1: Install a RuntimeClass +cat <<EOF | kubectl apply -f - +apiVersion: node.k8s.io/v1beta1 +kind: RuntimeClass +metadata: + name: gvisor +handler: runsc +EOF +} +``` + +2. Create a Pod with the gVisor Runtime Class + +[embedmd]:# (../test/e2e/runtimeclass-install.sh shell /{ # Step 2/ /^}/) +```shell +{ # Step 2: Create a pod +cat <<EOF | kubectl apply -f - +apiVersion: v1 +kind: Pod +metadata: + name: nginx-gvisor +spec: + runtimeClassName: gvisor + containers: + - name: nginx + image: nginx +EOF +} +``` + +3. Verify that the Pod is running + +[embedmd]:# (../test/e2e/runtimeclass-install.sh shell /{ # Step 3/ /^}/) +```shell +{ # Step 3: Get the pod +kubectl get pod nginx-gvisor -o wide +} +``` diff --git a/shim/runtime-handler-shim-v2-quickstart.md b/shim/runtime-handler-shim-v2-quickstart.md new file mode 100644 index 000000000..ca8336089 --- /dev/null +++ b/shim/runtime-handler-shim-v2-quickstart.md @@ -0,0 +1,232 @@ +# Runtime Handler Quickstart (Shim V2) + +This document describes how to install and run `containerd-shim-runsc-v1` using +the containerd runtime handler support. This requires containerd 1.2 or later. + +## Requirements + +- **runsc**: See the [gVisor documentation](https://github.com/google/gvisor) for information on how to install runsc. +- **containerd**: See the [containerd website](https://containerd.io/) for information on how to install containerd. + +## Install + +### Install containerd-shim-runsc-v1 + +1. Build and install `containerd-shim-runsc-v1`. + +<!-- TODO: Use a release once we have one available. --> +[embedmd]:# (../test/e2e/shim-install.sh shell /{ # Step 1\(dev\)/ /^}/) +```shell +{ # Step 1(dev): Build and install gvisor-containerd-shim and containerd-shim-runsc-v1 + make + sudo make install +} +``` + +### Configure containerd + +1. Update `/etc/containerd/config.toml`. Make sure `containerd-shim-runsc-v1` is + in `${PATH}`. + +[embedmd]:# (../test/e2e/runtime-handler-shim-v2/install.sh shell /{ # Step 1/ /^}/) +```shell +{ # Step 1: Create containerd config.toml +cat <<EOF | sudo tee /etc/containerd/config.toml +disabled_plugins = ["restart"] +[plugins.linux] + shim_debug = true +[plugins.cri.containerd.runtimes.runsc] + runtime_type = "io.containerd.runsc.v1" +EOF +} +``` + +2. Restart `containerd` + +```shell +sudo systemctl restart containerd +``` + +## Usage + +You can run containers in gVisor via containerd's CRI. + +### Install crictl + +1. Download and install the crictl binary: + +[embedmd]:# (../test/e2e/crictl-install.sh shell /{ # Step 1/ /^}/) +```shell +{ # Step 1: Download crictl +wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.13.0/crictl-v1.13.0-linux-amd64.tar.gz +tar xf crictl-v1.13.0-linux-amd64.tar.gz +sudo mv crictl /usr/local/bin +} +``` + +2. Write the crictl configuration file + +[embedmd]:# (../test/e2e/crictl-install.sh shell /{ # Step 2/ /^}/) +```shell +{ # Step 2: Configure crictl +cat <<EOF | sudo tee /etc/crictl.yaml +runtime-endpoint: unix:///run/containerd/containerd.sock +EOF +} +``` + +### Create the nginx Sandbox in gVisor + +1. Pull the nginx image + +[embedmd]:# (../test/e2e/runtime-handler/usage.sh shell /{ # Step 1/ /^}/) +```shell +{ # Step 1: Pull the nginx image +sudo crictl pull nginx +} +``` + +2. Create the sandbox creation request + +[embedmd]:# (../test/e2e/runtime-handler/usage.sh shell /{ # Step 2/ /^EOF\n}/) +```shell +{ # Step 2: Create sandbox.json +cat <<EOF | tee sandbox.json +{ + "metadata": { + "name": "nginx-sandbox", + "namespace": "default", + "attempt": 1, + "uid": "hdishd83djaidwnduwk28bcsb" + }, + "linux": { + }, + "log_directory": "/tmp" +} +EOF +} +``` + +3. Create the pod in gVisor + +[embedmd]:# (../test/e2e/runtime-handler/usage.sh shell /{ # Step 3/ /^}/) +```shell +{ # Step 3: Create the sandbox +SANDBOX_ID=$(sudo crictl runp --runtime runsc sandbox.json) +} +``` + +### Run the nginx Container in the Sandbox + +1. Create the nginx container creation request + +[embedmd]:# (../test/e2e/run-container.sh shell /{ # Step 1/ /^EOF\n}/) +```shell +{ # Step 1: Create nginx container config +cat <<EOF | tee container.json +{ + "metadata": { + "name": "nginx" + }, + "image":{ + "image": "nginx" + }, + "log_path":"nginx.0.log", + "linux": { + } +} +EOF +} +``` + +2. Create the nginx container + +[embedmd]:# (../test/e2e/run-container.sh shell /{ # Step 2/ /^}/) +```shell +{ # Step 2: Create nginx container +CONTAINER_ID=$(sudo crictl create ${SANDBOX_ID} container.json sandbox.json) +} +``` + +3. Start the nginx container + +[embedmd]:# (../test/e2e/run-container.sh shell /{ # Step 3/ /^}/) +```shell +{ # Step 3: Start nginx container +sudo crictl start ${CONTAINER_ID} +} +``` + +### Validate the container + +1. Inspect the created pod + +[embedmd]:# (../test/e2e/validate.sh shell /{ # Step 1/ /^}/) +```shell +{ # Step 1: Inspect the pod +sudo crictl inspectp ${SANDBOX_ID} +} +``` + +2. Inspect the nginx container + +[embedmd]:# (../test/e2e/validate.sh shell /{ # Step 2/ /^}/) +```shell +{ # Step 2: Inspect the container +sudo crictl inspect ${CONTAINER_ID} +} +``` + +3. Verify that nginx is running in gVisor + +[embedmd]:# (../test/e2e/validate.sh shell /{ # Step 3/ /^}/) +```shell +{ # Step 3: Check dmesg +sudo crictl exec ${CONTAINER_ID} dmesg | grep -i gvisor +} +``` + +### Set up the Kubernetes Runtime Class + +1. Install the Runtime Class for gVisor + +[embedmd]:# (../test/e2e/runtimeclass-install.sh shell /{ # Step 1/ /^}/) +```shell +{ # Step 1: Install a RuntimeClass +cat <<EOF | kubectl apply -f - +apiVersion: node.k8s.io/v1beta1 +kind: RuntimeClass +metadata: + name: gvisor +handler: runsc +EOF +} +``` + +2. Create a Pod with the gVisor Runtime Class + +[embedmd]:# (../test/e2e/runtimeclass-install.sh shell /{ # Step 2/ /^}/) +```shell +{ # Step 2: Create a pod +cat <<EOF | kubectl apply -f - +apiVersion: v1 +kind: Pod +metadata: + name: nginx-gvisor +spec: + runtimeClassName: gvisor + containers: + - name: nginx + image: nginx +EOF +} +``` + +3. Verify that the Pod is running + +[embedmd]:# (../test/e2e/runtimeclass-install.sh shell /{ # Step 3/ /^}/) +```shell +{ # Step 3: Get the pod +kubectl get pod nginx-gvisor -o wide +} +``` diff --git a/shim/untrusted-workload-quickstart.md b/shim/untrusted-workload-quickstart.md new file mode 100644 index 000000000..fb4441845 --- /dev/null +++ b/shim/untrusted-workload-quickstart.md @@ -0,0 +1,212 @@ +# Untrusted Workload Quickstart + +This document describes how to install and run the `gvisor-containerd-shim` +using the untrusted workload CRI extension. This requires containerd 1.1 or +later. + +*Note: The untrusted workload CRI extension is deprecated by containerd. If you +are using containerd 1.2, please consider using runtime handler.* + +## Requirements + +- **runsc**: See the [gVisor documentation](https://github.com/google/gvisor) for information on how to install runsc. +- **containerd**: See the [containerd website](https://containerd.io/) for information on how to install containerd. + +## Install + +### Install gvisor-containerd-shim + +1. Download the latest release of the `gvisor-containerd-shim`. See the + [releases page](https://github.com/google/gvisor-containerd-shim/releases) + +[embedmd]:# (../test/e2e/shim-install.sh shell /{ # Step 1/ /^}/) +```shell +{ # Step 1(release): Install gvisor-containerd-shim +LATEST_RELEASE=$(wget -qO - https://api.github.com/repos/google/gvisor-containerd-shim/releases | grep -oP '(?<="browser_download_url": ")https://[^"]*gvisor-containerd-shim.linux-amd64' | head -1) +wget -O gvisor-containerd-shim ${LATEST_RELEASE} +chmod +x gvisor-containerd-shim +sudo mv gvisor-containerd-shim /usr/local/bin/gvisor-containerd-shim +} +``` + +2. Create the configuration for the gvisor shim in + `/etc/containerd/gvisor-containerd-shim.toml`: + +[embedmd]:# (../test/e2e/shim-install.sh shell /{ # Step 2/ /^}/) +```shell +{ # Step 2: Create the gvisor-containerd-shim.toml +cat <<EOF | sudo tee /etc/containerd/gvisor-containerd-shim.toml +# This is the path to the default runc containerd-shim. +runc_shim = "/usr/local/bin/containerd-shim" +EOF +} +``` + +### Configure containerd + +1. Update `/etc/containerd/config.toml`. Be sure to update the path to + `gvisor-containerd-shim` and `runsc` if necessary: + +[embedmd]:# (../test/e2e/untrusted-workload/install.sh shell /{ # Step 1/ /^}/) +```shell +{ # Step 1: Create containerd config.toml +cat <<EOF | sudo tee /etc/containerd/config.toml +disabled_plugins = ["restart"] +[plugins.linux] + shim = "/usr/local/bin/gvisor-containerd-shim" + shim_debug = true +[plugins.cri.containerd.untrusted_workload_runtime] + runtime_type = "io.containerd.runtime.v1.linux" + runtime_engine = "/usr/local/bin/runsc" + runtime_root = "/run/containerd/runsc" +EOF +} +``` + +2. Restart `containerd` + +```shell +sudo systemctl restart containerd +``` + +## Usage + +You can run containers in gVisor via containerd's CRI. + +### Install crictl + +1. Download and install the crictl binary: + +[embedmd]:# (../test/e2e/crictl-install.sh shell /{ # Step 1/ /^}/) +```shell +{ # Step 1: Download crictl +wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.13.0/crictl-v1.13.0-linux-amd64.tar.gz +tar xf crictl-v1.13.0-linux-amd64.tar.gz +sudo mv crictl /usr/local/bin +} +``` + +2. Write the crictl configuration file + +[embedmd]:# (../test/e2e/crictl-install.sh shell /{ # Step 2/ /^}/) +```shell +{ # Step 2: Configure crictl +cat <<EOF | sudo tee /etc/crictl.yaml +runtime-endpoint: unix:///run/containerd/containerd.sock +EOF +} +``` + +### Create the nginx Sandbox in gVisor + +1. Pull the nginx image + +[embedmd]:# (../test/e2e/untrusted-workload/usage.sh shell /{ # Step 1/ /^}/) +```shell +{ # Step 1: Pull the nginx image +sudo crictl pull nginx +} +``` + +2. Create the sandbox creation request + +[embedmd]:# (../test/e2e/untrusted-workload/usage.sh shell /{ # Step 2/ /^EOF\n}/) +```shell +{ # Step 2: Create sandbox.json +cat <<EOF | tee sandbox.json +{ + "metadata": { + "name": "nginx-sandbox", + "namespace": "default", + "attempt": 1, + "uid": "hdishd83djaidwnduwk28bcsb" + }, + "annotations": { + "io.kubernetes.cri.untrusted-workload": "true" + }, + "linux": { + }, + "log_directory": "/tmp" +} +EOF +} +``` + +3. Create the pod in gVisor + +[embedmd]:# (../test/e2e/untrusted-workload/usage.sh shell /{ # Step 3/ /^}/) +```shell +{ # Step 3: Create the sandbox +SANDBOX_ID=$(sudo crictl runp sandbox.json) +} +``` + +### Run the nginx Container in the Sandbox + +1. Create the nginx container creation request + +[embedmd]:# (../test/e2e/run-container.sh shell /{ # Step 1/ /^EOF\n}/) +```shell +{ # Step 1: Create nginx container config +cat <<EOF | tee container.json +{ + "metadata": { + "name": "nginx" + }, + "image":{ + "image": "nginx" + }, + "log_path":"nginx.0.log", + "linux": { + } +} +EOF +} +``` + +2. Create the nginx container + +[embedmd]:# (../test/e2e/run-container.sh shell /{ # Step 2/ /^}/) +```shell +{ # Step 2: Create nginx container +CONTAINER_ID=$(sudo crictl create ${SANDBOX_ID} container.json sandbox.json) +} +``` + +3. Start the nginx container + +[embedmd]:# (../test/e2e/run-container.sh shell /{ # Step 3/ /^}/) +```shell +{ # Step 3: Start nginx container +sudo crictl start ${CONTAINER_ID} +} +``` + +### Validate the container + +1. Inspect the created pod + +[embedmd]:# (../test/e2e/validate.sh shell /{ # Step 1/ /^}/) +```shell +{ # Step 1: Inspect the pod +sudo crictl inspectp ${SANDBOX_ID} +} +``` + +2. Inspect the nginx container + +[embedmd]:# (../test/e2e/validate.sh shell /{ # Step 2/ /^}/) +```shell +{ # Step 2: Inspect the container +sudo crictl inspect ${CONTAINER_ID} +} +``` + +3. Verify that nginx is running in gVisor + +[embedmd]:# (../test/e2e/validate.sh shell /{ # Step 3/ /^}/) +```shell +{ # Step 3: Check dmesg +sudo crictl exec ${CONTAINER_ID} dmesg | grep -i gvisor +} +``` diff --git a/shim/v1/main.go b/shim/v1/main.go new file mode 100644 index 000000000..bdcf4e6d4 --- /dev/null +++ b/shim/v1/main.go @@ -0,0 +1,26 @@ +// Copyright 2018 The containerd Authors. +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/containerd/containerd/runtime/v2/shim" + + runsc "github.com/google/gvisor-containerd-shim/pkg/v2" +) + +func main() { + shim.Run("io.containerd.runsc.v1", runsc.New) +} diff --git a/shim/v2/config.go b/shim/v2/config.go new file mode 100644 index 000000000..a72cc7754 --- /dev/null +++ b/shim/v2/config.go @@ -0,0 +1,40 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "github.com/BurntSushi/toml" + +// config is the configuration for gvisor containerd shim. +type config struct { + // RuncShim is the shim binary path for standard containerd-shim for runc. + // When the runtime is `runc`, gvisor containerd shim will exec current + // process to standard containerd-shim. This is a work around for containerd + // 1.1. In containerd 1.2, containerd will choose different containerd-shims + // based on runtime. + RuncShim string `toml:"runc_shim"` + // RunscConfig is configuration for runsc. The key value will be converted + // to runsc flags --key=value directly. + RunscConfig map[string]string `toml:"runsc_config"` +} + +// loadConfig load gvisor containerd shim config from config file. +func loadConfig(path string) (*config, error) { + var c config + _, err := toml.DecodeFile(path, &c) + if err != nil { + return &c, err + } + return &c, nil +} diff --git a/shim/v2/main.go b/shim/v2/main.go new file mode 100644 index 000000000..500bdddee --- /dev/null +++ b/shim/v2/main.go @@ -0,0 +1,318 @@ +// Copyright 2018 The containerd Authors. +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "flag" + "fmt" + "net" + "os" + "os/exec" + "os/signal" + "path/filepath" + "runtime" + "runtime/debug" + "strings" + "sync" + "syscall" + "time" + + "github.com/containerd/containerd/events" + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/runtime/v1/linux/proc" + containerdshim "github.com/containerd/containerd/runtime/v1/shim" + shimapi "github.com/containerd/containerd/runtime/v1/shim/v1" + "github.com/containerd/ttrpc" + "github.com/containerd/typeurl" + ptypes "github.com/gogo/protobuf/types" + "github.com/opencontainers/runc/libcontainer/system" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" + + runsc "github.com/google/gvisor-containerd-shim/pkg/go-runsc" + "github.com/google/gvisor-containerd-shim/pkg/v1/shim" +) + +var ( + debugFlag bool + namespaceFlag string + socketFlag string + addressFlag string + workdirFlag string + runtimeRootFlag string + containerdBinaryFlag string + shimConfigFlag string +) + +// ShimConfigPath is the default shim config file path. +const ShimConfigPath = "/etc/containerd/gvisor-containerd-shim.toml" + +func init() { + flag.BoolVar(&debugFlag, "debug", false, "enable debug output in logs") + flag.StringVar(&namespaceFlag, "namespace", "", "namespace that owns the shim") + flag.StringVar(&socketFlag, "socket", "", "abstract socket path to serve") + flag.StringVar(&addressFlag, "address", "", "grpc address back to main containerd") + flag.StringVar(&workdirFlag, "workdir", "", "path used to storge large temporary data") + // Containerd default to runc, unless another runtime is explicitly specified. + // We keep the same default to make the default behavior consistent. + flag.StringVar(&runtimeRootFlag, "runtime-root", proc.RuncRoot, "root directory for the runtime") + // currently, the `containerd publish` utility is embedded in the daemon binary. + // The daemon invokes `containerd-shim -containerd-binary ...` with its own os.Executable() path. + flag.StringVar(&containerdBinaryFlag, "containerd-binary", "containerd", "path to containerd binary (used for `containerd publish`)") + flag.StringVar(&shimConfigFlag, "config", ShimConfigPath, "path to the shim configuration file") + flag.Parse() +} + +func main() { + // This is a hack. Exec current process to run standard containerd-shim + // if runtime root is not `runsc`. We don't need this for shim v2 api. + if filepath.Base(runtimeRootFlag) != "runsc" { + if err := executeRuncShim(); err != nil { + fmt.Fprintf(os.Stderr, "gvisor-containerd-shim: %s\n", err) + os.Exit(1) + } + } + + debug.SetGCPercent(40) + go func() { + for range time.Tick(30 * time.Second) { + debug.FreeOSMemory() + } + }() + + if debugFlag { + logrus.SetLevel(logrus.DebugLevel) + } + + if os.Getenv("GOMAXPROCS") == "" { + // If GOMAXPROCS hasn't been set, we default to a value of 2 to reduce + // the number of Go stacks present in the shim. + runtime.GOMAXPROCS(2) + } + + // Run regular shim if needed. + if err := executeShim(); err != nil { + fmt.Fprintf(os.Stderr, "gvisor-containerd-shim: %s\n", err) + os.Exit(1) + } +} + +// executeRuncShim execs current process to a containerd-shim process and +// retains all flags and envs. +func executeRuncShim() error { + c, err := loadConfig(shimConfigFlag) + if err != nil && !os.IsNotExist(err) { + return errors.Wrap(err, "failed to load shim config") + } + shimPath := c.RuncShim + if shimPath == "" { + shimPath, err = exec.LookPath("containerd-shim") + if err != nil { + return errors.Wrapf(err, "lookup containerd-shim") + } + } + + args := append([]string{shimPath}, os.Args[1:]...) + if err := syscall.Exec(shimPath, args, os.Environ()); err != nil { + return errors.Wrapf(err, "exec containerd-shim %q", shimPath) + } + return nil +} + +func executeShim() error { + // start handling signals as soon as possible so that things are properly reaped + // or if runtime exits before we hit the handler + signals, err := setupSignals() + if err != nil { + return err + } + dump := make(chan os.Signal, 32) + signal.Notify(dump, syscall.SIGUSR1) + + path, err := os.Getwd() + if err != nil { + return err + } + server, err := ttrpc.NewServer(ttrpc.WithServerHandshaker(ttrpc.UnixSocketRequireSameUser())) + if err != nil { + return errors.Wrap(err, "failed creating server") + } + c, err := loadConfig(shimConfigFlag) + if err != nil && !os.IsNotExist(err) { + return errors.Wrap(err, "failed to load shim config") + } + sv, err := shim.NewService( + shim.Config{ + Path: path, + Namespace: namespaceFlag, + WorkDir: workdirFlag, + RuntimeRoot: runtimeRootFlag, + RunscConfig: c.RunscConfig, + }, + &remoteEventsPublisher{address: addressFlag}, + ) + if err != nil { + return err + } + logrus.Debug("registering ttrpc server") + shimapi.RegisterShimService(server, sv) + + socket := socketFlag + if err := serve(server, socket); err != nil { + return err + } + logger := logrus.WithFields(logrus.Fields{ + "pid": os.Getpid(), + "path": path, + "namespace": namespaceFlag, + }) + go func() { + for range dump { + dumpStacks(logger) + } + }() + return handleSignals(logger, signals, server, sv) +} + +// serve serves the ttrpc API over a unix socket at the provided path +// this function does not block +func serve(server *ttrpc.Server, path string) error { + var ( + l net.Listener + err error + ) + if path == "" { + l, err = net.FileListener(os.NewFile(3, "socket")) + path = "[inherited from parent]" + } else { + if len(path) > 106 { + return errors.Errorf("%q: unix socket path too long (> 106)", path) + } + l, err = net.Listen("unix", "\x00"+path) + } + if err != nil { + return err + } + logrus.WithField("socket", path).Debug("serving api on unix socket") + go func() { + defer l.Close() + if err := server.Serve(context.Background(), l); err != nil && + !strings.Contains(err.Error(), "use of closed network connection") { + logrus.WithError(err).Fatal("gvisor-containerd-shim: ttrpc server failure") + } + }() + return nil +} + +// setupSignals creates a new signal handler for all signals and sets the shim as a +// sub-reaper so that the container processes are reparented +func setupSignals() (chan os.Signal, error) { + signals := make(chan os.Signal, 32) + signal.Notify(signals, unix.SIGTERM, unix.SIGINT, unix.SIGCHLD, unix.SIGPIPE) + // make sure runc is setup to use the monitor + // for waiting on processes + // TODO(random-liu): Move shim/reaper.go to a separate package. + runsc.Monitor = containerdshim.Default + // set the shim as the subreaper for all orphaned processes created by the container + if err := system.SetSubreaper(1); err != nil { + return nil, err + } + return signals, nil +} + +func handleSignals(logger *logrus.Entry, signals chan os.Signal, server *ttrpc.Server, sv *shim.Service) error { + var ( + termOnce sync.Once + done = make(chan struct{}) + ) + + for { + select { + case <-done: + return nil + case s := <-signals: + switch s { + case unix.SIGCHLD: + if err := containerdshim.Reap(); err != nil { + logger.WithError(err).Error("reap exit status") + } + case unix.SIGTERM, unix.SIGINT: + go termOnce.Do(func() { + ctx := context.TODO() + if err := server.Shutdown(ctx); err != nil { + logger.WithError(err).Error("failed to shutdown server") + } + // Ensure our child is dead if any + sv.Kill(ctx, &shimapi.KillRequest{ + Signal: uint32(syscall.SIGKILL), + All: true, + }) + sv.Delete(context.Background(), &ptypes.Empty{}) + close(done) + }) + case unix.SIGPIPE: + } + } + } +} + +func dumpStacks(logger *logrus.Entry) { + var ( + buf []byte + stackSize int + ) + bufferLen := 16384 + for stackSize == len(buf) { + buf = make([]byte, bufferLen) + stackSize = runtime.Stack(buf, true) + bufferLen *= 2 + } + buf = buf[:stackSize] + logger.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf) +} + +type remoteEventsPublisher struct { + address string +} + +func (l *remoteEventsPublisher) Publish(ctx context.Context, topic string, event events.Event) error { + ns, _ := namespaces.Namespace(ctx) + encoded, err := typeurl.MarshalAny(event) + if err != nil { + return err + } + data, err := encoded.Marshal() + if err != nil { + return err + } + cmd := exec.CommandContext(ctx, containerdBinaryFlag, "--address", l.address, "publish", "--topic", topic, "--namespace", ns) + cmd.Stdin = bytes.NewReader(data) + c, err := containerdshim.Default.Start(cmd) + if err != nil { + return err + } + status, err := containerdshim.Default.Wait(cmd, c) + if err != nil { + return err + } + if status != 0 { + return errors.New("failed to publish event") + } + return nil +} |