From d66aebb15a5804f84240dccac55cd56b1dde15e5 Mon Sep 17 00:00:00 2001 From: Kevin Krakauer Date: Fri, 30 Oct 2020 12:00:11 -0700 Subject: Add the gVisor admission webhook PiperOrigin-RevId: 339913577 --- webhook/BUILD | 28 ++++++ webhook/main.go | 24 +++++ webhook/pkg/cli/BUILD | 17 ++++ webhook/pkg/cli/cli.go | 115 +++++++++++++++++++++ webhook/pkg/injector/BUILD | 34 +++++++ webhook/pkg/injector/gencerts.sh | 71 +++++++++++++ webhook/pkg/injector/webhook.go | 211 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 500 insertions(+) create mode 100644 webhook/BUILD create mode 100644 webhook/main.go create mode 100644 webhook/pkg/cli/BUILD create mode 100644 webhook/pkg/cli/cli.go create mode 100644 webhook/pkg/injector/BUILD create mode 100755 webhook/pkg/injector/gencerts.sh create mode 100644 webhook/pkg/injector/webhook.go (limited to 'webhook') diff --git a/webhook/BUILD b/webhook/BUILD new file mode 100644 index 000000000..33c585504 --- /dev/null +++ b/webhook/BUILD @@ -0,0 +1,28 @@ +load("//images:defs.bzl", "docker_image") +load("//tools:defs.bzl", "go_binary", "pkg_tar") + +package(licenses = ["notice"]) + +docker_image( + name = "webhook_image", + data = ":files", + statements = ['ENTRYPOINT ["/webhook"]'], +) + +# files is the full file system of the webhook container. It is simply: +# / +# └─ webhook +pkg_tar( + name = "files", + srcs = [":webhook"], + extension = "tgz", + strip_prefix = "/third_party/gvisor/webhook", +) + +go_binary( + name = "webhook", + srcs = ["main.go"], + pure = "on", + static = "on", + deps = ["//webhook/pkg/cli"], +) diff --git a/webhook/main.go b/webhook/main.go new file mode 100644 index 000000000..220016543 --- /dev/null +++ b/webhook/main.go @@ -0,0 +1,24 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Binary main serves a mutating Kubernetes webhook. +package main + +import ( + "gvisor.dev/gvisor/webhook/pkg/cli" +) + +func main() { + cli.Main() +} diff --git a/webhook/pkg/cli/BUILD b/webhook/pkg/cli/BUILD new file mode 100644 index 000000000..ac093c556 --- /dev/null +++ b/webhook/pkg/cli/BUILD @@ -0,0 +1,17 @@ +load("//tools:defs.bzl", "go_library") + +package(licenses = ["notice"]) + +go_library( + name = "cli", + srcs = ["cli.go"], + visibility = ["//:sandbox"], + deps = [ + "//pkg/log", + "//webhook/pkg/injector", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + "@io_k8s_apimachinery//pkg/util/net:go_default_library", + "@io_k8s_client_go//kubernetes:go_default_library", + "@io_k8s_client_go//rest:go_default_library", + ], +) diff --git a/webhook/pkg/cli/cli.go b/webhook/pkg/cli/cli.go new file mode 100644 index 000000000..a07d341a2 --- /dev/null +++ b/webhook/pkg/cli/cli.go @@ -0,0 +1,115 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cli provides a CLI interface for a mutating Kubernetes webhook. +package cli + +import ( + "flag" + "fmt" + "net" + "net/http" + "os" + "strconv" + "strings" + + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/webhook/pkg/injector" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8snet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +var ( + address = flag.String("address", "", "The ip address the admission webhook serves on. If unspecified, a public address is selected automatically.") + port = flag.Int("port", 0, "The port the admission webhook serves on.") + podLabels = flag.String("pod-namespace-labels", "", "A comma-separated namespace label selector, the admission webhook will only take effect on pods in selected namespaces, e.g. `label1,label2`.") +) + +// Main runs the webhook. +func Main() { + flag.Parse() + + if err := run(); err != nil { + log.Warningf("%v", err) + os.Exit(1) + } +} + +func run() error { + log.Infof("Starting %s\n", injector.Name) + + // Create client config. + cfg, err := rest.InClusterConfig() + if err != nil { + return fmt.Errorf("create in cluster config: %w", err) + } + + // Create clientset. + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return fmt.Errorf("create kubernetes client: %w", err) + } + + if err := injector.CreateConfiguration(clientset, parsePodLabels()); err != nil { + return fmt.Errorf("create webhook configuration: %w", err) + } + + if err := startWebhookHTTPS(clientset); err != nil { + return fmt.Errorf("start webhook https server: %w", err) + } + + return nil +} + +func parsePodLabels() *metav1.LabelSelector { + rv := &metav1.LabelSelector{} + for _, s := range strings.Split(*podLabels, ",") { + req := metav1.LabelSelectorRequirement{ + Key: strings.TrimSpace(s), + Operator: "Exists", + } + rv.MatchExpressions = append(rv.MatchExpressions, req) + } + return rv +} + +func startWebhookHTTPS(clientset kubernetes.Interface) error { + log.Infof("Starting HTTPS handler") + defer log.Infof("Stopping HTTPS handler") + + if *address == "" { + ip, err := k8snet.ChooseHostInterface() + if err != nil { + return fmt.Errorf("select ip address: %w", err) + } + *address = ip.String() + } + mux := http.NewServeMux() + mux.Handle("/", http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + injector.Admit(w, r) + })) + server := &http.Server{ + // Listen on all addresses. + Addr: net.JoinHostPort(*address, strconv.Itoa(*port)), + TLSConfig: injector.GetTLSConfig(), + Handler: mux, + } + if err := server.ListenAndServeTLS("", ""); err != http.ErrServerClosed { + return fmt.Errorf("start HTTPS handler: %w", err) + } + return nil +} diff --git a/webhook/pkg/injector/BUILD b/webhook/pkg/injector/BUILD new file mode 100644 index 000000000..d296981be --- /dev/null +++ b/webhook/pkg/injector/BUILD @@ -0,0 +1,34 @@ +load("//tools:defs.bzl", "go_library") + +package(licenses = ["notice"]) + +go_library( + name = "injector", + srcs = [ + "certs.go", + "webhook.go", + ], + visibility = ["//:sandbox"], + deps = [ + "//pkg/log", + "@com_github_mattbaird_jsonpatch//:go_default_library", + "@io_k8s_api//admission/v1beta1:go_default_library", + "@io_k8s_api//admissionregistration/v1beta1:go_default_library", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/api/errors:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + "@io_k8s_client_go//kubernetes:go_default_library", + ], +) + +genrule( + name = "certs", + srcs = [":gencerts"], + outs = ["certs.go"], + cmd = "$$(cut -d ' ' -f 1 <<< \"$(locations :gencerts)\") $@", +) + +sh_binary( + name = "gencerts", + srcs = ["gencerts.sh"], +) diff --git a/webhook/pkg/injector/gencerts.sh b/webhook/pkg/injector/gencerts.sh new file mode 100755 index 000000000..f7fda4b63 --- /dev/null +++ b/webhook/pkg/injector/gencerts.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# Copyright 2020 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Generates the a CA cert, a server key, and a server cert signed by the CA. +# reference: +# https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/testcerts/gencerts.sh +set -euo pipefail + +# Do all the work in TMPDIR, then copy out generated code and delete TMPDIR. +declare -r OUTDIR="$(readlink -e .)" +declare -r TMPDIR="$(mktemp -d)" +cd "${TMPDIR}" +function cleanup() { + cd "${OUTDIR}" + rm -rf "${TMPDIR}" +} +trap cleanup EXIT + +declare -r CN_BASE="e2e" +declare -r CN="gvisor-injection-admission-webhook.e2e.svc" + +cat > server.conf << EOF +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, serverAuth +EOF + +declare -r OUTFILE="${TMPDIR}/certs.go" + +# We depend on OpenSSL being present. + +# Create a certificate authority. +openssl genrsa -out caKey.pem 2048 +openssl req -x509 -new -nodes -key caKey.pem -days 100000 -out caCert.pem -subj "/CN=${CN_BASE}_ca" -config server.conf + +# Create a server certificate. +openssl genrsa -out serverKey.pem 2048 +# Note the CN is the DNS name of the service of the webhook. +openssl req -new -key serverKey.pem -out server.csr -subj "/CN=${CN}" -config server.conf +openssl x509 -req -in server.csr -CA caCert.pem -CAkey caKey.pem -CAcreateserial -out serverCert.pem -days 100000 -extensions v3_req -extfile server.conf + +echo "package injector" > "${OUTFILE}" +echo "" >> "${OUTFILE}" +echo "// This file was generated using openssl by the gencerts.sh script." >> "${OUTFILE}" +for file in caKey caCert serverKey serverCert; do + DATA=$(cat "${file}.pem") + echo "" >> "${OUTFILE}" + echo "var $file = []byte(\`$DATA\`)" >> "${OUTFILE}" +done + +# Copy generated code into the output directory. +cp "${OUTFILE}" "${OUTDIR}/$1" diff --git a/webhook/pkg/injector/webhook.go b/webhook/pkg/injector/webhook.go new file mode 100644 index 000000000..614b5add7 --- /dev/null +++ b/webhook/pkg/injector/webhook.go @@ -0,0 +1,211 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package injector handles mutating webhook operations. +package injector + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "net/http" + "os" + + "github.com/mattbaird/jsonpatch" + "gvisor.dev/gvisor/pkg/log" + admv1beta1 "k8s.io/api/admission/v1beta1" + admregv1beta1 "k8s.io/api/admissionregistration/v1beta1" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeclientset "k8s.io/client-go/kubernetes" +) + +const ( + // Name is the name of the admission webhook service. The admission + // webhook must be exposed in the following service; this is mainly for + // the server certificate. + Name = "gvisor-injection-admission-webhook" + + // serviceNamespace is the namespace of the admission webhook service. + serviceNamespace = "e2e" + + fullName = Name + "." + serviceNamespace + ".svc" +) + +// CreateConfiguration creates MutatingWebhookConfiguration and registers the +// webhook admission controller with the kube-apiserver. The webhook will only +// take effect on pods in the namespaces selected by `podNsSelector`. If `podNsSelector` +// is empty, the webhook will take effect on all pods. +func CreateConfiguration(clientset kubeclientset.Interface, selector *metav1.LabelSelector) error { + fail := admregv1beta1.Fail + + config := &admregv1beta1.MutatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: Name, + }, + Webhooks: []admregv1beta1.MutatingWebhook{ + { + Name: fullName, + ClientConfig: admregv1beta1.WebhookClientConfig{ + Service: &admregv1beta1.ServiceReference{ + Name: Name, + Namespace: serviceNamespace, + }, + CABundle: caCert, + }, + Rules: []admregv1beta1.RuleWithOperations{ + { + Operations: []admregv1beta1.OperationType{ + admregv1beta1.Create, + }, + Rule: admregv1beta1.Rule{ + APIGroups: []string{"*"}, + APIVersions: []string{"*"}, + Resources: []string{"pods"}, + }, + }, + }, + FailurePolicy: &fail, + NamespaceSelector: selector, + }, + }, + } + log.Infof("Creating MutatingWebhookConfiguration %q", config.Name) + if _, err := clientset.AdmissionregistrationV1beta1().MutatingWebhookConfigurations().Create(config); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create MutatingWebhookConfiguration %q: %s", config.Name, err) + } + log.Infof("MutatingWebhookConfiguration %q already exists; use the existing one", config.Name) + } + return nil +} + +// GetTLSConfig retrieves the CA cert that signed the cert used by the webhook. +func GetTLSConfig() *tls.Config { + serverCert, err := tls.X509KeyPair(serverCert, serverKey) + if err != nil { + log.Warningf("Failed to generate X509 key pair: %v", err) + os.Exit(1) + } + return &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + } +} + +// Admit performs admission checks and mutations on Pods. +func Admit(writer http.ResponseWriter, req *http.Request) { + review := &admv1beta1.AdmissionReview{} + if err := json.NewDecoder(req.Body).Decode(review); err != nil { + log.Infof("Failed with error (%v) to decode Admit request: %+v", err, *req) + writer.WriteHeader(http.StatusBadRequest) + return + } + + log.Debugf("admitPod: %+v", review) + var err error + review.Response, err = admitPod(review.Request) + if err != nil { + log.Warningf("admitPod failed: %v", err) + review.Response = &admv1beta1.AdmissionResponse{ + Result: &metav1.Status{ + Reason: metav1.StatusReasonInvalid, + Message: err.Error(), + }, + } + sendResponse(writer, review) + return + } + + log.Debugf("Processed admission review: %+v", review) + sendResponse(writer, review) +} + +func sendResponse(writer http.ResponseWriter, response interface{}) { + b, err := json.Marshal(response) + if err != nil { + log.Warningf("Failed with error (%v) to marshal response: %+v", err, response) + writer.WriteHeader(http.StatusInternalServerError) + return + } + + writer.WriteHeader(http.StatusOK) + writer.Write(b) +} + +func admitPod(req *admv1beta1.AdmissionRequest) (*admv1beta1.AdmissionResponse, error) { + // Verify that the request is indeed a Pod. + resource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"} + if req.Resource != resource { + return nil, fmt.Errorf("unexpected resource %+v in pod admission", req.Resource) + } + + // Decode the request into a Pod. + pod := &v1.Pod{} + if err := json.Unmarshal(req.Object.Raw, pod); err != nil { + return nil, fmt.Errorf("failed to decode pod object %s/%s", req.Namespace, req.Name) + } + + // Copy first to change it. + podCopy := pod.DeepCopy() + updatePod(podCopy) + patch, err := createPatch(req.Object.Raw, podCopy) + if err != nil { + return nil, fmt.Errorf("failed to create patch for pod %s/%s (generatedName: %s)", pod.Namespace, pod.Name, pod.GenerateName) + } + + log.Debugf("Patched pod %s/%s (generateName: %s): %+v", pod.Namespace, pod.Name, pod.GenerateName, podCopy) + patchType := admv1beta1.PatchTypeJSONPatch + return &admv1beta1.AdmissionResponse{ + Allowed: true, + Patch: patch, + PatchType: &patchType, + }, nil +} + +func updatePod(pod *v1.Pod) { + gvisor := "gvisor" + pod.Spec.RuntimeClassName = &gvisor + + // We don't run SELinux test for gvisor. + // If SELinuxOptions are specified, this is usually for volume test to pass + // on SELinux. This can be safely ignored. + if pod.Spec.SecurityContext != nil && pod.Spec.SecurityContext.SELinuxOptions != nil { + pod.Spec.SecurityContext.SELinuxOptions = nil + } + for i := range pod.Spec.Containers { + c := &pod.Spec.Containers[i] + if c.SecurityContext != nil && c.SecurityContext.SELinuxOptions != nil { + c.SecurityContext.SELinuxOptions = nil + } + } + for i := range pod.Spec.InitContainers { + c := &pod.Spec.InitContainers[i] + if c.SecurityContext != nil && c.SecurityContext.SELinuxOptions != nil { + c.SecurityContext.SELinuxOptions = nil + } + } +} + +func createPatch(old []byte, newObj interface{}) ([]byte, error) { + new, err := json.Marshal(newObj) + if err != nil { + return nil, err + } + patch, err := jsonpatch.CreatePatch(old, new) + if err != nil { + return nil, err + } + return json.Marshal(patch) +} -- cgit v1.2.3