Add a functional vm_test for root_test.

This change renames the tools/images directory to tools/vm for clarity, and
adds a functional vm_test. Sharding is also added to the same test, and some
documentation added around key flags & variables to describe how they work.

Subsequent changes will add vm_tests for other cases, such as the runtime tests.

PiperOrigin-RevId: 307492245

14 files changed, 925 insertions, 0 deletions

diff --git a/tools/vm/BUILD b/tools/vm/BUILD
new file mode 100644
index 000000000..f7160c627
--- /dev/null
+++ b/tools/vm/BUILD
@@ -0,0 +1,57 @@
+load("//tools:defs.bzl", "cc_binary", "gtest")
+load("//tools/vm:defs.bzl", "vm_image", "vm_test")
+
+package(
+    default_visibility = ["//:sandbox"],
+    licenses = ["notice"],
+)
+
+sh_binary(
+    name = "zone",
+    srcs = ["zone.sh"],
+)
+
+sh_binary(
+    name = "builder",
+    srcs = ["build.sh"],
+)
+
+sh_binary(
+    name = "executer",
+    srcs = ["execute.sh"],
+)
+
+cc_binary(
+    name = "test",
+    testonly = 1,
+    srcs = ["test.cc"],
+    linkstatic = 1,
+    deps = [
+        gtest,
+        "//test/util:test_main",
+    ],
+)
+
+vm_image(
+    name = "ubuntu1604",
+    family = "ubuntu-1604-lts",
+    project = "ubuntu-os-cloud",
+    scripts = [
+        "//tools/vm/ubuntu1604",
+    ],
+)
+
+vm_image(
+    name = "ubuntu1804",
+    family = "ubuntu-1804-lts",
+    project = "ubuntu-os-cloud",
+    scripts = [
+        "//tools/vm/ubuntu1804",
+    ],
+)
+
+vm_test(
+    name = "vm_test",
+    shard_count = 2,
+    targets = [":test"],
+)
diff --git a/tools/vm/README.md b/tools/vm/README.md
new file mode 100644
index 000000000..898c95fca
--- /dev/null
+++ b/tools/vm/README.md
@@ -0,0 +1,42 @@
+# VM Images & Tests
+
+All commands in this directory require the `gcloud` project to be set.
+
+For example: `gcloud config set project gvisor-kokoro-testing`.
+
+Images can be generated by using the `vm_image` rule. This rule will generate a
+binary target that builds an image in an idempotent way, and can be referenced
+from other rules.
+
+For example:
+
+```
+vm_image(
+    name = "ubuntu",
+    project = "ubuntu-1604-lts",
+    family = "ubuntu-os-cloud",
+    scripts = [
+        "script.sh",
+        "other.sh",
+    ],
+)
+```
+
+These images can be built manually by executing the target. The output on
+`stdout` will be the image id (in the current project).
+
+Images are always named per the hash of all the hermetic input scripts. This
+allows images to be memoized quickly and easily.
+
+The `vm_test` rule can be used to execute a command remotely. This is still
+under development however, and will likely change over time.
+
+For example:
+
+```
+vm_test(
+    name = "mycommand",
+    image = ":ubuntu",
+    targets = [":test"],
+)
+```
diff --git a/tools/vm/build.sh b/tools/vm/build.sh
new file mode 100755
index 000000000..5d3dc0bbf
--- /dev/null
+++ b/tools/vm/build.sh
@@ -0,0 +1,117 @@
+#!/bin/bash
+
+# Copyright 2019 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is responsible for building a new GCP image that: 1) has nested
+# virtualization enabled, and 2) has been completely set up with the
+# image_setup.sh script. This script should be idempotent, as we memoize the
+# setup script with a hash and check for that name.
+
+set -eou pipefail
+
+# Parameters.
+declare -r USERNAME=${USERNAME:-test}
+declare -r IMAGE_PROJECT=${IMAGE_PROJECT:-ubuntu-os-cloud}
+declare -r IMAGE_FAMILY=${IMAGE_FAMILY:-ubuntu-1604-lts}
+declare -r ZONE=${ZONE:-us-central1-f}
+
+# Random names.
+declare -r DISK_NAME=$(mktemp -u disk-XXXXXX | tr A-Z a-z)
+declare -r SNAPSHOT_NAME=$(mktemp -u snapshot-XXXXXX | tr A-Z a-z)
+declare -r INSTANCE_NAME=$(mktemp -u build-XXXXXX | tr A-Z a-z)
+
+# Hash inputs in order to memoize the produced image.
+declare -r SETUP_HASH=$( (echo ${USERNAME} ${IMAGE_PROJECT} ${IMAGE_FAMILY} && cat "$@") | sha256sum - | cut -d' ' -f1 | cut -c 1-16)
+declare -r IMAGE_NAME=${IMAGE_FAMILY:-image}-${SETUP_HASH}
+
+# Does the image already exist? Skip the build.
+declare -r existing=$(set -x; gcloud compute images list --filter="name=(${IMAGE_NAME})" --format="value(name)")
+if ! [[ -z "${existing}" ]]; then
+  echo "${existing}"
+  exit 0
+fi
+
+# Standard arguments (applies only on script execution).
+declare -ar SSH_ARGS=("-o" "ConnectTimeout=60" "--")
+
+# gcloud has path errors; is this a result of being a genrule?
+export PATH=${PATH:-/bin:/usr/bin:/usr/local/bin}
+
+# Start a unique instance. Note that this instance will have a unique persistent
+# disk as it's boot disk with the same name as the instance.
+(set -x; gcloud compute instances create \
+    --quiet \
+    --image-project "${IMAGE_PROJECT}" \
+    --image-family "${IMAGE_FAMILY}" \
+    --boot-disk-size "200GB" \
+    --zone "${ZONE}" \
+    "${INSTANCE_NAME}" >/dev/null)
+function cleanup {
+  (set -x; gcloud compute instances delete --quiet --zone "${ZONE}" "${INSTANCE_NAME}")
+}
+trap cleanup EXIT
+
+# Wait for the instance to become available (up to 5 minutes).
+echo -n "Waiting for ${INSTANCE_NAME}"
+declare timeout=300
+declare success=0
+declare internal=""
+declare -r start=$(date +%s)
+declare -r end=$((${start}+${timeout}))
+while [[ "$(date +%s)" -lt "${end}" ]] && [[ "${success}" -lt 3 ]]; do
+  echo -n "."
+  if gcloud compute ssh --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- true 2>/dev/null; then
+    success=$((${success}+1))
+  elif gcloud compute ssh --internal-ip --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- true 2>/dev/null; then
+    success=$((${success}+1))
+    internal="--internal-ip"
+  fi
+done
+
+if [[ "${success}" -eq "0" ]]; then
+  echo "connect timed out after ${timeout} seconds."
+  exit 1
+else
+  echo "done."
+fi
+
+# Run the install scripts provided.
+for arg; do
+  (set -x; gcloud compute ssh ${internal} \
+      --zone "${ZONE}" \
+      "${USERNAME}"@"${INSTANCE_NAME}" -- \
+      "${SSH_ARGS[@]}" \
+      sudo bash - <"${arg}" >/dev/null)
+done
+
+# Stop the instance; required before creating an image.
+(set -x; gcloud compute instances stop --quiet --zone "${ZONE}" "${INSTANCE_NAME}" >/dev/null)
+
+# Create a snapshot of the instance disk.
+(set -x; gcloud compute disks snapshot \
+    --quiet \
+    --zone "${ZONE}" \
+    --snapshot-names="${SNAPSHOT_NAME}" \
+    "${INSTANCE_NAME}" >/dev/null)
+
+# Create the disk image.
+(set -x; gcloud compute images create \
+    --quiet \
+    --source-snapshot="${SNAPSHOT_NAME}" \
+    --licenses="https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" \
+    "${IMAGE_NAME}" >/dev/null)
+
+# Finish up.
+echo "${IMAGE_NAME}"
diff --git a/tools/vm/defs.bzl b/tools/vm/defs.bzl
new file mode 100644
index 000000000..24bf0aabc
--- /dev/null
+++ b/tools/vm/defs.bzl
@@ -0,0 +1,198 @@
+"""Image configuration. See README.md."""
+
+load("//tools:defs.bzl", "default_installer")
+
+# vm_image_builder is a rule that will construct a shell script that actually
+# generates a given VM image. Note that this does not _run_ the shell script
+# (although it can be run manually). It will be run manually during generation
+# of the vm_image target itself. This level of indirection is used so that the
+# build system itself only runs the builder once when multiple targets depend
+# on it, avoiding a set of races and conflicts.
+def _vm_image_builder_impl(ctx):
+    # Generate a binary that actually builds the image.
+    builder = ctx.actions.declare_file(ctx.label.name)
+    script_paths = []
+    for script in ctx.files.scripts:
+        script_paths.append(script.short_path)
+    builder_content = "\n".join([
+        "#!/bin/bash",
+        "export ZONE=$(%s)" % ctx.files.zone[0].short_path,
+        "export USERNAME=%s" % ctx.attr.username,
+        "export IMAGE_PROJECT=%s" % ctx.attr.project,
+        "export IMAGE_FAMILY=%s" % ctx.attr.family,
+        "%s %s" % (ctx.files._builder[0].short_path, " ".join(script_paths)),
+        "",
+    ])
+    ctx.actions.write(builder, builder_content, is_executable = True)
+
+    # Note that the scripts should only be files, and should not include any
+    # indirect transitive dependencies. The build script wouldn't work.
+    return [DefaultInfo(
+        executable = builder,
+        runfiles = ctx.runfiles(
+            files = ctx.files.scripts + ctx.files._builder + ctx.files.zone,
+        ),
+    )]
+
+vm_image_builder = rule(
+    attrs = {
+        "_builder": attr.label(
+            executable = True,
+            default = "//tools/vm:builder",
+            cfg = "host",
+        ),
+        "username": attr.string(default = "$(whoami)"),
+        "zone": attr.label(
+            executable = True,
+            default = "//tools/vm:zone",
+            cfg = "host",
+        ),
+        "family": attr.string(mandatory = True),
+        "project": attr.string(mandatory = True),
+        "scripts": attr.label_list(allow_files = True),
+    },
+    executable = True,
+    implementation = _vm_image_builder_impl,
+)
+
+# See vm_image_builder above.
+def _vm_image_impl(ctx):
+    # Run the builder to generate our output.
+    echo = ctx.actions.declare_file(ctx.label.name)
+    resolved_inputs, argv, runfiles_manifests = ctx.resolve_command(
+        command = "echo -ne \"#!/bin/bash\\necho $(%s)\\n\" > %s && chmod 0755 %s" % (
+            ctx.files.builder[0].path,
+            echo.path,
+            echo.path,
+        ),
+        tools = [ctx.attr.builder],
+    )
+    ctx.actions.run_shell(
+        tools = resolved_inputs,
+        outputs = [echo],
+        progress_message = "Building image...",
+        execution_requirements = {"local": "true"},
+        command = argv,
+        input_manifests = runfiles_manifests,
+    )
+
+    # Return just the echo command. All of the builder runfiles have been
+    # resolved and consumed in the generation of the trivial echo script.
+    return [DefaultInfo(executable = echo)]
+
+_vm_image_test = rule(
+    attrs = {
+        "builder": attr.label(
+            executable = True,
+            cfg = "host",
+        ),
+    },
+    test = True,
+    implementation = _vm_image_impl,
+)
+
+def vm_image(name, **kwargs):
+    vm_image_builder(
+        name = name + "_builder",
+        **kwargs
+    )
+    _vm_image_test(
+        name = name,
+        builder = ":" + name + "_builder",
+        tags = [
+            "local",
+            "manual",
+        ],
+    )
+
+def _vm_test_impl(ctx):
+    runner = ctx.actions.declare_file("%s-executer" % ctx.label.name)
+
+    # Note that the remote execution case must actually generate an
+    # intermediate target in order to collect all the relevant runfiles so that
+    # they can be copied over for remote execution.
+    runner_content = "\n".join([
+        "#!/bin/bash",
+        "export ZONE=$(%s)" % ctx.files.zone[0].short_path,
+        "export USERNAME=%s" % ctx.attr.username,
+        "export IMAGE=$(%s)" % ctx.files.image[0].short_path,
+        "export SUDO=%s" % "true" if ctx.attr.sudo else "false",
+        "%s %s" % (
+            ctx.executable.executer.short_path,
+            " ".join([
+                target.files_to_run.executable.short_path
+                for target in ctx.attr.targets
+            ]),
+        ),
+        "",
+    ])
+    ctx.actions.write(runner, runner_content, is_executable = True)
+
+    # Return with all transitive files.
+    runfiles = ctx.runfiles(
+        transitive_files = depset(transitive = [
+            depset(target.data_runfiles.files)
+            for target in ctx.attr.targets
+            if hasattr(target, "data_runfiles")
+        ]),
+        files = ctx.files.executer + ctx.files.zone + ctx.files.image +
+                ctx.files.targets,
+        collect_default = True,
+        collect_data = True,
+    )
+    return [DefaultInfo(executable = runner, runfiles = runfiles)]
+
+_vm_test = rule(
+    attrs = {
+        "image": attr.label(
+            executable = True,
+            default = "//tools/vm:ubuntu1804",
+            cfg = "host",
+        ),
+        "executer": attr.label(
+            executable = True,
+            default = "//tools/vm:executer",
+            cfg = "host",
+        ),
+        "username": attr.string(default = "$(whoami)"),
+        "zone": attr.label(
+            executable = True,
+            default = "//tools/vm:zone",
+            cfg = "host",
+        ),
+        "sudo": attr.bool(default = True),
+        "machine": attr.string(default = "n1-standard-1"),
+        "targets": attr.label_list(
+            mandatory = True,
+            allow_empty = False,
+            cfg = "target",
+        ),
+    },
+    test = True,
+    implementation = _vm_test_impl,
+)
+
+def vm_test(
+        installers = None,
+        **kwargs):
+    """Runs the given targets as a remote test.
+
+    Args:
+      installer: Script to run before all targets.
+      **kwargs: All test arguments. Should include targets and image.
+    """
+    targets = kwargs.pop("targets", [])
+    if installers == None:
+        installers = ["//tools/installers:head"]
+    targets = installers + targets
+    if default_installer():
+        targets = [default_installer()] + targets
+    _vm_test(
+        tags = [
+            "local",
+            "manual",
+        ],
+        targets = targets,
+        local = 1,
+        **kwargs
+    )
diff --git a/tools/vm/execute.sh b/tools/vm/execute.sh
new file mode 100755
index 000000000..1f1f3ce01
--- /dev/null
+++ b/tools/vm/execute.sh
@@ -0,0 +1,160 @@
+#!/bin/bash
+
+# Copyright 2019 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xeo pipefail
+
+# Required input.
+if ! [[ -v IMAGE ]]; then
+  echo "no image provided: set IMAGE."
+  exit 1
+fi
+
+# Parameters.
+declare -r USERNAME=${USERNAME:-test}
+declare -r KEYNAME=$(mktemp --tmpdir -u key-XXXXXX)
+declare -r SSHKEYS=$(mktemp --tmpdir -u sshkeys-XXXXXX)
+declare -r INSTANCE_NAME=$(mktemp -u test-XXXXXX | tr A-Z a-z)
+declare -r MACHINE=${MACHINE:-n1-standard-1}
+declare -r ZONE=${ZONE:-us-central1-f}
+declare -r SUDO=${SUDO:-false}
+
+# Standard arguments (applies only on script execution).
+declare -ar SSH_ARGS=("-o" "ConnectTimeout=60" "--")
+
+# This script is executed as a test rule, which will reset the value of HOME.
+# Unfortunately, it is needed to load the gconfig credentials. We will reset
+# HOME when we actually execute in the remote environment, defined below.
+export HOME=$(eval echo ~$(whoami))
+
+# Generate unique keys for this test.
+[[ -f "${KEYNAME}" ]] || ssh-keygen -t rsa -N "" -f "${KEYNAME}" -C "${USERNAME}"
+cat > "${SSHKEYS}" <<EOF
+${USERNAME}:$(cat ${KEYNAME}.pub)
+EOF
+
+# Start a unique instance. This means that we first generate a unique set of ssh
+# keys to ensure that only we have access to this instance. Note that we must
+# constrain ourselves to Haswell or greater in order to have nested
+# virtualization available.
+gcloud compute instances create \
+    --min-cpu-platform "Intel Haswell" \
+    --preemptible \
+    --no-scopes \
+    --metadata block-project-ssh-keys=TRUE \
+    --metadata-from-file ssh-keys="${SSHKEYS}" \
+    --machine-type "${MACHINE}" \
+    --image "${IMAGE}" \
+    --zone "${ZONE}" \
+    "${INSTANCE_NAME}"
+function cleanup {
+    gcloud compute instances delete --quiet --zone "${ZONE}" "${INSTANCE_NAME}"
+}
+trap cleanup EXIT
+
+# Wait for the instance to become available (up to 5 minutes).
+declare timeout=300
+declare success=0
+declare -r start=$(date +%s)
+declare -r end=$((${start}+${timeout}))
+while [[ "$(date +%s)" -lt "${end}" ]] && [[ "${success}" -lt 3 ]]; do
+  if gcloud compute ssh --ssh-key-file="${KEYNAME}" --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- true 2>/dev/null; then
+    success=$((${success}+1))
+  fi
+done
+if [[ "${success}" -eq "0" ]]; then
+  echo "connect timed out after ${timeout} seconds."
+  exit 1
+fi
+
+# Copy the local directory over.
+tar czf - --dereference --exclude=.git . |
+    gcloud compute ssh \
+        --ssh-key-file="${KEYNAME}" \
+        --zone "${ZONE}" \
+        "${USERNAME}"@"${INSTANCE_NAME}" -- \
+        "${SSH_ARGS[@]}" \
+        tar xzf -
+
+# Execute the command remotely.
+for cmd; do
+  # Setup relevant environment.
+  #
+  # N.B. This is not a complete test environment, but is complete enough to
+  # provide rudimentary sharding and test output support.
+  declare -a PREFIX=( "env" )
+  if [[ -v TEST_SHARD_INDEX ]]; then
+    PREFIX+=( "TEST_SHARD_INDEX=${TEST_SHARD_INDEX}" )
+  fi
+  if [[ -v TEST_SHARD_STATUS_FILE ]]; then
+    SHARD_STATUS_FILE=$(mktemp -u test-shard-status-XXXXXX)
+    PREFIX+=( "TEST_SHARD_STATUS_FILE=/tmp/${SHARD_STATUS_FILE}" )
+  fi
+  if [[ -v TEST_TOTAL_SHARDS ]]; then
+    PREFIX+=( "TEST_TOTAL_SHARDS=${TEST_TOTAL_SHARDS}" )
+  fi
+  if [[ -v TEST_TMPDIR ]]; then
+    REMOTE_TMPDIR=$(mktemp -u test-XXXXXX)
+    PREFIX+=( "TEST_TMPDIR=/tmp/${REMOTE_TMPDIR}" )
+    # Create remotely.
+    gcloud compute ssh \
+      --ssh-key-file="${KEYNAME}" \
+      --zone "${ZONE}" \
+      "${USERNAME}"@"${INSTANCE_NAME}" -- \
+      "${SSH_ARGS[@]}" \
+      mkdir -p "/tmp/${REMOTE_TMPDIR}"
+  fi
+  if [[ -v XML_OUTPUT_FILE ]]; then
+    TEST_XML_OUTPUT=$(mktemp -u xml-output-XXXXXX)
+    PREFIX+=( "XML_OUTPUT_FILE=/tmp/${TEST_XML_OUTPUT}" )
+  fi
+  if [[ "${SUDO}" == "true" ]]; then
+    PREFIX+=( "sudo" "-E" )
+  fi
+
+  # Execute the command.
+  gcloud compute ssh \
+    --ssh-key-file="${KEYNAME}" \
+    --zone "${ZONE}" \
+    "${USERNAME}"@"${INSTANCE_NAME}" -- \
+    "${SSH_ARGS[@]}" \
+    "${PREFIX[@]}" "${cmd}"
+
+  # Collect relevant results.
+  if [[ -v TEST_SHARD_STATUS_FILE ]]; then
+    gcloud compute scp \
+        --ssh-key-file="${KEYNAME}" \
+        --zone "${ZONE}" \
+        "${USERNAME}"@"${INSTANCE_NAME}":/tmp/"${SHARD_STATUS_FILE}" \
+        "${TEST_SHARD_STATUS_FILE}" 2>/dev/null || true # Allowed to fail.
+  fi
+  if [[ -v XML_OUTPUT_FILE ]]; then
+    gcloud compute scp \
+        --ssh-key-file="${KEYNAME}" \
+        --zone "${ZONE}" \
+        "${USERNAME}"@"${INSTANCE_NAME}":/tmp/"${TEST_XML_OUTPUT}" \
+        "${XML_OUTPUT_FILE}" 2>/dev/null || true # Allowed to fail.
+  fi
+
+  # Clean up the temporary directory.
+  if [[ -v TEST_TMPDIR ]]; then
+    gcloud compute ssh \
+      --ssh-key-file="${KEYNAME}" \
+      --zone "${ZONE}" \
+      "${USERNAME}"@"${INSTANCE_NAME}" -- \
+      "${SSH_ARGS[@]}" \
+      rm -rf "/tmp/${REMOTE_TMPDIR}"
+  fi
+done
diff --git a/tools/vm/test.cc b/tools/vm/test.cc
new file mode 100644
index 000000000..c0ceacda1
--- /dev/null
+++ b/tools/vm/test.cc
@@ -0,0 +1,27 @@
+// Copyright 2020 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "gtest/gtest.h"
+
+namespace {
+
+TEST(Image, Sanity0) {
+  // Do nothing (in shard 0).
+}
+
+TEST(Image, Sanity1) {
+  // Do nothing (in shard 1).
+}
+
+}  // namespace
diff --git a/tools/vm/ubuntu1604/10_core.sh b/tools/vm/ubuntu1604/10_core.sh
new file mode 100755
index 000000000..cd518d6ac
--- /dev/null
+++ b/tools/vm/ubuntu1604/10_core.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Copyright 2019 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xeo pipefail
+
+# Install all essential build tools.
+while true; do
+  if (apt-get update && apt-get install -y \
+      make \
+      git-core \
+      build-essential \
+      linux-headers-$(uname -r) \
+      pkg-config); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
+
+# Install a recent go toolchain.
+if ! [[ -d /usr/local/go ]]; then
+    wget https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz
+    tar -xvf go1.13.5.linux-amd64.tar.gz
+    mv go /usr/local
+fi
+
+# Link the Go binary from /usr/bin; replacing anything there.
+(cd /usr/bin && rm -f go && sudo ln -fs /usr/local/go/bin/go go)
diff --git a/tools/vm/ubuntu1604/20_bazel.sh b/tools/vm/ubuntu1604/20_bazel.sh
new file mode 100755
index 000000000..bb7afa676
--- /dev/null
+++ b/tools/vm/ubuntu1604/20_bazel.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Copyright 2019 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xeo pipefail
+
+declare -r BAZEL_VERSION=2.0.0
+
+# Install bazel dependencies.
+while true; do
+  if (apt-get update && apt-get install -y \
+      openjdk-8-jdk-headless \
+      unzip); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
+
+# Use the release installer.
+curl -L -o bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh
+chmod a+x bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh
+./bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh
+rm -f bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh
diff --git a/tools/vm/ubuntu1604/25_docker.sh b/tools/vm/ubuntu1604/25_docker.sh
new file mode 100755
index 000000000..11eea2d72
--- /dev/null
+++ b/tools/vm/ubuntu1604/25_docker.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Copyright 2019 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Add dependencies.
+while true; do
+  if (apt-get update && apt-get install -y \
+      apt-transport-https \
+      ca-certificates \
+      curl \
+      gnupg-agent \
+      software-properties-common); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
+
+# Install the key.
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+
+# Add the repository.
+add-apt-repository \
+   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+   $(lsb_release -cs) \
+   stable"
+
+# Install docker.
+while true; do
+  if (apt-get update && apt-get install -y \
+      docker-ce \
+      docker-ce-cli \
+      containerd.io); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
diff --git a/tools/vm/ubuntu1604/30_containerd.sh b/tools/vm/ubuntu1604/30_containerd.sh
new file mode 100755
index 000000000..fb3699c12
--- /dev/null
+++ b/tools/vm/ubuntu1604/30_containerd.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# Copyright 2019 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xeo pipefail
+
+# Helper for Go packages below.
+install_helper() {
+  PACKAGE="${1}"
+  TAG="${2}"
+  GOPATH="${3}"
+
+  # Clone the repository.
+  mkdir -p "${GOPATH}"/src/$(dirname "${PACKAGE}") && \
+     git clone https://"${PACKAGE}" "${GOPATH}"/src/"${PACKAGE}"
+
+  # Checkout and build the repository.
+  (cd "${GOPATH}"/src/"${PACKAGE}" && \
+      git checkout "${TAG}" && \
+      GOPATH="${GOPATH}" make && \
+      GOPATH="${GOPATH}" make install)
+}
+
+# Install dependencies for the crictl tests.
+while true; do
+  if (apt-get update && apt-get install -y \
+      btrfs-tools \
+      libseccomp-dev); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
+
+# Install containerd & cri-tools.
+GOPATH=$(mktemp -d --tmpdir gopathXXXXX)
+install_helper github.com/containerd/containerd v1.2.2 "${GOPATH}"
+install_helper github.com/kubernetes-sigs/cri-tools v1.11.0 "${GOPATH}"
+
+# Install gvisor-containerd-shim.
+declare -r base="https://storage.googleapis.com/cri-containerd-staging/gvisor-containerd-shim"
+declare -r latest=$(mktemp --tmpdir gvisor-containerd-shim-latest.XXXXXX)
+declare -r shim_path=$(mktemp --tmpdir gvisor-containerd-shim.XXXXXX)
+wget --no-verbose "${base}"/latest -O ${latest}
+wget --no-verbose "${base}"/gvisor-containerd-shim-$(cat ${latest}) -O ${shim_path}
+chmod +x ${shim_path}
+mv ${shim_path} /usr/local/bin
+
+# Configure containerd-shim.
+declare -r shim_config_path=/etc/containerd
+declare -r shim_config_tmp_path=$(mktemp --tmpdir gvisor-containerd-shim.XXXXXX.toml)
+mkdir -p ${shim_config_path}
+cat > ${shim_config_tmp_path} <<-EOF
+    runc_shim = "/usr/local/bin/containerd-shim"
+
+[runsc_config]
+    debug = "true"
+    debug-log = "/tmp/runsc-logs/"
+    strace = "true"
+    file-access = "shared"
+EOF
+mv ${shim_config_tmp_path} ${shim_config_path}
+
+# Configure CNI.
+(cd "${GOPATH}" && GOPATH="${GOPATH}" \
+    src/github.com/containerd/containerd/script/setup/install-cni)
+
+# Cleanup the above.
+rm -rf "${GOPATH}"
+rm -rf "${latest}"
+rm -rf "${shim_path}"
+rm -rf "${shim_config_tmp_path}"
diff --git a/tools/vm/ubuntu1604/40_kokoro.sh b/tools/vm/ubuntu1604/40_kokoro.sh
new file mode 100755
index 000000000..06a1e6c48
--- /dev/null
+++ b/tools/vm/ubuntu1604/40_kokoro.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# Copyright 2019 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xeo pipefail
+
+# Declare kokoro's required public keys.
+declare -r ssh_public_keys=(
+    "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDg7L/ZaEauETWrPklUTky3kvxqQfe2Ax/2CsSqhNIGNMnK/8d79CHlmY9+dE1FFQ/RzKNCaltgy7XcN/fCYiCZr5jm2ZtnLuGNOTzupMNhaYiPL419qmL+5rZXt4/dWTrsHbFRACxT8j51PcRMO5wgbL0Bg2XXimbx8kDFaurL2gqduQYqlu4lxWCaJqOL71WogcimeL63Nq/yeH5PJPWpqE4P9VUQSwAzBWFK/hLeds/AiP3MgVS65qHBnhq0JsHy8JQsqjZbG7Iidt/Ll0+gqzEbi62gDIcczG4KC0iOVzDDP/1BxDtt1lKeA23ll769Fcm3rJyoBMYxjvdw1TDx sabujp@trigger.mtv.corp.google.com"
+    "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNgGK/hCdjmulHfRE3hp4rZs38NCR8yAh0eDsztxqGcuXnuSnL7jOlRrbcQpremJ84omD4eKrIpwJUs+YokMdv4= sabujp@trigger.svl.corp.google.com"
+)
+
+# Install dependencies.
+while true; do
+  if (apt-get update && apt-get install -y \
+      rsync \
+      coreutils \
+      python-psutil \
+      qemu-kvm \
+      python-pip \
+      python3-pip \
+      zip); then
+    break
+  fi
+  result=$?
+  if [[ $result -ne 100 ]]; then
+    exit $result
+  fi
+done
+
+# junitparser is used to merge junit xml files.
+pip install junitparser
+
+# We need a kbuilder user.
+if useradd -c "kbuilder user" -m -s /bin/bash kbuilder; then
+    # User was added successfully; we add the relevant SSH keys here.
+    mkdir -p ~kbuilder/.ssh
+    (IFS=$'\n'; echo "${ssh_public_keys[*]}") > ~kbuilder/.ssh/authorized_keys
+    chmod 0600 ~kbuilder/.ssh/authorized_keys
+    chown -R kbuilder ~kbuilder/.ssh
+fi
+
+# Give passwordless sudo access.
+cat > /etc/sudoers.d/kokoro <<EOF
+kbuilder ALL=(ALL) NOPASSWD:ALL
+EOF
+
+# Ensure we can run Docker without sudo.
+usermod -aG docker kbuilder
+
+# Ensure that we can access kvm.
+usermod -aG kvm kbuilder
+
+# Ensure that /tmpfs exists and is writable by kokoro.
+#
+# Note that kokoro will typically attach a second disk (sdb) to the instance
+# that is used for the /tmpfs volume. In the future we could setup an init
+# script that formats and mounts this here; however, we don't expect our build
+# artifacts to be that large.
+mkdir -p /tmpfs && chmod 0777 /tmpfs && touch /tmpfs/READY
diff --git a/tools/vm/ubuntu1604/BUILD b/tools/vm/ubuntu1604/BUILD
new file mode 100644
index 000000000..ab1df0c4c
--- /dev/null
+++ b/tools/vm/ubuntu1604/BUILD
@@ -0,0 +1,7 @@
+package(licenses = ["notice"])
+
+filegroup(
+    name = "ubuntu1604",
+    srcs = glob(["*.sh"]),
+    visibility = ["//:sandbox"],
+)
diff --git a/tools/vm/ubuntu1804/BUILD b/tools/vm/ubuntu1804/BUILD
new file mode 100644
index 000000000..0c8856dde
--- /dev/null
+++ b/tools/vm/ubuntu1804/BUILD
@@ -0,0 +1,7 @@
+package(licenses = ["notice"])
+
+alias(
+    name = "ubuntu1804",
+    actual = "//tools/vm/ubuntu1604",
+    visibility = ["//:sandbox"],
+)
diff --git a/tools/vm/zone.sh b/tools/vm/zone.sh
new file mode 100755
index 000000000..79569fb19
--- /dev/null
+++ b/tools/vm/zone.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Copyright 2020 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+exec gcloud config get-value compute/zone