diff options
Diffstat (limited to 'tools/images')
| -rw-r--r-- | tools/images/BUILD | 9 | ||||
| -rw-r--r-- | tools/images/README.md | 42 | ||||
| -rwxr-xr-x | tools/images/build.sh | 36 | ||||
| -rw-r--r-- | tools/images/defs.bzl | 136 | ||||
| -rwxr-xr-x | tools/images/ubuntu1604/10_core.sh | 15 | ||||
| -rwxr-xr-x | tools/images/ubuntu1604/20_bazel.sh | 12 | ||||
| -rwxr-xr-x | tools/images/ubuntu1604/25_docker.sh | 33 | ||||
| -rwxr-xr-x | tools/images/ubuntu1604/30_containerd.sh | 12 | ||||
| -rwxr-xr-x | tools/images/ubuntu1604/40_kokoro.sh | 17 | ||||
| -rwxr-xr-x | tools/images/zone.sh | 17 |
10 files changed, 235 insertions, 94 deletions
diff --git a/tools/images/BUILD b/tools/images/BUILD index fe11f08a3..8d319e3e4 100644 --- a/tools/images/BUILD +++ b/tools/images/BUILD @@ -6,14 +6,9 @@ package( licenses = ["notice"], ) -genrule( +sh_binary( name = "zone", - outs = ["zone.txt"], - cmd = "gcloud config get-value compute/zone > $@", - tags = [ - "local", - "manual", - ], + srcs = ["zone.sh"], ) sh_binary( diff --git a/tools/images/README.md b/tools/images/README.md new file mode 100644 index 000000000..26c0f84f2 --- /dev/null +++ b/tools/images/README.md @@ -0,0 +1,42 @@ +# Images + +All commands in this directory require the `gcloud` project to be set. + +For example: `gcloud config set project gvisor-kokoro-testing`. + +Images can be generated by using the `vm_image` rule. This rule will generate a +binary target that builds an image in an idempotent way, and can be referenced +from other rules. + +For example: + +``` +vm_image( + name = "ubuntu", + project = "ubuntu-1604-lts", + family = "ubuntu-os-cloud", + scripts = [ + "script.sh", + "other.sh", + ], +) +``` + +These images can be built manually by executing the target. The output on +`stdout` will be the image id (in the current project). + +Images are always named per the hash of all the hermetic input scripts. This +allows images to be memoized quickly and easily. + +The `vm_test` rule can be used to execute a command remotely. This is still +under development however, and will likely change over time. + +For example: + +``` +vm_test( + name = "mycommand", + image = ":ubuntu", + targets = [":test"], +) +``` diff --git a/tools/images/build.sh b/tools/images/build.sh index be462d556..f39f723b8 100755 --- a/tools/images/build.sh +++ b/tools/images/build.sh @@ -19,7 +19,7 @@ # image_setup.sh script. This script should be idempotent, as we memoize the # setup script with a hash and check for that name. -set -xeou pipefail +set -eou pipefail # Parameters. declare -r USERNAME=${USERNAME:-test} @@ -34,10 +34,10 @@ declare -r INSTANCE_NAME=$(mktemp -u build-XXXXXX | tr A-Z a-z) # Hash inputs in order to memoize the produced image. declare -r SETUP_HASH=$( (echo ${USERNAME} ${IMAGE_PROJECT} ${IMAGE_FAMILY} && cat "$@") | sha256sum - | cut -d' ' -f1 | cut -c 1-16) -declare -r IMAGE_NAME=${IMAGE_FAMILY:-image-}${SETUP_HASH} +declare -r IMAGE_NAME=${IMAGE_FAMILY:-image}-${SETUP_HASH} # Does the image already exist? Skip the build. -declare -r existing=$(gcloud compute images list --filter="name=(${IMAGE_NAME})" --format="value(name)") +declare -r existing=$(set -x; gcloud compute images list --filter="name=(${IMAGE_NAME})" --format="value(name)") if ! [[ -z "${existing}" ]]; then echo "${existing}" exit 0 @@ -48,54 +48,66 @@ export PATH=${PATH:-/bin:/usr/bin:/usr/local/bin} # Start a unique instance. Note that this instance will have a unique persistent # disk as it's boot disk with the same name as the instance. -gcloud compute instances create \ +(set -x; gcloud compute instances create \ --quiet \ --image-project "${IMAGE_PROJECT}" \ --image-family "${IMAGE_FAMILY}" \ --boot-disk-size "200GB" \ --zone "${ZONE}" \ - "${INSTANCE_NAME}" >/dev/null + "${INSTANCE_NAME}" >/dev/null) function cleanup { - gcloud compute instances delete --quiet --zone "${ZONE}" "${INSTANCE_NAME}" + (set -x; gcloud compute instances delete --quiet --zone "${ZONE}" "${INSTANCE_NAME}") } trap cleanup EXIT # Wait for the instance to become available (up to 5 minutes). +echo -n "Waiting for ${INSTANCE_NAME}" declare timeout=300 declare success=0 +declare internal="" declare -r start=$(date +%s) declare -r end=$((${start}+${timeout})) while [[ "$(date +%s)" -lt "${end}" ]] && [[ "${success}" -lt 3 ]]; do + echo -n "." if gcloud compute ssh --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- env - true 2>/dev/null; then success=$((${success}+1)) + elif gcloud compute ssh --internal-ip --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- env - true 2>/dev/null; then + success=$((${success}+1)) + internal="--internal-ip" fi done + if [[ "${success}" -eq "0" ]]; then echo "connect timed out after ${timeout} seconds." exit 1 +else + echo "done." fi # Run the install scripts provided. for arg; do - gcloud compute ssh --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- sudo bash - <"${arg}" >/dev/null + (set -x; gcloud compute ssh ${internal} \ + --zone "${ZONE}" \ + "${USERNAME}"@"${INSTANCE_NAME}" -- \ + sudo bash - <"${arg}" >/dev/null) done # Stop the instance; required before creating an image. -gcloud compute instances stop --quiet --zone "${ZONE}" "${INSTANCE_NAME}" >/dev/null +(set -x; gcloud compute instances stop --quiet --zone "${ZONE}" "${INSTANCE_NAME}" >/dev/null) # Create a snapshot of the instance disk. -gcloud compute disks snapshot \ +(set -x; gcloud compute disks snapshot \ --quiet \ --zone "${ZONE}" \ --snapshot-names="${SNAPSHOT_NAME}" \ - "${INSTANCE_NAME}" >/dev/null + "${INSTANCE_NAME}" >/dev/null) # Create the disk image. -gcloud compute images create \ +(set -x; gcloud compute images create \ --quiet \ --source-snapshot="${SNAPSHOT_NAME}" \ --licenses="https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" \ - "${IMAGE_NAME}" >/dev/null + "${IMAGE_NAME}" >/dev/null) # Finish up. echo "${IMAGE_NAME}" diff --git a/tools/images/defs.bzl b/tools/images/defs.bzl index de365d153..2847e1847 100644 --- a/tools/images/defs.bzl +++ b/tools/images/defs.bzl @@ -1,76 +1,49 @@ -"""Image configuration. - -Images can be generated by using the vm_image rule. For example, - - vm_image( - name = "ubuntu", - project = "...", - family = "...", - scripts = [ - "script.sh", - "other.sh", - ], - ) - -This will always create an vm_image in the current default gcloud project. The -rule has a text file as its output containing the image name. This will enforce -serialization for all dependent rules. - -Images are always named per the hash of all the hermetic input scripts. This -allows images to be memoized quickly and easily. - -The vm_test rule can be used to execute a command remotely. For example, - - vm_test( - name = "mycommand", - image = ":myimage", - targets = [":test"], - ) -""" +"""Image configuration. See README.md.""" load("//tools:defs.bzl", "default_installer") -def _vm_image_impl(ctx): +# vm_image_builder is a rule that will construct a shell script that actually +# generates a given VM image. Note that this does not _run_ the shell script +# (although it can be run manually). It will be run manually during generation +# of the vm_image target itself. This level of indirection is used so that the +# build system itself only runs the builder once when multiple targets depend +# on it, avoiding a set of races and conflicts. +def _vm_image_builder_impl(ctx): + # Generate a binary that actually builds the image. + builder = ctx.actions.declare_file(ctx.label.name) script_paths = [] for script in ctx.files.scripts: script_paths.append(script.short_path) + builder_content = "\n".join([ + "#!/bin/bash", + "export ZONE=$(%s)" % ctx.files.zone[0].short_path, + "export USERNAME=%s" % ctx.attr.username, + "export IMAGE_PROJECT=%s" % ctx.attr.project, + "export IMAGE_FAMILY=%s" % ctx.attr.family, + "%s %s" % (ctx.files._builder[0].short_path, " ".join(script_paths)), + "", + ]) + ctx.actions.write(builder, builder_content, is_executable = True) - resolved_inputs, argv, runfiles_manifests = ctx.resolve_command( - command = "USERNAME=%s ZONE=$(cat %s) IMAGE_PROJECT=%s IMAGE_FAMILY=%s %s %s > %s" % - ( - ctx.attr.username, - ctx.files.zone[0].path, - ctx.attr.project, - ctx.attr.family, - ctx.executable.builder.path, - " ".join(script_paths), - ctx.outputs.out.path, - ), - tools = [ctx.attr.builder] + ctx.attr.scripts, - ) - - ctx.actions.run_shell( - tools = resolved_inputs, - outputs = [ctx.outputs.out], - progress_message = "Building image...", - execution_requirements = {"local": "true"}, - command = argv, - input_manifests = runfiles_manifests, - ) + # Note that the scripts should only be files, and should not include any + # indirect transitive dependencies. The build script wouldn't work. return [DefaultInfo( - files = depset([ctx.outputs.out]), - runfiles = ctx.runfiles(files = [ctx.outputs.out]), + executable = builder, + runfiles = ctx.runfiles( + files = ctx.files.scripts + ctx.files._builder + ctx.files.zone, + ), )] -_vm_image = rule( +vm_image_builder = rule( attrs = { - "builder": attr.label( + "_builder": attr.label( executable = True, default = "//tools/images:builder", cfg = "host", ), "username": attr.string(default = "$(whoami)"), "zone": attr.label( + executable = True, default = "//tools/images:zone", cfg = "host", ), @@ -78,20 +51,55 @@ _vm_image = rule( "project": attr.string(mandatory = True), "scripts": attr.label_list(allow_files = True), }, - outputs = { - "out": "%{name}.txt", + executable = True, + implementation = _vm_image_builder_impl, +) + +# See vm_image_builder above. +def _vm_image_impl(ctx): + # Run the builder to generate our output. + echo = ctx.actions.declare_file(ctx.label.name) + resolved_inputs, argv, runfiles_manifests = ctx.resolve_command( + command = "echo -ne \"#!/bin/bash\\necho $(%s)\\n\" > %s && chmod 0755 %s" % ( + ctx.files.builder[0].path, + echo.path, + echo.path, + ), + tools = [ctx.attr.builder], + ) + ctx.actions.run_shell( + tools = resolved_inputs, + outputs = [echo], + progress_message = "Building image...", + execution_requirements = {"local": "true"}, + command = argv, + input_manifests = runfiles_manifests, + ) + + # Return just the echo command. All of the builder runfiles have been + # resolved and consumed in the generation of the trivial echo script. + return [DefaultInfo(executable = echo)] + +_vm_image = rule( + attrs = { + "builder": attr.label( + executable = True, + cfg = "host", + ), }, + executable = True, implementation = _vm_image_impl, ) -def vm_image(**kwargs): - _vm_image( - tags = [ - "local", - "manual", - ], +def vm_image(name, **kwargs): + vm_image_builder( + name = name + "_builder", **kwargs ) + _vm_image( + name = name, + builder = ":" + name + "_builder", + ) def _vm_test_impl(ctx): runner = ctx.actions.declare_file("%s-executer" % ctx.label.name) diff --git a/tools/images/ubuntu1604/10_core.sh b/tools/images/ubuntu1604/10_core.sh index 46dda6bb1..cd518d6ac 100755 --- a/tools/images/ubuntu1604/10_core.sh +++ b/tools/images/ubuntu1604/10_core.sh @@ -17,7 +17,20 @@ set -xeo pipefail # Install all essential build tools. -apt-get update && apt-get -y install make git-core build-essential linux-headers-$(uname -r) pkg-config +while true; do + if (apt-get update && apt-get install -y \ + make \ + git-core \ + build-essential \ + linux-headers-$(uname -r) \ + pkg-config); then + break + fi + result=$? + if [[ $result -ne 100 ]]; then + exit $result + fi +done # Install a recent go toolchain. if ! [[ -d /usr/local/go ]]; then diff --git a/tools/images/ubuntu1604/20_bazel.sh b/tools/images/ubuntu1604/20_bazel.sh index b33e1656c..bb7afa676 100755 --- a/tools/images/ubuntu1604/20_bazel.sh +++ b/tools/images/ubuntu1604/20_bazel.sh @@ -19,7 +19,17 @@ set -xeo pipefail declare -r BAZEL_VERSION=2.0.0 # Install bazel dependencies. -apt-get update && apt-get install -y openjdk-8-jdk-headless unzip +while true; do + if (apt-get update && apt-get install -y \ + openjdk-8-jdk-headless \ + unzip); then + break + fi + result=$? + if [[ $result -ne 100 ]]; then + exit $result + fi +done # Use the release installer. curl -L -o bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh diff --git a/tools/images/ubuntu1604/25_docker.sh b/tools/images/ubuntu1604/25_docker.sh index 1d3defcd3..11eea2d72 100755 --- a/tools/images/ubuntu1604/25_docker.sh +++ b/tools/images/ubuntu1604/25_docker.sh @@ -15,12 +15,20 @@ # limitations under the License. # Add dependencies. -apt-get update && apt-get -y install \ - apt-transport-https \ - ca-certificates \ - curl \ - gnupg-agent \ - software-properties-common +while true; do + if (apt-get update && apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg-agent \ + software-properties-common); then + break + fi + result=$? + if [[ $result -ne 100 ]]; then + exit $result + fi +done # Install the key. curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - @@ -32,4 +40,15 @@ add-apt-repository \ stable" # Install docker. -apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io +while true; do + if (apt-get update && apt-get install -y \ + docker-ce \ + docker-ce-cli \ + containerd.io); then + break + fi + result=$? + if [[ $result -ne 100 ]]; then + exit $result + fi +done diff --git a/tools/images/ubuntu1604/30_containerd.sh b/tools/images/ubuntu1604/30_containerd.sh index a7472bd1c..fb3699c12 100755 --- a/tools/images/ubuntu1604/30_containerd.sh +++ b/tools/images/ubuntu1604/30_containerd.sh @@ -34,7 +34,17 @@ install_helper() { } # Install dependencies for the crictl tests. -apt-get install -y btrfs-tools libseccomp-dev +while true; do + if (apt-get update && apt-get install -y \ + btrfs-tools \ + libseccomp-dev); then + break + fi + result=$? + if [[ $result -ne 100 ]]; then + exit $result + fi +done # Install containerd & cri-tools. GOPATH=$(mktemp -d --tmpdir gopathXXXXX) diff --git a/tools/images/ubuntu1604/40_kokoro.sh b/tools/images/ubuntu1604/40_kokoro.sh index 5f2dfc858..06a1e6c48 100755 --- a/tools/images/ubuntu1604/40_kokoro.sh +++ b/tools/images/ubuntu1604/40_kokoro.sh @@ -23,7 +23,22 @@ declare -r ssh_public_keys=( ) # Install dependencies. -apt-get update && apt-get install -y rsync coreutils python-psutil qemu-kvm python-pip python3-pip zip +while true; do + if (apt-get update && apt-get install -y \ + rsync \ + coreutils \ + python-psutil \ + qemu-kvm \ + python-pip \ + python3-pip \ + zip); then + break + fi + result=$? + if [[ $result -ne 100 ]]; then + exit $result + fi +done # junitparser is used to merge junit xml files. pip install junitparser diff --git a/tools/images/zone.sh b/tools/images/zone.sh new file mode 100755 index 000000000..79569fb19 --- /dev/null +++ b/tools/images/zone.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Copyright 2020 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec gcloud config get-value compute/zone |