diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/image_build.sh | 98 | ||||
| -rw-r--r-- | tools/images/BUILD | 9 | ||||
| -rw-r--r-- | tools/images/README.md | 42 | ||||
| -rwxr-xr-x | tools/images/build.sh | 35 | ||||
| -rw-r--r-- | tools/images/defs.bzl | 136 | ||||
| -rwxr-xr-x | tools/images/zone.sh | 17 |
6 files changed, 154 insertions, 183 deletions
diff --git a/tools/image_build.sh b/tools/image_build.sh deleted file mode 100755 index 9b20a740d..000000000 --- a/tools/image_build.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is responsible for building a new GCP image that: 1) has nested -# virtualization enabled, and 2) has been completely set up with the -# image_setup.sh script. This script should be idempotent, as we memoize the -# setup script with a hash and check for that name. -# -# The GCP project name should be defined via a gcloud config. - -set -xeo pipefail - -# Parameters. -declare -r ZONE=${ZONE:-us-central1-f} -declare -r USERNAME=${USERNAME:-test} -declare -r IMAGE_PROJECT=${IMAGE_PROJECT:-ubuntu-os-cloud} -declare -r IMAGE_FAMILY=${IMAGE_FAMILY:-ubuntu-1604-lts} - -# Random names. -declare -r DISK_NAME=$(mktemp -u disk-XXXXXX | tr A-Z a-z) -declare -r SNAPSHOT_NAME=$(mktemp -u snapshot-XXXXXX | tr A-Z a-z) -declare -r INSTANCE_NAME=$(mktemp -u build-XXXXXX | tr A-Z a-z) - -# Hashes inputs. -declare -r SETUP_BLOB=$(echo ${ZONE} ${USERNAME} ${IMAGE_PROJECT} ${IMAGE_FAMILY} && sha256sum "$@") -declare -r SETUP_HASH=$(echo ${SETUP_BLOB} | sha256sum - | cut -d' ' -f1 | cut -c 1-16) -declare -r IMAGE_NAME=${IMAGE_NAME:-image-}${SETUP_HASH} - -# Does the image already exist? Skip the build. -declare -r existing=$(gcloud compute images list --filter="name=(${IMAGE_NAME})" --format="value(name)") -if ! [[ -z "${existing}" ]]; then - echo "${existing}" - exit 0 -fi - -# Set the zone for all actions. -gcloud config set compute/zone "${ZONE}" - -# Start a unique instance. Note that this instance will have a unique persistent -# disk as it's boot disk with the same name as the instance. -gcloud compute instances create \ - --quiet \ - --image-project "${IMAGE_PROJECT}" \ - --image-family "${IMAGE_FAMILY}" \ - --boot-disk-size "200GB" \ - "${INSTANCE_NAME}" -function cleanup { - gcloud compute instances delete --quiet "${INSTANCE_NAME}" -} -trap cleanup EXIT - -# Wait for the instance to become available. -declare attempts=0 -while [[ "${attempts}" -lt 30 ]]; do - attempts=$((${attempts}+1)) - if gcloud compute ssh "${USERNAME}"@"${INSTANCE_NAME}" -- true; then - break - fi -done -if [[ "${attempts}" -ge 30 ]]; then - echo "too many attempts: failed" - exit 1 -fi - -# Run the install scripts provided. -for arg; do - gcloud compute ssh "${USERNAME}"@"${INSTANCE_NAME}" -- sudo bash - <"${arg}" -done - -# Stop the instance; required before creating an image. -gcloud compute instances stop --quiet "${INSTANCE_NAME}" - -# Create a snapshot of the instance disk. -gcloud compute disks snapshot \ - --quiet \ - --zone="${ZONE}" \ - --snapshot-names="${SNAPSHOT_NAME}" \ - "${INSTANCE_NAME}" - -# Create the disk image. -gcloud compute images create \ - --quiet \ - --source-snapshot="${SNAPSHOT_NAME}" \ - --licenses="https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" \ - "${IMAGE_NAME}" diff --git a/tools/images/BUILD b/tools/images/BUILD index 66ffd02aa..8d319e3e4 100644 --- a/tools/images/BUILD +++ b/tools/images/BUILD @@ -6,14 +6,9 @@ package( licenses = ["notice"], ) -genrule( +sh_binary( name = "zone", - outs = ["zone.txt"], - cmd = "gcloud config get-value compute/zone > \"$@\"", - tags = [ - "local", - "manual", - ], + srcs = ["zone.sh"], ) sh_binary( diff --git a/tools/images/README.md b/tools/images/README.md new file mode 100644 index 000000000..26c0f84f2 --- /dev/null +++ b/tools/images/README.md @@ -0,0 +1,42 @@ +# Images + +All commands in this directory require the `gcloud` project to be set. + +For example: `gcloud config set project gvisor-kokoro-testing`. + +Images can be generated by using the `vm_image` rule. This rule will generate a +binary target that builds an image in an idempotent way, and can be referenced +from other rules. + +For example: + +``` +vm_image( + name = "ubuntu", + project = "ubuntu-1604-lts", + family = "ubuntu-os-cloud", + scripts = [ + "script.sh", + "other.sh", + ], +) +``` + +These images can be built manually by executing the target. The output on +`stdout` will be the image id (in the current project). + +Images are always named per the hash of all the hermetic input scripts. This +allows images to be memoized quickly and easily. + +The `vm_test` rule can be used to execute a command remotely. This is still +under development however, and will likely change over time. + +For example: + +``` +vm_test( + name = "mycommand", + image = ":ubuntu", + targets = [":test"], +) +``` diff --git a/tools/images/build.sh b/tools/images/build.sh index f89f39cbd..f39f723b8 100755 --- a/tools/images/build.sh +++ b/tools/images/build.sh @@ -19,7 +19,7 @@ # image_setup.sh script. This script should be idempotent, as we memoize the # setup script with a hash and check for that name. -set -xeou pipefail +set -eou pipefail # Parameters. declare -r USERNAME=${USERNAME:-test} @@ -34,10 +34,10 @@ declare -r INSTANCE_NAME=$(mktemp -u build-XXXXXX | tr A-Z a-z) # Hash inputs in order to memoize the produced image. declare -r SETUP_HASH=$( (echo ${USERNAME} ${IMAGE_PROJECT} ${IMAGE_FAMILY} && cat "$@") | sha256sum - | cut -d' ' -f1 | cut -c 1-16) -declare -r IMAGE_NAME=${IMAGE_FAMILY:-image-}${SETUP_HASH} +declare -r IMAGE_NAME=${IMAGE_FAMILY:-image}-${SETUP_HASH} # Does the image already exist? Skip the build. -declare -r existing=$(gcloud compute images list --filter="name=(${IMAGE_NAME})" --format="value(name)") +declare -r existing=$(set -x; gcloud compute images list --filter="name=(${IMAGE_NAME})" --format="value(name)") if ! [[ -z "${existing}" ]]; then echo "${existing}" exit 0 @@ -48,28 +48,30 @@ export PATH=${PATH:-/bin:/usr/bin:/usr/local/bin} # Start a unique instance. Note that this instance will have a unique persistent # disk as it's boot disk with the same name as the instance. -gcloud compute instances create \ +(set -x; gcloud compute instances create \ --quiet \ --image-project "${IMAGE_PROJECT}" \ --image-family "${IMAGE_FAMILY}" \ --boot-disk-size "200GB" \ --zone "${ZONE}" \ - "${INSTANCE_NAME}" >/dev/null + "${INSTANCE_NAME}" >/dev/null) function cleanup { - gcloud compute instances delete --quiet --zone "${ZONE}" "${INSTANCE_NAME}" + (set -x; gcloud compute instances delete --quiet --zone "${ZONE}" "${INSTANCE_NAME}") } trap cleanup EXIT # Wait for the instance to become available (up to 5 minutes). +echo -n "Waiting for ${INSTANCE_NAME}" declare timeout=300 declare success=0 declare internal="" declare -r start=$(date +%s) declare -r end=$((${start}+${timeout})) while [[ "$(date +%s)" -lt "${end}" ]] && [[ "${success}" -lt 3 ]]; do - if gcloud compute ssh --zone "${internal}" "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- env - true 2>/dev/null; then + echo -n "." + if gcloud compute ssh --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- env - true 2>/dev/null; then success=$((${success}+1)) - elif gcloud compute ssh --zone --internal-ip "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- env - true 2>/dev/null; then + elif gcloud compute ssh --internal-ip --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- env - true 2>/dev/null; then success=$((${success}+1)) internal="--internal-ip" fi @@ -78,29 +80,34 @@ done if [[ "${success}" -eq "0" ]]; then echo "connect timed out after ${timeout} seconds." exit 1 +else + echo "done." fi # Run the install scripts provided. for arg; do - gcloud compute ssh --zone "${internal}" "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- sudo bash - <"${arg}" >/dev/null + (set -x; gcloud compute ssh ${internal} \ + --zone "${ZONE}" \ + "${USERNAME}"@"${INSTANCE_NAME}" -- \ + sudo bash - <"${arg}" >/dev/null) done # Stop the instance; required before creating an image. -gcloud compute instances stop --quiet --zone "${ZONE}" "${INSTANCE_NAME}" >/dev/null +(set -x; gcloud compute instances stop --quiet --zone "${ZONE}" "${INSTANCE_NAME}" >/dev/null) # Create a snapshot of the instance disk. -gcloud compute disks snapshot \ +(set -x; gcloud compute disks snapshot \ --quiet \ --zone "${ZONE}" \ --snapshot-names="${SNAPSHOT_NAME}" \ - "${INSTANCE_NAME}" >/dev/null + "${INSTANCE_NAME}" >/dev/null) # Create the disk image. -gcloud compute images create \ +(set -x; gcloud compute images create \ --quiet \ --source-snapshot="${SNAPSHOT_NAME}" \ --licenses="https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" \ - "${IMAGE_NAME}" >/dev/null + "${IMAGE_NAME}" >/dev/null) # Finish up. echo "${IMAGE_NAME}" diff --git a/tools/images/defs.bzl b/tools/images/defs.bzl index de365d153..2847e1847 100644 --- a/tools/images/defs.bzl +++ b/tools/images/defs.bzl @@ -1,76 +1,49 @@ -"""Image configuration. - -Images can be generated by using the vm_image rule. For example, - - vm_image( - name = "ubuntu", - project = "...", - family = "...", - scripts = [ - "script.sh", - "other.sh", - ], - ) - -This will always create an vm_image in the current default gcloud project. The -rule has a text file as its output containing the image name. This will enforce -serialization for all dependent rules. - -Images are always named per the hash of all the hermetic input scripts. This -allows images to be memoized quickly and easily. - -The vm_test rule can be used to execute a command remotely. For example, - - vm_test( - name = "mycommand", - image = ":myimage", - targets = [":test"], - ) -""" +"""Image configuration. See README.md.""" load("//tools:defs.bzl", "default_installer") -def _vm_image_impl(ctx): +# vm_image_builder is a rule that will construct a shell script that actually +# generates a given VM image. Note that this does not _run_ the shell script +# (although it can be run manually). It will be run manually during generation +# of the vm_image target itself. This level of indirection is used so that the +# build system itself only runs the builder once when multiple targets depend +# on it, avoiding a set of races and conflicts. +def _vm_image_builder_impl(ctx): + # Generate a binary that actually builds the image. + builder = ctx.actions.declare_file(ctx.label.name) script_paths = [] for script in ctx.files.scripts: script_paths.append(script.short_path) + builder_content = "\n".join([ + "#!/bin/bash", + "export ZONE=$(%s)" % ctx.files.zone[0].short_path, + "export USERNAME=%s" % ctx.attr.username, + "export IMAGE_PROJECT=%s" % ctx.attr.project, + "export IMAGE_FAMILY=%s" % ctx.attr.family, + "%s %s" % (ctx.files._builder[0].short_path, " ".join(script_paths)), + "", + ]) + ctx.actions.write(builder, builder_content, is_executable = True) - resolved_inputs, argv, runfiles_manifests = ctx.resolve_command( - command = "USERNAME=%s ZONE=$(cat %s) IMAGE_PROJECT=%s IMAGE_FAMILY=%s %s %s > %s" % - ( - ctx.attr.username, - ctx.files.zone[0].path, - ctx.attr.project, - ctx.attr.family, - ctx.executable.builder.path, - " ".join(script_paths), - ctx.outputs.out.path, - ), - tools = [ctx.attr.builder] + ctx.attr.scripts, - ) - - ctx.actions.run_shell( - tools = resolved_inputs, - outputs = [ctx.outputs.out], - progress_message = "Building image...", - execution_requirements = {"local": "true"}, - command = argv, - input_manifests = runfiles_manifests, - ) + # Note that the scripts should only be files, and should not include any + # indirect transitive dependencies. The build script wouldn't work. return [DefaultInfo( - files = depset([ctx.outputs.out]), - runfiles = ctx.runfiles(files = [ctx.outputs.out]), + executable = builder, + runfiles = ctx.runfiles( + files = ctx.files.scripts + ctx.files._builder + ctx.files.zone, + ), )] -_vm_image = rule( +vm_image_builder = rule( attrs = { - "builder": attr.label( + "_builder": attr.label( executable = True, default = "//tools/images:builder", cfg = "host", ), "username": attr.string(default = "$(whoami)"), "zone": attr.label( + executable = True, default = "//tools/images:zone", cfg = "host", ), @@ -78,20 +51,55 @@ _vm_image = rule( "project": attr.string(mandatory = True), "scripts": attr.label_list(allow_files = True), }, - outputs = { - "out": "%{name}.txt", + executable = True, + implementation = _vm_image_builder_impl, +) + +# See vm_image_builder above. +def _vm_image_impl(ctx): + # Run the builder to generate our output. + echo = ctx.actions.declare_file(ctx.label.name) + resolved_inputs, argv, runfiles_manifests = ctx.resolve_command( + command = "echo -ne \"#!/bin/bash\\necho $(%s)\\n\" > %s && chmod 0755 %s" % ( + ctx.files.builder[0].path, + echo.path, + echo.path, + ), + tools = [ctx.attr.builder], + ) + ctx.actions.run_shell( + tools = resolved_inputs, + outputs = [echo], + progress_message = "Building image...", + execution_requirements = {"local": "true"}, + command = argv, + input_manifests = runfiles_manifests, + ) + + # Return just the echo command. All of the builder runfiles have been + # resolved and consumed in the generation of the trivial echo script. + return [DefaultInfo(executable = echo)] + +_vm_image = rule( + attrs = { + "builder": attr.label( + executable = True, + cfg = "host", + ), }, + executable = True, implementation = _vm_image_impl, ) -def vm_image(**kwargs): - _vm_image( - tags = [ - "local", - "manual", - ], +def vm_image(name, **kwargs): + vm_image_builder( + name = name + "_builder", **kwargs ) + _vm_image( + name = name, + builder = ":" + name + "_builder", + ) def _vm_test_impl(ctx): runner = ctx.actions.declare_file("%s-executer" % ctx.label.name) diff --git a/tools/images/zone.sh b/tools/images/zone.sh new file mode 100755 index 000000000..79569fb19 --- /dev/null +++ b/tools/images/zone.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Copyright 2020 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec gcloud config get-value compute/zone |