diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/build.sh | 49 | ||||
| -rwxr-xr-x | scripts/common.sh | 59 | ||||
| -rwxr-xr-x | scripts/common_bazel.sh | 37 | ||||
| -rwxr-xr-x | scripts/dev.sh | 73 | ||||
| -rwxr-xr-x | scripts/docker_tests.sh | 6 | ||||
| -rwxr-xr-x | scripts/go.sh | 11 | ||||
| -rwxr-xr-x | scripts/hostnet_tests.sh | 5 | ||||
| -rwxr-xr-x | scripts/kvm_tests.sh | 10 | ||||
| -rwxr-xr-x | scripts/overlay_tests.sh | 5 | ||||
| -rwxr-xr-x | scripts/release.sh | 6 | ||||
| -rwxr-xr-x | scripts/root_tests.sh | 6 |
11 files changed, 218 insertions, 49 deletions
diff --git a/scripts/build.sh b/scripts/build.sh index 293d87093..b3a6e4e7a 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -16,6 +16,9 @@ source $(dirname $0)/common.sh +# Install required packages for make_repository.sh et al. +sudo apt-get update && sudo apt-get install -y dpkg-sig coreutils apt-utils + # Build runsc. runsc=$(build -c opt //runsc) @@ -24,16 +27,19 @@ pkg=$(build -c opt --host_force_python=py2 //runsc:runsc-debian) # Build a repository, if the key is available. if [[ -v KOKORO_REPO_KEY ]]; then - repo=$(tools/make_repository.sh "${KOKORO_REPO_KEY}" gvisor-bot@google.com) + repo=$(tools/make_repository.sh "${KOKORO_KEYSTORE_DIR}/${KOKORO_REPO_KEY}" gvisor-bot@google.com main ${pkg}) fi # Install installs artifacts. install() { - mkdir -p $1 - cp "${runsc}" "$1"/runsc - sha512sum "$1"/runsc | awk '{print $1 " runsc"}' > "$1"/runsc.sha512 + local -r binaries_dir="$1" + local -r repo_dir="$2" + mkdir -p "${binaries_dir}" + cp -f "${runsc}" "${binaries_dir}"/runsc + sha512sum "${binaries_dir}"/runsc | awk '{print $1 " runsc"}' > "${binaries_dir}"/runsc.sha512 if [[ -v repo ]]; then - cp -a "${repo}" "${latest_dir}"/repo + rm -rf "${repo_dir}" && mkdir -p "$(dirname "${repo_dir}")" + cp -a "${repo}" "${repo_dir}" fi } @@ -41,22 +47,33 @@ install() { # current date. If the current commit happens to correpond to a tag, then we # will also move everything into a directory named after the given tag. if [[ -v KOKORO_ARTIFACTS_DIR ]]; then - if [[ "${KOKORO_BUILD_NIGHTLY}" == "true" ]]; then + if [[ "${KOKORO_BUILD_NIGHTLY:-false}" == "true" ]]; then # The "latest" directory and current date. - install "${KOKORO_ARTIFACTS_DIR}/nightly/latest" - install "${KOKORO_ARTIFACTS_DIR}/nightly/$(date -Idate)" + stamp="$(date -Idate)" + install "${KOKORO_ARTIFACTS_DIR}/nightly/latest" \ + "${KOKORO_ARTIFACTS_DIR}/dists/nightly/latest" + install "${KOKORO_ARTIFACTS_DIR}/nightly/${stamp}" \ + "${KOKORO_ARTIFACTS_DIR}/dists/nightly/${stamp}" else # Is it a tagged release? Build that instead. In that case, we also try to # update the base release directory, in case this is an update. Finally, we # update the "release" directory, which has the last released version. - tag="$(git describe --exact-match --tags HEAD)" - if ! [[ -z "${tag}" ]]; then - install "${KOKORO_ARTIFACTS_DIR}/${tag}" - base=$(echo "${tag}" | cut -d'.' -f1) - if [[ "${base}" != "${tag}" ]]; then - install "${KOKORO_ARTIFACTS_DIR}/${base}" - fi - install "${KOKORO_ARTIFACTS_DIR}/release" + tags="$(git tag --points-at HEAD)" + if ! [[ -z "${tags}" ]]; then + # Note that a given commit can match any number of tags. We have to + # iterate through all possible tags and produce associated artifacts. + for tag in ${tags}; do + name=$(echo "${tag}" | cut -d'-' -f2) + base=$(echo "${name}" | cut -d'.' -f1) + install "${KOKORO_ARTIFACTS_DIR}/release/${name}" \ + "${KOKORO_ARTIFACTS_DIR}/dists/${name}" + if [[ "${base}" != "${tag}" ]]; then + install "${KOKORO_ARTIFACTS_DIR}/release/${base}" \ + "${KOKORO_ARTIFACTS_DIR}/dists/${base}" + fi + install "${KOKORO_ARTIFACTS_DIR}/release/latest" \ + "${KOKORO_ARTIFACTS_DIR}/dists/latest" + done fi fi fi diff --git a/scripts/common.sh b/scripts/common.sh index f2b9e24d8..6dabad141 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -14,10 +14,67 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -xeo pipefail +set -xeou pipefail if [[ -f $(dirname $0)/common_google.sh ]]; then source $(dirname $0)/common_google.sh else source $(dirname $0)/common_bazel.sh fi + +# Ensure it attempts to collect logs in all cases. +trap collect_logs EXIT + +function set_runtime() { + RUNTIME=${1:-runsc} + RUNSC_BIN=/tmp/"${RUNTIME}"/runsc + RUNSC_LOGS_DIR="$(dirname ${RUNSC_BIN})"/logs + RUNSC_LOGS="${RUNSC_LOGS_DIR}"/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND% +} + +function test_runsc() { + test --test_arg=--runtime=${RUNTIME} "$@" +} + +function install_runsc_for_test() { + local -r test_name=$1 + shift + if [[ -z "${test_name}" ]]; then + echo "Missing mandatory test name" + exit 1 + fi + + # Add test to the name, so it doesn't conflict with other runtimes. + set_runtime $(find_branch_name)_"${test_name}" + + # ${RUNSC_TEST_NAME} is set by tests (see dockerutil) to pass the test name + # down to the runtime. + install_runsc "${RUNTIME}" \ + --TESTONLY-test-name-env=RUNSC_TEST_NAME \ + --debug \ + --strace \ + --log-packets \ + "$@" +} + +# Installs the runsc with given runtime name. set_runtime must have been called +# to set runtime and logs location. +function install_runsc() { + local -r runtime=$1 + shift + + # Prepare the runtime binary. + local -r output=$(build //runsc) + mkdir -p "$(dirname ${RUNSC_BIN})" + cp -f "${output}" "${RUNSC_BIN}" + chmod 0755 "${RUNSC_BIN}" + + # Install the runtime. + sudo "${RUNSC_BIN}" install --experimental=true --runtime="${runtime}" -- --debug-log "${RUNSC_LOGS}" "$@" + + # Clear old logs files that may exist. + sudo rm -f "${RUNSC_LOGS_DIR}"/* + + # Restart docker to pick up the new runtime configuration. + sudo systemctl restart docker +} diff --git a/scripts/common_bazel.sh b/scripts/common_bazel.sh index 42248cb25..dde0b51ed 100755 --- a/scripts/common_bazel.sh +++ b/scripts/common_bazel.sh @@ -48,20 +48,12 @@ fi # Wrap bazel. function build() { - bazel build "${BAZEL_RBE_FLAGS[@]}" "${BAZEL_RBE_AUTH_FLAGS[@]}" "${BAZEL_FLAGS[@]}" "$@" + bazel build "${BAZEL_RBE_FLAGS[@]}" "${BAZEL_RBE_AUTH_FLAGS[@]}" "${BAZEL_FLAGS[@]}" "$@" 2>&1 | + tee /dev/fd/2 | grep -E '^ bazel-bin/' | awk '{ print $1; }' } function test() { - (bazel test "${BAZEL_RBE_FLAGS[@]}" "${BAZEL_RBE_AUTH_FLAGS[@]}" "${BAZEL_FLAGS[@]}" "$@" && rc=0) || rc=$? - - # Zip out everything into a convenient form. - if [[ -v KOKORO_ARTIFACTS_DIR ]]; then - find -L "bazel-testlogs" -name "test.xml" -o -name "test.log" -o -name "outputs.zip" | - tar --create --files-from - --transform 's/test\./sponge_log./' | - tar --extract --directory ${KOKORO_ARTIFACTS_DIR} - fi - - return $rc + bazel test "${BAZEL_RBE_FLAGS[@]}" "${BAZEL_RBE_AUTH_FLAGS[@]}" "${BAZEL_FLAGS[@]}" "$@" } function run() { @@ -75,3 +67,26 @@ function run_as_root() { shift bazel run --run_under="sudo" "${binary}" -- "$@" } + +function collect_logs() { + # Zip out everything into a convenient form. + if [[ -v KOKORO_ARTIFACTS_DIR ]] && [[ -e bazel-testlogs ]]; then + # Move test logs to Kokoro directory. tar is used to conveniently perform + # renames while moving files. + find -L "bazel-testlogs" -name "test.xml" -o -name "test.log" -o -name "outputs.zip" | + tar --create --files-from - --transform 's/test\./sponge_log./' | + tar --extract --directory ${KOKORO_ARTIFACTS_DIR} + + # Collect sentry logs, if any. + if [[ -v RUNSC_LOGS_DIR ]] && [[ -d "${RUNSC_LOGS_DIR}" ]]; then + local -r logs=$(ls "${RUNSC_LOGS_DIR}") + if [[ -z "${logs}" ]]; then + tar --create --gzip --file="${KOKORO_ARTIFACTS_DIR}/${RUNTIME}.tar.gz" -C "${RUNSC_LOGS_DIR}" . + fi + fi + fi +} + +function find_branch_name() { + git branch --show-current || git rev-parse HEAD || bazel info workspace | xargs basename +} diff --git a/scripts/dev.sh b/scripts/dev.sh new file mode 100755 index 000000000..ee74dcb72 --- /dev/null +++ b/scripts/dev.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +# common.sh sets '-x', but it's annoying to see so much output. +set +x + +# Defaults +declare -i REFRESH=0 +declare NAME=$(find_branch_name) + +while [[ $# -gt 0 ]]; do + case "$1" in + --refresh) + REFRESH=1 + ;; + --help) + echo "Use this script to build and install runsc with Docker." + echo + echo "usage: $0 [--refresh] [runtime_name]" + exit 1 + ;; + *) + NAME=$1 + ;; + esac + shift +done + +set_runtime "${NAME}" +echo +echo "Using runtime=${RUNTIME}" +echo + +echo Building runsc... +# Build first and fail on error. $() prevents "set -e" from reporting errors. +build //runsc +declare OUTPUT="$(build //runsc)" + +if [[ ${REFRESH} -eq 0 ]]; then + install_runsc "${RUNTIME}" --net-raw + install_runsc "${RUNTIME}-d" --net-raw --debug --strace --log-packets + + echo + echo "Runtimes ${RUNTIME} and ${RUNTIME}-d (debug enabled) setup." + echo "Use --runtime="${RUNTIME}" with your Docker command." + echo " docker run --rm --runtime="${RUNTIME}" --rm hello-world" + echo + echo "If you rebuild, use $0 --refresh." + +else + mkdir -p "$(dirname ${RUNSC_BIN})" + cp -f ${OUTPUT} "${RUNSC_BIN}" + + echo + echo "Runtime ${RUNTIME} refreshed." +fi + +echo "Logs are in: ${RUNSC_LOGS_DIR}" diff --git a/scripts/docker_tests.sh b/scripts/docker_tests.sh index d6b18a35b..72ba05260 100755 --- a/scripts/docker_tests.sh +++ b/scripts/docker_tests.sh @@ -16,7 +16,5 @@ source $(dirname $0)/common.sh -# Install the runtime and perform basic tests. -run_as_root //runsc install --experimental=true -- --debug --strace --log-packets -sudo systemctl restart docker -test //test/image:image_test //test/e2e:integration_test +install_runsc_for_test docker +test_runsc //test/image:image_test //test/e2e:integration_test diff --git a/scripts/go.sh b/scripts/go.sh index e49d76c6d..0dbfb7747 100755 --- a/scripts/go.sh +++ b/scripts/go.sh @@ -29,6 +29,15 @@ git checkout go && git clean -f go build ./... # Push, if required. -if [[ "${KOKORO_GO_PUSH}" == "true" ]]; then +if [[ -v KOKORO_GO_PUSH ]] && [[ "${KOKORO_GO_PUSH}" == "true" ]]; then + if [[ -v KOKORO_GITHUB_ACCESS_TOKEN ]]; then + git config --global credential.helper cache + git credential approve <<EOF +protocol=https +host=github.com +username=$(cat "${KOKORO_KEYSTORE_DIR}/${KOKORO_GITHUB_ACCESS_TOKEN}") +password=x-oauth-basic +EOF + fi git push origin go:go fi diff --git a/scripts/hostnet_tests.sh b/scripts/hostnet_tests.sh index 0631c5510..41298293d 100755 --- a/scripts/hostnet_tests.sh +++ b/scripts/hostnet_tests.sh @@ -17,6 +17,5 @@ source $(dirname $0)/common.sh # Install the runtime and perform basic tests. -run_as_root //runsc install --experimental=true -- --debug --strace --log-packets --network=host -sudo systemctl restart docker -test --test_arg=-checkpoint=false //test/image:image_test //test/e2e:integration_test +install_runsc_for_test hostnet --network=host +test_runsc --test_arg=-checkpoint=false //test/image:image_test //test/e2e:integration_test diff --git a/scripts/kvm_tests.sh b/scripts/kvm_tests.sh index 5cb7aa007..5662401df 100755 --- a/scripts/kvm_tests.sh +++ b/scripts/kvm_tests.sh @@ -20,11 +20,9 @@ source $(dirname $0)/common.sh (lsmod | grep -E '^(kvm_intel|kvm_amd)') || sudo modprobe kvm sudo chmod a+rw /dev/kvm -# Run all KVM-tagged tests (locally). -test --test_strategy=standalone --test_tag_filters=requires-kvm //... -test --test_strategy=standalone //pkg/sentry/platform/kvm:kvm_test +# Run all KVM platform tests (locally). +run_as_root //pkg/sentry/platform/kvm:kvm_test # Install the KVM runtime and run all integration tests. -run_as_root //runsc install --experimental=true -- --debug --strace --log-packets --platform=kvm -sudo systemctl restart docker -test --test_strategy=standalone //test/image:image_test //test/e2e:integration_test +install_runsc_for_test kvm --platform=kvm +test_runsc //test/image:image_test //test/e2e:integration_test diff --git a/scripts/overlay_tests.sh b/scripts/overlay_tests.sh index 651a51f70..2a1f12c0b 100755 --- a/scripts/overlay_tests.sh +++ b/scripts/overlay_tests.sh @@ -17,6 +17,5 @@ source $(dirname $0)/common.sh # Install the runtime and perform basic tests. -run_as_root //runsc install --experimental=true -- --debug --strace --log-packets --overlay -sudo systemctl restart docker -test //test/image:image_test //test/e2e:integration_test +install_runsc_for_test overlay --overlay +test_runsc //test/image:image_test //test/e2e:integration_test diff --git a/scripts/release.sh b/scripts/release.sh index 422319500..b936bcc77 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -26,9 +26,13 @@ if ! [[ -v KOKORO_RELEASE_TAG ]]; then exit 1 fi +# Unless an explicit releaser is provided, use the bot e-mail. +declare -r KOKORO_RELEASE_AUTHOR=${KOKORO_RELEASE_AUTHOR:-gvisor-bot} +declare -r EMAIL=${EMAIL:-${KOKORO_RELEASE_AUTHOR}@google.com} + # Ensure we have an appropriate configuration for the tag. git config --get user.name || git config user.name "gVisor-bot" -git config --get user.email || git config user.email "gvisor-bot@google.com" +git config --get user.email || git config user.email "${EMAIL}" # Run the release tool, which pushes to the origin repository. tools/tag_release.sh "${KOKORO_RELEASE_COMMIT}" "${KOKORO_RELEASE_TAG}" diff --git a/scripts/root_tests.sh b/scripts/root_tests.sh index e42c0e3ec..4e4fcc76b 100755 --- a/scripts/root_tests.sh +++ b/scripts/root_tests.sh @@ -26,6 +26,6 @@ chmod +x ${shim_path} sudo mv ${shim_path} /usr/local/bin/gvisor-containerd-shim # Run the tests that require root. -run_as_root //runsc install --experimental=true -- --debug --strace --log-packets -sudo systemctl restart docker -run_as_root //test/root:root_test +install_runsc_for_test root +run_as_root //test/root:root_test --runtime=${RUNTIME} + |