diff options
Diffstat (limited to '.buildkite/hooks')
| -rw-r--r-- | .buildkite/hooks/post-command | 74 | ||||
| -rw-r--r-- | .buildkite/hooks/pre-command | 30 |
2 files changed, 104 insertions, 0 deletions
diff --git a/.buildkite/hooks/post-command b/.buildkite/hooks/post-command new file mode 100644 index 000000000..b0396bec7 --- /dev/null +++ b/.buildkite/hooks/post-command @@ -0,0 +1,74 @@ +# Upload test logs on failure, if there are any. +if test "${BUILDKITE_COMMAND_EXIT_STATUS}" -ne "0"; then + # Generate a metafile that ends with .output, and contains all the + # test failures that have been uploaded. These will all be sorted and + # aggregated by a failure stage in the build pipeline. + declare output=$(mktemp "${BUILDKITE_JOB_ID}".XXXXXX.output) + make -s testlogs 2>/dev/null | grep // | sort | uniq | ( + declare log_count=0 + while read target log; do + if test -z "${target}"; then + continue + fi + + # N.B. If *all* tests fail due to some common cause, then we will + # end up spending way too much time uploading logs. Instead, we just + # upload the first 10 and stop. That is hopefully enough to debug. + # + # We include this test in the metadata, but note that we cannot + # upload the actual test logs. The user should rerun locally. + log_count=$((${log_count}+1)) + if test "${log_count}" -ge 10; then + echo " * ${target} (no upload)" | tee -a "${output}" + else + buildkite-agent artifact upload "${log}" + echo " * [${target}](artifact://${log#/})" | tee -a "${output}" + fi + done + ) + + # Upload if we had outputs. + if test -s "${output}"; then + buildkite-agent artifact upload "${output}" + fi + rm -rf "${output}" + + # Attempt to clear the cache and shut down. + make clean || echo "make clean failed with code $?" + make bazel-shutdown || echo "make bazel-shutdown failed with code $?" +fi + +# Upload all profiles, and include in an annotation. +if test -d /tmp/profile; then + # Same as above. + declare profile_output=$(mktemp "${BUILDKITE_JOB_ID}".XXXXXX.profile_output) + for file in $(find /tmp/profile -name \*.pprof -print 2>/dev/null | sort); do + # Generate a link to speedscope, with a URL-encoded link to the BuildKite + # artifact location. Note that we use do a fixed URL encode below, since + # the link can be uniquely determined. If the storage location changes, + # this schema may break and these links may stop working. The artifacts + # uploaded however, will still work just fine. + profile_name="${file#/tmp/profile/}" + public_url="https://storage.googleapis.com/gvisor-buildkite/${BUILDKITE_BUILD_ID}/${BUILDKITE_JOB_ID}/${file#/}" + encoded_url=$(jq -rn --arg x "${public_url}" '$x|@uri') + encoded_title=$(jq -rn --arg x "${profile_name}" '$x|@uri') + profile_url="https://speedscope.app/#profileURL=${encoded_url}&title=${encoded_title}" + buildkite-agent artifact upload "${file}" + echo " * [${profile_name}](${profile_url}) ([pprof](artifact://${file#/}))" | tee -a "${profile_output}" + done + + # Upload if we had outputs. + if test -s "${profile_output}"; then + buildkite-agent artifact upload "${profile_output}" + fi + rm -rf "${profile_output}" + + # Remove stale profiles, which may be owned by root. + sudo rm -rf /tmp/profile +fi + +# Kill any running containers (clear state). +CONTAINERS="$(docker ps -q)" +if ! test -z "${CONTAINERS}"; then + docker container kill ${CONTAINERS} 2>/dev/null || true +fi diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100644 index 000000000..4f41fe021 --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,30 @@ +# Install packages we need. Docker must be installed and configured, +# as should Go itself. We just install some extra bits and pieces. +function install_pkgs() { + while true; do + if sudo apt-get update && sudo apt-get install -y "$@"; then + break + fi + done +} +install_pkgs graphviz jq curl binutils gnupg gnupg-agent linux-libc-dev \ + apt-transport-https ca-certificates software-properties-common + +# Setup for parallelization with PARTITION and TOTAL_PARTITIONS. +export PARTITION=${BUILDKITE_PARALLEL_JOB:-0} +PARTITION=$((${PARTITION}+1)) # 1-indexed, but PARALLEL_JOB is 0-indexed. +export TOTAL_PARTITIONS=${BUILDKITE_PARALLEL_JOB_COUNT:-1} + +# Ensure Docker has experimental enabled. +EXPERIMENTAL=$(sudo docker version --format='{{.Server.Experimental}}') +if test "${EXPERIMENTAL}" != "true"; then + make sudo TARGETS=//runsc:runsc ARGS="install --experimental=true" + sudo systemctl restart docker +fi + +# Helper for benchmarks, based on the branch. +if test "${BUILDKITE_BRANCH}" = "master"; then + export BENCHMARKS_OFFICIAL=true +else + export BENCHMARKS_OFFICIAL=false +fi
\ No newline at end of file |