diff options
Diffstat (limited to '.buildkite/hooks/post-command')
-rw-r--r-- | .buildkite/hooks/post-command | 60 |
1 files changed, 60 insertions, 0 deletions
diff --git a/.buildkite/hooks/post-command b/.buildkite/hooks/post-command new file mode 100644 index 000000000..8af1369a6 --- /dev/null +++ b/.buildkite/hooks/post-command @@ -0,0 +1,60 @@ +# Upload all relevant test failures. +make -s testlogs 2>/dev/null | grep // | sort | uniq | ( + declare log_count=0 + while read target log; do + if test -z "${target}"; then + continue + fi + + # N.B. If *all* tests fail due to some common cause, then we will + # end up spending way too much time uploading logs. Instead, we just + # upload the first 10 and stop. That is hopefully enough to debug. + # + # We include this test in the metadata, but note that we cannot + # upload the actual test logs. The user should rerun locally. + log_count=$((${log_count}+1)) + if test "${log_count}" -ge 10; then + echo " * ${target} (no upload)" | \ + buildkite-agent annotate --style error --context failures --append + else + buildkite-agent artifact upload "${log}" + echo " * [${target}](artifact://${log#/}) (${BUILDKITE_LABEL})" | \ + buildkite-agent annotate --style error --context failures --append + fi + done +) + +# Upload all profiles, and include in an annotation. +declare profile_output=$(mktemp --tmpdir) +for file in $(find /tmp/profile -name \*.pprof -print 2>/dev/null | sort); do + # Generate a link to the profile parsing function in gvisor.dev, which + # implicitly uses a prefix of https://storage.googleapis.com. Note that + # this relies on the specific BuildKite bucket location, and will break if + # this changes (although the artifacts will still exist and be just fine). + profile_name="${file#/tmp/profile/}" + profile_url="https://gvisor.dev/profile/gvisor-buildkite/${BUILDKITE_BUILD_ID}/${BUILDKITE_JOB_ID}/${file#/}/" + buildkite-agent artifact upload "${file}" + echo "<li><a href='${profile_url}'>${profile_name}</a></li>" >> "${profile_output}" +done + +# Upload if we had outputs. +if test -s "${profile_output}"; then + # Make the list a collapsible section in markdown. + sed -i "1s|^|<details><summary>${BUILDKITE_LABEL}</summary><ul>\n|" "${profile_output}" + echo "</ul></details>" >> "${profile_output}" + cat "${profile_output}" | buildkite-agent annotate --style info --context profiles --append +fi +rm -rf "${profile_output}" + +# Clean the bazel cache, if there's failure. +if test "${BUILDKITE_COMMAND_EXIT_STATUS}" -ne "0"; then + # Attempt to clear the cache and shut down. + make clean || echo "make clean failed with code $?" + make bazel-shutdown || echo "make bazel-shutdown failed with code $?" +fi + +# Kill any running containers (clear state). +CONTAINERS="$(docker ps -q)" +if ! test -z "${CONTAINERS}"; then + docker container kill ${CONTAINERS} 2>/dev/null || true +fi |