# Upload all relevant test failures. make -s testlogs 2>/dev/null | grep // | sort | uniq | ( declare log_count=0 while read target log; do if test -z "${target}"; then continue fi # N.B. If *all* tests fail due to some common cause, then we will # end up spending way too much time uploading logs. Instead, we just # upload the first 10 and stop. That is hopefully enough to debug. # # We include this test in the metadata, but note that we cannot # upload the actual test logs. The user should rerun locally. log_count=$((${log_count}+1)) if test "${log_count}" -ge 10; then echo " * ${target} (no upload)" | \ buildkite-agent annotate --style error --context failures --append else buildkite-agent artifact upload "${log}" echo " * [${target}](artifact://${log#/}) (${BUILDKITE_LABEL})" | \ buildkite-agent annotate --style error --context failures --append fi done ) # Upload all profiles, and include in an annotation. declare profile_output=$(mktemp --tmpdir) for file in $(find /tmp/profile -name \*.pprof -print 2>/dev/null | sort); do # Generate a link to the profile parsing function in gvisor.dev, which # implicitly uses a prefix of https://storage.googleapis.com. Note that # this relies on the specific BuildKite bucket location, and will break if # this changes (although the artifacts will still exist and be just fine). profile_name="${file#/tmp/profile/}" profile_url="https://gvisor.dev/profile/gvisor-buildkite/${BUILDKITE_BUILD_ID}/${BUILDKITE_JOB_ID}/${file#/}/" buildkite-agent artifact upload "${file}" echo "
  • ${profile_name}
  • " >> "${profile_output}" done # Upload if we had outputs. if test -s "${profile_output}"; then # Make the list a collapsible section in markdown. sed -i "1s|^|
    ${BUILDKITE_LABEL}
    " >> "${profile_output}" cat "${profile_output}" | buildkite-agent annotate --style info --context profiles --append fi rm -rf "${profile_output}" # Clean the bazel cache, if there's failure. if test "${BUILDKITE_COMMAND_EXIT_STATUS}" -ne "0"; then # Attempt to clear the cache and shut down. make clean || echo "make clean failed with code $?" make bazel-shutdown || echo "make bazel-shutdown failed with code $?" # Attempt to clear any Go cache. sudo rm -rf "${HOME}/.cache/go-build" sudo rm -rf "${HOME}/go" fi # Kill any running containers (clear state), except for "bootstrap". for container in $(docker ps -q); do maybe_kill="$(docker inspect -f '{{if ne "/bootstrap" .Name}}true{{ end }}' "${container}")" if test -n "${maybe_kill}"; then docker container kill "${container}" fi done