blob: 5e6db407a0f175f7ca28bd6908368f6f0489cb58 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
|
# Upload all relevant test failures.
make -s testlogs 2>/dev/null | grep // | sort | uniq | (
declare log_count=0
while read target log; do
if test -z "${target}"; then
continue
fi
# N.B. If *all* tests fail due to some common cause, then we will
# end up spending way too much time uploading logs. Instead, we just
# upload the first 10 and stop. That is hopefully enough to debug.
#
# We include this test in the metadata, but note that we cannot
# upload the actual test logs. The user should rerun locally.
log_count=$((${log_count}+1))
if test "${log_count}" -ge 10; then
echo " * ${target} (no upload)" | \
buildkite-agent annotate --style error --context failures --append
else
buildkite-agent artifact upload "${log}"
echo " * [${target}](artifact://${log#/}) (${BUILDKITE_LABEL})" | \
buildkite-agent annotate --style error --context failures --append
fi
done
)
# Upload all profiles, and include in an annotation.
declare profile_output=$(mktemp --tmpdir)
for file in $(find /tmp/profile -name \*.pprof -print 2>/dev/null | sort); do
# Generate a link to the profile parsing function in gvisor.dev, which
# implicitly uses a prefix of https://storage.googleapis.com. Note that
# this relies on the specific BuildKite bucket location, and will break if
# this changes (although the artifacts will still exist and be just fine).
profile_name="${file#/tmp/profile/}"
profile_url="https://gvisor.dev/profile/gvisor-buildkite/${BUILDKITE_BUILD_ID}/${BUILDKITE_JOB_ID}/${file#/}/"
buildkite-agent artifact upload "${file}"
echo "<li><a href='${profile_url}'>${profile_name}</a></li>" >> "${profile_output}"
done
# Upload if we had outputs.
if test -s "${profile_output}"; then
# Make the list a collapsible section in markdown.
sed -i "1s|^|<details><summary>${BUILDKITE_LABEL}</summary><ul>\n|" "${profile_output}"
echo "</ul></details>" >> "${profile_output}"
cat "${profile_output}" | buildkite-agent annotate --style info --context profiles --append
fi
rm -rf "${profile_output}"
# Clean the bazel cache, if there's failure.
if test "${BUILDKITE_COMMAND_EXIT_STATUS}" -ne "0"; then
# Attempt to clear the cache and shut down.
make clean || echo "make clean failed with code $?"
make bazel-shutdown || echo "make bazel-shutdown failed with code $?"
# Attempt to clear any Go cache.
sudo rm -rf "${HOME}/.cache/go-build"
sudo rm -rf "${HOME}/go"
fi
# Kill any running containers (clear state), except for "bootstrap".
for container in $(docker ps -q); do
maybe_kill="$(docker inspect -f '{{if ne "/bootstrap" .Name}}true{{ end }}' "${container}")"
if test -n "${maybe_kill}"; then
docker container kill "${container}"
fi
done
|
