2 files changed, 65 insertions, 11 deletions
diff --git a/.buildkite/hooks/post-command b/.buildkite/hooks/post-command
index ce3111f3c..5cd974002 100644
--- a/.buildkite/hooks/post-command
+++ b/.buildkite/hooks/post-command
@@ -1,17 +1,49 @@
-# Upload test logs on failure, if there are any.
-if [[ "${BUILDKITE_COMMAND_EXIT_STATUS}" -ne "0" ]]; then
+# Upload all relevant test failures.
+make -s testlogs 2>/dev/null | grep // | sort | uniq | (
   declare log_count=0
-  for log in $(make testlogs 2>/dev/null | sort | uniq); do
-    buildkite-agent artifact upload "${log}"
-    log_count=$((${log_count}+1))
+  while read target log; do
+    if test -z "${target}"; then
+      continue
+    fi
+
     # N.B. If *all* tests fail due to some common cause, then we will
     # end up spending way too much time uploading logs. Instead, we just
-    # upload the first 100 and stop. That is hopefully enough to debug.
-    if [[ "${log_count}" -ge 100 ]]; then
-      echo "Only uploaded first 100 failures; skipping the rest."
-      break
+    # upload the first 10 and stop. That is hopefully enough to debug.
+    #
+    # We include this test in the metadata, but note that we cannot
+    # upload the actual test logs. The user should rerun locally.
+    log_count=$((${log_count}+1))
+    if test "${log_count}" -ge 10; then
+      echo " * ${target} (no upload)" | \
+        buildkite-agent annotate --style error --context failures --append
+    else
+      buildkite-agent artifact upload "${log}"
+      echo " * [${target}](artifact://${log#/}) (${BUILDKITE_LABEL})" | \
+        buildkite-agent annotate --style error --context failures --append
     fi
   done
+)
+
+# Upload all profiles, and include in an annotation.
+declare profile_output=$(mktemp --tmpdir)
+for file in $(find /tmp/profile -name \*.pprof -print 2>/dev/null | sort); do
+  # Generate a link to the profile file at the top.
+  profile_name="${file#/tmp/profile/}"
+  buildkite-agent artifact upload "${file}"
+  echo "<li><a href='artifact://${file#/}'>${profile_name}</a></li>" >> "${profile_output}"
+done
+
+# Upload if we had outputs.
+if test -s "${profile_output}"; then
+  # Make the list a collapsible section in markdown.
+  sed -i "1s|^|<details><summary>${BUILDKITE_LABEL}</summary><ul>\n|" "${profile_output}"
+  echo "</ul></details>" >> "${profile_output}"
+  cat "${profile_output}" | buildkite-agent annotate --style info --context profiles --append
+fi
+rm -rf "${profile_output}"
+
+# Clean the bazel cache, if there's failure.
+if test "${BUILDKITE_COMMAND_EXIT_STATUS}" -ne "0"; then
   # Attempt to clear the cache and shut down.
   make clean || echo "make clean failed with code $?"
   make bazel-shutdown || echo "make bazel-shutdown failed with code $?"
@@ -19,6 +51,6 @@ fi
 
 # Kill any running containers (clear state).
 CONTAINERS="$(docker ps -q)"
-if ! [[ -z "${CONTAINERS}" ]]; then
+if ! test -z "${CONTAINERS}"; then
   docker container kill ${CONTAINERS} 2>/dev/null || true
-fi
-\ No newline at end of file
+fi
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 7d277202b..ba688f9ac 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -1,3 +1,15 @@
+# Install packages we need. Docker must be installed and configured,
+# as should Go itself. We just install some extra bits and pieces.
+function install_pkgs() {
+  while true; do
+    if sudo apt-get update && sudo apt-get install -y "$@"; then
+      break
+    fi
+  done
+}
+install_pkgs graphviz jq curl binutils gnupg gnupg-agent linux-libc-dev \
+  apt-transport-https ca-certificates software-properties-common
+
 # Setup for parallelization with PARTITION and TOTAL_PARTITIONS.
 export PARTITION=${BUILDKITE_PARALLEL_JOB:-0}
 PARTITION=$((${PARTITION}+1)) # 1-indexed, but PARALLEL_JOB is 0-indexed.
@@ -9,3 +21,13 @@ if test "${EXPERIMENTAL}" != "true"; then
   make sudo TARGETS=//runsc:runsc ARGS="install --experimental=true"
   sudo systemctl restart docker
 fi
+
+# Helper for benchmarks, based on the branch.
+if test "${BUILDKITE_BRANCH}" = "master"; then
+  export BENCHMARKS_OFFICIAL=true
+else
+  export BENCHMARKS_OFFICIAL=false
+fi
+
+# Clear existing profiles.
+sudo rm -rf /tmp/profile