Refactor kokoro/run_tests.sh

This will make it easier to add RBE to bazel.

PiperOrigin-RevId: 225865250
Change-Id: I530b5e09875267c18dc6e7e16590fe9e128253ac

1 files changed, 92 insertions, 66 deletions

diff --git a/kokoro/run_tests.sh b/kokoro/run_tests.sh
index c78a7be96..0acb92d7a 100755
--- a/kokoro/run_tests.sh
+++ b/kokoro/run_tests.sh
@@ -14,50 +14,55 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Fail on any error.
-set -e
-# Display commands to stderr.
-set -x
+# Fail on any error. Treat unset variables as error. Print commands as executed.
+set -eux
 
-# Install the latest version of Bazel.
-use_bazel.sh latest
 
-# Log the bazel path and version.
+###################
+# GLOBAL ENV VARS #
+###################
+
+readonly WORKSPACE_DIR="${PWD}/git/repo"
+
+# Random runtime name to avoid collisions.
+readonly RUNTIME="runsc_test_$((RANDOM))"
+
+
+#######################
+# BAZEL CONFIGURATION #
+#######################
+
+# Install the latest version of Bazel, and log the location and version.
+use_bazel.sh latest
 which bazel
 bazel version
 
-cd git/repo
 
-# Build everything except //test.
-bazel build //pkg/... //runsc/... //tools/...
+####################
+# Helper Functions #
+####################
 
-# Test use this variable to determine what runtime to use.
-runtime=runsc_test_$((RANDOM))
-sudo -n ./runsc/test/install.sh --runtime ${runtime}
+build_everything() {
+  cd ${WORKSPACE_DIR}
+  # TODO: Include "test" directory.
+  bazel build //pkg/... //runsc/... //tools/...
+}
 
-# Best effort to uninstall the runtime
-uninstallRuntime() {
-  sudo -n ./runsc/test/install.sh -u --runtime ${runtime}
+# Run simple tests runs the tests that require no special setup or
+# configuration.
+run_simple_tests() {
+  cd ${WORKSPACE_DIR}
+  # TODO: Include "test" directory.
+  bazel test --test_output=errors //pkg/... //runsc/... //tools/...
 }
 
-# Run the tests and upload results.
-#
-# We turn off "-e" flag because we must move the log files even if the test
-# fails.
-set +e
-
-# Note: We do not run the tests in the //test folder as these would take
-# too long.
-bazel test --test_output=errors //pkg/... //runsc/... //tools/...
-exit_code=${?}
-
-# This function spawns a subshell to install crictl and containerd.
-installCrictl() (
-  # Fail on any error.
-  set -e
-  # Display commands to stderr.
-  set -x
+install_runtime() {
+  cd ${WORKSPACE_DIR}
+  sudo -n ${WORKSPACE_DIR}/runsc/test/install.sh --runtime ${RUNTIME}
+}
 
+# Install dependencies for the crictl tests.
+install_crictl_test_deps() {
   # Install containerd.
   # libseccomp2 needs to be downgraded in order to install libseccomp-dev.
   sudo -n -E apt-get install -y --force-yes libseccomp2=2.1.1-1ubuntu1~trusty4
@@ -83,8 +88,8 @@ installCrictl() (
   # Install gvisor-containerd-shim.
   local latest=/tmp/gvisor-containerd-shim-latest
   local shim_path=/tmp/gvisor-containerd-shim
-  wget https://storage.googleapis.com/cri-containerd-staging/gvisor-containerd-shim/latest -O ${latest}
-  wget https://storage.googleapis.com/cri-containerd-staging/gvisor-containerd-shim/$(cat ${latest}) -O ${shim_path}
+  wget --no-verbose https://storage.googleapis.com/cri-containerd-staging/gvisor-containerd-shim/latest -O ${latest}
+  wget --no-verbose https://storage.googleapis.com/cri-containerd-staging/gvisor-containerd-shim/$(cat ${latest}) -O ${shim_path}
   chmod +x ${shim_path}
   sudo -n -E mv ${shim_path} /usr/local/bin
 
@@ -105,56 +110,77 @@ EOF
 
   # Configure CNI.
   sudo -n -E env PATH=${PATH} ${GOPATH}/src/github.com/containerd/containerd/script/setup/install-cni
-)
+}
 
-# Install containerd and crictl.
-if [[ ${exit_code} -eq 0 ]]; then
-  installCrictl
-  exit_code=${?}
-fi
+# Run the tests that require docker.
+run_docker_tests() {
+  cd ${WORKSPACE_DIR}
 
-# Execute local tests that require docker.
-if [[ ${exit_code} -eq 0 ]]; then
   # These names are used to exclude tests not supported in certain
   # configuration, e.g. save/restore not supported with hostnet.
   declare -a variations=("" "-kvm" "-hostnet" "-overlay")
   for v in "${variations[@]}"; do
     # Run runsc tests with docker that are tagged manual.
-    bazel test --test_output=errors --test_env=RUNSC_RUNTIME=${runtime}${v} \
+    bazel test --test_output=errors --test_env=RUNSC_RUNTIME="${RUNTIME}${v}" \
       //runsc/test/image:image_test \
       //runsc/test/integration:integration_test
-    exit_code=${?}
-    if [[ ${exit_code} -ne 0 ]]; then
-      break
-    fi
   done
-fi
+}
 
-# Execute local tests that require superuser.
-if [[ ${exit_code} -eq 0 ]]; then
+# Run the tests that require root.
+run_root_tests() {
+  cd ${WORKSPACE_DIR}
   bazel build //runsc/test/root:root_test
-  root_test=$(find -L ./bazel-bin/ -executable -type f -name root_test | grep __main__)
+  local root_test=$(find -L ./bazel-bin/ -executable -type f -name root_test | grep __main__)
   if [[ ! -f "${root_test}" ]]; then
-    uninstallRuntime
     echo "root_test executable not found"
     exit 1
   fi
-  sudo -n -E RUNSC_RUNTIME=${runtime} RUNSC_EXEC=/tmp/${runtime}/runsc ${root_test}
-  exit_code=${?}
-fi
-
-uninstallRuntime
-
-set -e
+  sudo -n -E RUNSC_RUNTIME="${RUNTIME}" RUNSC_EXEC=/tmp/"${RUNTIME}"/runsc ${root_test}
+}
 
 # Find and rename all test xml and log files so that Sponge can pick them up.
 # XML files must be named sponge_log.xml, and log files must be named
 # sponge_log.log. We move all such files into KOKORO_ARTIFACTS_DIR, in a
 # subdirectory named with the test name.
-for file in $(find -L "bazel-testlogs" -name "test.xml" -o -name "test.log"); do
-    newpath=${KOKORO_ARTIFACTS_DIR}/$(dirname ${file})
-    extension="${file##*.}"
-    mkdir -p "${newpath}" && cp "${file}" "${newpath}/sponge_log.${extension}"
-done
+upload_test_artifacts() {
+  cd ${WORKSPACE_DIR}
+  for file in $(find -L "bazel-testlogs" -name "test.xml" -o -name "test.log"); do
+      newpath=${KOKORO_ARTIFACTS_DIR}/$(dirname ${file})
+      extension="${file##*.}"
+      mkdir -p "${newpath}" && cp "${file}" "${newpath}/sponge_log.${extension}"
+  done
+}
+
+# Finish runs at exit, even in the event of an error, and uploads all test
+# artifacts.
+finish() {
+  # Grab the last exit code, we will return it.
+  local exit_code=${?}
+  upload_test_artifacts
+  exit ${exit_code}
+}
+
+########
+# MAIN #
+########
+
+main() {
+  # Register finish to run at exit.
+  trap finish EXIT
+
+  # Build and run the simple tests.
+  build_everything
+  run_simple_tests
+
+  # So far so good. Install more deps and run the integration tests.
+  install_runtime
+  install_crictl_test_deps
+  run_docker_tests
+  run_root_tests
+
+  # No need to call "finish" here, it will happen at exit.
+}
 
-exit ${exit_code}
+# Kick it off.
+main