Fix Kokoro revision and 'go get usage'

As a convenience for debugging, also factor the scripts such that
can be run without Kokoro. In the future, this may be used to add
additional presubmit hooks that run without Kokoro.

PiperOrigin-RevId: 251474868

4 files changed, 319 insertions, 307 deletions

diff --git a/kokoro/run_build.sh b/kokoro/run_build.sh
index 63fffda48..9deafe9bb 100755..120000
--- a/kokoro/run_build.sh
+++ b/kokoro/run_build.sh
@@ -1,42 +1 @@
-#!/bin/bash
-
-# Copyright 2018 The gVisor Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Fail on any error.
-set -e
-# Display commands to stderr.
-set -x
-
-# Install the latest version of Bazel.
-use_bazel.sh latest
-
-# Log the bazel path and version.
-which bazel
-bazel version
-
-cd git/repo
-
-# Build runsc.
-bazel build //runsc
-
-# Move the runsc binary into "latest" directory, and also a directory with the
-# current date.
-latest_dir="${KOKORO_ARTIFACTS_DIR}"/latest
-today_dir="${KOKORO_ARTIFACTS_DIR}"/"$(date -Idate)"
-mkdir -p "${latest_dir}" "${today_dir}"
-cp bazel-bin/runsc/linux_amd64_pure_stripped/runsc "${latest_dir}"
-sha512sum "${latest_dir}"/runsc | awk '{print $1 "  runsc"}' > "${latest_dir}"/runsc.sha512
-cp bazel-bin/runsc/linux_amd64_pure_stripped/runsc "${today_dir}"
-sha512sum "${today_dir}"/runsc | awk '{print $1 "  runsc"}' > "${today_dir}"/runsc.sha512
+../tools/run_build.sh
\ No newline at end of file
diff --git a/kokoro/run_tests.sh b/kokoro/run_tests.sh
index 6ff72ce1d..931cd2622 100755..120000
--- a/kokoro/run_tests.sh
+++ b/kokoro/run_tests.sh
@@ -1,265 +1 @@
-#!/bin/bash
-
-# Copyright 2018 The gVisor Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Fail on any error. Treat unset variables as error. Print commands as executed.
-set -eux
-
-
-###################
-# GLOBAL ENV VARS #
-###################
-
-readonly WORKSPACE_DIR="${PWD}/git/repo"
-
-# Used to configure RBE.
-readonly CLOUD_PROJECT_ID="gvisor-rbe"
-readonly RBE_PROJECT_ID="projects/${CLOUD_PROJECT_ID}/instances/default_instance"
-
-# Random runtime name to avoid collisions.
-readonly RUNTIME="runsc_test_$((RANDOM))"
-
-# Packages that will be built and tested.
-readonly BUILD_PACKAGES=("//...")
-readonly TEST_PACKAGES=("//pkg/..." "//runsc/..." "//tools/...")
-
-#######################
-# BAZEL CONFIGURATION #
-#######################
-
-# Install the latest version of Bazel, and log the location and version.
-use_bazel.sh latest
-which bazel
-bazel version
-
-# Load the kvm module
-sudo -n -E modprobe kvm
-
-# General Bazel build/test flags.
-BAZEL_BUILD_FLAGS=(
-  "--show_timestamps"
-  "--test_output=errors"
-  "--keep_going"
-  "--verbose_failures=true"
-)
-
-# Bazel build/test for RBE, a super-set of BAZEL_BUILD_FLAGS.
-BAZEL_BUILD_RBE_FLAGS=(
-  "${BAZEL_BUILD_FLAGS[@]}"
-  "--config=remote"
-  "--project_id=${CLOUD_PROJECT_ID}"
-  "--remote_instance_name=${RBE_PROJECT_ID}"
-  "--auth_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL}"
-)
-
-####################
-# Helper Functions #
-####################
-
-sanity_checks() {
-  cd ${WORKSPACE_DIR}
-  bazel run //:gazelle -- update-repos -from_file=go.mod
-  git diff --exit-code WORKSPACE
-}
-
-build_everything() {
-  FLAVOR="${1}"
-
-  cd ${WORKSPACE_DIR}
-  bazel build \
-    -c "${FLAVOR}" "${BAZEL_BUILD_RBE_FLAGS[@]}" \
-    "${BUILD_PACKAGES[@]}"
-}
-
-# Run simple tests runs the tests that require no special setup or
-# configuration.
-run_simple_tests() {
-  cd ${WORKSPACE_DIR}
-  bazel test \
-    "${BAZEL_BUILD_FLAGS[@]}" \
-    "${TEST_PACKAGES[@]}"
-}
-
-install_runtime() {
-  cd ${WORKSPACE_DIR}
-  sudo -n ${WORKSPACE_DIR}/runsc/test/install.sh --runtime ${RUNTIME}
-}
-
-# Install dependencies for the crictl tests.
-install_crictl_test_deps() {
-  # Install containerd.
-  sudo -n -E apt-get update
-  sudo -n -E apt-get install -y btrfs-tools libseccomp-dev
-  # go get will exit with a status of 1 despite succeeding, so ignore errors.
-  go get -d github.com/containerd/containerd || true
-  cd ${GOPATH}/src/github.com/containerd/containerd
-  git checkout v1.2.2
-  make
-  sudo -n -E make install
-
-  # Install crictl.
-  # go get will exit with a status of 1 despite succeeding, so ignore errors.
-  go get -d github.com/kubernetes-sigs/cri-tools || true
-  cd ${GOPATH}/src/github.com/kubernetes-sigs/cri-tools
-  git checkout tags/v1.11.0
-  make
-  sudo -n -E make install
-
-  # Install gvisor-containerd-shim.
-  local latest=/tmp/gvisor-containerd-shim-latest
-  local shim_path=/tmp/gvisor-containerd-shim
-  wget --no-verbose https://storage.googleapis.com/cri-containerd-staging/gvisor-containerd-shim/latest -O ${latest}
-  wget --no-verbose https://storage.googleapis.com/cri-containerd-staging/gvisor-containerd-shim/gvisor-containerd-shim-$(cat ${latest}) -O ${shim_path}
-  chmod +x ${shim_path}
-  sudo -n -E mv ${shim_path} /usr/local/bin
-
-  # Configure containerd-shim.
-  local shim_config_path=/etc/containerd
-  local shim_config_tmp_path=/tmp/gvisor-containerd-shim.toml
-  sudo -n -E mkdir -p ${shim_config_path}
-  cat > ${shim_config_tmp_path} <<-EOF
-    runc_shim = "/usr/local/bin/containerd-shim"
-
-    [runsc_config]
-      debug = "true"
-      debug-log = "/tmp/runsc-logs/"
-      strace = "true"
-      file-access = "shared"
-EOF
-  sudo mv ${shim_config_tmp_path} ${shim_config_path}
-
-  # Configure CNI.
-  sudo -n -E env PATH=${PATH} ${GOPATH}/src/github.com/containerd/containerd/script/setup/install-cni
-}
-
-# Run the tests that require docker.
-run_docker_tests() {
-  cd ${WORKSPACE_DIR}
-
-  # Run tests with a default runtime (runc).
-  bazel test \
-    "${BAZEL_BUILD_FLAGS[@]}" \
-    --test_env=RUNSC_RUNTIME="" \
-    --test_output=all \
-    //runsc/test/image:image_test
-
-  # These names are used to exclude tests not supported in certain
-  # configuration, e.g. save/restore not supported with hostnet.
-  declare -a variations=("" "-kvm" "-hostnet" "-overlay")
-  for v in "${variations[@]}"; do
-    # Run runsc tests with docker that are tagged manual.
-    bazel test \
-      "${BAZEL_BUILD_FLAGS[@]}" \
-      --test_env=RUNSC_RUNTIME="${RUNTIME}${v}" \
-      --test_output=all \
-      //runsc/test/image:image_test \
-      //runsc/test/integration:integration_test
-  done
-}
-
-# Run the tests that require root.
-run_root_tests() {
-  cd ${WORKSPACE_DIR}
-  bazel build //runsc/test/root:root_test
-  local root_test=$(find -L ./bazel-bin/ -executable -type f -name root_test | grep __main__)
-  if [[ ! -f "${root_test}" ]]; then
-    echo "root_test executable not found"
-    exit 1
-  fi
-  sudo -n -E RUNSC_RUNTIME="${RUNTIME}" RUNSC_EXEC=/tmp/"${RUNTIME}"/runsc ${root_test}
-}
-
-# Run syscall unit tests.
-run_syscall_tests() {
-  cd ${WORKSPACE_DIR}
-  bazel test "${BAZEL_BUILD_RBE_FLAGS[@]}" \
-    --test_tag_filters=runsc_ptrace //test/syscalls/...
-}
-
-run_runsc_do_tests() {
-  local runsc=$(find bazel-bin/runsc -type f -executable -name "runsc" | head -n1)
-
-  # run runsc do without root privileges.
-  unshare -Ur ${runsc} --network=none --TESTONLY-unsafe-nonroot do true
-  unshare -Ur ${runsc} --TESTONLY-unsafe-nonroot --network=host do --netns=false true
-
-  # run runsc do with root privileges.
-  sudo -n -E ${runsc} do true
-}
-
-# Find and rename all test xml and log files so that Sponge can pick them up.
-# XML files must be named sponge_log.xml, and log files must be named
-# sponge_log.log. We move all such files into KOKORO_ARTIFACTS_DIR, in a
-# subdirectory named with the test name.
-upload_test_artifacts() {
-  cd ${WORKSPACE_DIR}
-  find -L "bazel-testlogs" -name "test.xml" -o -name "test.log" -o -name "outputs.zip" |
-    tar --create --files-from - --transform 's/test\./sponge_log./' |
-    tar --extract --directory ${KOKORO_ARTIFACTS_DIR}
-  if [[ -d "/tmp/${RUNTIME}/logs" ]]; then
-    tar --create --gzip "--file=${KOKORO_ARTIFACTS_DIR}/runsc-logs.tar.gz" -C /tmp/ ${RUNTIME}/logs
-  fi
-}
-
-# Finish runs at exit, even in the event of an error, and uploads all test
-# artifacts.
-finish() {
-  # Grab the last exit code, we will return it.
-  local exit_code=${?}
-  upload_test_artifacts
-  exit ${exit_code}
-}
-
-# Run bazel in a docker container
-build_in_docker() {
-  cd ${WORKSPACE_DIR}
-  bazel clean
-  bazel shutdown
-  make
-  make runsc
-  make bazel-shutdown
-}
-
-########
-# MAIN #
-########
-
-main() {
-  # Register finish to run at exit.
-  trap finish EXIT
-
-  # Build and run the simple tests.
-  sanity_checks
-  build_everything opt
-  run_simple_tests
-
-  # So far so good. Install more deps and run the integration tests.
-  install_runtime
-  install_crictl_test_deps
-  run_docker_tests
-  run_root_tests
-
-  run_syscall_tests
-  run_runsc_do_tests
-
-  # Build other flavors too.
-  build_everything dbg
-
-  build_in_docker
-  # No need to call "finish" here, it will happen at exit.
-}
-
-# Kick it off.
-main
+../tools/run_tests.sh
\ No newline at end of file
diff --git a/tools/run_build.sh b/tools/run_build.sh
new file mode 100755
index 000000000..b6b446690
--- /dev/null
+++ b/tools/run_build.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Copyright 2018 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Fail on any error.
+set -e
+# Display commands to stderr.
+set -x
+
+# Install the latest version of Bazel and log the version.
+(which use_bazel.sh && use_bazel.sh latest) || which bazel
+bazel version
+
+# Switch into the workspace and checkout the appropriate commit.
+if [[ -v KOKORO_GIT_COMMIT ]]; then
+  cd git/repo && git checkout "${KOKORO_GIT_COMMIT}"
+fi
+
+# Build runsc.
+bazel build //runsc
+
+# Move the runsc binary into "latest" directory, and also a directory with the
+# current date.
+if [[ -v KOKORO_ARTIFACTS_DIR ]]; then
+  latest_dir="${KOKORO_ARTIFACTS_DIR}"/latest
+  today_dir="${KOKORO_ARTIFACTS_DIR}"/"$(date -Idate)"
+  mkdir -p "${latest_dir}" "${today_dir}"
+  cp bazel-bin/runsc/linux_amd64_pure_stripped/runsc "${latest_dir}"
+  sha512sum "${latest_dir}"/runsc | awk '{print $1 "  runsc"}' > "${latest_dir}"/runsc.sha512
+  cp bazel-bin/runsc/linux_amd64_pure_stripped/runsc "${today_dir}"
+  sha512sum "${today_dir}"/runsc | awk '{print $1 "  runsc"}' > "${today_dir}"/runsc.sha512
+fi
diff --git a/tools/run_tests.sh b/tools/run_tests.sh
new file mode 100755
index 000000000..c6e97dc95
--- /dev/null
+++ b/tools/run_tests.sh
@@ -0,0 +1,273 @@
+#!/bin/bash
+
+# Copyright 2018 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Fail on any error. Treat unset variables as error. Print commands as executed.
+set -eux
+
+###################
+# GLOBAL ENV VARS #
+###################
+
+if [[ -v KOKORO_GIT_COMMIT ]]; then
+  readonly WORKSPACE_DIR="${PWD}/git/repo"
+else
+  readonly WORKSPACE_DIR="${PWD}"
+fi
+
+# Used to configure RBE.
+readonly CLOUD_PROJECT_ID="gvisor-rbe"
+readonly RBE_PROJECT_ID="projects/${CLOUD_PROJECT_ID}/instances/default_instance"
+
+# Random runtime name to avoid collisions.
+readonly RUNTIME="runsc_test_$((RANDOM))"
+
+# Packages that will be built and tested.
+readonly BUILD_PACKAGES=("//...")
+readonly TEST_PACKAGES=("//pkg/..." "//runsc/..." "//tools/...")
+
+#######################
+# BAZEL CONFIGURATION #
+#######################
+
+# Install the latest version of Bazel and log the version.
+(which use_bazel.sh && use_bazel.sh latest) || which bazel
+bazel version
+
+# Checkout the appropriate commit.
+if [[ -v KOKORO_GIT_COMMIT ]]; then
+  (cd "${WORKSPACE_DIR}" && git checkout "${KOKORO_GIT_COMMIT}")
+fi
+
+# Load the kvm module.
+sudo -n -E modprobe kvm
+
+# General Bazel build/test flags.
+BAZEL_BUILD_FLAGS=(
+  "--show_timestamps"
+  "--test_output=errors"
+  "--keep_going"
+  "--verbose_failures=true"
+)
+
+# Bazel build/test for RBE, a super-set of BAZEL_BUILD_FLAGS.
+BAZEL_BUILD_RBE_FLAGS=(
+  "${BAZEL_BUILD_FLAGS[@]}"
+  "--config=remote"
+  "--project_id=${CLOUD_PROJECT_ID}"
+  "--remote_instance_name=${RBE_PROJECT_ID}"
+)
+if [[ -v KOKORO_BAZEL_AUTH_CREDENTIAL ]]; then
+  BAZEL_BUILD_RBE_FLAGS=(
+    "${BAZEL_BUILD_RBE_FLAGS[@]}"
+    "--auth_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL}"
+  )
+fi
+
+####################
+# Helper Functions #
+####################
+
+sanity_checks() {
+  cd ${WORKSPACE_DIR}
+  bazel run //:gazelle -- update-repos -from_file=go.mod
+  git diff --exit-code WORKSPACE
+}
+
+build_everything() {
+  FLAVOR="${1}"
+
+  cd ${WORKSPACE_DIR}
+  bazel build \
+    -c "${FLAVOR}" "${BAZEL_BUILD_RBE_FLAGS[@]}" \
+    "${BUILD_PACKAGES[@]}"
+}
+
+# Run simple tests runs the tests that require no special setup or
+# configuration.
+run_simple_tests() {
+  cd ${WORKSPACE_DIR}
+  bazel test \
+    "${BAZEL_BUILD_FLAGS[@]}" \
+    "${TEST_PACKAGES[@]}"
+}
+
+install_runtime() {
+  cd ${WORKSPACE_DIR}
+  sudo -n ${WORKSPACE_DIR}/runsc/test/install.sh --runtime ${RUNTIME}
+}
+
+# Install dependencies for the crictl tests.
+install_crictl_test_deps() {
+  sudo -n -E apt-get update
+  sudo -n -E apt-get install -y btrfs-tools libseccomp-dev
+
+  # Install containerd.
+  [[ -d containerd ]] || git clone https://github.com/containerd/containerd
+  (cd containerd && git checkout v1.2.2 && make && sudo -n -E make install)
+
+  # Install crictl.
+  [[ -d cri-tools ]] || git clone https://github.com/kubernetes-sigs/cri-tools
+  (cd cri-tools && git checkout tags/v1.11.0 && make && sudo -n -E make install)
+
+  # Install gvisor-containerd-shim.
+  local latest=/tmp/gvisor-containerd-shim-latest
+  local shim_path=/tmp/gvisor-containerd-shim
+  wget --no-verbose https://storage.googleapis.com/cri-containerd-staging/gvisor-containerd-shim/latest -O ${latest}
+  wget --no-verbose https://storage.googleapis.com/cri-containerd-staging/gvisor-containerd-shim/gvisor-containerd-shim-$(cat ${latest}) -O ${shim_path}
+  chmod +x ${shim_path}
+  sudo -n -E mv ${shim_path} /usr/local/bin
+
+  # Configure containerd-shim.
+  local shim_config_path=/etc/containerd
+  local shim_config_tmp_path=/tmp/gvisor-containerd-shim.toml
+  sudo -n -E mkdir -p ${shim_config_path}
+  cat > ${shim_config_tmp_path} <<-EOF
+    runc_shim = "/usr/local/bin/containerd-shim"
+
+    [runsc_config]
+      debug = "true"
+      debug-log = "/tmp/runsc-logs/"
+      strace = "true"
+      file-access = "shared"
+EOF
+  sudo mv ${shim_config_tmp_path} ${shim_config_path}
+
+  # Configure CNI.
+  sudo -n -E env PATH=${PATH} containerd/script/setup/install-cni
+}
+
+# Run the tests that require docker.
+run_docker_tests() {
+  cd ${WORKSPACE_DIR}
+
+  # Run tests with a default runtime (runc).
+  bazel test \
+    "${BAZEL_BUILD_FLAGS[@]}" \
+    --test_env=RUNSC_RUNTIME="" \
+    --test_output=all \
+    //runsc/test/image:image_test
+
+  # These names are used to exclude tests not supported in certain
+  # configuration, e.g. save/restore not supported with hostnet.
+  declare -a variations=("" "-kvm" "-hostnet" "-overlay")
+  for v in "${variations[@]}"; do
+    # Run runsc tests with docker that are tagged manual.
+    bazel test \
+      "${BAZEL_BUILD_FLAGS[@]}" \
+      --test_env=RUNSC_RUNTIME="${RUNTIME}${v}" \
+      --test_output=all \
+      //runsc/test/image:image_test \
+      //runsc/test/integration:integration_test
+  done
+}
+
+# Run the tests that require root.
+run_root_tests() {
+  cd ${WORKSPACE_DIR}
+  bazel build //runsc/test/root:root_test
+  local root_test=$(find -L ./bazel-bin/ -executable -type f -name root_test | grep __main__)
+  if [[ ! -f "${root_test}" ]]; then
+    echo "root_test executable not found"
+    exit 1
+  fi
+  sudo -n -E RUNSC_RUNTIME="${RUNTIME}" RUNSC_EXEC=/tmp/"${RUNTIME}"/runsc ${root_test}
+}
+
+# Run syscall unit tests.
+run_syscall_tests() {
+  cd ${WORKSPACE_DIR}
+  bazel test "${BAZEL_BUILD_RBE_FLAGS[@]}" \
+    --test_tag_filters=runsc_ptrace //test/syscalls/...
+}
+
+run_runsc_do_tests() {
+  local runsc=$(find bazel-bin/runsc -type f -executable -name "runsc" | head -n1)
+
+  # run runsc do without root privileges.
+  unshare -Ur ${runsc} --network=none --TESTONLY-unsafe-nonroot do true
+  unshare -Ur ${runsc} --TESTONLY-unsafe-nonroot --network=host do --netns=false true
+
+  # run runsc do with root privileges.
+  sudo -n -E ${runsc} do true
+}
+
+# Find and rename all test xml and log files so that Sponge can pick them up.
+# XML files must be named sponge_log.xml, and log files must be named
+# sponge_log.log. We move all such files into KOKORO_ARTIFACTS_DIR, in a
+# subdirectory named with the test name.
+upload_test_artifacts() {
+  # Skip if no kokoro directory.
+  [[ -v KOKORO_ARTIFACTS_DIR ]] || return
+
+  cd ${WORKSPACE_DIR}
+  find -L "bazel-testlogs" -name "test.xml" -o -name "test.log" -o -name "outputs.zip" |
+    tar --create --files-from - --transform 's/test\./sponge_log./' |
+    tar --extract --directory ${KOKORO_ARTIFACTS_DIR}
+  if [[ -d "/tmp/${RUNTIME}/logs" ]]; then
+    tar --create --gzip "--file=${KOKORO_ARTIFACTS_DIR}/runsc-logs.tar.gz" -C /tmp/ ${RUNTIME}/logs
+  fi
+}
+
+# Finish runs at exit, even in the event of an error, and uploads all test
+# artifacts.
+finish() {
+  # Grab the last exit code, we will return it.
+  local exit_code=${?}
+  upload_test_artifacts
+  exit ${exit_code}
+}
+
+# Run bazel in a docker container
+build_in_docker() {
+  cd ${WORKSPACE_DIR}
+  bazel clean
+  bazel shutdown
+  make
+  make runsc
+  make bazel-shutdown
+}
+
+########
+# MAIN #
+########
+
+main() {
+  # Register finish to run at exit.
+  trap finish EXIT
+
+  # Build and run the simple tests.
+  sanity_checks
+  build_everything opt
+  run_simple_tests
+
+  # So far so good. Install more deps and run the integration tests.
+  install_runtime
+  install_crictl_test_deps
+  run_docker_tests
+  run_root_tests
+
+  run_syscall_tests
+  run_runsc_do_tests
+
+  # Build other flavors too.
+  build_everything dbg
+
+  build_in_docker
+  # No need to call "finish" here, it will happen at exit.
+}
+
+# Kick it off.
+main