Merge branch 'master' into iptables-1-pkg

Change-Id: I7457a11de4725e1bf3811420c505d225b1cb6943

3 files changed, 410 insertions, 0 deletions

diff --git a/tools/go_branch.sh b/tools/go_branch.sh
new file mode 100755
index 000000000..8ea6a6d8d
--- /dev/null
+++ b/tools/go_branch.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+
+# Copyright 2019 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+
+# Discovery the package name from the go.mod file.
+declare -r gomod="$(pwd)/go.mod"
+declare -r module=$(cat "${gomod}" | grep -E "^module" | cut -d' ' -f2)
+
+# Check that gopath has been built.
+declare -r gopath_dir="$(pwd)/bazel-bin/gopath/src/${module}"
+if ! [ -d "${gopath_dir}" ]; then
+  echo "No gopath directory found; build the :gopath target." >&2
+  exit 1
+fi
+
+# Create a temporary working directory, and ensure that this directory and all
+# subdirectories are cleaned up upon exit.
+declare -r tmp_dir=$(mktemp -d)
+finish() {
+  cd # Leave tmp_dir.
+  rm -rf "${tmp_dir}"
+}
+trap finish EXIT
+
+# Record the current working commit.
+declare -r head=$(git describe --always)
+
+# We expect to have an existing go branch that we will use as the basis for
+# this commit. That branch may be empty, but it must exist.
+declare -r go_branch=$(git show-ref --hash origin/go)
+
+# Clone the current repository to the temporary directory, and check out the
+# current go_branch directory. We move to the new repository for convenience.
+declare -r repo_orig="$(pwd)"
+declare -r repo_new="${tmp_dir}/repository"
+git clone . "${repo_new}"
+cd "${repo_new}"
+
+# Setup the repository and checkout the branch.
+git config user.email "gvisor-bot@google.com"
+git config user.name "gVisor bot"
+git fetch origin "${go_branch}"
+git checkout -b go "${go_branch}"
+
+# Start working on a merge commit that combines the previous history with the
+# current history. Note that we don't actually want any changes yet.
+git merge --allow-unrelated-histories --no-commit --strategy ours ${head}
+
+# Sync the entire gopath_dir and go.mod.
+rsync --recursive --verbose --delete --exclude .git --exclude README.md -L "${gopath_dir}/" .
+cp "${gomod}" .
+
+# There are a few solitary files that can get left behind due to the way bazel
+# constructs the gopath target. Note that we don't find all Go files here
+# because they may correspond to unused templates, etc.
+cp "${repo_orig}"/runsc/*.go runsc/
+
+# Update the current working set and commit.
+git add . && git commit -m "Merge ${head} (automated)"
+
+# Push the branch back to the original repository.
+git remote add orig "${repo_orig}" && git push -f orig go:go
diff --git a/tools/run_build.sh b/tools/run_build.sh
new file mode 100755
index 000000000..d49a1d4be
--- /dev/null
+++ b/tools/run_build.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2018 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Fail on any error.
+set -e
+# Display commands to stderr.
+set -x
+
+# Install the latest version of Bazel and log the version.
+(which use_bazel.sh && use_bazel.sh latest) || which bazel
+bazel version
+
+# Switch into the workspace.
+if [[ -v KOKORO_GIT_COMMIT ]] && [[ -d git/repo ]]; then
+  cd git/repo
+elif [[ -v KOKORO_GIT_COMMIT ]] && [[ -d github/repo ]]; then
+  cd github/repo
+fi
+
+# Build runsc.
+bazel build //runsc
+
+# Move the runsc binary into "latest" directory, and also a directory with the
+# current date.
+if [[ -v KOKORO_ARTIFACTS_DIR ]]; then
+  latest_dir="${KOKORO_ARTIFACTS_DIR}"/latest
+  today_dir="${KOKORO_ARTIFACTS_DIR}"/"$(date -Idate)"
+  mkdir -p "${latest_dir}" "${today_dir}"
+  cp bazel-bin/runsc/linux_amd64_pure_stripped/runsc "${latest_dir}"
+  sha512sum "${latest_dir}"/runsc | awk '{print $1 "  runsc"}' > "${latest_dir}"/runsc.sha512
+  cp bazel-bin/runsc/linux_amd64_pure_stripped/runsc "${today_dir}"
+  sha512sum "${today_dir}"/runsc | awk '{print $1 "  runsc"}' > "${today_dir}"/runsc.sha512
+fi
diff --git a/tools/run_tests.sh b/tools/run_tests.sh
new file mode 100755
index 000000000..7a1f889dd
--- /dev/null
+++ b/tools/run_tests.sh
@@ -0,0 +1,288 @@
+#!/bin/bash
+
+# Copyright 2018 The gVisor Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Fail on any error. Treat unset variables as error. Print commands as executed.
+set -eux
+
+###################
+# GLOBAL ENV VARS #
+###################
+
+if [[ -v KOKORO_GIT_COMMIT ]] && [[ -d git/repo ]]; then
+  readonly WORKSPACE_DIR="${PWD}/git/repo"
+elif [[ -v KOKORO_GIT_COMMIT ]] && [[ -d github/repo ]]; then
+  readonly WORKSPACE_DIR="${PWD}/github/repo"
+else
+  readonly WORKSPACE_DIR="${PWD}"
+fi
+
+# Used to configure RBE.
+readonly CLOUD_PROJECT_ID="gvisor-rbe"
+readonly RBE_PROJECT_ID="projects/${CLOUD_PROJECT_ID}/instances/default_instance"
+
+# Random runtime name to avoid collisions.
+readonly RUNTIME="runsc_test_$((RANDOM))"
+
+# Packages that will be built and tested.
+readonly BUILD_PACKAGES=("//...")
+readonly TEST_PACKAGES=("//pkg/..." "//runsc/..." "//tools/...")
+
+#######################
+# BAZEL CONFIGURATION #
+#######################
+
+# Install the latest version of Bazel and log the version.
+(which use_bazel.sh && use_bazel.sh latest) || which bazel
+bazel version
+
+# Load the kvm module.
+sudo -n -E modprobe kvm
+
+# General Bazel build/test flags.
+BAZEL_BUILD_FLAGS=(
+  "--show_timestamps"
+  "--test_output=errors"
+  "--keep_going"
+  "--verbose_failures=true"
+)
+
+# Bazel build/test for RBE, a super-set of BAZEL_BUILD_FLAGS.
+BAZEL_BUILD_RBE_FLAGS=(
+  "${BAZEL_BUILD_FLAGS[@]}"
+  "--config=remote"
+  "--project_id=${CLOUD_PROJECT_ID}"
+  "--remote_instance_name=${RBE_PROJECT_ID}"
+)
+if [[ -v KOKORO_BAZEL_AUTH_CREDENTIAL ]]; then
+  BAZEL_BUILD_RBE_FLAGS=(
+    "${BAZEL_BUILD_RBE_FLAGS[@]}"
+    "--auth_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL}"
+  )
+fi
+
+####################
+# Helper Functions #
+####################
+
+sanity_checks() {
+  cd ${WORKSPACE_DIR}
+  bazel run //:gazelle -- update-repos -from_file=go.mod
+  git diff --exit-code WORKSPACE
+}
+
+build_everything() {
+  FLAVOR="${1}"
+
+  cd ${WORKSPACE_DIR}
+  bazel build \
+    -c "${FLAVOR}" "${BAZEL_BUILD_RBE_FLAGS[@]}" \
+    "${BUILD_PACKAGES[@]}"
+}
+
+# Run simple tests runs the tests that require no special setup or
+# configuration.
+run_simple_tests() {
+  cd ${WORKSPACE_DIR}
+  bazel test \
+    "${BAZEL_BUILD_FLAGS[@]}" \
+    "${TEST_PACKAGES[@]}"
+}
+
+install_runtime() {
+  cd ${WORKSPACE_DIR}
+  sudo -n ${WORKSPACE_DIR}/runsc/test/install.sh --runtime ${RUNTIME}
+}
+
+install_helper() {
+  PACKAGE="${1}"
+  TAG="${2}"
+  GOPATH="${3}"
+
+  # Clone the repository.
+  mkdir -p "${GOPATH}"/src/$(dirname "${PACKAGE}") && \
+     git clone https://"${PACKAGE}" "${GOPATH}"/src/"${PACKAGE}"
+
+  # Checkout and build the repository.
+  (cd "${GOPATH}"/src/"${PACKAGE}" && \
+      git checkout "${TAG}" && \
+      GOPATH="${GOPATH}" make && \
+      sudo -n -E env GOPATH="${GOPATH}" make install)
+}
+
+# Install dependencies for the crictl tests.
+install_crictl_test_deps() {
+  sudo -n -E apt-get update
+  sudo -n -E apt-get install -y btrfs-tools libseccomp-dev
+
+  # Install containerd & cri-tools.
+  GOPATH=$(mktemp -d --tmpdir gopathXXXXX)
+  install_helper github.com/containerd/containerd v1.2.2 "${GOPATH}"
+  install_helper github.com/kubernetes-sigs/cri-tools v1.11.0 "${GOPATH}"
+
+  # Install gvisor-containerd-shim.
+  local latest=/tmp/gvisor-containerd-shim-latest
+  local shim_path=/tmp/gvisor-containerd-shim
+  wget --no-verbose https://storage.googleapis.com/cri-containerd-staging/gvisor-containerd-shim/latest -O ${latest}
+  wget --no-verbose https://storage.googleapis.com/cri-containerd-staging/gvisor-containerd-shim/gvisor-containerd-shim-$(cat ${latest}) -O ${shim_path}
+  chmod +x ${shim_path}
+  sudo -n -E mv ${shim_path} /usr/local/bin
+
+  # Configure containerd-shim.
+  local shim_config_path=/etc/containerd
+  local shim_config_tmp_path=/tmp/gvisor-containerd-shim.toml
+  sudo -n -E mkdir -p ${shim_config_path}
+  cat > ${shim_config_tmp_path} <<-EOF
+    runc_shim = "/usr/local/bin/containerd-shim"
+
+    [runsc_config]
+      debug = "true"
+      debug-log = "/tmp/runsc-logs/"
+      strace = "true"
+      file-access = "shared"
+EOF
+  sudo mv ${shim_config_tmp_path} ${shim_config_path}
+
+  # Configure CNI.
+  (cd "${GOPATH}" && sudo -n -E env PATH="${PATH}" GOPATH="${GOPATH}" \
+      src/github.com/containerd/containerd/script/setup/install-cni)
+}
+
+# Run the tests that require docker.
+run_docker_tests() {
+  cd ${WORKSPACE_DIR}
+
+  # Run tests with a default runtime (runc).
+  bazel test \
+    "${BAZEL_BUILD_FLAGS[@]}" \
+    --test_env=RUNSC_RUNTIME="" \
+    --test_output=all \
+    //runsc/test/image:image_test
+
+  # These names are used to exclude tests not supported in certain
+  # configuration, e.g. save/restore not supported with hostnet.
+  declare -a variations=("" "-kvm" "-hostnet" "-overlay")
+  for v in "${variations[@]}"; do
+    # Change test names otherwise each run of tests will overwrite logs and
+    # results of the previous run.
+    sed -i "s/name = \"integration_test.*\"/name = \"integration_test${v}\"/" runsc/test/integration/BUILD
+    sed -i "s/name = \"image_test.*\"/name = \"image_test${v}\"/" runsc/test/image/BUILD
+    # Run runsc tests with docker that are tagged manual.
+    bazel test \
+      "${BAZEL_BUILD_FLAGS[@]}" \
+      --test_env=RUNSC_RUNTIME="${RUNTIME}${v}" \
+      --test_output=all \
+      //runsc/test/image:image_test${v} \
+      //runsc/test/integration:integration_test${v}
+  done
+}
+
+# Run the tests that require root.
+run_root_tests() {
+  cd ${WORKSPACE_DIR}
+  bazel build //runsc/test/root:root_test
+  local root_test=$(find -L ./bazel-bin/ -executable -type f -name root_test | grep __main__)
+  if [[ ! -f "${root_test}" ]]; then
+    echo "root_test executable not found"
+    exit 1
+  fi
+  sudo -n -E RUNSC_RUNTIME="${RUNTIME}" RUNSC_EXEC=/tmp/"${RUNTIME}"/runsc ${root_test}
+}
+
+# Run syscall unit tests.
+run_syscall_tests() {
+  cd ${WORKSPACE_DIR}
+  bazel test "${BAZEL_BUILD_RBE_FLAGS[@]}" \
+    --test_tag_filters=runsc_ptrace //test/syscalls/...
+}
+
+run_runsc_do_tests() {
+  local runsc=$(find bazel-bin/runsc -type f -executable -name "runsc" | head -n1)
+
+  # run runsc do without root privileges.
+  ${runsc} --rootless do true
+  ${runsc} --rootless --network=none do true
+
+  # run runsc do with root privileges.
+  sudo -n -E ${runsc} do true
+}
+
+# Find and rename all test xml and log files so that Sponge can pick them up.
+# XML files must be named sponge_log.xml, and log files must be named
+# sponge_log.log. We move all such files into KOKORO_ARTIFACTS_DIR, in a
+# subdirectory named with the test name.
+upload_test_artifacts() {
+  # Skip if no kokoro directory.
+  [[ -v KOKORO_ARTIFACTS_DIR ]] || return
+
+  cd ${WORKSPACE_DIR}
+  find -L "bazel-testlogs" -name "test.xml" -o -name "test.log" -o -name "outputs.zip" |
+    tar --create --files-from - --transform 's/test\./sponge_log./' |
+    tar --extract --directory ${KOKORO_ARTIFACTS_DIR}
+  if [[ -d "/tmp/${RUNTIME}/logs" ]]; then
+    tar --create --gzip "--file=${KOKORO_ARTIFACTS_DIR}/runsc-logs.tar.gz" -C /tmp/ ${RUNTIME}/logs
+  fi
+}
+
+# Finish runs at exit, even in the event of an error, and uploads all test
+# artifacts.
+finish() {
+  # Grab the last exit code, we will return it.
+  local exit_code=${?}
+  upload_test_artifacts
+  exit ${exit_code}
+}
+
+# Run bazel in a docker container
+build_in_docker() {
+  cd ${WORKSPACE_DIR}
+  bazel clean
+  bazel shutdown
+  make
+  make runsc
+  make bazel-shutdown
+}
+
+########
+# MAIN #
+########
+
+main() {
+  # Register finish to run at exit.
+  trap finish EXIT
+
+  # Build and run the simple tests.
+  sanity_checks
+  build_everything opt
+  run_simple_tests
+
+  # So far so good. Install more deps and run the integration tests.
+  install_runtime
+  install_crictl_test_deps
+  run_docker_tests
+  run_root_tests
+
+  run_syscall_tests
+  run_runsc_do_tests
+
+  # Build other flavors too.
+  build_everything dbg
+
+  build_in_docker
+  # No need to call "finish" here, it will happen at exit.
+}
+
+# Kick it off.
+main