From df125c986948fbbae2bc30de33213e2095762a86 Mon Sep 17 00:00:00 2001 From: Brad Burlage Date: Wed, 30 Oct 2019 16:14:30 -0700 Subject: Add Kokoro config for new runtime tests PiperOrigin-RevId: 277607217 --- kokoro/runtime_tests.cfg | 1 + 1 file changed, 1 insertion(+) create mode 100644 kokoro/runtime_tests.cfg (limited to 'kokoro') diff --git a/kokoro/runtime_tests.cfg b/kokoro/runtime_tests.cfg new file mode 100644 index 000000000..7d56d5aca --- /dev/null +++ b/kokoro/runtime_tests.cfg @@ -0,0 +1 @@ +build_file: "repo/scripts/runtime_tests.sh" -- cgit v1.2.3 From 493334f8b594eb1c2b0f5a6133dbedad4e0ecd32 Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Mon, 4 Nov 2019 15:59:11 -0800 Subject: kokoro: run KVM syscall tests We don't know how stable they are, so let's start with warning. PiperOrigin-RevId: 278484186 --- kokoro/syscall_kvm_tests.cfg | 9 +++++++++ scripts/syscall_kvm_tests.sh | 21 +++++++++++++++++++++ test/syscalls/linux/itimer.cc | 6 ++++++ 3 files changed, 36 insertions(+) create mode 100644 kokoro/syscall_kvm_tests.cfg create mode 100755 scripts/syscall_kvm_tests.sh (limited to 'kokoro') diff --git a/kokoro/syscall_kvm_tests.cfg b/kokoro/syscall_kvm_tests.cfg new file mode 100644 index 000000000..3b99e9c13 --- /dev/null +++ b/kokoro/syscall_kvm_tests.cfg @@ -0,0 +1,9 @@ +build_file: "repo/scripts/syscall_kvm_tests.sh" + +action { + define_artifacts { + regex: "**/sponge_log.xml" + regex: "**/sponge_log.log" + regex: "**/outputs.zip" + } +} diff --git a/scripts/syscall_kvm_tests.sh b/scripts/syscall_kvm_tests.sh new file mode 100755 index 000000000..de85daa5a --- /dev/null +++ b/scripts/syscall_kvm_tests.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +# TODO(b/112165693): "test --test_tag_filters=runsc_kvm" can be used +# when the "manual" tag will be removed for kvm tests. +test `bazel query "attr(tags, runsc_kvm, tests(//test/syscalls/...))"` diff --git a/test/syscalls/linux/itimer.cc b/test/syscalls/linux/itimer.cc index 930d2b940..b77e4cbd1 100644 --- a/test/syscalls/linux/itimer.cc +++ b/test/syscalls/linux/itimer.cc @@ -267,6 +267,9 @@ int TestSIGPROFFairness(absl::Duration sleep) { // Random save/restore is disabled as it introduces additional latency and // unpredictable distribution patterns. TEST(ItimerTest, DeliversSIGPROFToThreadsRoughlyFairlyActive_NoRandomSave) { + // TODO(b/143247272): CPU time accounting is inaccurate for the KVM platform. + SKIP_IF(GvisorPlatform() == Platform::kKVM); + pid_t child; int execve_errno; auto kill = ASSERT_NO_ERRNO_AND_VALUE( @@ -288,6 +291,9 @@ TEST(ItimerTest, DeliversSIGPROFToThreadsRoughlyFairlyActive_NoRandomSave) { // Random save/restore is disabled as it introduces additional latency and // unpredictable distribution patterns. TEST(ItimerTest, DeliversSIGPROFToThreadsRoughlyFairlyIdle_NoRandomSave) { + // TODO(b/143247272): CPU time accounting is inaccurate for the KVM platform. + SKIP_IF(GvisorPlatform() == Platform::kKVM); + pid_t child; int execve_errno; auto kill = ASSERT_NO_ERRNO_AND_VALUE( -- cgit v1.2.3 From e904823833bb166a514c98bd628704379de93b47 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Tue, 5 Nov 2019 15:06:06 -0800 Subject: Fix repository build scripts. This fixes a number of issues with the repository build process: * Fix the overall structure of the repository. * Fix the debian package description. * Fix the broken version number for packages. * Update the digest algorithm used for signing the release. I've validated that installation works from a separate staging bucket. Updates #852 PiperOrigin-RevId: 278716914 --- kokoro/build.cfg | 1 + runsc/BUILD | 10 ++++- runsc/debian/description | 6 +-- scripts/build.sh | 96 ++++++++++++++++++++++++++---------------------- tools/make_repository.sh | 67 +++++++++++++++++++++++---------- 5 files changed, 111 insertions(+), 69 deletions(-) (limited to 'kokoro') diff --git a/kokoro/build.cfg b/kokoro/build.cfg index 6c1d262d4..c9ceda947 100644 --- a/kokoro/build.cfg +++ b/kokoro/build.cfg @@ -19,5 +19,6 @@ action { regex: "**/runsc" regex: "**/runsc.*" regex: "**/dists/**" + regex: "**/pool/**" } } diff --git a/runsc/BUILD b/runsc/BUILD index e4e8e64a3..e5587421d 100644 --- a/runsc/BUILD +++ b/runsc/BUILD @@ -76,16 +76,24 @@ pkg_tar( genrule( name = "deb-version", + # Note that runsc must appear in the srcs parameter and not the tools + # parameter, otherwise it will not be stamped. This is reasonable, as tools + # may be encoded differently in the build graph (cached more aggressively + # because they are assumes to be hermetic). + srcs = [":runsc"], outs = ["version.txt"], cmd = "$(location :runsc) -version | grep 'runsc version' | sed 's/^[^0-9]*//' > $@", stamp = 1, - tools = [":runsc"], ) pkg_deb( name = "runsc-debian", architecture = "amd64", data = ":debian-data", + # Note that the description_file will be flatten (all newlines removed), + # and therefore it is kept to a simple one-line description. The expected + # format for debian packages is "short summary\nLonger explanation of + # tool." and this is impossible with the flattening. description_file = "debian/description", homepage = "https://gvisor.dev/", maintainer = "The gVisor Authors ", diff --git a/runsc/debian/description b/runsc/debian/description index 6e3b1b2c0..9e8e08805 100644 --- a/runsc/debian/description +++ b/runsc/debian/description @@ -1,5 +1 @@ -gVisor is a user-space kernel, written in Go, that implements a substantial -portion of the Linux system surface. It includes an Open Container Initiative -(OCI) runtime called runsc that provides an isolation boundary between the -application and the host kernel. The runsc runtime integrates with Docker and -Kubernetes, making it simple to run sandboxed containers. +gVisor container sandbox runtime diff --git a/scripts/build.sh b/scripts/build.sh index 0b3d1b316..8b2094cb0 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -17,63 +17,71 @@ source $(dirname $0)/common.sh # Install required packages for make_repository.sh et al. -sudo apt-get update && sudo apt-get install -y dpkg-sig coreutils apt-utils +sudo apt-get update && sudo apt-get install -y dpkg-sig coreutils apt-utils xz-utils # Build runsc. runsc=$(build -c opt //runsc) # Build packages. -pkg=$(build -c opt //runsc:runsc-debian) +pkgs=$(build -c opt //runsc:runsc-debian) + +# Stop here if we have no artifacts directory. +[[ -v KOKORO_ARTIFACTS_DIR ]] || exit 0 + +# install_raw installs raw artifacts. +install_raw() { + mkdir -p "$1" + cp -f "${runsc}" "$1"/runsc + sha512sum "$1"/runsc | awk '{print $1 " runsc"}' > "$1"/runsc.sha512 +} # Build a repository, if the key is available. +# +# Note that make_repository.sh script will install packages into the provided +# root, but will output to stdout a directory that can be copied arbitrarily +# into "${KOKORO_ARTIFACTS_DIR}"/dists/XXX. We do things this way because we +# will copy the same repository structure into multiple locations, below. if [[ -v KOKORO_REPO_KEY ]]; then - repo=$(tools/make_repository.sh "${KOKORO_KEYSTORE_DIR}/${KOKORO_REPO_KEY}" gvisor-bot@google.com main ${pkg}) + repo=$(tools/make_repository.sh \ + "${KOKORO_KEYSTORE_DIR}/${KOKORO_REPO_KEY}" \ + gvisor-bot@google.com \ + main \ + "${KOKORO_ARTIFACTS_DIR}" \ + ${pkgs}) fi -# Install installs artifacts. -install() { - local -r binaries_dir="$1" - local -r repo_dir="$2" - mkdir -p "${binaries_dir}" - cp -f "${runsc}" "${binaries_dir}"/runsc - sha512sum "${binaries_dir}"/runsc | awk '{print $1 " runsc"}' > "${binaries_dir}"/runsc.sha512 +# install_repo installs a repository. +# +# Note that packages are already installed, as noted above. +install_repo() { if [[ -v repo ]]; then - rm -rf "${repo_dir}" && mkdir -p "$(dirname "${repo_dir}")" - cp -a "${repo}" "${repo_dir}" + rm -rf "$1" && mkdir -p "$(dirname "$1")" && cp -a "${repo}" "$1" fi } -# Move the runsc binary into "latest" directory, and also a directory with the -# current date. If the current commit happens to correpond to a tag, then we -# will also move everything into a directory named after the given tag. -if [[ -v KOKORO_ARTIFACTS_DIR ]]; then - if [[ "${KOKORO_BUILD_NIGHTLY:-false}" == "true" ]]; then - # The "latest" directory and current date. - stamp="$(date -Idate)" - install "${KOKORO_ARTIFACTS_DIR}/nightly/latest" \ - "${KOKORO_ARTIFACTS_DIR}/dists/nightly/latest" - install "${KOKORO_ARTIFACTS_DIR}/nightly/${stamp}" \ - "${KOKORO_ARTIFACTS_DIR}/dists/nightly/${stamp}" - else - # Is it a tagged release? Build that instead. In that case, we also try to - # update the base release directory, in case this is an update. Finally, we - # update the "release" directory, which has the last released version. - tags="$(git tag --points-at HEAD)" - if ! [[ -z "${tags}" ]]; then - # Note that a given commit can match any number of tags. We have to - # iterate through all possible tags and produce associated artifacts. - for tag in ${tags}; do - name=$(echo "${tag}" | cut -d'-' -f2) - base=$(echo "${name}" | cut -d'.' -f1) - install "${KOKORO_ARTIFACTS_DIR}/release/${name}" \ - "${KOKORO_ARTIFACTS_DIR}/dists/${name}" - if [[ "${base}" != "${tag}" ]]; then - install "${KOKORO_ARTIFACTS_DIR}/release/${base}" \ - "${KOKORO_ARTIFACTS_DIR}/dists/${base}" - fi - install "${KOKORO_ARTIFACTS_DIR}/release/latest" \ - "${KOKORO_ARTIFACTS_DIR}/dists/latest" - done - fi +# If nightly, install only nightly artifacts. +if [[ "${KOKORO_BUILD_NIGHTLY:-false}" == "true" ]]; then + # The "latest" directory and current date. + stamp="$(date -Idate)" + install_raw "${KOKORO_ARTIFACTS_DIR}/nightly/latest" + install_raw "${KOKORO_ARTIFACTS_DIR}/nightly/${stamp}" + install_repo "${KOKORO_ARTIFACTS_DIR}/dists/nightly" +else + # We keep only the latest master raw release. + install_raw "${KOKORO_ARTIFACTS_DIR}/master/latest" + install_repo "${KOKORO_ARTIFACTS_DIR}/dists/master" + + # Is it a tagged release? Build that too. + tags="$(git tag --points-at HEAD)" + if ! [[ -z "${tags}" ]]; then + # Note that a given commit can match any number of tags. We have to iterate + # through all possible tags and produce associated artifacts. + for tag in ${tags}; do + name=$(echo "${tag}" | cut -d'-' -f2) + base=$(echo "${name}" | cut -d'.' -f1) + install_raw "${KOKORO_ARTIFACTS_DIR}/release/${name}" + install_repo "${KOKORO_ARTIFACTS_DIR}/dists/release" + install_repo "${KOKORO_ARTIFACTS_DIR}/dists/${base}" + done fi fi diff --git a/tools/make_repository.sh b/tools/make_repository.sh index 071f72b74..27ffbc9f3 100755 --- a/tools/make_repository.sh +++ b/tools/make_repository.sh @@ -17,13 +17,13 @@ # Parse arguments. We require more than two arguments, which are the private # keyring, the e-mail associated with the signer, and the list of packages. if [ "$#" -le 3 ]; then - echo "usage: $0 " + echo "usage: $0 " exit 1 fi -declare -r private_key=$(readlink -e "$1") -declare -r signer="$2" -declare -r component="$3" -shift; shift; shift +declare -r private_key=$(readlink -e "$1"); shift +declare -r signer="$1"; shift +declare -r component="$1"; shift +declare -r root="$1"; shift # Verbose from this point. set -xeo pipefail @@ -40,7 +40,7 @@ cleanup() { trap cleanup EXIT gpg --no-default-keyring --keyring "${keyring}" --import "${private_key}" >&2 -# Copy the packages, and ensure permissions are correct. +# Copy the packages into the root. for pkg in "$@"; do name=$(basename "${pkg}" .deb) name=$(basename "${name}" .changes) @@ -48,32 +48,61 @@ for pkg in "$@"; do if [[ "${name}" == "${arch}" ]]; then continue # Not a regular package. fi - mkdir -p "${tmpdir}"/"${component}"/binary-"${arch}" - cp -a "${pkg}" "${tmpdir}"/"${component}"/binary-"${arch}" + if [[ "${pkg}" =~ ^.*\.deb$ ]]; then + # Extract from the debian file. + version=$(dpkg --info "${pkg}" | grep -E 'Version:' | cut -d':' -f2) + elif [[ "${pkg}" =~ ^.*\.changes$ ]]; then + # Extract from the changes file. + version=$(grep -E 'Version:' "${pkg}" | cut -d':' -f2) + else + # Unsupported file type. + echo "Unknown file type: ${pkg}" + exit 1 + fi + version=${version// /} # Trim whitespace. + mkdir -p "${root}"/pool/"${version}"/binary-"${arch}" + cp -a "${pkg}" "${root}"/pool/"${version}"/binary-"${arch}" done -find "${tmpdir}" -type f -exec chmod 0644 {} \; -# Ensure there are no symlinks hanging around; these may be remnants of the -# build process. They may be useful for other things, but we are going to build -# an index of the actual packages here. -find "${tmpdir}" -type l -exec rm -f {} \; +# Ensure all permissions are correct. +find "${root}"/pool -type f -exec chmod 0644 {} \; # Sign all packages. -for file in "${tmpdir}"/"${component}"/binary-*/*.deb; do +for file in "${root}"/pool/*/binary-*/*.deb; do dpkg-sig -g "--no-default-keyring --keyring ${keyring}" --sign builder "${file}" >&2 done # Build the package list. -for dir in "${tmpdir}"/"${component}"/binary-*; do - (cd "${dir}" && apt-ftparchive packages . | gzip > Packages.gz) +declare arches=() +for dir in "${root}"/pool/*/binary-*; do + name=$(basename "${dir}") + arch=${name##binary-} + arches+=("${arch}") + repo_packages="${tmpdir}"/"${component}"/"${name}" + mkdir -p "${repo_packages}" + (cd "${root}" && apt-ftparchive --arch "${arch}" packages pool > "${repo_packages}"/Packages) + (cd "${repo_packages}" && cat Packages | gzip > Packages.gz) + (cd "${repo_packages}" && cat Packages | xz > Packages.xz) done # Build the release list. -(cd "${tmpdir}" && apt-ftparchive release . > Release) +cat > "${tmpdir}"/apt.conf < Release) +rm "${tmpdir}"/apt.conf # Sign the release. -(cd "${tmpdir}" && gpg --no-default-keyring --keyring "${keyring}" --clearsign -o InRelease Release >&2) -(cd "${tmpdir}" && gpg --no-default-keyring --keyring "${keyring}" -abs -o Release.gpg Release >&2) +declare -r digest_opts=("--digest-algo" "SHA512" "--cert-digest-algo" "SHA512") +(cd "${tmpdir}" && gpg --no-default-keyring --keyring "${keyring}" --clearsign "${digest_opts[@]}" -o InRelease Release >&2) +(cd "${tmpdir}" && gpg --no-default-keyring --keyring "${keyring}" -abs "${digest_opts[@]}" -o Release.gpg Release >&2) # Show the results. echo "${tmpdir}" -- cgit v1.2.3 From 50d6236111485acce0e728794c4f53884097ea7d Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Fri, 8 Nov 2019 14:07:34 -0800 Subject: Update kokoro images to install junitparser junitparser will be used to merge junit xml files. PiperOrigin-RevId: 279387305 --- kokoro/ubuntu1604/40_kokoro.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'kokoro') diff --git a/kokoro/ubuntu1604/40_kokoro.sh b/kokoro/ubuntu1604/40_kokoro.sh index 64772d74d..b132abcc8 100755 --- a/kokoro/ubuntu1604/40_kokoro.sh +++ b/kokoro/ubuntu1604/40_kokoro.sh @@ -23,7 +23,10 @@ declare -r ssh_public_keys=( ) # Install dependencies. -apt-get update && apt-get install -y rsync coreutils python-psutil qemu-kvm +apt-get update && apt-get install -y rsync coreutils python-psutil qemu-kvm python-pip + +# junitparser is used to merge junit xml files. +pip install junitparser # We need a kbuilder user. if useradd -c "kbuilder user" -m -s /bin/bash kbuilder; then -- cgit v1.2.3 From 14f4461f93a4c0014314a35a374ce07eec25636c Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Fri, 8 Nov 2019 15:43:47 -0800 Subject: kokoro: update images to install zip PiperOrigin-RevId: 279406266 --- kokoro/ubuntu1604/40_kokoro.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kokoro') diff --git a/kokoro/ubuntu1604/40_kokoro.sh b/kokoro/ubuntu1604/40_kokoro.sh index b132abcc8..3f50929d5 100755 --- a/kokoro/ubuntu1604/40_kokoro.sh +++ b/kokoro/ubuntu1604/40_kokoro.sh @@ -23,7 +23,7 @@ declare -r ssh_public_keys=( ) # Install dependencies. -apt-get update && apt-get install -y rsync coreutils python-psutil qemu-kvm python-pip +apt-get update && apt-get install -y rsync coreutils python-psutil qemu-kvm python-pip zip # junitparser is used to merge junit xml files. pip install junitparser -- cgit v1.2.3 From b6a00aa375e674617f1914b90db5ddb222b5a04e Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Wed, 20 Nov 2019 15:28:02 -0800 Subject: Use a GitHub credential for tagging a release. PiperOrigin-RevId: 281617882 --- kokoro/release.cfg | 14 ++++++++++++++ scripts/release.sh | 11 +++++++++++ 2 files changed, 25 insertions(+) (limited to 'kokoro') diff --git a/kokoro/release.cfg b/kokoro/release.cfg index b9d35bc51..5cec1790a 100644 --- a/kokoro/release.cfg +++ b/kokoro/release.cfg @@ -1 +1,15 @@ build_file: "repo/scripts/release.sh" + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73898 + keyname: "kokoro-github-access-token" + } + } +} + +env_vars { + key: "KOKORO_GITHUB_ACCESS_TOKEN" + value: "73898_kokoro-github-access-token" +} diff --git a/scripts/release.sh b/scripts/release.sh index b936bcc77..091abf87f 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -34,5 +34,16 @@ declare -r EMAIL=${EMAIL:-${KOKORO_RELEASE_AUTHOR}@google.com} git config --get user.name || git config user.name "gVisor-bot" git config --get user.email || git config user.email "${EMAIL}" +# Provide a credential if available. +if [[ -v KOKORO_GITHUB_ACCESS_TOKEN ]]; then + git config --global credential.helper cache + git credential approve < Date: Fri, 6 Dec 2019 20:11:51 -0800 Subject: Update Kokoro image to install Golang 1.13 PiperOrigin-RevId: 284308422 --- kokoro/ubuntu1604/10_core.sh | 4 ++-- kokoro/ubuntu1604/README.md | 34 ++++++++++++++++++++++++++++++++++ scripts/go.sh | 2 ++ 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 kokoro/ubuntu1604/README.md (limited to 'kokoro') diff --git a/kokoro/ubuntu1604/10_core.sh b/kokoro/ubuntu1604/10_core.sh index e87a6eee8..46dda6bb1 100755 --- a/kokoro/ubuntu1604/10_core.sh +++ b/kokoro/ubuntu1604/10_core.sh @@ -21,8 +21,8 @@ apt-get update && apt-get -y install make git-core build-essential linux-headers # Install a recent go toolchain. if ! [[ -d /usr/local/go ]]; then - wget https://dl.google.com/go/go1.12.linux-amd64.tar.gz - tar -xvf go1.12.linux-amd64.tar.gz + wget https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz + tar -xvf go1.13.5.linux-amd64.tar.gz mv go /usr/local fi diff --git a/kokoro/ubuntu1604/README.md b/kokoro/ubuntu1604/README.md new file mode 100644 index 000000000..64f913b9a --- /dev/null +++ b/kokoro/ubuntu1604/README.md @@ -0,0 +1,34 @@ +## Image Update + +After making changes to files in the directory, you must run the following +commands to update the image Kokoro uses: + +```shell +gcloud config set project gvisor-kokoro-testing +third_party/gvisor/kokoro/ubuntu1604/build.sh +third_party/gvisor/kokoro/ubuntu1804/build.sh +``` + +Note: the command above will change your default project for `gcloud`. Run +`gcloud config set project` again to revert back to your default project. + +Note: Files in `third_party/gvisor/kokoro/ubuntu1804/` as symlinks to +`ubuntu1604`, therefore both images must be updated. + +After the script finishes, the last few lines of the output will container the +image name. If the output was lost, you can run `build.sh` again to print the +image name. + +``` +NAME PROJECT FAMILY DEPRECATED STATUS +image-6777fa4666a968c8 gvisor-kokoro-testing READY ++ cleanup ++ gcloud compute instances delete --quiet build-tlfrdv +Deleted [https://www.googleapis.com/compute/v1/projects/gvisor-kokoro-testing/zones/us-central1-f/instances/build-tlfrdv]. +``` + +To setup Kokoro to use the new image, copy the image names to their +corresponding file below: + +* //devtools/kokoro/config/gcp/gvisor/ubuntu1604.gcl +* //devtools/kokoro/config/gcp/gvisor/ubuntu1804.gcl diff --git a/scripts/go.sh b/scripts/go.sh index 0dbfb7747..626ed8fa4 100755 --- a/scripts/go.sh +++ b/scripts/go.sh @@ -25,6 +25,8 @@ tools/go_branch.sh # Checkout the new branch. git checkout go && git clean -f +go version + # Build everything. go build ./... -- cgit v1.2.3 From 87337e92e3a65e69f6bfc6ac7d162c1b6ed18048 Mon Sep 17 00:00:00 2001 From: Brad Burlage Date: Tue, 10 Dec 2019 13:24:11 -0800 Subject: Add Kokoro configs for publishing Kythe xrefs. PiperOrigin-RevId: 284835614 --- kokoro/kythe/generate_xrefs.cfg | 28 ++++++++++++++++++++ kokoro/kythe/generate_xrefs.sh | 57 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 kokoro/kythe/generate_xrefs.cfg create mode 100644 kokoro/kythe/generate_xrefs.sh (limited to 'kokoro') diff --git a/kokoro/kythe/generate_xrefs.cfg b/kokoro/kythe/generate_xrefs.cfg new file mode 100644 index 000000000..03e65c54e --- /dev/null +++ b/kokoro/kythe/generate_xrefs.cfg @@ -0,0 +1,28 @@ +build_file: "gvisor/kokoro/kythe/generate_xrefs.sh" + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73898 + keyname: "kokoro-rbe-service-account" + } + } +} + +bazel_setting { + project_id: "gvisor-rbe" + local_execution: false + auth_credential: { + keystore_config_id: 73898 + keyname: "kokoro-rbe-service-account" + } + bes_backend_address: "buildeventservice.googleapis.com" + foundry_backend_address: "remotebuildexecution.googleapis.com" + upsalite_frontend_address: "https://source.cloud.google.com" +} + +action { + define_artifacts { + regex: "*.kzip" + } +} diff --git a/kokoro/kythe/generate_xrefs.sh b/kokoro/kythe/generate_xrefs.sh new file mode 100644 index 000000000..471b90e92 --- /dev/null +++ b/kokoro/kythe/generate_xrefs.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Install the latest version of Bazel. The default on Kokoro images is out of +# date. +if command -v use_bazel.sh >/dev/null; then + use_bazel.sh latest +fi +bazel version + +# We need to use python 3.6 (the Kokoro PY3 default is 3.4) to compile `//...` +# because benchmarktools requires a version of `requests` that is not a +# available in 3.4. +pyenv versions +pyenv global 3.6.1 + +readonly KYTHE_VERSION='v0.0.37' +readonly WORKDIR="$(mktemp -d)" +readonly KYTHE_DIR="${WORKDIR}/kythe-${KYTHE_VERSION}" +if [[ -n "$KOKORO_GIT_COMMIT" ]]; then + readonly KZIP_FILENAME="${KOKORO_ARTIFACTS_DIR}/${KOKORO_GIT_COMMIT}.kzip" +else + readonly KZIP_FILENAME="$(git rev-parse HEAD).kzip" +fi + +wget -q -O "${WORKDIR}/kythe.tar.gz" \ + "https://github.com/kythe/kythe/releases/download/${KYTHE_VERSION}/kythe-${KYTHE_VERSION}.tar.gz" +tar --no-same-owner -xzf "${WORKDIR}/kythe.tar.gz" --directory "$WORKDIR" + +if [[ -n "$KOKORO_ARTIFACTS_DIR" ]]; then + cd "${KOKORO_ARTIFACTS_DIR}/github/gvisor" +fi +bazel \ + --bazelrc="${KYTHE_DIR}/extractors.bazelrc" \ + build \ + --override_repository kythe_release="${KYTHE_DIR}" \ + --define=kythe_corpus=gvisor.dev \ + //... + +"${KYTHE_DIR}/tools/kzip" merge \ + --output "$KZIP_FILENAME" \ + $(find -L bazel-out/*/extra_actions/ -name '*.kzip') -- cgit v1.2.3 From a0aa784ecfb0d5ef94decf3c2be3e1cd44b6cbc6 Mon Sep 17 00:00:00 2001 From: Brad Burlage Date: Tue, 10 Dec 2019 14:54:04 -0800 Subject: Remove pyenv calls but log the python 3 version in use. Apparently our Kokoro VM images don't have pyenv -- I previously tested this on the Kokoro QA shared pool. PiperOrigin-RevId: 284855160 --- kokoro/kythe/generate_xrefs.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'kokoro') diff --git a/kokoro/kythe/generate_xrefs.sh b/kokoro/kythe/generate_xrefs.sh index 471b90e92..49186eeeb 100644 --- a/kokoro/kythe/generate_xrefs.sh +++ b/kokoro/kythe/generate_xrefs.sh @@ -23,11 +23,7 @@ if command -v use_bazel.sh >/dev/null; then fi bazel version -# We need to use python 3.6 (the Kokoro PY3 default is 3.4) to compile `//...` -# because benchmarktools requires a version of `requests` that is not a -# available in 3.4. -pyenv versions -pyenv global 3.6.1 +python3 -V readonly KYTHE_VERSION='v0.0.37' readonly WORKDIR="$(mktemp -d)" -- cgit v1.2.3 From e690651c67d38c2bd8532ddabd2967ebeef58c7e Mon Sep 17 00:00:00 2001 From: Brad Burlage Date: Wed, 11 Dec 2019 13:52:48 -0800 Subject: Run kythe build with -std=c++17 We seem to be getting some compiler errors when using the Kythe extractors.bazelrc. I'm not able to reproduce this on my local machine, so I'm hoping copying the cxxopt from gvisor.dev/pr/1350 will fix the build. PiperOrigin-RevId: 285054258 --- kokoro/kythe/generate_xrefs.sh | 1 + 1 file changed, 1 insertion(+) (limited to 'kokoro') diff --git a/kokoro/kythe/generate_xrefs.sh b/kokoro/kythe/generate_xrefs.sh index 49186eeeb..799467a34 100644 --- a/kokoro/kythe/generate_xrefs.sh +++ b/kokoro/kythe/generate_xrefs.sh @@ -46,6 +46,7 @@ bazel \ build \ --override_repository kythe_release="${KYTHE_DIR}" \ --define=kythe_corpus=gvisor.dev \ + --cxxopt=-std=c++17 \ //... "${KYTHE_DIR}/tools/kzip" merge \ -- cgit v1.2.3 From be2754a4b99cc92f13f479f74a5da8b0e6cb5839 Mon Sep 17 00:00:00 2001 From: Kevin Krakauer Date: Thu, 12 Dec 2019 14:40:36 -0800 Subject: Add iptables testing framework. It would be preferrable to test iptables via syscall tests, but there are some problems with that approach: * We're limited to loopback-only, as syscall tests involve only a single container. Other link interfaces (e.g. fdbased) should be tested. * We'd have to shell out to call iptables anyways, as the iptables syscall interface itself is too large and complex to work with alone. * Running the Linux/native version of the syscall test will require root, which is a pain to configure, is inherently unsafe, and could leave host iptables misconfigured. Using the go_test target allows there to be no new test runner. PiperOrigin-RevId: 285274275 --- WORKSPACE | 39 +++++++++ kokoro/iptables_tests.cfg | 10 +++ runsc/dockerutil/dockerutil.go | 10 +++ scripts/iptables_tests.sh | 27 ++++++ test/iptables/BUILD | 31 +++++++ test/iptables/README.md | 44 ++++++++++ test/iptables/filter_input.go | 124 ++++++++++++++++++++++++++++ test/iptables/iptables.go | 53 ++++++++++++ test/iptables/iptables_test.go | 179 ++++++++++++++++++++++++++++++++++++++++ test/iptables/iptables_util.go | 82 ++++++++++++++++++ test/iptables/runner/BUILD | 16 ++++ test/iptables/runner/Dockerfile | 4 + test/iptables/runner/main.go | 70 ++++++++++++++++ 13 files changed, 689 insertions(+) create mode 100644 kokoro/iptables_tests.cfg create mode 100755 scripts/iptables_tests.sh create mode 100644 test/iptables/BUILD create mode 100644 test/iptables/README.md create mode 100644 test/iptables/filter_input.go create mode 100644 test/iptables/iptables.go create mode 100644 test/iptables/iptables_test.go create mode 100644 test/iptables/iptables_util.go create mode 100644 test/iptables/runner/BUILD create mode 100644 test/iptables/runner/Dockerfile create mode 100644 test/iptables/runner/main.go (limited to 'kokoro') diff --git a/WORKSPACE b/WORKSPACE index 4561ed8fc..4b5a3bfe2 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -106,6 +106,45 @@ load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") rules_pkg_dependencies() +# Container rules. +http_archive( + name = "io_bazel_rules_docker", + sha256 = "14ac30773fdb393ddec90e158c9ec7ebb3f8a4fd533ec2abbfd8789ad81a284b", + strip_prefix = "rules_docker-0.12.1", + urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.12.1/rules_docker-v0.12.1.tar.gz"], +) + +load( + "@io_bazel_rules_docker//repositories:repositories.bzl", + container_repositories = "repositories", +) + +container_repositories() + +load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps") + +container_deps() + +load( + "@io_bazel_rules_docker//container:container.bzl", + "container_pull", +) + +# This container is built from the Dockerfile in test/iptables/runner. +container_pull( + name = "iptables-test", + registry = "gcr.io", + repository = "gvisor-presubmit/iptables-test", + digest = "sha256:a137d692a2eb9fc7bf95c5f4a568da090e2c31098e93634421ed88f3a3f1db65", +) + +load( + "@io_bazel_rules_docker//go:image.bzl", + _go_image_repos = "repositories", +) + +_go_image_repos() + # External repositories, in sorted order. go_repository( name = "com_github_cenkalti_backoff", diff --git a/kokoro/iptables_tests.cfg b/kokoro/iptables_tests.cfg new file mode 100644 index 000000000..7af20629a --- /dev/null +++ b/kokoro/iptables_tests.cfg @@ -0,0 +1,10 @@ +build_file: "repo/scripts/iptables_test.sh" + +action { + define_artifacts { + regex: "**/sponge_log.xml" + regex: "**/sponge_log.log" + regex: "**/outputs.zip" + regex: "**/runsc_logs_*.tar.gz" + } +} diff --git a/runsc/dockerutil/dockerutil.go b/runsc/dockerutil/dockerutil.go index 57f6ae8de..9b6346ca2 100644 --- a/runsc/dockerutil/dockerutil.go +++ b/runsc/dockerutil/dockerutil.go @@ -380,6 +380,16 @@ func (d *Docker) FindPort(sandboxPort int) (int, error) { return port, nil } +// FindIP returns the IP address of the container as a string. +func (d *Docker) FindIP() (string, error) { + const format = `{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}` + out, err := do("inspect", "-f", format, d.Name) + if err != nil { + return "", fmt.Errorf("error retrieving IP: %v", err) + } + return strings.TrimSpace(out), nil +} + // SandboxPid returns the PID to the sandbox process. func (d *Docker) SandboxPid() (int, error) { out, err := do("inspect", "-f={{.State.Pid}}", d.Name) diff --git a/scripts/iptables_tests.sh b/scripts/iptables_tests.sh new file mode 100755 index 000000000..c47cbd675 --- /dev/null +++ b/scripts/iptables_tests.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright 2018 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +install_runsc_for_test iptables + +# Build the docker image for the test. +run //test/iptables/runner --norun + +# TODO(gvisor.dev/issue/170): Also test this on runsc once iptables are better +# supported +test //test/iptables:iptables_test "--test_arg=--runtime=runc" \ + "--test_arg=--image=bazel/test/iptables/runner:runner" diff --git a/test/iptables/BUILD b/test/iptables/BUILD new file mode 100644 index 000000000..fa833c3b2 --- /dev/null +++ b/test/iptables/BUILD @@ -0,0 +1,31 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +package(licenses = ["notice"]) + +go_library( + name = "iptables", + srcs = [ + "filter_input.go", + "iptables.go", + "iptables_util.go", + ], + importpath = "gvisor.dev/gvisor/test/iptables", + visibility = ["//test/iptables:__subpackages__"], +) + +go_test( + name = "iptables_test", + srcs = [ + "iptables_test.go", + ], + embed = [":iptables"], + tags = [ + "local", + "manual", + ], + deps = [ + "//pkg/log", + "//runsc/dockerutil", + "//runsc/testutil", + ], +) diff --git a/test/iptables/README.md b/test/iptables/README.md new file mode 100644 index 000000000..b37cb2a96 --- /dev/null +++ b/test/iptables/README.md @@ -0,0 +1,44 @@ +# iptables Tests + +iptables tests are run via `scripts/iptables\_test.sh`. + +## Test Structure + +Each test implements `TestCase`, providing (1) a function to run inside the +container and (2) a function to run locally. Those processes are given each +others' IP addresses. The test succeeds when both functions succeed. + +The function inside the container (`ContainerAction`) typically sets some +iptables rules and then tries to send or receive packets. The local function +(`LocalAction`) will typically just send or receive packets. + +### Adding Tests + +1) Add your test to the `iptables` package. + +2) Register the test in an `init` function via `RegisterTestCase` (see +`filter_input.go` as an example). + +3) Add it to `iptables_test.go` (see the other tests in that file). + +Your test is now runnable with bazel! + +## Run individual tests + +Build the testing Docker container: + +```bash +$ bazel run //test/iptables/runner -- --norun +``` + +Run an individual test via: + +```bash +$ bazel test //test/iptables:iptables_test --test_filter= +``` + +To run an individual test with `runc`: + +```bash +$ bazel test //test/iptables:iptables_test --test_filter= --test_arg=--runtime=runc +``` diff --git a/test/iptables/filter_input.go b/test/iptables/filter_input.go new file mode 100644 index 000000000..923f44e68 --- /dev/null +++ b/test/iptables/filter_input.go @@ -0,0 +1,124 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iptables + +import ( + "fmt" + "net" + "time" +) + +const ( + dropPort = 2401 + acceptPort = 2402 + sendloopDuration = 2 * time.Second + network = "udp4" +) + +func init() { + RegisterTestCase(FilterInputDropUDP{}) + RegisterTestCase(FilterInputDropUDPPort{}) + RegisterTestCase(FilterInputDropDifferentUDPPort{}) +} + +// FilterInputDropUDP tests that we can drop UDP traffic. +type FilterInputDropUDP struct{} + +// Name implements TestCase.Name. +func (FilterInputDropUDP) Name() string { + return "FilterInputDropUDP" +} + +// ContainerAction implements TestCase.ContainerAction. +func (FilterInputDropUDP) ContainerAction(ip net.IP) error { + if err := filterTable("-A", "INPUT", "-p", "udp", "-j", "DROP"); err != nil { + return err + } + + // Listen for UDP packets on dropPort. + if err := listenUDP(dropPort, sendloopDuration); err == nil { + return fmt.Errorf("packets on port %d should have been dropped, but got a packet", dropPort) + } else if netErr, ok := err.(net.Error); !ok || !netErr.Timeout() { + return fmt.Errorf("error reading: %v", err) + } + + // At this point we know that reading timed out and never received a + // packet. + return nil +} + +// LocalAction implements TestCase.LocalAction. +func (FilterInputDropUDP) LocalAction(ip net.IP) error { + return sendUDPLoop(ip, dropPort, sendloopDuration) +} + +// FilterInputDropUDPPort tests that we can drop UDP traffic by port. +type FilterInputDropUDPPort struct{} + +// Name implements TestCase.Name. +func (FilterInputDropUDPPort) Name() string { + return "FilterInputDropUDPPort" +} + +// ContainerAction implements TestCase.ContainerAction. +func (FilterInputDropUDPPort) ContainerAction(ip net.IP) error { + if err := filterTable("-A", "INPUT", "-p", "udp", "-m", "udp", "--destination-port", fmt.Sprintf("%d", dropPort), "-j", "DROP"); err != nil { + return err + } + + // Listen for UDP packets on dropPort. + if err := listenUDP(dropPort, sendloopDuration); err == nil { + return fmt.Errorf("packets on port %d should have been dropped, but got a packet", dropPort) + } else if netErr, ok := err.(net.Error); !ok || !netErr.Timeout() { + return fmt.Errorf("error reading: %v", err) + } + + // At this point we know that reading timed out and never received a + // packet. + return nil +} + +// LocalAction implements TestCase.LocalAction. +func (FilterInputDropUDPPort) LocalAction(ip net.IP) error { + return sendUDPLoop(ip, dropPort, sendloopDuration) +} + +// FilterInputDropDifferentUDPPort tests that dropping traffic for a single UDP port +// doesn't drop packets on other ports. +type FilterInputDropDifferentUDPPort struct{} + +// Name implements TestCase.Name. +func (FilterInputDropDifferentUDPPort) Name() string { + return "FilterInputDropDifferentUDPPort" +} + +// ContainerAction implements TestCase.ContainerAction. +func (FilterInputDropDifferentUDPPort) ContainerAction(ip net.IP) error { + if err := filterTable("-A", "INPUT", "-p", "udp", "-m", "udp", "--destination-port", fmt.Sprintf("%d", dropPort), "-j", "DROP"); err != nil { + return err + } + + // Listen for UDP packets on another port. + if err := listenUDP(acceptPort, sendloopDuration); err != nil { + return fmt.Errorf("packets on port %d should be allowed, but encountered an error: %v", acceptPort, err) + } + + return nil +} + +// LocalAction implements TestCase.LocalAction. +func (FilterInputDropDifferentUDPPort) LocalAction(ip net.IP) error { + return sendUDPLoop(ip, acceptPort, sendloopDuration) +} diff --git a/test/iptables/iptables.go b/test/iptables/iptables.go new file mode 100644 index 000000000..2e565d988 --- /dev/null +++ b/test/iptables/iptables.go @@ -0,0 +1,53 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package iptables contains a set of iptables tests implemented as TestCases +package iptables + +import ( + "fmt" + "net" +) + +// IPExchangePort is the port the container listens on to receive the IP +// address of the local process. +const IPExchangePort = 2349 + +// A TestCase contains one action to run in the container and one to run +// locally. The actions run concurrently and each must succeed for the test +// pass. +type TestCase interface { + // Name returns the name of the test. + Name() string + + // ContainerAction runs inside the container. It receives the IP of the + // local process. + ContainerAction(ip net.IP) error + + // LocalAction runs locally. It receives the IP of the container. + LocalAction(ip net.IP) error +} + +// Tests maps test names to TestCase. +// +// New TestCases are added by calling RegisterTestCase in an init function. +var Tests = map[string]TestCase{} + +// RegisterTestCase registers tc so it can be run. +func RegisterTestCase(tc TestCase) { + if _, ok := Tests[tc.Name()]; ok { + panic(fmt.Sprintf("TestCase %s already registered.", tc.Name())) + } + Tests[tc.Name()] = tc +} diff --git a/test/iptables/iptables_test.go b/test/iptables/iptables_test.go new file mode 100644 index 000000000..bfbf1bb87 --- /dev/null +++ b/test/iptables/iptables_test.go @@ -0,0 +1,179 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iptables + +import ( + "fmt" + "net" + "os" + "path" + "testing" + "time" + + "flag" + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/runsc/dockerutil" + "gvisor.dev/gvisor/runsc/testutil" +) + +const timeout time.Duration = 10 * time.Second + +var image = flag.String("image", "bazel/test/iptables/runner:runner", "image to run tests in") + +type result struct { + output string + err error +} + +// singleTest runs a TestCase. Each test follows a pattern: +// - Create a container. +// - Get the container's IP. +// - Send the container our IP. +// - Start a new goroutine running the local action of the test. +// - Wait for both the container and local actions to finish. +// +// Container output is logged to $TEST_UNDECLARED_OUTPUTS_DIR if it exists, or +// to stderr. +func singleTest(test TestCase) error { + if _, ok := Tests[test.Name()]; !ok { + return fmt.Errorf("no test found with name %q. Has it been registered?", test.Name()) + } + + // Create and start the container. + cont := dockerutil.MakeDocker("gvisor-iptables") + defer cont.CleanUp() + resultChan := make(chan *result) + go func() { + output, err := cont.RunFg("--cap-add=NET_ADMIN", *image, "-name", test.Name()) + logContainer(output, err) + resultChan <- &result{output, err} + }() + + // Get the container IP. + ip, err := getIP(cont) + if err != nil { + return fmt.Errorf("failed to get container IP: %v", err) + } + + // Give the container our IP. + if err := sendIP(ip); err != nil { + return fmt.Errorf("failed to send IP to container: %v", err) + } + + // Run our side of the test. + errChan := make(chan error) + go func() { + errChan <- test.LocalAction(ip) + }() + + // Wait for both the container and local tests to finish. + var res *result + to := time.After(timeout) + for localDone := false; res == nil || !localDone; { + select { + case res = <-resultChan: + log.Infof("Container finished.") + case err, localDone = <-errChan: + log.Infof("Local finished.") + if err != nil { + return fmt.Errorf("local test failed: %v", err) + } + case <-to: + return fmt.Errorf("timed out after %f seconds", timeout.Seconds()) + } + } + + return res.err +} + +func getIP(cont dockerutil.Docker) (net.IP, error) { + // The container might not have started yet, so retry a few times. + var ipStr string + to := time.After(timeout) + for ipStr == "" { + ipStr, _ = cont.FindIP() + select { + case <-to: + return net.IP{}, fmt.Errorf("timed out getting IP after %f seconds", timeout.Seconds()) + default: + time.Sleep(250 * time.Millisecond) + } + } + ip := net.ParseIP(ipStr) + if ip == nil { + return net.IP{}, fmt.Errorf("invalid IP: %q", ipStr) + } + log.Infof("Container has IP of %s", ipStr) + return ip, nil +} + +func sendIP(ip net.IP) error { + contAddr := net.TCPAddr{ + IP: ip, + Port: IPExchangePort, + } + var conn *net.TCPConn + // The container may not be listening when we first connect, so retry + // upon error. + cb := func() error { + c, err := net.DialTCP("tcp4", nil, &contAddr) + conn = c + return err + } + if err := testutil.Poll(cb, timeout); err != nil { + return fmt.Errorf("timed out waiting to send IP, most recent error: %v", err) + } + if _, err := conn.Write([]byte{0}); err != nil { + return fmt.Errorf("error writing to container: %v", err) + } + return nil +} + +func logContainer(output string, err error) { + msg := fmt.Sprintf("Container error: %v\nContainer output:\n%v", err, output) + if artifactsDir := os.Getenv("TEST_UNDECLARED_OUTPUTS_DIR"); artifactsDir != "" { + fpath := path.Join(artifactsDir, "container.log") + if file, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE, 0644); err != nil { + log.Warningf("Failed to open log file %q: %v", fpath, err) + } else { + defer file.Close() + if _, err := file.Write([]byte(msg)); err == nil { + return + } + log.Warningf("Failed to write to log file %s: %v", fpath, err) + } + } + + // We couldn't write to the output directory -- just log to stderr. + log.Infof(msg) +} + +func TestFilterInputDropUDP(t *testing.T) { + if err := singleTest(FilterInputDropUDP{}); err != nil { + t.Fatal(err) + } +} + +func TestFilterInputDropUDPPort(t *testing.T) { + if err := singleTest(FilterInputDropUDPPort{}); err != nil { + t.Fatal(err) + } +} + +func TestFilterInputDropDifferentUDPPort(t *testing.T) { + if err := singleTest(FilterInputDropDifferentUDPPort{}); err != nil { + t.Fatal(err) + } +} diff --git a/test/iptables/iptables_util.go b/test/iptables/iptables_util.go new file mode 100644 index 000000000..3a4d11f1a --- /dev/null +++ b/test/iptables/iptables_util.go @@ -0,0 +1,82 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iptables + +import ( + "fmt" + "net" + "os/exec" + "time" +) + +const iptablesBinary = "iptables" + +// filterTable calls `iptables -t filter` with the given args. +func filterTable(args ...string) error { + args = append([]string{"-t", "filter"}, args...) + cmd := exec.Command(iptablesBinary, args...) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("error running iptables with args %v\nerror: %v\noutput: %s", args, err, string(out)) + } + return nil +} + +// listenUDP listens on a UDP port and returns the value of net.Conn.Read() for +// the first read on that port. +func listenUDP(port int, timeout time.Duration) error { + localAddr := net.UDPAddr{ + Port: port, + } + conn, err := net.ListenUDP(network, &localAddr) + if err != nil { + return err + } + defer conn.Close() + conn.SetDeadline(time.Now().Add(timeout)) + _, err = conn.Read([]byte{0}) + return err +} + +// sendUDPLoop sends 1 byte UDP packets repeatedly to the IP and port specified +// over a duration. +func sendUDPLoop(ip net.IP, port int, duration time.Duration) error { + // Send packets for a few seconds. + remote := net.UDPAddr{ + IP: ip, + Port: port, + } + conn, err := net.DialUDP(network, nil, &remote) + if err != nil { + return err + } + defer conn.Close() + + to := time.After(duration) + for timedOut := false; !timedOut; { + // This may return an error (connection refused) if the remote + // hasn't started listening yet or they're dropping our + // packets. So we ignore Write errors and depend on the remote + // to report a failure if it doesn't get a packet it needs. + conn.Write([]byte{0}) + select { + case <-to: + timedOut = true + default: + time.Sleep(200 * time.Millisecond) + } + } + + return nil +} diff --git a/test/iptables/runner/BUILD b/test/iptables/runner/BUILD new file mode 100644 index 000000000..1c59e26b9 --- /dev/null +++ b/test/iptables/runner/BUILD @@ -0,0 +1,16 @@ +load("@io_bazel_rules_docker//container:container.bzl", "container_image") +load("@io_bazel_rules_docker//go:image.bzl", "go_image") + +package(licenses = ["notice"]) + +container_image( + name = "iptables-base", + base = "@iptables-test//image", +) + +go_image( + name = "runner", + srcs = ["main.go"], + base = ":iptables-base", + deps = ["//test/iptables"], +) diff --git a/test/iptables/runner/Dockerfile b/test/iptables/runner/Dockerfile new file mode 100644 index 000000000..b77db44a1 --- /dev/null +++ b/test/iptables/runner/Dockerfile @@ -0,0 +1,4 @@ +# This Dockerfile builds the image hosted at +# gcr.io/gvisor-presubmit/iptables-test. +FROM ubuntu +RUN apt update && apt install -y iptables diff --git a/test/iptables/runner/main.go b/test/iptables/runner/main.go new file mode 100644 index 000000000..3c794114e --- /dev/null +++ b/test/iptables/runner/main.go @@ -0,0 +1,70 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main runs iptables tests from within a docker container. +package main + +import ( + "flag" + "fmt" + "log" + "net" + + "gvisor.dev/gvisor/test/iptables" +) + +var name = flag.String("name", "", "name of the test to run") + +func main() { + flag.Parse() + + // Find out which test we're running. + test, ok := iptables.Tests[*name] + if !ok { + log.Fatalf("No test found named %q", *name) + } + log.Printf("Running test %q", *name) + + // Get the IP of the local process. + ip, err := getIP() + if err != nil { + log.Fatal(err) + } + + // Run the test. + if err := test.ContainerAction(ip); err != nil { + log.Fatalf("Failed running test %q: %v", *name, err) + } +} + +// getIP listens for a connection from the local process and returns the source +// IP of that connection. +func getIP() (net.IP, error) { + localAddr := net.TCPAddr{ + Port: iptables.IPExchangePort, + } + listener, err := net.ListenTCP("tcp4", &localAddr) + if err != nil { + return net.IP{}, fmt.Errorf("failed listening for IP: %v", err) + } + defer listener.Close() + conn, err := listener.AcceptTCP() + if err != nil { + return net.IP{}, fmt.Errorf("failed accepting IP: %v", err) + } + defer conn.Close() + log.Printf("Connected to %v", conn.RemoteAddr()) + + return conn.RemoteAddr().(*net.TCPAddr).IP, nil +} -- cgit v1.2.3 From e548ce18051398fb3fe379326080411f59fda379 Mon Sep 17 00:00:00 2001 From: Zach Koopmans Date: Mon, 23 Dec 2019 11:48:03 -0800 Subject: Add python3-pip as dependency for Kokoro VM images. bm-tools requires python3 and pip3 in order to run tests. Add pip3 so that dependencies correctly install for Kokoro runs with bazel. PiperOrigin-RevId: 286923840 --- kokoro/ubuntu1604/40_kokoro.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kokoro') diff --git a/kokoro/ubuntu1604/40_kokoro.sh b/kokoro/ubuntu1604/40_kokoro.sh index 3f50929d5..5f2dfc858 100755 --- a/kokoro/ubuntu1604/40_kokoro.sh +++ b/kokoro/ubuntu1604/40_kokoro.sh @@ -23,7 +23,7 @@ declare -r ssh_public_keys=( ) # Install dependencies. -apt-get update && apt-get install -y rsync coreutils python-psutil qemu-kvm python-pip zip +apt-get update && apt-get install -y rsync coreutils python-psutil qemu-kvm python-pip python3-pip zip # junitparser is used to merge junit xml files. pip install junitparser -- cgit v1.2.3 From 290908fa8ae2363c3d2a7af7cef8d5dda622cde7 Mon Sep 17 00:00:00 2001 From: Fabricio Voznika Date: Thu, 9 Jan 2020 10:16:02 -0800 Subject: Configure issue reviver to run with Kokoro PiperOrigin-RevId: 288921032 --- kokoro/issue_reviver.cfg | 15 +++++++++++++++ scripts/issue_reviver.sh | 27 +++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 kokoro/issue_reviver.cfg create mode 100755 scripts/issue_reviver.sh (limited to 'kokoro') diff --git a/kokoro/issue_reviver.cfg b/kokoro/issue_reviver.cfg new file mode 100644 index 000000000..2370d9250 --- /dev/null +++ b/kokoro/issue_reviver.cfg @@ -0,0 +1,15 @@ +build_file: "repo/scripts/issue_reviver.sh" + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73898 + keyname: "kokoro-github-access-token" + } + } +} + +env_vars { + key: "KOKORO_GITHUB_ACCESS_TOKEN" + value: "73898_kokoro-github-access-token" +} diff --git a/scripts/issue_reviver.sh b/scripts/issue_reviver.sh new file mode 100755 index 000000000..bac9b9192 --- /dev/null +++ b/scripts/issue_reviver.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR=$(dirname $0) +source "${DIR}"/common.sh + +# Provide a credential file if available. +export OAUTH_TOKEN_FILE="" +if [[ -v KOKORO_GITHUB_ACCESS_TOKEN ]]; then + OAUTH_TOKEN_FILE="${KOKORO_KEYSTORE_DIR}/${KOKORO_GITHUB_ACCESS_TOKEN}" +fi + +REPO_ROOT=$(cd "$(dirname "${DIR}")"; pwd) +run //tools/issue_reviver:issue_reviver --path "${REPO_ROOT}" --oauth-token-file="${OAUTH_TOKEN_FILE}" -- cgit v1.2.3 From a944fcd94626bb278d5edd5453c5be16c72b7ee5 Mon Sep 17 00:00:00 2001 From: Nicolas Lacasse Date: Tue, 21 Jan 2020 16:38:42 -0800 Subject: Install Bazel 2.0.0 on kokoro images. PiperOrigin-RevId: 290850738 --- kokoro/ubuntu1604/20_bazel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kokoro') diff --git a/kokoro/ubuntu1604/20_bazel.sh b/kokoro/ubuntu1604/20_bazel.sh index b9a894024..b33e1656c 100755 --- a/kokoro/ubuntu1604/20_bazel.sh +++ b/kokoro/ubuntu1604/20_bazel.sh @@ -16,7 +16,7 @@ set -xeo pipefail -declare -r BAZEL_VERSION=0.29.1 +declare -r BAZEL_VERSION=2.0.0 # Install bazel dependencies. apt-get update && apt-get install -y openjdk-8-jdk-headless unzip -- cgit v1.2.3 From cb3906ae00575859a6910b8edc62ab9d531d1c85 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Wed, 22 Jan 2020 10:38:03 -0800 Subject: Add tools for generating images. This formalizes the adhoc scripts previously in kokoro. The image targets can be used by e.g. benchmarks in order to automated image prepation. PiperOrigin-RevId: 290982744 --- kokoro/ubuntu1604/10_core.sh | 30 ------ kokoro/ubuntu1604/20_bazel.sh | 28 ----- kokoro/ubuntu1604/25_docker.sh | 35 ------ kokoro/ubuntu1604/30_containerd.sh | 76 ------------- kokoro/ubuntu1604/40_kokoro.sh | 57 ---------- kokoro/ubuntu1604/README.md | 34 ------ kokoro/ubuntu1604/build.sh | 20 ---- kokoro/ubuntu1804/10_core.sh | 1 - kokoro/ubuntu1804/20_bazel.sh | 1 - kokoro/ubuntu1804/25_docker.sh | 1 - kokoro/ubuntu1804/30_containerd.sh | 1 - kokoro/ubuntu1804/40_kokoro.sh | 1 - kokoro/ubuntu1804/build.sh | 20 ---- tools/images/BUILD | 68 ++++++++++++ tools/images/build.sh | 101 ++++++++++++++++++ tools/images/defs.bzl | 178 +++++++++++++++++++++++++++++++ tools/images/execute.sh | 152 ++++++++++++++++++++++++++ tools/images/test.cc | 23 ++++ tools/images/ubuntu1604/10_core.sh | 30 ++++++ tools/images/ubuntu1604/20_bazel.sh | 28 +++++ tools/images/ubuntu1604/25_docker.sh | 35 ++++++ tools/images/ubuntu1604/30_containerd.sh | 76 +++++++++++++ tools/images/ubuntu1604/40_kokoro.sh | 57 ++++++++++ tools/images/ubuntu1604/BUILD | 7 ++ tools/images/ubuntu1804/BUILD | 7 ++ tools/installers/BUILD | 22 ++++ tools/installers/head.sh | 21 ++++ tools/installers/master.sh | 20 ++++ tools/installers/shim.sh | 24 +++++ 29 files changed, 849 insertions(+), 305 deletions(-) delete mode 100755 kokoro/ubuntu1604/10_core.sh delete mode 100755 kokoro/ubuntu1604/20_bazel.sh delete mode 100755 kokoro/ubuntu1604/25_docker.sh delete mode 100755 kokoro/ubuntu1604/30_containerd.sh delete mode 100755 kokoro/ubuntu1604/40_kokoro.sh delete mode 100644 kokoro/ubuntu1604/README.md delete mode 100755 kokoro/ubuntu1604/build.sh delete mode 120000 kokoro/ubuntu1804/10_core.sh delete mode 120000 kokoro/ubuntu1804/20_bazel.sh delete mode 120000 kokoro/ubuntu1804/25_docker.sh delete mode 120000 kokoro/ubuntu1804/30_containerd.sh delete mode 120000 kokoro/ubuntu1804/40_kokoro.sh delete mode 100755 kokoro/ubuntu1804/build.sh create mode 100644 tools/images/BUILD create mode 100755 tools/images/build.sh create mode 100644 tools/images/defs.bzl create mode 100755 tools/images/execute.sh create mode 100644 tools/images/test.cc create mode 100755 tools/images/ubuntu1604/10_core.sh create mode 100755 tools/images/ubuntu1604/20_bazel.sh create mode 100755 tools/images/ubuntu1604/25_docker.sh create mode 100755 tools/images/ubuntu1604/30_containerd.sh create mode 100755 tools/images/ubuntu1604/40_kokoro.sh create mode 100644 tools/images/ubuntu1604/BUILD create mode 100644 tools/images/ubuntu1804/BUILD create mode 100644 tools/installers/BUILD create mode 100755 tools/installers/head.sh create mode 100755 tools/installers/master.sh create mode 100755 tools/installers/shim.sh (limited to 'kokoro') diff --git a/kokoro/ubuntu1604/10_core.sh b/kokoro/ubuntu1604/10_core.sh deleted file mode 100755 index 46dda6bb1..000000000 --- a/kokoro/ubuntu1604/10_core.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -xeo pipefail - -# Install all essential build tools. -apt-get update && apt-get -y install make git-core build-essential linux-headers-$(uname -r) pkg-config - -# Install a recent go toolchain. -if ! [[ -d /usr/local/go ]]; then - wget https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz - tar -xvf go1.13.5.linux-amd64.tar.gz - mv go /usr/local -fi - -# Link the Go binary from /usr/bin; replacing anything there. -(cd /usr/bin && rm -f go && sudo ln -fs /usr/local/go/bin/go go) diff --git a/kokoro/ubuntu1604/20_bazel.sh b/kokoro/ubuntu1604/20_bazel.sh deleted file mode 100755 index b33e1656c..000000000 --- a/kokoro/ubuntu1604/20_bazel.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -xeo pipefail - -declare -r BAZEL_VERSION=2.0.0 - -# Install bazel dependencies. -apt-get update && apt-get install -y openjdk-8-jdk-headless unzip - -# Use the release installer. -curl -L -o bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh -chmod a+x bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh -./bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh -rm -f bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh diff --git a/kokoro/ubuntu1604/25_docker.sh b/kokoro/ubuntu1604/25_docker.sh deleted file mode 100755 index 1d3defcd3..000000000 --- a/kokoro/ubuntu1604/25_docker.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Add dependencies. -apt-get update && apt-get -y install \ - apt-transport-https \ - ca-certificates \ - curl \ - gnupg-agent \ - software-properties-common - -# Install the key. -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - -# Add the repository. -add-apt-repository \ - "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) \ - stable" - -# Install docker. -apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io diff --git a/kokoro/ubuntu1604/30_containerd.sh b/kokoro/ubuntu1604/30_containerd.sh deleted file mode 100755 index a7472bd1c..000000000 --- a/kokoro/ubuntu1604/30_containerd.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -xeo pipefail - -# Helper for Go packages below. -install_helper() { - PACKAGE="${1}" - TAG="${2}" - GOPATH="${3}" - - # Clone the repository. - mkdir -p "${GOPATH}"/src/$(dirname "${PACKAGE}") && \ - git clone https://"${PACKAGE}" "${GOPATH}"/src/"${PACKAGE}" - - # Checkout and build the repository. - (cd "${GOPATH}"/src/"${PACKAGE}" && \ - git checkout "${TAG}" && \ - GOPATH="${GOPATH}" make && \ - GOPATH="${GOPATH}" make install) -} - -# Install dependencies for the crictl tests. -apt-get install -y btrfs-tools libseccomp-dev - -# Install containerd & cri-tools. -GOPATH=$(mktemp -d --tmpdir gopathXXXXX) -install_helper github.com/containerd/containerd v1.2.2 "${GOPATH}" -install_helper github.com/kubernetes-sigs/cri-tools v1.11.0 "${GOPATH}" - -# Install gvisor-containerd-shim. -declare -r base="https://storage.googleapis.com/cri-containerd-staging/gvisor-containerd-shim" -declare -r latest=$(mktemp --tmpdir gvisor-containerd-shim-latest.XXXXXX) -declare -r shim_path=$(mktemp --tmpdir gvisor-containerd-shim.XXXXXX) -wget --no-verbose "${base}"/latest -O ${latest} -wget --no-verbose "${base}"/gvisor-containerd-shim-$(cat ${latest}) -O ${shim_path} -chmod +x ${shim_path} -mv ${shim_path} /usr/local/bin - -# Configure containerd-shim. -declare -r shim_config_path=/etc/containerd -declare -r shim_config_tmp_path=$(mktemp --tmpdir gvisor-containerd-shim.XXXXXX.toml) -mkdir -p ${shim_config_path} -cat > ${shim_config_tmp_path} <<-EOF - runc_shim = "/usr/local/bin/containerd-shim" - -[runsc_config] - debug = "true" - debug-log = "/tmp/runsc-logs/" - strace = "true" - file-access = "shared" -EOF -mv ${shim_config_tmp_path} ${shim_config_path} - -# Configure CNI. -(cd "${GOPATH}" && GOPATH="${GOPATH}" \ - src/github.com/containerd/containerd/script/setup/install-cni) - -# Cleanup the above. -rm -rf "${GOPATH}" -rm -rf "${latest}" -rm -rf "${shim_path}" -rm -rf "${shim_config_tmp_path}" diff --git a/kokoro/ubuntu1604/40_kokoro.sh b/kokoro/ubuntu1604/40_kokoro.sh deleted file mode 100755 index 5f2dfc858..000000000 --- a/kokoro/ubuntu1604/40_kokoro.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -xeo pipefail - -# Declare kokoro's required public keys. -declare -r ssh_public_keys=( - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDg7L/ZaEauETWrPklUTky3kvxqQfe2Ax/2CsSqhNIGNMnK/8d79CHlmY9+dE1FFQ/RzKNCaltgy7XcN/fCYiCZr5jm2ZtnLuGNOTzupMNhaYiPL419qmL+5rZXt4/dWTrsHbFRACxT8j51PcRMO5wgbL0Bg2XXimbx8kDFaurL2gqduQYqlu4lxWCaJqOL71WogcimeL63Nq/yeH5PJPWpqE4P9VUQSwAzBWFK/hLeds/AiP3MgVS65qHBnhq0JsHy8JQsqjZbG7Iidt/Ll0+gqzEbi62gDIcczG4KC0iOVzDDP/1BxDtt1lKeA23ll769Fcm3rJyoBMYxjvdw1TDx sabujp@trigger.mtv.corp.google.com" - "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNgGK/hCdjmulHfRE3hp4rZs38NCR8yAh0eDsztxqGcuXnuSnL7jOlRrbcQpremJ84omD4eKrIpwJUs+YokMdv4= sabujp@trigger.svl.corp.google.com" -) - -# Install dependencies. -apt-get update && apt-get install -y rsync coreutils python-psutil qemu-kvm python-pip python3-pip zip - -# junitparser is used to merge junit xml files. -pip install junitparser - -# We need a kbuilder user. -if useradd -c "kbuilder user" -m -s /bin/bash kbuilder; then - # User was added successfully; we add the relevant SSH keys here. - mkdir -p ~kbuilder/.ssh - (IFS=$'\n'; echo "${ssh_public_keys[*]}") > ~kbuilder/.ssh/authorized_keys - chmod 0600 ~kbuilder/.ssh/authorized_keys - chown -R kbuilder ~kbuilder/.ssh -fi - -# Give passwordless sudo access. -cat > /etc/sudoers.d/kokoro </dev/null +function cleanup { + gcloud compute instances delete --quiet --zone "${ZONE}" "${INSTANCE_NAME}" +} +trap cleanup EXIT + +# Wait for the instance to become available (up to 5 minutes). +declare timeout=300 +declare success=0 +declare -r start=$(date +%s) +declare -r end=$((${start}+${timeout})) +while [[ "$(date +%s)" -lt "${end}" ]] && [[ "${success}" -lt 3 ]]; do + if gcloud compute ssh --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- env - true 2>/dev/null; then + success=$((${success}+1)) + fi +done +if [[ "${success}" -eq "0" ]]; then + echo "connect timed out after ${timeout} seconds." + exit 1 +fi + +# Run the install scripts provided. +for arg; do + gcloud compute ssh --zone "${ZONE}" "${USERNAME}"@"${INSTANCE_NAME}" -- sudo bash - <"${arg}" >/dev/null +done + +# Stop the instance; required before creating an image. +gcloud compute instances stop --quiet --zone "${ZONE}" "${INSTANCE_NAME}" >/dev/null + +# Create a snapshot of the instance disk. +gcloud compute disks snapshot \ + --quiet \ + --zone "${ZONE}" \ + --snapshot-names="${SNAPSHOT_NAME}" \ + "${INSTANCE_NAME}" >/dev/null + +# Create the disk image. +gcloud compute images create \ + --quiet \ + --source-snapshot="${SNAPSHOT_NAME}" \ + --licenses="https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" \ + "${IMAGE_NAME}" >/dev/null + +# Finish up. +echo "${IMAGE_NAME}" diff --git a/tools/images/defs.bzl b/tools/images/defs.bzl new file mode 100644 index 000000000..d8e422a5d --- /dev/null +++ b/tools/images/defs.bzl @@ -0,0 +1,178 @@ +"""Image configuration. + +Images can be generated by using the vm_image rule. For example, + + vm_image( + name = "ubuntu", + project = "...", + family = "...", + scripts = [ + "script.sh", + "other.sh", + ], + ) + +This will always create an vm_image in the current default gcloud project. The +rule has a text file as its output containing the image name. This will enforce +serialization for all dependent rules. + +Images are always named per the hash of all the hermetic input scripts. This +allows images to be memoized quickly and easily. + +The vm_test rule can be used to execute a command remotely. For example, + + vm_test( + name = "mycommand", + image = ":myimage", + targets = [":test"], + ) +""" + +def _vm_image_impl(ctx): + script_paths = [] + for script in ctx.files.scripts: + script_paths.append(script.short_path) + + resolved_inputs, argv, runfiles_manifests = ctx.resolve_command( + command = "USERNAME=%s ZONE=$(cat %s) IMAGE_PROJECT=%s IMAGE_FAMILY=%s %s %s > %s" % + ( + ctx.attr.username, + ctx.files.zone[0].path, + ctx.attr.project, + ctx.attr.family, + ctx.executable.builder.path, + " ".join(script_paths), + ctx.outputs.out.path, + ), + tools = [ctx.attr.builder] + ctx.attr.scripts, + ) + + ctx.actions.run_shell( + tools = resolved_inputs, + outputs = [ctx.outputs.out], + progress_message = "Building image...", + execution_requirements = {"local": "true"}, + command = argv, + input_manifests = runfiles_manifests, + ) + return [DefaultInfo(files = depset([ctx.outputs.out]))] + +_vm_image = rule( + attrs = { + "builder": attr.label( + executable = True, + default = "//tools/images:builder", + cfg = "host", + ), + "username": attr.string(default = "$(whoami)"), + "zone": attr.label( + default = "//tools/images:zone", + cfg = "host", + ), + "family": attr.string(mandatory = True), + "project": attr.string(mandatory = True), + "scripts": attr.label_list(allow_files = True), + }, + outputs = { + "out": "%{name}.txt", + }, + implementation = _vm_image_impl, +) + +def vm_image(**kwargs): + _vm_image( + tags = [ + "local", + "manual", + ], + **kwargs + ) + +def _vm_test_impl(ctx): + runner = ctx.actions.declare_file("%s-executer" % ctx.label.name) + + # Note that the remote execution case must actually generate an + # intermediate target in order to collect all the relevant runfiles so that + # they can be copied over for remote execution. + runner_content = "\n".join([ + "#!/bin/bash", + "export ZONE=$(cat %s)" % ctx.files.zone[0].short_path, + "export USERNAME=%s" % ctx.attr.username, + "export IMAGE=$(cat %s)" % ctx.files.image[0].short_path, + "export SUDO=%s" % "true" if ctx.attr.sudo else "false", + "%s %s" % ( + ctx.executable.executer.short_path, + " ".join([ + target.files_to_run.executable.short_path + for target in ctx.attr.targets + ]), + ), + "", + ]) + ctx.actions.write(runner, runner_content, is_executable = True) + + # Return with all transitive files. + runfiles = ctx.runfiles( + transitive_files = depset(transitive = [ + depset(target.data_runfiles.files) + for target in ctx.attr.targets + if hasattr(target, "data_runfiles") + ]), + files = ctx.files.executer + ctx.files.zone + ctx.files.image + + ctx.files.targets, + collect_default = True, + collect_data = True, + ) + return [DefaultInfo(executable = runner, runfiles = runfiles)] + +_vm_test = rule( + attrs = { + "image": attr.label( + mandatory = True, + cfg = "host", + ), + "executer": attr.label( + executable = True, + default = "//tools/images:executer", + cfg = "host", + ), + "username": attr.string(default = "$(whoami)"), + "zone": attr.label( + default = "//tools/images:zone", + cfg = "host", + ), + "sudo": attr.bool(default = True), + "machine": attr.string(default = "n1-standard-1"), + "targets": attr.label_list( + mandatory = True, + allow_empty = False, + cfg = "target", + ), + }, + test = True, + implementation = _vm_test_impl, +) + +def vm_test( + installer = "//tools/installers:head", + **kwargs): + """Runs the given targets as a remote test. + + Args: + installer: Script to run before all targets. + **kwargs: All test arguments. Should include targets and image. + """ + targets = kwargs.pop("targets", []) + if installer: + targets = [installer] + targets + targets = [ + ] + targets + _vm_test( + tags = [ + "local", + "manual", + ], + targets = targets, + local = 1, + **kwargs + ) diff --git a/tools/images/execute.sh b/tools/images/execute.sh new file mode 100755 index 000000000..ba4b1ac0e --- /dev/null +++ b/tools/images/execute.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xeo pipefail + +# Required input. +if ! [[ -v IMAGE ]]; then + echo "no image provided: set IMAGE." + exit 1 +fi + +# Parameters. +declare -r USERNAME=${USERNAME:-test} +declare -r KEYNAME=$(mktemp --tmpdir -u key-XXXXXX) +declare -r SSHKEYS=$(mktemp --tmpdir -u sshkeys-XXXXXX) +declare -r INSTANCE_NAME=$(mktemp -u test-XXXXXX | tr A-Z a-z) +declare -r MACHINE=${MACHINE:-n1-standard-1} +declare -r ZONE=${ZONE:-us-central1-f} +declare -r SUDO=${SUDO:-false} + +# This script is executed as a test rule, which will reset the value of HOME. +# Unfortunately, it is needed to load the gconfig credentials. We will reset +# HOME when we actually execute in the remote environment, defined below. +export HOME=$(eval echo ~$(whoami)) + +# Generate unique keys for this test. +[[ -f "${KEYNAME}" ]] || ssh-keygen -t rsa -N "" -f "${KEYNAME}" -C "${USERNAME}" +cat > "${SSHKEYS}" </dev/null; then + success=$((${success}+1)) + fi +done +if [[ "${success}" -eq "0" ]]; then + echo "connect timed out after ${timeout} seconds." + exit 1 +fi + +# Copy the local directory over. +tar czf - --dereference --exclude=.git . | + gcloud compute ssh \ + --ssh-key-file="${KEYNAME}" \ + --zone "${ZONE}" \ + "${USERNAME}"@"${INSTANCE_NAME}" -- tar xzf - + +# Execute the command remotely. +for cmd; do + # Setup relevant environment. + # + # N.B. This is not a complete test environment, but is complete enough to + # provide rudimentary sharding and test output support. + declare -a PREFIX=( "env" ) + if [[ -v TEST_SHARD_INDEX ]]; then + PREFIX+=( "TEST_SHARD_INDEX=${TEST_SHARD_INDEX}" ) + fi + if [[ -v TEST_SHARD_STATUS_FILE ]]; then + SHARD_STATUS_FILE=$(mktemp -u test-shard-status-XXXXXX) + PREFIX+=( "TEST_SHARD_STATUS_FILE=/tmp/${SHARD_STATUS_FILE}" ) + fi + if [[ -v TEST_TOTAL_SHARDS ]]; then + PREFIX+=( "TEST_TOTAL_SHARDS=${TEST_TOTAL_SHARDS}" ) + fi + if [[ -v TEST_TMPDIR ]]; then + REMOTE_TMPDIR=$(mktemp -u test-XXXXXX) + PREFIX+=( "TEST_TMPDIR=/tmp/${REMOTE_TMPDIR}" ) + # Create remotely. + gcloud compute ssh \ + --ssh-key-file="${KEYNAME}" \ + --zone "${ZONE}" \ + "${USERNAME}"@"${INSTANCE_NAME}" -- \ + mkdir -p "/tmp/${REMOTE_TMPDIR}" + fi + if [[ -v XML_OUTPUT_FILE ]]; then + TEST_XML_OUTPUT=$(mktemp -u xml-output-XXXXXX) + PREFIX+=( "XML_OUTPUT_FILE=/tmp/${TEST_XML_OUTPUT}" ) + fi + if [[ "${SUDO}" == "true" ]]; then + PREFIX+=( "sudo" "-E" ) + fi + + # Execute the command. + gcloud compute ssh \ + --ssh-key-file="${KEYNAME}" \ + --zone "${ZONE}" \ + "${USERNAME}"@"${INSTANCE_NAME}" -- \ + "${PREFIX[@]}" "${cmd}" + + # Collect relevant results. + if [[ -v TEST_SHARD_STATUS_FILE ]]; then + gcloud compute scp \ + --ssh-key-file="${KEYNAME}" \ + --zone "${ZONE}" \ + "${USERNAME}"@"${INSTANCE_NAME}":/tmp/"${SHARD_STATUS_FILE}" \ + "${TEST_SHARD_STATUS_FILE}" 2>/dev/null || true # Allowed to fail. + fi + if [[ -v XML_OUTPUT_FILE ]]; then + gcloud compute scp \ + --ssh-key-file="${KEYNAME}" \ + --zone "${ZONE}" \ + "${USERNAME}"@"${INSTANCE_NAME}":/tmp/"${TEST_XML_OUTPUT}" \ + "${XML_OUTPUT_FILE}" 2>/dev/null || true # Allowed to fail. + fi + + # Clean up the temporary directory. + if [[ -v TEST_TMPDIR ]]; then + gcloud compute ssh \ + --ssh-key-file="${KEYNAME}" \ + --zone "${ZONE}" \ + "${USERNAME}"@"${INSTANCE_NAME}" -- \ + rm -rf "/tmp/${REMOTE_TMPDIR}" + fi +done diff --git a/tools/images/test.cc b/tools/images/test.cc new file mode 100644 index 000000000..4f31d93c5 --- /dev/null +++ b/tools/images/test.cc @@ -0,0 +1,23 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gtest/gtest.h" + +namespace { + +TEST(Image, Sanity) { + // Do nothing. +} + +} // namespace diff --git a/tools/images/ubuntu1604/10_core.sh b/tools/images/ubuntu1604/10_core.sh new file mode 100755 index 000000000..46dda6bb1 --- /dev/null +++ b/tools/images/ubuntu1604/10_core.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xeo pipefail + +# Install all essential build tools. +apt-get update && apt-get -y install make git-core build-essential linux-headers-$(uname -r) pkg-config + +# Install a recent go toolchain. +if ! [[ -d /usr/local/go ]]; then + wget https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz + tar -xvf go1.13.5.linux-amd64.tar.gz + mv go /usr/local +fi + +# Link the Go binary from /usr/bin; replacing anything there. +(cd /usr/bin && rm -f go && sudo ln -fs /usr/local/go/bin/go go) diff --git a/tools/images/ubuntu1604/20_bazel.sh b/tools/images/ubuntu1604/20_bazel.sh new file mode 100755 index 000000000..b33e1656c --- /dev/null +++ b/tools/images/ubuntu1604/20_bazel.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xeo pipefail + +declare -r BAZEL_VERSION=2.0.0 + +# Install bazel dependencies. +apt-get update && apt-get install -y openjdk-8-jdk-headless unzip + +# Use the release installer. +curl -L -o bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh +chmod a+x bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh +./bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh +rm -f bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh diff --git a/tools/images/ubuntu1604/25_docker.sh b/tools/images/ubuntu1604/25_docker.sh new file mode 100755 index 000000000..1d3defcd3 --- /dev/null +++ b/tools/images/ubuntu1604/25_docker.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Add dependencies. +apt-get update && apt-get -y install \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg-agent \ + software-properties-common + +# Install the key. +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + +# Add the repository. +add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + +# Install docker. +apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io diff --git a/tools/images/ubuntu1604/30_containerd.sh b/tools/images/ubuntu1604/30_containerd.sh new file mode 100755 index 000000000..a7472bd1c --- /dev/null +++ b/tools/images/ubuntu1604/30_containerd.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xeo pipefail + +# Helper for Go packages below. +install_helper() { + PACKAGE="${1}" + TAG="${2}" + GOPATH="${3}" + + # Clone the repository. + mkdir -p "${GOPATH}"/src/$(dirname "${PACKAGE}") && \ + git clone https://"${PACKAGE}" "${GOPATH}"/src/"${PACKAGE}" + + # Checkout and build the repository. + (cd "${GOPATH}"/src/"${PACKAGE}" && \ + git checkout "${TAG}" && \ + GOPATH="${GOPATH}" make && \ + GOPATH="${GOPATH}" make install) +} + +# Install dependencies for the crictl tests. +apt-get install -y btrfs-tools libseccomp-dev + +# Install containerd & cri-tools. +GOPATH=$(mktemp -d --tmpdir gopathXXXXX) +install_helper github.com/containerd/containerd v1.2.2 "${GOPATH}" +install_helper github.com/kubernetes-sigs/cri-tools v1.11.0 "${GOPATH}" + +# Install gvisor-containerd-shim. +declare -r base="https://storage.googleapis.com/cri-containerd-staging/gvisor-containerd-shim" +declare -r latest=$(mktemp --tmpdir gvisor-containerd-shim-latest.XXXXXX) +declare -r shim_path=$(mktemp --tmpdir gvisor-containerd-shim.XXXXXX) +wget --no-verbose "${base}"/latest -O ${latest} +wget --no-verbose "${base}"/gvisor-containerd-shim-$(cat ${latest}) -O ${shim_path} +chmod +x ${shim_path} +mv ${shim_path} /usr/local/bin + +# Configure containerd-shim. +declare -r shim_config_path=/etc/containerd +declare -r shim_config_tmp_path=$(mktemp --tmpdir gvisor-containerd-shim.XXXXXX.toml) +mkdir -p ${shim_config_path} +cat > ${shim_config_tmp_path} <<-EOF + runc_shim = "/usr/local/bin/containerd-shim" + +[runsc_config] + debug = "true" + debug-log = "/tmp/runsc-logs/" + strace = "true" + file-access = "shared" +EOF +mv ${shim_config_tmp_path} ${shim_config_path} + +# Configure CNI. +(cd "${GOPATH}" && GOPATH="${GOPATH}" \ + src/github.com/containerd/containerd/script/setup/install-cni) + +# Cleanup the above. +rm -rf "${GOPATH}" +rm -rf "${latest}" +rm -rf "${shim_path}" +rm -rf "${shim_config_tmp_path}" diff --git a/tools/images/ubuntu1604/40_kokoro.sh b/tools/images/ubuntu1604/40_kokoro.sh new file mode 100755 index 000000000..5f2dfc858 --- /dev/null +++ b/tools/images/ubuntu1604/40_kokoro.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xeo pipefail + +# Declare kokoro's required public keys. +declare -r ssh_public_keys=( + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDg7L/ZaEauETWrPklUTky3kvxqQfe2Ax/2CsSqhNIGNMnK/8d79CHlmY9+dE1FFQ/RzKNCaltgy7XcN/fCYiCZr5jm2ZtnLuGNOTzupMNhaYiPL419qmL+5rZXt4/dWTrsHbFRACxT8j51PcRMO5wgbL0Bg2XXimbx8kDFaurL2gqduQYqlu4lxWCaJqOL71WogcimeL63Nq/yeH5PJPWpqE4P9VUQSwAzBWFK/hLeds/AiP3MgVS65qHBnhq0JsHy8JQsqjZbG7Iidt/Ll0+gqzEbi62gDIcczG4KC0iOVzDDP/1BxDtt1lKeA23ll769Fcm3rJyoBMYxjvdw1TDx sabujp@trigger.mtv.corp.google.com" + "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNgGK/hCdjmulHfRE3hp4rZs38NCR8yAh0eDsztxqGcuXnuSnL7jOlRrbcQpremJ84omD4eKrIpwJUs+YokMdv4= sabujp@trigger.svl.corp.google.com" +) + +# Install dependencies. +apt-get update && apt-get install -y rsync coreutils python-psutil qemu-kvm python-pip python3-pip zip + +# junitparser is used to merge junit xml files. +pip install junitparser + +# We need a kbuilder user. +if useradd -c "kbuilder user" -m -s /bin/bash kbuilder; then + # User was added successfully; we add the relevant SSH keys here. + mkdir -p ~kbuilder/.ssh + (IFS=$'\n'; echo "${ssh_public_keys[*]}") > ~kbuilder/.ssh/authorized_keys + chmod 0600 ~kbuilder/.ssh/authorized_keys + chown -R kbuilder ~kbuilder/.ssh +fi + +# Give passwordless sudo access. +cat > /etc/sudoers.d/kokoro < Date: Thu, 23 Jan 2020 10:58:18 -0800 Subject: Try running kythe build on RBE. Also add our RBE project/instance to the --config=remote defaults. PiperOrigin-RevId: 291201426 --- .bazelrc | 20 +++++++------------- kokoro/kythe/generate_xrefs.sh | 3 ++- scripts/common_bazel.sh | 9 +-------- 3 files changed, 10 insertions(+), 22 deletions(-) (limited to 'kokoro') diff --git a/.bazelrc b/.bazelrc index 7f87e94b1..9c35c5e7b 100644 --- a/.bazelrc +++ b/.bazelrc @@ -20,6 +20,13 @@ build --stamp --workspace_status_command tools/workspace_status.sh # Enable remote execution so actions are performed on the remote systems. build:remote --remote_executor=grpcs://remotebuildexecution.googleapis.com +build:remote --project_id=gvisor-rbe +build:remote --remote_instance_name=projects/gvisor-rbe/instances/default_instance +# Enable authentication. This will pick up application default credentials by +# default. You can use --google_credentials=some_file.json to use a service +# account credential instead. +build:remote --google_default_credentials=true +build:remote --auth_scope="https://www.googleapis.com/auth/cloud-source-tools" # Add a custom platform and toolchain that builds in a privileged docker # container, which is required by our syscall tests. @@ -27,25 +34,12 @@ build:remote --host_platform=//test:rbe_ubuntu1604 build:remote --extra_toolchains=//test:cc-toolchain-clang-x86_64-default build:remote --extra_execution_platforms=//test:rbe_ubuntu1604 build:remote --platforms=//test:rbe_ubuntu1604 - -# Use default image for crosstool toolchain. build:remote --crosstool_top=@rbe_default//cc:toolchain - -# Default parallelism and timeout for remote jobs. build:remote --jobs=50 build:remote --remote_timeout=3600 - # RBE requires a strong hash function, such as SHA256. startup --host_jvm_args=-Dbazel.DigestFunction=SHA256 -# Enable authentication. This will pick up application default credentials by -# default. You can use --google_credentials=some_file.json to use a service -# account credential instead. -build:remote --google_default_credentials=true - -# Auth scope needed for authentication with RBE. -build:remote --auth_scope="https://www.googleapis.com/auth/cloud-source-tools" - # Set flags for uploading to BES in order to view results in the Bazel Build # Results UI. build:results --bes_backend="buildeventservice.googleapis.com" diff --git a/kokoro/kythe/generate_xrefs.sh b/kokoro/kythe/generate_xrefs.sh index 799467a34..d2ca95c68 100644 --- a/kokoro/kythe/generate_xrefs.sh +++ b/kokoro/kythe/generate_xrefs.sh @@ -25,7 +25,7 @@ bazel version python3 -V -readonly KYTHE_VERSION='v0.0.37' +readonly KYTHE_VERSION='v0.0.39' readonly WORKDIR="$(mktemp -d)" readonly KYTHE_DIR="${WORKDIR}/kythe-${KYTHE_VERSION}" if [[ -n "$KOKORO_GIT_COMMIT" ]]; then @@ -47,6 +47,7 @@ bazel \ --override_repository kythe_release="${KYTHE_DIR}" \ --define=kythe_corpus=gvisor.dev \ --cxxopt=-std=c++17 \ + --config=remote \ //... "${KYTHE_DIR}/tools/kzip" merge \ diff --git a/scripts/common_bazel.sh b/scripts/common_bazel.sh index bbc1a038e..a473a88a4 100755 --- a/scripts/common_bazel.sh +++ b/scripts/common_bazel.sh @@ -32,18 +32,11 @@ declare -r BAZEL_FLAGS=( "--keep_going" "--verbose_failures=true" ) -if [[ -v KOKORO_BAZEL_AUTH_CREDENTIAL ]] || [[ -v RBE_PROJECT_ID ]]; then - declare -r RBE_PROJECT_ID="${RBE_PROJECT_ID:-gvisor-rbe}" - declare -r BAZEL_RBE_FLAGS=( - "--config=remote" - "--project_id=${RBE_PROJECT_ID}" - "--remote_instance_name=projects/${RBE_PROJECT_ID}/instances/default_instance" - ) -fi if [[ -v KOKORO_BAZEL_AUTH_CREDENTIAL ]]; then declare -r BAZEL_RBE_AUTH_FLAGS=( "--auth_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL}" ) + declare -r BAZEL_RBE_FLAGS=("--config=remote") fi # Wrap bazel. -- cgit v1.2.3 From 14d2ed1ad7785a54b35ef7ee949d3cf89a87e66d Mon Sep 17 00:00:00 2001 From: Brad Burlage Date: Thu, 23 Jan 2020 14:12:40 -0800 Subject: Fix kythe build. * Pass --auth_credentials now that we're using RBE * Fix kzips not being uploaded to the root of the GCS bucket PiperOrigin-RevId: 291241757 --- kokoro/kythe/generate_xrefs.cfg | 3 ++- kokoro/kythe/generate_xrefs.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'kokoro') diff --git a/kokoro/kythe/generate_xrefs.cfg b/kokoro/kythe/generate_xrefs.cfg index 03e65c54e..ccf657983 100644 --- a/kokoro/kythe/generate_xrefs.cfg +++ b/kokoro/kythe/generate_xrefs.cfg @@ -23,6 +23,7 @@ bazel_setting { action { define_artifacts { - regex: "*.kzip" + regex: "**/*.kzip" + fail_if_no_artifacts: true } } diff --git a/kokoro/kythe/generate_xrefs.sh b/kokoro/kythe/generate_xrefs.sh index d2ca95c68..4c104afdb 100644 --- a/kokoro/kythe/generate_xrefs.sh +++ b/kokoro/kythe/generate_xrefs.sh @@ -48,6 +48,7 @@ bazel \ --define=kythe_corpus=gvisor.dev \ --cxxopt=-std=c++17 \ --config=remote \ + --auth_credentials="${KOKORO_BAZEL_AUTH_CREDENTIAL}" \ //... "${KYTHE_DIR}/tools/kzip" merge \ -- cgit v1.2.3 From 24cfbf4b981a76e46cab47650ef514835990b72e Mon Sep 17 00:00:00 2001 From: Brad Burlage Date: Fri, 24 Jan 2020 11:44:31 -0800 Subject: Fix corpus_name to match our ingestion config[1]. PiperOrigin-RevId: 291412676 --- kokoro/kythe/generate_xrefs.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'kokoro') diff --git a/kokoro/kythe/generate_xrefs.sh b/kokoro/kythe/generate_xrefs.sh index 4c104afdb..7a0fbb3cd 100644 --- a/kokoro/kythe/generate_xrefs.sh +++ b/kokoro/kythe/generate_xrefs.sh @@ -16,8 +16,6 @@ set -ex -# Install the latest version of Bazel. The default on Kokoro images is out of -# date. if command -v use_bazel.sh >/dev/null; then use_bazel.sh latest fi @@ -45,7 +43,7 @@ bazel \ --bazelrc="${KYTHE_DIR}/extractors.bazelrc" \ build \ --override_repository kythe_release="${KYTHE_DIR}" \ - --define=kythe_corpus=gvisor.dev \ + --define=kythe_corpus=github.com/google/gvisor \ --cxxopt=-std=c++17 \ --config=remote \ --auth_credentials="${KOKORO_BAZEL_AUTH_CREDENTIAL}" \ -- cgit v1.2.3 From bc3a24d62788bd6b881afeda230f1a5550a5709a Mon Sep 17 00:00:00 2001 From: gVisor bot Date: Fri, 31 Jan 2020 11:49:37 -0800 Subject: Internal change. PiperOrigin-RevId: 292587459 --- kokoro/kythe/generate_xrefs.sh | 2 +- pkg/sentry/fs/g3doc/inotify.md | 16 ++++++++-------- pkg/sentry/mm/README.md | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) (limited to 'kokoro') diff --git a/kokoro/kythe/generate_xrefs.sh b/kokoro/kythe/generate_xrefs.sh index 7a0fbb3cd..323b0f77b 100644 --- a/kokoro/kythe/generate_xrefs.sh +++ b/kokoro/kythe/generate_xrefs.sh @@ -23,7 +23,7 @@ bazel version python3 -V -readonly KYTHE_VERSION='v0.0.39' +readonly KYTHE_VERSION='v0.0.41' readonly WORKDIR="$(mktemp -d)" readonly KYTHE_DIR="${WORKDIR}/kythe-${KYTHE_VERSION}" if [[ -n "$KOKORO_GIT_COMMIT" ]]; then diff --git a/pkg/sentry/fs/g3doc/inotify.md b/pkg/sentry/fs/g3doc/inotify.md index 71a577d9d..85063d4e6 100644 --- a/pkg/sentry/fs/g3doc/inotify.md +++ b/pkg/sentry/fs/g3doc/inotify.md @@ -112,11 +112,11 @@ attempts to queue a new event, it is already holding `fs.Watches.mu`. If we used `Inotify.mu` to also protect the event queue, this would violate the above lock ordering. -[dirent]: https://github.com/google/gvisor/blob/master/+/master/pkg/sentry/fs/dirent.go -[event]: https://github.com/google/gvisor/blob/master/+/master/pkg/sentry/fs/inotify_event.go -[fd_table]: https://github.com/google/gvisor/blob/master/+/master/pkg/sentry/kernel/fd_table.go -[inode]: https://github.com/google/gvisor/blob/master/+/master/pkg/sentry/fs/inode.go -[inode_watches]: https://github.com/google/gvisor/blob/master/+/master/pkg/sentry/fs/inode_inotify.go -[inotify]: https://github.com/google/gvisor/blob/master/+/master/pkg/sentry/fs/inotify.go -[syscall_dir]: https://github.com/google/gvisor/blob/master/+/master/pkg/sentry/syscalls/linux/ -[watch]: https://github.com/google/gvisor/blob/master/+/master/pkg/sentry/fs/inotify_watch.go +[dirent]: https://github.com/google/gvisor/blob/master/pkg/sentry/fs/dirent.go +[event]: https://github.com/google/gvisor/blob/master/pkg/sentry/fs/inotify_event.go +[fd_table]: https://github.com/google/gvisor/blob/master/pkg/sentry/kernel/fd_table.go +[inode]: https://github.com/google/gvisor/blob/master/pkg/sentry/fs/inode.go +[inode_watches]: https://github.com/google/gvisor/blob/master/pkg/sentry/fs/inode_inotify.go +[inotify]: https://github.com/google/gvisor/blob/master/pkg/sentry/fs/inotify.go +[syscall_dir]: https://github.com/google/gvisor/blob/master/pkg/sentry/syscalls/linux/ +[watch]: https://github.com/google/gvisor/blob/master/pkg/sentry/fs/inotify_watch.go diff --git a/pkg/sentry/mm/README.md b/pkg/sentry/mm/README.md index e1322e373..f4d43d927 100644 --- a/pkg/sentry/mm/README.md +++ b/pkg/sentry/mm/README.md @@ -274,7 +274,7 @@ In the sentry: methods [`platform.AddressSpace.MapFile` and `platform.AddressSpace.Unmap`][platform]. -[memmap]: https://github.com/google/gvisor/blob/master/+/master/pkg/sentry/memmap/memmap.go -[mm]: https://github.com/google/gvisor/blob/master/+/master/pkg/sentry/mm/mm.go -[pgalloc]: https://github.com/google/gvisor/blob/master/+/master/pkg/sentry/pgalloc/pgalloc.go -[platform]: https://github.com/google/gvisor/blob/master/+/master/pkg/sentry/platform/platform.go +[memmap]: https://github.com/google/gvisor/blob/master/pkg/sentry/memmap/memmap.go +[mm]: https://github.com/google/gvisor/blob/master/pkg/sentry/mm/mm.go +[pgalloc]: https://github.com/google/gvisor/blob/master/pkg/sentry/pgalloc/pgalloc.go +[platform]: https://github.com/google/gvisor/blob/master/pkg/sentry/platform/platform.go -- cgit v1.2.3 From 6cd7901d7d5f9639e95fff3d8927ba8856a83f91 Mon Sep 17 00:00:00 2001 From: Brad Burlage Date: Mon, 3 Feb 2020 15:30:42 -0800 Subject: Add 1 Kokoro job per runtime test. PiperOrigin-RevId: 293019326 --- kokoro/runtime_tests/go1.12.cfg | 6 ++++++ kokoro/runtime_tests/java11.cfg | 6 ++++++ kokoro/runtime_tests/nodejs12.4.0.cfg | 6 ++++++ kokoro/runtime_tests/php7.3.6.cfg | 6 ++++++ kokoro/runtime_tests/python3.7.3.cfg | 6 ++++++ kokoro/runtime_tests/runtime_tests.sh | 25 +++++++++++++++++++++++++ scripts/runtime_tests.sh | 25 ------------------------- 7 files changed, 55 insertions(+), 25 deletions(-) create mode 100644 kokoro/runtime_tests/go1.12.cfg create mode 100644 kokoro/runtime_tests/java11.cfg create mode 100644 kokoro/runtime_tests/nodejs12.4.0.cfg create mode 100644 kokoro/runtime_tests/php7.3.6.cfg create mode 100644 kokoro/runtime_tests/python3.7.3.cfg create mode 100755 kokoro/runtime_tests/runtime_tests.sh delete mode 100755 scripts/runtime_tests.sh (limited to 'kokoro') diff --git a/kokoro/runtime_tests/go1.12.cfg b/kokoro/runtime_tests/go1.12.cfg new file mode 100644 index 000000000..024740ab2 --- /dev/null +++ b/kokoro/runtime_tests/go1.12.cfg @@ -0,0 +1,6 @@ +build_file: "github/kokoro/runtime_tests/runtime_tests.sh" + +env_vars { + key: "RUNTIME_TEST_NAME" + value: "go1.12" +} \ No newline at end of file diff --git a/kokoro/runtime_tests/java11.cfg b/kokoro/runtime_tests/java11.cfg new file mode 100644 index 000000000..f01d26153 --- /dev/null +++ b/kokoro/runtime_tests/java11.cfg @@ -0,0 +1,6 @@ +build_file: "github/kokoro/runtime_tests/runtime_tests.sh" + +env_vars { + key: "RUNTIME_TEST_NAME" + value: "java11" +} \ No newline at end of file diff --git a/kokoro/runtime_tests/nodejs12.4.0.cfg b/kokoro/runtime_tests/nodejs12.4.0.cfg new file mode 100644 index 000000000..d4861fb07 --- /dev/null +++ b/kokoro/runtime_tests/nodejs12.4.0.cfg @@ -0,0 +1,6 @@ +build_file: "github/kokoro/runtime_tests/runtime_tests.sh" + +env_vars { + key: "RUNTIME_TEST_NAME" + value: "nodejs12.4.0" +} \ No newline at end of file diff --git a/kokoro/runtime_tests/php7.3.6.cfg b/kokoro/runtime_tests/php7.3.6.cfg new file mode 100644 index 000000000..b737ed9cb --- /dev/null +++ b/kokoro/runtime_tests/php7.3.6.cfg @@ -0,0 +1,6 @@ +build_file: "github/kokoro/runtime_tests/runtime_tests.sh" + +env_vars { + key: "RUNTIME_TEST_NAME" + value: "php7.3.6" +} \ No newline at end of file diff --git a/kokoro/runtime_tests/python3.7.3.cfg b/kokoro/runtime_tests/python3.7.3.cfg new file mode 100644 index 000000000..971fcba05 --- /dev/null +++ b/kokoro/runtime_tests/python3.7.3.cfg @@ -0,0 +1,6 @@ +build_file: "github/kokoro/runtime_tests/runtime_tests.sh" + +env_vars { + key: "RUNTIME_TEST_NAME" + value: "python3.7.3" +} \ No newline at end of file diff --git a/kokoro/runtime_tests/runtime_tests.sh b/kokoro/runtime_tests/runtime_tests.sh new file mode 100755 index 000000000..9ee991e42 --- /dev/null +++ b/kokoro/runtime_tests/runtime_tests.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +if [ ! -v RUNTIME_TEST_NAME ]; then + echo 'Must set $RUNTIME_TEST_NAME' >&2 + exit 1 +fi + +install_runsc_for_test runtimes +test_runsc "//test/runtimes:${RUNTIME_TEST_NAME}_test" diff --git a/scripts/runtime_tests.sh b/scripts/runtime_tests.sh deleted file mode 100755 index 9ee991e42..000000000 --- a/scripts/runtime_tests.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source $(dirname $0)/common.sh - -if [ ! -v RUNTIME_TEST_NAME ]; then - echo 'Must set $RUNTIME_TEST_NAME' >&2 - exit 1 -fi - -install_runsc_for_test runtimes -test_runsc "//test/runtimes:${RUNTIME_TEST_NAME}_test" -- cgit v1.2.3 From 37abbbc547d9d78e0bf42f192403c8eca30593d8 Mon Sep 17 00:00:00 2001 From: Eyal Soha Date: Wed, 5 Feb 2020 11:17:14 -0800 Subject: Add packetdrill tests to presubmit and CI testing PiperOrigin-RevId: 293409718 --- kokoro/packetdrill_tests.cfg | 9 +++++++++ scripts/packetdrill_tests.sh | 20 ++++++++++++++++++++ test/packetdrill/defs.bzl | 6 ++++-- test/packetdrill/packetdrill_test.sh | 20 ++++++++++++++++---- 4 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 kokoro/packetdrill_tests.cfg create mode 100755 scripts/packetdrill_tests.sh (limited to 'kokoro') diff --git a/kokoro/packetdrill_tests.cfg b/kokoro/packetdrill_tests.cfg new file mode 100644 index 000000000..258d7deb4 --- /dev/null +++ b/kokoro/packetdrill_tests.cfg @@ -0,0 +1,9 @@ +build_file: "repo/scripts/packetdrill_tests.sh" + +action { + define_artifacts { + regex: "**/sponge_log.xml" + regex: "**/sponge_log.log" + regex: "**/outputs.zip" + } +} diff --git a/scripts/packetdrill_tests.sh b/scripts/packetdrill_tests.sh new file mode 100755 index 000000000..fc6bef79c --- /dev/null +++ b/scripts/packetdrill_tests.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +install_runsc_for_test runsc-d +test_runsc $(bazel query "attr(tags, manual, tests(//test/packetdrill/...))") diff --git a/test/packetdrill/defs.bzl b/test/packetdrill/defs.bzl index 582f97e0c..8623ce7b1 100644 --- a/test/packetdrill/defs.bzl +++ b/test/packetdrill/defs.bzl @@ -15,7 +15,7 @@ def _packetdrill_test_impl(ctx): # Make sure that everything is readable here. "find . -type f -exec chmod a+rx {} \\;", "find . -type d -exec chmod a+rx {} \\;", - "%s %s --init_script %s -- %s\n" % ( + "%s %s --init_script %s $@ -- %s\n" % ( test_runner.short_path, " ".join(ctx.attr.flags), ctx.files._init_script[0].short_path, @@ -76,7 +76,9 @@ def packetdrill_netstack_test(name, **kwargs): kwargs["tags"] = _PACKETDRILL_TAGS _packetdrill_test( name = name + "_netstack_test", - flags = ["--dut_platform", "netstack"], + # This is the default runtime unless + # "--test_arg=--runtime=OTHER_RUNTIME" is used to override the value. + flags = ["--dut_platform", "netstack", "--runtime", "runsc-d"], **kwargs ) diff --git a/test/packetdrill/packetdrill_test.sh b/test/packetdrill/packetdrill_test.sh index 614d94d74..0b22dfd5c 100755 --- a/test/packetdrill/packetdrill_test.sh +++ b/test/packetdrill/packetdrill_test.sh @@ -29,7 +29,7 @@ function failure() { } trap 'failure ${LINENO} "$BASH_COMMAND"' ERR -declare -r LONGOPTS="dut_platform:,init_script:" +declare -r LONGOPTS="dut_platform:,init_script:,runtime:" # Don't use declare below so that the error from getopt will end the script. PARSED=$(getopt --options "" --longoptions=$LONGOPTS --name "$0" -- "$@") @@ -39,6 +39,7 @@ eval set -- "$PARSED" while true; do case "$1" in --dut_platform) + # Either "linux" or "netstack". declare -r DUT_PLATFORM="$2" shift 2 ;; @@ -46,6 +47,13 @@ while true; do declare -r INIT_SCRIPT="$2" shift 2 ;; + --runtime) + # Not readonly because there might be multiple --runtime arguments and we + # want to use just the last one. Only used if --dut_platform is + # "netstack". + declare RUNTIME="$2" + shift 2 + ;; --) shift break @@ -61,9 +69,13 @@ declare -r scripts="$@" # Check that the required flags are defined in a way that is safe for "set -u". if [[ "${DUT_PLATFORM-}" == "netstack" ]]; then - declare -r RUNTIME="--runtime runsc-d" + if [[ -z "${RUNTIME-}" ]]; then + echo "FAIL: Missing --runtime argument: ${RUNTIME-}" + exit 2 + fi + declare -r RUNTIME_ARG="--runtime ${RUNTIME}" elif [[ "${DUT_PLATFORM-}" == "linux" ]]; then - declare -r RUNTIME="" + declare -r RUNTIME_ARG="" else echo "FAIL: Bad or missing --dut_platform argument: ${DUT_PLATFORM-}" exit 2 @@ -143,7 +155,7 @@ done docker pull "${IMAGE_TAG}" # Create the DUT container and connect to network. -DUT=$(docker create ${RUNTIME} --privileged --rm \ +DUT=$(docker create ${RUNTIME_ARG} --privileged --rm \ --stop-timeout ${TIMEOUT} -it ${IMAGE_TAG}) docker network connect "${CTRL_NET}" \ --ip "${CTRL_NET_PREFIX}${DUT_NET_SUFFIX}" "${DUT}" \ -- cgit v1.2.3 From bc504d52026790b7deed6506371ebc6d5d17c948 Mon Sep 17 00:00:00 2001 From: Brad Burlage Date: Mon, 10 Feb 2020 13:06:57 -0800 Subject: Fix build_file in runtimes_tests. PiperOrigin-RevId: 294286242 --- kokoro/runtime_tests/go1.12.cfg | 4 ++-- kokoro/runtime_tests/java11.cfg | 4 ++-- kokoro/runtime_tests/nodejs12.4.0.cfg | 4 ++-- kokoro/runtime_tests/php7.3.6.cfg | 4 ++-- kokoro/runtime_tests/python3.7.3.cfg | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) (limited to 'kokoro') diff --git a/kokoro/runtime_tests/go1.12.cfg b/kokoro/runtime_tests/go1.12.cfg index 024740ab2..164ddc18f 100644 --- a/kokoro/runtime_tests/go1.12.cfg +++ b/kokoro/runtime_tests/go1.12.cfg @@ -1,6 +1,6 @@ -build_file: "github/kokoro/runtime_tests/runtime_tests.sh" +build_file: "github/github/kokoro/runtime_tests/runtime_tests.sh" env_vars { key: "RUNTIME_TEST_NAME" value: "go1.12" -} \ No newline at end of file +} diff --git a/kokoro/runtime_tests/java11.cfg b/kokoro/runtime_tests/java11.cfg index f01d26153..4957d4794 100644 --- a/kokoro/runtime_tests/java11.cfg +++ b/kokoro/runtime_tests/java11.cfg @@ -1,6 +1,6 @@ -build_file: "github/kokoro/runtime_tests/runtime_tests.sh" +build_file: "github/github/kokoro/runtime_tests/runtime_tests.sh" env_vars { key: "RUNTIME_TEST_NAME" value: "java11" -} \ No newline at end of file +} diff --git a/kokoro/runtime_tests/nodejs12.4.0.cfg b/kokoro/runtime_tests/nodejs12.4.0.cfg index d4861fb07..1df343f95 100644 --- a/kokoro/runtime_tests/nodejs12.4.0.cfg +++ b/kokoro/runtime_tests/nodejs12.4.0.cfg @@ -1,6 +1,6 @@ -build_file: "github/kokoro/runtime_tests/runtime_tests.sh" +build_file: "github/github/kokoro/runtime_tests/runtime_tests.sh" env_vars { key: "RUNTIME_TEST_NAME" value: "nodejs12.4.0" -} \ No newline at end of file +} diff --git a/kokoro/runtime_tests/php7.3.6.cfg b/kokoro/runtime_tests/php7.3.6.cfg index b737ed9cb..8e3667125 100644 --- a/kokoro/runtime_tests/php7.3.6.cfg +++ b/kokoro/runtime_tests/php7.3.6.cfg @@ -1,6 +1,6 @@ -build_file: "github/kokoro/runtime_tests/runtime_tests.sh" +build_file: "github/github/kokoro/runtime_tests/runtime_tests.sh" env_vars { key: "RUNTIME_TEST_NAME" value: "php7.3.6" -} \ No newline at end of file +} diff --git a/kokoro/runtime_tests/python3.7.3.cfg b/kokoro/runtime_tests/python3.7.3.cfg index 971fcba05..0ca70d5bb 100644 --- a/kokoro/runtime_tests/python3.7.3.cfg +++ b/kokoro/runtime_tests/python3.7.3.cfg @@ -1,6 +1,6 @@ -build_file: "github/kokoro/runtime_tests/runtime_tests.sh" +build_file: "github/github/kokoro/runtime_tests/runtime_tests.sh" env_vars { key: "RUNTIME_TEST_NAME" value: "python3.7.3" -} \ No newline at end of file +} -- cgit v1.2.3 From cf1e50a80976fae95eef4ab05d961200b04e2346 Mon Sep 17 00:00:00 2001 From: gVisor bot Date: Wed, 12 Feb 2020 16:26:08 -0800 Subject: Minor runtime test fixes. * Allow scripts/common.sh to be sourced from outside the scripts/ directory * Fix passing empty args to Bazel, which causes the tool to exit with a failure even if the command succeeds. PiperOrigin-RevId: 294785456 --- kokoro/runtime_tests/runtime_tests.sh | 6 +++++- scripts/common.sh | 12 +++++++++++- scripts/common_build.sh | 27 ++++++++++++++++----------- 3 files changed, 32 insertions(+), 13 deletions(-) (limited to 'kokoro') diff --git a/kokoro/runtime_tests/runtime_tests.sh b/kokoro/runtime_tests/runtime_tests.sh index 9ee991e42..73a58f806 100755 --- a/kokoro/runtime_tests/runtime_tests.sh +++ b/kokoro/runtime_tests/runtime_tests.sh @@ -14,7 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -source $(dirname $0)/common.sh +# Run in the root of the repo. +cd "$(dirname "$0")" +cd "$(git rev-parse --show-toplevel)" + +source scripts/common.sh if [ ! -v RUNTIME_TEST_NAME ]; then echo 'Must set $RUNTIME_TEST_NAME' >&2 diff --git a/scripts/common.sh b/scripts/common.sh index cd91b9f8e..3ca699e4a 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -16,7 +16,17 @@ set -xeou pipefail -source $(dirname $0)/common_build.sh +# Get the path to the directory this script lives in. +# If this script is being called with `source`, $0 will be the path of the +# *sourcing* script, so we can't use `dirname $0` to find scripts in this +# directory. +if [[ -v BASH_SOURCE && "$0" != "$BASH_SOURCE" ]]; then + declare -r script_dir="$(dirname "$BASH_SOURCE")" +else + declare -r script_dir="$(dirname "$0")" +fi + +source "${script_dir}/common_build.sh" # Ensure it attempts to collect logs in all cases. trap collect_logs EXIT diff --git a/scripts/common_build.sh b/scripts/common_build.sh index 2c2a826c7..ae8b67383 100755 --- a/scripts/common_build.sh +++ b/scripts/common_build.sh @@ -14,8 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Install the latest version of Bazel and log the version. -(which use_bazel.sh && use_bazel.sh latest) || which bazel +which bazel bazel version # Switch into the workspace; only necessary if run with kokoro. @@ -26,27 +25,30 @@ elif [[ -v KOKORO_GIT_COMMIT ]] && [[ -d github/repo ]]; then fi # Set the standard bazel flags. -declare -r BAZEL_FLAGS=( +declare -a BAZEL_FLAGS=( "--show_timestamps" "--test_output=errors" "--keep_going" "--verbose_failures=true" ) -BAZEL_RBE_AUTH_FLAGS="" -BAZEL_RBE_FLAGS="" if [[ -v KOKORO_BAZEL_AUTH_CREDENTIAL ]]; then - declare -r BAZEL_RBE_AUTH_FLAGS="--auth_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL}" - declare -r BAZEL_RBE_FLAGS="--config=remote" + BAZEL_FLAGS+=( + "--auth_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL}" + "--config=remote" + ) fi +declare -r BAZEL_FLAGS # Wrap bazel. function build() { - bazel build "${BAZEL_RBE_FLAGS}" "${BAZEL_RBE_AUTH_FLAGS}" "${BAZEL_FLAGS[@]}" "$@" 2>&1 | - tee /dev/fd/2 | grep -E '^ bazel-bin/' | awk '{ print $1; }' + bazel build "${BAZEL_FLAGS[@]}" "$@" 2>&1 \ + | tee /dev/fd/2 \ + | grep -E '^ bazel-bin/' \ + | awk '{ print $1; }' } function test() { - bazel test "${BAZEL_RBE_FLAGS}" "${BAZEL_RBE_AUTH_FLAGS}" "${BAZEL_FLAGS[@]}" "$@" + bazel test "${BAZEL_FLAGS[@]}" "$@" } function run() { @@ -95,5 +97,8 @@ function collect_logs() { } function find_branch_name() { - git branch --show-current || git rev-parse HEAD || bazel info workspace | xargs basename + git branch --show-current \ + || git rev-parse HEAD \ + || bazel info workspace \ + | xargs basename } -- cgit v1.2.3 From 8dae8a10f01b49d9b28c1ae72b67cb5b83238963 Mon Sep 17 00:00:00 2001 From: gVisor bot Date: Tue, 18 Feb 2020 17:03:59 -0800 Subject: Internal change. PiperOrigin-RevId: 295857142 --- kokoro/benchmarks/benchmarks.cfg | 26 ++++++++++++++++++++++++++ kokoro/benchmarks/benchmarks.sh | 25 +++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 kokoro/benchmarks/benchmarks.cfg create mode 100644 kokoro/benchmarks/benchmarks.sh (limited to 'kokoro') diff --git a/kokoro/benchmarks/benchmarks.cfg b/kokoro/benchmarks/benchmarks.cfg new file mode 100644 index 000000000..9b9073e62 --- /dev/null +++ b/kokoro/benchmarks/benchmarks.cfg @@ -0,0 +1,26 @@ +build_file : 'github/github/kokoro/benchmarks/benchmarks.sh' + + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id : 73898 + keyname : 'kokoro-rbe-service-account' + }, + } +} + +env_vars { + key : 'PROJECT' + value : 'gvisor-kokoro-testing' +} + +env_vars { + key : 'ZONE' + value : 'us-central1-b' +} + +env_vars { + key : 'KOKORO_SERVICE_ACCOUNT' + value : '73898_kokoro-rbe-service-account' +} diff --git a/kokoro/benchmarks/benchmarks.sh b/kokoro/benchmarks/benchmarks.sh new file mode 100644 index 000000000..a0317db02 --- /dev/null +++ b/kokoro/benchmarks/benchmarks.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2020 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run in the root of the repo. +cd "$(dirname "$0")" + +KEY_PATH=${KEY_PATH:-"${KOKORO_KEYSTORE_DIR}/${KOKORO_SERVICE_ACCOUNT}"} + +gcloud auth activate-service-account --key-file "${KEY_PATH}" + +gcloud compute instances list + -- cgit v1.2.3 From 90d0851c0e37146158e4f46f6aa1707d786fb48a Mon Sep 17 00:00:00 2001 From: gVisor bot Date: Wed, 19 Feb 2020 11:35:16 -0800 Subject: Internal change. PiperOrigin-RevId: 296016675 --- kokoro/benchmark.sh | 25 +++++++++++++++++++++++++ kokoro/benchmark_tests.cfg | 26 ++++++++++++++++++++++++++ kokoro/benchmarks/benchmarks.cfg | 26 -------------------------- kokoro/benchmarks/benchmarks.sh | 25 ------------------------- 4 files changed, 51 insertions(+), 51 deletions(-) create mode 100644 kokoro/benchmark.sh create mode 100644 kokoro/benchmark_tests.cfg delete mode 100644 kokoro/benchmarks/benchmarks.cfg delete mode 100644 kokoro/benchmarks/benchmarks.sh (limited to 'kokoro') diff --git a/kokoro/benchmark.sh b/kokoro/benchmark.sh new file mode 100644 index 000000000..a0317db02 --- /dev/null +++ b/kokoro/benchmark.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2020 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run in the root of the repo. +cd "$(dirname "$0")" + +KEY_PATH=${KEY_PATH:-"${KOKORO_KEYSTORE_DIR}/${KOKORO_SERVICE_ACCOUNT}"} + +gcloud auth activate-service-account --key-file "${KEY_PATH}" + +gcloud compute instances list + diff --git a/kokoro/benchmark_tests.cfg b/kokoro/benchmark_tests.cfg new file mode 100644 index 000000000..76ba30752 --- /dev/null +++ b/kokoro/benchmark_tests.cfg @@ -0,0 +1,26 @@ +build_file : 'github/github/kokoro/benchmarks.sh' + + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id : 73898 + keyname : 'kokoro-rbe-service-account' + }, + } +} + +env_vars { + key : 'PROJECT' + value : 'gvisor-kokoro-testing' +} + +env_vars { + key : 'ZONE' + value : 'us-central1-b' +} + +env_vars { + key : 'KOKORO_SERVICE_ACCOUNT' + value : '73898_kokoro-rbe-service-account' +} diff --git a/kokoro/benchmarks/benchmarks.cfg b/kokoro/benchmarks/benchmarks.cfg deleted file mode 100644 index 9b9073e62..000000000 --- a/kokoro/benchmarks/benchmarks.cfg +++ /dev/null @@ -1,26 +0,0 @@ -build_file : 'github/github/kokoro/benchmarks/benchmarks.sh' - - -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id : 73898 - keyname : 'kokoro-rbe-service-account' - }, - } -} - -env_vars { - key : 'PROJECT' - value : 'gvisor-kokoro-testing' -} - -env_vars { - key : 'ZONE' - value : 'us-central1-b' -} - -env_vars { - key : 'KOKORO_SERVICE_ACCOUNT' - value : '73898_kokoro-rbe-service-account' -} diff --git a/kokoro/benchmarks/benchmarks.sh b/kokoro/benchmarks/benchmarks.sh deleted file mode 100644 index a0317db02..000000000 --- a/kokoro/benchmarks/benchmarks.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Copyright 2020 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Run in the root of the repo. -cd "$(dirname "$0")" - -KEY_PATH=${KEY_PATH:-"${KOKORO_KEYSTORE_DIR}/${KOKORO_SERVICE_ACCOUNT}"} - -gcloud auth activate-service-account --key-file "${KEY_PATH}" - -gcloud compute instances list - -- cgit v1.2.3 From ddc02e489cfe3990f76882463c0ad386710a9a94 Mon Sep 17 00:00:00 2001 From: gVisor bot Date: Wed, 19 Feb 2020 13:11:38 -0800 Subject: Internal change. PiperOrigin-RevId: 296039442 --- kokoro/benchmark_tests.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kokoro') diff --git a/kokoro/benchmark_tests.cfg b/kokoro/benchmark_tests.cfg index 76ba30752..6a5d9b114 100644 --- a/kokoro/benchmark_tests.cfg +++ b/kokoro/benchmark_tests.cfg @@ -1,4 +1,4 @@ -build_file : 'github/github/kokoro/benchmarks.sh' +build_file : 'repo/kokoro/benchmarks.sh' before_action { -- cgit v1.2.3 From 3a20eccf8b2d30eeff60f616a3e1d0d15c6ffac4 Mon Sep 17 00:00:00 2001 From: gVisor bot Date: Wed, 19 Feb 2020 14:17:12 -0800 Subject: Internal change. PiperOrigin-RevId: 296055452 --- kokoro/benchmark.sh | 25 ------------------------- kokoro/benchmark_tests.cfg | 2 +- scripts/benchmark.sh | 25 +++++++++++++++++++++++++ 3 files changed, 26 insertions(+), 26 deletions(-) delete mode 100644 kokoro/benchmark.sh create mode 100644 scripts/benchmark.sh (limited to 'kokoro') diff --git a/kokoro/benchmark.sh b/kokoro/benchmark.sh deleted file mode 100644 index a0317db02..000000000 --- a/kokoro/benchmark.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Copyright 2020 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Run in the root of the repo. -cd "$(dirname "$0")" - -KEY_PATH=${KEY_PATH:-"${KOKORO_KEYSTORE_DIR}/${KOKORO_SERVICE_ACCOUNT}"} - -gcloud auth activate-service-account --key-file "${KEY_PATH}" - -gcloud compute instances list - diff --git a/kokoro/benchmark_tests.cfg b/kokoro/benchmark_tests.cfg index 6a5d9b114..c48518a05 100644 --- a/kokoro/benchmark_tests.cfg +++ b/kokoro/benchmark_tests.cfg @@ -1,4 +1,4 @@ -build_file : 'repo/kokoro/benchmarks.sh' +build_file : 'repo/scripts/benchmark.sh' before_action { diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh new file mode 100644 index 000000000..a0317db02 --- /dev/null +++ b/scripts/benchmark.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2020 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run in the root of the repo. +cd "$(dirname "$0")" + +KEY_PATH=${KEY_PATH:-"${KOKORO_KEYSTORE_DIR}/${KOKORO_SERVICE_ACCOUNT}"} + +gcloud auth activate-service-account --key-file "${KEY_PATH}" + +gcloud compute instances list + -- cgit v1.2.3 From ededa90d07e4df3eb3fe8a52a0afbcdaf82e8df5 Mon Sep 17 00:00:00 2001 From: Brad Burlage Date: Mon, 24 Feb 2020 14:21:18 -0800 Subject: Internal change. PiperOrigin-RevId: 296972565 --- kokoro/runtime_tests/go1.12.cfg | 10 ++++++++++ kokoro/runtime_tests/java11.cfg | 10 ++++++++++ kokoro/runtime_tests/nodejs12.4.0.cfg | 10 ++++++++++ kokoro/runtime_tests/php7.3.6.cfg | 10 ++++++++++ kokoro/runtime_tests/python3.7.3.cfg | 10 ++++++++++ scripts/common_build.sh | 12 ++++++++++-- 6 files changed, 60 insertions(+), 2 deletions(-) (limited to 'kokoro') diff --git a/kokoro/runtime_tests/go1.12.cfg b/kokoro/runtime_tests/go1.12.cfg index 164ddc18f..fd4911e88 100644 --- a/kokoro/runtime_tests/go1.12.cfg +++ b/kokoro/runtime_tests/go1.12.cfg @@ -4,3 +4,13 @@ env_vars { key: "RUNTIME_TEST_NAME" value: "go1.12" } + +action { + define_artifacts { + regex: "**/sponge_log.xml" + regex: "**/sponge_log.log" + regex: "**/outputs.zip" + regex: "**/runsc" + regex: "**/runsc.*" + } +} \ No newline at end of file diff --git a/kokoro/runtime_tests/java11.cfg b/kokoro/runtime_tests/java11.cfg index 4957d4794..7f8611a08 100644 --- a/kokoro/runtime_tests/java11.cfg +++ b/kokoro/runtime_tests/java11.cfg @@ -4,3 +4,13 @@ env_vars { key: "RUNTIME_TEST_NAME" value: "java11" } + +action { + define_artifacts { + regex: "**/sponge_log.xml" + regex: "**/sponge_log.log" + regex: "**/outputs.zip" + regex: "**/runsc" + regex: "**/runsc.*" + } +} \ No newline at end of file diff --git a/kokoro/runtime_tests/nodejs12.4.0.cfg b/kokoro/runtime_tests/nodejs12.4.0.cfg index 1df343f95..c67ad5567 100644 --- a/kokoro/runtime_tests/nodejs12.4.0.cfg +++ b/kokoro/runtime_tests/nodejs12.4.0.cfg @@ -4,3 +4,13 @@ env_vars { key: "RUNTIME_TEST_NAME" value: "nodejs12.4.0" } + +action { + define_artifacts { + regex: "**/sponge_log.xml" + regex: "**/sponge_log.log" + regex: "**/outputs.zip" + regex: "**/runsc" + regex: "**/runsc.*" + } +} \ No newline at end of file diff --git a/kokoro/runtime_tests/php7.3.6.cfg b/kokoro/runtime_tests/php7.3.6.cfg index 8e3667125..f266c5e26 100644 --- a/kokoro/runtime_tests/php7.3.6.cfg +++ b/kokoro/runtime_tests/php7.3.6.cfg @@ -4,3 +4,13 @@ env_vars { key: "RUNTIME_TEST_NAME" value: "php7.3.6" } + +action { + define_artifacts { + regex: "**/sponge_log.xml" + regex: "**/sponge_log.log" + regex: "**/outputs.zip" + regex: "**/runsc" + regex: "**/runsc.*" + } +} \ No newline at end of file diff --git a/kokoro/runtime_tests/python3.7.3.cfg b/kokoro/runtime_tests/python3.7.3.cfg index 0ca70d5bb..574add152 100644 --- a/kokoro/runtime_tests/python3.7.3.cfg +++ b/kokoro/runtime_tests/python3.7.3.cfg @@ -4,3 +4,13 @@ env_vars { key: "RUNTIME_TEST_NAME" value: "python3.7.3" } + +action { + define_artifacts { + regex: "**/sponge_log.xml" + regex: "**/sponge_log.log" + regex: "**/outputs.zip" + regex: "**/runsc" + regex: "**/runsc.*" + } +} \ No newline at end of file diff --git a/scripts/common_build.sh b/scripts/common_build.sh index ae8b67383..3be0bb21c 100755 --- a/scripts/common_build.sh +++ b/scripts/common_build.sh @@ -70,7 +70,9 @@ function collect_logs() { for d in `find -L "bazel-testlogs" -name 'shard_*_of_*' | xargs dirname | sort | uniq`; do junitparser merge `find $d -name test.xml` $d/test.xml cat $d/shard_*_of_*/test.log > $d/test.log - ls -l $d/shard_*_of_*/test.outputs/outputs.zip && zip -r -1 $d/outputs.zip $d/shard_*_of_*/test.outputs/outputs.zip + if ls -l $d/shard_*_of_*/test.outputs/outputs.zip 2>/dev/null; then + zip -r -1 "$d/outputs.zip" $d/shard_*_of_*/test.outputs/outputs.zip + fi done find -L "bazel-testlogs" -name 'shard_*_of_*' | xargs rm -rf # Move test logs to Kokoro directory. tar is used to conveniently perform @@ -90,7 +92,13 @@ function collect_logs() { echo " gsutil cp gs://gvisor/logs/${KOKORO_BUILD_ARTIFACTS_SUBDIR}/${archive} /tmp" echo " https://storage.cloud.google.com/gvisor/logs/${KOKORO_BUILD_ARTIFACTS_SUBDIR}/${archive}" fi - tar --create --gzip --file="${KOKORO_ARTIFACTS_DIR}/${archive}" -C "${RUNSC_LOGS_DIR}" . + time tar \ + --verbose \ + --create \ + --gzip \ + --file="${KOKORO_ARTIFACTS_DIR}/${archive}" \ + --directory "${RUNSC_LOGS_DIR}" \ + . fi fi fi -- cgit v1.2.3 From 5ee9bbb15d90b5f116d9482e932e0823cffd9f6e Mon Sep 17 00:00:00 2001 From: Michael Pratt Date: Wed, 11 Mar 2020 14:38:04 -0700 Subject: Upgrade Kythe PiperOrigin-RevId: 300409401 --- kokoro/kythe/generate_xrefs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kokoro') diff --git a/kokoro/kythe/generate_xrefs.sh b/kokoro/kythe/generate_xrefs.sh index 323b0f77b..2f531aa72 100644 --- a/kokoro/kythe/generate_xrefs.sh +++ b/kokoro/kythe/generate_xrefs.sh @@ -23,7 +23,7 @@ bazel version python3 -V -readonly KYTHE_VERSION='v0.0.41' +readonly KYTHE_VERSION='v0.0.43' readonly WORKDIR="$(mktemp -d)" readonly KYTHE_DIR="${WORKDIR}/kythe-${KYTHE_VERSION}" if [[ -n "$KOKORO_GIT_COMMIT" ]]; then -- cgit v1.2.3 From fd84cddab0dc175d5050e2ac46f8520f632d1550 Mon Sep 17 00:00:00 2001 From: Zach Koopmans Date: Wed, 11 Mar 2020 14:44:57 -0700 Subject: Internal change PiperOrigin-RevId: 300410856 --- kokoro/benchmark_tests.cfg | 6 +++--- scripts/benchmark.sh | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'kokoro') diff --git a/kokoro/benchmark_tests.cfg b/kokoro/benchmark_tests.cfg index c48518a05..7e2c8acad 100644 --- a/kokoro/benchmark_tests.cfg +++ b/kokoro/benchmark_tests.cfg @@ -5,7 +5,7 @@ before_action { fetch_keystore { keystore_resource { keystore_config_id : 73898 - keyname : 'kokoro-rbe-service-account' + keyname : 'gvisor-benchmarks-service-account' }, } } @@ -21,6 +21,6 @@ env_vars { } env_vars { - key : 'KOKORO_SERVICE_ACCOUNT' - value : '73898_kokoro-rbe-service-account' + key : 'GCLOUD_CREDENTIALS' + value : '73898_gvisor-benchmarks-service-account' } diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index a0317db02..79ff198d5 100644 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -17,9 +17,10 @@ # Run in the root of the repo. cd "$(dirname "$0")" -KEY_PATH=${KEY_PATH:-"${KOKORO_KEYSTORE_DIR}/${KOKORO_SERVICE_ACCOUNT}"} +export GOOGLE_APPLICATION_CREDENTIALS="${KOKORO_KEYSTORE_DIR}/${GCLOUD_CREDENTIALS}" -gcloud auth activate-service-account --key-file "${KEY_PATH}" +gcloud auth activate-service-account --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" gcloud compute instances list +bq show gvisor-benchmarks:test.test -- cgit v1.2.3 From d8cf54f20f2dd7348d06796f29a42991086c3f81 Mon Sep 17 00:00:00 2001 From: Zach Koopmans Date: Fri, 13 Mar 2020 15:17:01 -0700 Subject: Internal change PiperOrigin-RevId: 300836270 --- kokoro/benchmark_tests.cfg | 2 +- scripts/benchmark.sh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'kokoro') diff --git a/kokoro/benchmark_tests.cfg b/kokoro/benchmark_tests.cfg index 7e2c8acad..f85cc9681 100644 --- a/kokoro/benchmark_tests.cfg +++ b/kokoro/benchmark_tests.cfg @@ -12,7 +12,7 @@ before_action { env_vars { key : 'PROJECT' - value : 'gvisor-kokoro-testing' + value : 'gvisor-benchmarks' } env_vars { diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 032899386..06d44f914 100644 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -23,6 +23,9 @@ export GOOGLE_APPLICATION_CREDENTIALS="${KOKORO_KEYSTORE_DIR}/${GCLOUD_CREDENTIA gcloud auth activate-service-account \ --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" +gcloud config set project ${PROJECT} +gcloud config set compute/zone ${ZONE} + bazel run //benchmarks:benchmarks -- \ --verbose \ run-gcp \ -- cgit v1.2.3 From 1cc5a71a0e2100c89c97ffd12a38143907b33630 Mon Sep 17 00:00:00 2001 From: Kevin Krakauer Date: Tue, 17 Mar 2020 16:21:36 -0700 Subject: iptables: fix type in script name, mark some new tests as skipped PiperOrigin-RevId: 301476456 --- kokoro/iptables_tests.cfg | 2 +- test/iptables/iptables_test.go | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) (limited to 'kokoro') diff --git a/kokoro/iptables_tests.cfg b/kokoro/iptables_tests.cfg index 7af20629a..a30d82591 100644 --- a/kokoro/iptables_tests.cfg +++ b/kokoro/iptables_tests.cfg @@ -1,4 +1,4 @@ -build_file: "repo/scripts/iptables_test.sh" +build_file: "repo/scripts/iptables_tests.sh" action { define_artifacts { diff --git a/test/iptables/iptables_test.go b/test/iptables/iptables_test.go index 73ba8b447..7f1f70606 100644 --- a/test/iptables/iptables_test.go +++ b/test/iptables/iptables_test.go @@ -191,6 +191,7 @@ func TestFilterInputDropOnlyUDP(t *testing.T) { } func TestNATRedirectUDPPort(t *testing.T) { + // TODO(gvisor.dev/issue/170): Enable when supported. t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATRedirectUDPPort{}); err != nil { t.Fatal(err) @@ -198,6 +199,7 @@ func TestNATRedirectUDPPort(t *testing.T) { } func TestNATRedirectTCPPort(t *testing.T) { + // TODO(gvisor.dev/issue/170): Enable when supported. t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATRedirectTCPPort{}); err != nil { t.Fatal(err) @@ -205,6 +207,7 @@ func TestNATRedirectTCPPort(t *testing.T) { } func TestNATDropUDP(t *testing.T) { + // TODO(gvisor.dev/issue/170): Enable when supported. t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATDropUDP{}); err != nil { t.Fatal(err) @@ -212,6 +215,7 @@ func TestNATDropUDP(t *testing.T) { } func TestNATAcceptAll(t *testing.T) { + // TODO(gvisor.dev/issue/170): Enable when supported. t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATAcceptAll{}); err != nil { t.Fatal(err) @@ -255,6 +259,7 @@ func TestFilterInputReturnUnderflow(t *testing.T) { } func TestFilterOutputDropTCPDestPort(t *testing.T) { + // TODO(gvisor.dev/issue/170): Enable when supported. t.Skip("filter OUTPUT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(FilterOutputDropTCPDestPort{}); err != nil { t.Fatal(err) @@ -262,6 +267,7 @@ func TestFilterOutputDropTCPDestPort(t *testing.T) { } func TestFilterOutputDropTCPSrcPort(t *testing.T) { + // TODO(gvisor.dev/issue/170): Enable when supported. t.Skip("filter OUTPUT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(FilterOutputDropTCPSrcPort{}); err != nil { t.Fatal(err) @@ -329,42 +335,56 @@ func TestOutputInvertDestination(t *testing.T) { } func TestNATOutRedirectIP(t *testing.T) { + // TODO(gvisor.dev/issue/170): Enable when supported. + t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATOutRedirectIP{}); err != nil { t.Fatal(err) } } func TestNATOutDontRedirectIP(t *testing.T) { + // TODO(gvisor.dev/issue/170): Enable when supported. + t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATOutDontRedirectIP{}); err != nil { t.Fatal(err) } } func TestNATOutRedirectInvert(t *testing.T) { + // TODO(gvisor.dev/issue/170): Enable when supported. + t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATOutRedirectInvert{}); err != nil { t.Fatal(err) } } func TestNATPreRedirectIP(t *testing.T) { + // TODO(gvisor.dev/issue/170): Enable when supported. + t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATPreRedirectIP{}); err != nil { t.Fatal(err) } } func TestNATPreDontRedirectIP(t *testing.T) { + // TODO(gvisor.dev/issue/170): Enable when supported. + t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATPreDontRedirectIP{}); err != nil { t.Fatal(err) } } func TestNATPreRedirectInvert(t *testing.T) { + // TODO(gvisor.dev/issue/170): Enable when supported. + t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATPreRedirectInvert{}); err != nil { t.Fatal(err) } } func TestNATRedirectRequiresProtocol(t *testing.T) { + // TODO(gvisor.dev/issue/170): Enable when supported. + t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATRedirectRequiresProtocol{}); err != nil { t.Fatal(err) } -- cgit v1.2.3 From fc16e64396cf534bc4336e6bc0396a2f0f621e70 Mon Sep 17 00:00:00 2001 From: Kevin Krakauer Date: Wed, 18 Mar 2020 13:06:55 -0700 Subject: Automated rollback of changelist 301476456 PiperOrigin-RevId: 301650898 --- kokoro/iptables_tests.cfg | 2 +- test/iptables/iptables_test.go | 20 -------------------- 2 files changed, 1 insertion(+), 21 deletions(-) (limited to 'kokoro') diff --git a/kokoro/iptables_tests.cfg b/kokoro/iptables_tests.cfg index a30d82591..7af20629a 100644 --- a/kokoro/iptables_tests.cfg +++ b/kokoro/iptables_tests.cfg @@ -1,4 +1,4 @@ -build_file: "repo/scripts/iptables_tests.sh" +build_file: "repo/scripts/iptables_test.sh" action { define_artifacts { diff --git a/test/iptables/iptables_test.go b/test/iptables/iptables_test.go index 7f1f70606..73ba8b447 100644 --- a/test/iptables/iptables_test.go +++ b/test/iptables/iptables_test.go @@ -191,7 +191,6 @@ func TestFilterInputDropOnlyUDP(t *testing.T) { } func TestNATRedirectUDPPort(t *testing.T) { - // TODO(gvisor.dev/issue/170): Enable when supported. t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATRedirectUDPPort{}); err != nil { t.Fatal(err) @@ -199,7 +198,6 @@ func TestNATRedirectUDPPort(t *testing.T) { } func TestNATRedirectTCPPort(t *testing.T) { - // TODO(gvisor.dev/issue/170): Enable when supported. t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATRedirectTCPPort{}); err != nil { t.Fatal(err) @@ -207,7 +205,6 @@ func TestNATRedirectTCPPort(t *testing.T) { } func TestNATDropUDP(t *testing.T) { - // TODO(gvisor.dev/issue/170): Enable when supported. t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATDropUDP{}); err != nil { t.Fatal(err) @@ -215,7 +212,6 @@ func TestNATDropUDP(t *testing.T) { } func TestNATAcceptAll(t *testing.T) { - // TODO(gvisor.dev/issue/170): Enable when supported. t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATAcceptAll{}); err != nil { t.Fatal(err) @@ -259,7 +255,6 @@ func TestFilterInputReturnUnderflow(t *testing.T) { } func TestFilterOutputDropTCPDestPort(t *testing.T) { - // TODO(gvisor.dev/issue/170): Enable when supported. t.Skip("filter OUTPUT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(FilterOutputDropTCPDestPort{}); err != nil { t.Fatal(err) @@ -267,7 +262,6 @@ func TestFilterOutputDropTCPDestPort(t *testing.T) { } func TestFilterOutputDropTCPSrcPort(t *testing.T) { - // TODO(gvisor.dev/issue/170): Enable when supported. t.Skip("filter OUTPUT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(FilterOutputDropTCPSrcPort{}); err != nil { t.Fatal(err) @@ -335,56 +329,42 @@ func TestOutputInvertDestination(t *testing.T) { } func TestNATOutRedirectIP(t *testing.T) { - // TODO(gvisor.dev/issue/170): Enable when supported. - t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATOutRedirectIP{}); err != nil { t.Fatal(err) } } func TestNATOutDontRedirectIP(t *testing.T) { - // TODO(gvisor.dev/issue/170): Enable when supported. - t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATOutDontRedirectIP{}); err != nil { t.Fatal(err) } } func TestNATOutRedirectInvert(t *testing.T) { - // TODO(gvisor.dev/issue/170): Enable when supported. - t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATOutRedirectInvert{}); err != nil { t.Fatal(err) } } func TestNATPreRedirectIP(t *testing.T) { - // TODO(gvisor.dev/issue/170): Enable when supported. - t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATPreRedirectIP{}); err != nil { t.Fatal(err) } } func TestNATPreDontRedirectIP(t *testing.T) { - // TODO(gvisor.dev/issue/170): Enable when supported. - t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATPreDontRedirectIP{}); err != nil { t.Fatal(err) } } func TestNATPreRedirectInvert(t *testing.T) { - // TODO(gvisor.dev/issue/170): Enable when supported. - t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATPreRedirectInvert{}); err != nil { t.Fatal(err) } } func TestNATRedirectRequiresProtocol(t *testing.T) { - // TODO(gvisor.dev/issue/170): Enable when supported. - t.Skip("NAT isn't supported yet (gvisor.dev/issue/170).") if err := singleTest(NATRedirectRequiresProtocol{}); err != nil { t.Fatal(err) } -- cgit v1.2.3 From e0fbcdcb7f8a3e18946ccd5d5b98ea3adadee5ba Mon Sep 17 00:00:00 2001 From: Eyal Soha Date: Thu, 19 Mar 2020 12:32:15 -0700 Subject: Add packetimpact tests to presubmit and CI testing PiperOrigin-RevId: 301872161 --- kokoro/packetimpact_tests.cfg | 9 +++++++++ scripts/packetimpact_tests.sh | 20 ++++++++++++++++++++ test/packetimpact/tests/defs.bzl | 4 ++-- 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 kokoro/packetimpact_tests.cfg create mode 100755 scripts/packetimpact_tests.sh (limited to 'kokoro') diff --git a/kokoro/packetimpact_tests.cfg b/kokoro/packetimpact_tests.cfg new file mode 100644 index 000000000..db86b52d5 --- /dev/null +++ b/kokoro/packetimpact_tests.cfg @@ -0,0 +1,9 @@ +build_file: "repo/scripts/packetimpact_tests.sh" + +action { + define_artifacts { + regex: "**/sponge_log.xml" + regex: "**/sponge_log.log" + regex: "**/outputs.zip" + } +} diff --git a/scripts/packetimpact_tests.sh b/scripts/packetimpact_tests.sh new file mode 100755 index 000000000..027d11e64 --- /dev/null +++ b/scripts/packetimpact_tests.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +install_runsc_for_test runsc-d +test_runsc $(bazel query "attr(tags, packetimpact, tests(//test/packetimpact/...))") diff --git a/test/packetimpact/tests/defs.bzl b/test/packetimpact/tests/defs.bzl index 3baac567a..1b4213d9b 100644 --- a/test/packetimpact/tests/defs.bzl +++ b/test/packetimpact/tests/defs.bzl @@ -71,7 +71,7 @@ def packetimpact_linux_test(name, testbench_binary, **kwargs): name = name + "_linux_test", testbench_binary = testbench_binary, flags = ["--dut_platform", "linux"], - tags = PACKETIMPACT_TAGS, + tags = PACKETIMPACT_TAGS + ["packetimpact"], **kwargs ) @@ -89,7 +89,7 @@ def packetimpact_netstack_test(name, testbench_binary, **kwargs): # This is the default runtime unless # "--test_arg=--runtime=OTHER_RUNTIME" is used to override the value. flags = ["--dut_platform", "netstack", "--runtime=runsc-d"], - tags = PACKETIMPACT_TAGS, + tags = PACKETIMPACT_TAGS + ["packetimpact"], **kwargs ) -- cgit v1.2.3 From 238e80fe38670faed5418444ce9fb2eb84e6d5d7 Mon Sep 17 00:00:00 2001 From: Michael Pratt Date: Thu, 19 Mar 2020 14:09:03 -0700 Subject: Automated rollback of changelist 300409401 PiperOrigin-RevId: 301891849 --- kokoro/kythe/generate_xrefs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kokoro') diff --git a/kokoro/kythe/generate_xrefs.sh b/kokoro/kythe/generate_xrefs.sh index 2f531aa72..323b0f77b 100644 --- a/kokoro/kythe/generate_xrefs.sh +++ b/kokoro/kythe/generate_xrefs.sh @@ -23,7 +23,7 @@ bazel version python3 -V -readonly KYTHE_VERSION='v0.0.43' +readonly KYTHE_VERSION='v0.0.41' readonly WORKDIR="$(mktemp -d)" readonly KYTHE_DIR="${WORKDIR}/kythe-${KYTHE_VERSION}" if [[ -n "$KOKORO_GIT_COMMIT" ]]; then -- cgit v1.2.3 From 8c35614760a194d52a706748adb9af3a28fb864b Mon Sep 17 00:00:00 2001 From: Kevin Krakauer Date: Mon, 23 Mar 2020 13:35:10 -0700 Subject: iptables: enable iptables tests as nonblocking PiperOrigin-RevId: 302506064 --- kokoro/iptables_tests.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kokoro') diff --git a/kokoro/iptables_tests.cfg b/kokoro/iptables_tests.cfg index 7af20629a..a30d82591 100644 --- a/kokoro/iptables_tests.cfg +++ b/kokoro/iptables_tests.cfg @@ -1,4 +1,4 @@ -build_file: "repo/scripts/iptables_test.sh" +build_file: "repo/scripts/iptables_tests.sh" action { define_artifacts { -- cgit v1.2.3 From 9ba3086d9dfc7e9a4a957446f443786b179cd84e Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Mon, 20 Apr 2020 11:08:01 -0700 Subject: Move runtime_tests.sh to align with other scripts. PiperOrigin-RevId: 307435879 --- kokoro/runtime_tests/go1.12.cfg | 2 +- kokoro/runtime_tests/java11.cfg | 2 +- kokoro/runtime_tests/nodejs12.4.0.cfg | 2 +- kokoro/runtime_tests/php7.3.6.cfg | 2 +- kokoro/runtime_tests/python3.7.3.cfg | 2 +- scripts/runtime_tests.sh | 26 ++++++++++++++++++++++++++ 6 files changed, 31 insertions(+), 5 deletions(-) create mode 100755 scripts/runtime_tests.sh (limited to 'kokoro') diff --git a/kokoro/runtime_tests/go1.12.cfg b/kokoro/runtime_tests/go1.12.cfg index fd4911e88..04bfe2868 100644 --- a/kokoro/runtime_tests/go1.12.cfg +++ b/kokoro/runtime_tests/go1.12.cfg @@ -1,4 +1,4 @@ -build_file: "github/github/kokoro/runtime_tests/runtime_tests.sh" +build_file: "github/github/scripts/runtime_tests.sh" env_vars { key: "RUNTIME_TEST_NAME" diff --git a/kokoro/runtime_tests/java11.cfg b/kokoro/runtime_tests/java11.cfg index 7f8611a08..c82855cd2 100644 --- a/kokoro/runtime_tests/java11.cfg +++ b/kokoro/runtime_tests/java11.cfg @@ -1,4 +1,4 @@ -build_file: "github/github/kokoro/runtime_tests/runtime_tests.sh" +build_file: "github/github/scripts/runtime_tests.sh" env_vars { key: "RUNTIME_TEST_NAME" diff --git a/kokoro/runtime_tests/nodejs12.4.0.cfg b/kokoro/runtime_tests/nodejs12.4.0.cfg index c67ad5567..5512db5df 100644 --- a/kokoro/runtime_tests/nodejs12.4.0.cfg +++ b/kokoro/runtime_tests/nodejs12.4.0.cfg @@ -1,4 +1,4 @@ -build_file: "github/github/kokoro/runtime_tests/runtime_tests.sh" +build_file: "github/github/scripts/runtime_tests.sh" env_vars { key: "RUNTIME_TEST_NAME" diff --git a/kokoro/runtime_tests/php7.3.6.cfg b/kokoro/runtime_tests/php7.3.6.cfg index f266c5e26..bc9ac92aa 100644 --- a/kokoro/runtime_tests/php7.3.6.cfg +++ b/kokoro/runtime_tests/php7.3.6.cfg @@ -1,4 +1,4 @@ -build_file: "github/github/kokoro/runtime_tests/runtime_tests.sh" +build_file: "github/github/scripts/runtime_tests.sh" env_vars { key: "RUNTIME_TEST_NAME" diff --git a/kokoro/runtime_tests/python3.7.3.cfg b/kokoro/runtime_tests/python3.7.3.cfg index 574add152..12eb13860 100644 --- a/kokoro/runtime_tests/python3.7.3.cfg +++ b/kokoro/runtime_tests/python3.7.3.cfg @@ -1,4 +1,4 @@ -build_file: "github/github/kokoro/runtime_tests/runtime_tests.sh" +build_file: "github/github/scripts/runtime_tests.sh" env_vars { key: "RUNTIME_TEST_NAME" diff --git a/scripts/runtime_tests.sh b/scripts/runtime_tests.sh new file mode 100755 index 000000000..350a59f7c --- /dev/null +++ b/scripts/runtime_tests.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +# Check that a runtime is provided. +if [ ! -v RUNTIME_TEST_NAME ]; then + echo "Must set $RUNTIME_TEST_NAME" >&2 + exit 1 +fi + +install_runsc_for_test runtimes +test_runsc "//test/runtimes:${RUNTIME_TEST_NAME}_test" -- cgit v1.2.3 From a27d6329df96d50b5b5cd4b550ca024cc3f0b16c Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Wed, 22 Apr 2020 10:13:40 -0700 Subject: Remove unnecessary kokoro configurations. PiperOrigin-RevId: 307841689 --- kokoro/benchmark_tests.cfg | 26 ----------------- kokoro/build.cfg | 24 ---------------- kokoro/build_tests.cfg | 1 - kokoro/common.cfg | 29 ------------------- kokoro/do_tests.cfg | 9 ------ kokoro/docker_tests.cfg | 10 ------- kokoro/go.cfg | 20 ------------- kokoro/go_tests.cfg | 1 - kokoro/hostnet_tests.cfg | 10 ------- kokoro/iptables_tests.cfg | 10 ------- kokoro/issue_reviver.cfg | 15 ---------- kokoro/kvm_tests.cfg | 10 ------- kokoro/kythe/generate_xrefs.cfg | 29 ------------------- kokoro/kythe/generate_xrefs.sh | 54 ----------------------------------- kokoro/make_tests.cfg | 9 ------ kokoro/overlay_tests.cfg | 10 ------- kokoro/packetdrill_tests.cfg | 9 ------ kokoro/packetimpact_tests.cfg | 9 ------ kokoro/release.cfg | 15 ---------- kokoro/root_tests.cfg | 10 ------- kokoro/runtime_tests.cfg | 1 - kokoro/runtime_tests/go1.12.cfg | 16 ----------- kokoro/runtime_tests/java11.cfg | 16 ----------- kokoro/runtime_tests/nodejs12.4.0.cfg | 16 ----------- kokoro/runtime_tests/php7.3.6.cfg | 16 ----------- kokoro/runtime_tests/python3.7.3.cfg | 16 ----------- kokoro/runtime_tests/runtime_tests.sh | 29 ------------------- kokoro/simple_tests.cfg | 9 ------ kokoro/swgso_tests.cfg | 9 ------ kokoro/syscall_kvm_tests.cfg | 9 ------ kokoro/syscall_tests.cfg | 9 ------ scripts/benchmarks.sh | 53 ---------------------------------- 32 files changed, 509 deletions(-) delete mode 100644 kokoro/benchmark_tests.cfg delete mode 100644 kokoro/build.cfg delete mode 100644 kokoro/build_tests.cfg delete mode 100644 kokoro/common.cfg delete mode 100644 kokoro/do_tests.cfg delete mode 100644 kokoro/docker_tests.cfg delete mode 100644 kokoro/go.cfg delete mode 100644 kokoro/go_tests.cfg delete mode 100644 kokoro/hostnet_tests.cfg delete mode 100644 kokoro/iptables_tests.cfg delete mode 100644 kokoro/issue_reviver.cfg delete mode 100644 kokoro/kvm_tests.cfg delete mode 100644 kokoro/kythe/generate_xrefs.cfg delete mode 100644 kokoro/kythe/generate_xrefs.sh delete mode 100644 kokoro/make_tests.cfg delete mode 100644 kokoro/overlay_tests.cfg delete mode 100644 kokoro/packetdrill_tests.cfg delete mode 100644 kokoro/packetimpact_tests.cfg delete mode 100644 kokoro/release.cfg delete mode 100644 kokoro/root_tests.cfg delete mode 100644 kokoro/runtime_tests.cfg delete mode 100644 kokoro/runtime_tests/go1.12.cfg delete mode 100644 kokoro/runtime_tests/java11.cfg delete mode 100644 kokoro/runtime_tests/nodejs12.4.0.cfg delete mode 100644 kokoro/runtime_tests/php7.3.6.cfg delete mode 100644 kokoro/runtime_tests/python3.7.3.cfg delete mode 100755 kokoro/runtime_tests/runtime_tests.sh delete mode 100644 kokoro/simple_tests.cfg delete mode 100644 kokoro/swgso_tests.cfg delete mode 100644 kokoro/syscall_kvm_tests.cfg delete mode 100644 kokoro/syscall_tests.cfg delete mode 100755 scripts/benchmarks.sh (limited to 'kokoro') diff --git a/kokoro/benchmark_tests.cfg b/kokoro/benchmark_tests.cfg deleted file mode 100644 index f85cc9681..000000000 --- a/kokoro/benchmark_tests.cfg +++ /dev/null @@ -1,26 +0,0 @@ -build_file : 'repo/scripts/benchmark.sh' - - -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id : 73898 - keyname : 'gvisor-benchmarks-service-account' - }, - } -} - -env_vars { - key : 'PROJECT' - value : 'gvisor-benchmarks' -} - -env_vars { - key : 'ZONE' - value : 'us-central1-b' -} - -env_vars { - key : 'GCLOUD_CREDENTIALS' - value : '73898_gvisor-benchmarks-service-account' -} diff --git a/kokoro/build.cfg b/kokoro/build.cfg deleted file mode 100644 index c9ceda947..000000000 --- a/kokoro/build.cfg +++ /dev/null @@ -1,24 +0,0 @@ -build_file: "repo/scripts/build.sh" - -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73898 - keyname: "kokoro-repo-key" - } - } -} - -env_vars { - key: "KOKORO_REPO_KEY" - value: "73898_kokoro-repo-key" -} - -action { - define_artifacts { - regex: "**/runsc" - regex: "**/runsc.*" - regex: "**/dists/**" - regex: "**/pool/**" - } -} diff --git a/kokoro/build_tests.cfg b/kokoro/build_tests.cfg deleted file mode 100644 index c64b7e679..000000000 --- a/kokoro/build_tests.cfg +++ /dev/null @@ -1 +0,0 @@ -build_file: "repo/scripts/build.sh" diff --git a/kokoro/common.cfg b/kokoro/common.cfg deleted file mode 100644 index 669a2e458..000000000 --- a/kokoro/common.cfg +++ /dev/null @@ -1,29 +0,0 @@ -# Give Kokoro access to Remote Build Executor (RBE) service account key. -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73898 - keyname: "kokoro-rbe-service-account" - } - } -} - -# Configure bazel to access RBE. -bazel_setting { - # Our GCP project name. - project_id: "gvisor-rbe" - - # Use RBE for execution as well as caching. - local_execution: false - - # This must match the values in the job config. - auth_credential: { - keystore_config_id: 73898 - keyname: "kokoro-rbe-service-account" - } - - # Do not change unless you know what you are doing. - bes_backend_address: "buildeventservice.googleapis.com" - foundry_backend_address: "remotebuildexecution.googleapis.com" - upsalite_frontend_address: "https://source.cloud.google.com" -} diff --git a/kokoro/do_tests.cfg b/kokoro/do_tests.cfg deleted file mode 100644 index b45ec0b42..000000000 --- a/kokoro/do_tests.cfg +++ /dev/null @@ -1,9 +0,0 @@ -build_file: "repo/scripts/do_tests.sh" - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - } -} diff --git a/kokoro/docker_tests.cfg b/kokoro/docker_tests.cfg deleted file mode 100644 index 0a0ef87ed..000000000 --- a/kokoro/docker_tests.cfg +++ /dev/null @@ -1,10 +0,0 @@ -build_file: "repo/scripts/docker_tests.sh" - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - regex: "**/runsc_logs_*.tar.gz" - } -} diff --git a/kokoro/go.cfg b/kokoro/go.cfg deleted file mode 100644 index b9c1fcb12..000000000 --- a/kokoro/go.cfg +++ /dev/null @@ -1,20 +0,0 @@ -build_file: "repo/scripts/go.sh" - -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73898 - keyname: "kokoro-github-access-token" - } - } -} - -env_vars { - key: "KOKORO_GITHUB_ACCESS_TOKEN" - value: "73898_kokoro-github-access-token" -} - -env_vars { - key: "KOKORO_GO_PUSH" - value: "true" -} diff --git a/kokoro/go_tests.cfg b/kokoro/go_tests.cfg deleted file mode 100644 index 5eb51041a..000000000 --- a/kokoro/go_tests.cfg +++ /dev/null @@ -1 +0,0 @@ -build_file: "repo/scripts/go.sh" diff --git a/kokoro/hostnet_tests.cfg b/kokoro/hostnet_tests.cfg deleted file mode 100644 index 520dc55a3..000000000 --- a/kokoro/hostnet_tests.cfg +++ /dev/null @@ -1,10 +0,0 @@ -build_file: "repo/scripts/hostnet_tests.sh" - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - regex: "**/runsc_logs_*.tar.gz" - } -} diff --git a/kokoro/iptables_tests.cfg b/kokoro/iptables_tests.cfg deleted file mode 100644 index a30d82591..000000000 --- a/kokoro/iptables_tests.cfg +++ /dev/null @@ -1,10 +0,0 @@ -build_file: "repo/scripts/iptables_tests.sh" - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - regex: "**/runsc_logs_*.tar.gz" - } -} diff --git a/kokoro/issue_reviver.cfg b/kokoro/issue_reviver.cfg deleted file mode 100644 index 2370d9250..000000000 --- a/kokoro/issue_reviver.cfg +++ /dev/null @@ -1,15 +0,0 @@ -build_file: "repo/scripts/issue_reviver.sh" - -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73898 - keyname: "kokoro-github-access-token" - } - } -} - -env_vars { - key: "KOKORO_GITHUB_ACCESS_TOKEN" - value: "73898_kokoro-github-access-token" -} diff --git a/kokoro/kvm_tests.cfg b/kokoro/kvm_tests.cfg deleted file mode 100644 index 1feb60c8a..000000000 --- a/kokoro/kvm_tests.cfg +++ /dev/null @@ -1,10 +0,0 @@ -build_file: "repo/scripts/kvm_tests.sh" - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - regex: "**/runsc_logs_*.tar.gz" - } -} diff --git a/kokoro/kythe/generate_xrefs.cfg b/kokoro/kythe/generate_xrefs.cfg deleted file mode 100644 index ccf657983..000000000 --- a/kokoro/kythe/generate_xrefs.cfg +++ /dev/null @@ -1,29 +0,0 @@ -build_file: "gvisor/kokoro/kythe/generate_xrefs.sh" - -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73898 - keyname: "kokoro-rbe-service-account" - } - } -} - -bazel_setting { - project_id: "gvisor-rbe" - local_execution: false - auth_credential: { - keystore_config_id: 73898 - keyname: "kokoro-rbe-service-account" - } - bes_backend_address: "buildeventservice.googleapis.com" - foundry_backend_address: "remotebuildexecution.googleapis.com" - upsalite_frontend_address: "https://source.cloud.google.com" -} - -action { - define_artifacts { - regex: "**/*.kzip" - fail_if_no_artifacts: true - } -} diff --git a/kokoro/kythe/generate_xrefs.sh b/kokoro/kythe/generate_xrefs.sh deleted file mode 100644 index 323b0f77b..000000000 --- a/kokoro/kythe/generate_xrefs.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex - -if command -v use_bazel.sh >/dev/null; then - use_bazel.sh latest -fi -bazel version - -python3 -V - -readonly KYTHE_VERSION='v0.0.41' -readonly WORKDIR="$(mktemp -d)" -readonly KYTHE_DIR="${WORKDIR}/kythe-${KYTHE_VERSION}" -if [[ -n "$KOKORO_GIT_COMMIT" ]]; then - readonly KZIP_FILENAME="${KOKORO_ARTIFACTS_DIR}/${KOKORO_GIT_COMMIT}.kzip" -else - readonly KZIP_FILENAME="$(git rev-parse HEAD).kzip" -fi - -wget -q -O "${WORKDIR}/kythe.tar.gz" \ - "https://github.com/kythe/kythe/releases/download/${KYTHE_VERSION}/kythe-${KYTHE_VERSION}.tar.gz" -tar --no-same-owner -xzf "${WORKDIR}/kythe.tar.gz" --directory "$WORKDIR" - -if [[ -n "$KOKORO_ARTIFACTS_DIR" ]]; then - cd "${KOKORO_ARTIFACTS_DIR}/github/gvisor" -fi -bazel \ - --bazelrc="${KYTHE_DIR}/extractors.bazelrc" \ - build \ - --override_repository kythe_release="${KYTHE_DIR}" \ - --define=kythe_corpus=github.com/google/gvisor \ - --cxxopt=-std=c++17 \ - --config=remote \ - --auth_credentials="${KOKORO_BAZEL_AUTH_CREDENTIAL}" \ - //... - -"${KYTHE_DIR}/tools/kzip" merge \ - --output "$KZIP_FILENAME" \ - $(find -L bazel-out/*/extra_actions/ -name '*.kzip') diff --git a/kokoro/make_tests.cfg b/kokoro/make_tests.cfg deleted file mode 100644 index d973130ff..000000000 --- a/kokoro/make_tests.cfg +++ /dev/null @@ -1,9 +0,0 @@ -build_file: "repo/scripts/make_tests.sh" - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - } -} diff --git a/kokoro/overlay_tests.cfg b/kokoro/overlay_tests.cfg deleted file mode 100644 index 6a2ddbd03..000000000 --- a/kokoro/overlay_tests.cfg +++ /dev/null @@ -1,10 +0,0 @@ -build_file: "repo/scripts/overlay_tests.sh" - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - regex: "**/runsc_logs_*.tar.gz" - } -} diff --git a/kokoro/packetdrill_tests.cfg b/kokoro/packetdrill_tests.cfg deleted file mode 100644 index 258d7deb4..000000000 --- a/kokoro/packetdrill_tests.cfg +++ /dev/null @@ -1,9 +0,0 @@ -build_file: "repo/scripts/packetdrill_tests.sh" - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - } -} diff --git a/kokoro/packetimpact_tests.cfg b/kokoro/packetimpact_tests.cfg deleted file mode 100644 index db86b52d5..000000000 --- a/kokoro/packetimpact_tests.cfg +++ /dev/null @@ -1,9 +0,0 @@ -build_file: "repo/scripts/packetimpact_tests.sh" - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - } -} diff --git a/kokoro/release.cfg b/kokoro/release.cfg deleted file mode 100644 index 5cec1790a..000000000 --- a/kokoro/release.cfg +++ /dev/null @@ -1,15 +0,0 @@ -build_file: "repo/scripts/release.sh" - -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73898 - keyname: "kokoro-github-access-token" - } - } -} - -env_vars { - key: "KOKORO_GITHUB_ACCESS_TOKEN" - value: "73898_kokoro-github-access-token" -} diff --git a/kokoro/root_tests.cfg b/kokoro/root_tests.cfg deleted file mode 100644 index 28351695c..000000000 --- a/kokoro/root_tests.cfg +++ /dev/null @@ -1,10 +0,0 @@ -build_file: "repo/scripts/root_tests.sh" - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - regex: "**/runsc_logs_*.tar.gz" - } -} diff --git a/kokoro/runtime_tests.cfg b/kokoro/runtime_tests.cfg deleted file mode 100644 index 7d56d5aca..000000000 --- a/kokoro/runtime_tests.cfg +++ /dev/null @@ -1 +0,0 @@ -build_file: "repo/scripts/runtime_tests.sh" diff --git a/kokoro/runtime_tests/go1.12.cfg b/kokoro/runtime_tests/go1.12.cfg deleted file mode 100644 index 04bfe2868..000000000 --- a/kokoro/runtime_tests/go1.12.cfg +++ /dev/null @@ -1,16 +0,0 @@ -build_file: "github/github/scripts/runtime_tests.sh" - -env_vars { - key: "RUNTIME_TEST_NAME" - value: "go1.12" -} - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - regex: "**/runsc" - regex: "**/runsc.*" - } -} \ No newline at end of file diff --git a/kokoro/runtime_tests/java11.cfg b/kokoro/runtime_tests/java11.cfg deleted file mode 100644 index c82855cd2..000000000 --- a/kokoro/runtime_tests/java11.cfg +++ /dev/null @@ -1,16 +0,0 @@ -build_file: "github/github/scripts/runtime_tests.sh" - -env_vars { - key: "RUNTIME_TEST_NAME" - value: "java11" -} - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - regex: "**/runsc" - regex: "**/runsc.*" - } -} \ No newline at end of file diff --git a/kokoro/runtime_tests/nodejs12.4.0.cfg b/kokoro/runtime_tests/nodejs12.4.0.cfg deleted file mode 100644 index 5512db5df..000000000 --- a/kokoro/runtime_tests/nodejs12.4.0.cfg +++ /dev/null @@ -1,16 +0,0 @@ -build_file: "github/github/scripts/runtime_tests.sh" - -env_vars { - key: "RUNTIME_TEST_NAME" - value: "nodejs12.4.0" -} - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - regex: "**/runsc" - regex: "**/runsc.*" - } -} \ No newline at end of file diff --git a/kokoro/runtime_tests/php7.3.6.cfg b/kokoro/runtime_tests/php7.3.6.cfg deleted file mode 100644 index bc9ac92aa..000000000 --- a/kokoro/runtime_tests/php7.3.6.cfg +++ /dev/null @@ -1,16 +0,0 @@ -build_file: "github/github/scripts/runtime_tests.sh" - -env_vars { - key: "RUNTIME_TEST_NAME" - value: "php7.3.6" -} - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - regex: "**/runsc" - regex: "**/runsc.*" - } -} \ No newline at end of file diff --git a/kokoro/runtime_tests/python3.7.3.cfg b/kokoro/runtime_tests/python3.7.3.cfg deleted file mode 100644 index 12eb13860..000000000 --- a/kokoro/runtime_tests/python3.7.3.cfg +++ /dev/null @@ -1,16 +0,0 @@ -build_file: "github/github/scripts/runtime_tests.sh" - -env_vars { - key: "RUNTIME_TEST_NAME" - value: "python3.7.3" -} - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - regex: "**/runsc" - regex: "**/runsc.*" - } -} \ No newline at end of file diff --git a/kokoro/runtime_tests/runtime_tests.sh b/kokoro/runtime_tests/runtime_tests.sh deleted file mode 100755 index 73a58f806..000000000 --- a/kokoro/runtime_tests/runtime_tests.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Run in the root of the repo. -cd "$(dirname "$0")" -cd "$(git rev-parse --show-toplevel)" - -source scripts/common.sh - -if [ ! -v RUNTIME_TEST_NAME ]; then - echo 'Must set $RUNTIME_TEST_NAME' >&2 - exit 1 -fi - -install_runsc_for_test runtimes -test_runsc "//test/runtimes:${RUNTIME_TEST_NAME}_test" diff --git a/kokoro/simple_tests.cfg b/kokoro/simple_tests.cfg deleted file mode 100644 index 32e0a9431..000000000 --- a/kokoro/simple_tests.cfg +++ /dev/null @@ -1,9 +0,0 @@ -build_file: "repo/scripts/simple_tests.sh" - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - } -} diff --git a/kokoro/swgso_tests.cfg b/kokoro/swgso_tests.cfg deleted file mode 100644 index 101a9c607..000000000 --- a/kokoro/swgso_tests.cfg +++ /dev/null @@ -1,9 +0,0 @@ -build_file: "repo/scripts/swgso_tests.sh" - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - } -} diff --git a/kokoro/syscall_kvm_tests.cfg b/kokoro/syscall_kvm_tests.cfg deleted file mode 100644 index 3b99e9c13..000000000 --- a/kokoro/syscall_kvm_tests.cfg +++ /dev/null @@ -1,9 +0,0 @@ -build_file: "repo/scripts/syscall_kvm_tests.sh" - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - } -} diff --git a/kokoro/syscall_tests.cfg b/kokoro/syscall_tests.cfg deleted file mode 100644 index ee6e4a3a4..000000000 --- a/kokoro/syscall_tests.cfg +++ /dev/null @@ -1,9 +0,0 @@ -build_file: "repo/scripts/syscall_tests.sh" - -action { - define_artifacts { - regex: "**/sponge_log.xml" - regex: "**/sponge_log.log" - regex: "**/outputs.zip" - } -} diff --git a/scripts/benchmarks.sh b/scripts/benchmarks.sh deleted file mode 100755 index 6b9065b07..000000000 --- a/scripts/benchmarks.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#!/usr/bin/env bash - -if [ "$#" -lt "1" ]; then - echo "usage: $0 <--mock |--env=> ..." - echo "example: $0 --mock --runs=8" - exit 1 -fi - -source $(dirname $0)/common.sh - -readonly TIMESTAMP=`date "+%Y%m%d-%H%M%S"` -readonly OUTDIR="$(mktemp --tmpdir -d run-${TIMESTAMP}-XXX)" -readonly DEFAULT_RUNTIMES="--runtime=runc --runtime=runsc --runtime=runsc-kvm" -readonly ALL_RUNTIMES="--runtime=runc --runtime=runsc --runtime=runsc-kvm" - -run //benchmarks:perf -- run "$@" ${DEFAULT_RUNTIMES} 'fio.(read|write)' --metric=bandwidth --size=5g --ioengine=sync --blocksize=1m > "${OUTDIR}/fio.csv" -run //benchmarks:perf -- run "$@" ${DEFAULT_RUNTIMES} fio.rand --metric=bandwidth --size=5g --ioengine=sync --blocksize=4k --time=30 > "${OUTDIR}/tmp_fio.csv" -cat "${OUTDIR}/tmp_fio.csv" | grep "\(runc\|runsc\)" >> "${OUTDIR}/fio.csv" && rm "${OUTDIR}/tmp_fio.csv" - -run //benchmarks:perf -- run "$@" ${DEFAULT_RUNTIMES} 'fio.(read|write)' --metric=bandwidth --tmpfs=True --size=5g --ioengine=sync --blocksize=1m > "${OUTDIR}/fio-tmpfs.csv" -run //benchmarks:perf -- run "$@" ${DEFAULT_RUNTIMES} fio.rand --metric=bandwidth --tmpfs=True --size=5g --ioengine=sync --blocksize=4k --time=30 > "${OUTDIR}/tmp_fio.csv" -cat "${OUTDIR}/tmp_fio.csv" | grep "\(runc\|runsc\)" >> "${OUTDIR}/fio-tmpfs.csv" && rm "${OUTDIR}/tmp_fio.csv" - -run //benchmarks:perf -- run "$@" ${DEFAULT_RUNTIMES} startup --count=50 > "${OUTDIR}/startup.csv" -run //benchmarks:perf -- run "$@" ${DEFAULT_RUNTIMES} density > "${OUTDIR}/density.csv" - -run //benchmarks:perf -- run "$@" ${DEFAULT_RUNTIMES} sysbench.cpu --threads=1 --max_prime=50000 --options='--max-time=5' > "${OUTDIR}/sysbench-cpu.csv" -run //benchmarks:perf -- run "$@" ${DEFAULT_RUNTIMES} sysbench.memory --threads=1 --options='--memory-block-size=1M --memory-total-size=500G' > "${OUTDIR}/sysbench-memory.csv" -run //benchmarks:perf -- run "$@" ${ALL_RUNTIMES} syscall > "${OUTDIR}/syscall.csv" -run //benchmarks:perf -- run "$@" ${DEFAULT_RUNTIMES} 'network.(upload|download)' --runs=20 > "${OUTDIR}/iperf.csv" -run //benchmarks:perf -- run "$@" ${DEFAULT_RUNTIMES} ml.tensorflow > "${OUTDIR}/tensorflow.csv" -run //benchmarks:perf -- run "$@" ${DEFAULT_RUNTIMES} media.ffmpeg > "${OUTDIR}/ffmpeg.csv" -run //benchmarks:perf -- run "$@" ${DEFAULT_RUNTIMES} http.httpd --path=latin100k.txt --connections=1 --connections=5 --connections=10 --connections=25 > "${OUTDIR}/httpd100k.csv" -run //benchmarks:perf -- run "$@" ${DEFAULT_RUNTIMES} http.httpd --path=latin10240k.txt --connections=1 --connections=5 --connections=10 --connections=25 > "${OUTDIR}/httpd10240k.csv" -run //benchmarks:perf -- run "$@" ${DEFAULT_RUNTIMES} redis > "${OUTDIR}/redis.csv" -run //benchmarks:perf -- run "$@" ${DEFAULT_RUNTIMES} 'http.(ruby|node)' > "${OUTDIR}/applications.csv" - -echo "${OUTPUT}" && exit 0 -- cgit v1.2.3