From 67a2ab1438cdccbe045143bbfaa807cf83110ebc Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Tue, 3 Sep 2019 22:01:34 -0700 Subject: Impose order on test scripts. The simple test script has gotten out of control. Shard this script into different pieces and attempt to impose order on overall test structure. This change helps lay some of the foundations for future improvements. * The runsc/test directories are moved into just test/. * The runsc/test/testutil package is split into logical pieces. * The scripts/ directory contains new top-level targets. * Each test is now responsible for building targets it requires. * The install functionality is moved into `runsc` itself for simplicity. * The existing kokoro run_tests.sh file now just calls all (can be split). After this change is merged, I will create multiple distinct workflows for Kokoro, one for each of the scripts currently targeted by `run_tests.sh` today, which should dramatically reduce the time-to-run for the Kokoro tests, and provides a better foundation for further improvements to the infrastructure. PiperOrigin-RevId: 267081397 --- scripts/build.sh | 62 ++++++++++++++++++++++++++++++++++++++ scripts/common.sh | 23 +++++++++++++++ scripts/common_bazel.sh | 77 ++++++++++++++++++++++++++++++++++++++++++++++++ scripts/do_tests.sh | 27 +++++++++++++++++ scripts/docker_tests.sh | 22 ++++++++++++++ scripts/go.sh | 34 +++++++++++++++++++++ scripts/hostnet_tests.sh | 22 ++++++++++++++ scripts/kvm_tests.sh | 30 +++++++++++++++++++ scripts/make_tests.sh | 24 +++++++++++++++ scripts/overlay_tests.sh | 22 ++++++++++++++ scripts/release.sh | 34 +++++++++++++++++++++ scripts/root_tests.sh | 31 +++++++++++++++++++ scripts/simple_tests.sh | 20 +++++++++++++ scripts/syscall_tests.sh | 20 +++++++++++++ 14 files changed, 448 insertions(+) create mode 100755 scripts/build.sh create mode 100755 scripts/common.sh create mode 100755 scripts/common_bazel.sh create mode 100755 scripts/do_tests.sh create mode 100755 scripts/docker_tests.sh create mode 100755 scripts/go.sh create mode 100755 scripts/hostnet_tests.sh create mode 100755 scripts/kvm_tests.sh create mode 100755 scripts/make_tests.sh create mode 100755 scripts/overlay_tests.sh create mode 100755 scripts/release.sh create mode 100755 scripts/root_tests.sh create mode 100755 scripts/simple_tests.sh create mode 100755 scripts/syscall_tests.sh (limited to 'scripts') diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 000000000..dae3460af --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Copyright 2018 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +# Build runsc. +runsc=$(build -c opt //runsc) + +# Build packages. +pkg=$(build -c opt --host_force_python=py2 //runsc:debian) + +# Build a repository, if the key is available. +if [[ -v KOKORO_REPO_KEY ]]; then + repo=$(tools/make_repository.sh "${KOKORO_REPO_KEY}" gvisor-bot@google.com) +fi + +# Install installs artifacts. +install() { + mkdir -p $1 + cp "${runsc}" "$1"/runsc + sha512sum "$1"/runsc | awk '{print $1 " runsc"}' > "$1"/runsc.sha512 + if [[ -v repo ]]; then + cp -a "${repo}" "${latest_dir}"/repo + fi +} + +# Move the runsc binary into "latest" directory, and also a directory with the +# current date. If the current commit happens to correpond to a tag, then we +# will also move everything into a directory named after the given tag. +if [[ -v KOKORO_ARTIFACTS_DIR ]]; then + if [[ "${KOKORO_BUILD_NIGHTLY}" == "true" ]]; then + # The "latest" directory and current date. + install "${KOKORO_ARTIFACTS_DIR}/nightly/latest" + install "${KOKORO_ARTIFACTS_DIR}/nightly/$(date -Idate)" + else + # Is it a tagged release? Build that instead. In that case, we also try to + # update the base release directory, in case this is an update. Finally, we + # update the "release" directory, which has the last released version. + tag="$(git describe --exact-match --tags HEAD)" + if ! [[ -z "${tag}" ]]; then + install "${KOKORO_ARTIFACTS_DIR}/${tag}" + base=$(echo "${tag}" | cut -d'.' -f1) + if [[ "${base}" != "${tag}" ]]; then + install "${KOKORO_ARTIFACTS_DIR}/${base}" + fi + install "${KOKORO_ARTIFACTS_DIR}/release" + fi + fi +fi diff --git a/scripts/common.sh b/scripts/common.sh new file mode 100755 index 000000000..f2b9e24d8 --- /dev/null +++ b/scripts/common.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xeo pipefail + +if [[ -f $(dirname $0)/common_google.sh ]]; then + source $(dirname $0)/common_google.sh +else + source $(dirname $0)/common_bazel.sh +fi diff --git a/scripts/common_bazel.sh b/scripts/common_bazel.sh new file mode 100755 index 000000000..42248cb25 --- /dev/null +++ b/scripts/common_bazel.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install the latest version of Bazel and log the version. +(which use_bazel.sh && use_bazel.sh latest) || which bazel +bazel version + +# Switch into the workspace; only necessary if run with kokoro. +if [[ -v KOKORO_GIT_COMMIT ]] && [[ -d git/repo ]]; then + cd git/repo +elif [[ -v KOKORO_GIT_COMMIT ]] && [[ -d github/repo ]]; then + cd github/repo +fi + +# Set the standard bazel flags. +declare -r BAZEL_FLAGS=( + "--show_timestamps" + "--test_output=errors" + "--keep_going" + "--verbose_failures=true" +) +if [[ -v KOKORO_BAZEL_AUTH_CREDENTIAL ]] || [[ -v RBE_PROJECT_ID ]]; then + declare -r RBE_PROJECT_ID="${RBE_PROJECT_ID:-gvisor-rbe}" + declare -r BAZEL_RBE_FLAGS=( + "--config=remote" + "--project_id=${RBE_PROJECT_ID}" + "--remote_instance_name=projects/${RBE_PROJECT_ID}/instances/default_instance" + ) +fi +if [[ -v KOKORO_BAZEL_AUTH_CREDENTIAL ]]; then + declare -r BAZEL_RBE_AUTH_FLAGS=( + "--auth_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL}" + ) +fi + +# Wrap bazel. +function build() { + bazel build "${BAZEL_RBE_FLAGS[@]}" "${BAZEL_RBE_AUTH_FLAGS[@]}" "${BAZEL_FLAGS[@]}" "$@" +} + +function test() { + (bazel test "${BAZEL_RBE_FLAGS[@]}" "${BAZEL_RBE_AUTH_FLAGS[@]}" "${BAZEL_FLAGS[@]}" "$@" && rc=0) || rc=$? + + # Zip out everything into a convenient form. + if [[ -v KOKORO_ARTIFACTS_DIR ]]; then + find -L "bazel-testlogs" -name "test.xml" -o -name "test.log" -o -name "outputs.zip" | + tar --create --files-from - --transform 's/test\./sponge_log./' | + tar --extract --directory ${KOKORO_ARTIFACTS_DIR} + fi + + return $rc +} + +function run() { + local binary=$1 + shift + bazel run "${binary}" -- "$@" +} + +function run_as_root() { + local binary=$1 + shift + bazel run --run_under="sudo" "${binary}" -- "$@" +} diff --git a/scripts/do_tests.sh b/scripts/do_tests.sh new file mode 100755 index 000000000..a3a387c37 --- /dev/null +++ b/scripts/do_tests.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +# Build runsc. +build //runsc + +# run runsc do without root privileges. +run //runsc --rootless do true +run //runsc --rootless --network=none do true + +# run runsc do with root privileges. +run_as_root //runsc do true diff --git a/scripts/docker_tests.sh b/scripts/docker_tests.sh new file mode 100755 index 000000000..d6b18a35b --- /dev/null +++ b/scripts/docker_tests.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +# Install the runtime and perform basic tests. +run_as_root //runsc install --experimental=true -- --debug --strace --log-packets +sudo systemctl restart docker +test //test/image:image_test //test/e2e:integration_test diff --git a/scripts/go.sh b/scripts/go.sh new file mode 100755 index 000000000..e49d76c6d --- /dev/null +++ b/scripts/go.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +# Build the go path. +build :gopath + +# Build the synthetic branch. +tools/go_branch.sh + +# Checkout the new branch. +git checkout go && git clean -f + +# Build everything. +go build ./... + +# Push, if required. +if [[ "${KOKORO_GO_PUSH}" == "true" ]]; then + git push origin go:go +fi diff --git a/scripts/hostnet_tests.sh b/scripts/hostnet_tests.sh new file mode 100755 index 000000000..0631c5510 --- /dev/null +++ b/scripts/hostnet_tests.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +# Install the runtime and perform basic tests. +run_as_root //runsc install --experimental=true -- --debug --strace --log-packets --network=host +sudo systemctl restart docker +test --test_arg=-checkpoint=false //test/image:image_test //test/e2e:integration_test diff --git a/scripts/kvm_tests.sh b/scripts/kvm_tests.sh new file mode 100755 index 000000000..5cb7aa007 --- /dev/null +++ b/scripts/kvm_tests.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +# Ensure that KVM is loaded, and we can use it. +(lsmod | grep -E '^(kvm_intel|kvm_amd)') || sudo modprobe kvm +sudo chmod a+rw /dev/kvm + +# Run all KVM-tagged tests (locally). +test --test_strategy=standalone --test_tag_filters=requires-kvm //... +test --test_strategy=standalone //pkg/sentry/platform/kvm:kvm_test + +# Install the KVM runtime and run all integration tests. +run_as_root //runsc install --experimental=true -- --debug --strace --log-packets --platform=kvm +sudo systemctl restart docker +test --test_strategy=standalone //test/image:image_test //test/e2e:integration_test diff --git a/scripts/make_tests.sh b/scripts/make_tests.sh new file mode 100755 index 000000000..0fa1248be --- /dev/null +++ b/scripts/make_tests.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +top_level=$(git rev-parse --show-toplevel 2>/dev/null) +[[ $? -eq 0 ]] && cd "${top_level}" || exit 1 + +make +make runsc +make bazel-shutdown diff --git a/scripts/overlay_tests.sh b/scripts/overlay_tests.sh new file mode 100755 index 000000000..651a51f70 --- /dev/null +++ b/scripts/overlay_tests.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +# Install the runtime and perform basic tests. +run_as_root //runsc install --experimental=true -- --debug --strace --log-packets --overlay +sudo systemctl restart docker +test //test/image:image_test //test/e2e:integration_test diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 000000000..422319500 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2018 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +# Tag a release only if provided. +if ! [[ -v KOKORO_RELEASE_COMMIT ]]; then + echo "No KOKORO_RELEASE_COMMIT provided." >&2 + exit 1 +fi +if ! [[ -v KOKORO_RELEASE_TAG ]]; then + echo "No KOKORO_RELEASE_TAG provided." >&2 + exit 1 +fi + +# Ensure we have an appropriate configuration for the tag. +git config --get user.name || git config user.name "gVisor-bot" +git config --get user.email || git config user.email "gvisor-bot@google.com" + +# Run the release tool, which pushes to the origin repository. +tools/tag_release.sh "${KOKORO_RELEASE_COMMIT}" "${KOKORO_RELEASE_TAG}" diff --git a/scripts/root_tests.sh b/scripts/root_tests.sh new file mode 100755 index 000000000..e42c0e3ec --- /dev/null +++ b/scripts/root_tests.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +# Reinstall the latest containerd shim. +declare -r base="https://storage.googleapis.com/cri-containerd-staging/gvisor-containerd-shim" +declare -r latest=$(mktemp --tmpdir gvisor-containerd-shim-latest.XXXXXX) +declare -r shim_path=$(mktemp --tmpdir gvisor-containerd-shim.XXXXXX) +wget --no-verbose "${base}"/latest -O ${latest} +wget --no-verbose "${base}"/gvisor-containerd-shim-$(cat ${latest}) -O ${shim_path} +chmod +x ${shim_path} +sudo mv ${shim_path} /usr/local/bin/gvisor-containerd-shim + +# Run the tests that require root. +run_as_root //runsc install --experimental=true -- --debug --strace --log-packets +sudo systemctl restart docker +run_as_root //test/root:root_test diff --git a/scripts/simple_tests.sh b/scripts/simple_tests.sh new file mode 100755 index 000000000..585216aae --- /dev/null +++ b/scripts/simple_tests.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +# Run all simple tests (locally). +test //pkg/... //runsc/... //tools/... diff --git a/scripts/syscall_tests.sh b/scripts/syscall_tests.sh new file mode 100755 index 000000000..a131b2d50 --- /dev/null +++ b/scripts/syscall_tests.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +# Run all ptrace-variants of the system call tests. +test --test_tag_filters=runsc_ptrace //test/syscalls/... -- cgit v1.2.3 From bcddd0a4778916f6a5347246a81fbe236050d2c4 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Wed, 4 Sep 2019 18:47:47 -0700 Subject: Fix continuous build breakage PiperOrigin-RevId: 267277711 --- scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/build.sh b/scripts/build.sh index dae3460af..293d87093 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -20,7 +20,7 @@ source $(dirname $0)/common.sh runsc=$(build -c opt //runsc) # Build packages. -pkg=$(build -c opt --host_force_python=py2 //runsc:debian) +pkg=$(build -c opt --host_force_python=py2 //runsc:runsc-debian) # Build a repository, if the key is available. if [[ -v KOKORO_REPO_KEY ]]; then -- cgit v1.2.3 From 91518fd553b828ce9f3fa84d91f20a45a8f0c81d Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Wed, 4 Sep 2019 22:24:42 -0700 Subject: Fix build when no tags are present This should correct the continuous build. --- scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/build.sh b/scripts/build.sh index 293d87093..ee2de3b94 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -49,7 +49,7 @@ if [[ -v KOKORO_ARTIFACTS_DIR ]]; then # Is it a tagged release? Build that instead. In that case, we also try to # update the base release directory, in case this is an update. Finally, we # update the "release" directory, which has the last released version. - tag="$(git describe --exact-match --tags HEAD)" + tag="$(git describe --exact-match --tags HEAD || true)" if ! [[ -z "${tag}" ]]; then install "${KOKORO_ARTIFACTS_DIR}/${tag}" base=$(echo "${tag}" | cut -d'.' -f1) -- cgit v1.2.3 From 1a0a940587e4db8923ca81b78d7bba395eb56ce1 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Thu, 5 Sep 2019 16:37:06 -0700 Subject: Fix repository build scripts This has the following fixes: * Packages are passed to the tools/make_repository.sh command. * All matching tags are built, for commits with multiple. * The binary path is generated by the build command. * Output from signing the repository is supressed. * Allow a release author. Change-Id: I2d08954ba76e35612f352be99d5bb99080f80892 --- scripts/build.sh | 32 +++++++++++++++++++------------- scripts/common_bazel.sh | 3 ++- scripts/release.sh | 6 +++++- tools/make_repository.sh | 10 +++++----- 4 files changed, 31 insertions(+), 20 deletions(-) (limited to 'scripts') diff --git a/scripts/build.sh b/scripts/build.sh index ee2de3b94..d5dd14acc 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -24,16 +24,17 @@ pkg=$(build -c opt --host_force_python=py2 //runsc:runsc-debian) # Build a repository, if the key is available. if [[ -v KOKORO_REPO_KEY ]]; then - repo=$(tools/make_repository.sh "${KOKORO_REPO_KEY}" gvisor-bot@google.com) + repo=$(tools/make_repository.sh "${KOKORO_REPO_KEY}" gvisor-bot@google.com ${pkg}) fi # Install installs artifacts. install() { - mkdir -p $1 - cp "${runsc}" "$1"/runsc - sha512sum "$1"/runsc | awk '{print $1 " runsc"}' > "$1"/runsc.sha512 + local dir="$1" + mkdir -p "${dir}" + cp -f "${runsc}" "${dir}"/runsc + sha512sum "${dir}"/runsc | awk '{print $1 " runsc"}' > "${dir}"/runsc.sha512 if [[ -v repo ]]; then - cp -a "${repo}" "${latest_dir}"/repo + rm -rf "${dir}"/repo && cp -a "${repo}" "$dir"/repo fi } @@ -49,14 +50,19 @@ if [[ -v KOKORO_ARTIFACTS_DIR ]]; then # Is it a tagged release? Build that instead. In that case, we also try to # update the base release directory, in case this is an update. Finally, we # update the "release" directory, which has the last released version. - tag="$(git describe --exact-match --tags HEAD || true)" - if ! [[ -z "${tag}" ]]; then - install "${KOKORO_ARTIFACTS_DIR}/${tag}" - base=$(echo "${tag}" | cut -d'.' -f1) - if [[ "${base}" != "${tag}" ]]; then - install "${KOKORO_ARTIFACTS_DIR}/${base}" - fi - install "${KOKORO_ARTIFACTS_DIR}/release" + tags="$(git tag --points-at HEAD)" + if ! [[ -z "${tags}" ]]; then + # Note that a given commit can match any number of tags. We have to + # iterate through all possible tags and produce associated artifacts. + for tag in ${tags}; do + name=$(echo "${tag}" | cut -d'-' -f2) + base=$(echo "${name}" | cut -d'.' -f1) + install "${KOKORO_ARTIFACTS_DIR}/release/${name}" + if [[ "${base}" != "${tag}" ]]; then + install "${KOKORO_ARTIFACTS_DIR}/release/${base}" + fi + install "${KOKORO_ARTIFACTS_DIR}/release/latest" + done fi fi fi diff --git a/scripts/common_bazel.sh b/scripts/common_bazel.sh index 42248cb25..5340c7323 100755 --- a/scripts/common_bazel.sh +++ b/scripts/common_bazel.sh @@ -48,7 +48,8 @@ fi # Wrap bazel. function build() { - bazel build "${BAZEL_RBE_FLAGS[@]}" "${BAZEL_RBE_AUTH_FLAGS[@]}" "${BAZEL_FLAGS[@]}" "$@" + bazel build "${BAZEL_RBE_FLAGS[@]}" "${BAZEL_RBE_AUTH_FLAGS[@]}" "${BAZEL_FLAGS[@]}" "$@" 2>&1 | + tee /dev/fd/2 | grep -E '^ bazel-bin/' | awk '{ print $1; }' } function test() { diff --git a/scripts/release.sh b/scripts/release.sh index 422319500..b936bcc77 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -26,9 +26,13 @@ if ! [[ -v KOKORO_RELEASE_TAG ]]; then exit 1 fi +# Unless an explicit releaser is provided, use the bot e-mail. +declare -r KOKORO_RELEASE_AUTHOR=${KOKORO_RELEASE_AUTHOR:-gvisor-bot} +declare -r EMAIL=${EMAIL:-${KOKORO_RELEASE_AUTHOR}@google.com} + # Ensure we have an appropriate configuration for the tag. git config --get user.name || git config user.name "gVisor-bot" -git config --get user.email || git config user.email "gvisor-bot@google.com" +git config --get user.email || git config user.email "${EMAIL}" # Run the release tool, which pushes to the origin repository. tools/tag_release.sh "${KOKORO_RELEASE_COMMIT}" "${KOKORO_RELEASE_TAG}" diff --git a/tools/make_repository.sh b/tools/make_repository.sh index bf9c50d74..ccebe27b3 100755 --- a/tools/make_repository.sh +++ b/tools/make_repository.sh @@ -37,10 +37,10 @@ cleanup() { rm -f "${keyring}" } trap cleanup EXIT -gpg --no-default-keyring --keyring "${keyring}" --import "${private_key}" +gpg --no-default-keyring --keyring "${keyring}" --import "${private_key}" >&2 # Export the public key from the keyring. -gpg --no-default-keyring --keyring "${keyring}" --armor --export "${signer}" > "${tmpdir}"/keyFile +gpg --no-default-keyring --keyring "${keyring}" --armor --export "${signer}" > "${tmpdir}"/keyFile >&2 # Copy the packages, and ensure permissions are correct. cp -a "$@" "${tmpdir}" && chmod 0644 "${tmpdir}"/* @@ -52,7 +52,7 @@ find "${tmpdir}" -type l -exec rm -f {} \; # Sign all packages. for file in "${tmpdir}"/*.deb; do - dpkg-sig -g "--no-default-keyring --keyring ${keyring}" --sign builder "${file}" + dpkg-sig -g "--no-default-keyring --keyring ${keyring}" --sign builder "${file}" >&2 done # Build the package list. @@ -62,8 +62,8 @@ done (cd "${tmpdir}" && apt-ftparchive release . > Release) # Sign the release. -(cd "${tmpdir}" && gpg --no-default-keyring --keyring "${keyring}" --clearsign -o InRelease Release) -(cd "${tmpdir}" && gpg --no-default-keyring --keyring "${keyring}" -abs -o Release.gpg Release) +(cd "${tmpdir}" && gpg --no-default-keyring --keyring "${keyring}" --clearsign -o InRelease Release >&2) +(cd "${tmpdir}" && gpg --no-default-keyring --keyring "${keyring}" -abs -o Release.gpg Release >&2) # Show the results. echo "${tmpdir}" -- cgit v1.2.3 From 849c57314f6b6f0d1ebcfa2e68762b8ea95f5948 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Tue, 10 Sep 2019 00:37:46 -0700 Subject: Fix minor Kokoro issues. A recent Kokoro change pointed to go_tests.cfg (in line with the other configurations), which unfortunately broke the presubmits. This change also enabled the KVM tests, which were still using a remote execution strategy. This fixes both of these issues and allows presubmits to pass. One additional test was caught with this case, which seems to have been broken. It's unclear why this was not being caught. PiperOrigin-RevId: 268166291 --- kokoro/go_test.cfg | 1 - kokoro/go_tests.cfg | 1 + scripts/common_bazel.sh | 2 +- scripts/kvm_tests.sh | 7 +++---- test/root/cgroup_test.go | 18 ++++++------------ test/syscalls/linux/proc_net.cc | 2 +- tools/go_branch.sh | 6 +++++- 7 files changed, 17 insertions(+), 20 deletions(-) delete mode 100644 kokoro/go_test.cfg create mode 100644 kokoro/go_tests.cfg (limited to 'scripts') diff --git a/kokoro/go_test.cfg b/kokoro/go_test.cfg deleted file mode 100644 index 5eb51041a..000000000 --- a/kokoro/go_test.cfg +++ /dev/null @@ -1 +0,0 @@ -build_file: "repo/scripts/go.sh" diff --git a/kokoro/go_tests.cfg b/kokoro/go_tests.cfg new file mode 100644 index 000000000..5eb51041a --- /dev/null +++ b/kokoro/go_tests.cfg @@ -0,0 +1 @@ +build_file: "repo/scripts/go.sh" diff --git a/scripts/common_bazel.sh b/scripts/common_bazel.sh index 5340c7323..dc0e2041d 100755 --- a/scripts/common_bazel.sh +++ b/scripts/common_bazel.sh @@ -56,7 +56,7 @@ function test() { (bazel test "${BAZEL_RBE_FLAGS[@]}" "${BAZEL_RBE_AUTH_FLAGS[@]}" "${BAZEL_FLAGS[@]}" "$@" && rc=0) || rc=$? # Zip out everything into a convenient form. - if [[ -v KOKORO_ARTIFACTS_DIR ]]; then + if [[ -v KOKORO_ARTIFACTS_DIR ]] && [[ -e bazel-testlogs ]]; then find -L "bazel-testlogs" -name "test.xml" -o -name "test.log" -o -name "outputs.zip" | tar --create --files-from - --transform 's/test\./sponge_log./' | tar --extract --directory ${KOKORO_ARTIFACTS_DIR} diff --git a/scripts/kvm_tests.sh b/scripts/kvm_tests.sh index 5cb7aa007..b6d787f0f 100755 --- a/scripts/kvm_tests.sh +++ b/scripts/kvm_tests.sh @@ -20,11 +20,10 @@ source $(dirname $0)/common.sh (lsmod | grep -E '^(kvm_intel|kvm_amd)') || sudo modprobe kvm sudo chmod a+rw /dev/kvm -# Run all KVM-tagged tests (locally). -test --test_strategy=standalone --test_tag_filters=requires-kvm //... -test --test_strategy=standalone //pkg/sentry/platform/kvm:kvm_test +# Run all KVM platform tests (locally). +run_as_root //pkg/sentry/platform/kvm:kvm_test # Install the KVM runtime and run all integration tests. run_as_root //runsc install --experimental=true -- --debug --strace --log-packets --platform=kvm sudo systemctl restart docker -test --test_strategy=standalone //test/image:image_test //test/e2e:integration_test +test //test/image:image_test //test/e2e:integration_test diff --git a/test/root/cgroup_test.go b/test/root/cgroup_test.go index cc7e8583e..76f1e4f2a 100644 --- a/test/root/cgroup_test.go +++ b/test/root/cgroup_test.go @@ -62,6 +62,12 @@ func TestCgroup(t *testing.T) { } d := dockerutil.MakeDocker("cgroup-test") + // This is not a comprehensive list of attributes. + // + // Note that we are specifically missing cpusets, which fail if specified. + // In any case, it's unclear if cpusets can be reliably tested here: these + // are often run on a single core virtual machine, and there is only a single + // CPU available in our current set, and every container's set. attrs := []struct { arg string ctrl string @@ -87,18 +93,6 @@ func TestCgroup(t *testing.T) { file: "cpu.cfs_quota_us", want: "3000", }, - { - arg: "--cpuset-cpus=0", - ctrl: "cpuset", - file: "cpuset.cpus", - want: "0", - }, - { - arg: "--cpuset-mems=0", - ctrl: "cpuset", - file: "cpuset.mems", - want: "0", - }, { arg: "--kernel-memory=100MB", ctrl: "memory", diff --git a/test/syscalls/linux/proc_net.cc b/test/syscalls/linux/proc_net.cc index c097af196..efdaf202b 100644 --- a/test/syscalls/linux/proc_net.cc +++ b/test/syscalls/linux/proc_net.cc @@ -28,7 +28,7 @@ TEST(ProcNetIfInet6, Format) { EXPECT_THAT(ifinet6, ::testing::MatchesRegex( // Ex: "00000000000000000000000000000001 01 80 10 80 lo\n" - "^([a-f\\d]{32}( [a-f\\d]{2}){4} +[a-z][a-z\\d]*\\n)+$")); + "^([a-f0-9]{32}( [a-f0-9]{2}){4} +[a-z][a-z0-9]*\n)+$")); } TEST(ProcSysNetIpv4Sack, Exists) { diff --git a/tools/go_branch.sh b/tools/go_branch.sh index d9e79401d..ddb9b6e7b 100755 --- a/tools/go_branch.sh +++ b/tools/go_branch.sh @@ -59,7 +59,11 @@ git checkout -b go "${go_branch}" # Start working on a merge commit that combines the previous history with the # current history. Note that we don't actually want any changes yet. -git merge --allow-unrelated-histories --no-commit --strategy ours ${head} +# +# N.B. The git behavior changed at some point and the relevant flag was added +# to allow for override, so try the only behavior first then pass the flag. +git merge --no-commit --strategy ours ${head} || \ + git merge --allow-unrelated-histories --no-commit --strategy ours ${head} # Sync the entire gopath_dir and go.mod. rsync --recursive --verbose --delete --exclude .git --exclude README.md -L "${gopath_dir}/" . -- cgit v1.2.3 From c06ef5131f1ccd3106ccf4fa4e787db079db2d96 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Wed, 11 Sep 2019 18:48:26 -0700 Subject: Fix authorization for continuous integration. The credentials must be explicitly refreshed for pushing to the repository on the Go branch. PiperOrigin-RevId: 268589817 --- kokoro/go.cfg | 14 ++++++++++++++ scripts/go.sh | 9 +++++++++ 2 files changed, 23 insertions(+) (limited to 'scripts') diff --git a/kokoro/go.cfg b/kokoro/go.cfg index d1577252a..759e16152 100644 --- a/kokoro/go.cfg +++ b/kokoro/go.cfg @@ -1,5 +1,19 @@ build_file: "repo/scripts/go.sh" +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73898 + keyname: "kokoro-github-access-token" + } + } +} + +env_vars { + key: "KOKORO_GITHUB_ACCESS_TOKEN" + value: "$KOKORO_ROOT/src/keystore/73898_kokoro-github-access-token" +} + env_vars { key: "KOKORO_GO_PUSH" value: "true" diff --git a/scripts/go.sh b/scripts/go.sh index e49d76c6d..83a667640 100755 --- a/scripts/go.sh +++ b/scripts/go.sh @@ -30,5 +30,14 @@ go build ./... # Push, if required. if [[ "${KOKORO_GO_PUSH}" == "true" ]]; then + if [[ -v KOKORO_GITHUB_ACCESS_TOKEN ]]; then + git config --global credential.helper cache + git credential approve < Date: Wed, 11 Sep 2019 21:03:12 -0700 Subject: Update key environment variables. PiperOrigin-RevId: 268604220 --- kokoro/build.cfg | 2 +- kokoro/go.cfg | 2 +- scripts/build.sh | 2 +- scripts/go.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/kokoro/build.cfg b/kokoro/build.cfg index d67af4694..d462d839c 100644 --- a/kokoro/build.cfg +++ b/kokoro/build.cfg @@ -11,7 +11,7 @@ before_action { env_vars { key: "KOKORO_REPO_KEY" - value: "$KOKORO_ROOT/src/keystore/73898_kokoro-repo-key" + value: "73898_kokoro-repo-key" } action { diff --git a/kokoro/go.cfg b/kokoro/go.cfg index 759e16152..b9c1fcb12 100644 --- a/kokoro/go.cfg +++ b/kokoro/go.cfg @@ -11,7 +11,7 @@ before_action { env_vars { key: "KOKORO_GITHUB_ACCESS_TOKEN" - value: "$KOKORO_ROOT/src/keystore/73898_kokoro-github-access-token" + value: "73898_kokoro-github-access-token" } env_vars { diff --git a/scripts/build.sh b/scripts/build.sh index d5dd14acc..81023aab3 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -24,7 +24,7 @@ pkg=$(build -c opt --host_force_python=py2 //runsc:runsc-debian) # Build a repository, if the key is available. if [[ -v KOKORO_REPO_KEY ]]; then - repo=$(tools/make_repository.sh "${KOKORO_REPO_KEY}" gvisor-bot@google.com ${pkg}) + repo=$(tools/make_repository.sh "${KOKORO_KEYSTORE_DIR}/${KOKORO_REPO_KEY}" gvisor-bot@google.com ${pkg}) fi # Install installs artifacts. diff --git a/scripts/go.sh b/scripts/go.sh index 83a667640..f24fad04c 100755 --- a/scripts/go.sh +++ b/scripts/go.sh @@ -35,7 +35,7 @@ if [[ "${KOKORO_GO_PUSH}" == "true" ]]; then git credential approve < Date: Wed, 11 Sep 2019 21:49:18 -0700 Subject: Ensure appropriate tools are installed on image. PiperOrigin-RevId: 268608466 --- scripts/build.sh | 3 +++ 1 file changed, 3 insertions(+) (limited to 'scripts') diff --git a/scripts/build.sh b/scripts/build.sh index 81023aab3..1d2d2a6d6 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -16,6 +16,9 @@ source $(dirname $0)/common.sh +# Install required packages for make_repository.sh et al. +sudo apt-get update && sudo apt-get install -y dpkg-sig coreutils gpg apt-utils + # Build runsc. runsc=$(build -c opt //runsc) -- cgit v1.2.3 From 69f2c41b7acc4b72df9fe5fad984e05f78bfe6cd Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Wed, 11 Sep 2019 22:29:07 -0700 Subject: Drop unavailable package. PiperOrigin-RevId: 268614014 --- scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/build.sh b/scripts/build.sh index 1d2d2a6d6..4a1cf730c 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -17,7 +17,7 @@ source $(dirname $0)/common.sh # Install required packages for make_repository.sh et al. -sudo apt-get update && sudo apt-get install -y dpkg-sig coreutils gpg apt-utils +sudo apt-get update && sudo apt-get install -y dpkg-sig coreutils apt-utils # Build runsc. runsc=$(build -c opt //runsc) -- cgit v1.2.3 From 574eda88808138b2dd72ebe6bbca80a09a13c7fb Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Thu, 12 Sep 2019 13:43:07 -0700 Subject: Update repository directory structure. Currently it will not work with apt out of the box, as we require the dists/ prefix, along with a distribution name. This tweaks the overall structure to allow for the same URL prefix to be used for all repositories, and enables multiple architectures. Fixes #852 PiperOrigin-RevId: 268756104 --- scripts/build.sh | 28 ++++++++++++++++++---------- tools/make_repository.sh | 21 +++++++++++++++------ 2 files changed, 33 insertions(+), 16 deletions(-) (limited to 'scripts') diff --git a/scripts/build.sh b/scripts/build.sh index 4a1cf730c..d73eaee77 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -32,12 +32,14 @@ fi # Install installs artifacts. install() { - local dir="$1" - mkdir -p "${dir}" - cp -f "${runsc}" "${dir}"/runsc - sha512sum "${dir}"/runsc | awk '{print $1 " runsc"}' > "${dir}"/runsc.sha512 + local -r binaries_dir="$1" + local -r repo_dir="$2" + mkdir -p "${binaries_dir}" + cp -f "${runsc}" "${binaries_dir}"/runsc + sha512sum "${binaries_dir}"/runsc | awk '{print $1 " runsc"}' > "${binaries_dir}"/runsc.sha512 if [[ -v repo ]]; then - rm -rf "${dir}"/repo && cp -a "${repo}" "$dir"/repo + rm -rf "${repo_dir}" && mkdir -p "$(dirname "${repo_dir}")" + cp -a "${repo}" "${repo_dir}" fi } @@ -47,8 +49,11 @@ install() { if [[ -v KOKORO_ARTIFACTS_DIR ]]; then if [[ "${KOKORO_BUILD_NIGHTLY}" == "true" ]]; then # The "latest" directory and current date. - install "${KOKORO_ARTIFACTS_DIR}/nightly/latest" - install "${KOKORO_ARTIFACTS_DIR}/nightly/$(date -Idate)" + stamp="$(date -Idate)" + install "${KOKORO_ARTIFACTS_DIR}/nightly/latest" \ + "${KOKORO_ARTIFACTS_DIR}/dists/nightly/main" + install "${KOKORO_ARTIFACTS_DIR}/nightly/${stamp}" \ + "${KOKORO_ARTIFACTS_DIR}/dists/nightly/${stamp}" else # Is it a tagged release? Build that instead. In that case, we also try to # update the base release directory, in case this is an update. Finally, we @@ -60,11 +65,14 @@ if [[ -v KOKORO_ARTIFACTS_DIR ]]; then for tag in ${tags}; do name=$(echo "${tag}" | cut -d'-' -f2) base=$(echo "${name}" | cut -d'.' -f1) - install "${KOKORO_ARTIFACTS_DIR}/release/${name}" + install "${KOKORO_ARTIFACTS_DIR}/release/${name}" \ + "${KOKORO_ARTIFACTS_DIR}/dists/${name}/main" if [[ "${base}" != "${tag}" ]]; then - install "${KOKORO_ARTIFACTS_DIR}/release/${base}" + install "${KOKORO_ARTIFACTS_DIR}/release/${base}" \ + "${KOKORO_ARTIFACTS_DIR}/dists/${base}/main" fi - install "${KOKORO_ARTIFACTS_DIR}/release/latest" + install "${KOKORO_ARTIFACTS_DIR}/release/latest" \ + "${KOKORO_ARTIFACTS_DIR}/dists/latest/main" done fi fi diff --git a/tools/make_repository.sh b/tools/make_repository.sh index ccebe27b3..b16ac6311 100755 --- a/tools/make_repository.sh +++ b/tools/make_repository.sh @@ -39,11 +39,18 @@ cleanup() { trap cleanup EXIT gpg --no-default-keyring --keyring "${keyring}" --import "${private_key}" >&2 -# Export the public key from the keyring. -gpg --no-default-keyring --keyring "${keyring}" --armor --export "${signer}" > "${tmpdir}"/keyFile >&2 - # Copy the packages, and ensure permissions are correct. -cp -a "$@" "${tmpdir}" && chmod 0644 "${tmpdir}"/* +for pkg in "$@"; do + name=$(basename "${pkg}" .deb) + name=$(basename "${name}" .changes) + arch=${name##*_} + if [[ "${name}" == "${arch}" ]]; then + continue # Not a regular package. + fi + mkdir -p "${tmpdir}"/binary-"${arch}" + cp -a "${pkg}" "${tmpdir}"/binary-"${arch}" +done +find "${tmpdir}" -type f -exec chmod 0644 {} \; # Ensure there are no symlinks hanging around; these may be remnants of the # build process. They may be useful for other things, but we are going to build @@ -51,12 +58,14 @@ cp -a "$@" "${tmpdir}" && chmod 0644 "${tmpdir}"/* find "${tmpdir}" -type l -exec rm -f {} \; # Sign all packages. -for file in "${tmpdir}"/*.deb; do +for file in "${tmpdir}"/binary-*/*.deb; do dpkg-sig -g "--no-default-keyring --keyring ${keyring}" --sign builder "${file}" >&2 done # Build the package list. -(cd "${tmpdir}" && apt-ftparchive packages . | gzip > Packages.gz) +for dir in "${tmpdir}"/binary-*; do + (cd "${dir}" && apt-ftparchive packages . | gzip > Packages.gz) +done # Build the release list. (cd "${tmpdir}" && apt-ftparchive release . > Release) -- cgit v1.2.3 From 010b0932583711ab3f6a88b1136cf8d87c2a53d2 Mon Sep 17 00:00:00 2001 From: Fabricio Voznika Date: Mon, 16 Sep 2019 08:15:40 -0700 Subject: Bring back to life features lost in recent refactor - Sandbox logs are generated when running tests - Kokoro uploads the sandbox logs - Supports multiple parallel runs - Revive script to install locally built runsc with docker PiperOrigin-RevId: 269337274 --- CONTRIBUTING.md | 32 +++++++++++++++++++ runsc/boot/config.go | 26 ++++++++++----- runsc/container/container.go | 9 +++++- runsc/dockerutil/dockerutil.go | 15 ++++++--- runsc/main.go | 4 ++- runsc/sandbox/sandbox.go | 10 +++++- runsc/specutils/specutils.go | 16 +++++++++- scripts/common.sh | 59 +++++++++++++++++++++++++++++++++- scripts/common_bazel.sh | 34 ++++++++++++++------ scripts/dev.sh | 72 ++++++++++++++++++++++++++++++++++++++++++ scripts/docker_tests.sh | 6 ++-- scripts/go.sh | 2 +- scripts/hostnet_tests.sh | 5 ++- scripts/kvm_tests.sh | 5 ++- scripts/overlay_tests.sh | 5 ++- scripts/root_tests.sh | 6 ++-- test/root/main_test.go | 5 +-- 17 files changed, 265 insertions(+), 46 deletions(-) create mode 100755 scripts/dev.sh (limited to 'scripts') diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 638942a42..5d46168bc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,6 +83,8 @@ Rules: ### Code reviews +Before sending code reviews, run `bazel test ...` to ensure tests are passing. + Code changes are accepted via [pull request][github]. When approved, the change will be submitted by a team member and automatically @@ -100,6 +102,36 @@ form `b/1234`. These correspond to bugs in our internal bug tracker. Eventually these bugs will be moved to the GitHub Issues, but until then they can simply be ignored. +### Build and test with Docker + +`scripts/dev.sh` is a convenient script that builds and installs `runsc` as a +new Docker runtime for you. The scripts tries to extract the runtime name from +your local environment and will print it at the end. You can also customize it. +The script creates one regular runtime and another with debug flags enabled. +Here are a few examples: + +```bash +# Default case (inside branch my-branch) +$ scripts/dev.sh +... +Runtimes my-branch and my-branch-d (debug enabled) setup. +Use --runtime=my-branch with your Docker command. + docker run --rm --runtime=my-branch --rm hello-world + +If you rebuild, use scripts/dev.sh --refresh. +Logs are in: /tmp/my-branch/logs + +# --refresh just updates the runtime binary and doesn't restart docker. +$ git/my_branch> scripts/dev.sh --refresh + +# Using a custom runtime name +$ git/my_branch> scripts/dev.sh my-runtime +... +Runtimes my-runtime and my-runtime-d (debug enabled) setup. +Use --runtime=my-runtime with your Docker command. + docker run --rm --runtime=my-runtime --rm hello-world +``` + ### The small print Contributions made by corporations are covered by a different agreement than the diff --git a/runsc/boot/config.go b/runsc/boot/config.go index 05b8f8761..31103367d 100644 --- a/runsc/boot/config.go +++ b/runsc/boot/config.go @@ -211,12 +211,6 @@ type Config struct { // RestoreFile is the path to the saved container image RestoreFile string - // TestOnlyAllowRunAsCurrentUserWithoutChroot should only be used in - // tests. It allows runsc to start the sandbox process as the current - // user, and without chrooting the sandbox process. This can be - // necessary in test environments that have limited capabilities. - TestOnlyAllowRunAsCurrentUserWithoutChroot bool - // NumNetworkChannels controls the number of AF_PACKET sockets that map // to the same underlying network device. This allows netstack to better // scale for high throughput use cases. @@ -233,6 +227,19 @@ type Config struct { // ReferenceLeakMode sets reference leak check mode ReferenceLeakMode refs.LeakMode + + // TestOnlyAllowRunAsCurrentUserWithoutChroot should only be used in + // tests. It allows runsc to start the sandbox process as the current + // user, and without chrooting the sandbox process. This can be + // necessary in test environments that have limited capabilities. + TestOnlyAllowRunAsCurrentUserWithoutChroot bool + + // TestOnlyTestNameEnv should only be used in tests. It looks up for the + // test name in the container environment variables and adds it to the debug + // log file name. This is done to help identify the log with the test when + // multiple tests are run in parallel, since there is no way to pass + // parameters to the runtime from docker. + TestOnlyTestNameEnv string } // ToFlags returns a slice of flags that correspond to the given Config. @@ -261,9 +268,12 @@ func (c *Config) ToFlags() []string { "--alsologtostderr=" + strconv.FormatBool(c.AlsoLogToStderr), "--ref-leak-mode=" + refsLeakModeToString(c.ReferenceLeakMode), } + // Only include these if set since it is never to be used by users. if c.TestOnlyAllowRunAsCurrentUserWithoutChroot { - // Only include if set since it is never to be used by users. - f = append(f, "-TESTONLY-unsafe-nonroot=true") + f = append(f, "--TESTONLY-unsafe-nonroot=true") + } + if len(c.TestOnlyTestNameEnv) != 0 { + f = append(f, "--TESTONLY-test-name-env="+c.TestOnlyTestNameEnv) } return f } diff --git a/runsc/container/container.go b/runsc/container/container.go index 00f1b1de9..a721c1c31 100644 --- a/runsc/container/container.go +++ b/runsc/container/container.go @@ -946,7 +946,14 @@ func (c *Container) createGoferProcess(spec *specs.Spec, conf *boot.Config, bund } if conf.DebugLog != "" { - debugLogFile, err := specutils.DebugLogFile(conf.DebugLog, "gofer") + test := "" + if len(conf.TestOnlyTestNameEnv) != 0 { + // Fetch test name if one is provided and the test only flag was set. + if t, ok := specutils.EnvVar(spec.Process.Env, conf.TestOnlyTestNameEnv); ok { + test = t + } + } + debugLogFile, err := specutils.DebugLogFile(conf.DebugLog, "gofer", test) if err != nil { return nil, nil, fmt.Errorf("opening debug log file in %q: %v", conf.DebugLog, err) } diff --git a/runsc/dockerutil/dockerutil.go b/runsc/dockerutil/dockerutil.go index 41f5fe1e8..c073d8f75 100644 --- a/runsc/dockerutil/dockerutil.go +++ b/runsc/dockerutil/dockerutil.go @@ -240,7 +240,7 @@ func (d *Docker) Stop() error { // Run calls 'docker run' with the arguments provided. The container starts // running in the background and the call returns immediately. func (d *Docker) Run(args ...string) error { - a := []string{"run", "--runtime", d.Runtime, "--name", d.Name, "-d"} + a := d.runArgs("-d") a = append(a, args...) _, err := do(a...) if err == nil { @@ -251,7 +251,7 @@ func (d *Docker) Run(args ...string) error { // RunWithPty is like Run but with an attached pty. func (d *Docker) RunWithPty(args ...string) (*exec.Cmd, *os.File, error) { - a := []string{"run", "--runtime", d.Runtime, "--name", d.Name, "-it"} + a := d.runArgs("-it") a = append(a, args...) return doWithPty(a...) } @@ -259,8 +259,7 @@ func (d *Docker) RunWithPty(args ...string) (*exec.Cmd, *os.File, error) { // RunFg calls 'docker run' with the arguments provided in the foreground. It // blocks until the container exits and returns the output. func (d *Docker) RunFg(args ...string) (string, error) { - a := []string{"run", "--runtime", d.Runtime, "--name", d.Name} - a = append(a, args...) + a := d.runArgs(args...) out, err := do(a...) if err == nil { d.logDockerID() @@ -268,6 +267,14 @@ func (d *Docker) RunFg(args ...string) (string, error) { return string(out), err } +func (d *Docker) runArgs(args ...string) []string { + // Environment variable RUNSC_TEST_NAME is picked up by the runtime and added + // to the log name, so one can easily identify the corresponding logs for + // this test. + rv := []string{"run", "--runtime", d.Runtime, "--name", d.Name, "-e", "RUNSC_TEST_NAME=" + d.Name} + return append(rv, args...) +} + // Logs calls 'docker logs'. func (d *Docker) Logs() (string, error) { return do("logs", d.Name) diff --git a/runsc/main.go b/runsc/main.go index 0ff68160d..ff74c0a3d 100644 --- a/runsc/main.go +++ b/runsc/main.go @@ -79,6 +79,7 @@ var ( // Test flags, not to be used outside tests, ever. testOnlyAllowRunAsCurrentUserWithoutChroot = flag.Bool("TESTONLY-unsafe-nonroot", false, "TEST ONLY; do not ever use! This skips many security measures that isolate the host from the sandbox.") + testOnlyTestNameEnv = flag.String("TESTONLY-test-name-env", "", "TEST ONLY; do not ever use! Used for automated tests to improve logging.") ) func main() { @@ -211,6 +212,7 @@ func main() { ReferenceLeakMode: refsLeakMode, TestOnlyAllowRunAsCurrentUserWithoutChroot: *testOnlyAllowRunAsCurrentUserWithoutChroot, + TestOnlyTestNameEnv: *testOnlyTestNameEnv, } if len(*straceSyscalls) != 0 { conf.StraceSyscalls = strings.Split(*straceSyscalls, ",") @@ -244,7 +246,7 @@ func main() { e = newEmitter(*debugLogFormat, f) } else if *debugLog != "" { - f, err := specutils.DebugLogFile(*debugLog, subcommand) + f, err := specutils.DebugLogFile(*debugLog, subcommand, "" /* name */) if err != nil { cmd.Fatalf("error opening debug log file in %q: %v", *debugLog, err) } diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index df3c0c5ef..4c6c83fbd 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -351,7 +351,15 @@ func (s *Sandbox) createSandboxProcess(conf *boot.Config, args *Args, startSyncF nextFD++ } if conf.DebugLog != "" { - debugLogFile, err := specutils.DebugLogFile(conf.DebugLog, "boot") + test := "" + if len(conf.TestOnlyTestNameEnv) == 0 { + // Fetch test name if one is provided and the test only flag was set. + if t, ok := specutils.EnvVar(args.Spec.Process.Env, conf.TestOnlyTestNameEnv); ok { + test = t + } + } + + debugLogFile, err := specutils.DebugLogFile(conf.DebugLog, "boot", test) if err != nil { return fmt.Errorf("opening debug log file in %q: %v", conf.DebugLog, err) } diff --git a/runsc/specutils/specutils.go b/runsc/specutils/specutils.go index df435f88d..cb9e58dfb 100644 --- a/runsc/specutils/specutils.go +++ b/runsc/specutils/specutils.go @@ -399,13 +399,15 @@ func WaitForReady(pid int, timeout time.Duration, ready func() (bool, error)) er // - %TIMESTAMP%: is replaced with a timestamp using the following format: // // - %COMMAND%: is replaced with 'command' -func DebugLogFile(logPattern, command string) (*os.File, error) { +// - %TEST%: is replaced with 'test' (omitted by default) +func DebugLogFile(logPattern, command, test string) (*os.File, error) { if strings.HasSuffix(logPattern, "/") { // Default format: /runsc.log.. logPattern += "runsc.log.%TIMESTAMP%.%COMMAND%" } logPattern = strings.Replace(logPattern, "%TIMESTAMP%", time.Now().Format("20060102-150405.000000"), -1) logPattern = strings.Replace(logPattern, "%COMMAND%", command, -1) + logPattern = strings.Replace(logPattern, "%TEST%", test, -1) dir := filepath.Dir(logPattern) if err := os.MkdirAll(dir, 0775); err != nil { @@ -542,3 +544,15 @@ func GetParentPid(pid int) (int, error) { return ppid, nil } + +// EnvVar looks for a varible value in the env slice assuming the following +// format: "NAME=VALUE". +func EnvVar(env []string, name string) (string, bool) { + prefix := name + "=" + for _, e := range env { + if strings.HasPrefix(e, prefix) { + return strings.TrimPrefix(e, prefix), true + } + } + return "", false +} diff --git a/scripts/common.sh b/scripts/common.sh index f2b9e24d8..6dabad141 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -14,10 +14,67 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -xeo pipefail +set -xeou pipefail if [[ -f $(dirname $0)/common_google.sh ]]; then source $(dirname $0)/common_google.sh else source $(dirname $0)/common_bazel.sh fi + +# Ensure it attempts to collect logs in all cases. +trap collect_logs EXIT + +function set_runtime() { + RUNTIME=${1:-runsc} + RUNSC_BIN=/tmp/"${RUNTIME}"/runsc + RUNSC_LOGS_DIR="$(dirname ${RUNSC_BIN})"/logs + RUNSC_LOGS="${RUNSC_LOGS_DIR}"/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND% +} + +function test_runsc() { + test --test_arg=--runtime=${RUNTIME} "$@" +} + +function install_runsc_for_test() { + local -r test_name=$1 + shift + if [[ -z "${test_name}" ]]; then + echo "Missing mandatory test name" + exit 1 + fi + + # Add test to the name, so it doesn't conflict with other runtimes. + set_runtime $(find_branch_name)_"${test_name}" + + # ${RUNSC_TEST_NAME} is set by tests (see dockerutil) to pass the test name + # down to the runtime. + install_runsc "${RUNTIME}" \ + --TESTONLY-test-name-env=RUNSC_TEST_NAME \ + --debug \ + --strace \ + --log-packets \ + "$@" +} + +# Installs the runsc with given runtime name. set_runtime must have been called +# to set runtime and logs location. +function install_runsc() { + local -r runtime=$1 + shift + + # Prepare the runtime binary. + local -r output=$(build //runsc) + mkdir -p "$(dirname ${RUNSC_BIN})" + cp -f "${output}" "${RUNSC_BIN}" + chmod 0755 "${RUNSC_BIN}" + + # Install the runtime. + sudo "${RUNSC_BIN}" install --experimental=true --runtime="${runtime}" -- --debug-log "${RUNSC_LOGS}" "$@" + + # Clear old logs files that may exist. + sudo rm -f "${RUNSC_LOGS_DIR}"/* + + # Restart docker to pick up the new runtime configuration. + sudo systemctl restart docker +} diff --git a/scripts/common_bazel.sh b/scripts/common_bazel.sh index dc0e2041d..dde0b51ed 100755 --- a/scripts/common_bazel.sh +++ b/scripts/common_bazel.sh @@ -53,16 +53,7 @@ function build() { } function test() { - (bazel test "${BAZEL_RBE_FLAGS[@]}" "${BAZEL_RBE_AUTH_FLAGS[@]}" "${BAZEL_FLAGS[@]}" "$@" && rc=0) || rc=$? - - # Zip out everything into a convenient form. - if [[ -v KOKORO_ARTIFACTS_DIR ]] && [[ -e bazel-testlogs ]]; then - find -L "bazel-testlogs" -name "test.xml" -o -name "test.log" -o -name "outputs.zip" | - tar --create --files-from - --transform 's/test\./sponge_log./' | - tar --extract --directory ${KOKORO_ARTIFACTS_DIR} - fi - - return $rc + bazel test "${BAZEL_RBE_FLAGS[@]}" "${BAZEL_RBE_AUTH_FLAGS[@]}" "${BAZEL_FLAGS[@]}" "$@" } function run() { @@ -76,3 +67,26 @@ function run_as_root() { shift bazel run --run_under="sudo" "${binary}" -- "$@" } + +function collect_logs() { + # Zip out everything into a convenient form. + if [[ -v KOKORO_ARTIFACTS_DIR ]] && [[ -e bazel-testlogs ]]; then + # Move test logs to Kokoro directory. tar is used to conveniently perform + # renames while moving files. + find -L "bazel-testlogs" -name "test.xml" -o -name "test.log" -o -name "outputs.zip" | + tar --create --files-from - --transform 's/test\./sponge_log./' | + tar --extract --directory ${KOKORO_ARTIFACTS_DIR} + + # Collect sentry logs, if any. + if [[ -v RUNSC_LOGS_DIR ]] && [[ -d "${RUNSC_LOGS_DIR}" ]]; then + local -r logs=$(ls "${RUNSC_LOGS_DIR}") + if [[ -z "${logs}" ]]; then + tar --create --gzip --file="${KOKORO_ARTIFACTS_DIR}/${RUNTIME}.tar.gz" -C "${RUNSC_LOGS_DIR}" . + fi + fi + fi +} + +function find_branch_name() { + git branch --show-current || git rev-parse HEAD || bazel info workspace | xargs basename +} diff --git a/scripts/dev.sh b/scripts/dev.sh new file mode 100755 index 000000000..64151c558 --- /dev/null +++ b/scripts/dev.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source $(dirname $0)/common.sh + +# common.sh sets '-x', but it's annoying to see so much output. +set +x + +# Defaults +declare -i REFRESH=0 +declare NAME=$(find_branch_name) + +while [[ $# -gt 0 ]]; do + case "$1" in + --refresh) + REFRESH=1 + ;; + --help) + echo "Use this script to build and install runsc with Docker." + echo + echo "usage: $0 [--refresh] [runtime_name]" + exit 1 + ;; + *) + NAME=$1 + ;; + esac + shift +done + +set_runtime "${NAME}" +echo +echo "Using runtime=${RUNTIME}" +echo + +echo Building runsc... +# Build first and fail on error. $() prevents "set -e" from reporting errors. +build //runsc +declare OUTPUT="$(build //runsc)" + +if [[ ${REFRESH} -eq 0 ]]; then + install_runsc "${RUNTIME}" --net-raw + install_runsc "${RUNTIME}-d" --net-raw --debug --strace --log-packets + + echo + echo "Runtimes ${RUNTIME} and ${RUNTIME}-d (debug enabled) setup." + echo "Use --runtime="${RUNTIME}" with your Docker command." + echo " docker run --rm --runtime="${RUNTIME}" --rm hello-world" + echo + echo "If you rebuild, use $0 --refresh." + +else + cp -f ${OUTPUT} "${RUNSC_BIN}" + + echo + echo "Runtime ${RUNTIME} refreshed." +fi + +echo "Logs are in: ${RUNSC_LOGS_DIR}" diff --git a/scripts/docker_tests.sh b/scripts/docker_tests.sh index d6b18a35b..72ba05260 100755 --- a/scripts/docker_tests.sh +++ b/scripts/docker_tests.sh @@ -16,7 +16,5 @@ source $(dirname $0)/common.sh -# Install the runtime and perform basic tests. -run_as_root //runsc install --experimental=true -- --debug --strace --log-packets -sudo systemctl restart docker -test //test/image:image_test //test/e2e:integration_test +install_runsc_for_test docker +test_runsc //test/image:image_test //test/e2e:integration_test diff --git a/scripts/go.sh b/scripts/go.sh index f24fad04c..0dbfb7747 100755 --- a/scripts/go.sh +++ b/scripts/go.sh @@ -29,7 +29,7 @@ git checkout go && git clean -f go build ./... # Push, if required. -if [[ "${KOKORO_GO_PUSH}" == "true" ]]; then +if [[ -v KOKORO_GO_PUSH ]] && [[ "${KOKORO_GO_PUSH}" == "true" ]]; then if [[ -v KOKORO_GITHUB_ACCESS_TOKEN ]]; then git config --global credential.helper cache git credential approve < Date: Tue, 17 Sep 2019 17:24:39 -0700 Subject: scripts/build.sh: fix kokoro failure "KOKORO_BUILD_NIGHTLY: unbound variable" PiperOrigin-RevId: 269690988 --- scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/build.sh b/scripts/build.sh index d73eaee77..5021dda49 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -47,7 +47,7 @@ install() { # current date. If the current commit happens to correpond to a tag, then we # will also move everything into a directory named after the given tag. if [[ -v KOKORO_ARTIFACTS_DIR ]]; then - if [[ "${KOKORO_BUILD_NIGHTLY}" == "true" ]]; then + if [[ "${KOKORO_BUILD_NIGHTLY:-false}" == "true" ]]; then # The "latest" directory and current date. stamp="$(date -Idate)" install "${KOKORO_ARTIFACTS_DIR}/nightly/latest" \ -- cgit v1.2.3 From 461123ea3510a401423181e8ea8f2cae27fcbc8f Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Wed, 18 Sep 2019 14:55:55 -0700 Subject: Move the component into the repository structure. The RELEASE file must be at the top-level for the signed repository to work correctly. PiperOrigin-RevId: 269897109 --- scripts/build.sh | 10 +++++----- tools/make_repository.sh | 15 ++++++++------- 2 files changed, 13 insertions(+), 12 deletions(-) (limited to 'scripts') diff --git a/scripts/build.sh b/scripts/build.sh index 5021dda49..b3a6e4e7a 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -27,7 +27,7 @@ pkg=$(build -c opt --host_force_python=py2 //runsc:runsc-debian) # Build a repository, if the key is available. if [[ -v KOKORO_REPO_KEY ]]; then - repo=$(tools/make_repository.sh "${KOKORO_KEYSTORE_DIR}/${KOKORO_REPO_KEY}" gvisor-bot@google.com ${pkg}) + repo=$(tools/make_repository.sh "${KOKORO_KEYSTORE_DIR}/${KOKORO_REPO_KEY}" gvisor-bot@google.com main ${pkg}) fi # Install installs artifacts. @@ -51,7 +51,7 @@ if [[ -v KOKORO_ARTIFACTS_DIR ]]; then # The "latest" directory and current date. stamp="$(date -Idate)" install "${KOKORO_ARTIFACTS_DIR}/nightly/latest" \ - "${KOKORO_ARTIFACTS_DIR}/dists/nightly/main" + "${KOKORO_ARTIFACTS_DIR}/dists/nightly/latest" install "${KOKORO_ARTIFACTS_DIR}/nightly/${stamp}" \ "${KOKORO_ARTIFACTS_DIR}/dists/nightly/${stamp}" else @@ -66,13 +66,13 @@ if [[ -v KOKORO_ARTIFACTS_DIR ]]; then name=$(echo "${tag}" | cut -d'-' -f2) base=$(echo "${name}" | cut -d'.' -f1) install "${KOKORO_ARTIFACTS_DIR}/release/${name}" \ - "${KOKORO_ARTIFACTS_DIR}/dists/${name}/main" + "${KOKORO_ARTIFACTS_DIR}/dists/${name}" if [[ "${base}" != "${tag}" ]]; then install "${KOKORO_ARTIFACTS_DIR}/release/${base}" \ - "${KOKORO_ARTIFACTS_DIR}/dists/${base}/main" + "${KOKORO_ARTIFACTS_DIR}/dists/${base}" fi install "${KOKORO_ARTIFACTS_DIR}/release/latest" \ - "${KOKORO_ARTIFACTS_DIR}/dists/latest/main" + "${KOKORO_ARTIFACTS_DIR}/dists/latest" done fi fi diff --git a/tools/make_repository.sh b/tools/make_repository.sh index b16ac6311..071f72b74 100755 --- a/tools/make_repository.sh +++ b/tools/make_repository.sh @@ -16,13 +16,14 @@ # Parse arguments. We require more than two arguments, which are the private # keyring, the e-mail associated with the signer, and the list of packages. -if [ "$#" -le 2 ]; then - echo "usage: $0 " +if [ "$#" -le 3 ]; then + echo "usage: $0 " exit 1 fi declare -r private_key=$(readlink -e "$1") declare -r signer="$2" -shift; shift +declare -r component="$3" +shift; shift; shift # Verbose from this point. set -xeo pipefail @@ -47,8 +48,8 @@ for pkg in "$@"; do if [[ "${name}" == "${arch}" ]]; then continue # Not a regular package. fi - mkdir -p "${tmpdir}"/binary-"${arch}" - cp -a "${pkg}" "${tmpdir}"/binary-"${arch}" + mkdir -p "${tmpdir}"/"${component}"/binary-"${arch}" + cp -a "${pkg}" "${tmpdir}"/"${component}"/binary-"${arch}" done find "${tmpdir}" -type f -exec chmod 0644 {} \; @@ -58,12 +59,12 @@ find "${tmpdir}" -type f -exec chmod 0644 {} \; find "${tmpdir}" -type l -exec rm -f {} \; # Sign all packages. -for file in "${tmpdir}"/binary-*/*.deb; do +for file in "${tmpdir}"/"${component}"/binary-*/*.deb; do dpkg-sig -g "--no-default-keyring --keyring ${keyring}" --sign builder "${file}" >&2 done # Build the package list. -for dir in "${tmpdir}"/binary-*; do +for dir in "${tmpdir}"/"${component}"/binary-*; do (cd "${dir}" && apt-ftparchive packages . | gzip > Packages.gz) done -- cgit v1.2.3 From a1f84469218d148b1f8210370cb08677b0d74f49 Mon Sep 17 00:00:00 2001 From: Fabricio Voznika Date: Wed, 18 Sep 2019 16:54:29 -0700 Subject: Fix dev.sh --refresh to create target dir PiperOrigin-RevId: 269921234 --- scripts/dev.sh | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/dev.sh b/scripts/dev.sh index 64151c558..ee74dcb72 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -63,6 +63,7 @@ if [[ ${REFRESH} -eq 0 ]]; then echo "If you rebuild, use $0 --refresh." else + mkdir -p "$(dirname ${RUNSC_BIN})" cp -f ${OUTPUT} "${RUNSC_BIN}" echo -- cgit v1.2.3 From 2fb34c8d5ccf13388371437d128cc95d577fbc8a Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Tue, 24 Sep 2019 19:03:26 -0700 Subject: test: don't use designated initializers This change fixes compile errors: pty.cc:1460:7: error: expected primary-expression before '.' token ... PiperOrigin-RevId: 271033729 --- scripts/make_tests.sh | 1 + test/syscalls/linux/pty.cc | 28 ++++++++++++---------------- 2 files changed, 13 insertions(+), 16 deletions(-) (limited to 'scripts') diff --git a/scripts/make_tests.sh b/scripts/make_tests.sh index 0fa1248be..79426756d 100755 --- a/scripts/make_tests.sh +++ b/scripts/make_tests.sh @@ -21,4 +21,5 @@ top_level=$(git rev-parse --show-toplevel 2>/dev/null) make make runsc +make BAZEL_OPTIONS="build //..." bazel make bazel-shutdown diff --git a/test/syscalls/linux/pty.cc b/test/syscalls/linux/pty.cc index 286388316..bf32efe1e 100644 --- a/test/syscalls/linux/pty.cc +++ b/test/syscalls/linux/pty.cc @@ -1292,10 +1292,9 @@ TEST_F(JobControlTest, ReleaseTTY) { // Make sure we're ignoring SIGHUP, which will be sent to this process once we // disconnect they TTY. - struct sigaction sa = { - .sa_handler = SIG_IGN, - .sa_flags = 0, - }; + struct sigaction sa = {}; + sa.sa_handler = SIG_IGN; + sa.sa_flags = 0; sigemptyset(&sa.sa_mask); struct sigaction old_sa; EXPECT_THAT(sigaction(SIGHUP, &sa, &old_sa), SyscallSucceeds()); @@ -1362,10 +1361,9 @@ TEST_F(JobControlTest, ReleaseTTYSignals) { ASSERT_THAT(ioctl(slave_.get(), TIOCSCTTY, 0), SyscallSucceeds()); received = 0; - struct sigaction sa = { - .sa_handler = sig_handler, - .sa_flags = 0, - }; + struct sigaction sa = {}; + sa.sa_handler = sig_handler; + sa.sa_flags = 0; sigemptyset(&sa.sa_mask); sigaddset(&sa.sa_mask, SIGHUP); sigaddset(&sa.sa_mask, SIGCONT); @@ -1403,10 +1401,9 @@ TEST_F(JobControlTest, ReleaseTTYSignals) { // Make sure we're ignoring SIGHUP, which will be sent to this process once we // disconnect they TTY. - struct sigaction sighup_sa = { - .sa_handler = SIG_IGN, - .sa_flags = 0, - }; + struct sigaction sighup_sa = {}; + sighup_sa.sa_handler = SIG_IGN; + sighup_sa.sa_flags = 0; sigemptyset(&sighup_sa.sa_mask); struct sigaction old_sa; EXPECT_THAT(sigaction(SIGHUP, &sighup_sa, &old_sa), SyscallSucceeds()); @@ -1456,10 +1453,9 @@ TEST_F(JobControlTest, SetForegroundProcessGroup) { ASSERT_THAT(ioctl(slave_.get(), TIOCSCTTY, 0), SyscallSucceeds()); // Ignore SIGTTOU so that we don't stop ourself when calling tcsetpgrp. - struct sigaction sa = { - .sa_handler = SIG_IGN, - .sa_flags = 0, - }; + struct sigaction sa = {}; + sa.sa_handler = SIG_IGN; + sa.sa_flags = 0; sigemptyset(&sa.sa_mask); sigaction(SIGTTOU, &sa, NULL); -- cgit v1.2.3 From 129c67d68ee2db4aa3a45ab6970e7d26348ce5ef Mon Sep 17 00:00:00 2001 From: Fabricio Voznika Date: Wed, 25 Sep 2019 14:31:40 -0700 Subject: Fix runsc log collection in kokoro PiperOrigin-RevId: 271207152 --- runsc/sandbox/sandbox.go | 2 +- scripts/common_bazel.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index 4c6c83fbd..ee9327fc8 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -352,7 +352,7 @@ func (s *Sandbox) createSandboxProcess(conf *boot.Config, args *Args, startSyncF } if conf.DebugLog != "" { test := "" - if len(conf.TestOnlyTestNameEnv) == 0 { + if len(conf.TestOnlyTestNameEnv) != 0 { // Fetch test name if one is provided and the test only flag was set. if t, ok := specutils.EnvVar(args.Spec.Process.Env, conf.TestOnlyTestNameEnv); ok { test = t diff --git a/scripts/common_bazel.sh b/scripts/common_bazel.sh index dde0b51ed..ea2291a4d 100755 --- a/scripts/common_bazel.sh +++ b/scripts/common_bazel.sh @@ -80,7 +80,7 @@ function collect_logs() { # Collect sentry logs, if any. if [[ -v RUNSC_LOGS_DIR ]] && [[ -d "${RUNSC_LOGS_DIR}" ]]; then local -r logs=$(ls "${RUNSC_LOGS_DIR}") - if [[ -z "${logs}" ]]; then + if [[ "${logs}" ]]; then tar --create --gzip --file="${KOKORO_ARTIFACTS_DIR}/${RUNTIME}.tar.gz" -C "${RUNSC_LOGS_DIR}" . fi fi -- cgit v1.2.3 From 3221e8372cbd41bbe74d0bef82519de6e2852e13 Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Thu, 26 Sep 2019 14:35:57 -0700 Subject: kokoro: don't force to use python2 https://github.com/bazelbuild/bazel/issues/7899 was fixed and we don't need this hack anymore. PiperOrigin-RevId: 271434565 --- scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/build.sh b/scripts/build.sh index b3a6e4e7a..0b3d1b316 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -23,7 +23,7 @@ sudo apt-get update && sudo apt-get install -y dpkg-sig coreutils apt-utils runsc=$(build -c opt //runsc) # Build packages. -pkg=$(build -c opt --host_force_python=py2 //runsc:runsc-debian) +pkg=$(build -c opt //runsc:runsc-debian) # Build a repository, if the key is available. if [[ -v KOKORO_REPO_KEY ]]; then -- cgit v1.2.3 From 739f53fc17e3e3ed82dc2bc920f49fa91738a437 Mon Sep 17 00:00:00 2001 From: Fabricio Voznika Date: Tue, 1 Oct 2019 13:49:35 -0700 Subject: Add runsc logs to kokoro artifacts PiperOrigin-RevId: 272286122 --- kokoro/docker_tests.cfg | 1 + kokoro/hostnet_tests.cfg | 1 + kokoro/kvm_tests.cfg | 1 + kokoro/overlay_tests.cfg | 1 + kokoro/root_tests.cfg | 1 + scripts/common_bazel.sh | 11 +++++++++-- 6 files changed, 14 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/kokoro/docker_tests.cfg b/kokoro/docker_tests.cfg index 717d71dd3..0a0ef87ed 100644 --- a/kokoro/docker_tests.cfg +++ b/kokoro/docker_tests.cfg @@ -5,5 +5,6 @@ action { regex: "**/sponge_log.xml" regex: "**/sponge_log.log" regex: "**/outputs.zip" + regex: "**/runsc_logs_*.tar.gz" } } diff --git a/kokoro/hostnet_tests.cfg b/kokoro/hostnet_tests.cfg index 532755f4a..520dc55a3 100644 --- a/kokoro/hostnet_tests.cfg +++ b/kokoro/hostnet_tests.cfg @@ -5,5 +5,6 @@ action { regex: "**/sponge_log.xml" regex: "**/sponge_log.log" regex: "**/outputs.zip" + regex: "**/runsc_logs_*.tar.gz" } } diff --git a/kokoro/kvm_tests.cfg b/kokoro/kvm_tests.cfg index 54365c2b2..1feb60c8a 100644 --- a/kokoro/kvm_tests.cfg +++ b/kokoro/kvm_tests.cfg @@ -5,5 +5,6 @@ action { regex: "**/sponge_log.xml" regex: "**/sponge_log.log" regex: "**/outputs.zip" + regex: "**/runsc_logs_*.tar.gz" } } diff --git a/kokoro/overlay_tests.cfg b/kokoro/overlay_tests.cfg index abd96f60c..6a2ddbd03 100644 --- a/kokoro/overlay_tests.cfg +++ b/kokoro/overlay_tests.cfg @@ -5,5 +5,6 @@ action { regex: "**/sponge_log.xml" regex: "**/sponge_log.log" regex: "**/outputs.zip" + regex: "**/runsc_logs_*.tar.gz" } } diff --git a/kokoro/root_tests.cfg b/kokoro/root_tests.cfg index 20b97766a..28351695c 100644 --- a/kokoro/root_tests.cfg +++ b/kokoro/root_tests.cfg @@ -5,5 +5,6 @@ action { regex: "**/sponge_log.xml" regex: "**/sponge_log.log" regex: "**/outputs.zip" + regex: "**/runsc_logs_*.tar.gz" } } diff --git a/scripts/common_bazel.sh b/scripts/common_bazel.sh index ea2291a4d..f8ec967b1 100755 --- a/scripts/common_bazel.sh +++ b/scripts/common_bazel.sh @@ -79,9 +79,16 @@ function collect_logs() { # Collect sentry logs, if any. if [[ -v RUNSC_LOGS_DIR ]] && [[ -d "${RUNSC_LOGS_DIR}" ]]; then - local -r logs=$(ls "${RUNSC_LOGS_DIR}") + # Check if the directory is empty or not (only the first line it needed). + local -r logs=$(ls "${RUNSC_LOGS_DIR}" | head -n1) if [[ "${logs}" ]]; then - tar --create --gzip --file="${KOKORO_ARTIFACTS_DIR}/${RUNTIME}.tar.gz" -C "${RUNSC_LOGS_DIR}" . + local -r archive=runsc_logs_"${RUNTIME}".tar.gz + if [[ -v KOKORO_BUILD_ARTIFACTS_SUBDIR ]]; then + echo "runsc logs will be uploaded to:" + echo " gsutil cp gs://gvisor/logs/${KOKORO_BUILD_ARTIFACTS_SUBDIR}/${archive} /tmp" + echo " https://storage.cloud.google.com/gvisor/logs/${KOKORO_BUILD_ARTIFACTS_SUBDIR}/${archive}" + fi + tar --create --gzip --file="${KOKORO_ARTIFACTS_DIR}/${archive}" -C "${RUNSC_LOGS_DIR}" . fi fi fi -- cgit v1.2.3 From 03ce4dd86c9acd6b6148f68d5d2cf025d8c254bb Mon Sep 17 00:00:00 2001 From: Michael Pratt Date: Tue, 1 Oct 2019 16:44:27 -0700 Subject: Remove extra --rm PiperOrigin-RevId: 272324038 --- scripts/dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/dev.sh b/scripts/dev.sh index ee74dcb72..c67003018 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -58,7 +58,7 @@ if [[ ${REFRESH} -eq 0 ]]; then echo echo "Runtimes ${RUNTIME} and ${RUNTIME}-d (debug enabled) setup." echo "Use --runtime="${RUNTIME}" with your Docker command." - echo " docker run --rm --runtime="${RUNTIME}" --rm hello-world" + echo " docker run --rm --runtime="${RUNTIME}" hello-world" echo echo "If you rebuild, use $0 --refresh." -- cgit v1.2.3