From 849c57314f6b6f0d1ebcfa2e68762b8ea95f5948 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Tue, 10 Sep 2019 00:37:46 -0700 Subject: Fix minor Kokoro issues. A recent Kokoro change pointed to go_tests.cfg (in line with the other configurations), which unfortunately broke the presubmits. This change also enabled the KVM tests, which were still using a remote execution strategy. This fixes both of these issues and allows presubmits to pass. One additional test was caught with this case, which seems to have been broken. It's unclear why this was not being caught. PiperOrigin-RevId: 268166291 --- kokoro/go_test.cfg | 1 - kokoro/go_tests.cfg | 1 + scripts/common_bazel.sh | 2 +- scripts/kvm_tests.sh | 7 +++---- test/root/cgroup_test.go | 18 ++++++------------ test/syscalls/linux/proc_net.cc | 2 +- tools/go_branch.sh | 6 +++++- 7 files changed, 17 insertions(+), 20 deletions(-) delete mode 100644 kokoro/go_test.cfg create mode 100644 kokoro/go_tests.cfg diff --git a/kokoro/go_test.cfg b/kokoro/go_test.cfg deleted file mode 100644 index 5eb51041a..000000000 --- a/kokoro/go_test.cfg +++ /dev/null @@ -1 +0,0 @@ -build_file: "repo/scripts/go.sh" diff --git a/kokoro/go_tests.cfg b/kokoro/go_tests.cfg new file mode 100644 index 000000000..5eb51041a --- /dev/null +++ b/kokoro/go_tests.cfg @@ -0,0 +1 @@ +build_file: "repo/scripts/go.sh" diff --git a/scripts/common_bazel.sh b/scripts/common_bazel.sh index 5340c7323..dc0e2041d 100755 --- a/scripts/common_bazel.sh +++ b/scripts/common_bazel.sh @@ -56,7 +56,7 @@ function test() { (bazel test "${BAZEL_RBE_FLAGS[@]}" "${BAZEL_RBE_AUTH_FLAGS[@]}" "${BAZEL_FLAGS[@]}" "$@" && rc=0) || rc=$? # Zip out everything into a convenient form. - if [[ -v KOKORO_ARTIFACTS_DIR ]]; then + if [[ -v KOKORO_ARTIFACTS_DIR ]] && [[ -e bazel-testlogs ]]; then find -L "bazel-testlogs" -name "test.xml" -o -name "test.log" -o -name "outputs.zip" | tar --create --files-from - --transform 's/test\./sponge_log./' | tar --extract --directory ${KOKORO_ARTIFACTS_DIR} diff --git a/scripts/kvm_tests.sh b/scripts/kvm_tests.sh index 5cb7aa007..b6d787f0f 100755 --- a/scripts/kvm_tests.sh +++ b/scripts/kvm_tests.sh @@ -20,11 +20,10 @@ source $(dirname $0)/common.sh (lsmod | grep -E '^(kvm_intel|kvm_amd)') || sudo modprobe kvm sudo chmod a+rw /dev/kvm -# Run all KVM-tagged tests (locally). -test --test_strategy=standalone --test_tag_filters=requires-kvm //... -test --test_strategy=standalone //pkg/sentry/platform/kvm:kvm_test +# Run all KVM platform tests (locally). +run_as_root //pkg/sentry/platform/kvm:kvm_test # Install the KVM runtime and run all integration tests. run_as_root //runsc install --experimental=true -- --debug --strace --log-packets --platform=kvm sudo systemctl restart docker -test --test_strategy=standalone //test/image:image_test //test/e2e:integration_test +test //test/image:image_test //test/e2e:integration_test diff --git a/test/root/cgroup_test.go b/test/root/cgroup_test.go index cc7e8583e..76f1e4f2a 100644 --- a/test/root/cgroup_test.go +++ b/test/root/cgroup_test.go @@ -62,6 +62,12 @@ func TestCgroup(t *testing.T) { } d := dockerutil.MakeDocker("cgroup-test") + // This is not a comprehensive list of attributes. + // + // Note that we are specifically missing cpusets, which fail if specified. + // In any case, it's unclear if cpusets can be reliably tested here: these + // are often run on a single core virtual machine, and there is only a single + // CPU available in our current set, and every container's set. attrs := []struct { arg string ctrl string @@ -87,18 +93,6 @@ func TestCgroup(t *testing.T) { file: "cpu.cfs_quota_us", want: "3000", }, - { - arg: "--cpuset-cpus=0", - ctrl: "cpuset", - file: "cpuset.cpus", - want: "0", - }, - { - arg: "--cpuset-mems=0", - ctrl: "cpuset", - file: "cpuset.mems", - want: "0", - }, { arg: "--kernel-memory=100MB", ctrl: "memory", diff --git a/test/syscalls/linux/proc_net.cc b/test/syscalls/linux/proc_net.cc index c097af196..efdaf202b 100644 --- a/test/syscalls/linux/proc_net.cc +++ b/test/syscalls/linux/proc_net.cc @@ -28,7 +28,7 @@ TEST(ProcNetIfInet6, Format) { EXPECT_THAT(ifinet6, ::testing::MatchesRegex( // Ex: "00000000000000000000000000000001 01 80 10 80 lo\n" - "^([a-f\\d]{32}( [a-f\\d]{2}){4} +[a-z][a-z\\d]*\\n)+$")); + "^([a-f0-9]{32}( [a-f0-9]{2}){4} +[a-z][a-z0-9]*\n)+$")); } TEST(ProcSysNetIpv4Sack, Exists) { diff --git a/tools/go_branch.sh b/tools/go_branch.sh index d9e79401d..ddb9b6e7b 100755 --- a/tools/go_branch.sh +++ b/tools/go_branch.sh @@ -59,7 +59,11 @@ git checkout -b go "${go_branch}" # Start working on a merge commit that combines the previous history with the # current history. Note that we don't actually want any changes yet. -git merge --allow-unrelated-histories --no-commit --strategy ours ${head} +# +# N.B. The git behavior changed at some point and the relevant flag was added +# to allow for override, so try the only behavior first then pass the flag. +git merge --no-commit --strategy ours ${head} || \ + git merge --allow-unrelated-histories --no-commit --strategy ours ${head} # Sync the entire gopath_dir and go.mod. rsync --recursive --verbose --delete --exclude .git --exclude README.md -L "${gopath_dir}/" . -- cgit v1.2.3 From d24be656c9321de8b65e9d21c41f756056847eb1 Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Wed, 11 Sep 2019 00:07:25 -0700 Subject: Update required Bazel version in README. PiperOrigin-RevId: 268397389 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d102845ac..7ab76d305 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Make sure the following dependencies are installed: * Linux 4.14.77+ ([older linux][old-linux]) * [git][git] -* [Bazel][bazel] 0.23.0+ +* [Bazel][bazel] 0.28.0+ * [Python][python] * [Docker version 17.09.0 or greater][docker] * Gold linker (e.g. `binutils-gold` package on Ubuntu) -- cgit v1.2.3 From f52dd451ed4c517d81dc3274568ea214fb218e71 Mon Sep 17 00:00:00 2001 From: Nicolas Lacasse Date: Wed, 11 Sep 2019 10:18:16 -0700 Subject: Bump bazel, rules_go, gazelle, and go toolchain to latest versions. PiperOrigin-RevId: 268486127 --- WORKSPACE | 14 +++++++------- kokoro/ubuntu1604/20_bazel.sh | 4 +--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 26e76e2d7..27f24d044 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -3,19 +3,19 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "io_bazel_rules_go", - sha256 = "313f2c7a23fecc33023563f082f381a32b9b7254f727a7dd2d6380ccc6dfe09b", + sha256 = "ae8c36ff6e565f674c7a3692d6a9ea1096e4c1ade497272c2108a810fb39acd2", urls = [ - "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/rules_go/releases/download/0.19.3/rules_go-0.19.3.tar.gz", - "https://github.com/bazelbuild/rules_go/releases/download/0.19.3/rules_go-0.19.3.tar.gz", + "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/rules_go/releases/download/0.19.4/rules_go-0.19.4.tar.gz", + "https://github.com/bazelbuild/rules_go/releases/download/0.19.4/rules_go-0.19.4.tar.gz", ], ) http_archive( name = "bazel_gazelle", - sha256 = "be9296bfd64882e3c08e3283c58fcb461fa6dd3c171764fcc4cf322f60615a9b", + sha256 = "7fc87f4170011201b1690326e8c16c5d802836e3a0d617d8f75c3af2b23180c4", urls = [ - "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/bazel-gazelle/releases/download/0.18.1/bazel-gazelle-0.18.1.tar.gz", - "https://github.com/bazelbuild/bazel-gazelle/releases/download/0.18.1/bazel-gazelle-0.18.1.tar.gz", + "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/bazel-gazelle/releases/download/0.18.2/bazel-gazelle-0.18.2.tar.gz", + "https://github.com/bazelbuild/bazel-gazelle/releases/download/0.18.2/bazel-gazelle-0.18.2.tar.gz", ], ) @@ -24,7 +24,7 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies", "go_register_to go_rules_dependencies() go_register_toolchains( - go_version = "1.12.9", + go_version = "1.13", nogo = "@//:nogo", ) diff --git a/kokoro/ubuntu1604/20_bazel.sh b/kokoro/ubuntu1604/20_bazel.sh index 74b4b8be2..b9a894024 100755 --- a/kokoro/ubuntu1604/20_bazel.sh +++ b/kokoro/ubuntu1604/20_bazel.sh @@ -16,9 +16,7 @@ set -xeo pipefail -# We need to install a specific version of bazel due to a bug with the RBE -# environment not respecting the dockerPrivileged configuration. -declare -r BAZEL_VERSION=0.28.1 +declare -r BAZEL_VERSION=0.29.1 # Install bazel dependencies. apt-get update && apt-get install -y openjdk-8-jdk-headless unzip -- cgit v1.2.3 From c06ef5131f1ccd3106ccf4fa4e787db079db2d96 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Wed, 11 Sep 2019 18:48:26 -0700 Subject: Fix authorization for continuous integration. The credentials must be explicitly refreshed for pushing to the repository on the Go branch. PiperOrigin-RevId: 268589817 --- kokoro/go.cfg | 14 ++++++++++++++ scripts/go.sh | 9 +++++++++ 2 files changed, 23 insertions(+) diff --git a/kokoro/go.cfg b/kokoro/go.cfg index d1577252a..759e16152 100644 --- a/kokoro/go.cfg +++ b/kokoro/go.cfg @@ -1,5 +1,19 @@ build_file: "repo/scripts/go.sh" +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73898 + keyname: "kokoro-github-access-token" + } + } +} + +env_vars { + key: "KOKORO_GITHUB_ACCESS_TOKEN" + value: "$KOKORO_ROOT/src/keystore/73898_kokoro-github-access-token" +} + env_vars { key: "KOKORO_GO_PUSH" value: "true" diff --git a/scripts/go.sh b/scripts/go.sh index e49d76c6d..83a667640 100755 --- a/scripts/go.sh +++ b/scripts/go.sh @@ -30,5 +30,14 @@ go build ./... # Push, if required. if [[ "${KOKORO_GO_PUSH}" == "true" ]]; then + if [[ -v KOKORO_GITHUB_ACCESS_TOKEN ]]; then + git config --global credential.helper cache + git credential approve < Date: Wed, 11 Sep 2019 21:03:12 -0700 Subject: Update key environment variables. PiperOrigin-RevId: 268604220 --- kokoro/build.cfg | 2 +- kokoro/go.cfg | 2 +- scripts/build.sh | 2 +- scripts/go.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kokoro/build.cfg b/kokoro/build.cfg index d67af4694..d462d839c 100644 --- a/kokoro/build.cfg +++ b/kokoro/build.cfg @@ -11,7 +11,7 @@ before_action { env_vars { key: "KOKORO_REPO_KEY" - value: "$KOKORO_ROOT/src/keystore/73898_kokoro-repo-key" + value: "73898_kokoro-repo-key" } action { diff --git a/kokoro/go.cfg b/kokoro/go.cfg index 759e16152..b9c1fcb12 100644 --- a/kokoro/go.cfg +++ b/kokoro/go.cfg @@ -11,7 +11,7 @@ before_action { env_vars { key: "KOKORO_GITHUB_ACCESS_TOKEN" - value: "$KOKORO_ROOT/src/keystore/73898_kokoro-github-access-token" + value: "73898_kokoro-github-access-token" } env_vars { diff --git a/scripts/build.sh b/scripts/build.sh index d5dd14acc..81023aab3 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -24,7 +24,7 @@ pkg=$(build -c opt --host_force_python=py2 //runsc:runsc-debian) # Build a repository, if the key is available. if [[ -v KOKORO_REPO_KEY ]]; then - repo=$(tools/make_repository.sh "${KOKORO_REPO_KEY}" gvisor-bot@google.com ${pkg}) + repo=$(tools/make_repository.sh "${KOKORO_KEYSTORE_DIR}/${KOKORO_REPO_KEY}" gvisor-bot@google.com ${pkg}) fi # Install installs artifacts. diff --git a/scripts/go.sh b/scripts/go.sh index 83a667640..f24fad04c 100755 --- a/scripts/go.sh +++ b/scripts/go.sh @@ -35,7 +35,7 @@ if [[ "${KOKORO_GO_PUSH}" == "true" ]]; then git credential approve <