diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/bazel.mk | 55 | ||||
| -rw-r--r-- | tools/bazel_gazelle.patch | 24 | ||||
| -rw-r--r-- | tools/bazeldefs/defs.bzl | 2 | ||||
| -rw-r--r-- | tools/defs.bzl | 4 | ||||
| -rwxr-xr-x | tools/go_branch.sh | 2 | ||||
| -rwxr-xr-x | tools/installers/containerd.sh | 15 |
6 files changed, 79 insertions, 23 deletions
diff --git a/tools/bazel.mk b/tools/bazel.mk index 3a7de427f..ca5621a9c 100644 --- a/tools/bazel.mk +++ b/tools/bazel.mk @@ -36,11 +36,14 @@ DOCKER_PRIVILEGED := --privileged BAZEL_CACHE := $(shell readlink -m ~/.cache/bazel/) GCLOUD_CONFIG := $(shell readlink -m ~/.config/gcloud/) DOCKER_SOCKET := /var/run/docker.sock -DOCKER_CONFIG := /etc/docker/daemon.json +DOCKER_CONFIG := /etc/docker # Bazel flags. BAZEL := bazel $(STARTUP_OPTIONS) -OPTIONS += --color=no --curses=no +BASE_OPTIONS := --color=no --curses=no +ifneq (,$(BAZEL_CONFIG)) +BASE_OPTIONS += --config=$(BAZEL_CONFIG) +endif # Basic options. UID := $(shell id -u ${USER}) @@ -72,7 +75,7 @@ endif # out of disk space. ifneq ($(UID),0) USERADD_DOCKER += useradd -l --uid $(UID) --non-unique --no-create-home \ - --gid $(GID) $(USERADD_OPTIONS) -d $(HOME) $(USER) && + --gid $(GID) $(USERADD_OPTIONS) -d $(HOME) $(USER) && endif ifneq ($(GID),0) GROUPADD_DOCKER += groupadd --gid $(GID) --non-unique $(USER) && @@ -81,8 +84,6 @@ endif # Add docker passthrough options. ifneq ($(DOCKER_PRIVILEGED),) FULL_DOCKER_RUN_OPTIONS += -v "$(DOCKER_SOCKET):$(DOCKER_SOCKET)" -# TODO(gvisor.dev/issue/1624): Remove docker config volume. This is required -# temporarily for checking VFS1 vs VFS2 by some tests. FULL_DOCKER_RUN_OPTIONS += -v "$(DOCKER_CONFIG):$(DOCKER_CONFIG)" FULL_DOCKER_RUN_OPTIONS += $(DOCKER_PRIVILEGED) FULL_DOCKER_EXEC_OPTIONS += $(DOCKER_PRIVILEGED) @@ -105,11 +106,6 @@ FULL_DOCKER_RUN_OPTIONS += --group-add $(KVM_GROUP) endif endif -# Load the appropriate config. -ifneq (,$(BAZEL_CONFIG)) -OPTIONS += --config=$(BAZEL_CONFIG) -endif - bazel-image: load-default @if docker ps --all | grep $(BUILDER_NAME); then docker rm -f $(BUILDER_NAME); fi docker run --user 0:0 --entrypoint "" --name $(BUILDER_NAME) \ @@ -161,20 +157,30 @@ bazel-alias: ## Emits an alias that can be used within the shell. .PHONY: bazel-alias bazel-server: ## Ensures that the server exists. Used as an internal target. - @docker exec $(FULL_DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) true || $(MAKE) bazel-server-start + @docker exec $(FULL_DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) true >&2 || $(MAKE) bazel-server-start >&2 .PHONY: bazel-server -build_cmd = docker exec $(FULL_DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) sh -o pipefail -c '$(BAZEL) build $(OPTIONS) "$(TARGETS)"' +# build_cmd builds the given targets in the bazel-server container. +build_cmd = docker exec $(FULL_DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) sh -o pipefail -c \ + '$(BAZEL) build $(BASE_OPTIONS) $(OPTIONS) "$(TARGETS)"' -build_paths = $(build_cmd) 2>&1 \ - | tee /proc/self/fd/2 \ +# build_paths extracts the built binary from the bazel stderr output. +# +# This could be alternately done by parsing the bazel build event stream, but +# this is a complex schema, and begs the question: what will build the thing +# that parses the output? Bazel? Do we need a separate bootstrapping build +# command here? Yikes, let's just stick with the ugly shell pipeline. +# +# The last line is used to prevent terminal shenanigans. +build_paths = command_line=$$( $(build_cmd) 2>&1 \ | grep -A1 -E '^Target' \ | grep -E '^ ($(subst $(SPACE),|,$(BUILD_ROOTS)))' \ | sed "s/ /\n/g" \ | strings -n 10 \ | awk '{$$1=$$1};1' \ | xargs -n 1 -I {} readlink -f "{}" \ - | xargs -n 1 -I {} sh -c "$(1)" + | xargs -n 1 -I {} echo "$(1)" ) && \ + (set -xeuo pipefail; eval $${command_line}) build: bazel-server @$(call build_cmd) @@ -194,12 +200,21 @@ sudo: bazel-server @$(call build_paths,sudo -E {} $(ARGS)) .PHONY: sudo -test: OPTIONS += --test_output=errors --keep_going --verbose_failures=true test: bazel-server - @docker exec $(FULL_DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) $(BAZEL) test $(OPTIONS) $(TARGETS) + @docker exec $(FULL_DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) \ + $(BAZEL) test $(BASE_OPTIONS) \ + --test_output=errors --keep_going --verbose_failures=true \ + --build_event_json_file=.build_events.json \ + $(OPTIONS) $(TARGETS) .PHONY: test -query: - @$(MAKE) bazel-server >&2 # If we need to start, ensure stdout is not polluted. - @docker exec $(FULL_DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) sh -o pipefail -c '$(BAZEL) query $(OPTIONS) "$(TARGETS)" 2>/dev/null' +testlogs: + @cat .build_events.json | jq -r \ + 'select(.testSummary?.overallStatus? | tostring | test("(FAILED|FLAKY|TIMEOUT)")) | .testSummary.failed | .[] | .uri' | \ + awk -Ffile:// '{print $$2;}' +.PHONY: testlogs + +query: bazel-server + @docker exec $(FULL_DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) sh -o pipefail -c \ + '$(BAZEL) query $(BASE_OPTIONS) $(OPTIONS) "$(TARGETS)" 2>/dev/null' .PHONY: query diff --git a/tools/bazel_gazelle.patch b/tools/bazel_gazelle.patch new file mode 100644 index 000000000..e35f38933 --- /dev/null +++ b/tools/bazel_gazelle.patch @@ -0,0 +1,24 @@ +diff -r -u2 a/language/go/resolve.go b/language/go/resolve.go +--- a/language/go/resolve.go 2020-10-02 14:22:18.000000000 -0700 ++++ b/language/go/resolve.go 2020-11-17 19:40:59.770648029 -0800 +@@ -20,5 +20,4 @@ + "fmt" + "go/build" +- "log" + "path" + "regexp" +@@ -80,5 +79,5 @@ + resolve = ResolveGo + } +- deps, errs := imports.Map(func(imp string) (string, error) { ++ deps, _ := imports.Map(func(imp string) (string, error) { + l, err := resolve(c, ix, rc, imp, from) + if err == skipImportError { +@@ -95,7 +94,4 @@ + return l.String(), nil + }) +- for _, err := range errs { +- log.Print(err) +- } + if !deps.IsEmpty() { + if r.Kind() == "go_proto_library" { diff --git a/tools/bazeldefs/defs.bzl b/tools/bazeldefs/defs.bzl index c2f94bb9c..51e17a79a 100644 --- a/tools/bazeldefs/defs.bzl +++ b/tools/bazeldefs/defs.bzl @@ -7,6 +7,8 @@ build_test = _build_test bzl_library = _bzl_library rbe_platform = native.platform rbe_toolchain = native.toolchain +more_shards = 4 +most_shards = 8 def short_path(path): return path diff --git a/tools/defs.bzl b/tools/defs.bzl index 2c8129e7e..b6f188aeb 100644 --- a/tools/defs.bzl +++ b/tools/defs.bzl @@ -8,7 +8,7 @@ change for Google-internal and bazel-compatible rules. load("//tools/go_stateify:defs.bzl", "go_stateify") load("//tools/go_marshal:defs.bzl", "go_marshal", "marshal_deps", "marshal_test_deps") load("//tools/nogo:defs.bzl", "nogo_test") -load("//tools/bazeldefs:defs.bzl", _build_test = "build_test", _bzl_library = "bzl_library", _coreutil = "coreutil", _default_installer = "default_installer", _default_net_util = "default_net_util", _proto_library = "proto_library", _rbe_platform = "rbe_platform", _rbe_toolchain = "rbe_toolchain", _select_arch = "select_arch", _select_system = "select_system", _short_path = "short_path") +load("//tools/bazeldefs:defs.bzl", _build_test = "build_test", _bzl_library = "bzl_library", _coreutil = "coreutil", _default_installer = "default_installer", _default_net_util = "default_net_util", _more_shards = "more_shards", _most_shards = "most_shards", _proto_library = "proto_library", _rbe_platform = "rbe_platform", _rbe_toolchain = "rbe_toolchain", _select_arch = "select_arch", _select_system = "select_system", _short_path = "short_path") load("//tools/bazeldefs:cc.bzl", _cc_binary = "cc_binary", _cc_flags_supplier = "cc_flags_supplier", _cc_grpc_library = "cc_grpc_library", _cc_library = "cc_library", _cc_proto_library = "cc_proto_library", _cc_test = "cc_test", _cc_toolchain = "cc_toolchain", _gbenchmark = "gbenchmark", _grpcpp = "grpcpp", _gtest = "gtest", _vdso_linker_option = "vdso_linker_option") load("//tools/bazeldefs:go.bzl", _gazelle = "gazelle", _go_binary = "go_binary", _go_embed_data = "go_embed_data", _go_grpc_and_proto_libraries = "go_grpc_and_proto_libraries", _go_library = "go_library", _go_path = "go_path", _go_proto_library = "go_proto_library", _go_test = "go_test", _select_goarch = "select_goarch", _select_goos = "select_goos") load("//tools/bazeldefs:pkg.bzl", _pkg_deb = "pkg_deb", _pkg_tar = "pkg_tar") @@ -26,6 +26,8 @@ short_path = _short_path rbe_platform = _rbe_platform rbe_toolchain = _rbe_toolchain coreutil = _coreutil +more_shards = _more_shards +most_shards = _most_shards # C++ rules. cc_binary = _cc_binary diff --git a/tools/go_branch.sh b/tools/go_branch.sh index 71d036b12..768a37b9a 100755 --- a/tools/go_branch.sh +++ b/tools/go_branch.sh @@ -39,7 +39,7 @@ declare tmp_dir tmp_dir=$(mktemp -d) readonly tmp_dir finish() { - cd # Leave tmp_dir. + cd / # Leave tmp_dir. rm -rf "${tmp_dir}" } trap finish EXIT diff --git a/tools/installers/containerd.sh b/tools/installers/containerd.sh index 6b7bb261c..5520a447c 100755 --- a/tools/installers/containerd.sh +++ b/tools/installers/containerd.sh @@ -43,10 +43,23 @@ install_helper() { make install) } +# Figure out were btrfs headers are. +# +# Ubuntu 16.04 has only btrfs-tools, while 18.04 has a transitional package, +# and later versions no longer have the transitional package. +source /etc/os-release +declare BTRFS_DEV +if [[ "${VERSION_ID%.*}" -le "18" ]]; then + BTRFS_DEV="btrfs-tools" +else + BTRFS_DEV="libbtrfs-dev" +fi +readonly BTRFS_DEV + # Install dependencies for the crictl tests. while true; do if (apt-get update && apt-get install -y \ - btrfs-tools \ + "${BTRFS_DEV}" \ libseccomp-dev); then break fi |