diff options
Diffstat (limited to 'tools')
140 files changed, 0 insertions, 17073 deletions
diff --git a/tools/BUILD b/tools/BUILD deleted file mode 100644 index 3861ff2a5..000000000 --- a/tools/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -load("//tools:defs.bzl", "bzl_library") - -package(licenses = ["notice"]) - -bzl_library( - name = "defs_bzl", - srcs = ["defs.bzl"], - visibility = [ - "//:sandbox", - ], -) - -bzl_library( - name = "deps_bzl", - srcs = ["deps.bzl"], - visibility = [ - "//:sandbox", - ], -) diff --git a/tools/bazel.mk b/tools/bazel.mk deleted file mode 100644 index 60b50cfb0..000000000 --- a/tools/bazel.mk +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/make -f - -# Copyright 2018 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## -## Docker options. -## -## This file supports targets that wrap bazel in a running Docker -## container to simplify development. Some options are available to -## control the behavior of this container: -## -## USER - The in-container user. -## DOCKER_RUN_OPTIONS - Options for the container (default: --privileged, required for tests). -## DOCKER_NAME - The container name (default: gvisor-bazel-HASH). -## DOCKER_PRIVILEGED - Docker privileged flags (default: --privileged). -## BAZEL_CACHE - The bazel cache directory (default: detected). -## GCLOUD_CONFIG - The gcloud config directory (detect: detected). -## DOCKER_SOCKET - The Docker socket (default: detected). -## -## To opt out of these wrappers, set DOCKER_BUILD=false. -DOCKER_BUILD := true -ifeq ($(DOCKER_BUILD),true) --include bazel-server -endif - -# See base Makefile. -BRANCH_NAME := $(shell (git branch --show-current 2>/dev/null || \ - git rev-parse --abbrev-ref HEAD 2>/dev/null) | \ - xargs -n 1 basename 2>/dev/null) -BUILD_ROOTS := bazel-bin/ bazel-out/ - -# Bazel container configuration (see below). -USER := $(shell whoami) -HASH := $(shell readlink -m $(CURDIR) | md5sum | cut -c1-8) -BUILDER_NAME := gvisor-builder-$(HASH)-$(ARCH) -DOCKER_NAME := gvisor-bazel-$(HASH)-$(ARCH) -DOCKER_PRIVILEGED := --privileged -BAZEL_CACHE := $(HOME)/.cache/bazel/ -GCLOUD_CONFIG := $(HOME)/.config/gcloud/ -DOCKER_SOCKET := /var/run/docker.sock -DOCKER_CONFIG := /etc/docker - -## -## Bazel helpers. -## -## Bazel will be run with standard flags. You can specify the following flags -## to control which flags are passed: -## -## STARTUP_OPTIONS - Startup options passed to Bazel. -## -STARTUP_OPTIONS := -BAZEL_OPTIONS := -BAZEL := bazel $(STARTUP_OPTIONS) -BASE_OPTIONS := --color=no --curses=no -TEST_OPTIONS := $(BASE_OPTIONS) \ - --test_output=errors \ - --keep_going \ - --verbose_failures=true \ - --build_event_json_file=.build_events.json - -# Basic options. -UID := $(shell id -u ${USER}) -GID := $(shell id -g ${USER}) -USERADD_OPTIONS := -DOCKER_RUN_OPTIONS := -DOCKER_RUN_OPTIONS += --rm -DOCKER_RUN_OPTIONS += --user $(UID):$(GID) -DOCKER_RUN_OPTIONS += --entrypoint "" -DOCKER_RUN_OPTIONS += --init -DOCKER_RUN_OPTIONS += -v "$(shell readlink -m $(BAZEL_CACHE)):$(BAZEL_CACHE)" -DOCKER_RUN_OPTIONS += -v "$(shell readlink -m $(GCLOUD_CONFIG)):$(GCLOUD_CONFIG)" -DOCKER_RUN_OPTIONS += -v "/tmp:/tmp" -DOCKER_EXEC_OPTIONS := --user $(UID):$(GID) -DOCKER_EXEC_OPTIONS += --interactive -ifeq (true,$(shell test -t 0 && echo true)) -DOCKER_EXEC_OPTIONS += --tty -endif - -# Add basic UID/GID options. -# -# Note that USERADD_DOCKER and GROUPADD_DOCKER are both defined as "deferred" -# variables in Make terminology, that is they will be expanded at time of use -# and may include other variables, including those defined below. -# -# NOTE: we pass -l to useradd below because otherwise you can hit a bug -# best described here: -# https://github.com/moby/moby/issues/5419#issuecomment-193876183 -# TLDR; trying to add to /var/log/lastlog (sparse file) runs the machine out -# out of disk space. -ifneq ($(UID),0) -USERADD_DOCKER += useradd -l --uid $(UID) --non-unique --no-create-home \ - --gid $(GID) $(USERADD_OPTIONS) -d $(HOME) $(USER) && -endif -ifneq ($(GID),0) -GROUPADD_DOCKER += groupadd --gid $(GID) --non-unique $(USER) && -endif - -# Add docker passthrough options. -ifneq ($(DOCKER_PRIVILEGED),) -DOCKER_RUN_OPTIONS += -v "$(DOCKER_SOCKET):$(DOCKER_SOCKET)" -DOCKER_RUN_OPTIONS += -v "$(DOCKER_CONFIG):$(DOCKER_CONFIG)" -DOCKER_RUN_OPTIONS += $(DOCKER_PRIVILEGED) -DOCKER_EXEC_OPTIONS += $(DOCKER_PRIVILEGED) -DOCKER_GROUP := $(shell stat -c '%g' $(DOCKER_SOCKET)) -ifneq ($(GID),$(DOCKER_GROUP)) -USERADD_OPTIONS += --groups $(DOCKER_GROUP) -GROUPADD_DOCKER += groupadd --gid $(DOCKER_GROUP) --non-unique docker-$(HASH) && -DOCKER_RUN_OPTIONS += --group-add $(DOCKER_GROUP) -endif -endif - -# Add KVM passthrough options. -ifneq (,$(wildcard /dev/kvm)) -DOCKER_RUN_OPTIONS += --device=/dev/kvm -KVM_GROUP := $(shell stat -c '%g' /dev/kvm) -ifneq ($(GID),$(KVM_GROUP)) -USERADD_OPTIONS += --groups $(KVM_GROUP) -GROUPADD_DOCKER += groupadd --gid $(KVM_GROUP) --non-unique kvm-$(HASH) && -DOCKER_RUN_OPTIONS += --group-add $(KVM_GROUP) -endif -endif - -# Top-level functions. -# -# This command runs a bazel server, and the container sticks around -# until the bazel server exits. This should ensure that it does not -# exit in the middle of running a build, but also it won't stick around -# forever. The build commands wrap around an appropriate exec into the -# container in order to perform work via the bazel client. -ifeq ($(DOCKER_BUILD),true) -wrapper = docker exec $(DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) $(1) -else -wrapper = $(1) -endif - -bazel-shutdown: ## Shuts down a running bazel server. - @$(call wrapper,$(BAZEL) shutdown) -.PHONY: bazel-shutdown - -bazel-alias: ## Emits an alias that can be used within the shell. - @echo "alias bazel='$(call wrapper,$(BAZEL))'" -.PHONY: bazel-alias - -bazel-image: load-default ## Ensures that the local builder exists. - @$(call header,DOCKER BUILD) - @docker rm -f $(BUILDER_NAME) 2>/dev/null || true - @docker run --user 0:0 --entrypoint "" --name $(BUILDER_NAME) gvisor.dev/images/default \ - bash -c "$(GROUPADD_DOCKER) $(USERADD_DOCKER) if test -e /dev/kvm; then chmod a+rw /dev/kvm; fi" >&2 - @docker commit $(BUILDER_NAME) gvisor.dev/images/builder >&2 -.PHONY: bazel-image - -ifneq (true,$(shell $(wrapper echo true))) -bazel-server: bazel-image ## Ensures that the server exists. - @$(call header,DOCKER RUN) - @docker rm -f $(DOCKER_NAME) 2>/dev/null || true - @mkdir -p $(BAZEL_CACHE) - @mkdir -p $(GCLOUD_CONFIG) - @docker run -d --name $(DOCKER_NAME) \ - -v "$(CURDIR):$(CURDIR)" \ - --workdir "$(CURDIR)" \ - $(DOCKER_RUN_OPTIONS) \ - gvisor.dev/images/builder \ - bash -c "set -x; tail -f --pid=\$$($(BAZEL) info server_pid) /dev/null" -else -bazel-server: - @ -endif -.PHONY: bazel-server - -# build_paths extracts the built binary from the bazel stderr output. -# -# This could be alternately done by parsing the bazel build event stream, but -# this is a complex schema, and begs the question: what will build the thing -# that parses the output? Bazel? Do we need a separate bootstrapping build -# command here? Yikes, let's just stick with the ugly shell pipeline. -# -# The last line is used to prevent terminal shenanigans. -build_paths = \ - (set -euo pipefail; \ - $(call wrapper,$(BAZEL) build $(BASE_OPTIONS) $(BAZEL_OPTIONS) $(1)) 2>&1 \ - | tee /dev/fd/2 \ - | sed -n -e '/^Target/,$$p' \ - | sed -n -e '/^ \($(subst /,\/,$(subst $(SPACE),\|,$(BUILD_ROOTS)))\)/p' \ - | sed -e 's/ /\n/g' \ - | awk '{$$1=$$1};1' \ - | strings \ - | xargs -r -n 1 -I {} readlink -f "{}" \ - | xargs -r -n 1 -I {} bash -c 'set -xeuo pipefail; $(2)') - -clean = $(call header,CLEAN) && $(call wrapper,$(BAZEL) clean) -build = $(call header,BUILD $(1)) && $(call build_paths,$(1),echo {}) -copy = $(call header,COPY $(1) $(2)) && $(call build_paths,$(1),cp -fa {} $(2)) -run = $(call header,RUN $(1) $(2)) && $(call build_paths,$(1),{} $(2)) -sudo = $(call header,SUDO $(1) $(2)) && $(call build_paths,$(1),sudo -E {} $(2)) -test = $(call header,TEST $(1)) && $(call wrapper,$(BAZEL) test $(TEST_OPTIONS) $(1)) - -clean: ## Cleans the bazel cache. - @$(call clean) -.PHONY: clean - -testlogs: ## Returns the most recent set of test logs. - @if test -f .build_events.json; then \ - cat .build_events.json | jq -r \ - 'select(.testSummary?.overallStatus? | tostring | test("(FAILED|FLAKY|TIMEOUT)")) | "\(.id.testSummary.label) \(.testSummary.failed[].uri)"' | \ - sed -e 's|file://||'; \ - fi -.PHONY: testlogs diff --git a/tools/bazel_gazelle_generate.patch b/tools/bazel_gazelle_generate.patch deleted file mode 100644 index fd1e1bda6..000000000 --- a/tools/bazel_gazelle_generate.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/language/go/generate.go b/language/go/generate.go -index 2892948..feb4ad6 100644 ---- a/language/go/generate.go -+++ b/language/go/generate.go -@@ -691,6 +691,10 @@ func (g *generator) setImportAttrs(r *rule.Rule, importPath string) { - } - - func (g *generator) commonVisibility(importPath string) []string { -+ if importPath == "golang.org/x/tools/go/analysis/internal/facts" { -+ // Imported by nogo main. We add a visibility exception. -+ return []string{"//visibility:public"} -+ } - // If the Bazel package name (rel) contains "internal", add visibility for - // subpackages of the parent. - // If the import path contains "internal" but rel does not, this is diff --git a/tools/bazeldefs/BUILD b/tools/bazeldefs/BUILD deleted file mode 100644 index 24e6f8a94..000000000 --- a/tools/bazeldefs/BUILD +++ /dev/null @@ -1,56 +0,0 @@ -load("//tools:defs.bzl", "bzl_library", "go_proto_library") - -package( - default_visibility = ["//:sandbox"], - licenses = ["notice"], -) - -bzl_library( - name = "platforms_bzl", - srcs = ["platforms.bzl"], - visibility = ["//visibility:private"], -) - -bzl_library( - name = "tags_bzl", - srcs = ["tags.bzl"], - visibility = ["//visibility:private"], -) - -bzl_library( - name = "defs_bzl", - srcs = ["defs.bzl"], - visibility = ["//visibility:private"], -) - -config_setting( - name = "linux_arm64_cross", - values = { - "cpu": "aarch64", - "host_cpu": "k8", - }, - visibility = ["//visibility:private"], -) - -config_setting( - name = "linux_amd64_cross", - values = { - "cpu": "k8", - "host_cpu": "aarch64", - }, - visibility = ["//visibility:private"], -) - -genrule( - name = "version", - outs = ["version.txt"], - cmd = "cat bazel-out/stable-status.txt | grep STABLE_VERSION | cut -d' ' -f2- | sed 's/^[^[:digit:]]*//g' >$@", - stamp = True, - visibility = ["//:sandbox"], -) - -go_proto_library( - name = "worker_protocol_go_proto", - importpath = "gvisor.dev/bazel/worker_protocol_go_proto", - proto = "@bazel_tools//src/main/protobuf:worker_protocol_proto", -) diff --git a/tools/bazeldefs/cc.bzl b/tools/bazeldefs/cc.bzl deleted file mode 100644 index 2831eac5f..000000000 --- a/tools/bazeldefs/cc.bzl +++ /dev/null @@ -1,51 +0,0 @@ -"""C++ rules.""" - -load("@rules_cc//cc:defs.bzl", _cc_binary = "cc_binary", _cc_library = "cc_library", _cc_proto_library = "cc_proto_library", _cc_test = "cc_test") -load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", _cc_grpc_library = "cc_grpc_library") - -cc_library = _cc_library -cc_proto_library = _cc_proto_library -cc_test = _cc_test -cc_toolchain = "@bazel_tools//tools/cpp:current_cc_toolchain" -gtest = "@com_google_googletest//:gtest" -gbenchmark = "@com_google_benchmark//:benchmark" -grpcpp = "@com_github_grpc_grpc//:grpc++" -vdso_linker_option = "-fuse-ld=gold " - -def _cc_flags_supplier_impl(ctx): - variables = platform_common.TemplateVariableInfo({ - "CC_FLAGS": "", - }) - return [variables] - -cc_flags_supplier = rule( - implementation = _cc_flags_supplier_impl, -) - -def cc_grpc_library(name, **kwargs): - _cc_grpc_library(name = name, grpc_only = True, **kwargs) - -def cc_binary(name, static = False, **kwargs): - """Run cc_binary. - - Args: - name: name of the target. - static: make a static binary if True - **kwargs: the rest of the args. - """ - if static: - # How to statically link a c++ program that uses threads, like for gRPC: - # https://gcc.gnu.org/legacy-ml/gcc-help/2010-05/msg00029.html - if "linkopts" not in kwargs: - kwargs["linkopts"] = [] - kwargs["linkopts"] += [ - "-static", - "-lstdc++", - "-Wl,--whole-archive", - "-lpthread", - "-Wl,--no-whole-archive", - ] - _cc_binary( - name = name, - **kwargs - ) diff --git a/tools/bazeldefs/defs.bzl b/tools/bazeldefs/defs.bzl deleted file mode 100644 index 7875bbaea..000000000 --- a/tools/bazeldefs/defs.bzl +++ /dev/null @@ -1,81 +0,0 @@ -"""Meta and miscellaneous rules.""" - -load("@bazel_skylib//rules:build_test.bzl", _build_test = "build_test") -load("@bazel_skylib//:bzl_library.bzl", _bzl_library = "bzl_library") - -build_test = _build_test -bzl_library = _bzl_library -more_shards = 4 -most_shards = 8 -version = "//tools/bazeldefs:version" - -def short_path(path): - return path - -def proto_library(name, has_services = None, **kwargs): - native.proto_library( - name = name, - **kwargs - ) - -def select_arch(amd64 = "amd64", arm64 = "arm64", default = None, **kwargs): - values = { - "@bazel_tools//src/conditions:linux_x86_64": amd64, - "@bazel_tools//src/conditions:linux_aarch64": arm64, - } - if default: - values["//conditions:default"] = default - return select(values, **kwargs) - -def select_system(linux = ["__linux__"], **kwargs): - return linux # Only Linux supported. - -def default_installer(): - return None - -def default_net_util(): - return [] # Nothing needed. - -def coreutil(): - return [] # Nothing needed. - -def select_native_vs_cross(native = [], amd64 = [], arm64 = [], cross = []): - values = { - "//tools/bazeldefs:linux_arm64_cross": arm64 + cross, - "//tools/bazeldefs:linux_amd64_cross": amd64 + cross, - "//conditions:default": native, - } - return select(values) - -def arch_genrule(name, srcs, outs, cmd, tools): - """Runs a gen command on the target architecture. - - If the target architecture isn't match the host architecture, it will build - a command for the target architecture and run it via qemu. - - The native genrule runs the command on the host architecture. - - Args: - name: name of generated target. - srcs: A list of inputs for this rule. - cmd: The command to run. It has to contain " QEMU " before executed binaries. - outs: A list of files generated by this rule. - tools: A list of tool dependencies for this rule. - """ - qemu_arm64 = "qemu-aarch64-static" - qemu_amd64 = "qemu-x86_64-static" - srcs = select_native_vs_cross( - cross = srcs + tools, - native = srcs, - ) - tools = select_native_vs_cross( - cross = [], - native = tools, - ) - cmd = select_native_vs_cross( - arm64 = cmd.replace("QEMU", qemu_arm64), - amd64 = cmd.replace("QEMU", qemu_amd64), - native = cmd.replace("QEMU", ""), - cross = "", - ) - native.genrule(name = name, srcs = srcs, outs = outs, cmd = cmd, tools = tools) diff --git a/tools/bazeldefs/go.bzl b/tools/bazeldefs/go.bzl deleted file mode 100644 index da027846b..000000000 --- a/tools/bazeldefs/go.bzl +++ /dev/null @@ -1,147 +0,0 @@ -"""Go rules.""" - -load("@bazel_gazelle//:def.bzl", _gazelle = "gazelle") -load("@io_bazel_rules_go//go:def.bzl", "GoLibrary", _go_binary = "go_binary", _go_context = "go_context", _go_embed_data = "go_embed_data", _go_library = "go_library", _go_path = "go_path", _go_test = "go_test") -load("@io_bazel_rules_go//proto:def.bzl", _go_grpc_library = "go_grpc_library", _go_proto_library = "go_proto_library") -load("//tools/bazeldefs:defs.bzl", "select_arch", "select_system") - -gazelle = _gazelle -go_embed_data = _go_embed_data -go_path = _go_path -bazel_worker_proto = "//tools/bazeldefs:worker_protocol_go_proto" - -def _go_proto_or_grpc_library(go_library_func, name, **kwargs): - if "importpath" in kwargs: - # If importpath is explicit, pass straight through. - go_library_func(name = name, **kwargs) - return - deps = [ - dep.replace("_proto", "_go_proto") - for dep in (kwargs.pop("deps", []) or []) - ] - go_library_func( - name = name + "_go_proto", - importpath = "gvisor.dev/gvisor/" + native.package_name() + "/" + name + "_go_proto", - proto = ":" + name + "_proto", - deps = deps, - **kwargs - ) - -def go_proto_library(name, **kwargs): - _go_proto_or_grpc_library(_go_proto_library, name, **kwargs) - -def go_grpc_and_proto_libraries(name, **kwargs): - _go_proto_or_grpc_library(_go_grpc_library, name, **kwargs) - -def go_binary(name, static = False, pure = False, x_defs = None, system_malloc = False, **kwargs): - """Build a go binary. - - Args: - name: name of the target. - static: build a static binary. - pure: build without cgo. - x_defs: additional definitions. - **kwargs: rest of the arguments are passed to _go_binary. - """ - if static: - kwargs["static"] = "on" - if pure: - kwargs["pure"] = "on" - _go_binary( - name = name, - x_defs = x_defs, - **kwargs - ) - -def go_importpath(target): - """Returns the importpath for the target.""" - return target[GoLibrary].importpath - -def go_library(name, arch_deps = [], **kwargs): - _go_library( - name = name, - importpath = "gvisor.dev/gvisor/" + native.package_name(), - **kwargs - ) - -def go_test(name, pure = False, library = None, **kwargs): - """Build a go test. - - Args: - name: name of the output binary. - pure: should it be built without cgo. - library: the library to embed. - **kwargs: rest of the arguments to pass to _go_test. - """ - if pure: - kwargs["pure"] = "on" - if library: - kwargs["embed"] = [library] - _go_test( - name = name, - **kwargs - ) - -def go_rule(rule, implementation, **kwargs): - """Wraps a rule definition with Go attributes. - - Args: - rule: rule function (typically rule or aspect). - implementation: implementation function. - **kwargs: other arguments to pass to rule. - - Returns: - The result of invoking the rule. - """ - attrs = kwargs.pop("attrs", dict()) - attrs["_go_context_data"] = attr.label(default = "@io_bazel_rules_go//:go_context_data") - attrs["_stdlib"] = attr.label(default = "@io_bazel_rules_go//:stdlib") - toolchains = kwargs.get("toolchains", []) + ["@io_bazel_rules_go//go:toolchain"] - return rule(implementation, attrs = attrs, toolchains = toolchains, **kwargs) - -def go_embed_libraries(target): - if hasattr(target.attr, "embed"): - return target.attr.embed - return [] - -def go_context(ctx, goos = None, goarch = None, std = False): - """Extracts a standard Go context struct. - - Args: - ctx: the starlark context (required). - goos: the GOOS value. - goarch: the GOARCH value. - std: ignored. - - Returns: - A context Go struct with pointers to Go toolchain components. - """ - - # We don't change anything for the standard library analysis. All Go files - # are available in all instances. Note that this includes the standard - # library sources, which are analyzed by nogo. - go_ctx = _go_context(ctx) - if goos == None: - goos = go_ctx.sdk.goos - elif goos != go_ctx.sdk.goos: - fail("Internal GOOS (%s) doesn't match GoSdk GOOS (%s)." % (goos, go_ctx.sdk.goos)) - if goarch == None: - goarch = go_ctx.sdk.goarch - elif goarch != go_ctx.sdk.goarch: - fail("Internal GOARCH (%s) doesn't match GoSdk GOARCH (%s)." % (goarch, go_ctx.sdk.goarch)) - return struct( - go = go_ctx.go, - env = go_ctx.env, - nogo_args = [], - stdlib_srcs = go_ctx.sdk.srcs, - runfiles = depset([go_ctx.go] + go_ctx.sdk.srcs + go_ctx.sdk.tools + go_ctx.stdlib.libs), - goos = go_ctx.sdk.goos, - goarch = go_ctx.sdk.goarch, - gotags = go_ctx.tags, - ) - -def select_goarch(): - return select_arch(arm64 = "arm64", amd64 = "amd64") - -def select_goos(): - return select_system(linux = "linux") diff --git a/tools/bazeldefs/pkg.bzl b/tools/bazeldefs/pkg.bzl deleted file mode 100644 index ccc9bdeef..000000000 --- a/tools/bazeldefs/pkg.bzl +++ /dev/null @@ -1,7 +0,0 @@ -"""Packaging rules.""" - -# N.B. We refer to pkg_deb_impl to avoid the macro, which cannot use select. -load("@rules_pkg//:pkg.bzl", _pkg_deb = "pkg_deb_impl", _pkg_tar = "pkg_tar") - -pkg_deb = _pkg_deb -pkg_tar = _pkg_tar diff --git a/tools/bazeldefs/platforms.bzl b/tools/bazeldefs/platforms.bzl deleted file mode 100644 index 165b22311..000000000 --- a/tools/bazeldefs/platforms.bzl +++ /dev/null @@ -1,9 +0,0 @@ -"""List of platforms.""" - -# Platform to associated tags. -platforms = { - "ptrace": [], - "kvm": [], -} - -default_platform = "ptrace" diff --git a/tools/bazeldefs/tags.bzl b/tools/bazeldefs/tags.bzl deleted file mode 100644 index 6564c3b25..000000000 --- a/tools/bazeldefs/tags.bzl +++ /dev/null @@ -1,60 +0,0 @@ -"""List of special Go suffixes.""" - -def explode(tagset, suffixes): - """explode combines tagset and suffixes in all ways. - - Args: - tagset: Original suffixes. - suffixes: Suffixes to combine before and after. - - Returns: - The set of possible combinations. - """ - result = [t for t in tagset] - result += [s for s in suffixes] - for t in tagset: - result += [t + s for s in suffixes] - result += [s + t for s in suffixes] - return result - -archs = [ - "_386", - "_aarch64", - "_amd64", - "_arm", - "_arm64", - "_mips", - "_mips64", - "_mips64le", - "_mipsle", - "_ppc64", - "_ppc64le", - "_riscv64", - "_s390x", - "_sparc64", - "_x86", - - # Pseudo-architectures to group by word side. - "_32bit", - "_64bit", -] - -oses = [ - "_linux", -] - -generic = [ - "_impl", - "_race", - "_norace", - "_unsafe", - "_opts", -] - -# State explosion? Sure. This is approximately: -# len(archs) * (1 + 2 * len(oses) * (1 + 2 * len(generic)) -# -# This evaluates to 495 at the time of writing. So it's a lot of different -# combinations, but not so much that it will cause issues. We can probably add -# quite a few more variants before this becomes a genuine problem. -go_suffixes = explode(explode(archs, oses), generic) diff --git a/tools/bigquery/BUILD b/tools/bigquery/BUILD deleted file mode 100644 index 2b116fe0d..000000000 --- a/tools/bigquery/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -package(licenses = ["notice"]) - -go_library( - name = "bigquery", - testonly = 1, - srcs = ["bigquery.go"], - nogo = False, # FIXME(b/184974218): Analysis failing for cloud libraries. - visibility = [ - "//:sandbox", - ], - deps = [ - "@com_google_cloud_go_bigquery//:go_default_library", - "@org_golang_google_api//option:go_default_library", - "@org_golang_x_oauth2//:go_default_library", - ], -) diff --git a/tools/bigquery/bigquery.go b/tools/bigquery/bigquery.go deleted file mode 100644 index 935154acc..000000000 --- a/tools/bigquery/bigquery.go +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package bigquery defines a BigQuery schema for benchmarks. -// -// This package contains a schema for BigQuery and methods for publishing -// benchmark data into tables. -package bigquery - -import ( - "context" - "fmt" - "strconv" - "strings" - "time" - - bq "cloud.google.com/go/bigquery" - "google.golang.org/api/option" -) - -// Suite is the top level structure for a benchmark run. BigQuery -// will infer the schema from this. -type Suite struct { - Name string `bq:"name"` - Conditions []*Condition `bq:"conditions"` - Benchmarks []*Benchmark `bq:"benchmarks"` - Official bool `bq:"official"` - Timestamp time.Time `bq:"timestamp"` -} - -// Benchmark represents an individual benchmark in a suite. -type Benchmark struct { - Name string `bq:"name"` - Condition []*Condition `bq:"condition"` - Metric []*Metric `bq:"metric"` -} - -// Condition represents qualifiers for the benchmark or suite. For example: -// Get_Pid/1/real_time would have Benchmark Name "Get_Pid" with "1" -// and "real_time" parameters as conditions. Suite conditions include -// information such as the CL number and platform name. -type Condition struct { - Name string `bq:"name"` - Value string `bq:"value"` -} - -// Metric holds the actual metric data and unit information for this benchmark. -type Metric struct { - Name string `bq:"name"` - Unit string `bq:"unit"` - Sample float64 `bq:"sample"` -} - -// InitBigQuery initializes a BigQuery dataset/table in the project. If the dataset/table already exists, it is not duplicated. -func InitBigQuery(ctx context.Context, projectID, datasetID, tableID string, opts []option.ClientOption) error { - client, err := bq.NewClient(ctx, projectID, opts...) - if err != nil { - return fmt.Errorf("failed to initialize client on project %s: %v", projectID, err) - } - defer client.Close() - - dataset := client.Dataset(datasetID) - if err := dataset.Create(ctx, nil); err != nil && !checkDuplicateError(err) { - return fmt.Errorf("failed to create dataset: %s: %v", datasetID, err) - } - - table := dataset.Table(tableID) - schema, err := bq.InferSchema(Suite{}) - if err != nil { - return fmt.Errorf("failed to infer schema: %v", err) - } - - if err := table.Create(ctx, &bq.TableMetadata{Schema: schema}); err != nil && !checkDuplicateError(err) { - return fmt.Errorf("failed to create table: %s: %v", tableID, err) - } - return nil -} - -// AddCondition adds a condition to an existing Benchmark. -func (bm *Benchmark) AddCondition(name, value string) { - bm.Condition = append(bm.Condition, &Condition{ - Name: name, - Value: value, - }) -} - -// AddMetric adds a metric to an existing Benchmark. -func (bm *Benchmark) AddMetric(metricName, unit string, sample float64) { - m := &Metric{ - Name: metricName, - Unit: unit, - Sample: sample, - } - bm.Metric = append(bm.Metric, m) -} - -// NewBenchmark initializes a new benchmark. -func NewBenchmark(name string, iters int) *Benchmark { - return &Benchmark{ - Name: name, - Metric: make([]*Metric, 0), - Condition: []*Condition{ - { - Name: "iterations", - Value: strconv.Itoa(iters), - }, - }, - } -} - -// NewBenchmarkWithMetric creates a new sending to BigQuery, initialized with a -// single iteration and single metric. -func NewBenchmarkWithMetric(name, metric, unit string, value float64) *Benchmark { - b := NewBenchmark(name, 1) - b.AddMetric(metric, unit, value) - return b -} - -// NewSuite initializes a new Suite. -func NewSuite(name string, official bool) *Suite { - return &Suite{ - Name: name, - Timestamp: time.Now().UTC(), - Benchmarks: make([]*Benchmark, 0), - Conditions: make([]*Condition, 0), - Official: official, - } -} - -// SendBenchmarks sends the slice of benchmarks to the BigQuery dataset/table. -func SendBenchmarks(ctx context.Context, suite *Suite, projectID, datasetID, tableID string, opts []option.ClientOption) error { - client, err := bq.NewClient(ctx, projectID, opts...) - if err != nil { - return fmt.Errorf("failed to initialize client on project: %s: %v", projectID, err) - } - defer client.Close() - - uploader := client.Dataset(datasetID).Table(tableID).Uploader() - if err = uploader.Put(ctx, suite); err != nil { - return fmt.Errorf("failed to upload benchmarks %s to project %s, table %s.%s: %v", suite.Name, projectID, datasetID, tableID, err) - } - - return nil -} - -// BigQuery will error "409" for duplicate tables and datasets. -func checkDuplicateError(err error) bool { - return strings.Contains(err.Error(), "googleapi: Error 409: Already Exists") -} diff --git a/tools/checkescape/BUILD b/tools/checkescape/BUILD deleted file mode 100644 index 940538b9e..000000000 --- a/tools/checkescape/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -package(licenses = ["notice"]) - -go_library( - name = "checkescape", - srcs = ["checkescape.go"], - nogo = False, - visibility = ["//tools/nogo:__subpackages__"], - deps = [ - "@org_golang_x_tools//go/analysis:go_default_library", - "@org_golang_x_tools//go/analysis/passes/buildssa:go_default_library", - "@org_golang_x_tools//go/ssa:go_default_library", - ], -) diff --git a/tools/checkescape/checkescape.go b/tools/checkescape/checkescape.go deleted file mode 100644 index c788654a8..000000000 --- a/tools/checkescape/checkescape.go +++ /dev/null @@ -1,923 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package checkescape allows recursive escape analysis for hot paths. -// -// The analysis tracks multiple types of escapes, in two categories. First, -// 'hard' escapes are explicit allocations. Second, 'soft' escapes are -// interface dispatches or dynamic function dispatches; these don't necessarily -// escape but they *may* escape. The analysis is capable of making assertions -// recursively: soft escapes cannot be analyzed in this way, and therefore -// count as escapes for recursive purposes. -// -// The different types of escapes are as follows, with the category in -// parentheses: -// -// heap: A direct allocation is made on the heap (hard). -// builtin: A call is made to a built-in allocation function (hard). -// stack: A stack split as part of a function preamble (soft). -// interface: A call is made via an interface which *may* escape (soft). -// dynamic: A dynamic function is dispatched which *may* escape (soft). -// -// To the use the package, annotate a function-level comment with either the -// line "// +checkescape" or "// +checkescape:OPTION[,OPTION]". In the second -// case, the OPTION field is either a type above, or one of: -// -// local: Escape analysis is limited to local hard escapes only. -// all: All the escapes are included. -// hard: All hard escapes are included. -// -// If the "// +checkescape" annotation is provided, this is equivalent to -// provided the local and hard options. -// -// Some examples of this syntax are: -// -// +checkescape:all - Analyzes for all escapes in this function and all calls. -// +checkescape:local - Analyzes only for default local hard escapes. -// +checkescape:heap - Only analyzes for heap escapes. -// +checkescape:interface,dynamic - Only checks for dynamic calls and interface calls. -// +checkescape - Does the same as +checkescape:local,hard. -// -// Note that all of the above can be inverted by using +mustescape. The -// +checkescape keyword will ensure failure if the class of escape occurs, -// whereas +mustescape will fail if the given class of escape does not occur. -// -// Local exemptions can be made by a comment of the form "// escapes: reason." -// This must appear on the line of the escape and will also apply to callers of -// the function as well (for non-local escape analysis). -package checkescape - -import ( - "bufio" - "bytes" - "flag" - "fmt" - "go/ast" - "go/token" - "go/types" - "io" - "log" - "os" - "os/exec" - "path/filepath" - "strings" - - "golang.org/x/tools/go/analysis" - "golang.org/x/tools/go/analysis/passes/buildssa" - "golang.org/x/tools/go/ssa" -) - -const ( - // magic is the magic annotation. - magic = "// +checkescape" - - // magicParams is the magic annotation with specific parameters. - magicParams = magic + ":" - - // testMagic is the test magic annotation (parameters required). - testMagic = "// +mustescape:" - - // exempt is the exemption annotation. - exempt = "// escapes" -) - -var ( - // Binary is the binary under analysis. - // - // See Reader, below. - binary = flag.String("binary", "", "binary under analysis") - - // Reader is the input stream. - // - // This may be set instead of Binary. - Reader io.Reader - - // objdumpTool is the tool used to dump a binary. - objdumpTool = flag.String("objdump_tool", "", "tool used to dump a binary") -) - -// EscapeReason is an escape reason. -// -// This is a simple enum. -type EscapeReason int - -const ( - allocation EscapeReason = iota - builtin - interfaceInvoke - dynamicCall - stackSplit - unknownPackage - reasonCount // Count for below. -) - -// String returns the string for the EscapeReason. -// -// Note that this also implicitly defines the reverse string -> EscapeReason -// mapping, which is the word before the colon (computed below). -func (e EscapeReason) String() string { - switch e { - case interfaceInvoke: - return "interface: call to potentially allocating function" - case unknownPackage: - return "unknown: no package information available" - case allocation: - return "heap: explicit allocation" - case builtin: - return "builtin: call to potentially allocating builtin" - case dynamicCall: - return "dynamic: call to potentially allocating function" - case stackSplit: - return "stack: possible split on function entry" - default: - panic(fmt.Sprintf("unknown reason: %d", e)) - } -} - -var hardReasons = []EscapeReason{ - allocation, - builtin, -} - -var softReasons = []EscapeReason{ - interfaceInvoke, - unknownPackage, - dynamicCall, - stackSplit, -} - -var allReasons = append(hardReasons, softReasons...) - -var escapeTypes = func() map[string]EscapeReason { - result := make(map[string]EscapeReason) - for _, r := range allReasons { - parts := strings.Split(r.String(), ":") - result[parts[0]] = r // Key before ':'. - } - return result -}() - -// escapingBuiltins are builtins known to escape. -// -// These are lowered at an earlier stage of compilation to explicit function -// calls, but are not available for recursive analysis. -var escapingBuiltins = []string{ - "append", - "makemap", - "newobject", - "mallocgc", -} - -// packageEscapeFacts is the set of all functions in a package, and whether or -// not they recursively pass escape analysis. -// -// All the type names for receivers are encoded in the full key. The key -// represents the fully qualified package and type name used at link time. -// -// Note that each Escapes object is a summary. Local findings may be reported -// using more detailed information. -type packageEscapeFacts struct { - Funcs map[string]Escapes -} - -// AFact implements analysis.Fact.AFact. -func (*packageEscapeFacts) AFact() {} - -// Analyzer includes specific results. -var Analyzer = &analysis.Analyzer{ - Name: "checkescape", - Doc: "escape analysis checks based on +checkescape annotations", - Run: runSelectEscapes, - Requires: []*analysis.Analyzer{buildssa.Analyzer}, - FactTypes: []analysis.Fact{(*packageEscapeFacts)(nil)}, -} - -// EscapeAnalyzer includes all local escape results. -var EscapeAnalyzer = &analysis.Analyzer{ - Name: "checkescape", - Doc: "complete local escape analysis results (requires Analyzer facts)", - Run: runAllEscapes, - Requires: []*analysis.Analyzer{buildssa.Analyzer}, -} - -// LinePosition is a low-resolution token.Position. -// -// This is used to match against possible exemptions placed in the source. -type LinePosition struct { - Filename string - Line int -} - -// String implements fmt.Stringer.String. -func (e LinePosition) String() string { - return fmt.Sprintf("%s:%d", e.Filename, e.Line) -} - -// Simplified returns the simplified name. -func (e LinePosition) Simplified() string { - return fmt.Sprintf("%s:%d", filepath.Base(e.Filename), e.Line) -} - -// CallSite is a single call site. -// -// These can be chained. -type CallSite struct { - LocalPos token.Pos - Resolved LinePosition -} - -// IsValid indicates whether the CallSite is valid or not. -func (cs *CallSite) IsValid() bool { - return cs.LocalPos.IsValid() -} - -// Escapes is a collection of escapes. -// -// We record at most one escape for each reason, but record the number of -// escapes that were omitted. -// -// This object should be used to summarize all escapes for a single line (local -// analysis) or a single function (package facts). -// -// All fields are exported for gob. -type Escapes struct { - CallSites [reasonCount][]CallSite - Details [reasonCount]string - Omitted [reasonCount]int -} - -// add is called by Add and Merge. -func (es *Escapes) add(r EscapeReason, detail string, omitted int, callSites ...CallSite) { - if es.CallSites[r] != nil { - // We will either be replacing the current escape or dropping - // the added one. Either way, we increment omitted by the - // appropriate amount. - es.Omitted[r]++ - // If the callSites in the other is only a single element, then - // we will universally favor this. This provides the cleanest - // set of escapes to summarize, and more importantly: if there - if len(es.CallSites) == 1 || len(callSites) != 1 { - return - } - } - es.Details[r] = detail - es.CallSites[r] = callSites - es.Omitted[r] += omitted -} - -// Add adds a single escape. -func (es *Escapes) Add(r EscapeReason, detail string, callSites ...CallSite) { - es.add(r, detail, 0, callSites...) -} - -// IsEmpty returns true iff this Escapes is empty. -func (es *Escapes) IsEmpty() bool { - for _, cs := range es.CallSites { - if cs != nil { - return false - } - } - return true -} - -// Filter filters out all escapes except those matches the given reasons. -// -// If local is set, then non-local escapes will also be filtered. -func (es *Escapes) Filter(reasons []EscapeReason, local bool) { -FilterReasons: - for r := EscapeReason(0); r < reasonCount; r++ { - for i := 0; i < len(reasons); i++ { - if r == reasons[i] { - continue FilterReasons - } - } - // Zap this reason. - es.CallSites[r] = nil - es.Details[r] = "" - es.Omitted[r] = 0 - } - if !local { - return - } - for r := EscapeReason(0); r < reasonCount; r++ { - // Is does meet our local requirement? - if len(es.CallSites[r]) > 1 { - es.CallSites[r] = nil - es.Details[r] = "" - es.Omitted[r] = 0 - } - } -} - -// MergeWithCall merges these escapes with another. -// -// If callSite is nil, no call is added. -func (es *Escapes) MergeWithCall(other Escapes, callSite CallSite) { - for r := EscapeReason(0); r < reasonCount; r++ { - if other.CallSites[r] != nil { - // Construct our new call chain. - newCallSites := other.CallSites[r] - if callSite.IsValid() { - newCallSites = append([]CallSite{callSite}, newCallSites...) - } - // Add (potentially replacing) the underlying escape. - es.add(r, other.Details[r], other.Omitted[r], newCallSites...) - } - } -} - -// Reportf will call Reportf for each class of escapes. -func (es *Escapes) Reportf(pass *analysis.Pass) { - var b bytes.Buffer // Reused for all escapes. - for r := EscapeReason(0); r < reasonCount; r++ { - if es.CallSites[r] == nil { - continue - } - b.Reset() - fmt.Fprintf(&b, "%s ", r.String()) - if es.Omitted[r] > 0 { - fmt.Fprintf(&b, "(%d omitted) ", es.Omitted[r]) - } - for _, cs := range es.CallSites[r][1:] { - fmt.Fprintf(&b, "→ %s ", cs.Resolved.String()) - } - fmt.Fprintf(&b, "→ %s", es.Details[r]) - pass.Reportf(es.CallSites[r][0].LocalPos, b.String()) - } -} - -// MergeAll merges a sequence of escapes. -func MergeAll(others []Escapes) (es Escapes) { - for _, other := range others { - es.MergeWithCall(other, CallSite{}) - } - return -} - -// loadObjdump reads the objdump output. -// -// This records if there is a call any function for every source line. It is -// used only to remove false positives for escape analysis. The call will be -// elided if escape analysis is able to put the object on the heap exclusively. -// -// Note that the map uses <basename.go>:<line> because that is all that is -// provided in the objdump format. Since this is all local, it is sufficient. -func loadObjdump() (map[string][]string, error) { - var ( - args []string - stdin io.Reader - ) - if *binary != "" { - args = append(args, *binary) - } else if Reader != nil { - stdin = Reader - } else { - // We have no input stream or binary. - return nil, fmt.Errorf("no binary or reader provided") - } - - // Construct our command. - cmd := exec.Command(*objdumpTool, args...) - cmd.Stdin = stdin - cmd.Stderr = os.Stderr - out, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - if err := cmd.Start(); err != nil { - return nil, err - } - - // Identify calls by address or name. Note that this is also - // constructed dynamically below, as we encounted the addresses. - // This is because some of the functions (duffzero) may have - // jump targets in the middle of the function itself. - funcsAllowed := map[string]struct{}{ - "runtime.duffzero": {}, - "runtime.duffcopy": {}, - "runtime.racefuncenter": {}, - "runtime.gcWriteBarrier": {}, - "runtime.retpolineAX": {}, - "runtime.retpolineBP": {}, - "runtime.retpolineBX": {}, - "runtime.retpolineCX": {}, - "runtime.retpolineDI": {}, - "runtime.retpolineDX": {}, - "runtime.retpolineR10": {}, - "runtime.retpolineR11": {}, - "runtime.retpolineR12": {}, - "runtime.retpolineR13": {}, - "runtime.retpolineR14": {}, - "runtime.retpolineR15": {}, - "runtime.retpolineR8": {}, - "runtime.retpolineR9": {}, - "runtime.retpolineSI": {}, - "runtime.stackcheck": {}, - "runtime.settls": {}, - } - addrsAllowed := make(map[string]struct{}) - - // Build the map. - nextFunc := "" // For funcsAllowed. - m := make(map[string][]string) - r := bufio.NewReader(out) -NextLine: - for { - line, err := r.ReadString('\n') - if err != nil && err != io.EOF { - return nil, err - } - fields := strings.Fields(line) - - // Is this an "allowed" function definition? - if len(fields) >= 2 && fields[0] == "TEXT" { - nextFunc = strings.TrimSuffix(fields[1], "(SB)") - if _, ok := funcsAllowed[nextFunc]; !ok { - nextFunc = "" // Don't record addresses. - } - } - if nextFunc != "" && len(fields) > 2 { - // Save the given address (in hex form, as it appears). - addrsAllowed[fields[1]] = struct{}{} - } - - // We recognize lines corresponding to actual code (not the - // symbol name or other metadata) and annotate them if they - // correspond to an explicit CALL instruction. We assume that - // the lack of a CALL for a given line is evidence that escape - // analysis has eliminated an allocation. - // - // Lines look like this (including the first space): - // gohacks_unsafe.go:33 0xa39 488b442408 MOVQ 0x8(SP), AX - if len(fields) >= 5 && line[0] == ' ' { - if !strings.Contains(fields[3], "CALL") { - continue - } - site := fields[0] - target := strings.TrimSuffix(fields[4], "(SB)") - - // Ignore strings containing allowed functions. - if _, ok := funcsAllowed[target]; ok { - continue - } - if _, ok := addrsAllowed[target]; ok { - continue - } - if len(fields) > 5 { - // This may be a future relocation. Some - // objdump versions describe this differently. - // If it contains any of the functions allowed - // above as a string, we let it go. - softTarget := strings.Join(fields[5:], " ") - for name := range funcsAllowed { - if strings.Contains(softTarget, name) { - continue NextLine - } - } - } - - // Does this exist already? - existing, ok := m[site] - if !ok { - existing = make([]string, 0, 1) - } - for _, other := range existing { - if target == other { - continue NextLine - } - } - existing = append(existing, target) - m[site] = existing // Update. - } - if err == io.EOF { - break - } - } - - // Zap any accidental false positives. - final := make(map[string][]string) - for site, calls := range m { - filteredCalls := make([]string, 0, len(calls)) - for _, call := range calls { - if _, ok := addrsAllowed[call]; ok { - continue // Omit this call. - } - filteredCalls = append(filteredCalls, call) - } - final[site] = filteredCalls - } - - // Wait for the dump to finish. - if err := cmd.Wait(); err != nil { - return nil, err - } - - return final, nil -} - -// poser is a type that implements Pos. -type poser interface { - Pos() token.Pos -} - -// runSelectEscapes runs with only select escapes. -func runSelectEscapes(pass *analysis.Pass) (interface{}, error) { - return run(pass, false) -} - -// runAllEscapes runs with all escapes included. -func runAllEscapes(pass *analysis.Pass) (interface{}, error) { - return run(pass, true) -} - -// findReasons extracts reasons from the function. -func findReasons(pass *analysis.Pass, fdecl *ast.FuncDecl) ([]EscapeReason, bool, map[EscapeReason]bool) { - // Is there a comment? - if fdecl.Doc == nil { - return nil, false, nil - } - var ( - reasons []EscapeReason - local bool - testReasons = make(map[EscapeReason]bool) // reason -> local? - ) - // Scan all lines. - found := false - for _, c := range fdecl.Doc.List { - // Does the comment contain a +checkescape line? - if !strings.HasPrefix(c.Text, magic) && !strings.HasPrefix(c.Text, testMagic) { - continue - } - if c.Text == magic { - // Default: hard reasons, local only. - reasons = hardReasons - local = true - } else if strings.HasPrefix(c.Text, magicParams) { - // Extract specific reasons. - types := strings.Split(c.Text[len(magicParams):], ",") - found = true // For below. - for i := 0; i < len(types); i++ { - if types[i] == "local" { - // Limit search to local escapes. - local = true - } else if types[i] == "all" { - // Append all reasons. - reasons = append(reasons, allReasons...) - } else if types[i] == "hard" { - // Append all hard reasons. - reasons = append(reasons, hardReasons...) - } else { - r, ok := escapeTypes[types[i]] - if !ok { - // This is not a valid escape reason. - pass.Reportf(fdecl.Pos(), "unknown reason: %v", types[i]) - continue - } - reasons = append(reasons, r) - } - } - } else if strings.HasPrefix(c.Text, testMagic) { - types := strings.Split(c.Text[len(testMagic):], ",") - local := false - for i := 0; i < len(types); i++ { - if types[i] == "local" { - local = true - } else { - r, ok := escapeTypes[types[i]] - if !ok { - // This is not a valid escape reason. - pass.Reportf(fdecl.Pos(), "unknown reason: %v", types[i]) - continue - } - if v, ok := testReasons[r]; ok && v { - // Already registered as local. - continue - } - testReasons[r] = local - } - } - } - } - if len(reasons) == 0 && found { - // A magic annotation was provided, but no reasons. - pass.Reportf(fdecl.Pos(), "no reasons provided") - } - return reasons, local, testReasons -} - -// run performs the analysis. -func run(pass *analysis.Pass, localEscapes bool) (interface{}, error) { - calls, callsErr := loadObjdump() - if callsErr != nil { - // Note that if this analysis fails, then we don't actually - // fail the analyzer itself. We simply report every possible - // escape. In most cases this will work just fine. - log.Printf("WARNING: unable to load objdump: %v", callsErr) - } - allEscapes := make(map[string][]Escapes) - mergedEscapes := make(map[string]Escapes) - linePosition := func(inst, parent poser) LinePosition { - p := pass.Fset.Position(inst.Pos()) - if (p.Filename == "" || p.Line == 0) && parent != nil { - p = pass.Fset.Position(parent.Pos()) - } - return LinePosition{ - Filename: p.Filename, - Line: p.Line, - } - } - callSite := func(inst ssa.Instruction) CallSite { - return CallSite{ - LocalPos: inst.Pos(), - Resolved: linePosition(inst, inst.Parent()), - } - } - hasCall := func(inst poser) (string, bool) { - p := linePosition(inst, nil) - if callsErr != nil { - // See above: we don't have access to the binary - // itself, so need to include every possible call. - return fmt.Sprintf("(possible, unable to load objdump: %v)", callsErr), true - } - s, ok := calls[p.Simplified()] - if !ok { - return "", false - } - // Join all calls together. - return strings.Join(s, " or "), true - } - state := pass.ResultOf[buildssa.Analyzer].(*buildssa.SSA) - - // Build the exception list. - exemptions := make(map[LinePosition]string) - for _, f := range pass.Files { - for _, cg := range f.Comments { - for _, c := range cg.List { - p := pass.Fset.Position(c.Slash) - if strings.HasPrefix(strings.ToLower(c.Text), exempt) { - exemptions[LinePosition{ - Filename: p.Filename, - Line: p.Line, - }] = c.Text[len(exempt):] - } - } - } - } - - var loadFunc func(*ssa.Function) Escapes // Used below. - analyzeInstruction := func(inst ssa.Instruction) (es Escapes) { - cs := callSite(inst) - if _, ok := exemptions[cs.Resolved]; ok { - return // No escape. - } - switch x := inst.(type) { - case *ssa.Call: - if x.Call.IsInvoke() { - // This is an interface dispatch. There is no - // way to know if this is actually escaping or - // not, since we don't know the underlying - // type. - call, _ := hasCall(inst) - es.Add(interfaceInvoke, call, cs) - return - } - switch x := x.Call.Value.(type) { - case *ssa.Function: - if x.Pkg == nil { - // Can't resolve the package. - es.Add(unknownPackage, "no package", cs) - return - } - - // Is this a local function? If yes, call the - // function to load the local function. The - // local escapes are the escapes found in the - // local function. - if x.Pkg.Pkg == pass.Pkg { - es.MergeWithCall(loadFunc(x), cs) - return - } - - // If this package is the atomic package, the implementation - // may be replaced by instrinsics that don't have analysis. - if x.Pkg.Pkg.Path() == "sync/atomic" { - return - } - - // Recursively collect information. - var imp packageEscapeFacts - if !pass.ImportPackageFact(x.Pkg.Pkg, &imp) { - // Unable to import the dependency; we must - // declare these as escaping. - es.Add(unknownPackage, "no analysis", cs) - return - } - - // The escapes of this instruction are the - // escapes of the called function directly. - // Note that this may record many escapes. - es.MergeWithCall(imp.Funcs[x.RelString(x.Pkg.Pkg)], cs) - return - case *ssa.Builtin: - // Ignore elided escapes. - if _, has := hasCall(inst); !has { - return - } - - // Check if the builtin is escaping. - for _, name := range escapingBuiltins { - if x.Name() == name { - es.Add(builtin, name, cs) - return - } - } - default: - // All dynamic calls are counted as soft - // escapes. They are similar to interface - // dispatches. We cannot actually look up what - // this refers to using static analysis alone. - call, _ := hasCall(inst) - es.Add(dynamicCall, call, cs) - } - case *ssa.Alloc: - // Ignore non-heap allocations. - if !x.Heap { - return - } - - // Ignore elided escapes. - call, has := hasCall(inst) - if !has { - return - } - - // This is a real heap allocation. - es.Add(allocation, call, cs) - case *ssa.MakeMap: - es.Add(builtin, "makemap", cs) - case *ssa.MakeSlice: - es.Add(builtin, "makeslice", cs) - case *ssa.MakeClosure: - es.Add(builtin, "makeclosure", cs) - case *ssa.MakeChan: - es.Add(builtin, "makechan", cs) - } - return - } - - var analyzeBasicBlock func(*ssa.BasicBlock) []Escapes // Recursive. - analyzeBasicBlock = func(block *ssa.BasicBlock) (rval []Escapes) { - for _, inst := range block.Instrs { - if es := analyzeInstruction(inst); !es.IsEmpty() { - rval = append(rval, es) - } - } - return - } - - loadFunc = func(fn *ssa.Function) Escapes { - // Is this already available? - name := fn.RelString(pass.Pkg) - if es, ok := mergedEscapes[name]; ok { - return es - } - - // In the case of a true cycle, we assume that the current - // function itself has no escapes. - // - // When evaluating the function again, the proper escapes will - // be filled in here. - allEscapes[name] = nil - mergedEscapes[name] = Escapes{} - - // Perform the basic analysis. - var es []Escapes - if fn.Recover != nil { - es = append(es, analyzeBasicBlock(fn.Recover)...) - } - for _, block := range fn.Blocks { - es = append(es, analyzeBasicBlock(block)...) - } - - // Check for a stack split. - if call, has := hasCall(fn); has { - var ss Escapes - ss.Add(stackSplit, call, CallSite{ - LocalPos: fn.Pos(), - Resolved: linePosition(fn, fn.Parent()), - }) - es = append(es, ss) - } - - // Save the result and return. - // - // Note that we merge the result when saving to the facts. It - // doesn't really matter the specific escapes, as long as we - // have recorded all the appropriate classes of escapes. - summary := MergeAll(es) - allEscapes[name] = es - mergedEscapes[name] = summary - return summary - } - - // Complete all local functions. - for _, fn := range state.SrcFuncs { - loadFunc(fn) - } - - if !localEscapes { - // Export all findings for future packages. We only do this in - // non-local escapes mode, and expect to run this analysis - // after the SelectAnalysis. - pass.ExportPackageFact(&packageEscapeFacts{ - Funcs: mergedEscapes, - }) - } - - // Scan all functions for violations. - for _, f := range pass.Files { - // Scan all declarations. - for _, decl := range f.Decls { - // Function declaration? - fdecl, ok := decl.(*ast.FuncDecl) - if !ok { - continue - } - var ( - reasons []EscapeReason - local bool - testReasons map[EscapeReason]bool - ) - if localEscapes { - // Find all hard escapes. - reasons = hardReasons - } else { - // Find all declared reasons. - reasons, local, testReasons = findReasons(pass, fdecl) - } - - // Scan for matches. - fn := pass.TypesInfo.Defs[fdecl.Name].(*types.Func) - fv := state.Pkg.Prog.FuncValue(fn) - if fv == nil { - continue - } - name := fv.RelString(pass.Pkg) - all, allOk := allEscapes[name] - merged, mergedOk := mergedEscapes[name] - if !allOk || !mergedOk { - pass.Reportf(fdecl.Pos(), "internal error: function %s not found.", name) - continue - } - - // Filter reasons and report. - // - // For the findings, we use all escapes. - for _, es := range all { - es.Filter(reasons, local) - es.Reportf(pass) - } - - // Scan for test (required) matches. - // - // For tests we need only the merged escapes. - testReasonsFound := make(map[EscapeReason]bool) - for r := EscapeReason(0); r < reasonCount; r++ { - if merged.CallSites[r] == nil { - continue - } - // Is this local? - wantLocal, ok := testReasons[r] - isLocal := len(merged.CallSites[r]) == 1 - testReasonsFound[r] = isLocal - if !ok { - continue - } - if isLocal == wantLocal { - delete(testReasons, r) - } - } - for reason, local := range testReasons { - // We didn't find the escapes we wanted. - pass.Reportf(fdecl.Pos(), fmt.Sprintf("testescapes not found: reason=%s, local=%t", reason, local)) - } - if len(testReasons) > 0 { - // Report for debugging. - merged.Reportf(pass) - } - } - } - - return nil, nil -} diff --git a/tools/checkescape/test1/BUILD b/tools/checkescape/test1/BUILD deleted file mode 100644 index 783403247..000000000 --- a/tools/checkescape/test1/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -package(licenses = ["notice"]) - -go_library( - name = "test1", - srcs = ["test1.go"], - visibility = ["//tools/checkescape/test2:__pkg__"], -) diff --git a/tools/checkescape/test1/test1.go b/tools/checkescape/test1/test1.go deleted file mode 100644 index f46eba39b..000000000 --- a/tools/checkescape/test1/test1.go +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package test1 is a test package. -package test1 - -import ( - "fmt" -) - -// Interface is a generic interface. -type Interface interface { - Foo() -} - -// Type is a concrete implementation of Interface. -type Type struct { - A uint64 - B uint64 -} - -// Foo implements Interface.Foo. -//go:nosplit -func (t Type) Foo() { - fmt.Printf("%v", t) // Never executed. -} - -// InterfaceFunction is passed an interface argument. -// +checkescape:all,hard -//go:nosplit -func InterfaceFunction(i Interface) { - // Do nothing; exported for tests. -} - -// TypeFunction is passed a concrete pointer argument. -// +checkesacape:all,hard -//go:nosplit -func TypeFunction(t *Type) { -} - -// BuiltinMap creates a new map. -// +mustescape:local,builtin -//go:noinline -//go:nosplit -func BuiltinMap(x int) map[string]bool { - return make(map[string]bool) -} - -// +mustescape:builtin -//go:noinline -//go:nosplit -func builtinMapRec(x int) map[string]bool { - return BuiltinMap(x) -} - -// BuiltinClosure returns a closure around x. -// +mustescape:local,builtin -//go:noinline -//go:nosplit -func BuiltinClosure(x int) func() { - return func() { - fmt.Printf("%v", x) - } -} - -// +mustescape:builtin -//go:noinline -//go:nosplit -func builtinClosureRec(x int) func() { - return BuiltinClosure(x) -} - -// BuiltinMakeSlice makes a new slice. -// +mustescape:local,builtin -//go:noinline -//go:nosplit -func BuiltinMakeSlice(x int) []byte { - return make([]byte, x) -} - -// +mustescape:builtin -//go:noinline -//go:nosplit -func builtinMakeSliceRec(x int) []byte { - return BuiltinMakeSlice(x) -} - -// BuiltinAppend calls append on a slice. -// +mustescape:local,builtin -//go:noinline -//go:nosplit -func BuiltinAppend(x []byte) []byte { - return append(x, 0) -} - -// +mustescape:builtin -//go:noinline -//go:nosplit -func builtinAppendRec() []byte { - return BuiltinAppend(nil) -} - -// BuiltinChan makes a channel. -// +mustescape:local,builtin -//go:noinline -//go:nosplit -func BuiltinChan() chan int { - return make(chan int) -} - -// +mustescape:builtin -//go:noinline -//go:nosplit -func builtinChanRec() chan int { - return BuiltinChan() -} - -// Heap performs an explicit heap allocation. -// +mustescape:local,heap -//go:noinline -//go:nosplit -func Heap() *Type { - var t Type - return &t -} - -// +mustescape:heap -//go:noinline -//go:nosplit -func heapRec() *Type { - return Heap() -} - -// Dispatch dispatches via an interface. -// +mustescape:local,interface -//go:noinline -//go:nosplit -func Dispatch(i Interface) { - i.Foo() -} - -// +mustescape:interface -//go:noinline -//go:nosplit -func dispatchRec(i Interface) { - Dispatch(i) -} - -// Dynamic invokes a dynamic function. -// +mustescape:local,dynamic -//go:noinline -//go:nosplit -func Dynamic(f func()) { - f() -} - -// +mustescape:dynamic -//go:noinline -//go:nosplit -func dynamicRec(f func()) { - Dynamic(f) -} - -//go:noinline -//go:nosplit -func internalFunc() { -} - -// Split includes a guaranteed stack split. -// +mustescape:local,stack -//go:noinline -func Split() { - internalFunc() -} - -// +mustescape:stack -//go:noinline -//go:nosplit -func splitRec() { - Split() -} diff --git a/tools/checkescape/test2/BUILD b/tools/checkescape/test2/BUILD deleted file mode 100644 index 5a11e4b43..000000000 --- a/tools/checkescape/test2/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -package(licenses = ["notice"]) - -go_library( - name = "test2", - srcs = ["test2.go"], - deps = ["//tools/checkescape/test1"], -) diff --git a/tools/checkescape/test2/test2.go b/tools/checkescape/test2/test2.go deleted file mode 100644 index 067d5a1f4..000000000 --- a/tools/checkescape/test2/test2.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package test2 is a test package that imports test1. -package test2 - -import ( - "gvisor.dev/gvisor/tools/checkescape/test1" -) - -// +checkescape:all -//go:nosplit -func interfaceFunctionCrossPkg() { - var i test1.Interface - test1.InterfaceFunction(i) -} - -// +checkesacape:all -//go:nosplit -func typeFunctionCrossPkg() { - var t test1.Type - test1.TypeFunction(&t) -} - -// +mustescape:builtin -//go:noinline -func builtinMapCrossPkg(x int) map[string]bool { - return test1.BuiltinMap(x) -} - -// +mustescape:builtin -//go:noinline -func builtinClosureCrossPkg(x int) func() { - return test1.BuiltinClosure(x) -} - -// +mustescape:builtin -//go:noinline -func builtinMakeSliceCrossPkg(x int) []byte { - return test1.BuiltinMakeSlice(x) -} - -// +mustescape:builtin -//go:noinline -func builtinAppendCrossPkg() []byte { - return test1.BuiltinAppend(nil) -} - -// +mustescape:builtin -//go:noinline -func builtinChanCrossPkg() chan int { - return test1.BuiltinChan() -} - -// +mustescape:heap -//go:noinline -func heapCrossPkg() *test1.Type { - return test1.Heap() -} - -// +mustescape:interface -//go:noinline -func dispatchCrossPkg(i test1.Interface) { - test1.Dispatch(i) -} - -// +mustescape:dynamic -//go:noinline -func dynamicCrossPkg(f func()) { - test1.Dynamic(f) -} - -// +mustescape:stack -//go:noinline -//go:nosplit -func splitCrosssPkt() { - test1.Split() -} diff --git a/tools/checklocks/BUILD b/tools/checklocks/BUILD deleted file mode 100644 index 7d4c63dc7..000000000 --- a/tools/checklocks/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -package(licenses = ["notice"]) - -go_library( - name = "checklocks", - srcs = ["checklocks.go"], - nogo = False, - visibility = ["//tools/nogo:__subpackages__"], - deps = [ - "//pkg/log", - "@org_golang_x_tools//go/analysis:go_default_library", - "@org_golang_x_tools//go/analysis/passes/buildssa:go_default_library", - "@org_golang_x_tools//go/ssa:go_default_library", - ], -) diff --git a/tools/checklocks/README.md b/tools/checklocks/README.md deleted file mode 100644 index dfb0275ab..000000000 --- a/tools/checklocks/README.md +++ /dev/null @@ -1,142 +0,0 @@ -# CheckLocks Analyzer - -<!--* freshness: { owner: 'gvisor-eng' reviewed: '2020-10-05' } *--> - -Checklocks is a nogo analyzer that at compile time uses Go's static analysis -tools to identify and flag cases where a field that is guarded by a mutex in the -same struct is accessed outside of a mutex lock. - -The analyzer relies on explicit '// +checklocks:<mutex-name>' kind of -annotations to identify fields that should be checked for access. - -Individual struct members may be protected by annotations that indicate how they -must be accessed. These annotations are of the form: - -```go -type foo struct { - mu sync.Mutex - // +checklocks:mu - bar int - - foo int // No annotation on foo means it's not guarded by mu. - - secondMu sync.Mutex - - // Multiple annotations indicate that both must be held but the - // checker does not assert any lock ordering. - // +checklocks:secondMu - // +checklocks:mu - foobar int -} -``` - -The checklocks annotation may also apply to functions. For example: - -```go -// +checklocks:f.mu -func (f *foo) doThingLocked() { } -``` - -This will check that the "f.mu" is locked for any calls, where possible. - -In case of functions which initialize structs that may have annotations one can -use the following annotation on the function to disable reporting by the lock -checker. The lock checker will still track any mutexes acquired or released but -won't report any failures for this function for unguarded field access. - -```go -// +checklocks:ignore -func newXXX() *X { -... -} -``` - -***The checker treats both 'sync.Mutex' and 'sync.RWMutex' identically, i.e, as -a sync.Mutex. The checker does not distinguish between read locks vs. exclusive -locks and treats all locks as exclusive locks***. - -For cases the checker is able to correctly handle today please see test/test.go. - -The checklocks check also flags any invalid annotations where the mutex -annotation refers either to something that is not a 'sync.Mutex' or -'sync.RWMutex' or where the field does not exist at all. This will prevent the -annotations from becoming stale over time as fields are renamed, etc. - -# Currently not supported - -1. The analyzer does not correctly handle deferred functions. e.g The following - code is not correctly checked by the analyzer. The defer call is never - evaluated. As a result if the lock was to be say unlocked twice via deferred - functions it would not be caught by the analyzer. - - Similarly deferred anonymous functions are not evaluated either. - -```go -type A struct { - mu sync.Mutex - - // +checklocks:mu - x int -} - -func abc() { - var a A - a.mu.Lock() - defer a.mu.Unlock() - defer a.mu.Unlock() - a.x = 1 -} -``` - -1. Anonymous functions are not correctly evaluated. The analyzer does not - currently support specifying annotations on anonymous functions as a result - evaluation of a function that accesses protected fields will fail. - -```go -type A struct { - mu sync.Mutex - - // +checklocks:mu - x int -} - -func abc() { - var a A - f := func() { a.x = 1 } <=== This line will be flagged by analyzer - a.mu.Lock() - f() - a.mu.Unlock() -} - -``` - -# Explicitly Not Supported - -1. Checking for embedded mutexes as sync.Locker rather than directly as - 'sync.Mutex'. In other words, the checker will not track mutex Lock and - Unlock() methods where the mutex is behind an interface dispatch. - -An example that we won't handle is shown below (this in fact will fail to -build): - -```go -type A struct { - mu sync.Locker - - // +checklocks:mu - x int -} - -func abc() { - mu sync.Mutex - a := A{mu: &mu} - a.x = 1 // This won't be flagged by copylocks checker. -} - -``` - -1. The checker will not support guards on anything other than the cases - described above. For example, global mutexes cannot be referred to by - checklocks. Only struct members can be used. - -2. The checker will not support checking for lock ordering violations. diff --git a/tools/checklocks/checklocks.go b/tools/checklocks/checklocks.go deleted file mode 100644 index 1e877d394..000000000 --- a/tools/checklocks/checklocks.go +++ /dev/null @@ -1,758 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package checklocks performs lock analysis to identify and flag unprotected -// access to field annotated with a '// +checklocks:<mutex-name>' annotation. -// -// For detailed ussage refer to README.md in the same directory. -package checklocks - -import ( - "bytes" - "fmt" - "go/ast" - "go/token" - "go/types" - "reflect" - "regexp" - "strconv" - "strings" - - "golang.org/x/tools/go/analysis" - "golang.org/x/tools/go/analysis/passes/buildssa" - "golang.org/x/tools/go/ssa" - "gvisor.dev/gvisor/pkg/log" -) - -const ( - checkLocksAnnotation = "// +checklocks:" - checkLocksIgnore = "// +checklocksignore" - checkLocksFail = "// +checklocksfail" -) - -// Analyzer is the main entrypoint. -var Analyzer = &analysis.Analyzer{ - Name: "checklocks", - Doc: "checks lock preconditions on functions and fields", - Run: run, - Requires: []*analysis.Analyzer{buildssa.Analyzer}, - FactTypes: []analysis.Fact{(*lockFieldFacts)(nil), (*lockFunctionFacts)(nil)}, -} - -// lockFieldFacts apply on every struct field protected by a lock or that is a -// lock. -type lockFieldFacts struct { - // GuardedBy tracks the names and field numbers that guard this field. - GuardedBy map[string]int - - // IsMutex is true if the field is of type sync.Mutex. - IsMutex bool - - // IsRWMutex is true if the field is of type sync.RWMutex. - IsRWMutex bool - - // FieldNumber is the number of this field in the struct. - FieldNumber int -} - -// AFact implements analysis.Fact.AFact. -func (*lockFieldFacts) AFact() {} - -type functionGuard struct { - // ParameterNumber is the index of the object that contains the guarding mutex. - // This is required during SSA analysis as field names and parameters names are - // not available in SSA. For example, from the example below ParameterNumber would - // be 1 and FieldNumber would correspond to the field number of 'mu' within b's type. - // - // //+checklocks:b.mu - // func (a *A) method(b *B, c *C) { - // ... - // } - ParameterNumber int - - // FieldNumber is the field index of the mutex in the parameter's struct - // type. Refer to example above for more details. - FieldNumber int -} - -// lockFunctionFacts apply on every method. -type lockFunctionFacts struct { - // GuardedBy tracks the names and number of parameter (including receiver) - // lockFuncfields that guard calls to this function. - // The key is the name specified in the checklocks annotation. e.g given - // the following code. - // ``` - // type A struct { - // mu sync.Mutex - // a int - // } - // - // // +checklocks:a.mu - // func xyz(a *A) {..} - // ``` - // - // '`+checklocks:a.mu' will result in an entry in this map as shown below. - // GuardedBy: {"a.mu" => {ParameterNumber: 0, FieldNumber: 0} - GuardedBy map[string]functionGuard -} - -// AFact implements analysis.Fact.AFact. -func (*lockFunctionFacts) AFact() {} - -type positionKey string - -// toPositionKey converts from a token.Position to a key we can use to track -// failures as the position of the failure annotation is not the same as the -// position of the actual failure (different column/offsets). Hence we ignore -// these fields and only use the file/line numbers to track failures. -func toPositionKey(position token.Position) positionKey { - return positionKey(fmt.Sprintf("%s:%d", position.Filename, position.Line)) -} - -type failData struct { - pos token.Pos - count int -} - -func (f failData) String() string { - return fmt.Sprintf("pos: %d, count: %d", f.pos, f.count) -} - -type passContext struct { - pass *analysis.Pass - - // exemptions tracks functions that should be exempted from lock checking due - // to '// +checklocksignore' annotation. - exemptions map[types.Object]struct{} - - failures map[positionKey]*failData -} - -var ( - mutexRE = regexp.MustCompile("((.*/)|^)sync.(CrossGoroutineMutex|Mutex)") - rwMutexRE = regexp.MustCompile("((.*/)|^)sync.(CrossGoroutineRWMutex|RWMutex)") -) - -func (pc *passContext) extractFieldAnnotations(field *ast.Field, fieldType *types.Var) *lockFieldFacts { - s := fieldType.Type().String() - // We use HasSuffix below because fieldType can be fully qualified with the - // package name eg for the gvisor sync package mutex fields have the type: - // "<package path>/sync/sync.Mutex" - switch { - case mutexRE.Match([]byte(s)): - return &lockFieldFacts{IsMutex: true} - case rwMutexRE.Match([]byte(s)): - return &lockFieldFacts{IsRWMutex: true} - default: - } - if field.Doc == nil { - return nil - } - fieldFacts := &lockFieldFacts{GuardedBy: make(map[string]int)} - for _, l := range field.Doc.List { - if strings.HasPrefix(l.Text, checkLocksAnnotation) { - guardName := strings.TrimPrefix(l.Text, checkLocksAnnotation) - if _, ok := fieldFacts.GuardedBy[guardName]; ok { - pc.pass.Reportf(field.Pos(), "annotation %s specified more than once.", l.Text) - continue - } - fieldFacts.GuardedBy[guardName] = -1 - } - } - - return fieldFacts -} - -func (pc *passContext) findField(v ssa.Value, fieldNumber int) types.Object { - structType, ok := v.Type().Underlying().(*types.Struct) - if !ok { - structType = v.Type().Underlying().(*types.Pointer).Elem().Underlying().(*types.Struct) - } - return structType.Field(fieldNumber) -} - -// findAndExportStructFacts finds any struct fields that are annotated with the -// "// +checklocks:" annotation and exports relevant facts about the fields to -// be used in later analysis. -func (pc *passContext) findAndExportStructFacts(ss *ast.StructType, structType *types.Struct) { - type fieldRef struct { - fieldObj *types.Var - facts *lockFieldFacts - } - mutexes := make(map[string]*fieldRef) - rwMutexes := make(map[string]*fieldRef) - guardedFields := make(map[string]*fieldRef) - for i, field := range ss.Fields.List { - fieldObj := structType.Field(i) - fieldFacts := pc.extractFieldAnnotations(field, fieldObj) - if fieldFacts == nil { - continue - } - fieldFacts.FieldNumber = i - - ref := &fieldRef{fieldObj, fieldFacts} - if fieldFacts.IsMutex { - mutexes[fieldObj.Name()] = ref - } - if fieldFacts.IsRWMutex { - rwMutexes[fieldObj.Name()] = ref - } - if len(fieldFacts.GuardedBy) != 0 { - guardedFields[fieldObj.Name()] = ref - } - } - - // Export facts about all mutexes. - for _, f := range mutexes { - pc.pass.ExportObjectFact(f.fieldObj, f.facts) - } - // Export facts about all rwMutexes. - for _, f := range rwMutexes { - pc.pass.ExportObjectFact(f.fieldObj, f.facts) - } - - // Validate that guarded fields annotations refer to actual mutexes or - // rwMutexes in the struct. - for _, gf := range guardedFields { - for g := range gf.facts.GuardedBy { - if f, ok := mutexes[g]; ok { - gf.facts.GuardedBy[g] = f.facts.FieldNumber - } else if f, ok := rwMutexes[g]; ok { - gf.facts.GuardedBy[g] = f.facts.FieldNumber - } else { - pc.maybeFail(gf.fieldObj.Pos(), false /* isExempted */, "invalid mutex guard, no such mutex %s in struct %s", g, structType.String()) - continue - } - // Export guarded field fact. - pc.pass.ExportObjectFact(gf.fieldObj, gf.facts) - } - } -} - -func (pc *passContext) findAndExportFuncFacts(d *ast.FuncDecl) { - log.Debugf("finding and exporting function facts\n") - // for each function definition, check for +checklocks:mu annotation, which - // means that the function must be called with that lock held. - fnObj := pc.pass.TypesInfo.ObjectOf(d.Name) - funcFacts := lockFunctionFacts{GuardedBy: make(map[string]functionGuard)} - var ( - ignore bool - ignorePos token.Pos - ) - -outerLoop: - for _, l := range d.Doc.List { - if strings.HasPrefix(l.Text, checkLocksIgnore) { - pc.exemptions[fnObj] = struct{}{} - ignore = true - ignorePos = l.Pos() - continue - } - if strings.HasPrefix(l.Text, checkLocksAnnotation) { - guardName := strings.TrimPrefix(l.Text, checkLocksAnnotation) - if _, ok := funcFacts.GuardedBy[guardName]; ok { - pc.pass.Reportf(l.Pos(), "annotation %s specified more than once.", l.Text) - continue - } - - found := false - x := strings.Split(guardName, ".") - if len(x) != 2 { - pc.pass.Reportf(l.Pos(), "checklocks mutex annotation should be of the form 'a.b'") - continue - } - paramName, fieldName := x[0], x[1] - log.Debugf("paramName: %s, fieldName: %s", paramName, fieldName) - var paramList []*ast.Field - if d.Recv != nil { - paramList = append(paramList, d.Recv.List...) - } - if d.Type.Params != nil { - paramList = append(paramList, d.Type.Params.List...) - } - for paramNum, field := range paramList { - log.Debugf("field names: %+v", field.Names) - if len(field.Names) == 0 { - log.Debugf("skipping because parameter is unnamed", paramName) - continue - } - nameExists := false - for _, name := range field.Names { - if name.Name == paramName { - nameExists = true - } - } - if !nameExists { - log.Debugf("skipping because parameter name(s) does not match : %s", paramName) - continue - } - ptrType, ok := pc.pass.TypesInfo.TypeOf(field.Type).Underlying().(*types.Pointer) - if !ok { - // Since mutexes cannot be copied we only care about parameters that - // are pointer types when checking for guards. - pc.pass.Reportf(l.Pos(), "annotation %s incorrectly specified, parameter name does not refer to a pointer type", l.Text) - continue outerLoop - } - - structType, ok := ptrType.Elem().Underlying().(*types.Struct) - if !ok { - pc.pass.Reportf(l.Pos(), "annotation %s incorrectly specified, parameter name does not refer to a pointer to a struct", l.Text) - continue outerLoop - } - - for i := 0; i < structType.NumFields(); i++ { - if structType.Field(i).Name() == fieldName { - var fieldFacts lockFieldFacts - pc.pass.ImportObjectFact(structType.Field(i), &fieldFacts) - if !fieldFacts.IsMutex && !fieldFacts.IsRWMutex { - pc.pass.Reportf(l.Pos(), "field %s of param %s is not a mutex or an rwmutex", paramName, structType.Field(i)) - continue outerLoop - } - funcFacts.GuardedBy[guardName] = functionGuard{ParameterNumber: paramNum, FieldNumber: i} - found = true - continue outerLoop - } - } - if !found { - pc.pass.Reportf(l.Pos(), "annotation refers to a non-existent field %s in %s", guardName, structType) - continue outerLoop - } - } - if !found { - pc.pass.Reportf(l.Pos(), "annotation refers to a non-existent parameter %s", paramName) - } - } - } - - if len(funcFacts.GuardedBy) == 0 { - return - } - if ignore { - pc.pass.Reportf(ignorePos, "//+checklocksignore cannot be specified with other annotations on the function") - } - funcObj, ok := pc.pass.TypesInfo.Defs[d.Name].(*types.Func) - if !ok { - panic(fmt.Sprintf("function type information missing for %+v", d)) - } - log.Debugf("export fact for d: %+v, funcObj: %+v, funcFacts: %+v\n", d, funcObj, funcFacts) - pc.pass.ExportObjectFact(funcObj, &funcFacts) -} - -type mutexState struct { - // lockedMutexes is used to track which mutexes in a given struct are - // currently locked using the field number of the mutex as the key. - lockedMutexes map[int]struct{} -} - -// locksHeld tracks all currently held locks. -type locksHeld struct { - locks map[ssa.Value]mutexState -} - -// Same returns true if the locks held by other and l are the same. -func (l *locksHeld) Same(other *locksHeld) bool { - return reflect.DeepEqual(l.locks, other.locks) -} - -// Copy creates a copy of all the lock state held by l. -func (l *locksHeld) Copy() *locksHeld { - out := &locksHeld{locks: make(map[ssa.Value]mutexState)} - for ssaVal, mState := range l.locks { - newLM := make(map[int]struct{}) - for k, v := range mState.lockedMutexes { - newLM[k] = v - } - out.locks[ssaVal] = mutexState{lockedMutexes: newLM} - } - return out -} - -func isAlias(first, second ssa.Value) bool { - if first == second { - return true - } - switch x := first.(type) { - case *ssa.Field: - if y, ok := second.(*ssa.Field); ok { - return x.Field == y.Field && isAlias(x.X, y.X) - } - case *ssa.FieldAddr: - if y, ok := second.(*ssa.FieldAddr); ok { - return x.Field == y.Field && isAlias(x.X, y.X) - } - case *ssa.Index: - if y, ok := second.(*ssa.Index); ok { - return isAlias(x.Index, y.Index) && isAlias(x.X, y.X) - } - case *ssa.IndexAddr: - if y, ok := second.(*ssa.IndexAddr); ok { - return isAlias(x.Index, y.Index) && isAlias(x.X, y.X) - } - case *ssa.UnOp: - if y, ok := second.(*ssa.UnOp); ok { - return isAlias(x.X, y.X) - } - } - return false -} - -// checkBasicBlocks traverses the control flow graph starting at a set of given -// block and checks each instruction for allowed operations. -// -// funcFact are the exported facts for the enclosing function for these basic -// blocks. -func (pc *passContext) checkBasicBlocks(blocks []*ssa.BasicBlock, recoverBlock *ssa.BasicBlock, fn *ssa.Function, funcFact lockFunctionFacts) { - if len(blocks) == 0 { - return - } - - // mutexes is used to track currently locked sync.Mutexes/sync.RWMutexes for a - // given *struct identified by ssa.Value. - seen := make(map[*ssa.BasicBlock]*locksHeld) - var scan func(block *ssa.BasicBlock, parent *locksHeld) - scan = func(block *ssa.BasicBlock, parent *locksHeld) { - _, isExempted := pc.exemptions[block.Parent().Object()] - if oldLocksHeld, ok := seen[block]; ok { - if oldLocksHeld.Same(parent) { - return - } - pc.maybeFail(block.Instrs[0].Pos(), isExempted, "failure entering a block %+v with different sets of lock held, oldLocks: %+v, parentLocks: %+v", block, oldLocksHeld, parent) - return - } - seen[block] = parent - var lh = parent.Copy() - for _, inst := range block.Instrs { - pc.checkInstruction(inst, isExempted, lh) - } - for _, b := range block.Succs { - scan(b, lh) - } - } - - // Initialize lh with any preconditions that require locks to be held for the - // method to be invoked. - lh := &locksHeld{locks: make(map[ssa.Value]mutexState)} - for _, fg := range funcFact.GuardedBy { - // The first is the method object itself so we skip that when looking - // for receiver/function parameters. - log.Debugf("fn: %s, fn.Operands() == %+v", fn, fn.Operands(nil)) - r := fn.Params[fg.ParameterNumber] - guardObj := findField(r, fg.FieldNumber) - var fieldFacts lockFieldFacts - pc.pass.ImportObjectFact(guardObj, &fieldFacts) - if fieldFacts.IsMutex || fieldFacts.IsRWMutex { - m, ok := lh.locks[r] - if !ok { - m = mutexState{lockedMutexes: make(map[int]struct{})} - lh.locks[r] = m - } - m.lockedMutexes[fieldFacts.FieldNumber] = struct{}{} - } else { - panic(fmt.Sprintf("function: %+v has an invalid guard that is not a mutex: %+v", fn, guardObj)) - } - } - - // Start scanning from the first basic block. - scan(blocks[0], lh) - - // Validate that all blocks were touched. - for _, b := range blocks { - if _, ok := seen[b]; !ok && b != recoverBlock { - panic(fmt.Sprintf("block %+v was not visited during checkBasicBlocks", b)) - } - } -} - -func (pc *passContext) checkInstruction(inst ssa.Instruction, isExempted bool, lh *locksHeld) { - log.Debugf("checking instruction: %s, isExempted: %t", inst, isExempted) - switch x := inst.(type) { - case *ssa.Field: - pc.checkFieldAccess(inst, x.X, x.Field, isExempted, lh) - case *ssa.FieldAddr: - pc.checkFieldAccess(inst, x.X, x.Field, isExempted, lh) - case *ssa.Call: - pc.checkFunctionCall(x, isExempted, lh) - } -} - -func findField(v ssa.Value, field int) types.Object { - structType, ok := v.Type().Underlying().(*types.Struct) - if !ok { - ptrType, ok := v.Type().Underlying().(*types.Pointer) - if !ok { - return nil - } - structType = ptrType.Elem().Underlying().(*types.Struct) - } - return structType.Field(field) -} - -func (pc *passContext) maybeFail(pos token.Pos, isExempted bool, fmtStr string, args ...interface{}) { - posKey := toPositionKey(pc.pass.Fset.Position(pos)) - log.Debugf("maybeFail: pos: %d, positionKey: %s", pos, posKey) - if fData, ok := pc.failures[posKey]; ok { - fData.count-- - if fData.count == 0 { - delete(pc.failures, posKey) - } - return - } - if !isExempted { - pc.pass.Reportf(pos, fmt.Sprintf(fmtStr, args...)) - } -} - -func (pc *passContext) checkFieldAccess(inst ssa.Instruction, structObj ssa.Value, field int, isExempted bool, lh *locksHeld) { - var fieldFacts lockFieldFacts - fieldObj := findField(structObj, field) - pc.pass.ImportObjectFact(fieldObj, &fieldFacts) - log.Debugf("fieldObj: %s, fieldFacts: %+v", fieldObj, fieldFacts) - for _, guardFieldNumber := range fieldFacts.GuardedBy { - guardObj := findField(structObj, guardFieldNumber) - var guardfieldFacts lockFieldFacts - pc.pass.ImportObjectFact(guardObj, &guardfieldFacts) - log.Debugf("guardObj: %s, guardFieldFacts: %+v", guardObj, guardfieldFacts) - if guardfieldFacts.IsMutex || guardfieldFacts.IsRWMutex { - log.Debugf("guard is a mutex") - m, ok := lh.locks[structObj] - if !ok { - pc.maybeFail(inst.Pos(), isExempted, "invalid field access, %s must be locked when accessing %s", guardObj.Name(), fieldObj.Name()) - continue - } - if _, ok := m.lockedMutexes[guardfieldFacts.FieldNumber]; !ok { - pc.maybeFail(inst.Pos(), isExempted, "invalid field access, %s must be locked when accessing %s", guardObj.Name(), fieldObj.Name()) - } - } else { - panic("incorrect guard that is not a mutex or an RWMutex") - } - } -} - -func (pc *passContext) checkFunctionCall(call *ssa.Call, isExempted bool, lh *locksHeld) { - // See: https://godoc.org/golang.org/x/tools/go/ssa#CallCommon - // - // 1. "call" mode: when Method is nil (!IsInvoke), a CallCommon represents an ordinary - // function call of the value in Value, which may be a *Builtin, a *Function or any - // other value of kind 'func'. - // - // Value may be one of: - // (a) a *Function, indicating a statically dispatched call - // to a package-level function, an anonymous function, or - // a method of a named type. - // - // (b) a *MakeClosure, indicating an immediately applied - // function literal with free variables. - // - // (c) a *Builtin, indicating a statically dispatched call - // to a built-in function. - // - // (d) any other value, indicating a dynamically dispatched - // function call. - fn, ok := call.Common().Value.(*ssa.Function) - if !ok { - return - } - if fn.Object() == nil { - return - } - - // Check if the function should be called with any locks held. - var funcFact lockFunctionFacts - pc.pass.ImportObjectFact(fn.Object(), &funcFact) - if len(funcFact.GuardedBy) > 0 { - for _, fg := range funcFact.GuardedBy { - // The first is the method object itself so we skip that when looking - // for receiver/function parameters. - r := (*call.Value().Operands(nil)[fg.ParameterNumber+1]) - guardObj := findField(r, fg.FieldNumber) - if guardObj == nil { - continue - } - var fieldFacts lockFieldFacts - pc.pass.ImportObjectFact(guardObj, &fieldFacts) - if fieldFacts.IsMutex || fieldFacts.IsRWMutex { - heldMutexes, ok := lh.locks[r] - if !ok { - log.Debugf("fn: %s, funcFact: %+v", fn, funcFact) - pc.maybeFail(call.Pos(), isExempted, "invalid function call %s must be held", guardObj.Name()) - continue - } - if _, ok := heldMutexes.lockedMutexes[fg.FieldNumber]; !ok { - log.Debugf("fn: %s, funcFact: %+v", fn, funcFact) - pc.maybeFail(call.Pos(), isExempted, "invalid function call %s must be held", guardObj.Name()) - } - } else { - panic(fmt.Sprintf("function: %+v has an invalid guard that is not a mutex: %+v", fn, guardObj)) - } - } - } - - // Check if it's a method dispatch for something in the sync package. - // See: https://godoc.org/golang.org/x/tools/go/ssa#Function - if fn.Package() != nil && fn.Package().Pkg.Name() == "sync" && fn.Signature.Recv() != nil { - r, ok := call.Common().Args[0].(*ssa.FieldAddr) - if !ok { - return - } - guardObj := findField(r.X, r.Field) - var fieldFacts lockFieldFacts - pc.pass.ImportObjectFact(guardObj, &fieldFacts) - if fieldFacts.IsMutex || fieldFacts.IsRWMutex { - switch fn.Name() { - case "Lock", "RLock": - obj := r.X - m := mutexState{lockedMutexes: make(map[int]struct{})} - for k, v := range lh.locks { - if isAlias(r.X, k) { - obj = k - m = v - } - } - if _, ok := m.lockedMutexes[r.Field]; ok { - // Double locking a mutex that is already locked. - pc.maybeFail(call.Pos(), isExempted, "trying to a lock %s when already locked", guardObj.Name()) - return - } - m.lockedMutexes[r.Field] = struct{}{} - lh.locks[obj] = m - case "Unlock", "RUnlock": - // Find the associated locker object. - var ( - obj ssa.Value - m mutexState - ) - for k, v := range lh.locks { - if isAlias(r.X, k) { - obj = k - m = v - break - } - } - if _, ok := m.lockedMutexes[r.Field]; !ok { - pc.maybeFail(call.Pos(), isExempted, "trying to unlock a mutex %s that is already unlocked", guardObj.Name()) - return - } - delete(m.lockedMutexes, r.Field) - if len(m.lockedMutexes) == 0 { - delete(lh.locks, obj) - } - case "RLocker", "DowngradeLock", "TryLock", "TryRLock": - // we explicitly ignore this for now. - default: - panic(fmt.Sprintf("unexpected mutex/rwmutex method invoked: %s", fn.Name())) - } - } - } -} - -func run(pass *analysis.Pass) (interface{}, error) { - pc := &passContext{ - pass: pass, - exemptions: make(map[types.Object]struct{}), - failures: make(map[positionKey]*failData), - } - - // Find all line failure annotations. - for _, f := range pass.Files { - for _, cg := range f.Comments { - for _, c := range cg.List { - if strings.Contains(c.Text, checkLocksFail) { - cnt := 1 - if strings.Contains(c.Text, checkLocksFail+":") { - parts := strings.SplitAfter(c.Text, checkLocksFail+":") - parsedCount, err := strconv.Atoi(parts[1]) - if err != nil { - pc.pass.Reportf(c.Pos(), "invalid checklocks annotation : %s", err) - continue - } - cnt = parsedCount - } - position := toPositionKey(pass.Fset.Position(c.Pos())) - pc.failures[position] = &failData{pos: c.Pos(), count: cnt} - } - } - } - } - - // Find all struct declarations and export any relevant facts. - for _, f := range pass.Files { - for _, decl := range f.Decls { - d, ok := decl.(*ast.GenDecl) - // A GenDecl node (generic declaration node) represents an import, - // constant, type or variable declaration. We only care about struct - // declarations so skip any declaration that doesn't declare a new type. - if !ok || d.Tok != token.TYPE { - continue - } - - for _, gs := range d.Specs { - ts := gs.(*ast.TypeSpec) - ss, ok := ts.Type.(*ast.StructType) - if !ok { - continue - } - structType := pass.TypesInfo.TypeOf(ts.Name).Underlying().(*types.Struct) - pc.findAndExportStructFacts(ss, structType) - } - } - } - - // Find all method calls and export any relevant facts. - for _, f := range pass.Files { - for _, decl := range f.Decls { - d, ok := decl.(*ast.FuncDecl) - // Ignore any non function declarations and any functions that do not have - // any comments. - if !ok || d.Doc == nil { - continue - } - pc.findAndExportFuncFacts(d) - } - } - - // log all known facts and all failures if debug logging is enabled. - allFacts := pass.AllObjectFacts() - for i := range allFacts { - log.Debugf("fact.object: %+v, fact.Fact: %+v", allFacts[i].Object, allFacts[i].Fact) - } - log.Debugf("all expected failures: %+v", pc.failures) - - // Scan all code looking for invalid accesses. - state := pass.ResultOf[buildssa.Analyzer].(*buildssa.SSA) - for _, fn := range state.SrcFuncs { - var funcFact lockFunctionFacts - // Anonymous(closures) functions do not have an object() but do show up in - // the SSA. - if obj := fn.Object(); obj != nil { - pc.pass.ImportObjectFact(fn.Object(), &funcFact) - } - - log.Debugf("checking function: %s", fn) - var b bytes.Buffer - ssa.WriteFunction(&b, fn) - log.Debugf("function SSA: %s", b.String()) - if fn.Recover != nil { - pc.checkBasicBlocks([]*ssa.BasicBlock{fn.Recover}, nil, fn, funcFact) - } - pc.checkBasicBlocks(fn.Blocks, fn.Recover, fn, funcFact) - } - - // Scan for remaining failures we expect. - for _, failure := range pc.failures { - // We are missing expect failures, report as much as possible. - pass.Reportf(failure.pos, "expected %d failures", failure.count) - } - - return nil, nil -} diff --git a/tools/checklocks/test/BUILD b/tools/checklocks/test/BUILD deleted file mode 100644 index b055e71d9..000000000 --- a/tools/checklocks/test/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -package(licenses = ["notice"]) - -go_library( - name = "test", - srcs = ["test.go"], -) diff --git a/tools/checklocks/test/test.go b/tools/checklocks/test/test.go deleted file mode 100644 index 05693c183..000000000 --- a/tools/checklocks/test/test.go +++ /dev/null @@ -1,362 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package test is a test package. -package test - -import ( - "math/rand" - "sync" -) - -type oneGuarded struct { - mu sync.Mutex - // +checklocks:mu - guardedField int - - unguardedField int -} - -func testAccessOne() { - var tc oneGuarded - // Valid access - tc.mu.Lock() - tc.guardedField = 1 - tc.unguardedField = 1 - tc.mu.Unlock() - - // Valid access as unguarded field is not protected by mu. - tc.unguardedField = 2 - - // Invalid access - tc.guardedField = 2 // +checklocksfail - - // Invalid read of a guarded field. - x := tc.guardedField // +checklocksfail - _ = x -} - -func testFunctionCallsNoParameters() { - // Couple of regular function calls with no parameters. - funcCallWithValidAccess() - funcCallWithInvalidAccess() -} - -func funcCallWithValidAccess() { - var tc2 oneGuarded - // Valid tc2 access - tc2.mu.Lock() - tc2.guardedField = 1 - tc2.mu.Unlock() -} - -func funcCallWithInvalidAccess() { - var tc oneGuarded - var tc2 oneGuarded - // Invalid access, wrong mutex is held. - tc.mu.Lock() - tc2.guardedField = 2 // +checklocksfail - tc.mu.Unlock() -} - -func testParameterPassing() { - var tc oneGuarded - - // Valid call where a guardedField is passed to a function as a parameter. - tc.mu.Lock() - nestedWithGuardByAddr(&tc.guardedField, &tc.unguardedField) - tc.mu.Unlock() - - // Invalid call where a guardedField is passed to a function as a parameter - // without holding locks. - nestedWithGuardByAddr(&tc.guardedField, &tc.unguardedField) // +checklocksfail - - // Valid call where a guardedField is passed to a function as a parameter. - tc.mu.Lock() - nestedWithGuardByValue(tc.guardedField, tc.unguardedField) - tc.mu.Unlock() - - // Invalid call where a guardedField is passed to a function as a parameter - // without holding locks. - nestedWithGuardByValue(tc.guardedField, tc.unguardedField) // +checklocksfail -} - -func nestedWithGuardByAddr(guardedField, unguardedField *int) { - *guardedField = 4 - *unguardedField = 5 -} - -func nestedWithGuardByValue(guardedField, unguardedField int) { - // read the fields to keep SA4009 static analyzer happy. - _ = guardedField - _ = unguardedField - guardedField = 4 - unguardedField = 5 -} - -type twoGuarded struct { - mu sync.Mutex - // +checklocks:mu - guardedField1 int - // +checklocks:mu - guardedField2 int -} - -type twoLocks struct { - mu sync.Mutex - secondMu sync.Mutex - - // +checklocks:mu - guardedField1 int - // +checklocks:secondMu - guardedField2 int -} - -type twoLocksDoubleGuard struct { - mu sync.Mutex - secondMu sync.Mutex - - // +checklocks:mu - // +checklocks:secondMu - doubleGuardedField int -} - -func testTwoLocksDoubleGuard() { - var tc twoLocksDoubleGuard - - // Double guarded field - tc.mu.Lock() - tc.secondMu.Lock() - tc.doubleGuardedField = 1 - tc.secondMu.Unlock() - - // This should fail as we released the secondMu. - tc.doubleGuardedField = 2 // +checklocksfail - tc.mu.Unlock() - - // This should fail as well as now we are not holding any locks. - // - // This line triggers two failures one for each mutex, hence the 2 after - // fail. - tc.doubleGuardedField = 3 // +checklocksfail:2 -} - -type rwGuarded struct { - rwMu sync.RWMutex - - // +checklocks:rwMu - rwGuardedField int -} - -func testRWGuarded() { - var tc rwGuarded - - // Assignment w/ exclusive lock should pass. - tc.rwMu.Lock() - tc.rwGuardedField = 1 - tc.rwMu.Unlock() - - // Assignment w/ RWLock should pass as we don't differentiate between - // Lock/RLock. - tc.rwMu.RLock() - tc.rwGuardedField = 2 - tc.rwMu.RUnlock() - - // Assignment w/o hold Lock() should fail. - tc.rwGuardedField = 3 // +checklocksfail - - // Reading w/o holding lock should fail. - x := tc.rwGuardedField + 3 // +checklocksfail - _ = x -} - -type nestedFields struct { - mu sync.Mutex - - // +checklocks:mu - nestedStruct struct { - nested1 int - nested2 int - } -} - -func testNestedStructGuards() { - var tc nestedFields - // Valid access with mu held. - tc.mu.Lock() - tc.nestedStruct.nested1 = 1 - tc.nestedStruct.nested2 = 2 - tc.mu.Unlock() - - // Invalid access to nested1 wihout holding mu. - tc.nestedStruct.nested1 = 1 // +checklocksfail -} - -type testCaseMethods struct { - mu sync.Mutex - - // +checklocks:mu - guardedField int -} - -func (t *testCaseMethods) Method() { - // Valid access - t.mu.Lock() - t.guardedField = 1 - t.mu.Unlock() - - // invalid access - t.guardedField = 2 // +checklocksfail -} - -// +checklocks:t.mu -func (t *testCaseMethods) MethodLocked(a, b, c int) { - t.guardedField = 3 -} - -// +checklocksignore -func (t *testCaseMethods) IgnoredMethod() { - // Invalid access but should not fail as the function is annotated - // with "// +checklocksignore" - t.guardedField = 2 -} - -func testMethodCalls() { - var tc2 testCaseMethods - - // Valid use, tc2.Method acquires lock. - tc2.Method() - - // Valid access tc2.mu is held before calling tc2.MethodLocked. - tc2.mu.Lock() - tc2.MethodLocked(1, 2, 3) - tc2.mu.Unlock() - - // Invalid access no locks are being held. - tc2.MethodLocked(4, 5, 6) // +checklocksfail -} - -type noMutex struct { - f int - g int -} - -func (n noMutex) method() { - n.f = 1 - n.f = n.g -} - -func testNoMutex() { - var n noMutex - n.method() -} - -func testMultiple() { - var tc1, tc2, tc3 testCaseMethods - - tc1.mu.Lock() - - // Valid access we are holding tc1's lock. - tc1.guardedField = 1 - - // Invalid access we are not holding tc2 or tc3's lock. - tc2.guardedField = 2 // +checklocksfail - tc3.guardedField = 3 // +checklocksfail - tc1.mu.Unlock() -} - -func testConditionalBranchingLocks() { - var tc2 testCaseMethods - x := rand.Intn(10) - if x%2 == 1 { - tc2.mu.Lock() - } - // This is invalid access as tc2.mu is not held if we never entered - // the if block. - tc2.guardedField = 1 // +checklocksfail - - var tc3 testCaseMethods - if x%2 == 1 { - tc3.mu.Lock() - } else { - tc3.mu.Lock() - } - // This is valid as tc3.mu is held in if and else blocks. - tc3.guardedField = 1 -} - -type testMethodWithParams struct { - mu sync.Mutex - - // +checklocks:mu - guardedField int -} - -type ptrToTestMethodWithParams *testMethodWithParams - -// +checklocks:t.mu -// +checklocks:a.mu -func (t *testMethodWithParams) methodLockedWithParams(a *testMethodWithParams, b *testMethodWithParams) { - t.guardedField = a.guardedField - b.guardedField = a.guardedField // +checklocksfail -} - -// +checklocks:t.mu -// +checklocks:a.mu -// +checklocks:b.mu -func (t *testMethodWithParams) methodLockedWithPtrType(a *testMethodWithParams, b ptrToTestMethodWithParams) { - t.guardedField = a.guardedField - b.guardedField = a.guardedField -} - -// +checklocks:a.mu -func standaloneFunctionWithGuard(a *testMethodWithParams) { - a.guardedField = 1 - a.mu.Unlock() - a.guardedField = 1 // +checklocksfail -} - -type testMethodWithEmbedded struct { - mu sync.Mutex - - // +checklocks:mu - guardedField int - p *testMethodWithParams -} - -// +checklocks:t.mu -func (t *testMethodWithEmbedded) DoLocked() { - var a, b testMethodWithParams - t.guardedField = 1 - a.mu.Lock() - b.mu.Lock() - t.p.methodLockedWithParams(&a, &b) // +checklocksfail - a.mu.Unlock() - b.mu.Unlock() -} - -// UnsupportedLockerExample is a test that verifies that trying to annotate a -// field that is not a sync.Mutex/RWMutex results in a failure. -type UnsupportedLockerExample struct { - mu sync.Locker - - // +checklocks:mu - x int // +checklocksfail -} - -func abc() { - var mu sync.Mutex - a := UnsupportedLockerExample{mu: &mu} - a.x = 1 -} diff --git a/tools/checkunsafe/BUILD b/tools/checkunsafe/BUILD deleted file mode 100644 index 0bb07b415..000000000 --- a/tools/checkunsafe/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -package(licenses = ["notice"]) - -go_library( - name = "checkunsafe", - srcs = ["check_unsafe.go"], - nogo = False, - visibility = ["//tools/nogo:__subpackages__"], - deps = [ - "@org_golang_x_tools//go/analysis:go_default_library", - ], -) diff --git a/tools/checkunsafe/check_unsafe.go b/tools/checkunsafe/check_unsafe.go deleted file mode 100644 index 4ccd7cc5a..000000000 --- a/tools/checkunsafe/check_unsafe.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package checkunsafe allows unsafe imports only in files named appropriately. -package checkunsafe - -import ( - "fmt" - "path" - "strconv" - "strings" - - "golang.org/x/tools/go/analysis" -) - -// Analyzer defines the entrypoint. -var Analyzer = &analysis.Analyzer{ - Name: "checkunsafe", - Doc: "allows unsafe use only in specified files", - Run: run, -} - -func run(pass *analysis.Pass) (interface{}, error) { - for _, f := range pass.Files { - for _, imp := range f.Imports { - // Is this an unsafe import? - pkg, err := strconv.Unquote(imp.Path.Value) - if err != nil || pkg != "unsafe" { - continue - } - - // Extract the filename. - filename := pass.Fset.File(imp.Pos()).Name() - - // Allow files named _unsafe.go or _test.go to opt out. - if strings.HasSuffix(filename, "_unsafe.go") || strings.HasSuffix(filename, "_test.go") { - continue - } - - // Throw the error. - pass.Reportf(imp.Pos(), fmt.Sprintf("package unsafe imported by %s; must end with _unsafe.go", path.Base(filename))) - } - } - return nil, nil -} diff --git a/tools/defs.bzl b/tools/defs.bzl deleted file mode 100644 index 27542a2f5..000000000 --- a/tools/defs.bzl +++ /dev/null @@ -1,349 +0,0 @@ -"""Wrappers for common build rules. - -These wrappers apply common BUILD configurations (e.g., proto_library -automagically creating cc_ and go_ proto targets) and act as a single point of -change for Google-internal and bazel-compatible rules. -""" - -load("//tools/go_stateify:defs.bzl", "go_stateify") -load("//tools/go_marshal:defs.bzl", "go_marshal", "marshal_deps", "marshal_test_deps") -load("//tools/nogo:defs.bzl", "nogo_test") -load("//tools/bazeldefs:defs.bzl", _arch_genrule = "arch_genrule", _build_test = "build_test", _bzl_library = "bzl_library", _coreutil = "coreutil", _default_installer = "default_installer", _default_net_util = "default_net_util", _more_shards = "more_shards", _most_shards = "most_shards", _proto_library = "proto_library", _select_arch = "select_arch", _select_system = "select_system", _short_path = "short_path", _version = "version") -load("//tools/bazeldefs:cc.bzl", _cc_binary = "cc_binary", _cc_flags_supplier = "cc_flags_supplier", _cc_grpc_library = "cc_grpc_library", _cc_library = "cc_library", _cc_proto_library = "cc_proto_library", _cc_test = "cc_test", _cc_toolchain = "cc_toolchain", _gbenchmark = "gbenchmark", _grpcpp = "grpcpp", _gtest = "gtest", _vdso_linker_option = "vdso_linker_option") -load("//tools/bazeldefs:go.bzl", _bazel_worker_proto = "bazel_worker_proto", _gazelle = "gazelle", _go_binary = "go_binary", _go_embed_data = "go_embed_data", _go_grpc_and_proto_libraries = "go_grpc_and_proto_libraries", _go_library = "go_library", _go_path = "go_path", _go_proto_library = "go_proto_library", _go_rule = "go_rule", _go_test = "go_test", _select_goarch = "select_goarch", _select_goos = "select_goos") -load("//tools/bazeldefs:pkg.bzl", _pkg_deb = "pkg_deb", _pkg_tar = "pkg_tar") -load("//tools/bazeldefs:platforms.bzl", _default_platform = "default_platform", _platforms = "platforms") -load("//tools/bazeldefs:tags.bzl", "go_suffixes") - -# Core rules. -arch_genrule = _arch_genrule -build_test = _build_test -bzl_library = _bzl_library -default_installer = _default_installer -default_net_util = _default_net_util -select_arch = _select_arch -select_system = _select_system -short_path = _short_path -coreutil = _coreutil -more_shards = _more_shards -most_shards = _most_shards -version = _version - -# C++ rules. -cc_binary = _cc_binary -cc_flags_supplier = _cc_flags_supplier -cc_grpc_library = _cc_grpc_library -cc_library = _cc_library -cc_test = _cc_test -cc_toolchain = _cc_toolchain -gbenchmark = _gbenchmark -gtest = _gtest -grpcpp = _grpcpp -vdso_linker_option = _vdso_linker_option - -# Go rules. -gazelle = _gazelle -go_path = _go_path -select_goos = _select_goos -select_goarch = _select_goarch -go_embed_data = _go_embed_data -go_proto_library = _go_proto_library -bazel_worker_proto = _bazel_worker_proto - -# Packaging rules. -pkg_deb = _pkg_deb -pkg_tar = _pkg_tar - -# Platform options. -default_platform = _default_platform -platforms = _platforms - -def _go_add_tags(ctx): - """ Adds tags to the given source file. """ - output = ctx.outputs.out - runner = ctx.actions.declare_file(ctx.label.name + ".sh") - lines = ["#!/bin/bash"] - lines += ["echo '// +build %s' >> %s" % (tag, output.path) for tag in ctx.attr.go_tags] - lines.append("echo '' >> %s" % output.path) - lines += ["cat %s >> %s" % (f.path, output.path) for f in ctx.files.src] - lines.append("") - ctx.actions.write(runner, "\n".join(lines), is_executable = True) - ctx.actions.run( - inputs = ctx.files.src, - outputs = [output], - executable = runner, - ) - return [DefaultInfo( - files = depset([output]), - )] - -go_add_tags = _go_rule( - rule, - implementation = _go_add_tags, - attrs = { - "go_tags": attr.string_list(doc = "Go build tags to be added.", mandatory = True), - "src": attr.label(doc = "Source file.", allow_single_file = True, mandatory = True), - "out": attr.output(doc = "Output file.", mandatory = True), - }, -) - -def go_binary(name, nogo = True, pure = False, static = False, x_defs = None, **kwargs): - """Wraps the standard go_binary. - - Args: - name: the rule name. - nogo: enable nogo analysis. - pure: build a pure Go (no CGo) binary. - static: build a static binary. - x_defs: additional linker definitions. - **kwargs: standard go_binary arguments. - """ - _go_binary( - name = name, - pure = pure, - static = static, - x_defs = x_defs, - **kwargs - ) - if nogo: - # Note that the nogo rule applies only for go_library and go_test - # targets, therefore we construct a library from the binary sources. - # This is done because the binary may not be in a form that objdump - # supports (i.e. a pure Go binary). - _go_library( - name = name + "_nogo_library", - srcs = kwargs.get("srcs", []), - deps = kwargs.get("deps", []), - testonly = 1, - ) - nogo_test( - name = name + "_nogo", - config = "//:nogo_config", - srcs = kwargs.get("srcs", []), - deps = [":" + name + "_nogo_library"], - tags = ["nogo"], - ) - -def calculate_sets(srcs): - """Calculates special Go sets for templates. - - Args: - srcs: the full set of Go sources. - - Returns: - A dictionary of the form: - - "": [src1.go, src2.go] - "suffix": [src3suffix.go, src4suffix.go] - - Note that suffix will typically start with '_'. - """ - result = dict() - for file in srcs: - if not file.endswith(".go"): - continue - target = "" - for suffix in go_suffixes: - if file.endswith(suffix + ".go"): - target = suffix - if not target in result: - result[target] = [file] - else: - result[target].append(file) - return result - -def go_imports(name, src, out): - """Simplify a single Go source file by eliminating unused imports.""" - native.genrule( - name = name, - srcs = [src], - outs = [out], - tools = ["@org_golang_x_tools//cmd/goimports:goimports"], - cmd = ("$(location @org_golang_x_tools//cmd/goimports:goimports) $(SRCS) > $@"), - ) - -def go_library(name, srcs, deps = [], imports = [], stateify = True, marshal = False, marshal_debug = False, nogo = True, **kwargs): - """Wraps the standard go_library and does stateification and marshalling. - - The recommended way is to use this rule with mostly identical configuration as the native - go_library rule. - - These definitions provide additional flags (stateify, marshal) that can be used - with the generators to automatically supplement the library code. - - load("//tools:defs.bzl", "go_library") - - go_library( - name = "foo", - srcs = ["foo.go"], - ) - - Args: - name: the rule name. - srcs: the library sources. - deps: the library dependencies. - imports: imports required for stateify. - stateify: whether statify is enabled (default: true). - marshal: whether marshal is enabled (default: false). - marshal_debug: whether the gomarshal tools emits debugging output (default: false). - nogo: enable nogo analysis. - **kwargs: standard go_library arguments. - """ - all_srcs = srcs - all_deps = deps - dirname, _, _ = native.package_name().rpartition("/") - full_pkg = dirname + "/" + name - if stateify: - # Only do stateification for non-state packages without manual autogen. - # First, we need to segregate the input files via the special suffixes, - # and calculate the final output set. - state_sets = calculate_sets(srcs) - for (suffix, src_subset) in state_sets.items(): - go_stateify( - name = name + suffix + "_state_autogen_with_imports", - srcs = src_subset, - imports = imports, - package = full_pkg, - out = name + suffix + "_state_autogen_with_imports.go", - ) - go_imports( - name = name + suffix + "_state_autogen", - src = name + suffix + "_state_autogen_with_imports.go", - out = name + suffix + "_state_autogen.go", - ) - all_srcs = all_srcs + [ - name + suffix + "_state_autogen.go" - for suffix in state_sets.keys() - ] - - if "//pkg/state" not in all_deps: - all_deps = all_deps + ["//pkg/state"] - - if marshal: - # See above. - marshal_sets = calculate_sets(srcs) - for (suffix, src_subset) in marshal_sets.items(): - go_marshal( - name = name + suffix + "_abi_autogen", - srcs = src_subset, - debug = select({ - "//tools/go_marshal:marshal_config_verbose": True, - "//conditions:default": marshal_debug, - }), - imports = imports, - package = name, - ) - extra_deps = [ - dep - for dep in marshal_deps - if not dep in all_deps - ] - all_deps = all_deps + extra_deps - all_srcs = all_srcs + [ - name + suffix + "_abi_autogen_unsafe.go" - for suffix in marshal_sets.keys() - ] - - _go_library( - name = name, - srcs = all_srcs, - deps = all_deps, - **kwargs - ) - if nogo: - nogo_test( - name = name + "_nogo", - config = "//:nogo_config", - srcs = all_srcs, - deps = [":" + name], - tags = ["nogo"], - ) - - if marshal: - # Ignore importpath for go_test. - kwargs.pop("importpath", None) - - # See above. - marshal_sets = calculate_sets(srcs) - for (suffix, _) in marshal_sets.items(): - _go_test( - name = name + suffix + "_abi_autogen_test", - srcs = [ - name + suffix + "_abi_autogen_test.go", - name + suffix + "_abi_autogen_unconditional_test.go", - ], - library = ":" + name, - deps = marshal_test_deps, - **kwargs - ) - -def go_test(name, nogo = True, **kwargs): - """Wraps the standard go_test. - - Args: - name: the rule name. - nogo: enable nogo analysis. - **kwargs: standard go_test arguments. - """ - _go_test( - name = name, - **kwargs - ) - if nogo: - nogo_test( - name = name + "_nogo", - config = "//:nogo_config", - srcs = kwargs.get("srcs", []), - deps = [":" + name], - tags = ["nogo"], - ) - -def proto_library(name, srcs, deps = None, has_services = 0, **kwargs): - """Wraps the standard proto_library. - - Given a proto_library named "foo", this produces up to five different - targets: - - foo_proto: proto_library rule. - - foo_go_proto: go_proto_library rule. - - foo_cc_proto: cc_proto_library rule. - - foo_go_grpc_proto: go_grpc_library rule. - - foo_cc_grpc_proto: cc_grpc_library rule. - - Args: - name: the name to which _proto, _go_proto, etc, will be appended. - srcs: the proto sources. - deps: for the proto library and the go_proto_library. - has_services: 1 to build gRPC code, otherwise 0. - **kwargs: standard proto_library arguments. - """ - _proto_library( - name = name + "_proto", - srcs = srcs, - deps = deps, - has_services = has_services, - **kwargs - ) - if has_services: - _go_grpc_and_proto_libraries( - name = name, - deps = deps, - **kwargs - ) - else: - _go_proto_library( - name = name, - deps = deps, - **kwargs - ) - _cc_proto_library( - name = name + "_cc_proto", - deps = [":" + name + "_proto"], - **kwargs - ) - if has_services: - _cc_grpc_library( - name = name + "_cc_grpc_proto", - srcs = [":" + name + "_proto"], - deps = [":" + name + "_cc_proto"], - **kwargs - ) diff --git a/tools/deps.bzl b/tools/deps.bzl deleted file mode 100644 index 91442617c..000000000 --- a/tools/deps.bzl +++ /dev/null @@ -1,119 +0,0 @@ -"""Rules for dependency checking.""" - -# DepsInfo provides a list of dependencies found when building a target. -DepsInfo = provider( - "lists dependencies encountered while building", - fields = { - "nodes": "a dict from targets to a list of their dependencies", - }, -) - -def _deps_check_impl(target, ctx): - # Check the target's dependencies and add any of our own deps. - deps = [] - for dep in ctx.rule.attr.deps: - deps.append(dep) - nodes = {} - if len(deps) != 0: - nodes[target] = deps - - # Keep and propagate each dep's providers. - for dep in ctx.rule.attr.deps: - nodes.update(dep[DepsInfo].nodes) - - return [DepsInfo(nodes = nodes)] - -_deps_check = aspect( - implementation = _deps_check_impl, - attr_aspects = ["deps"], -) - -def _is_allowed(target, allowlist, prefixes): - # Check for allowed prefixes. - for prefix in prefixes: - workspace, pfx = prefix.split("//", 1) - if len(workspace) > 0 and workspace[0] == "@": - workspace = workspace[1:] - if target.workspace_name == workspace and target.package.startswith(pfx): - return True - - # Check the allowlist. - for allowed in allowlist: - if target == allowed.label: - return True - - return False - -def _deps_test_impl(ctx): - nodes = {} - for target in ctx.attr.targets: - for (node_target, node_deps) in target[DepsInfo].nodes.items(): - # Ignore any disallowed targets. This generates more useful error - # messages. Consider the case where A dependes on B and B depends - # on C, and both B and C are disallowed. Avoid emitting an error - # that B depends on C, when the real issue is that A depends on B. - if not _is_allowed(node_target.label, ctx.attr.allowed, ctx.attr.allowed_prefixes) and node_target.label != target.label: - continue - bad_deps = [] - for dep in node_deps: - if not _is_allowed(dep.label, ctx.attr.allowed, ctx.attr.allowed_prefixes): - bad_deps.append(dep) - if len(bad_deps) > 0: - nodes[node_target] = bad_deps - - # If there aren't any violations, write a passing test. - if len(nodes) == 0: - ctx.actions.write( - output = ctx.outputs.executable, - content = "#!/bin/bash\n\nexit 0\n", - ) - return [] - - # If we're here, we've found at least one violation. - script_lines = [ - "#!/bin/bash", - "echo Invalid dependencies found. If you\\'re sure you want to add dependencies,", - "echo modify this target.", - "echo", - ] - - # List the violations. - for target, deps in nodes.items(): - script_lines.append( - 'echo "{target} depends on:"'.format(target = target.label), - ) - for dep in deps: - script_lines.append('echo "\t{dep}"'.format(dep = dep.label)) - - # The test must fail. - script_lines.append("exit 1\n") - - ctx.actions.write( - output = ctx.outputs.executable, - content = "\n".join(script_lines), - ) - return [] - -# Checks that targets only depend on an allowlist of other targets. Targets can -# be specified directly, or prefixes can be used to allow entire packages or -# directory trees. -# -# This recursively checks the "deps" attribute of each target, dependencies -# expressed other ways are not checked. For example, protobuf targets pull in -# protobuf code, but aren't analyzed by deps_test. -deps_test = rule( - implementation = _deps_test_impl, - attrs = { - "targets": attr.label_list( - doc = "The targets to check the transitive dependencies of.", - aspects = [_deps_check], - ), - "allowed": attr.label_list( - doc = "The allowed dependency targets.", - ), - "allowed_prefixes": attr.string_list( - doc = "Any packages beginning with these prefixes are allowed.", - ), - }, - test = True, -) diff --git a/tools/github/BUILD b/tools/github/BUILD deleted file mode 100644 index a345debf6..000000000 --- a/tools/github/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -load("//tools:defs.bzl", "go_binary") - -package(licenses = ["notice"]) - -go_binary( - name = "github", - srcs = ["main.go"], - nogo = False, - deps = [ - "//tools/github/reviver", - "@com_github_google_go_github_v32//github:go_default_library", - "@org_golang_x_oauth2//:go_default_library", - ], -) diff --git a/tools/github/main.go b/tools/github/main.go deleted file mode 100644 index dfb4c769d..000000000 --- a/tools/github/main.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Binary github is the entry point for GitHub utilities. -package main - -import ( - "context" - "flag" - "fmt" - "io/ioutil" - "log" - "os" - "strings" - - "github.com/google/go-github/github" - "golang.org/x/oauth2" - "gvisor.dev/gvisor/tools/github/reviver" -) - -var ( - owner string - repo string - tokenFile string - paths stringList - commit string - dryRun bool -) - -type stringList []string - -func (s *stringList) String() string { - return strings.Join(*s, ",") -} - -func (s *stringList) Set(value string) error { - *s = append(*s, value) - return nil -} - -// Keep the options simple for now. Supports only a single path and repo. -func init() { - flag.StringVar(&owner, "owner", "", "GitHub project org/owner") - flag.StringVar(&repo, "repo", "", "GitHub repo") - flag.StringVar(&tokenFile, "oauth-token-file", "", "file containing the GitHub token (or GITHUB_TOKEN is set)") - flag.Var(&paths, "path", "path(s) to scan (required for revive)") - flag.BoolVar(&dryRun, "dry-run", false, "just print changes to be made") -} - -func filterPaths(paths []string) (existing []string) { - for _, path := range paths { - if _, err := os.Stat(path); err != nil { - log.Printf("WARNING: skipping %v: %v", path, err) - continue - } - existing = append(existing, path) - } - return -} - -func main() { - // Set defaults from the environment. - repository := os.Getenv("GITHUB_REPOSITORY") - if parts := strings.SplitN(repository, "/", 2); len(parts) == 2 { - owner = parts[0] - repo = parts[1] - } - - // Parse flags. - flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), "usage: %s [options] <command>\n", os.Args[0]) - fmt.Fprintf(flag.CommandLine.Output(), "commands: revive, nogo\n") - flag.PrintDefaults() - } - flag.Parse() - args := flag.Args() - if len(args) != 1 { - fmt.Fprintf(flag.CommandLine.Output(), "extra arguments: %s\n", strings.Join(args[1:], ", ")) - flag.Usage() - os.Exit(1) - } - - // Check for mandatory parameters. - command := args[0] - if len(owner) == 0 { - fmt.Fprintln(flag.CommandLine.Output(), "missing --owner option.") - flag.Usage() - os.Exit(1) - } - if len(repo) == 0 { - fmt.Fprintln(flag.CommandLine.Output(), "missing --repo option.") - flag.Usage() - os.Exit(1) - } - filteredPaths := filterPaths(paths) - if len(filteredPaths) == 0 { - fmt.Fprintln(flag.CommandLine.Output(), "no valid --path options provided.") - flag.Usage() - os.Exit(1) - } - - // The access token may be passed as a file so it doesn't show up in - // command line arguments. It also may be provided through the - // environment to faciliate use through GitHub's CI system. - token := os.Getenv("GITHUB_TOKEN") - if len(tokenFile) != 0 { - bytes, err := ioutil.ReadFile(tokenFile) - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - token = string(bytes) - } - var client *github.Client - if len(token) == 0 { - // Client is unauthenticated. - client = github.NewClient(nil) - } else { - // Using the above token. - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - tc := oauth2.NewClient(context.Background(), ts) - client = github.NewClient(tc) - } - - switch command { - case "revive": - // Load existing GitHub bugs. - bugger, err := reviver.NewGitHubBugger(client, owner, repo, dryRun) - if err != nil { - fmt.Fprintf(os.Stderr, "Error getting github issues: %v\n", err) - os.Exit(1) - } - // Scan the provided path. - rev := reviver.New(filteredPaths, []reviver.Bugger{bugger}) - if errs := rev.Run(); len(errs) > 0 { - fmt.Fprintf(os.Stderr, "Encountered %d errors:\n", len(errs)) - for _, err := range errs { - fmt.Fprintf(os.Stderr, "\t%v\n", err) - } - os.Exit(1) - } - default: - // Not a known command. - fmt.Fprintf(flag.CommandLine.Output(), "unknown command: %s\n", command) - flag.Usage() - os.Exit(1) - } -} diff --git a/tools/github/reviver/BUILD b/tools/github/reviver/BUILD deleted file mode 100644 index fc54782f5..000000000 --- a/tools/github/reviver/BUILD +++ /dev/null @@ -1,27 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "reviver", - srcs = [ - "github.go", - "reviver.go", - ], - nogo = False, - visibility = [ - "//tools/github:__subpackages__", - ], - deps = ["@com_github_google_go_github_v32//github:go_default_library"], -) - -go_test( - name = "reviver_test", - size = "small", - srcs = [ - "github_test.go", - "reviver_test.go", - ], - library = ":reviver", - nogo = False, -) diff --git a/tools/github/reviver/github.go b/tools/github/reviver/github.go deleted file mode 100644 index b360f0544..000000000 --- a/tools/github/reviver/github.go +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package reviver - -import ( - "context" - "fmt" - "strconv" - "strings" - "time" - - "github.com/google/go-github/github" -) - -// GitHubBugger implements Bugger interface for github issues. -type GitHubBugger struct { - owner string - repo string - dryRun bool - - client *github.Client - issues map[int]*github.Issue -} - -// NewGitHubBugger creates a new GitHubBugger. -func NewGitHubBugger(client *github.Client, owner, repo string, dryRun bool) (*GitHubBugger, error) { - b := &GitHubBugger{ - owner: owner, - repo: repo, - dryRun: dryRun, - issues: map[int]*github.Issue{}, - client: client, - } - if err := b.load(); err != nil { - return nil, err - } - return b, nil -} - -func (b *GitHubBugger) load() error { - err := processAllPages(func(listOpts github.ListOptions) (*github.Response, error) { - opts := &github.IssueListByRepoOptions{State: "open", ListOptions: listOpts} - tmps, resp, err := b.client.Issues.ListByRepo(context.Background(), b.owner, b.repo, opts) - if err != nil { - return resp, err - } - for _, issue := range tmps { - b.issues[issue.GetNumber()] = issue - } - return resp, nil - }) - if err != nil { - return err - } - - fmt.Printf("Loaded %d issues from github.com/%s/%s\n", len(b.issues), b.owner, b.repo) - return nil -} - -// Activate implements Bugger.Activate. -func (b *GitHubBugger) Activate(todo *Todo) (bool, error) { - id, err := parseIssueNo(todo.Issue) - if err != nil { - return true, err - } - if id <= 0 { - return false, nil - } - - // Check against active issues cache. - if _, ok := b.issues[id]; ok { - fmt.Printf("%q is active: OK\n", todo.Issue) - return true, nil - } - - fmt.Printf("%q is not active: reopening issue %d\n", todo.Issue, id) - - // Format comment with TODO locations and search link. - comment := strings.Builder{} - fmt.Fprintln(&comment, "There are TODOs still referencing this issue:") - for _, l := range todo.Locations { - fmt.Fprintf(&comment, - "1. [%s:%d](https://github.com/%s/%s/blob/HEAD/%s#L%d): %s\n", - l.File, l.Line, b.owner, b.repo, l.File, l.Line, l.Comment) - } - fmt.Fprintf(&comment, - "\n\nSearch [TODO](https://github.com/%s/%s/search?q=%%22%s%%22)", b.owner, b.repo, todo.Issue) - - if b.dryRun { - fmt.Printf("[dry-run: skipping change to issue %d]\n%s\n=======================\n", id, comment.String()) - return true, nil - } - - ctx := context.Background() - req := &github.IssueRequest{State: github.String("open")} - _, _, err = b.client.Issues.Edit(ctx, b.owner, b.repo, id, req) - if err != nil { - return true, fmt.Errorf("failed to reactivate issue %d: %v", id, err) - } - - _, _, err = b.client.Issues.AddLabelsToIssue(ctx, b.owner, b.repo, id, []string{"revived"}) - if err != nil { - return true, fmt.Errorf("failed to set label on issue %d: %v", id, err) - } - - cmt := &github.IssueComment{ - Body: github.String(comment.String()), - Reactions: &github.Reactions{Confused: github.Int(1)}, - } - if _, _, err := b.client.Issues.CreateComment(ctx, b.owner, b.repo, id, cmt); err != nil { - return true, fmt.Errorf("failed to add comment to issue %d: %v", id, err) - } - - return true, nil -} - -var issuePrefixes = []string{ - "gvisor.dev/issue/", - "gvisor.dev/issues/", -} - -// parseIssueNo parses the issue number out of the issue url. -// -// 0 is returned if url does not correspond to an issue. -func parseIssueNo(url string) (int, error) { - // First check if I can handle the TODO. - var idStr string - for _, p := range issuePrefixes { - if str := strings.TrimPrefix(url, p); len(str) < len(url) { - idStr = str - break - } - } - if len(idStr) == 0 { - return 0, nil - } - - id, err := strconv.ParseInt(strings.TrimRight(idStr, "/"), 10, 64) - if err != nil { - return 0, err - } - return int(id), nil -} - -func processAllPages(fn func(github.ListOptions) (*github.Response, error)) error { - opts := github.ListOptions{PerPage: 1000} - for { - resp, err := fn(opts) - if err != nil { - if rateErr, ok := err.(*github.RateLimitError); ok { - duration := rateErr.Rate.Reset.Sub(time.Now()) - if duration > 5*time.Minute { - return fmt.Errorf("Rate limited for too long: %v", duration) - } - fmt.Printf("Rate limited, sleeping for: %v\n", duration) - time.Sleep(duration) - continue - } - return err - } - if resp.NextPage == 0 { - return nil - } - opts.Page = resp.NextPage - } -} diff --git a/tools/github/reviver/github_test.go b/tools/github/reviver/github_test.go deleted file mode 100644 index 5df7e3624..000000000 --- a/tools/github/reviver/github_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package reviver - -import ( - "testing" -) - -func TestParseIssueNo(t *testing.T) { - testCases := []struct { - issue string - expectErr bool - expected int - }{ - { - issue: "gvisor.dev/issue/123", - expected: 123, - }, - { - issue: "gvisor.dev/issue/123/", - expected: 123, - }, - { - issue: "not a url", - expected: 0, - }, - { - issue: "gvisor.dev/issue//", - expectErr: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.issue, func(t *testing.T) { - id, err := parseIssueNo(tc.issue) - if err != nil && !tc.expectErr { - t.Errorf("got error: %v", err) - } else if tc.expected != id { - t.Errorf("got: %v, want: %v", id, tc.expected) - } - }) - } -} diff --git a/tools/github/reviver/reviver.go b/tools/github/reviver/reviver.go deleted file mode 100644 index 2af7f0d59..000000000 --- a/tools/github/reviver/reviver.go +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package reviver scans the code looking for TODOs and pass them to registered -// Buggers to ensure TODOs point to active issues. -package reviver - -import ( - "bufio" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "sync" -) - -// regexTodo matches a TODO or FIXME comment. -var regexTodo = regexp.MustCompile(`(\/\/|#)\s*(TODO|FIXME)\(([a-zA-Z0-9.\/]+)\):\s*(.+)`) - -// Bugger interface is called for every TODO found in the code. If it can handle -// the TODO, it must return true. If it returns false, the next Bugger is -// called. If no Bugger handles the TODO, it's dropped on the floor. -type Bugger interface { - Activate(todo *Todo) (bool, error) -} - -// Location saves the location where the TODO was found. -type Location struct { - Comment string - File string - Line uint -} - -// Todo represents a unique TODO. There can be several TODOs pointing to the -// same issue in the code. They are all grouped together. -type Todo struct { - Issue string - Locations []Location -} - -// Reviver scans the given paths for TODOs and calls Buggers to handle them. -type Reviver struct { - paths []string - buggers []Bugger - - mu sync.Mutex - todos map[string]*Todo - errs []error -} - -// New create a new Reviver. -func New(paths []string, buggers []Bugger) *Reviver { - return &Reviver{ - paths: paths, - buggers: buggers, - todos: map[string]*Todo{}, - } -} - -// Run runs. It returns all errors found during processing, it doesn't stop -// on errors. -func (r *Reviver) Run() []error { - // Process each directory in parallel. - wg := sync.WaitGroup{} - for _, path := range r.paths { - wg.Add(1) - go func(path string) { - defer wg.Done() - r.processPath(path, &wg) - }(path) - } - - wg.Wait() - - r.mu.Lock() - defer r.mu.Unlock() - - fmt.Printf("Processing %d TODOs (%d errors)...\n", len(r.todos), len(r.errs)) - dropped := 0 - for _, todo := range r.todos { - ok, err := r.processTodo(todo) - if err != nil { - r.errs = append(r.errs, err) - } - if !ok { - dropped++ - } - } - fmt.Printf("Processed %d TODOs, %d were skipped (%d errors)\n", len(r.todos)-dropped, dropped, len(r.errs)) - - return r.errs -} - -func (r *Reviver) processPath(path string, wg *sync.WaitGroup) { - fmt.Printf("Processing dir %q\n", path) - fis, err := ioutil.ReadDir(path) - if err != nil { - r.addErr(fmt.Errorf("error processing dir %q: %v", path, err)) - return - } - - for _, fi := range fis { - childPath := filepath.Join(path, fi.Name()) - switch { - case fi.Mode().IsDir(): - wg.Add(1) - go func() { - defer wg.Done() - r.processPath(childPath, wg) - }() - - case fi.Mode().IsRegular(): - file, err := os.Open(childPath) - if err != nil { - r.addErr(err) - continue - } - - scanner := bufio.NewScanner(file) - lineno := uint(0) - for scanner.Scan() { - lineno++ - line := scanner.Text() - if todo := r.processLine(line, childPath, lineno); todo != nil { - r.addTodo(todo) - } - } - } - } -} - -func (r *Reviver) processLine(line, path string, lineno uint) *Todo { - matches := regexTodo.FindStringSubmatch(line) - if matches == nil { - return nil - } - if len(matches) != 5 { - panic(fmt.Sprintf("regex returned wrong matches for %q: %v", line, matches)) - } - return &Todo{ - Issue: matches[3], - Locations: []Location{ - { - File: path, - Line: lineno, - Comment: matches[4], - }, - }, - } -} - -func (r *Reviver) addTodo(newTodo *Todo) { - r.mu.Lock() - defer r.mu.Unlock() - - if todo := r.todos[newTodo.Issue]; todo == nil { - r.todos[newTodo.Issue] = newTodo - } else { - todo.Locations = append(todo.Locations, newTodo.Locations...) - } -} - -func (r *Reviver) addErr(err error) { - r.mu.Lock() - defer r.mu.Unlock() - r.errs = append(r.errs, err) -} - -func (r *Reviver) processTodo(todo *Todo) (bool, error) { - for _, bugger := range r.buggers { - ok, err := bugger.Activate(todo) - if err != nil { - return false, err - } - if ok { - return true, nil - } - } - return false, nil -} diff --git a/tools/github/reviver/reviver_test.go b/tools/github/reviver/reviver_test.go deleted file mode 100644 index 851306c9d..000000000 --- a/tools/github/reviver/reviver_test.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package reviver - -import ( - "testing" -) - -func TestProcessLine(t *testing.T) { - for _, tc := range []struct { - line string - want *Todo - }{ - { - line: "// TODO(foobar.com/issue/123): comment, bla. blabla.", - want: &Todo{ - Issue: "foobar.com/issue/123", - Locations: []Location{ - {Comment: "comment, bla. blabla."}, - }, - }, - }, - { - line: "// TODO(foobar.com/issues/123): comment, bla. blabla.", - want: &Todo{ - Issue: "foobar.com/issues/123", - Locations: []Location{ - {Comment: "comment, bla. blabla."}, - }, - }, - }, - { - line: "// FIXME(b/123): internal bug", - want: &Todo{ - Issue: "b/123", - Locations: []Location{ - {Comment: "internal bug"}, - }, - }, - }, - { - line: "TODO(issue): not todo", - }, - { - line: "FIXME(issue): not todo", - }, - { - line: "// TODO (issue): not todo", - }, - { - line: "// TODO(issue) not todo", - }, - { - line: "// todo(issue): not todo", - }, - { - line: "// TODO(issue):", - }, - } { - t.Logf("Testing: %s", tc.line) - r := Reviver{} - got := r.processLine(tc.line, "test", 0) - if got == nil { - if tc.want != nil { - t.Errorf("failed to process line, want: %+v", tc.want) - } - } else { - if tc.want == nil { - t.Errorf("expected error, got: %+v", got) - continue - } - if got.Issue != tc.want.Issue { - t.Errorf("wrong issue, got: %v, want: %v", got.Issue, tc.want.Issue) - } - if len(got.Locations) != len(tc.want.Locations) { - t.Errorf("wrong number of locations, got: %v, want: %v, locations: %+v", len(got.Locations), len(tc.want.Locations), got.Locations) - } - for i, wantLoc := range tc.want.Locations { - if got.Locations[i].Comment != wantLoc.Comment { - t.Errorf("wrong comment, got: %v, want: %v", got.Locations[i].Comment, wantLoc.Comment) - } - } - } - } -} diff --git a/tools/go_branch.sh b/tools/go_branch.sh deleted file mode 100755 index 392e40619..000000000 --- a/tools/go_branch.sh +++ /dev/null @@ -1,171 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -xeou pipefail - -# Remember our current directory. -declare orig_dir -orig_dir=$(pwd) -readonly orig_dir - -# Record the current working commit. -declare head -head=$(git describe --always) -readonly head - -# Create a temporary working directory, and ensure that this directory and all -# subdirectories are cleaned up upon exit. -declare tmp_dir -tmp_dir=$(mktemp -d) -readonly tmp_dir -finish() { - cd "${orig_dir}" # Leave tmp_dir. - rm -rf "${tmp_dir}" # Remove all contents. - git checkout -f "${head}" # Restore commit. -} -trap finish EXIT - -# Discover the package name from the go.mod file. -declare module origpwd othersrc -module=$(cat go.mod | grep -E "^module" | cut -d' ' -f2) -origpwd=$(pwd) -othersrc=("go.mod" "go.sum" "AUTHORS" "LICENSE") -readonly module origpwd othersrc - -# Build an amd64 & arm64 gopath. -declare -r go_amd64="${tmp_dir}/amd64" -declare -r go_arm64="${tmp_dir}/arm64" -make build BAZEL_OPTIONS="" TARGETS="//:gopath" -rsync --recursive --delete --copy-links bazel-bin/gopath/ "${go_amd64}" -make build BAZEL_OPTIONS=--config=cross-aarch64 TARGETS="//:gopath" 2>/dev/null -rsync --recursive --delete --copy-links bazel-bin/gopath/ "${go_arm64}" - -# Strip irrelevant files, i.e. use only arm64 files from the arm64 build. -# This is because bazel may generate incorrect files for non-target platforms -# as a workaround. See pkg/sentry/loader/vdsodata as an example. -find "${go_amd64}/src/${module}" -name '*_arm64*.go' -exec rm -f {} \; -find "${go_amd64}/src/${module}" -name '*_arm64*.s' -exec rm -f {} \; -find "${go_arm64}/src/${module}" -name '*_amd64*.go' -exec rm -f {} \; -find "${go_arm64}/src/${module}" -name '*_amd64*.s' -exec rm -f {} \; - -# See below. The certs.go file is pseudo-random, and therefore will also -# differ between the branches. Since we merge, it only has to come from one. -# We arbitrarily keep the one from the amd64 branch, and drop the arm64 one. -rm -f "${go_arm64}/src/${module}/webhook/pkg/injector/certs.go" - -# Check that all files are compatible. This means that if the files exist in -# both architectures, then they must be identical. The only ones that we expect -# to exist in a single architecture (due to binary builds) may be different. -function cross_check() { - (cd "${1}" && find "src/${module}" -type f | \ - xargs -n 1 -I {} sh -c "diff '${1}/{}' '${2}/{}' 2>/dev/null; test \$? -ne 1") -} -cross_check "${go_arm64}" "${go_amd64}" -cross_check "${go_amd64}" "${go_arm64}" - -# Merge the two for a complete set of source files. -declare -r go_merged="${tmp_dir}/merged" -rsync --recursive "${go_amd64}/" "${go_merged}" -rsync --recursive "${go_arm64}/" "${go_merged}" - -# We expect to have an existing go branch that we will use as the basis for this -# commit. That branch may be empty, but it must exist. We search for this branch -# using the local branch, the "origin" branch, and other remotes, in order. -git fetch --all -declare go_branch -go_branch=$( \ - git show-ref --hash refs/heads/go || \ - git show-ref --hash refs/remotes/origin/go || \ - git show-ref --hash go | head -n 1 \ -) -readonly go_branch - -# Clone the current repository to the temporary directory, and check out the -# current go_branch directory. We move to the new repository for convenience. -declare repo_orig -repo_orig="$(pwd)" -readonly repo_orig -declare -r repo_new="${tmp_dir}/repository" -git clone . "${repo_new}" -cd "${repo_new}" - -# Setup the repository and checkout the branch. -git config user.email "gvisor-bot@google.com" -git config user.name "gVisor bot" -git fetch origin "${go_branch}" -git checkout -b go "${go_branch}" - -# Start working on a merge commit that combines the previous history with the -# current history. Note that we don't actually want any changes yet. -# -# N.B. The git behavior changed at some point and the relevant flag was added -# to allow for override, so try the only behavior first then pass the flag. -git merge --no-commit --strategy ours "${head}" || \ - git merge --allow-unrelated-histories --no-commit --strategy ours "${head}" - -# Normalize the permissions on the old branch. Note that they should be -# normalized if constructed by this tool, but we do so before the rsync. -find . -type f -exec chmod 0644 {} \; -find . -type d -exec chmod 0755 {} \; - -# Sync the entire gopath. Note that we exclude auto-generated source files that -# will change here. Otherwise, it adds a tremendous amount of noise to commits. -# If this file disappears in the future, then presumably we will still delete -# the underlying directory. -declare -r gopath="${go_merged}/src/${module}" -rsync --recursive --delete \ - --exclude .git \ - --exclude webhook/pkg/injector/certs.go \ - "${gopath}/" . - -# Add additional files. -for file in "${othersrc[@]}"; do - cp "${origpwd}"/"${file}" . -done - -# Construct a new README.md. -cat > README.md <<EOF -# gVisor - -This branch is a synthetic branch, containing only Go sources, that is -compatible with standard Go tools. See the master branch for authoritative -sources and tests. -EOF - -# There are a few solitary files that can get left behind due to the way bazel -# constructs the gopath target. Note that we don't find all Go files here -# because they may correspond to unused templates, etc. -declare -ar binaries=( "runsc" "shim" "webhook" ) -for target in "${binaries[@]}"; do - mkdir -p "${target}" - cp "${repo_orig}/${target}"/*.go "${target}/" -done - -# Normalize all permissions. The way bazel constructs the :gopath tree may leave -# some strange permissions on files. We don't have anything in this tree that -# should be execution, only the Go source files, README.md, and ${othersrc}. -find . -type f -exec chmod 0644 {} \; -find . -type d -exec chmod 0755 {} \; - -# Update the current working set and commit. -# If the current working commit has already been committed to the remote go -# branch, then we have nothing to commit here. So allow empty commit. This can -# occur when this script is run parallely (via pull_request and push events) -# and the push workflow finishes before the pull_request workflow can run this. -git add --all && git commit --allow-empty -m "Merge ${head} (automated)" - -# Push the branch back to the original repository. -git remote add orig "${repo_orig}" && git push -f orig go:go diff --git a/tools/go_generics/BUILD b/tools/go_generics/BUILD deleted file mode 100644 index 807c08ead..000000000 --- a/tools/go_generics/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -load("//tools:defs.bzl", "bzl_library", "go_binary") - -package(licenses = ["notice"]) - -go_binary( - name = "go_generics", - srcs = [ - "generics.go", - "imports.go", - "remove.go", - ], - visibility = ["//:sandbox"], - deps = ["//tools/go_generics/globals"], -) - -bzl_library( - name = "defs_bzl", - srcs = ["defs.bzl"], - visibility = ["//visibility:private"], -) diff --git a/tools/go_generics/defs.bzl b/tools/go_generics/defs.bzl deleted file mode 100644 index 50e2546bf..000000000 --- a/tools/go_generics/defs.bzl +++ /dev/null @@ -1,127 +0,0 @@ -"""Generics support via go_generics. - -A Go template is similar to a go library, except that it has certain types that -can be replaced before usage. For example, one could define a templatized List -struct, whose elements are of type T, then instantiate that template for -T=segment, where "segment" is the concrete type. -""" - -TemplateInfo = provider( - "Information about a go_generics template.", - fields = { - "unsafe": "whether the template requires unsafe code", - "types": "required types", - "opt_types": "optional types", - "consts": "required consts", - "opt_consts": "optional consts", - "deps": "package dependencies", - "template": "merged template source file", - }, -) - -def _go_template_impl(ctx): - srcs = ctx.files.srcs - template = ctx.actions.declare_file(ctx.label.name + "_template.go") - args = ["-o=%s" % template.path] + [f.path for f in srcs] - - ctx.actions.run( - inputs = srcs, - outputs = [template], - mnemonic = "GoGenericsTemplate", - progress_message = "Building Go template %s" % ctx.label, - arguments = args, - executable = ctx.executable._tool, - ) - - return [TemplateInfo( - types = ctx.attr.types, - opt_types = ctx.attr.opt_types, - consts = ctx.attr.consts, - opt_consts = ctx.attr.opt_consts, - deps = ctx.attr.deps, - template = template, - )] - -go_template = rule( - implementation = _go_template_impl, - attrs = { - "srcs": attr.label_list(doc = "the list of source files that comprise the template", mandatory = True, allow_files = True), - "deps": attr.label_list(doc = "the standard dependency list", allow_files = True, cfg = "target"), - "types": attr.string_list(doc = "the list of generic types in the template that are required to be specified"), - "opt_types": attr.string_list(doc = "the list of generic types in the template that can but aren't required to be specified"), - "consts": attr.string_list(doc = "the list of constants in the template that are required to be specified"), - "opt_consts": attr.string_list(doc = "the list of constants in the template that can but aren't required to be specified"), - "_tool": attr.label(executable = True, cfg = "host", default = Label("//tools/go_generics/go_merge")), - }, -) - -def _go_template_instance_impl(ctx): - info = ctx.attr.template[TemplateInfo] - output = ctx.outputs.out - - # Check that all required types are defined. - for t in info.types: - if t not in ctx.attr.types: - fail("Missing value for type %s in %s" % (t, ctx.attr.template.label)) - - # Check that all defined types are expected by the template. - for t in ctx.attr.types: - if (t not in info.types) and (t not in info.opt_types): - fail("Type %s is not a parameter to %s" % (t, ctx.attr.template.label)) - - # Check that all required consts are defined. - for t in info.consts: - if t not in ctx.attr.consts: - fail("Missing value for constant %s in %s" % (t, ctx.attr.template.label)) - - # Check that all defined consts are expected by the template. - for t in ctx.attr.consts: - if (t not in info.consts) and (t not in info.opt_consts): - fail("Const %s is not a parameter to %s" % (t, ctx.attr.template.label)) - - # Build the argument list. - args = ["-i=%s" % info.template.path, "-o=%s" % output.path] - if ctx.attr.package: - args.append("-p=%s" % ctx.attr.package) - - if len(ctx.attr.prefix) > 0: - args.append("-prefix=%s" % ctx.attr.prefix) - - if len(ctx.attr.suffix) > 0: - args.append("-suffix=%s" % ctx.attr.suffix) - - args += [("-t=%s=%s" % (p[0], p[1])) for p in ctx.attr.types.items()] - args += [("-c=%s=%s" % (p[0], p[1])) for p in ctx.attr.consts.items()] - args += [("-import=%s=%s" % (p[0], p[1])) for p in ctx.attr.imports.items()] - - if ctx.attr.anon: - args.append("-anon") - - ctx.actions.run( - inputs = [info.template], - outputs = [output], - mnemonic = "GoGenericsInstance", - progress_message = "Building Go template instance %s" % ctx.label, - arguments = args, - executable = ctx.executable._tool, - ) - - return [DefaultInfo( - files = depset([output]), - )] - -go_template_instance = rule( - implementation = _go_template_instance_impl, - attrs = { - "template": attr.label(doc = "the label of the template to be instantiated", mandatory = True), - "prefix": attr.string(doc = "a prefix to be added to globals in the template"), - "suffix": attr.string(doc = "a suffix to be added to globals in the template"), - "types": attr.string_dict(doc = "the map from generic type names to concrete ones"), - "consts": attr.string_dict(doc = "the map from constant names to their values"), - "imports": attr.string_dict(doc = "the map from imports used in types/consts to their import paths"), - "anon": attr.bool(doc = "whether anoymous fields should be processed", mandatory = False, default = False), - "package": attr.string(doc = "the package for the generated source file", mandatory = False), - "out": attr.output(doc = "output file", mandatory = True), - "_tool": attr.label(executable = True, cfg = "host", default = Label("//tools/go_generics")), - }, -) diff --git a/tools/go_generics/generics.go b/tools/go_generics/generics.go deleted file mode 100644 index 30584006c..000000000 --- a/tools/go_generics/generics.go +++ /dev/null @@ -1,286 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// go_generics reads a Go source file and writes a new version of that file with -// a few transformations applied to each. Namely: -// -// 1. Global types can be explicitly renamed with the -t option. For example, -// if -t=A=B is passed in, all references to A will be replaced with -// references to B; a function declaration like: -// -// func f(arg *A) -// -// would be renamed to: -// -// func f(arg *B) -// -// 2. Global type definitions and their method sets will be removed when they're -// being renamed with -t. For example, if -t=A=B is passed in, the following -// definition and methods that existed in the input file wouldn't exist at -// all in the output file: -// -// type A struct{} -// -// func (*A) f() {} -// -// 3. All global types, variables, constants and functions (not methods) are -// prefixed and suffixed based on the option -prefix and -suffix arguments. -// For example, if -suffix=A is passed in, the following globals: -// -// func f() -// type t struct{} -// -// would be renamed to: -// -// func fA() -// type tA struct{} -// -// Some special tags are also modified. For example: -// -// "state:.(t)" -// -// would become: -// -// "state:.(tA)" -// -// 4. The package is renamed to the value via the -p argument. -// 5. Value of constants can be modified with -c argument. -// -// Note that not just the top-level declarations are renamed, all references to -// them are also properly renamed as well, taking into account visibility rules -// and shadowing. For example, if -suffix=A is passed in, the following: -// -// var b = 100 -// -// func f() { -// g(b) -// b := 0 -// g(b) -// } -// -// Would be replaced with: -// -// var bA = 100 -// -// func f() { -// g(bA) -// b := 0 -// g(b) -// } -// -// Note that the second call to g() kept "b" as an argument because it refers to -// the local variable "b". -// -// Note that go_generics can handle anonymous fields with renamed types if -// -anon is passed in, however it does not perform strict checking on parameter -// types that share the same name as the global type and therefore will rename -// them as well. -// -// You can see an example in the tools/go_generics/generics_tests/interface test. -package main - -import ( - "bytes" - "flag" - "fmt" - "go/ast" - "go/format" - "go/parser" - "go/token" - "io/ioutil" - "os" - "regexp" - "strings" - - "gvisor.dev/gvisor/tools/go_generics/globals" -) - -var ( - input = flag.String("i", "", "input `file`") - output = flag.String("o", "", "output `file`") - suffix = flag.String("suffix", "", "`suffix` to add to each global symbol") - prefix = flag.String("prefix", "", "`prefix` to add to each global symbol") - packageName = flag.String("p", "main", "output package `name`") - printAST = flag.Bool("ast", false, "prints the AST") - processAnon = flag.Bool("anon", false, "process anonymous fields") - types = make(mapValue) - consts = make(mapValue) - imports = make(mapValue) -) - -// mapValue implements flag.Value. We use a mapValue flag instead of a regular -// string flag when we want to allow more than one instance of the flag. For -// example, we allow several "-t A=B" arguments, and will rename them all. -type mapValue map[string]string - -func (m mapValue) String() string { - var b bytes.Buffer - first := true - for k, v := range m { - if !first { - b.WriteRune(',') - } else { - first = false - } - b.WriteString(k) - b.WriteRune('=') - b.WriteString(v) - } - return b.String() -} - -func (m mapValue) Set(s string) error { - sep := strings.Index(s, "=") - if sep == -1 { - return fmt.Errorf("missing '=' from '%s'", s) - } - - m[s[:sep]] = s[sep+1:] - - return nil -} - -// stateTagRegexp matches against the 'typed' state tags. -var stateTagRegexp = regexp.MustCompile(`^(.*[^a-z0-9_])state:"\.\(([^\)]*)\)"(.*)$`) - -var identifierRegexp = regexp.MustCompile(`^(.*[^a-zA-Z_])([a-zA-Z_][a-zA-Z0-9_]*)(.*)$`) - -func main() { - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage: %s [options]\n", os.Args[0]) - flag.PrintDefaults() - } - - flag.Var(types, "t", "rename type A to B when `A=B` is passed in. Multiple such mappings are allowed.") - flag.Var(consts, "c", "reassign constant A to value B when `A=B` is passed in. Multiple such mappings are allowed.") - flag.Var(imports, "import", "specifies the import libraries to use when types are not local. `name=path` specifies that 'name', used in types as name.type, refers to the package living in 'path'.") - flag.Parse() - - if *input == "" || *output == "" { - flag.Usage() - os.Exit(1) - } - - // Parse the input file. - fset := token.NewFileSet() - f, err := parser.ParseFile(fset, *input, nil, parser.ParseComments|parser.DeclarationErrors|parser.SpuriousErrors) - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } - - // Print the AST if requested. - if *printAST { - ast.Print(fset, f) - } - - cmap := ast.NewCommentMap(fset, f, f.Comments) - - // Update imports based on what's used in types and consts. - maps := []mapValue{types, consts} - importDecl, err := updateImports(maps, imports) - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } - types = maps[0] - consts = maps[1] - - // Reassign all specified constants. - for _, decl := range f.Decls { - d, ok := decl.(*ast.GenDecl) - if !ok || d.Tok != token.CONST { - continue - } - - for _, gs := range d.Specs { - s := gs.(*ast.ValueSpec) - for i, id := range s.Names { - if n, ok := consts[id.Name]; ok { - s.Values[i] = &ast.BasicLit{Value: n} - } - } - } - } - - // Go through all globals and their uses in the AST and rename the types - // with explicitly provided names, and rename all types, variables, - // consts and functions with the provided prefix and suffix. - globals.Visit(fset, f, func(ident *ast.Ident, kind globals.SymKind) { - if n, ok := types[ident.Name]; ok && kind == globals.KindType { - ident.Name = n - } else { - switch kind { - case globals.KindType, globals.KindVar, globals.KindConst, globals.KindFunction: - if ident.Name != "_" && !(ident.Name == "init" && kind == globals.KindFunction) { - ident.Name = *prefix + ident.Name + *suffix - } - case globals.KindTag: - // Modify the state tag appropriately. - if m := stateTagRegexp.FindStringSubmatch(ident.Name); m != nil { - if t := identifierRegexp.FindStringSubmatch(m[2]); t != nil { - typeName := *prefix + t[2] + *suffix - if n, ok := types[t[2]]; ok { - typeName = n - } - ident.Name = m[1] + `state:".(` + t[1] + typeName + t[3] + `)"` + m[3] - } - } - } - } - }, *processAnon) - - // Remove the definition of all types that are being remapped. - set := make(typeSet) - for _, v := range types { - set[v] = struct{}{} - } - removeTypes(set, f) - - // Add the new imports, if any, to the top. - if importDecl != nil { - newDecls := make([]ast.Decl, 0, len(f.Decls)+1) - newDecls = append(newDecls, importDecl) - newDecls = append(newDecls, f.Decls...) - f.Decls = newDecls - } - - // Update comments to remove the ones potentially associated with the - // type T that we removed. - f.Comments = cmap.Filter(f).Comments() - - // If there are file (package) comments, delete them. - if f.Doc != nil { - for i, cg := range f.Comments { - if cg == f.Doc { - f.Comments = append(f.Comments[:i], f.Comments[i+1:]...) - break - } - } - } - - // Write the output file. - f.Name.Name = *packageName - - var buf bytes.Buffer - if err := format.Node(&buf, fset, f); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } - - if err := ioutil.WriteFile(*output, buf.Bytes(), 0644); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } -} diff --git a/tools/go_generics/globals/BUILD b/tools/go_generics/globals/BUILD deleted file mode 100644 index 38caa3ce7..000000000 --- a/tools/go_generics/globals/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -package(licenses = ["notice"]) - -go_library( - name = "globals", - srcs = [ - "globals_visitor.go", - "scope.go", - ], - stateify = False, - visibility = ["//tools/go_generics:__pkg__"], -) diff --git a/tools/go_generics/globals/globals_visitor.go b/tools/go_generics/globals/globals_visitor.go deleted file mode 100644 index 883f21ebe..000000000 --- a/tools/go_generics/globals/globals_visitor.go +++ /dev/null @@ -1,597 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package globals provides an AST visitor that calls the visit function for all -// global identifiers. -package globals - -import ( - "fmt" - - "go/ast" - "go/token" - "path/filepath" - "strconv" -) - -// globalsVisitor holds the state used while traversing the nodes of a file in -// search of globals. -// -// The visitor does two passes on the global declarations: the first one adds -// all globals to the global scope (since Go allows references to globals that -// haven't been declared yet), and the second one calls f() for the definition -// and uses of globals found in the first pass. -// -// The implementation correctly handles cases when globals are aliased by -// locals; in such cases, f() is not called. -type globalsVisitor struct { - // file is the file whose nodes are being visited. - file *ast.File - - // fset is the file set the file being visited belongs to. - fset *token.FileSet - - // f is the visit function to be called when a global symbol is reached. - f func(*ast.Ident, SymKind) - - // scope is the current scope as nodes are visited. - scope *scope - - // processAnon indicates whether we should process anonymous struct fields. - // It does not perform strict checking on parameter types that share the same name - // as the global type and therefore will rename them as well. - processAnon bool -} - -// unexpected is called when an unexpected node appears in the AST. It dumps -// the location of the associated token and panics because this should only -// happen when there is a bug in the traversal code. -func (v *globalsVisitor) unexpected(p token.Pos) { - panic(fmt.Sprintf("Unable to parse at %v", v.fset.Position(p))) -} - -// pushScope creates a new scope and pushes it to the top of the scope stack. -func (v *globalsVisitor) pushScope() { - v.scope = newScope(v.scope) -} - -// popScope removes the scope created by the last call to pushScope. -func (v *globalsVisitor) popScope() { - v.scope = v.scope.outer -} - -// visitType is called when an expression is known to be a type, for example, -// on the first argument of make(). It visits all children nodes and reports -// any globals. -func (v *globalsVisitor) visitType(ge ast.Expr) { - switch e := ge.(type) { - case *ast.Ident: - if s := v.scope.deepLookup(e.Name); s != nil && s.scope.isGlobal() { - v.f(e, s.kind) - } - - case *ast.SelectorExpr: - id := GetIdent(e.X) - if id == nil { - v.unexpected(e.X.Pos()) - } - - case *ast.StarExpr: - v.visitType(e.X) - case *ast.ParenExpr: - v.visitType(e.X) - case *ast.ChanType: - v.visitType(e.Value) - case *ast.Ellipsis: - v.visitType(e.Elt) - case *ast.ArrayType: - v.visitExpr(e.Len) - v.visitType(e.Elt) - case *ast.MapType: - v.visitType(e.Key) - v.visitType(e.Value) - case *ast.StructType: - v.visitFields(e.Fields, KindUnknown) - case *ast.FuncType: - v.visitFields(e.Params, KindUnknown) - v.visitFields(e.Results, KindUnknown) - case *ast.InterfaceType: - v.visitFields(e.Methods, KindUnknown) - default: - v.unexpected(ge.Pos()) - } -} - -// visitFields visits all fields, and add symbols if kind isn't KindUnknown. -func (v *globalsVisitor) visitFields(l *ast.FieldList, kind SymKind) { - if l == nil { - return - } - - for _, f := range l.List { - if kind != KindUnknown { - for _, n := range f.Names { - v.scope.add(n.Name, kind, n.Pos()) - } - } - v.visitType(f.Type) - if f.Tag != nil { - tag := ast.NewIdent(f.Tag.Value) - v.f(tag, KindTag) - // Replace the tag if updated. - if tag.Name != f.Tag.Value { - f.Tag.Value = tag.Name - } - } - } -} - -// visitGenDecl is called when a generic declaration is encountered, for example, -// on variable, constant and type declarations. It adds all newly defined -// symbols to the current scope and reports them if the current scope is the -// global one. -func (v *globalsVisitor) visitGenDecl(d *ast.GenDecl) { - switch d.Tok { - case token.IMPORT: - case token.TYPE: - for _, gs := range d.Specs { - s := gs.(*ast.TypeSpec) - v.scope.add(s.Name.Name, KindType, s.Name.Pos()) - if v.scope.isGlobal() { - v.f(s.Name, KindType) - } - v.visitType(s.Type) - } - case token.CONST, token.VAR: - kind := KindConst - if d.Tok == token.VAR { - kind = KindVar - } - - for _, gs := range d.Specs { - s := gs.(*ast.ValueSpec) - if s.Type != nil { - v.visitType(s.Type) - } - - for _, e := range s.Values { - v.visitExpr(e) - } - - for _, n := range s.Names { - if v.scope.isGlobal() { - v.f(n, kind) - } - v.scope.add(n.Name, kind, n.Pos()) - } - } - default: - v.unexpected(d.Pos()) - } -} - -// isViableType determines if the given expression is a viable type expression, -// that is, if it could be interpreted as a type, for example, sync.Mutex, -// myType, func(int)int, as opposed to -1, 2 * 2, a + b, etc. -func (v *globalsVisitor) isViableType(expr ast.Expr) bool { - switch e := expr.(type) { - case *ast.Ident: - // This covers the plain identifier case. When we see it, we - // have to check if it resolves to a type; if the symbol is not - // known, we'll claim it's viable as a type. - s := v.scope.deepLookup(e.Name) - return s == nil || s.kind == KindType - - case *ast.ChanType, *ast.ArrayType, *ast.MapType, *ast.StructType, *ast.FuncType, *ast.InterfaceType, *ast.Ellipsis: - // This covers the following cases: - // 1. ChanType: - // chan T - // <-chan T - // chan<- T - // 2. ArrayType: - // [Expr]T - // 3. MapType: - // map[T]U - // 4. StructType: - // struct { Fields } - // 5. FuncType: - // func(Fields)Returns - // 6. Interface: - // interface { Fields } - // 7. Ellipsis: - // ...T - return true - - case *ast.SelectorExpr: - // The only case in which an expression involving a selector can - // be a type is if it has the following form X.T, where X is an - // import, and T is a type exported by X. - // - // There's no way to know whether T is a type because we don't - // parse imports. So we just claim that this is a viable type; - // it doesn't affect the general result because we don't visit - // imported symbols. - id := GetIdent(e.X) - if id == nil { - return false - } - - s := v.scope.deepLookup(id.Name) - return s != nil && s.kind == KindImport - - case *ast.StarExpr: - // This covers the *T case. The expression is a viable type if - // T is. - return v.isViableType(e.X) - - case *ast.ParenExpr: - // This covers the (T) case. The expression is a viable type if - // T is. - return v.isViableType(e.X) - - default: - return false - } -} - -// visitCallExpr visits a "call expression" which can be either a -// function/method call (e.g., f(), pkg.f(), obj.f(), etc.) call or a type -// conversion (e.g., int32(1), (*sync.Mutex)(ptr), etc.). -func (v *globalsVisitor) visitCallExpr(e *ast.CallExpr) { - if v.isViableType(e.Fun) { - v.visitType(e.Fun) - } else { - v.visitExpr(e.Fun) - } - - // If the function being called is new or make, the first argument is - // a type, so it needs to be visited as such. - first := 0 - if id := GetIdent(e.Fun); id != nil && (id.Name == "make" || id.Name == "new") { - if len(e.Args) > 0 { - v.visitType(e.Args[0]) - } - first = 1 - } - - for i := first; i < len(e.Args); i++ { - v.visitExpr(e.Args[i]) - } -} - -// visitExpr visits all nodes of an expression, and reports any globals that it -// finds. -func (v *globalsVisitor) visitExpr(ge ast.Expr) { - switch e := ge.(type) { - case nil: - case *ast.Ident: - if s := v.scope.deepLookup(e.Name); s != nil && s.scope.isGlobal() { - v.f(e, s.kind) - } - - case *ast.BasicLit: - case *ast.CompositeLit: - v.visitType(e.Type) - for _, ne := range e.Elts { - v.visitExpr(ne) - } - case *ast.FuncLit: - v.pushScope() - v.visitFields(e.Type.Params, KindParameter) - v.visitFields(e.Type.Results, KindResult) - v.visitBlockStmt(e.Body) - v.popScope() - - case *ast.BinaryExpr: - v.visitExpr(e.X) - v.visitExpr(e.Y) - - case *ast.CallExpr: - v.visitCallExpr(e) - - case *ast.IndexExpr: - v.visitExpr(e.X) - v.visitExpr(e.Index) - - case *ast.KeyValueExpr: - v.visitExpr(e.Value) - - case *ast.ParenExpr: - v.visitExpr(e.X) - - case *ast.SelectorExpr: - v.visitExpr(e.X) - if v.processAnon { - v.visitExpr(e.Sel) - } - - case *ast.SliceExpr: - v.visitExpr(e.X) - v.visitExpr(e.Low) - v.visitExpr(e.High) - v.visitExpr(e.Max) - - case *ast.StarExpr: - v.visitExpr(e.X) - - case *ast.TypeAssertExpr: - v.visitExpr(e.X) - if e.Type != nil { - v.visitType(e.Type) - } - - case *ast.UnaryExpr: - v.visitExpr(e.X) - - default: - v.unexpected(ge.Pos()) - } -} - -// GetIdent returns the identifier associated with the given expression by -// removing parentheses if needed. -func GetIdent(expr ast.Expr) *ast.Ident { - switch e := expr.(type) { - case *ast.Ident: - return e - case *ast.ParenExpr: - return GetIdent(e.X) - default: - return nil - } -} - -// visitStmt visits all nodes of a statement, and reports any globals that it -// finds. It also adds to the current scope new symbols defined/declared. -func (v *globalsVisitor) visitStmt(gs ast.Stmt) { - switch s := gs.(type) { - case nil, *ast.BranchStmt, *ast.EmptyStmt: - case *ast.AssignStmt: - for _, e := range s.Rhs { - v.visitExpr(e) - } - - // We visit the LHS after the RHS because the symbols we'll - // potentially add to the table aren't meant to be visible to - // the RHS. - for _, e := range s.Lhs { - if s.Tok == token.DEFINE { - if n := GetIdent(e); n != nil { - v.scope.add(n.Name, KindVar, n.Pos()) - } - } - v.visitExpr(e) - } - - case *ast.BlockStmt: - v.visitBlockStmt(s) - - case *ast.DeclStmt: - v.visitGenDecl(s.Decl.(*ast.GenDecl)) - - case *ast.DeferStmt: - v.visitCallExpr(s.Call) - - case *ast.ExprStmt: - v.visitExpr(s.X) - - case *ast.ForStmt: - v.pushScope() - v.visitStmt(s.Init) - v.visitExpr(s.Cond) - v.visitStmt(s.Post) - v.visitBlockStmt(s.Body) - v.popScope() - - case *ast.GoStmt: - v.visitCallExpr(s.Call) - - case *ast.IfStmt: - v.pushScope() - v.visitStmt(s.Init) - v.visitExpr(s.Cond) - v.visitBlockStmt(s.Body) - v.visitStmt(s.Else) - v.popScope() - - case *ast.IncDecStmt: - v.visitExpr(s.X) - - case *ast.LabeledStmt: - v.visitStmt(s.Stmt) - - case *ast.RangeStmt: - v.pushScope() - v.visitExpr(s.X) - if s.Tok == token.DEFINE { - if n := GetIdent(s.Key); n != nil { - v.scope.add(n.Name, KindVar, n.Pos()) - } - - if n := GetIdent(s.Value); n != nil { - v.scope.add(n.Name, KindVar, n.Pos()) - } - } - v.visitExpr(s.Key) - v.visitExpr(s.Value) - v.visitBlockStmt(s.Body) - v.popScope() - - case *ast.ReturnStmt: - for _, r := range s.Results { - v.visitExpr(r) - } - - case *ast.SelectStmt: - for _, ns := range s.Body.List { - c := ns.(*ast.CommClause) - - v.pushScope() - v.visitStmt(c.Comm) - for _, bs := range c.Body { - v.visitStmt(bs) - } - v.popScope() - } - - case *ast.SendStmt: - v.visitExpr(s.Chan) - v.visitExpr(s.Value) - - case *ast.SwitchStmt: - v.pushScope() - v.visitStmt(s.Init) - v.visitExpr(s.Tag) - for _, ns := range s.Body.List { - c := ns.(*ast.CaseClause) - v.pushScope() - for _, ce := range c.List { - v.visitExpr(ce) - } - for _, bs := range c.Body { - v.visitStmt(bs) - } - v.popScope() - } - v.popScope() - - case *ast.TypeSwitchStmt: - v.pushScope() - v.visitStmt(s.Init) - v.visitStmt(s.Assign) - for _, ns := range s.Body.List { - c := ns.(*ast.CaseClause) - v.pushScope() - for _, ce := range c.List { - v.visitType(ce) - } - for _, bs := range c.Body { - v.visitStmt(bs) - } - v.popScope() - } - v.popScope() - - default: - v.unexpected(gs.Pos()) - } -} - -// visitBlockStmt visits all statements in the block, adding symbols to a newly -// created scope. -func (v *globalsVisitor) visitBlockStmt(s *ast.BlockStmt) { - v.pushScope() - for _, c := range s.List { - v.visitStmt(c) - } - v.popScope() -} - -// visitFuncDecl is called when a function or method declaration is encountered. -// it creates a new scope for the function [optional] receiver, parameters and -// results, and visits all children nodes. -func (v *globalsVisitor) visitFuncDecl(d *ast.FuncDecl) { - // We don't report methods. - if d.Recv == nil { - v.f(d.Name, KindFunction) - } - - v.pushScope() - v.visitFields(d.Recv, KindReceiver) - v.visitFields(d.Type.Params, KindParameter) - v.visitFields(d.Type.Results, KindResult) - if d.Body != nil { - v.visitBlockStmt(d.Body) - } - v.popScope() -} - -// globalsFromDecl is called in the first, and adds symbols to global scope. -func (v *globalsVisitor) globalsFromGenDecl(d *ast.GenDecl) { - switch d.Tok { - case token.IMPORT: - for _, gs := range d.Specs { - s := gs.(*ast.ImportSpec) - if s.Name == nil { - str, _ := strconv.Unquote(s.Path.Value) - v.scope.add(filepath.Base(str), KindImport, s.Path.Pos()) - } else if s.Name.Name != "_" { - v.scope.add(s.Name.Name, KindImport, s.Name.Pos()) - } - } - case token.TYPE: - for _, gs := range d.Specs { - s := gs.(*ast.TypeSpec) - v.scope.add(s.Name.Name, KindType, s.Name.Pos()) - } - case token.CONST, token.VAR: - kind := KindConst - if d.Tok == token.VAR { - kind = KindVar - } - - for _, s := range d.Specs { - for _, n := range s.(*ast.ValueSpec).Names { - v.scope.add(n.Name, kind, n.Pos()) - } - } - default: - v.unexpected(d.Pos()) - } -} - -// visit implements the visiting of globals. It does performs the two passes -// described in the description of the globalsVisitor struct. -func (v *globalsVisitor) visit() { - // Gather all symbols in the global scope. This excludes methods. - v.pushScope() - for _, gd := range v.file.Decls { - switch d := gd.(type) { - case *ast.GenDecl: - v.globalsFromGenDecl(d) - case *ast.FuncDecl: - if d.Recv == nil { - v.scope.add(d.Name.Name, KindFunction, d.Name.Pos()) - } - default: - v.unexpected(gd.Pos()) - } - } - - // Go through the contents of the declarations. - for _, gd := range v.file.Decls { - switch d := gd.(type) { - case *ast.GenDecl: - v.visitGenDecl(d) - case *ast.FuncDecl: - v.visitFuncDecl(d) - } - } -} - -// Visit traverses the provided AST and calls f() for each identifier that -// refers to global names. The global name must be defined in the file itself. -// -// The function f() is allowed to modify the identifier, for example, to rename -// uses of global references. -func Visit(fset *token.FileSet, file *ast.File, f func(*ast.Ident, SymKind), processAnon bool) { - v := globalsVisitor{ - fset: fset, - file: file, - f: f, - processAnon: processAnon, - } - - v.visit() -} diff --git a/tools/go_generics/globals/scope.go b/tools/go_generics/globals/scope.go deleted file mode 100644 index eec93534b..000000000 --- a/tools/go_generics/globals/scope.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package globals - -import ( - "go/token" -) - -// SymKind specifies the kind of a global symbol. For example, a variable, const -// function, etc. -type SymKind int - -// Constants for different kinds of symbols. -const ( - KindUnknown SymKind = iota - KindImport - KindType - KindVar - KindConst - KindFunction - KindReceiver - KindParameter - KindResult - KindTag -) - -type symbol struct { - kind SymKind - pos token.Pos - scope *scope -} - -type scope struct { - outer *scope - syms map[string]*symbol -} - -func newScope(outer *scope) *scope { - return &scope{ - outer: outer, - syms: make(map[string]*symbol), - } -} - -func (s *scope) isGlobal() bool { - return s.outer == nil -} - -func (s *scope) lookup(n string) *symbol { - return s.syms[n] -} - -func (s *scope) deepLookup(n string) *symbol { - for x := s; x != nil; x = x.outer { - if sym := x.lookup(n); sym != nil { - return sym - } - } - return nil -} - -func (s *scope) add(name string, kind SymKind, pos token.Pos) { - if s.syms[name] != nil { - return - } - - s.syms[name] = &symbol{ - kind: kind, - pos: pos, - scope: s, - } -} diff --git a/tools/go_generics/go_merge/BUILD b/tools/go_generics/go_merge/BUILD deleted file mode 100644 index 5e0487e93..000000000 --- a/tools/go_generics/go_merge/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -load("//tools:defs.bzl", "go_binary") - -package(licenses = ["notice"]) - -go_binary( - name = "go_merge", - srcs = ["main.go"], - visibility = ["//:sandbox"], - deps = [ - "//tools/tags", - ], -) diff --git a/tools/go_generics/go_merge/main.go b/tools/go_generics/go_merge/main.go deleted file mode 100644 index 801f2354f..000000000 --- a/tools/go_generics/go_merge/main.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bytes" - "flag" - "fmt" - "go/ast" - "go/format" - "go/parser" - "go/token" - "os" - "path/filepath" - "strconv" - "strings" - - "gvisor.dev/gvisor/tools/tags" -) - -var ( - output = flag.String("o", "", "output `file`") -) - -func fatalf(s string, args ...interface{}) { - fmt.Fprintf(os.Stderr, s, args...) - os.Exit(1) -} - -func main() { - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage: %s [options] <input1> [<input2> ...]\n", os.Args[0]) - flag.PrintDefaults() - } - - flag.Parse() - if *output == "" || len(flag.Args()) == 0 { - flag.Usage() - os.Exit(1) - } - - // Load all files. - files := make(map[string]*ast.File) - fset := token.NewFileSet() - var name string - for _, fname := range flag.Args() { - f, err := parser.ParseFile(fset, fname, nil, parser.ParseComments|parser.DeclarationErrors|parser.SpuriousErrors) - if err != nil { - fatalf("%v\n", err) - } - - files[fname] = f - if name == "" { - name = f.Name.Name - } else if name != f.Name.Name { - fatalf("Expected '%s' for package name instead of '%s'.\n", name, f.Name.Name) - } - } - - // Merge all files into one. - pkg := &ast.Package{ - Name: name, - Files: files, - } - f := ast.MergePackageFiles(pkg, ast.FilterUnassociatedComments|ast.FilterFuncDuplicates|ast.FilterImportDuplicates) - - // Create a new declaration slice with all imports at the top, merging any - // redundant imports. - imports := make(map[string]*ast.ImportSpec) - var importNames []string // Keep imports in the original order to get deterministic output. - var anonImports []*ast.ImportSpec - for _, d := range f.Decls { - if g, ok := d.(*ast.GenDecl); ok && g.Tok == token.IMPORT { - for _, s := range g.Specs { - i := s.(*ast.ImportSpec) - p, _ := strconv.Unquote(i.Path.Value) - var n string - if i.Name == nil { - n = filepath.Base(p) - } else { - n = i.Name.Name - } - if n == "_" { - anonImports = append(anonImports, i) - } else { - if i2, ok := imports[n]; ok { - if first, second := i.Path.Value, i2.Path.Value; first != second { - fatalf("Conflicting paths for import name '%s': '%s' vs. '%s'\n", n, first, second) - } - } else { - imports[n] = i - importNames = append(importNames, n) - } - } - } - } - } - newDecls := make([]ast.Decl, 0, len(f.Decls)) - if l := len(imports) + len(anonImports); l > 0 { - // Non-NoPos Lparen is needed for Go to recognize more than one spec in - // ast.GenDecl.Specs. - d := &ast.GenDecl{ - Tok: token.IMPORT, - Lparen: token.NoPos + 1, - Specs: make([]ast.Spec, 0, l), - } - for _, i := range importNames { - d.Specs = append(d.Specs, imports[i]) - } - for _, i := range anonImports { - d.Specs = append(d.Specs, i) - } - newDecls = append(newDecls, d) - } - for _, d := range f.Decls { - if g, ok := d.(*ast.GenDecl); !ok || g.Tok != token.IMPORT { - newDecls = append(newDecls, d) - } - } - f.Decls = newDecls - - // Write the output file. - var buf bytes.Buffer - if err := format.Node(&buf, fset, f); err != nil { - fatalf("fomatting: %v\n", err) - } - outf, err := os.OpenFile(*output, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - fatalf("opening output: %v\n", err) - } - defer outf.Close() - if t := tags.Aggregate(flag.Args()); len(t) > 0 { - fmt.Fprintf(outf, "%s\n\n", strings.Join(t.Lines(), "\n")) - } - if _, err := outf.Write(buf.Bytes()); err != nil { - fatalf("write: %v\n", err) - } -} diff --git a/tools/go_generics/imports.go b/tools/go_generics/imports.go deleted file mode 100644 index 370650e46..000000000 --- a/tools/go_generics/imports.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bytes" - "fmt" - "go/ast" - "go/format" - "go/parser" - "go/token" - "sort" - "strconv" - - "gvisor.dev/gvisor/tools/go_generics/globals" -) - -type importedPackage struct { - newName string - path string -} - -// updateImportIdent modifies the given import identifier with the new name -// stored in the used map. If the identifier doesn't exist in the used map yet, -// a new name is generated and inserted into the map. -func updateImportIdent(orig string, imports mapValue, id *ast.Ident, used map[string]*importedPackage) error { - importName := id.Name - - // If the name is already in the table, just use the new name. - m := used[importName] - if m != nil { - id.Name = m.newName - return nil - } - - // Create a new entry in the used map. - path := imports[importName] - if path == "" { - return fmt.Errorf("unknown path to package '%s', used in '%s'", importName, orig) - } - - m = &importedPackage{ - newName: fmt.Sprintf("__generics_imported%d", len(used)), - path: strconv.Quote(path), - } - used[importName] = m - - id.Name = m.newName - - return nil -} - -// convertExpression creates a new string that is a copy of the input one with -// all imports references renamed to the names in the "used" map. If the -// referenced import isn't in "used" yet, a new one is created based on the path -// in "imports" and stored in "used". For example, if string s is -// "math.MaxUint32-math.MaxUint16+10", it would be converted to -// "x.MaxUint32-x.MathUint16+10", where x is a generated name. -func convertExpression(s string, imports mapValue, used map[string]*importedPackage) (string, error) { - // Parse the expression in the input string. - expr, err := parser.ParseExpr(s) - if err != nil { - return "", fmt.Errorf("unable to parse \"%s\": %v", s, err) - } - - // Go through the AST and update references. - var retErr error - ast.Inspect(expr, func(n ast.Node) bool { - switch x := n.(type) { - case *ast.SelectorExpr: - if id := globals.GetIdent(x.X); id != nil { - if err := updateImportIdent(s, imports, id, used); err != nil { - retErr = err - } - return false - } - } - return true - }) - if retErr != nil { - return "", retErr - } - - // Convert the modified AST back to a string. - fset := token.NewFileSet() - var buf bytes.Buffer - if err := format.Node(&buf, fset, expr); err != nil { - return "", err - } - - return string(buf.Bytes()), nil -} - -// updateImports replaces all maps in the input slice with copies where the -// mapped values have had all references to imported packages renamed to -// generated names. It also returns an import declaration for all the renamed -// import packages. -// -// For example, if the input maps contains A=math.B and C=math.D, the updated -// maps will instead contain A=__generics_imported0.B and -// C=__generics_imported0.C, and the 'import __generics_imported0 "math"' would -// be returned as the import declaration. -func updateImports(maps []mapValue, imports mapValue) (ast.Decl, error) { - importsUsed := make(map[string]*importedPackage) - - // Update all maps. - for i, m := range maps { - newMap := make(mapValue) - for n, e := range m { - updated, err := convertExpression(e, imports, importsUsed) - if err != nil { - return nil, err - } - - newMap[n] = updated - } - maps[i] = newMap - } - - // Nothing else to do if no imports are used in the expressions. - if len(importsUsed) == 0 { - return nil, nil - } - var names []string - for n := range importsUsed { - names = append(names, n) - } - // Sort the new imports for deterministic build outputs. - sort.Strings(names) - - // Create spec array for each new import. - specs := make([]ast.Spec, 0, len(importsUsed)) - for _, n := range names { - i := importsUsed[n] - specs = append(specs, &ast.ImportSpec{ - Name: &ast.Ident{Name: i.newName}, - Path: &ast.BasicLit{Value: i.path}, - }) - } - - return &ast.GenDecl{ - Tok: token.IMPORT, - Specs: specs, - Lparen: token.NoPos + 1, - }, nil -} diff --git a/tools/go_generics/remove.go b/tools/go_generics/remove.go deleted file mode 100644 index 568a6bbd3..000000000 --- a/tools/go_generics/remove.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "go/ast" - "go/token" -) - -type typeSet map[string]struct{} - -// isTypeOrPointerToType determines if the given AST expression represents a -// type or a pointer to a type that exists in the provided type set. -func isTypeOrPointerToType(set typeSet, expr ast.Expr, starCount int) bool { - switch e := expr.(type) { - case *ast.Ident: - _, ok := set[e.Name] - return ok - case *ast.StarExpr: - if starCount > 1 { - return false - } - return isTypeOrPointerToType(set, e.X, starCount+1) - case *ast.ParenExpr: - return isTypeOrPointerToType(set, e.X, starCount) - default: - return false - } -} - -// isMethodOf determines if the given function declaration is a method of one -// of the types in the provided type set. To do that, it checks if the function -// has a receiver and that its type is either T or *T, where T is a type that -// exists in the set. This is per the spec: -// -// That parameter section must declare a single parameter, the receiver. Its -// type must be of the form T or *T (possibly using parentheses) where T is a -// type name. The type denoted by T is called the receiver base type; it must -// not be a pointer or interface type and it must be declared in the same -// package as the method. -func isMethodOf(set typeSet, f *ast.FuncDecl) bool { - // If the function doesn't have exactly one receiver, then it's - // definitely not a method. - if f.Recv == nil || len(f.Recv.List) != 1 { - return false - } - - return isTypeOrPointerToType(set, f.Recv.List[0].Type, 0) -} - -// removeTypeDefinitions removes the definition of all types contained in the -// provided type set. -func removeTypeDefinitions(set typeSet, d *ast.GenDecl) { - if d.Tok != token.TYPE { - return - } - - i := 0 - for _, gs := range d.Specs { - s := gs.(*ast.TypeSpec) - if _, ok := set[s.Name.Name]; !ok { - d.Specs[i] = gs - i++ - } - } - - d.Specs = d.Specs[:i] -} - -// removeTypes removes from the AST the definition of all types and their -// method sets that are contained in the provided type set. -func removeTypes(set typeSet, f *ast.File) { - // Go through the top-level declarations. - i := 0 - for _, decl := range f.Decls { - keep := true - switch d := decl.(type) { - case *ast.GenDecl: - countBefore := len(d.Specs) - removeTypeDefinitions(set, d) - keep = countBefore == 0 || len(d.Specs) > 0 - case *ast.FuncDecl: - keep = !isMethodOf(set, d) - } - - if keep { - f.Decls[i] = decl - i++ - } - } - - f.Decls = f.Decls[:i] -} diff --git a/tools/go_generics/rules_tests/BUILD b/tools/go_generics/rules_tests/BUILD deleted file mode 100644 index 8a329dfc6..000000000 --- a/tools/go_generics/rules_tests/BUILD +++ /dev/null @@ -1,43 +0,0 @@ -load("//tools:defs.bzl", "go_test") -load("//tools/go_generics:defs.bzl", "go_template", "go_template_instance") - -package(licenses = ["notice"]) - -go_template_instance( - name = "instance", - out = "instance_test.go", - consts = { - "n": "20", - "m": "\"test\"", - "o": "math.MaxUint64", - }, - imports = { - "math": "math", - }, - package = "template_test", - template = ":test_template", - types = { - "t": "int", - }, -) - -go_template( - name = "test_template", - srcs = [ - "template.go", - ], - opt_consts = [ - "n", - "m", - "o", - ], - opt_types = ["t"], -) - -go_test( - name = "template_test", - srcs = [ - "instance_test.go", - "template_test.go", - ], -) diff --git a/tools/go_generics/rules_tests/template.go b/tools/go_generics/rules_tests/template.go deleted file mode 100644 index aace61da1..000000000 --- a/tools/go_generics/rules_tests/template.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package template - -type t float - -const ( - n t = 10.1 - m = "abc" - o = 0 -) - -func max(a, b t) t { - if a > b { - return a - } - return b -} - -func add(a t) t { - return a + n -} - -func getName() string { - return m -} - -func getMax() uint64 { - return o -} diff --git a/tools/go_generics/rules_tests/template_test.go b/tools/go_generics/rules_tests/template_test.go deleted file mode 100644 index b2a3446ef..000000000 --- a/tools/go_generics/rules_tests/template_test.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package template_test - -import ( - "math" - "testing" -) - -func TestMax(t *testing.T) { - var a int = max(10, 20) - if a != 20 { - t.Errorf("Bad result of max, got %v, want %v", a, 20) - } -} - -func TestIntConst(t *testing.T) { - var a int = add(10) - if a != 30 { - t.Errorf("Bad result of add, got %v, want %v", a, 30) - } -} - -func TestStrConst(t *testing.T) { - v := getName() - if v != "test" { - t.Errorf("Bad name, got %v, want %v", v, "test") - } -} - -func TestImport(t *testing.T) { - v := getMax() - if v != math.MaxUint64 { - t.Errorf("Bad max value, got %v, want %v", v, uint64(math.MaxUint64)) - } -} diff --git a/tools/go_generics/tests/BUILD b/tools/go_generics/tests/BUILD deleted file mode 100644 index 7547a6b53..000000000 --- a/tools/go_generics/tests/BUILD +++ /dev/null @@ -1,7 +0,0 @@ -load("//tools:defs.bzl", "bzl_library") - -bzl_library( - name = "defs_bzl", - srcs = ["defs.bzl"], - visibility = ["//visibility:private"], -) diff --git a/tools/go_generics/tests/all_stmts/BUILD b/tools/go_generics/tests/all_stmts/BUILD deleted file mode 100644 index a4a7c775a..000000000 --- a/tools/go_generics/tests/all_stmts/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -load("//tools/go_generics/tests:defs.bzl", "go_generics_test") - -go_generics_test( - name = "all_stmts", - inputs = ["input.go"], - output = "output.go", - types = { - "T": "Q", - }, -) - -# @unused -glaze_ignore = [ - "input.go", - "output.go", -] diff --git a/tools/go_generics/tests/all_stmts/input.go b/tools/go_generics/tests/all_stmts/input.go deleted file mode 100644 index 7ebe7c40e..000000000 --- a/tools/go_generics/tests/all_stmts/input.go +++ /dev/null @@ -1,292 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tests - -import ( - "sync" -) - -type T int - -func h(T) { -} - -type s struct { - a, b int - c []int -} - -func g(T) *s { - return &s{} -} - -func f() (T, []int) { - // Branch. - goto T - goto R - - // Labeled. -T: - _ = T(0) - - // Empty. -R: - ; - - // Assignment with definition. - a, b, c := T(1), T(2), T(3) - _, _, _ = a, b, c - - // Assignment without definition. - g(T(0)).a, g(T(1)).b, c = int(T(1)), int(T(2)), T(3) - _, _, _ = a, b, c - - // Block. - { - var T T - T = 0 - _ = T - } - - // Declarations. - type Type T - const Const T = 10 - var g1 func(T, int, ...T) (int, T) - var v T - var w = T(0) - { - var T struct { - f []T - } - _ = T - } - - // Defer. - defer g1(T(0), 1) - - // Expression. - h(v + w + T(1)) - - // For statements. - for i := T(0); i < T(10); i++ { - var T func(int) T - v := T(0) - _ = v - } - - for { - var T func(int) T - v := T(0) - _ = v - } - - // Go. - go g1(T(0), 1) - - // If statements. - if a != T(1) { - var T func(int) T - v := T(0) - _ = v - } - - if a := T(0); a != T(1) { - var T func(int) T - v := T(0) - _ = v - } - - if a := T(0); a != T(1) { - var T func(int) T - v := T(0) - _ = v - } else if b := T(0); b != T(1) { - var T func(int) T - v := T(0) - _ = v - } else if T := T(0); T != 1 { - T++ - _ = T - } else { - T-- - _ = T - } - - if a := T(0); a != T(1) { - var T func(int) T - v := T(0) - _ = v - } else { - var T func(int) T - v := T(0) - _ = v - } - - // Inc/Dec statements. - (*(*T)(nil))++ - (*(*T)(nil))-- - - // Range statements. - for g(T(0)).a, g(T(1)).b = range g(T(10)).c { - var d T - _ = d - } - - for T, b := range g(T(10)).c { - _ = T - _ = b - } - - // Select statement. - { - var fch func(T) chan int - - select { - case <-fch(T(30)): - var T T - T = 0 - _ = T - default: - var T T - T = 0 - _ = T - case T := <-fch(T(30)): - T = 0 - _ = T - case g(T(0)).a = <-fch(T(30)): - var T T - T = 0 - _ = T - case fch(T(30)) <- int(T(0)): - var T T - T = 0 - _ = T - } - } - - // Send statements. - { - var ch chan T - var fch func(T) chan int - - ch <- T(0) - fch(T(1)) <- g(T(10)).a - } - - // Switch statements. - { - var a T - var b int - switch { - case a == T(0): - var T T - T = 0 - _ = T - case a < T(0), b < g(T(10)).a: - var T T - T = 0 - _ = T - default: - var T T - T = 0 - _ = T - } - } - - switch T(g(T(10)).a) { - case T(0): - var T T - T = 0 - _ = T - case T(1), T(g(T(10)).a): - var T T - T = 0 - _ = T - default: - var T T - T = 0 - _ = T - } - - switch b := g(T(10)); T(b.a) + T(10) { - case T(0): - var T T - T = 0 - _ = T - case T(1), T(g(T(10)).a): - var T T - T = 0 - _ = T - default: - var T T - T = 0 - _ = T - } - - // Type switch statements. - { - var interfaceFunc func(T) interface{} - - switch interfaceFunc(T(0)).(type) { - case *T, T, int: - var T T - T = 0 - _ = T - case sync.Mutex, **T: - var T T - T = 0 - _ = T - default: - var T T - T = 0 - _ = T - } - - switch x := interfaceFunc(T(0)).(type) { - case *T, T, int: - var T T - T = 0 - _ = T - _ = x - case sync.Mutex, **T: - var T T - T = 0 - _ = T - default: - var T T - T = 0 - _ = T - } - - switch t := T(0); x := interfaceFunc(T(0) + t).(type) { - case *T, T, int: - var T T - T = 0 - _ = T - _ = x - case sync.Mutex, **T: - var T T - T = 0 - _ = T - default: - var T T - T = 0 - _ = T - } - } - - // Return statement. - return T(10), g(T(11)).c -} diff --git a/tools/go_generics/tests/all_stmts/output.go b/tools/go_generics/tests/all_stmts/output.go deleted file mode 100644 index a33944d85..000000000 --- a/tools/go_generics/tests/all_stmts/output.go +++ /dev/null @@ -1,290 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "sync" -) - -func h(Q) { -} - -type s struct { - a, b int - c []int -} - -func g(Q) *s { - return &s{} -} - -func f() (Q, []int) { - // Branch. - goto T - goto R - - // Labeled. -T: - _ = Q(0) - - // Empty. -R: - ; - - // Assignment with definition. - a, b, c := Q(1), Q(2), Q(3) - _, _, _ = a, b, c - - // Assignment without definition. - g(Q(0)).a, g(Q(1)).b, c = int(Q(1)), int(Q(2)), Q(3) - _, _, _ = a, b, c - - // Block. - { - var T Q - T = 0 - _ = T - } - - // Declarations. - type Type Q - const Const Q = 10 - var g1 func(Q, int, ...Q) (int, Q) - var v Q - var w = Q(0) - { - var T struct { - f []Q - } - _ = T - } - - // Defer. - defer g1(Q(0), 1) - - // Expression. - h(v + w + Q(1)) - - // For statements. - for i := Q(0); i < Q(10); i++ { - var T func(int) Q - v := T(0) - _ = v - } - - for { - var T func(int) Q - v := T(0) - _ = v - } - - // Go. - go g1(Q(0), 1) - - // If statements. - if a != Q(1) { - var T func(int) Q - v := T(0) - _ = v - } - - if a := Q(0); a != Q(1) { - var T func(int) Q - v := T(0) - _ = v - } - - if a := Q(0); a != Q(1) { - var T func(int) Q - v := T(0) - _ = v - } else if b := Q(0); b != Q(1) { - var T func(int) Q - v := T(0) - _ = v - } else if T := Q(0); T != 1 { - T++ - _ = T - } else { - T-- - _ = T - } - - if a := Q(0); a != Q(1) { - var T func(int) Q - v := T(0) - _ = v - } else { - var T func(int) Q - v := T(0) - _ = v - } - - // Inc/Dec statements. - (*(*Q)(nil))++ - (*(*Q)(nil))-- - - // Range statements. - for g(Q(0)).a, g(Q(1)).b = range g(Q(10)).c { - var d Q - _ = d - } - - for T, b := range g(Q(10)).c { - _ = T - _ = b - } - - // Select statement. - { - var fch func(Q) chan int - - select { - case <-fch(Q(30)): - var T Q - T = 0 - _ = T - default: - var T Q - T = 0 - _ = T - case T := <-fch(Q(30)): - T = 0 - _ = T - case g(Q(0)).a = <-fch(Q(30)): - var T Q - T = 0 - _ = T - case fch(Q(30)) <- int(Q(0)): - var T Q - T = 0 - _ = T - } - } - - // Send statements. - { - var ch chan Q - var fch func(Q) chan int - - ch <- Q(0) - fch(Q(1)) <- g(Q(10)).a - } - - // Switch statements. - { - var a Q - var b int - switch { - case a == Q(0): - var T Q - T = 0 - _ = T - case a < Q(0), b < g(Q(10)).a: - var T Q - T = 0 - _ = T - default: - var T Q - T = 0 - _ = T - } - } - - switch Q(g(Q(10)).a) { - case Q(0): - var T Q - T = 0 - _ = T - case Q(1), Q(g(Q(10)).a): - var T Q - T = 0 - _ = T - default: - var T Q - T = 0 - _ = T - } - - switch b := g(Q(10)); Q(b.a) + Q(10) { - case Q(0): - var T Q - T = 0 - _ = T - case Q(1), Q(g(Q(10)).a): - var T Q - T = 0 - _ = T - default: - var T Q - T = 0 - _ = T - } - - // Type switch statements. - { - var interfaceFunc func(Q) interface{} - - switch interfaceFunc(Q(0)).(type) { - case *Q, Q, int: - var T Q - T = 0 - _ = T - case sync.Mutex, **Q: - var T Q - T = 0 - _ = T - default: - var T Q - T = 0 - _ = T - } - - switch x := interfaceFunc(Q(0)).(type) { - case *Q, Q, int: - var T Q - T = 0 - _ = T - _ = x - case sync.Mutex, **Q: - var T Q - T = 0 - _ = T - default: - var T Q - T = 0 - _ = T - } - - switch t := Q(0); x := interfaceFunc(Q(0) + t).(type) { - case *Q, Q, int: - var T Q - T = 0 - _ = T - _ = x - case sync.Mutex, **Q: - var T Q - T = 0 - _ = T - default: - var T Q - T = 0 - _ = T - } - } - - // Return statement. - return Q(10), g(Q(11)).c -} diff --git a/tools/go_generics/tests/all_types/BUILD b/tools/go_generics/tests/all_types/BUILD deleted file mode 100644 index 60b1fd314..000000000 --- a/tools/go_generics/tests/all_types/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -load("//tools/go_generics/tests:defs.bzl", "go_generics_test") - -go_generics_test( - name = "all_types", - inputs = ["input.go"], - output = "output.go", - types = { - "T": "Q", - }, -) - -# @unused -glaze_ignore = [ - "input.go", - "output.go", -] diff --git a/tools/go_generics/tests/all_types/input.go b/tools/go_generics/tests/all_types/input.go deleted file mode 100644 index 6f85bbb69..000000000 --- a/tools/go_generics/tests/all_types/input.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tests - -import ( - "./lib" -) - -type T int - -type newType struct { - a T - b lib.T - c *T - d (T) - e chan T - f <-chan T - g chan<- T - h []T - i [10]T - j map[T]T - k func(T, T) (T, T) - l interface { - f(T) - } - m struct { - T - a T - } -} - -func f(...T) { -} diff --git a/tools/go_generics/tests/all_types/lib/lib.go b/tools/go_generics/tests/all_types/lib/lib.go deleted file mode 100644 index 99edb371f..000000000 --- a/tools/go_generics/tests/all_types/lib/lib.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package lib - -// T is a test type. -type T int32 diff --git a/tools/go_generics/tests/all_types/output.go b/tools/go_generics/tests/all_types/output.go deleted file mode 100644 index c0bbebfe7..000000000 --- a/tools/go_generics/tests/all_types/output.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "./lib" -) - -type newType struct { - a Q - b lib.T - c *Q - d (Q) - e chan Q - f <-chan Q - g chan<- Q - h []Q - i [10]Q - j map[Q]Q - k func(Q, Q) (Q, Q) - l interface { - f(Q) - } - m struct { - Q - a Q - } -} - -func f(...Q) { -} diff --git a/tools/go_generics/tests/anon/BUILD b/tools/go_generics/tests/anon/BUILD deleted file mode 100644 index ef24f4b25..000000000 --- a/tools/go_generics/tests/anon/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -load("//tools/go_generics/tests:defs.bzl", "go_generics_test") - -go_generics_test( - name = "anon", - anon = True, - inputs = ["input.go"], - output = "output.go", - suffix = "New", - types = { - "T": "Q", - }, -) - -# @unused -glaze_ignore = [ - "input.go", - "output.go", -] diff --git a/tools/go_generics/tests/anon/input.go b/tools/go_generics/tests/anon/input.go deleted file mode 100644 index 44086d522..000000000 --- a/tools/go_generics/tests/anon/input.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tests - -type T interface { - Apply(T) T -} - -type Foo struct { - T - Bar map[string]T `json:"bar,omitempty"` -} - -type Baz struct { - T someTypeNotT -} - -func (f Foo) GetBar(name string) T { - b, ok := f.Bar[name] - if ok { - b = f.Apply(b) - } else { - b = f.T - } - return b -} - -func foobar() { - a := Baz{} - a.T = 0 // should not be renamed, this is a limitation - - b := otherpkg.UnrelatedType{} - b.T = 0 // should not be renamed, this is a limitation -} diff --git a/tools/go_generics/tests/anon/output.go b/tools/go_generics/tests/anon/output.go deleted file mode 100644 index 7fa791853..000000000 --- a/tools/go_generics/tests/anon/output.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -type FooNew struct { - Q - Bar map[string]Q `json:"bar,omitempty"` -} - -type BazNew struct { - T someTypeNotT -} - -func (f FooNew) GetBar(name string) Q { - b, ok := f.Bar[name] - if ok { - b = f.Apply(b) - } else { - b = f.Q - } - return b -} - -func foobarNew() { - a := BazNew{} - a.Q = 0 - - b := otherpkg.UnrelatedType{} - b.Q = 0 -} diff --git a/tools/go_generics/tests/consts/BUILD b/tools/go_generics/tests/consts/BUILD deleted file mode 100644 index fd7caccad..000000000 --- a/tools/go_generics/tests/consts/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -load("//tools/go_generics/tests:defs.bzl", "go_generics_test") - -go_generics_test( - name = "consts", - consts = { - "c1": "20", - "z": "600", - "v": "3.3", - "s": "\"def\"", - "A": "20", - "C": "100", - "S": "\"def\"", - "T": "\"ABC\"", - }, - inputs = ["input.go"], - output = "output.go", -) - -# @unused -glaze_ignore = [ - "input.go", - "output.go", -] diff --git a/tools/go_generics/tests/consts/input.go b/tools/go_generics/tests/consts/input.go deleted file mode 100644 index 04b95fcc6..000000000 --- a/tools/go_generics/tests/consts/input.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tests - -const c1 = 10 -const x, y, z = 100, 200, 300 -const v float32 = 1.0 + 2.0 -const s = "abc" -const ( - A = 10 - B, C, D = 10, 20, 30 - S = "abc" - T, U, V string = "abc", "def", "ghi" -) diff --git a/tools/go_generics/tests/consts/output.go b/tools/go_generics/tests/consts/output.go deleted file mode 100644 index 18d316cc9..000000000 --- a/tools/go_generics/tests/consts/output.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -const c1 = 20 -const x, y, z = 100, 200, 600 -const v float32 = 3.3 -const s = "def" -const ( - A = 20 - B, C, D = 10, 100, 30 - S = "def" - T, U, V string = "ABC", "def", "ghi" -) diff --git a/tools/go_generics/tests/defs.bzl b/tools/go_generics/tests/defs.bzl deleted file mode 100644 index 6277c3947..000000000 --- a/tools/go_generics/tests/defs.bzl +++ /dev/null @@ -1,67 +0,0 @@ -"""Generics tests.""" - -load("//tools/go_generics:defs.bzl", "go_template", "go_template_instance") - -def _go_generics_test_impl(ctx): - runner = ctx.actions.declare_file(ctx.label.name) - runner_content = "\n".join([ - "#!/bin/bash", - "exec diff --ignore-blank-lines --ignore-matching-lines=^[[:space:]]*// %s %s" % ( - ctx.files.template_output[0].short_path, - ctx.files.expected_output[0].short_path, - ), - "", - ]) - ctx.actions.write(runner, runner_content, is_executable = True) - return [DefaultInfo( - executable = runner, - runfiles = ctx.runfiles( - files = ctx.files.template_output + ctx.files.expected_output, - collect_default = True, - collect_data = True, - ), - )] - -_go_generics_test = rule( - implementation = _go_generics_test_impl, - attrs = { - "template_output": attr.label(mandatory = True, allow_single_file = True), - "expected_output": attr.label(mandatory = True, allow_single_file = True), - }, - test = True, -) - -def go_generics_test(name, inputs, output, types = None, consts = None, **kwargs): - """Instantiates a generics test. - - Args: - name: the name of the test. - inputs: all the input files. - output: the output files. - types: the template types (dictionary). - consts: the template consts (dictionary). - **kwargs: additional arguments for the template_instance. - """ - if types == None: - types = dict() - if consts == None: - consts = dict() - go_template( - name = name + "_template", - srcs = inputs, - types = types.keys(), - consts = consts.keys(), - ) - go_template_instance( - name = name + "_output", - template = ":" + name + "_template", - out = name + "_output.go", - types = types, - consts = consts, - **kwargs - ) - _go_generics_test( - name = name + "_test", - template_output = name + "_output.go", - expected_output = output, - ) diff --git a/tools/go_generics/tests/imports/BUILD b/tools/go_generics/tests/imports/BUILD deleted file mode 100644 index a86223d41..000000000 --- a/tools/go_generics/tests/imports/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -load("//tools/go_generics/tests:defs.bzl", "go_generics_test") - -go_generics_test( - name = "imports", - consts = { - "n": "math.Uint32", - "m": "math.Uint64", - }, - imports = { - "sync": "sync", - "math": "mymathpath", - }, - inputs = ["input.go"], - output = "output.go", - types = { - "T": "sync.Mutex", - }, -) - -# @unused -glaze_ignore = [ - "input.go", - "output.go", -] diff --git a/tools/go_generics/tests/imports/input.go b/tools/go_generics/tests/imports/input.go deleted file mode 100644 index 0f032c2a1..000000000 --- a/tools/go_generics/tests/imports/input.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tests - -type T int - -var global T - -const ( - m = 0 - n = 0 -) diff --git a/tools/go_generics/tests/imports/output.go b/tools/go_generics/tests/imports/output.go deleted file mode 100644 index 2488ca58c..000000000 --- a/tools/go_generics/tests/imports/output.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - __generics_imported1 "mymathpath" - __generics_imported0 "sync" -) - -var global __generics_imported0.Mutex - -const ( - m = __generics_imported1.Uint64 - n = __generics_imported1.Uint32 -) diff --git a/tools/go_generics/tests/remove_typedef/BUILD b/tools/go_generics/tests/remove_typedef/BUILD deleted file mode 100644 index 46457cec6..000000000 --- a/tools/go_generics/tests/remove_typedef/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -load("//tools/go_generics/tests:defs.bzl", "go_generics_test") - -go_generics_test( - name = "remove_typedef", - inputs = ["input.go"], - output = "output.go", - types = { - "T": "U", - }, -) - -# @unused -glaze_ignore = [ - "input.go", - "output.go", -] diff --git a/tools/go_generics/tests/remove_typedef/input.go b/tools/go_generics/tests/remove_typedef/input.go deleted file mode 100644 index cf632bae7..000000000 --- a/tools/go_generics/tests/remove_typedef/input.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tests - -func f(T) Q { - return Q{} -} - -type T struct{} - -type Q struct{} - -func (*T) f() { -} - -func (T) g() { -} - -func (*Q) f(T) T { - return T{} -} - -func (*Q) g(T) *T { - return nil -} diff --git a/tools/go_generics/tests/remove_typedef/output.go b/tools/go_generics/tests/remove_typedef/output.go deleted file mode 100644 index d44fd8e1c..000000000 --- a/tools/go_generics/tests/remove_typedef/output.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -func f(U) Q { - return Q{} -} - -type Q struct{} - -func (*Q) f(U) U { - return U{} -} - -func (*Q) g(U) *U { - return nil -} diff --git a/tools/go_generics/tests/simple/BUILD b/tools/go_generics/tests/simple/BUILD deleted file mode 100644 index 4b9265ea4..000000000 --- a/tools/go_generics/tests/simple/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -load("//tools/go_generics/tests:defs.bzl", "go_generics_test") - -go_generics_test( - name = "simple", - inputs = ["input.go"], - output = "output.go", - suffix = "New", - types = { - "T": "Q", - }, -) - -# @unused -glaze_ignore = [ - "input.go", - "output.go", -] diff --git a/tools/go_generics/tests/simple/input.go b/tools/go_generics/tests/simple/input.go deleted file mode 100644 index 2a917f16c..000000000 --- a/tools/go_generics/tests/simple/input.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tests - -type T int - -var global T - -func f(_ T, a int) { -} - -func g(a T, b int) { - var c T - _ = c - - d := (*T)(nil) - _ = d -} - -type R struct { - T - a *T -} - -var ( - Z *T = (*T)(nil) -) - -const ( - X T = (T)(0) -) - -type Y T diff --git a/tools/go_generics/tests/simple/output.go b/tools/go_generics/tests/simple/output.go deleted file mode 100644 index 6bfa0b25b..000000000 --- a/tools/go_generics/tests/simple/output.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -var globalNew Q - -func fNew(_ Q, a int) { -} - -func gNew(a Q, b int) { - var c Q - _ = c - - d := (*Q)(nil) - _ = d -} - -type RNew struct { - Q - a *Q -} - -var ( - ZNew *Q = (*Q)(nil) -) - -const ( - XNew Q = (Q)(0) -) - -type YNew Q diff --git a/tools/go_marshal/BUILD b/tools/go_marshal/BUILD deleted file mode 100644 index 85a1adf66..000000000 --- a/tools/go_marshal/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -load("//tools:defs.bzl", "bzl_library", "go_binary") - -licenses(["notice"]) - -go_binary( - name = "go_marshal", - srcs = ["main.go"], - visibility = ["//:sandbox"], - deps = [ - "//tools/go_marshal/gomarshal", - ], -) - -config_setting( - name = "marshal_config_verbose", - values = {"define": "gomarshal=verbose"}, - visibility = ["//:sandbox"], -) - -bzl_library( - name = "defs_bzl", - srcs = ["defs.bzl"], - visibility = ["//visibility:private"], -) diff --git a/tools/go_marshal/README.md b/tools/go_marshal/README.md deleted file mode 100644 index bbd4c9f48..000000000 --- a/tools/go_marshal/README.md +++ /dev/null @@ -1,145 +0,0 @@ -This package implements the go_marshal utility. - -# Overview - -`go_marshal` is a code generation utility similar to `go_stateify` for -marshalling go data structures to and from memory. - -`go_marshal` attempts to improve on `binary.Write` and the sentry's -`binary.Marshal` by moving the expensive use of reflection from runtime to -compile-time. - -`go_marshal` automatically generates implementations for `marshal.Marshallable` -interface. Data structures that require custom serialization can be accomodated -through a manual implementation this interface. - -Data structures can be flagged for code generation by adding a struct-level -comment `// +marshal`. For additional details and options, see the documentation -for the `marshal.Marshallable` interface. - -# Usage - -See `defs.bzl`: a new rule is provided, `go_marshal`. - -Under the hood, the `go_marshal` rule is used to generate a file that will -appear in a Go target; the output file should appear explicitly in a srcs list. -For example (note that the above is the preferred method): - -``` -load("<PKGPATH>/gvisor/tools/go_marshal:defs.bzl", "go_marshal") - -go_marshal( - name = "foo_abi", - srcs = ["foo.go"], - out = "foo_abi.go", - package = "foo", -) - -go_library( - name = "foo", - srcs = [ - "foo.go", - "foo_abi.go", - ], - ... -) -``` - -As part of the interface generation, `go_marshal` also generates some tests for -sanity checking the struct definitions for potential alignment issues, and a -simple round-trip test through Marshal/Unmarshal to verify the implementation. -These tests use reflection to verify properties of the ABI struct, and should be -considered part of the generated interfaces (but are too expensive to execute at -runtime). Ensure these tests run at some point. - -# Restrictions - -Not all valid go type definitions can be used with `go_marshal`. `go_marshal` is -intended for ABI structs, which have these additional restrictions: - -- At the moment, `go_marshal` only supports struct declarations. - -- Structs are marshalled as packed types. This means no implicit padding is - inserted between fields shorter than the platform register size. For - alignment, manually insert padding fields. - -- Structs used with `go_marshal` must have a compile-time static size. This - means no dynamically sizes fields like slices or strings. Use statically - sized array (byte arrays for strings) instead. - -- No pointers, channel, map or function pointer fields, and no fields that are - arrays of these types. These don't make sense in an ABI data structure. - -- We could support opaque pointers as `uintptr`, but this is currently not - implemented. Implementing this would require handling the architecture - dependent native pointer size. - -- Fields must either be a primitive integer type (`byte`, - `[u]int{8,16,32,64}`), or of a type that implements `marshal.Marshallable`. - -- `int` and `uint` fields are not allowed. Use an explicitly-sized numeric - type. - -- `float*` fields are currently not supported, but could be if necessary. - -# Appendix - -## Working with Non-Packed Structs - -ABI structs must generally be packed types, meaning they should have no implicit -padding between short fields. However, if a field is tagged -`marshal:"unaligned"`, `go_marshal` will fall back to a safer but slower -mechanism to deal with potentially unaligned fields. - -Note that the non-packed property is inheritted by any other struct that embeds -this struct, since the `go_marshal` tool currently can't reason about alignments -for embedded structs that are not aligned. - -Because of this, it's generally best to avoid using `marshal:"unaligned"` and -insert explicit padding fields instead. - -## Working with dynamically sized structs - -While `go_marshal` seamlessly supports statically sized structs (which most ABI -structs are), it can also used for other uses cases where marshalling is -required. There is some provision to partially support dynamically sized structs -that may not be ABI structs. A user can define a dynamic struct and define -`SizeBytes()`, `MarshalBytes(dst)` and `UnmarshalBytes(src)` for it. Then user -can then add a comment above the struct like `// +marshal dynamic` while will -make `go_marshal` autogenerate the remaining methods required to complete the -`Marshallable` interface. This feature is currently only available for structs -and can not be used alongside the Slice API. - -## Modifying the `go_marshal` Tool - -The following are some guidelines for modifying the `go_marshal` tool: - -- The `go_marshal` tool currently does a single pass over all types requesting - code generation, in arbitrary order. This means the generated code can't - directly obtain information about embedded marshallable types at - compile-time. One way to work around this restriction is to add a new - Marshallable interface method providing this piece of information, and - calling it from the generated code. Use this sparingly, as we want to rely - on compile-time information as much as possible for performance. - -- No runtime reflection in the code generated for the marshallable interface. - The entire point of the tool is to avoid runtime reflection. The generated - tests may use reflection. - -## Debugging - -To enable debugging output from the go-marshal tool, use one of the following -options, depending on how go-marshal is being invoked: - -- Pass `--define gomarshal=verbose` to the bazel command. Note that this can - generate a lot of output depending on what's being compiled, as this will - enable debugging for all packages built by the command. - -- Set `marshal_debug = True` on the top-level `go_library` BUILD rule. - -- Set `debug = True` on the `go_marshal` BUILD rule. - -- Pass `-debug` to the go-marshal tool invocation. - -If bazel complains about stdout output being too large, set a larger value -through `--experimental_ui_max_stdouterr_bytes`, or `-1` for unlimited output. diff --git a/tools/go_marshal/analysis/BUILD b/tools/go_marshal/analysis/BUILD deleted file mode 100644 index c2a4d45c4..000000000 --- a/tools/go_marshal/analysis/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -licenses(["notice"]) - -go_library( - name = "analysis", - testonly = 1, - srcs = ["analysis_unsafe.go"], - visibility = [ - "//:sandbox", - ], -) diff --git a/tools/go_marshal/analysis/analysis_unsafe.go b/tools/go_marshal/analysis/analysis_unsafe.go deleted file mode 100644 index 7a3d6bbea..000000000 --- a/tools/go_marshal/analysis/analysis_unsafe.go +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package analysis implements common functionality used by generated -// go_marshal tests. -package analysis - -// All functions in this package are unsafe and are not intended for general -// consumption. They contain sharp edge cases and the caller is responsible for -// ensuring none of them are hit. Callers must be carefully to pass in only sane -// arguments. Failure to do so may cause panics at best and arbitrary memory -// corruption at worst. -// -// Never use outside of tests. - -import ( - "fmt" - "math/rand" - "reflect" - "testing" - "unsafe" -) - -// RandomizeValue assigns random value(s) to an abitrary type. This is intended -// for used with ABI structs from go_marshal, meaning the typical restrictions -// apply (fixed-size types, no pointers, maps, channels, etc), and should only -// be used on zeroed values to avoid overwriting pointers to active go objects. -// -// Internally, we populate the type with random data by doing an unsafe cast to -// access the underlying memory of the type and filling it as if it were a byte -// slice. This almost gets us what we want, but padding fields named "_" are -// normally not accessible, so we walk the type and recursively zero all "_" -// fields. -// -// Precondition: x must be a pointer. x must not contain any valid -// pointers to active go objects (pointer fields aren't allowed in ABI -// structs anyways), or we'd be violating the go runtime contract and -// the GC may malfunction. -func RandomizeValue(x interface{}) { - v := reflect.Indirect(reflect.ValueOf(x)) - if !v.CanSet() { - panic("RandomizeType() called with an unaddressable value. You probably need to pass a pointer to the argument") - } - - // Cast the underlying memory for the type into a byte slice. - var b []byte - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&b)) - // Note: v.UnsafeAddr panics if x is passed by value. x should be a pointer. - hdr.Data = v.UnsafeAddr() - hdr.Len = int(v.Type().Size()) - hdr.Cap = hdr.Len - - // Fill the byte slice with random data, which in effect fills the type with - // random values. - n, err := rand.Read(b) - if err != nil || n != len(b) { - panic("unreachable") - } - - // Normally, padding fields are not accessible, so zero them out. - reflectZeroPaddingFields(v.Type(), b, false) -} - -// reflectZeroPaddingFields assigns zero values to padding fields for the value -// of type r, represented by the memory in data. Padding fields are defined as -// fields with the name "_". If zero is true, the immediate value itself is -// zeroed. In addition, the type is recursively scanned for padding fields in -// inner types. -// -// This is used for zeroing padding fields after calling RandomizeValue. -func reflectZeroPaddingFields(r reflect.Type, data []byte, zero bool) { - if zero { - for i := range data { - data[i] = 0 - } - } - switch r.Kind() { - case reflect.Int8, reflect.Uint8, reflect.Int16, reflect.Uint16, reflect.Int32, reflect.Uint32, reflect.Int64, reflect.Uint64: - // These types are explicitly allowed in an ABI type, but we don't need - // to recurse further as they're scalar types. - case reflect.Struct: - for i, numFields := 0, r.NumField(); i < numFields; i++ { - f := r.Field(i) - off := f.Offset - len := f.Type.Size() - window := data[off : off+len] - reflectZeroPaddingFields(f.Type, window, f.Name == "_") - } - case reflect.Array: - eLen := int(r.Elem().Size()) - if int(r.Size()) != eLen*r.Len() { - panic("Array has unexpected size?") - } - for i, n := 0, r.Len(); i < n; i++ { - reflectZeroPaddingFields(r.Elem(), data[i*eLen:(i+1)*eLen], false) - } - default: - panic(fmt.Sprintf("Type %v not allowed in ABI struct", r.Kind())) - - } -} - -// AlignmentCheck ensures the definition of the type represented by typ doesn't -// cause the go compiler to emit implicit padding between elements of the type -// (i.e. fields in a struct). -// -// AlignmentCheck doesn't explicitly recurse for embedded structs because any -// struct present in an ABI struct must also be Marshallable, and therefore -// they're aligned by definition (or their alignment check would have failed). -func AlignmentCheck(t *testing.T, typ reflect.Type) (ok bool, delta uint64) { - switch typ.Kind() { - case reflect.Int8, reflect.Uint8, reflect.Int16, reflect.Uint16, reflect.Int32, reflect.Uint32, reflect.Int64, reflect.Uint64: - // Primitive types are always considered well aligned. Primitive types - // that are fields in structs are checked independently, this branch - // exists to handle recursive calls to alignmentCheck. - case reflect.Struct: - xOff := 0 - nextXOff := 0 - skipNext := false - for i, numFields := 0, typ.NumField(); i < numFields; i++ { - xOff = nextXOff - f := typ.Field(i) - fmt.Printf("Checking alignment of %s.%s @ %d [+%d]...\n", typ.Name(), f.Name, f.Offset, f.Type.Size()) - nextXOff = int(f.Offset + f.Type.Size()) - - if f.Name == "_" { - // Padding fields need not be aligned. - fmt.Printf("Padding field of type %v\n", f.Type) - continue - } - - if tag, ok := f.Tag.Lookup("marshal"); ok && tag == "unaligned" { - skipNext = true - continue - } - - if skipNext { - skipNext = false - fmt.Printf("Skipping alignment check for field %s.%s explicitly marked as unaligned.\n", typ.Name(), f.Name) - continue - } - - if xOff != int(f.Offset) { - implicitPad := int(f.Offset) - xOff - t.Fatalf("Suspect offset for field %s.%s, detected an implicit %d byte padding from offset %d to %d; either add %d bytes of explicit padding before this field or tag it as `marshal:\"unaligned\"`.", typ.Name(), f.Name, implicitPad, xOff, f.Offset, implicitPad) - } - } - - // Ensure structs end on a byte explicitly defined by the type. - if typ.NumField() > 0 && nextXOff != int(typ.Size()) { - implicitPad := int(typ.Size()) - nextXOff - f := typ.Field(typ.NumField() - 1) // Final field - if tag, ok := f.Tag.Lookup("marshal"); ok && tag == "unaligned" { - // Final field explicitly marked unaligned. - break - } - t.Fatalf("Suspect offset for field %s.%s at the end of %s, detected an implicit %d byte padding from offset %d to %d at the end of the struct; either add %d bytes of explict padding at end of the struct or tag the final field %s as `marshal:\"unaligned\"`.", - typ.Name(), f.Name, typ.Name(), implicitPad, nextXOff, typ.Size(), implicitPad, f.Name) - } - case reflect.Array: - // Independent arrays are also always considered well aligned. We only - // need to worry about their alignment when they're embedded in structs, - // which we handle above. - default: - t.Fatalf("Unsupported type in ABI struct while checking for field alignment for type: %v", typ.Kind()) - } - return true, uint64(typ.Size()) -} diff --git a/tools/go_marshal/defs.bzl b/tools/go_marshal/defs.bzl deleted file mode 100644 index 9f620cb76..000000000 --- a/tools/go_marshal/defs.bzl +++ /dev/null @@ -1,67 +0,0 @@ -"""Marshal is a tool for generating marshalling interfaces for Go types.""" - -def _go_marshal_impl(ctx): - """Execute the go_marshal tool.""" - output = ctx.outputs.lib - output_test = ctx.outputs.test - output_test_unconditional = ctx.outputs.test_unconditional - - # Run the marshal command. - args = ["-output=%s" % output.path] - args.append("-pkg=%s" % ctx.attr.package) - args.append("-output_test=%s" % output_test.path) - args.append("-output_test_unconditional=%s" % output_test_unconditional.path) - - if ctx.attr.debug: - args += ["-debug"] - - args += ["--"] - for src in ctx.attr.srcs: - args += [f.path for f in src.files.to_list()] - ctx.actions.run( - inputs = ctx.files.srcs, - outputs = [output, output_test, output_test_unconditional], - mnemonic = "GoMarshal", - progress_message = "go_marshal: %s" % ctx.label, - arguments = args, - executable = ctx.executable._tool, - ) - -# Generates save and restore logic from a set of Go files. -# -# Args: -# name: the name of the rule. -# srcs: the input source files. These files should include all structs in the -# package that need to be saved. -# imports: an optional list of extra, non-aliased, Go-style absolute import -# paths. -# out: the name of the generated file output. This must not conflict with any -# other files and must be added to the srcs of the relevant go_library. -# package: the package name for the input sources. -go_marshal = rule( - implementation = _go_marshal_impl, - attrs = { - "srcs": attr.label_list(mandatory = True, allow_files = True), - "imports": attr.string_list(mandatory = False), - "package": attr.string(mandatory = True), - "debug": attr.bool(doc = "enable debugging output from the go_marshal tool"), - "_tool": attr.label(executable = True, cfg = "host", default = Label("//tools/go_marshal:go_marshal")), - }, - outputs = { - "lib": "%{name}_unsafe.go", - "test": "%{name}_test.go", - "test_unconditional": "%{name}_unconditional_test.go", - }, -) - -# marshal_deps are the dependencies requied by generated code. -marshal_deps = [ - "//pkg/gohacks", - "//pkg/hostarch", - "//pkg/marshal", -] - -# marshal_test_deps are required by test targets. -marshal_test_deps = [ - "//tools/go_marshal/analysis", -] diff --git a/tools/go_marshal/gomarshal/BUILD b/tools/go_marshal/gomarshal/BUILD deleted file mode 100644 index c2747d94c..000000000 --- a/tools/go_marshal/gomarshal/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -licenses(["notice"]) - -go_library( - name = "gomarshal", - srcs = [ - "generator.go", - "generator_interfaces.go", - "generator_interfaces_array_newtype.go", - "generator_interfaces_dynamic.go", - "generator_interfaces_primitive_newtype.go", - "generator_interfaces_struct.go", - "generator_tests.go", - "util.go", - ], - stateify = False, - visibility = [ - "//:sandbox", - ], - deps = ["//tools/tags"], -) diff --git a/tools/go_marshal/gomarshal/generator.go b/tools/go_marshal/gomarshal/generator.go deleted file mode 100644 index 00961c90d..000000000 --- a/tools/go_marshal/gomarshal/generator.go +++ /dev/null @@ -1,584 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package gomarshal implements the go_marshal code generator. See README.md. -package gomarshal - -import ( - "bytes" - "fmt" - "go/ast" - "go/parser" - "go/token" - "os" - "sort" - "strings" - - "gvisor.dev/gvisor/tools/tags" -) - -// List of identifiers we use in generated code that may conflict with a -// similarly-named source identifier. Abort gracefully when we see these to -// avoid potentially confusing compilation failures in generated code. -// -// This only applies to import aliases at the moment. All other identifiers -// are qualified by a receiver argument, since they're struct fields. -// -// All recievers are single letters, so we don't allow import aliases to be a -// single letter. -var badIdents = []string{ - "addr", "blk", "buf", "cc", "dst", "dsts", "count", "err", "hdr", "idx", - "inner", "length", "limit", "ptr", "size", "src", "srcs", "val", - // All single-letter identifiers. -} - -// Constructed fromt badIdents in init(). -var badIdentsMap map[string]struct{} - -func init() { - badIdentsMap = make(map[string]struct{}) - for _, ident := range badIdents { - badIdentsMap[ident] = struct{}{} - } -} - -// Generator drives code generation for a single invocation of the go_marshal -// utility. -// -// The Generator holds arguments passed to the tool, and drives parsing, -// processing and code Generator for all types marked with +marshal declared in -// the input files. -// -// See Generator.run() as the entry point. -type Generator struct { - // Paths to input go source files. - inputs []string - // Output file to write generated go source. - output *os.File - // Output file to write generated tests. - outputTest *os.File - // Output file to write unconditionally generated tests. - outputTestUC *os.File - // Package name for the generated file. - pkg string - // Set of extra packages to import in the generated file. - imports *importTable -} - -// NewGenerator creates a new code Generator. -func NewGenerator(srcs []string, out, outTest, outTestUnconditional, pkg string, imports []string) (*Generator, error) { - f, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - return nil, fmt.Errorf("couldn't open output file %q: %w", out, err) - } - fTest, err := os.OpenFile(outTest, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - return nil, fmt.Errorf("couldn't open test output file %q: %w", out, err) - } - fTestUC, err := os.OpenFile(outTestUnconditional, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - return nil, fmt.Errorf("couldn't open unconditional test output file %q: %w", out, err) - } - g := Generator{ - inputs: srcs, - output: f, - outputTest: fTest, - outputTestUC: fTestUC, - pkg: pkg, - imports: newImportTable(), - } - for _, i := range imports { - // All imports on the extra imports list are unconditionally marked as - // used, so that they're always added to the generated code. - g.imports.add(i).markUsed() - } - - // The following imports may or may not be used by the generated code, - // depending on what's required for the target types. Don't mark these as - // used by default. - g.imports.add("io") - g.imports.add("reflect") - g.imports.add("runtime") - g.imports.add("unsafe") - g.imports.add("gvisor.dev/gvisor/pkg/gohacks") - g.imports.add("gvisor.dev/gvisor/pkg/hostarch") - g.imports.add("gvisor.dev/gvisor/pkg/marshal") - return &g, nil -} - -// writeHeader writes the header for the generated source file. The header -// includes the package name, package level comments and import statements. -func (g *Generator) writeHeader() error { - var b sourceBuffer - b.emit("// Automatically generated marshal implementation. See tools/go_marshal.\n\n") - - // Emit build tags. - b.emit("// If there are issues with build tag aggregation, see\n") - b.emit("// tools/go_marshal/gomarshal/generator.go:writeHeader(). The build tags here\n") - b.emit("// come from the input set of files used to generate this file. This input set\n") - b.emit("// is filtered based on pre-defined file suffixes related to build tags, see \n") - b.emit("// tools/defs.bzl:calculate_sets().\n\n") - - if t := tags.Aggregate(g.inputs); len(t) > 0 { - b.emit(strings.Join(t.Lines(), "\n")) - b.emit("\n\n") - } - - // Package header. - b.emit("package %s\n\n", g.pkg) - if err := b.write(g.output); err != nil { - return err - } - - return g.imports.write(g.output) -} - -// writeTypeChecks writes a statement to force the compiler to perform a type -// check for all Marshallable types referenced by the generated code. -func (g *Generator) writeTypeChecks(ms map[string]struct{}) error { - if len(ms) == 0 { - return nil - } - - msl := make([]string, 0, len(ms)) - for m := range ms { - msl = append(msl, m) - } - sort.Strings(msl) - - var buf bytes.Buffer - fmt.Fprint(&buf, "// Marshallable types used by this file.\n") - - for _, m := range msl { - fmt.Fprintf(&buf, "var _ marshal.Marshallable = (*%s)(nil)\n", m) - } - fmt.Fprint(&buf, "\n") - - _, err := fmt.Fprint(g.output, buf.String()) - return err -} - -// parse processes all input files passed this generator and produces a set of -// parsed go ASTs. -func (g *Generator) parse() ([]*ast.File, []*token.FileSet, error) { - debugf("go_marshal invoked with %d input files:\n", len(g.inputs)) - for _, path := range g.inputs { - debugf(" %s\n", path) - } - - files := make([]*ast.File, 0, len(g.inputs)) - fsets := make([]*token.FileSet, 0, len(g.inputs)) - - for _, path := range g.inputs { - fset := token.NewFileSet() - f, err := parser.ParseFile(fset, path, nil, parser.ParseComments) - if err != nil { - // Not a valid input file? - return nil, nil, fmt.Errorf("input %q can't be parsed: %w", path, err) - } - - if debugEnabled() { - debugf("AST for %q:\n", path) - ast.Print(fset, f) - } - - files = append(files, f) - fsets = append(fsets, fset) - } - - return files, fsets, nil -} - -// sliceAPI carries information about the '+marshal slice' directive. -type sliceAPI struct { - // Comment node in the AST containing the +marshal tag. - comment *ast.Comment - // Identifier fragment to use when naming generated functions for the slice - // API. - ident string - // Whether the generated functions should reference the newtype name, or the - // inner type name. Only meaningful on newtype declarations on primitives. - inner bool -} - -// marshallableType carries information about a type marked with the '+marshal' -// directive. -type marshallableType struct { - spec *ast.TypeSpec - slice *sliceAPI - recv string - dynamic bool -} - -func newMarshallableType(fset *token.FileSet, tagLine *ast.Comment, spec *ast.TypeSpec) *marshallableType { - mt := &marshallableType{ - spec: spec, - slice: nil, - } - - var unhandledTags []string - - for _, tag := range strings.Fields(strings.TrimPrefix(tagLine.Text, "// +marshal")) { - if strings.HasPrefix(tag, "slice:") { - tokens := strings.Split(tag, ":") - if len(tokens) < 2 || len(tokens) > 3 { - abortAt(fset.Position(tagLine.Slash), fmt.Sprintf("+marshal directive has invalid 'slice' clause. Expecting format 'slice:<IDENTIFIER>[:inner]', got '%v'", tag)) - } - if len(tokens[1]) == 0 { - abortAt(fset.Position(tagLine.Slash), "+marshal slice directive has empty identifier argument. Expecting '+marshal slice:identifier'") - } - - sa := &sliceAPI{ - comment: tagLine, - ident: tokens[1], - } - mt.slice = sa - - if len(tokens) == 3 { - if tokens[2] != "inner" { - abortAt(fset.Position(tagLine.Slash), "+marshal slice directive has an invalid argument. Expecting '+marshal slice:<IDENTIFIER>[:inner]'") - } - sa.inner = true - } - - continue - } else if tag == "dynamic" { - mt.dynamic = true - continue - } - - unhandledTags = append(unhandledTags, tag) - } - - if len(unhandledTags) > 0 { - abortAt(fset.Position(tagLine.Slash), fmt.Sprintf("+marshal directive contained the following unknown clauses: %v", strings.Join(unhandledTags, " "))) - } - - return mt -} - -// collectMarshallableTypes walks the parsed AST and collects a list of type -// declarations for which we need to generate the Marshallable interface. -func (g *Generator) collectMarshallableTypes(a *ast.File, f *token.FileSet) map[*ast.TypeSpec]*marshallableType { - recv := make(map[string]string) // Type name to recevier name. - types := make(map[*ast.TypeSpec]*marshallableType) - for _, decl := range a.Decls { - gdecl, ok := decl.(*ast.GenDecl) - // Type declaration? - if !ok || gdecl.Tok != token.TYPE { - // Is this a function declaration? We remember receiver names. - d, ok := decl.(*ast.FuncDecl) - if ok && d.Recv != nil && len(d.Recv.List) == 1 { - // Accept concrete methods & pointer methods. - ident, ok := d.Recv.List[0].Type.(*ast.Ident) - if !ok { - var st *ast.StarExpr - st, ok = d.Recv.List[0].Type.(*ast.StarExpr) - if ok { - ident, ok = st.X.(*ast.Ident) - } - } - // The receiver name may be not present. - if ok && len(d.Recv.List[0].Names) == 1 { - // Recover the type receiver name in this case. - recv[ident.Name] = d.Recv.List[0].Names[0].Name - } - } - debugfAt(f.Position(decl.Pos()), "Skipping declaration since it's not a type declaration.\n") - continue - } - // Does it have a comment? - if gdecl.Doc == nil { - debugfAt(f.Position(gdecl.Pos()), "Skipping declaration since it doesn't have a comment.\n") - continue - } - // Does the comment contain a "+marshal" line? - marked := false - var tagLine *ast.Comment - for _, c := range gdecl.Doc.List { - if strings.HasPrefix(c.Text, "// +marshal") { - marked = true - tagLine = c - break - } - } - if !marked { - debugfAt(f.Position(gdecl.Pos()), "Skipping declaration since it doesn't have a comment containing +marshal line.\n") - continue - } - for _, spec := range gdecl.Specs { - // We already confirmed we're in a type declaration earlier, so this - // cast will succeed. - t := spec.(*ast.TypeSpec) - switch t.Type.(type) { - case *ast.StructType: - debugfAt(f.Position(t.Pos()), "Collected marshallable struct %s.\n", t.Name.Name) - case *ast.Ident: // Newtype on primitive. - debugfAt(f.Position(t.Pos()), "Collected marshallable newtype on primitive %s.\n", t.Name.Name) - case *ast.ArrayType: // Newtype on array. - debugfAt(f.Position(t.Pos()), "Collected marshallable newtype on array %s.\n", t.Name.Name) - default: - // A user specifically requested marshalling on this type, but we - // don't support it. - abortAt(f.Position(t.Pos()), fmt.Sprintf("Marshalling codegen was requested on type '%s', but go-marshal doesn't support this kind of declaration.\n", t.Name)) - } - types[t] = newMarshallableType(f, tagLine, t) - } - } - // Update the types with the last seen receiver. As long as the - // receiver name is consistent for the type, then we will generate - // code that is still consistent with itself. - for t, mt := range types { - r, ok := recv[t.Name.Name] - if !ok { - mt.recv = receiverName(t) // Default. - continue - } - mt.recv = r // Last seen. - } - return types -} - -// collectImports collects all imports from all input source files. Some of -// these imports are copied to the generated output, if they're referenced by -// the generated code. -// -// collectImports de-duplicates imports while building the list, and ensures -// identifiers in the generated code don't conflict with any imported package -// names. -func (g *Generator) collectImports(a *ast.File, f *token.FileSet) map[string]importStmt { - is := make(map[string]importStmt) - for _, decl := range a.Decls { - gdecl, ok := decl.(*ast.GenDecl) - // Import statement? - if !ok || gdecl.Tok != token.IMPORT { - continue - } - for _, spec := range gdecl.Specs { - i := g.imports.addFromSpec(spec.(*ast.ImportSpec), f) - debugf("Collected import '%s' as '%s'\n", i.path, i.name) - - // Make sure we have an import that doesn't use any local names that - // would conflict with identifiers in the generated code. - if len(i.name) == 1 && i.name != "_" { - abortAt(f.Position(spec.Pos()), fmt.Sprintf("Import has a single character local name '%s'; this may conflict with code generated by go_marshal, use a multi-character import alias", i.name)) - } - if _, ok := badIdentsMap[i.name]; ok { - abortAt(f.Position(spec.Pos()), fmt.Sprintf("Import name '%s' is likely to conflict with code generated by go_marshal, use a different import alias", i.name)) - } - } - } - return is - -} - -func (g *Generator) generateOne(t *marshallableType, fset *token.FileSet) *interfaceGenerator { - i := newInterfaceGenerator(t.spec, t.recv, fset) - if t.dynamic { - if t.slice != nil { - abortAt(fset.Position(t.slice.comment.Slash), "Slice API is not supported for dynamic types because it assumes that each slice element is statically sized.") - } - // No validation needed, assume the user knows what they are doing. - i.emitMarshallableForDynamicType() - return i - } - switch ty := t.spec.Type.(type) { - case *ast.StructType: - i.validateStruct(t.spec, ty) - i.emitMarshallableForStruct(ty) - if t.slice != nil { - i.emitMarshallableSliceForStruct(ty, t.slice) - } - case *ast.Ident: - i.validatePrimitiveNewtype(ty) - i.emitMarshallableForPrimitiveNewtype(ty) - if t.slice != nil { - i.emitMarshallableSliceForPrimitiveNewtype(ty, t.slice) - } - case *ast.ArrayType: - i.validateArrayNewtype(t.spec.Name, ty) - // After validate, we can safely call arrayLen. - i.emitMarshallableForArrayNewtype(t.spec.Name, ty, ty.Elt.(*ast.Ident)) - if t.slice != nil { - abortAt(fset.Position(t.slice.comment.Slash), "Array type marked as '+marshal slice:...', but this is not supported. Perhaps fold one of the dimensions?") - } - default: - // This should've been filtered out by collectMarshallabeTypes. - panic(fmt.Sprintf("Unexpected type %+v", ty)) - } - return i -} - -// generateOneTestSuite generates a test suite for the automatically generated -// implementations type t. -func (g *Generator) generateOneTestSuite(t *marshallableType) *testGenerator { - i := newTestGenerator(t.spec, t.recv) - i.emitTests(t.slice) - return i -} - -// Run is the entry point to code generation using g. -// -// Run parses all input source files specified in g and emits generated code. -func (g *Generator) Run() error { - // Parse our input source files into ASTs and token sets. - asts, fsets, err := g.parse() - if err != nil { - return err - } - - if len(asts) != len(fsets) { - panic("ASTs and FileSets don't match") - } - - // Map of imports in source files; key = local package name, value = import - // path. - is := make(map[string]importStmt) - for i, a := range asts { - // Collect all imports from the source files. We may need to copy some - // of these to the generated code if they're referenced. This has to be - // done before the loop below because we need to process all ASTs before - // we start requesting imports to be copied one by one as we encounter - // them in each generated source. - for name, i := range g.collectImports(a, fsets[i]) { - is[name] = i - } - } - - var impls []*interfaceGenerator - var ts []*testGenerator - // Set of Marshallable types referenced by generated code. - ms := make(map[string]struct{}) - for i, a := range asts { - // Collect type declarations marked for code generation and generate - // Marshallable interfaces. - var sortedTypes []*marshallableType - for _, t := range g.collectMarshallableTypes(a, fsets[i]) { - sortedTypes = append(sortedTypes, t) - } - sort.Slice(sortedTypes, func(x, y int) bool { - // Sort by type name, which should be unique within a package. - return sortedTypes[x].spec.Name.String() < sortedTypes[y].spec.Name.String() - }) - for _, t := range sortedTypes { - impl := g.generateOne(t, fsets[i]) - // Collect Marshallable types referenced by the generated code. - for ref := range impl.ms { - ms[ref] = struct{}{} - } - impls = append(impls, impl) - // Collect imports referenced by the generated code and add them to - // the list of imports we need to copy to the generated code. - for name := range impl.is { - if !g.imports.markUsed(name) { - panic(fmt.Sprintf("Generated code for '%s' referenced a non-existent import with local name '%s'. Either go-marshal needs to add an import to the generated file, or a package in an input source file has a package name differ from the final component of its path, which go-marshal doesn't know how to detect; use an import alias to work around this limitation.", impl.typeName(), name)) - } - } - // Do not generate tests for dynamic types because they inherently - // violate some go_marshal requirements. - if !t.dynamic { - ts = append(ts, g.generateOneTestSuite(t)) - } - } - } - - // Write output file header. These include things like package name and - // import statements. - if err := g.writeHeader(); err != nil { - return err - } - - // Write type checks for referenced marshallable types to output file. - if err := g.writeTypeChecks(ms); err != nil { - return err - } - - // Write generated interfaces to output file. - for _, i := range impls { - if err := i.write(g.output); err != nil { - return err - } - } - - // Write generated tests to test file. - return g.writeTests(ts) -} - -// writeTests outputs tests for the generated interface implementations to a go -// source file. -func (g *Generator) writeTests(ts []*testGenerator) error { - var b sourceBuffer - - // Write the unconditional test file. This file is always compiled, - // regardless of what build tags were specified on the original input - // files. We use this file to guarantee we never end up with an empty test - // file, as that causes the build to fail with "no tests/benchmarks/examples - // found". - // - // There's no easy way to determine ahead of time if we'll end up with an - // empty build file since build constraints can arbitrarily cause some of - // the original types to be not defined. We also have no way to tell bazel - // to omit the entire test suite since the output files are already defined - // before go-marshal is called. - b.emit("// Automatically generated marshal tests. See tools/go_marshal.\n\n") - b.emit("package %s\n\n", g.pkg) - b.emit("func Example() {\n") - b.inIndent(func() { - b.emit("// This example is intentionally empty, and ensures this package contains at\n") - b.emit("// least one testable entity. go-marshal is forced to emit a test package if the\n") - b.emit("// input package is marked marshallable, but emitting no testable entities \n") - b.emit("// results in a build failure.\n") - }) - b.emit("}\n") - if err := b.write(g.outputTestUC); err != nil { - return err - } - - // Now generate the real test file that contains the real types we - // processed. These need to be conditionally compiled according to the build - // tags, as the original types may not be defined under all build - // configurations. - - b.reset() - b.emit("// Automatically generated marshal tests. See tools/go_marshal.\n\n") - - // Emit build tags. - if t := tags.Aggregate(g.inputs); len(t) > 0 { - b.emit(strings.Join(t.Lines(), "\n")) - b.emit("\n\n") - } - - b.emit("package %s\n\n", g.pkg) - if err := b.write(g.outputTest); err != nil { - return err - } - - // Collect and write test import statements. - imports := newImportTable() - for _, t := range ts { - imports.merge(t.imports) - } - - if err := imports.write(g.outputTest); err != nil { - return err - } - - // Write test functions. - for _, t := range ts { - if err := t.write(g.outputTest); err != nil { - return err - } - } - return nil -} diff --git a/tools/go_marshal/gomarshal/generator_interfaces.go b/tools/go_marshal/gomarshal/generator_interfaces.go deleted file mode 100644 index 3e643e77f..000000000 --- a/tools/go_marshal/gomarshal/generator_interfaces.go +++ /dev/null @@ -1,276 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gomarshal - -import ( - "fmt" - "go/ast" - "go/token" - "strings" -) - -// interfaceGenerator generates marshalling interfaces for a single type. -// -// getState is not thread-safe. -type interfaceGenerator struct { - sourceBuffer - - // The type we're serializing. - t *ast.TypeSpec - - // Receiver argument for generated methods. - r string - - // FileSet containing the tokens for the type we're processing. - f *token.FileSet - - // is records external packages referenced by the generated implementation. - is map[string]struct{} - - // ms records Marshallable types referenced by the generated implementation - // of t's interfaces. - ms map[string]struct{} - - // as records fields in t that are potentially not packed. The key is the - // accessor for the field. - as map[string]struct{} -} - -// typeName returns the name of the type this g represents. -func (g *interfaceGenerator) typeName() string { - return g.t.Name.Name -} - -// newinterfaceGenerator creates a new interface generator. -func newInterfaceGenerator(t *ast.TypeSpec, r string, fset *token.FileSet) *interfaceGenerator { - g := &interfaceGenerator{ - t: t, - r: r, - f: fset, - is: make(map[string]struct{}), - ms: make(map[string]struct{}), - as: make(map[string]struct{}), - } - g.recordUsedMarshallable(g.typeName()) - return g -} - -func (g *interfaceGenerator) recordUsedMarshallable(m string) { - g.ms[m] = struct{}{} - -} - -func (g *interfaceGenerator) recordUsedImport(i string) { - g.is[i] = struct{}{} -} - -func (g *interfaceGenerator) recordPotentiallyNonPackedField(fieldName string) { - g.as[fieldName] = struct{}{} -} - -// abortAt aborts the go_marshal tool with the given error message, with a -// reference position to the input source. Same as abortAt, but uses g to -// resolve p to position. -func (g *interfaceGenerator) abortAt(p token.Pos, msg string) { - abortAt(g.f.Position(p), msg) -} - -// scalarSize returns the size of type identified by t. If t isn't a primitive -// type, the size isn't known at code generation time, and must be resolved via -// the marshal.Marshallable interface. -func (g *interfaceGenerator) scalarSize(t *ast.Ident) (size int, unknownSize bool) { - switch t.Name { - case "int8", "uint8", "byte": - return 1, false - case "int16", "uint16": - return 2, false - case "int32", "uint32": - return 4, false - case "int64", "uint64": - return 8, false - default: - return 0, true - } -} - -func (g *interfaceGenerator) shift(bufVar string, n int) { - g.emit("%s = %s[%d:]\n", bufVar, bufVar, n) -} - -func (g *interfaceGenerator) shiftDynamic(bufVar, name string) { - g.emit("%s = %s[%s.SizeBytes():]\n", bufVar, bufVar, name) -} - -// marshalScalar writes a single scalar to a byte slice. -func (g *interfaceGenerator) marshalScalar(accessor, typ, bufVar string) { - switch typ { - case "int8", "uint8", "byte": - g.emit("%s[0] = byte(%s)\n", bufVar, accessor) - g.shift(bufVar, 1) - case "int16", "uint16": - g.recordUsedImport("hostarch") - g.emit("hostarch.ByteOrder.PutUint16(%s[:2], uint16(%s))\n", bufVar, accessor) - g.shift(bufVar, 2) - case "int32", "uint32": - g.recordUsedImport("hostarch") - g.emit("hostarch.ByteOrder.PutUint32(%s[:4], uint32(%s))\n", bufVar, accessor) - g.shift(bufVar, 4) - case "int64", "uint64": - g.recordUsedImport("hostarch") - g.emit("hostarch.ByteOrder.PutUint64(%s[:8], uint64(%s))\n", bufVar, accessor) - g.shift(bufVar, 8) - default: - g.emit("%s.MarshalBytes(%s[:%s.SizeBytes()])\n", accessor, bufVar, accessor) - g.shiftDynamic(bufVar, accessor) - } -} - -// unmarshalScalar reads a single scalar from a byte slice. -func (g *interfaceGenerator) unmarshalScalar(accessor, typ, bufVar string) { - switch typ { - case "byte": - g.emit("%s = %s[0]\n", accessor, bufVar) - g.shift(bufVar, 1) - case "int8", "uint8": - g.emit("%s = %s(%s[0])\n", accessor, typ, bufVar) - g.shift(bufVar, 1) - case "int16", "uint16": - g.recordUsedImport("hostarch") - g.emit("%s = %s(hostarch.ByteOrder.Uint16(%s[:2]))\n", accessor, typ, bufVar) - g.shift(bufVar, 2) - case "int32", "uint32": - g.recordUsedImport("hostarch") - g.emit("%s = %s(hostarch.ByteOrder.Uint32(%s[:4]))\n", accessor, typ, bufVar) - g.shift(bufVar, 4) - case "int64", "uint64": - g.recordUsedImport("hostarch") - g.emit("%s = %s(hostarch.ByteOrder.Uint64(%s[:8]))\n", accessor, typ, bufVar) - g.shift(bufVar, 8) - default: - g.emit("%s.UnmarshalBytes(%s[:%s.SizeBytes()])\n", accessor, bufVar, accessor) - g.shiftDynamic(bufVar, accessor) - g.recordPotentiallyNonPackedField(accessor) - } -} - -// emitCastToByteSlice unsafely casts an arbitrary type's underlying memory to a -// byte slice, bypassing escape analysis. The caller is responsible for ensuring -// srcPtr lives until they're done with dstVar, the runtime does not consider -// dstVar dependent on srcPtr due to the escape analysis bypass. -// -// srcPtr must be a pointer. -// -// This function uses internally uses the identifier "hdr", and cannot be used -// in a context where it is already bound. -func (g *interfaceGenerator) emitCastToByteSlice(srcPtr, dstVar, lenExpr string) { - g.recordUsedImport("gohacks") - g.emit("// Construct a slice backed by dst's underlying memory.\n") - g.emit("var %s []byte\n", dstVar) - g.emit("hdr := (*reflect.SliceHeader)(unsafe.Pointer(&%s))\n", dstVar) - g.emit("hdr.Data = uintptr(gohacks.Noescape(unsafe.Pointer(%s)))\n", srcPtr) - g.emit("hdr.Len = %s\n", lenExpr) - g.emit("hdr.Cap = %s\n\n", lenExpr) -} - -// emitCastToByteSlice unsafely casts a slice with elements of an abitrary type -// to a byte slice. As part of the cast, the byte slice is made to look -// independent of the src slice by bypassing escape analysis. This means the -// byte slice can be used without causing the source to escape. The caller is -// responsible for ensuring srcPtr lives until they're done with dstVar, as the -// runtime no longer considers dstVar dependent on srcPtr and is free to GC it. -// -// srcPtr must be a pointer. -// -// This function uses internally uses the identifiers "ptr", "val" and "hdr", -// and cannot be used in a context where these identifiers are already bound. -func (g *interfaceGenerator) emitCastSliceToByteSlice(srcPtr, dstVar, lenExpr string) { - g.emitNoEscapeSliceDataPointer(srcPtr, "val") - - g.emit("// Construct a slice backed by dst's underlying memory.\n") - g.emit("var %s []byte\n", dstVar) - g.emit("hdr := (*reflect.SliceHeader)(unsafe.Pointer(&%s))\n", dstVar) - g.emit("hdr.Data = uintptr(val)\n") - g.emit("hdr.Len = %s\n", lenExpr) - g.emit("hdr.Cap = %s\n\n", lenExpr) -} - -// emitNoEscapeSliceDataPointer unsafely casts a slice's data pointer to an -// unsafe.Pointer, bypassing escape analysis. The caller is responsible for -// ensuring srcPtr lives until they're done with dstVar, as the runtime no -// longer considers dstVar dependent on srcPtr and is free to GC it. -// -// srcPtr must be a pointer. -// -// This function uses internally uses the identifier "ptr" cannot be used in a -// context where this identifier is already bound. -func (g *interfaceGenerator) emitNoEscapeSliceDataPointer(srcPtr, dstVar string) { - g.recordUsedImport("gohacks") - g.emit("ptr := unsafe.Pointer(%s)\n", srcPtr) - g.emit("%s := gohacks.Noescape(unsafe.Pointer((*reflect.SliceHeader)(ptr).Data))\n\n", dstVar) -} - -func (g *interfaceGenerator) emitKeepAlive(ptrVar string) { - g.emit("// Since we bypassed the compiler's escape analysis, indicate that %s\n", ptrVar) - g.emit("// must live until the use above.\n") - g.emit("runtime.KeepAlive(%s) // escapes: replaced by intrinsic.\n", ptrVar) -} - -func (g *interfaceGenerator) expandBinaryExpr(b *strings.Builder, e *ast.BinaryExpr) { - switch x := e.X.(type) { - case *ast.BinaryExpr: - // Recursively expand sub-expression. - g.expandBinaryExpr(b, x) - case *ast.Ident: - fmt.Fprintf(b, "%s", x.Name) - case *ast.BasicLit: - fmt.Fprintf(b, "%s", x.Value) - default: - g.abortAt(e.Pos(), "Cannot convert binary expression to output code. Go-marshal currently only handles simple expressions of literals, constants and basic identifiers") - } - - fmt.Fprintf(b, "%s", e.Op) - - switch y := e.Y.(type) { - case *ast.BinaryExpr: - // Recursively expand sub-expression. - g.expandBinaryExpr(b, y) - case *ast.Ident: - fmt.Fprintf(b, "%s", y.Name) - case *ast.BasicLit: - fmt.Fprintf(b, "%s", y.Value) - default: - g.abortAt(e.Pos(), "Cannot convert binary expression to output code. Go-marshal currently only handles simple expressions of literals, constants and basic identifiers") - } -} - -// arrayLenExpr returns a string containing a valid golang expression -// representing the length of array a. The returned expression should be treated -// as a single value, and will be already parenthesized as required. -func (g *interfaceGenerator) arrayLenExpr(a *ast.ArrayType) string { - var b strings.Builder - - switch l := a.Len.(type) { - case *ast.Ident: - fmt.Fprintf(&b, "%s", l.Name) - case *ast.BasicLit: - fmt.Fprintf(&b, "%s", l.Value) - case *ast.BinaryExpr: - g.expandBinaryExpr(&b, l) - return fmt.Sprintf("(%s)", b.String()) - default: - g.abortAt(l.Pos(), "Cannot convert this array len expression to output code. Go-marshal currently only handles simple expressions of literals, constants and basic identifiers") - } - return b.String() -} diff --git a/tools/go_marshal/gomarshal/generator_interfaces_array_newtype.go b/tools/go_marshal/gomarshal/generator_interfaces_array_newtype.go deleted file mode 100644 index bd7741ae5..000000000 --- a/tools/go_marshal/gomarshal/generator_interfaces_array_newtype.go +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This file contains the bits of the code generator specific to marshalling -// newtypes on arrays. - -package gomarshal - -import ( - "fmt" - "go/ast" -) - -func (g *interfaceGenerator) validateArrayNewtype(n *ast.Ident, a *ast.ArrayType) { - if a.Len == nil { - g.abortAt(a.Pos(), fmt.Sprintf("Dynamically sized slice '%s' cannot be marshalled, arrays must be statically sized", n.Name)) - } - - if _, ok := a.Elt.(*ast.Ident); !ok { - g.abortAt(a.Elt.Pos(), fmt.Sprintf("Marshalling not supported for arrays with %s elements, array elements must be primitive types", kindString(a.Elt))) - } -} - -func (g *interfaceGenerator) emitMarshallableForArrayNewtype(n *ast.Ident, a *ast.ArrayType, elt *ast.Ident) { - g.recordUsedImport("gohacks") - g.recordUsedImport("hostarch") - g.recordUsedImport("io") - g.recordUsedImport("marshal") - g.recordUsedImport("reflect") - g.recordUsedImport("runtime") - g.recordUsedImport("unsafe") - - lenExpr := g.arrayLenExpr(a) - - g.emit("// SizeBytes implements marshal.Marshallable.SizeBytes.\n") - g.emit("//go:nosplit\n") - g.emit("func (%s *%s) SizeBytes() int {\n", g.r, g.typeName()) - g.inIndent(func() { - if size, dynamic := g.scalarSize(elt); !dynamic { - g.emit("return %d * %s\n", size, lenExpr) - } else { - g.emit("return (*%s)(nil).SizeBytes() * %s\n", n.Name, lenExpr) - } - }) - g.emit("}\n\n") - - g.emit("// MarshalBytes implements marshal.Marshallable.MarshalBytes.\n") - g.emit("func (%s *%s) MarshalBytes(dst []byte) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("for idx := 0; idx < %s; idx++ {\n", lenExpr) - g.inIndent(func() { - g.marshalScalar(fmt.Sprintf("%s[idx]", g.r), elt.Name, "dst") - }) - g.emit("}\n") - }) - g.emit("}\n\n") - - g.emit("// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes.\n") - g.emit("func (%s *%s) UnmarshalBytes(src []byte) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("for idx := 0; idx < %s; idx++ {\n", lenExpr) - g.inIndent(func() { - g.unmarshalScalar(fmt.Sprintf("%s[idx]", g.r), elt.Name, "src") - }) - g.emit("}\n") - }) - g.emit("}\n\n") - - g.emit("// Packed implements marshal.Marshallable.Packed.\n") - g.emit("//go:nosplit\n") - g.emit("func (%s *%s) Packed() bool {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("// Array newtypes are always packed.\n") - g.emit("return true\n") - }) - g.emit("}\n\n") - - g.emit("// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe.\n") - g.emit("func (%s *%s) MarshalUnsafe(dst []byte) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&%s[0]), uintptr(%s.SizeBytes()))\n", g.r, g.r) - }) - g.emit("}\n\n") - - g.emit("// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe.\n") - g.emit("func (%s *%s) UnmarshalUnsafe(src []byte) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("gohacks.Memmove(unsafe.Pointer(%s), unsafe.Pointer(&src[0]), uintptr(%s.SizeBytes()))\n", g.r, g.r) - }) - g.emit("}\n\n") - - g.emit("// CopyOutN implements marshal.Marshallable.CopyOutN.\n") - g.emit("//go:nosplit\n") - g.emit("func (%s *%s) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emitCastToByteSlice(g.r, "buf", fmt.Sprintf("%s.SizeBytes()", g.r)) - - g.emit("length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay.\n") - g.emitKeepAlive(g.r) - g.emit("return length, err\n") - }) - g.emit("}\n\n") - - g.emit("// CopyOut implements marshal.Marshallable.CopyOut.\n") - g.emit("//go:nosplit\n") - g.emit("func (%s *%s) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("return %s.CopyOutN(cc, addr, %s.SizeBytes())\n", g.r, g.r) - }) - g.emit("}\n\n") - - g.emit("// CopyIn implements marshal.Marshallable.CopyIn.\n") - g.emit("//go:nosplit\n") - g.emit("func (%s *%s) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emitCastToByteSlice(g.r, "buf", fmt.Sprintf("%s.SizeBytes()", g.r)) - - g.emit("length, err := cc.CopyInBytes(addr, buf) // escapes: okay.\n") - g.emitKeepAlive(g.r) - g.emit("return length, err\n") - }) - g.emit("}\n\n") - - g.emit("// WriteTo implements io.WriterTo.WriteTo.\n") - g.emit("func (%s *%s) WriteTo(w io.Writer) (int64, error) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emitCastToByteSlice(g.r, "buf", fmt.Sprintf("%s.SizeBytes()", g.r)) - - g.emit("length, err := w.Write(buf)\n") - g.emitKeepAlive(g.r) - g.emit("return int64(length), err\n") - - }) - g.emit("}\n\n") -} diff --git a/tools/go_marshal/gomarshal/generator_interfaces_dynamic.go b/tools/go_marshal/gomarshal/generator_interfaces_dynamic.go deleted file mode 100644 index 345020ddc..000000000 --- a/tools/go_marshal/gomarshal/generator_interfaces_dynamic.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2021 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gomarshal - -func (g *interfaceGenerator) emitMarshallableForDynamicType() { - // The user writes their own MarshalBytes, UnmarshalBytes and SizeBytes for - // dynamic types. Generate the rest using these definitions. - - g.emit("// Packed implements marshal.Marshallable.Packed.\n") - g.emit("//go:nosplit\n") - g.emit("func (%s *%s) Packed() bool {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("// Type %s is dynamic so it might have slice/string headers. Hence, it is not packed.\n", g.typeName()) - g.emit("return false\n") - }) - g.emit("}\n\n") - - g.emit("// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe.\n") - g.emit("func (%s *%s) MarshalUnsafe(dst []byte) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("// Type %s doesn't have a packed layout in memory, fallback to MarshalBytes.\n", g.typeName()) - g.emit("%s.MarshalBytes(dst)\n", g.r) - }) - g.emit("}\n\n") - - g.emit("// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe.\n") - g.emit("func (%s *%s) UnmarshalUnsafe(src []byte) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("// Type %s doesn't have a packed layout in memory, fallback to UnmarshalBytes.\n", g.typeName()) - g.emit("%s.UnmarshalBytes(src)\n", g.r) - }) - g.emit("}\n\n") - - g.emit("// CopyOutN implements marshal.Marshallable.CopyOutN.\n") - g.emit("//go:nosplit\n") - g.recordUsedImport("marshal") - g.recordUsedImport("hostarch") - g.emit("func (%s *%s) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("// Type %s doesn't have a packed layout in memory, fall back to MarshalBytes.\n", g.typeName()) - g.emit("buf := cc.CopyScratchBuffer(%s.SizeBytes()) // escapes: okay.\n", g.r) - g.emit("%s.MarshalBytes(buf) // escapes: fallback.\n", g.r) - g.emit("return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay.\n") - }) - g.emit("}\n\n") - - g.emit("// CopyOut implements marshal.Marshallable.CopyOut.\n") - g.emit("//go:nosplit\n") - g.recordUsedImport("marshal") - g.recordUsedImport("hostarch") - g.emit("func (%s *%s) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("return %s.CopyOutN(cc, addr, %s.SizeBytes())\n", g.r, g.r) - }) - g.emit("}\n\n") - - g.emit("// CopyIn implements marshal.Marshallable.CopyIn.\n") - g.emit("//go:nosplit\n") - g.recordUsedImport("marshal") - g.recordUsedImport("hostarch") - g.emit("func (%s *%s) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("// Type %s doesn't have a packed layout in memory, fall back to UnmarshalBytes.\n", g.typeName()) - g.emit("buf := cc.CopyScratchBuffer(%s.SizeBytes()) // escapes: okay.\n", g.r) - g.emit("length, err := cc.CopyInBytes(addr, buf) // escapes: okay.\n") - g.emit("// Unmarshal unconditionally. If we had a short copy-in, this results in a\n") - g.emit("// partially unmarshalled struct.\n") - g.emit("%s.UnmarshalBytes(buf) // escapes: fallback.\n", g.r) - g.emit("return length, err\n") - }) - g.emit("}\n\n") - - g.emit("// WriteTo implements io.WriterTo.WriteTo.\n") - g.recordUsedImport("io") - g.emit("func (%s *%s) WriteTo(writer io.Writer) (int64, error) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("// Type %s doesn't have a packed layout in memory, fall back to MarshalBytes.\n", g.typeName()) - g.emit("buf := make([]byte, %s.SizeBytes())\n", g.r) - g.emit("%s.MarshalBytes(buf)\n", g.r) - g.emit("length, err := writer.Write(buf)\n") - g.emit("return int64(length), err\n") - }) - g.emit("}\n\n") -} diff --git a/tools/go_marshal/gomarshal/generator_interfaces_primitive_newtype.go b/tools/go_marshal/gomarshal/generator_interfaces_primitive_newtype.go deleted file mode 100644 index ba4b7324e..000000000 --- a/tools/go_marshal/gomarshal/generator_interfaces_primitive_newtype.go +++ /dev/null @@ -1,285 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This file contains the bits of the code generator specific to marshalling -// newtypes on primitives. - -package gomarshal - -import ( - "fmt" - "go/ast" -) - -// marshalPrimitiveScalar writes a single primitive variable to a byte -// slice. -func (g *interfaceGenerator) marshalPrimitiveScalar(accessor, typ, bufVar string) { - switch typ { - case "int8", "uint8", "byte": - g.emit("%s[0] = byte(*%s)\n", bufVar, accessor) - case "int16", "uint16": - g.recordUsedImport("hostarch") - g.emit("hostarch.ByteOrder.PutUint16(%s[:2], uint16(*%s))\n", bufVar, accessor) - case "int32", "uint32": - g.recordUsedImport("hostarch") - g.emit("hostarch.ByteOrder.PutUint32(%s[:4], uint32(*%s))\n", bufVar, accessor) - case "int64", "uint64": - g.recordUsedImport("hostarch") - g.emit("hostarch.ByteOrder.PutUint64(%s[:8], uint64(*%s))\n", bufVar, accessor) - default: - g.emit("// Explicilty cast to the underlying type before dispatching to\n") - g.emit("// MarshalBytes, so we don't recursively call %s.MarshalBytes\n", accessor) - g.emit("inner := (*%s)(%s)\n", typ, accessor) - g.emit("inner.MarshalBytes(%s[:%s.SizeBytes()])\n", bufVar, accessor) - } -} - -// unmarshalPrimitiveScalar read a single primitive variable from a byte slice. -func (g *interfaceGenerator) unmarshalPrimitiveScalar(accessor, typ, bufVar, typeCast string) { - switch typ { - case "byte": - g.emit("*%s = %s(%s[0])\n", accessor, typeCast, bufVar) - case "int8", "uint8": - g.emit("*%s = %s(%s(%s[0]))\n", accessor, typeCast, typ, bufVar) - case "int16", "uint16": - g.recordUsedImport("hostarch") - g.emit("*%s = %s(%s(hostarch.ByteOrder.Uint16(%s[:2])))\n", accessor, typeCast, typ, bufVar) - case "int32", "uint32": - g.recordUsedImport("hostarch") - g.emit("*%s = %s(%s(hostarch.ByteOrder.Uint32(%s[:4])))\n", accessor, typeCast, typ, bufVar) - case "int64", "uint64": - g.recordUsedImport("hostarch") - g.emit("*%s = %s(%s(hostarch.ByteOrder.Uint64(%s[:8])))\n", accessor, typeCast, typ, bufVar) - default: - g.emit("// Explicilty cast to the underlying type before dispatching to\n") - g.emit("// UnmarshalBytes, so we don't recursively call %s.UnmarshalBytes\n", accessor) - g.emit("inner := (*%s)(%s)\n", typ, accessor) - g.emit("inner.UnmarshalBytes(%s[:%s.SizeBytes()])\n", bufVar, accessor) - } -} - -func (g *interfaceGenerator) validatePrimitiveNewtype(t *ast.Ident) { - switch t.Name { - case "int8", "uint8", "byte", "int16", "uint16", "int32", "uint32", "int64", "uint64": - // These are the only primitive types we're allow. Below, we provide - // suggestions for some disallowed types and reject them, then attempt - // to marshal any remaining types by invoking the marshal.Marshallable - // interface on them. If these types don't actually implement - // marshal.Marshallable, compilation of the generated code will fail - // with an appropriate error message. - return - case "int": - g.abortAt(t.Pos(), "Type 'int' has ambiguous width, use int32 or int64") - case "uint": - g.abortAt(t.Pos(), "Type 'uint' has ambiguous width, use uint32 or uint64") - case "string": - g.abortAt(t.Pos(), "Type 'string' is dynamically-sized and cannot be marshalled, use a fixed size byte array '[...]byte' instead") - default: - debugfAt(g.f.Position(t.Pos()), fmt.Sprintf("Found derived type '%s', will attempt dispatch via marshal.Marshallable.\n", t.Name)) - } -} - -// emitMarshallableForPrimitiveNewtype outputs code to implement the -// marshal.Marshallable interface for a newtype on a primitive. Primitive -// newtypes are always packed, so we can omit the various fallbacks required for -// non-packed structs. -func (g *interfaceGenerator) emitMarshallableForPrimitiveNewtype(nt *ast.Ident) { - g.recordUsedImport("gohacks") - g.recordUsedImport("hostarch") - g.recordUsedImport("io") - g.recordUsedImport("marshal") - g.recordUsedImport("reflect") - g.recordUsedImport("runtime") - g.recordUsedImport("unsafe") - - g.emit("// SizeBytes implements marshal.Marshallable.SizeBytes.\n") - g.emit("//go:nosplit\n") - g.emit("func (%s *%s) SizeBytes() int {\n", g.r, g.typeName()) - g.inIndent(func() { - if size, dynamic := g.scalarSize(nt); !dynamic { - g.emit("return %d\n", size) - } else { - g.emit("return (*%s)(nil).SizeBytes()\n", nt.Name) - } - }) - g.emit("}\n\n") - - g.emit("// MarshalBytes implements marshal.Marshallable.MarshalBytes.\n") - g.emit("func (%s *%s) MarshalBytes(dst []byte) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.marshalPrimitiveScalar(g.r, nt.Name, "dst") - }) - g.emit("}\n\n") - - g.emit("// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes.\n") - g.emit("func (%s *%s) UnmarshalBytes(src []byte) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.unmarshalPrimitiveScalar(g.r, nt.Name, "src", g.typeName()) - }) - g.emit("}\n\n") - - g.emit("// Packed implements marshal.Marshallable.Packed.\n") - g.emit("//go:nosplit\n") - g.emit("func (%s *%s) Packed() bool {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("// Scalar newtypes are always packed.\n") - g.emit("return true\n") - }) - g.emit("}\n\n") - - g.emit("// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe.\n") - g.emit("func (%s *%s) MarshalUnsafe(dst []byte) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(%s), uintptr(%s.SizeBytes()))\n", g.r, g.r) - }) - g.emit("}\n\n") - - g.emit("// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe.\n") - g.emit("func (%s *%s) UnmarshalUnsafe(src []byte) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("gohacks.Memmove(unsafe.Pointer(%s), unsafe.Pointer(&src[0]), uintptr(%s.SizeBytes()))\n", g.r, g.r) - }) - g.emit("}\n\n") - - g.emit("// CopyOutN implements marshal.Marshallable.CopyOutN.\n") - g.emit("//go:nosplit\n") - g.emit("func (%s *%s) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emitCastToByteSlice(g.r, "buf", fmt.Sprintf("%s.SizeBytes()", g.r)) - - g.emit("length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay.\n") - g.emitKeepAlive(g.r) - g.emit("return length, err\n") - }) - g.emit("}\n\n") - - g.emit("// CopyOut implements marshal.Marshallable.CopyOut.\n") - g.emit("//go:nosplit\n") - g.emit("func (%s *%s) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("return %s.CopyOutN(cc, addr, %s.SizeBytes())\n", g.r, g.r) - }) - g.emit("}\n\n") - - g.emit("// CopyIn implements marshal.Marshallable.CopyIn.\n") - g.emit("//go:nosplit\n") - g.emit("func (%s *%s) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emitCastToByteSlice(g.r, "buf", fmt.Sprintf("%s.SizeBytes()", g.r)) - - g.emit("length, err := cc.CopyInBytes(addr, buf) // escapes: okay.\n") - g.emitKeepAlive(g.r) - g.emit("return length, err\n") - }) - g.emit("}\n\n") - - g.emit("// WriteTo implements io.WriterTo.WriteTo.\n") - g.emit("func (%s *%s) WriteTo(w io.Writer) (int64, error) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emitCastToByteSlice(g.r, "buf", fmt.Sprintf("%s.SizeBytes()", g.r)) - - g.emit("length, err := w.Write(buf)\n") - g.emitKeepAlive(g.r) - g.emit("return int64(length), err\n") - - }) - g.emit("}\n\n") -} - -func (g *interfaceGenerator) emitMarshallableSliceForPrimitiveNewtype(nt *ast.Ident, slice *sliceAPI) { - g.recordUsedImport("marshal") - g.recordUsedImport("hostarch") - g.recordUsedImport("reflect") - g.recordUsedImport("runtime") - g.recordUsedImport("unsafe") - - eltType := g.typeName() - if slice.inner { - eltType = nt.Name - } - - g.emit("// Copy%sIn copies in a slice of %s objects from the task's memory.\n", slice.ident, eltType) - g.emit("//go:nosplit\n") - g.emit("func Copy%sIn(cc marshal.CopyContext, addr hostarch.Addr, dst []%s) (int, error) {\n", slice.ident, eltType) - g.inIndent(func() { - g.emit("count := len(dst)\n") - g.emit("if count == 0 {\n") - g.inIndent(func() { - g.emit("return 0, nil\n") - }) - g.emit("}\n") - g.emit("size := (*%s)(nil).SizeBytes()\n\n", g.typeName()) - - g.emitCastSliceToByteSlice("&dst", "buf", "size * count") - - g.emit("length, err := cc.CopyInBytes(addr, buf) // escapes: okay.\n") - g.emitKeepAlive("dst") - g.emit("return length, err\n") - }) - g.emit("}\n\n") - - g.emit("// Copy%sOut copies a slice of %s objects to the task's memory.\n", slice.ident, eltType) - g.emit("//go:nosplit\n") - g.emit("func Copy%sOut(cc marshal.CopyContext, addr hostarch.Addr, src []%s) (int, error) {\n", slice.ident, eltType) - g.inIndent(func() { - g.emit("count := len(src)\n") - g.emit("if count == 0 {\n") - g.inIndent(func() { - g.emit("return 0, nil\n") - }) - g.emit("}\n") - g.emit("size := (*%s)(nil).SizeBytes()\n\n", g.typeName()) - - g.emitCastSliceToByteSlice("&src", "buf", "size * count") - - g.emit("length, err := cc.CopyOutBytes(addr, buf) // escapes: okay.\n") - g.emitKeepAlive("src") - g.emit("return length, err\n") - }) - g.emit("}\n\n") - - g.emit("// MarshalUnsafe%s is like %s.MarshalUnsafe, but for a []%s.\n", slice.ident, g.typeName(), g.typeName()) - g.emit("func MarshalUnsafe%s(src []%s, dst []byte) (int, error) {\n", slice.ident, g.typeName()) - g.inIndent(func() { - g.emit("count := len(src)\n") - g.emit("if count == 0 {\n") - g.inIndent(func() { - g.emit("return 0, nil\n") - }) - g.emit("}\n") - g.emit("size := (*%s)(nil).SizeBytes()\n\n", g.typeName()) - - g.emit("dst = dst[:size*count]\n") - g.emit("gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&src[0]), uintptr(len(dst)))\n") - g.emit("return size*count, nil\n") - }) - g.emit("}\n\n") - - g.emit("// UnmarshalUnsafe%s is like %s.UnmarshalUnsafe, but for a []%s.\n", slice.ident, g.typeName(), g.typeName()) - g.emit("func UnmarshalUnsafe%s(dst []%s, src []byte) (int, error) {\n", slice.ident, g.typeName()) - g.inIndent(func() { - g.emit("count := len(dst)\n") - g.emit("if count == 0 {\n") - g.inIndent(func() { - g.emit("return 0, nil\n") - }) - g.emit("}\n") - g.emit("size := (*%s)(nil).SizeBytes()\n\n", g.typeName()) - - g.emit("src = src[:(size*count)]\n") - g.emit("gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&src[0]), uintptr(len(src)))\n") - g.emit("return size*count, nil\n") - }) - g.emit("}\n\n") -} diff --git a/tools/go_marshal/gomarshal/generator_interfaces_struct.go b/tools/go_marshal/gomarshal/generator_interfaces_struct.go deleted file mode 100644 index 4c47218f1..000000000 --- a/tools/go_marshal/gomarshal/generator_interfaces_struct.go +++ /dev/null @@ -1,616 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This file contains the bits of the code generator specific to marshalling -// structs. - -package gomarshal - -import ( - "fmt" - "go/ast" - "sort" - "strings" -) - -func (g *interfaceGenerator) fieldAccessor(n *ast.Ident) string { - return fmt.Sprintf("%s.%s", g.r, n.Name) -} - -// areFieldsPackedExpression returns a go expression checking whether g.t's fields are -// packed. Returns "", false if g.t has no fields that may be potentially -// packed, otherwise returns <clause>, true, where <clause> is an expression -// like "t.a.Packed() && t.b.Packed() && t.c.Packed()". -func (g *interfaceGenerator) areFieldsPackedExpression() (string, bool) { - if len(g.as) == 0 { - return "", false - } - - cs := make([]string, 0, len(g.as)) - for accessor := range g.as { - cs = append(cs, fmt.Sprintf("%s.Packed()", accessor)) - } - // Sort expressions for determinstic build outputs. - sort.Strings(cs) - return strings.Join(cs, " && "), true -} - -// validateStruct ensures the type we're working with can be marshalled. These -// checks are done ahead of time and in one place so we can make assumptions -// later. -func (g *interfaceGenerator) validateStruct(ts *ast.TypeSpec, st *ast.StructType) { - forEachStructField(st, func(f *ast.Field) { - fieldDispatcher{ - primitive: func(_, t *ast.Ident) { - g.validatePrimitiveNewtype(t) - }, - selector: func(_, _, _ *ast.Ident) { - // No validation to perform on selector fields. However this - // callback must still be provided. - }, - array: func(n *ast.Ident, a *ast.ArrayType, _ *ast.Ident) { - g.validateArrayNewtype(n, a) - }, - unhandled: func(_ *ast.Ident) { - g.abortAt(f.Pos(), fmt.Sprintf("Marshalling not supported for %s fields", kindString(f.Type))) - }, - }.dispatch(f) - }) -} - -func (g *interfaceGenerator) isStructPacked(st *ast.StructType) bool { - packed := true - forEachStructField(st, func(f *ast.Field) { - if f.Tag != nil { - if f.Tag.Value == "`marshal:\"unaligned\"`" { - if packed { - debugfAt(g.f.Position(g.t.Pos()), - fmt.Sprintf("Marking type '%s' as not packed due to tag `marshal:\"unaligned\"`.\n", g.t.Name)) - packed = false - } - } - } - }) - return packed -} - -func (g *interfaceGenerator) emitMarshallableForStruct(st *ast.StructType) { - thisPacked := g.isStructPacked(st) - - g.emit("// SizeBytes implements marshal.Marshallable.SizeBytes.\n") - g.emit("func (%s *%s) SizeBytes() int {\n", g.r, g.typeName()) - g.inIndent(func() { - primitiveSize := 0 - var dynamicSizeTerms []string - - forEachStructField(st, fieldDispatcher{ - primitive: func(_, t *ast.Ident) { - if size, dynamic := g.scalarSize(t); !dynamic { - primitiveSize += size - } else { - g.recordUsedMarshallable(t.Name) - dynamicSizeTerms = append(dynamicSizeTerms, fmt.Sprintf("(*%s)(nil).SizeBytes()", t.Name)) - } - }, - selector: func(_, tX, tSel *ast.Ident) { - tName := fmt.Sprintf("%s.%s", tX.Name, tSel.Name) - g.recordUsedImport(tX.Name) - g.recordUsedMarshallable(tName) - dynamicSizeTerms = append(dynamicSizeTerms, fmt.Sprintf("(*%s)(nil).SizeBytes()", tName)) - }, - array: func(_ *ast.Ident, a *ast.ArrayType, t *ast.Ident) { - lenExpr := g.arrayLenExpr(a) - if size, dynamic := g.scalarSize(t); !dynamic { - dynamicSizeTerms = append(dynamicSizeTerms, fmt.Sprintf("%d*%s", size, lenExpr)) - } else { - g.recordUsedMarshallable(t.Name) - dynamicSizeTerms = append(dynamicSizeTerms, fmt.Sprintf("(*%s)(nil).SizeBytes()*%s", t.Name, lenExpr)) - } - }, - }.dispatch) - g.emit("return %d", primitiveSize) - if len(dynamicSizeTerms) > 0 { - g.incIndent() - } - { - for _, d := range dynamicSizeTerms { - g.emitNoIndent(" +\n") - g.emit(d) - } - } - if len(dynamicSizeTerms) > 0 { - g.decIndent() - } - }) - g.emit("\n}\n\n") - - g.emit("// MarshalBytes implements marshal.Marshallable.MarshalBytes.\n") - g.emit("func (%s *%s) MarshalBytes(dst []byte) {\n", g.r, g.typeName()) - g.inIndent(func() { - forEachStructField(st, fieldDispatcher{ - primitive: func(n, t *ast.Ident) { - if n.Name == "_" { - g.emit("// Padding: dst[:sizeof(%s)] ~= %s(0)\n", t.Name, t.Name) - if len, dynamic := g.scalarSize(t); !dynamic { - g.shift("dst", len) - } else { - // We can't use shiftDynamic here because we don't have - // an instance of the dynamic type we can reference here - // (since the version in this struct is anonymous). Use - // a typed nil pointer to call SizeBytes() instead. - g.emit("dst = dst[(*%s)(nil).SizeBytes():]\n", t.Name) - } - return - } - g.marshalScalar(g.fieldAccessor(n), t.Name, "dst") - }, - selector: func(n, tX, tSel *ast.Ident) { - if n.Name == "_" { - g.emit("// Padding: dst[:sizeof(%s)] ~= %s(0)\n", tX.Name, tSel.Name) - g.emit("dst = dst[(*%s.%s)(nil).SizeBytes():]\n", tX.Name, tSel.Name) - return - } - g.marshalScalar(g.fieldAccessor(n), fmt.Sprintf("%s.%s", tX.Name, tSel.Name), "dst") - }, - array: func(n *ast.Ident, a *ast.ArrayType, t *ast.Ident) { - lenExpr := g.arrayLenExpr(a) - if n.Name == "_" { - g.emit("// Padding: dst[:sizeof(%s)*%s] ~= [%s]%s{0}\n", t.Name, lenExpr, lenExpr, t.Name) - if size, dynamic := g.scalarSize(t); !dynamic { - g.emit("dst = dst[%d*(%s):]\n", size, lenExpr) - } else { - // We can't use shiftDynamic here because we don't have - // an instance of the dynamic type we can reference here - // (since the version in this struct is anonymous). Use - // a typed nil pointer to call SizeBytes() instead. - g.emit("dst = dst[(*%s)(nil).SizeBytes()*(%s):]\n", t.Name, lenExpr) - } - return - } - - g.emit("for idx := 0; idx < %s; idx++ {\n", lenExpr) - g.inIndent(func() { - g.marshalScalar(fmt.Sprintf("%s[idx]", g.fieldAccessor(n)), t.Name, "dst") - }) - g.emit("}\n") - }, - }.dispatch) - }) - g.emit("}\n\n") - - g.emit("// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes.\n") - g.emit("func (%s *%s) UnmarshalBytes(src []byte) {\n", g.r, g.typeName()) - g.inIndent(func() { - forEachStructField(st, fieldDispatcher{ - primitive: func(n, t *ast.Ident) { - if n.Name == "_" { - g.emit("// Padding: var _ %s ~= src[:sizeof(%s)]\n", t.Name, t.Name) - if len, dynamic := g.scalarSize(t); !dynamic { - g.shift("src", len) - } else { - // We don't have an instance of the dynamic type we can - // reference here (since the version in this struct is - // anonymous). Use a typed nil pointer to call - // SizeBytes() instead. - g.shiftDynamic("src", fmt.Sprintf("(*%s)(nil)", t.Name)) - g.recordPotentiallyNonPackedField(fmt.Sprintf("(*%s)(nil)", t.Name)) - } - return - } - g.unmarshalScalar(g.fieldAccessor(n), t.Name, "src") - }, - selector: func(n, tX, tSel *ast.Ident) { - if n.Name == "_" { - g.emit("// Padding: %s ~= src[:sizeof(%s.%s)]\n", g.fieldAccessor(n), tX.Name, tSel.Name) - g.emit("src = src[(*%s.%s)(nil).SizeBytes():]\n", tX.Name, tSel.Name) - g.recordPotentiallyNonPackedField(fmt.Sprintf("(*%s.%s)(nil)", tX.Name, tSel.Name)) - return - } - g.unmarshalScalar(g.fieldAccessor(n), fmt.Sprintf("%s.%s", tX.Name, tSel.Name), "src") - }, - array: func(n *ast.Ident, a *ast.ArrayType, t *ast.Ident) { - lenExpr := g.arrayLenExpr(a) - if n.Name == "_" { - g.emit("// Padding: ~ copy([%s]%s(%s), src[:sizeof(%s)*%s])\n", lenExpr, t.Name, g.fieldAccessor(n), t.Name, lenExpr) - if size, dynamic := g.scalarSize(t); !dynamic { - g.emit("src = src[%d*(%s):]\n", size, lenExpr) - } else { - // We can't use shiftDynamic here because we don't have - // an instance of the dynamic type we can referece here - // (since the version in this struct is anonymous). Use - // a typed nil pointer to call SizeBytes() instead. - g.emit("src = src[(*%s)(nil).SizeBytes()*(%s):]\n", t.Name, lenExpr) - } - return - } - - g.emit("for idx := 0; idx < %s; idx++ {\n", lenExpr) - g.inIndent(func() { - g.unmarshalScalar(fmt.Sprintf("%s[idx]", g.fieldAccessor(n)), t.Name, "src") - }) - g.emit("}\n") - }, - }.dispatch) - }) - g.emit("}\n\n") - - g.emit("// Packed implements marshal.Marshallable.Packed.\n") - g.emit("//go:nosplit\n") - g.emit("func (%s *%s) Packed() bool {\n", g.r, g.typeName()) - g.inIndent(func() { - expr, fieldsMaybePacked := g.areFieldsPackedExpression() - switch { - case !thisPacked: - g.emit("return false\n") - case fieldsMaybePacked: - g.emit("return %s\n", expr) - default: - g.emit("return true\n") - - } - }) - g.emit("}\n\n") - - g.emit("// MarshalUnsafe implements marshal.Marshallable.MarshalUnsafe.\n") - g.emit("func (%s *%s) MarshalUnsafe(dst []byte) {\n", g.r, g.typeName()) - g.inIndent(func() { - fallback := func() { - g.emit("// Type %s doesn't have a packed layout in memory, fallback to MarshalBytes.\n", g.typeName()) - g.emit("%s.MarshalBytes(dst)\n", g.r) - } - if thisPacked { - g.recordUsedImport("gohacks") - g.recordUsedImport("unsafe") - if cond, ok := g.areFieldsPackedExpression(); ok { - g.emit("if %s {\n", cond) - g.inIndent(func() { - g.emit("gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(%s), uintptr(%s.SizeBytes()))\n", g.r, g.r) - }) - g.emit("} else {\n") - g.inIndent(fallback) - g.emit("}\n") - } else { - g.emit("gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(%s), uintptr(%s.SizeBytes()))\n", g.r, g.r) - } - } else { - fallback() - } - }) - g.emit("}\n\n") - - g.emit("// UnmarshalUnsafe implements marshal.Marshallable.UnmarshalUnsafe.\n") - g.emit("func (%s *%s) UnmarshalUnsafe(src []byte) {\n", g.r, g.typeName()) - g.inIndent(func() { - fallback := func() { - g.emit("// Type %s doesn't have a packed layout in memory, fallback to UnmarshalBytes.\n", g.typeName()) - g.emit("%s.UnmarshalBytes(src)\n", g.r) - } - if thisPacked { - g.recordUsedImport("gohacks") - if cond, ok := g.areFieldsPackedExpression(); ok { - g.emit("if %s {\n", cond) - g.inIndent(func() { - g.emit("gohacks.Memmove(unsafe.Pointer(%s), unsafe.Pointer(&src[0]), uintptr(%s.SizeBytes()))\n", g.r, g.r) - }) - g.emit("} else {\n") - g.inIndent(fallback) - g.emit("}\n") - } else { - g.emit("gohacks.Memmove(unsafe.Pointer(%s), unsafe.Pointer(&src[0]), uintptr(%s.SizeBytes()))\n", g.r, g.r) - } - } else { - fallback() - } - }) - g.emit("}\n\n") - g.emit("// CopyOutN implements marshal.Marshallable.CopyOutN.\n") - g.emit("//go:nosplit\n") - g.recordUsedImport("marshal") - g.recordUsedImport("hostarch") - g.emit("func (%s *%s) CopyOutN(cc marshal.CopyContext, addr hostarch.Addr, limit int) (int, error) {\n", g.r, g.typeName()) - g.inIndent(func() { - fallback := func() { - g.emit("// Type %s doesn't have a packed layout in memory, fall back to MarshalBytes.\n", g.typeName()) - g.emit("buf := cc.CopyScratchBuffer(%s.SizeBytes()) // escapes: okay.\n", g.r) - g.emit("%s.MarshalBytes(buf) // escapes: fallback.\n", g.r) - g.emit("return cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay.\n") - } - if thisPacked { - g.recordUsedImport("reflect") - g.recordUsedImport("runtime") - g.recordUsedImport("unsafe") - if cond, ok := g.areFieldsPackedExpression(); ok { - g.emit("if !%s {\n", cond) - g.inIndent(fallback) - g.emit("}\n\n") - } - // Fast serialization. - g.emitCastToByteSlice(g.r, "buf", fmt.Sprintf("%s.SizeBytes()", g.r)) - - g.emit("length, err := cc.CopyOutBytes(addr, buf[:limit]) // escapes: okay.\n") - g.emitKeepAlive(g.r) - g.emit("return length, err\n") - } else { - fallback() - } - }) - g.emit("}\n\n") - - g.emit("// CopyOut implements marshal.Marshallable.CopyOut.\n") - g.emit("//go:nosplit\n") - g.recordUsedImport("marshal") - g.recordUsedImport("hostarch") - g.emit("func (%s *%s) CopyOut(cc marshal.CopyContext, addr hostarch.Addr) (int, error) {\n", g.r, g.typeName()) - g.inIndent(func() { - g.emit("return %s.CopyOutN(cc, addr, %s.SizeBytes())\n", g.r, g.r) - }) - g.emit("}\n\n") - - g.emit("// CopyIn implements marshal.Marshallable.CopyIn.\n") - g.emit("//go:nosplit\n") - g.recordUsedImport("marshal") - g.recordUsedImport("hostarch") - g.emit("func (%s *%s) CopyIn(cc marshal.CopyContext, addr hostarch.Addr) (int, error) {\n", g.r, g.typeName()) - g.inIndent(func() { - fallback := func() { - g.emit("// Type %s doesn't have a packed layout in memory, fall back to UnmarshalBytes.\n", g.typeName()) - g.emit("buf := cc.CopyScratchBuffer(%s.SizeBytes()) // escapes: okay.\n", g.r) - g.emit("length, err := cc.CopyInBytes(addr, buf) // escapes: okay.\n") - g.emit("// Unmarshal unconditionally. If we had a short copy-in, this results in a\n") - g.emit("// partially unmarshalled struct.\n") - g.emit("%s.UnmarshalBytes(buf) // escapes: fallback.\n", g.r) - g.emit("return length, err\n") - } - if thisPacked { - g.recordUsedImport("reflect") - g.recordUsedImport("runtime") - g.recordUsedImport("unsafe") - if cond, ok := g.areFieldsPackedExpression(); ok { - g.emit("if !%s {\n", cond) - g.inIndent(fallback) - g.emit("}\n\n") - } - // Fast deserialization. - g.emitCastToByteSlice(g.r, "buf", fmt.Sprintf("%s.SizeBytes()", g.r)) - - g.emit("length, err := cc.CopyInBytes(addr, buf) // escapes: okay.\n") - g.emitKeepAlive(g.r) - g.emit("return length, err\n") - } else { - fallback() - } - }) - g.emit("}\n\n") - - g.emit("// WriteTo implements io.WriterTo.WriteTo.\n") - g.recordUsedImport("io") - g.emit("func (%s *%s) WriteTo(writer io.Writer) (int64, error) {\n", g.r, g.typeName()) - g.inIndent(func() { - fallback := func() { - g.emit("// Type %s doesn't have a packed layout in memory, fall back to MarshalBytes.\n", g.typeName()) - g.emit("buf := make([]byte, %s.SizeBytes())\n", g.r) - g.emit("%s.MarshalBytes(buf)\n", g.r) - g.emit("length, err := writer.Write(buf)\n") - g.emit("return int64(length), err\n") - } - if thisPacked { - g.recordUsedImport("reflect") - g.recordUsedImport("runtime") - g.recordUsedImport("unsafe") - if cond, ok := g.areFieldsPackedExpression(); ok { - g.emit("if !%s {\n", cond) - g.inIndent(fallback) - g.emit("}\n\n") - } - // Fast serialization. - g.emitCastToByteSlice(g.r, "buf", fmt.Sprintf("%s.SizeBytes()", g.r)) - - g.emit("length, err := writer.Write(buf)\n") - g.emitKeepAlive(g.r) - g.emit("return int64(length), err\n") - } else { - fallback() - } - }) - g.emit("}\n\n") -} - -func (g *interfaceGenerator) emitMarshallableSliceForStruct(st *ast.StructType, slice *sliceAPI) { - thisPacked := g.isStructPacked(st) - - if slice.inner { - abortAt(g.f.Position(slice.comment.Slash), fmt.Sprintf("The ':inner' argument to '+marshal slice:%s:inner' is only applicable to newtypes on primitives. Remove it from this struct declaration.", slice.ident)) - } - - g.recordUsedImport("marshal") - g.recordUsedImport("hostarch") - - g.emit("// Copy%sIn copies in a slice of %s objects from the task's memory.\n", slice.ident, g.typeName()) - g.emit("func Copy%sIn(cc marshal.CopyContext, addr hostarch.Addr, dst []%s) (int, error) {\n", slice.ident, g.typeName()) - g.inIndent(func() { - g.emit("count := len(dst)\n") - g.emit("if count == 0 {\n") - g.inIndent(func() { - g.emit("return 0, nil\n") - }) - g.emit("}\n") - g.emit("size := (*%s)(nil).SizeBytes()\n\n", g.typeName()) - - fallback := func() { - g.emit("// Type %s doesn't have a packed layout in memory, fall back to UnmarshalBytes.\n", g.typeName()) - g.emit("buf := cc.CopyScratchBuffer(size * count)\n") - g.emit("length, err := cc.CopyInBytes(addr, buf)\n\n") - - g.emit("// Unmarshal as much as possible, even on error. First handle full objects.\n") - g.emit("limit := length/size\n") - g.emit("for idx := 0; idx < limit; idx++ {\n") - g.inIndent(func() { - g.emit("dst[idx].UnmarshalBytes(buf[size*idx:size*(idx+1)])\n") - }) - g.emit("}\n\n") - - g.emit("// Handle any final partial object. buf is guaranteed to be long enough for the\n") - g.emit("// final element, but may not contain valid data for the entire range. This may\n") - g.emit("// result in unmarshalling zero values for some parts of the object.\n") - g.emit("if length%size != 0 {\n") - g.inIndent(func() { - g.emit("idx := limit\n") - g.emit("dst[idx].UnmarshalBytes(buf[size*idx:size*(idx+1)])\n") - }) - g.emit("}\n\n") - - g.emit("return length, err\n") - } - if thisPacked { - g.recordUsedImport("reflect") - g.recordUsedImport("runtime") - g.recordUsedImport("unsafe") - if _, ok := g.areFieldsPackedExpression(); ok { - g.emit("if !dst[0].Packed() {\n") - g.inIndent(fallback) - g.emit("}\n\n") - } - // Fast deserialization. - g.emitCastSliceToByteSlice("&dst", "buf", "size * count") - - g.emit("length, err := cc.CopyInBytes(addr, buf)\n") - g.emitKeepAlive("dst") - g.emit("return length, err\n") - } else { - fallback() - } - }) - g.emit("}\n\n") - - g.emit("// Copy%sOut copies a slice of %s objects to the task's memory.\n", slice.ident, g.typeName()) - g.emit("func Copy%sOut(cc marshal.CopyContext, addr hostarch.Addr, src []%s) (int, error) {\n", slice.ident, g.typeName()) - g.inIndent(func() { - g.emit("count := len(src)\n") - g.emit("if count == 0 {\n") - g.inIndent(func() { - g.emit("return 0, nil\n") - }) - g.emit("}\n") - g.emit("size := (*%s)(nil).SizeBytes()\n\n", g.typeName()) - - fallback := func() { - g.emit("// Type %s doesn't have a packed layout in memory, fall back to MarshalBytes.\n", g.typeName()) - g.emit("buf := cc.CopyScratchBuffer(size * count)\n") - g.emit("for idx := 0; idx < count; idx++ {\n") - g.inIndent(func() { - g.emit("src[idx].MarshalBytes(buf[size*idx:size*(idx+1)])\n") - }) - g.emit("}\n") - g.emit("return cc.CopyOutBytes(addr, buf)\n") - } - if thisPacked { - g.recordUsedImport("reflect") - g.recordUsedImport("runtime") - g.recordUsedImport("unsafe") - if _, ok := g.areFieldsPackedExpression(); ok { - g.emit("if !src[0].Packed() {\n") - g.inIndent(fallback) - g.emit("}\n\n") - } - // Fast serialization. - g.emitCastSliceToByteSlice("&src", "buf", "size * count") - - g.emit("length, err := cc.CopyOutBytes(addr, buf)\n") - g.emitKeepAlive("src") - g.emit("return length, err\n") - } else { - fallback() - } - }) - g.emit("}\n\n") - - g.emit("// MarshalUnsafe%s is like %s.MarshalUnsafe, but for a []%s.\n", slice.ident, g.typeName(), g.typeName()) - g.emit("func MarshalUnsafe%s(src []%s, dst []byte) (int, error) {\n", slice.ident, g.typeName()) - g.inIndent(func() { - g.emit("count := len(src)\n") - g.emit("if count == 0 {\n") - g.inIndent(func() { - g.emit("return 0, nil\n") - }) - g.emit("}\n") - g.emit("size := (*%s)(nil).SizeBytes()\n\n", g.typeName()) - - fallback := func() { - g.emit("// Type %s doesn't have a packed layout in memory, fall back to MarshalBytes.\n", g.typeName()) - g.emit("for idx := 0; idx < count; idx++ {\n") - g.inIndent(func() { - g.emit("src[idx].MarshalBytes(dst[size*idx:(size)*(idx+1)])\n") - }) - g.emit("}\n") - g.emit("return size * count, nil\n") - } - if thisPacked { - g.recordUsedImport("reflect") - g.recordUsedImport("runtime") - g.recordUsedImport("unsafe") - g.recordUsedImport("gohacks") - if _, ok := g.areFieldsPackedExpression(); ok { - g.emit("if !src[0].Packed() {\n") - g.inIndent(fallback) - g.emit("}\n\n") - } - g.emit("dst = dst[:size*count]\n") - g.emit("gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&src[0]), uintptr(len(dst)))\n") - g.emit("return size * count, nil\n") - } else { - fallback() - } - }) - g.emit("}\n\n") - - g.emit("// UnmarshalUnsafe%s is like %s.UnmarshalUnsafe, but for a []%s.\n", slice.ident, g.typeName(), g.typeName()) - g.emit("func UnmarshalUnsafe%s(dst []%s, src []byte) (int, error) {\n", slice.ident, g.typeName()) - g.inIndent(func() { - g.emit("count := len(dst)\n") - g.emit("if count == 0 {\n") - g.inIndent(func() { - g.emit("return 0, nil\n") - }) - g.emit("}\n") - g.emit("size := (*%s)(nil).SizeBytes()\n\n", g.typeName()) - - fallback := func() { - g.emit("// Type %s doesn't have a packed layout in memory, fall back to UnmarshalBytes.\n", g.typeName()) - g.emit("for idx := 0; idx < count; idx++ {\n") - g.inIndent(func() { - g.emit("dst[idx].UnmarshalBytes(src[size*idx:size*(idx+1)])\n") - }) - g.emit("}\n") - g.emit("return size * count, nil\n") - } - if thisPacked { - g.recordUsedImport("gohacks") - g.recordUsedImport("reflect") - g.recordUsedImport("runtime") - if _, ok := g.areFieldsPackedExpression(); ok { - g.emit("if !dst[0].Packed() {\n") - g.inIndent(fallback) - g.emit("}\n\n") - } - - g.emit("src = src[:(size*count)]\n") - g.emit("gohacks.Memmove(unsafe.Pointer(&dst[0]), unsafe.Pointer(&src[0]), uintptr(len(src)))\n") - - g.emit("return count*size, nil\n") - } else { - fallback() - } - }) - g.emit("}\n\n") -} diff --git a/tools/go_marshal/gomarshal/generator_tests.go b/tools/go_marshal/gomarshal/generator_tests.go deleted file mode 100644 index 8f93a1de5..000000000 --- a/tools/go_marshal/gomarshal/generator_tests.go +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gomarshal - -import ( - "fmt" - "go/ast" - "io" - "strings" -) - -var standardImports = []string{ - "bytes", - "fmt", - "reflect", - "testing", - - "gvisor.dev/gvisor/tools/go_marshal/analysis", -} - -var sliceAPIImports = []string{ - "encoding/binary", - "gvisor.dev/gvisor/pkg/hostarch", -} - -type testGenerator struct { - sourceBuffer - - // The type we're serializing. - t *ast.TypeSpec - - // Receiver argument for generated methods. - r string - - // Imports used by generated code. - imports *importTable - - // Import statement for the package declaring the type we generated code - // for. We need this to construct test instances for the type, since the - // tests aren't written in the same package. - decl *importStmt -} - -func newTestGenerator(t *ast.TypeSpec, r string) *testGenerator { - g := &testGenerator{ - t: t, - r: r, - imports: newImportTable(), - } - - for _, i := range standardImports { - g.imports.add(i).markUsed() - } - // These imports are used if a type requests the slice API. Don't - // mark them as used by default. - for _, i := range sliceAPIImports { - g.imports.add(i) - } - - return g -} - -func (g *testGenerator) typeName() string { - return g.t.Name.Name -} - -func (g *testGenerator) testFuncName(base string) string { - return fmt.Sprintf("%s%s", base, strings.Title(g.t.Name.Name)) -} - -func (g *testGenerator) inTestFunction(name string, body func()) { - g.emit("func %s(t *testing.T) {\n", g.testFuncName(name)) - g.inIndent(body) - g.emit("}\n\n") -} - -func (g *testGenerator) emitTestNonZeroSize() { - g.inTestFunction("TestSizeNonZero", func() { - g.emit("var x %v\n", g.typeName()) - g.emit("if x.SizeBytes() == 0 {\n") - g.inIndent(func() { - g.emit("t.Fatal(\"Marshallable.SizeBytes() should not return zero\")\n") - }) - g.emit("}\n") - }) -} - -func (g *testGenerator) emitTestSuspectAlignment() { - g.inTestFunction("TestSuspectAlignment", func() { - g.emit("var x %v\n", g.typeName()) - g.emit("analysis.AlignmentCheck(t, reflect.TypeOf(x))\n") - }) -} - -func (g *testGenerator) emitTestMarshalUnmarshalPreservesData() { - g.inTestFunction("TestSafeMarshalUnmarshalPreservesData", func() { - g.emit("var x, y, z, yUnsafe, zUnsafe %s\n", g.typeName()) - g.emit("analysis.RandomizeValue(&x)\n\n") - - g.emit("buf := make([]byte, x.SizeBytes())\n") - g.emit("x.MarshalBytes(buf)\n") - g.emit("bufUnsafe := make([]byte, x.SizeBytes())\n") - g.emit("x.MarshalUnsafe(bufUnsafe)\n\n") - - g.emit("y.UnmarshalBytes(buf)\n") - g.emit("if !reflect.DeepEqual(x, y) {\n") - g.inIndent(func() { - g.emit("t.Fatal(fmt.Sprintf(\"Data corrupted across MarshalBytes/UnmarshalBytes cycle:\\nBefore: %+v\\nAfter: %+v\\n\", x, y))\n") - }) - g.emit("}\n") - g.emit("yUnsafe.UnmarshalBytes(bufUnsafe)\n") - g.emit("if !reflect.DeepEqual(x, yUnsafe) {\n") - g.inIndent(func() { - g.emit("t.Fatal(fmt.Sprintf(\"Data corrupted across MarshalUnsafe/UnmarshalBytes cycle:\\nBefore: %+v\\nAfter: %+v\\n\", x, yUnsafe))\n") - }) - g.emit("}\n\n") - - g.emit("z.UnmarshalUnsafe(buf)\n") - g.emit("if !reflect.DeepEqual(x, z) {\n") - g.inIndent(func() { - g.emit("t.Fatal(fmt.Sprintf(\"Data corrupted across MarshalBytes/UnmarshalUnsafe cycle:\\nBefore: %+v\\nAfter: %+v\\n\", x, z))\n") - }) - g.emit("}\n") - g.emit("zUnsafe.UnmarshalUnsafe(bufUnsafe)\n") - g.emit("if !reflect.DeepEqual(x, zUnsafe) {\n") - g.inIndent(func() { - g.emit("t.Fatal(fmt.Sprintf(\"Data corrupted across MarshalUnsafe/UnmarshalUnsafe cycle:\\nBefore: %+v\\nAfter: %+v\\n\", x, zUnsafe))\n") - }) - g.emit("}\n") - }) -} - -func (g *testGenerator) emitTestMarshalUnmarshalSlicePreservesData(slice *sliceAPI) { - for _, name := range []string{"binary", "hostarch"} { - if !g.imports.markUsed(name) { - panic(fmt.Sprintf("Generated test for '%s' referenced a non-existent import with local name '%s'", g.typeName(), name)) - } - } - - g.inTestFunction("TestSafeMarshalUnmarshalSlicePreservesData", func() { - g.emit("var x, y, yUnsafe [8]%s\n", g.typeName()) - g.emit("analysis.RandomizeValue(&x)\n\n") - g.emit("size := (*%s)(nil).SizeBytes() * len(x)\n", g.typeName()) - g.emit("buf := bytes.NewBuffer(make([]byte, size))\n") - g.emit("buf.Reset()\n") - g.emit("if err := binary.Write(buf, hostarch.ByteOrder, x[:]); err != nil {\n") - g.inIndent(func() { - g.emit("t.Fatal(fmt.Sprintf(\"binary.Write failed: %v\", err))\n") - }) - g.emit("}\n") - g.emit("bufUnsafe := make([]byte, size)\n") - g.emit("MarshalUnsafe%s(x[:], bufUnsafe)\n\n", slice.ident) - - g.emit("UnmarshalUnsafe%s(y[:], buf.Bytes())\n", slice.ident) - g.emit("if !reflect.DeepEqual(x, y) {\n") - g.inIndent(func() { - g.emit("t.Fatal(fmt.Sprintf(\"Data corrupted across binary.Write/UnmarshalUnsafeSlice cycle:\\nBefore: %+v\\nAfter: %+v\\n\", x, y))\n") - }) - g.emit("}\n") - g.emit("UnmarshalUnsafe%s(yUnsafe[:], bufUnsafe)\n", slice.ident) - g.emit("if !reflect.DeepEqual(x, yUnsafe) {\n") - g.inIndent(func() { - g.emit("t.Fatal(fmt.Sprintf(\"Data corrupted across MarshalUnsafeSlice/UnmarshalUnsafeSlice cycle:\\nBefore: %+v\\nAfter: %+v\\n\", x, yUnsafe))\n") - }) - g.emit("}\n\n") - }) -} - -func (g *testGenerator) emitTestWriteToUnmarshalPreservesData() { - g.inTestFunction("TestWriteToUnmarshalPreservesData", func() { - g.emit("var x, y, yUnsafe %s\n", g.typeName()) - g.emit("analysis.RandomizeValue(&x)\n\n") - - g.emit("var buf bytes.Buffer\n\n") - - g.emit("x.WriteTo(&buf)\n") - g.emit("y.UnmarshalBytes(buf.Bytes())\n\n") - g.emit("yUnsafe.UnmarshalUnsafe(buf.Bytes())\n\n") - - g.emit("if !reflect.DeepEqual(x, y) {\n") - g.inIndent(func() { - g.emit("t.Fatal(fmt.Sprintf(\"Data corrupted across WriteTo/UnmarshalBytes cycle:\\nBefore: %+v\\nAfter: %+v\\n\", x, y))\n") - }) - g.emit("}\n") - g.emit("if !reflect.DeepEqual(x, yUnsafe) {\n") - g.inIndent(func() { - g.emit("t.Fatal(fmt.Sprintf(\"Data corrupted across WriteTo/UnmarshalUnsafe cycle:\\nBefore: %+v\\nAfter: %+v\\n\", x, yUnsafe))\n") - }) - g.emit("}\n") - }) -} - -func (g *testGenerator) emitTestSizeBytesOnTypedNilPtr() { - g.inTestFunction("TestSizeBytesOnTypedNilPtr", func() { - g.emit("var x %s\n", g.typeName()) - g.emit("sizeFromConcrete := x.SizeBytes()\n") - g.emit("sizeFromTypedNilPtr := (*%s)(nil).SizeBytes()\n\n", g.typeName()) - - g.emit("if sizeFromTypedNilPtr != sizeFromConcrete {\n") - g.inIndent(func() { - g.emit("t.Fatalf(\"SizeBytes() on typed nil pointer (%v) doesn't match size returned by a concrete object (%v).\\n\", sizeFromTypedNilPtr, sizeFromConcrete)\n") - }) - g.emit("}\n") - }) -} - -func (g *testGenerator) emitTests(slice *sliceAPI) { - g.emitTestNonZeroSize() - g.emitTestSuspectAlignment() - g.emitTestMarshalUnmarshalPreservesData() - g.emitTestWriteToUnmarshalPreservesData() - g.emitTestSizeBytesOnTypedNilPtr() - - if slice != nil { - g.emitTestMarshalUnmarshalSlicePreservesData(slice) - } -} - -func (g *testGenerator) write(out io.Writer) error { - return g.sourceBuffer.write(out) -} diff --git a/tools/go_marshal/gomarshal/util.go b/tools/go_marshal/gomarshal/util.go deleted file mode 100644 index 6a42691cd..000000000 --- a/tools/go_marshal/gomarshal/util.go +++ /dev/null @@ -1,503 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gomarshal - -import ( - "bytes" - "flag" - "fmt" - "go/ast" - "go/token" - "io" - "os" - "path" - "reflect" - "sort" - "strings" -) - -var debug = flag.Bool("debug", false, "enables debugging output") - -// receiverName returns an appropriate receiver name given a type spec. -func receiverName(t *ast.TypeSpec) string { - if len(t.Name.Name) < 1 { - // Zero length type name? - panic("unreachable") - } - return strings.ToLower(t.Name.Name[:1]) -} - -// kindString returns a user-friendly representation of an AST expr type. -func kindString(e ast.Expr) string { - switch e.(type) { - case *ast.Ident: - return "scalar" - case *ast.ArrayType: - return "array" - case *ast.StructType: - return "struct" - case *ast.StarExpr: - return "pointer" - case *ast.FuncType: - return "function" - case *ast.InterfaceType: - return "interface" - case *ast.MapType: - return "map" - case *ast.ChanType: - return "channel" - default: - return reflect.TypeOf(e).String() - } -} - -func forEachStructField(st *ast.StructType, fn func(f *ast.Field)) { - for _, field := range st.Fields.List { - fn(field) - } -} - -// fieldDispatcher is a collection of callbacks for handling different types of -// fields in a struct declaration. -type fieldDispatcher struct { - primitive func(n, t *ast.Ident) - selector func(n, tX, tSel *ast.Ident) - array func(n *ast.Ident, a *ast.ArrayType, t *ast.Ident) - unhandled func(n *ast.Ident) -} - -// Precondition: All dispatch callbacks that will be invoked must be -// provided. -func (fd fieldDispatcher) dispatch(f *ast.Field) { - // Each field declaration may actually be multiple declarations of the same - // type. For example, consider: - // - // type Point struct { - // x, y, z int - // } - // - // We invoke the call-backs once per such instance. - - // Handle embedded fields. Embedded fields have no names, but can be - // referenced by the type name. - if len(f.Names) < 1 { - switch v := f.Type.(type) { - case *ast.Ident: - fd.primitive(v, v) - case *ast.SelectorExpr: - fd.selector(v.Sel, v.X.(*ast.Ident), v.Sel) - default: - // Note: Arrays can't be embedded, which is handled here. - panic(fmt.Sprintf("Attempted to dispatch on embedded field of unsupported kind: %#v", f.Type)) - } - return - } - - // Non-embedded field. - for _, name := range f.Names { - switch v := f.Type.(type) { - case *ast.Ident: - fd.primitive(name, v) - case *ast.SelectorExpr: - fd.selector(name, v.X.(*ast.Ident), v.Sel) - case *ast.ArrayType: - switch t := v.Elt.(type) { - case *ast.Ident: - fd.array(name, v, t) - default: - // Should be handled with a better error message during validate. - panic(fmt.Sprintf("Array element type is of unsupported kind. Expected *ast.Ident, got %v", t)) - } - default: - fd.unhandled(name) - } - } -} - -// debugEnabled indicates whether debugging is enabled for gomarshal. -func debugEnabled() bool { - return *debug -} - -// abort aborts the go_marshal tool with the given error message. -func abort(msg string) { - if !strings.HasSuffix(msg, "\n") { - msg += "\n" - } - fmt.Print(msg) - os.Exit(1) -} - -// abortAt aborts the go_marshal tool with the given error message, with -// a reference position to the input source. -func abortAt(p token.Position, msg string) { - abort(fmt.Sprintf("%v:\n %s\n", p, msg)) -} - -// debugf conditionally prints a debug message. -func debugf(f string, a ...interface{}) { - if debugEnabled() { - fmt.Printf(f, a...) - } -} - -// debugfAt conditionally prints a debug message with a reference to a position -// in the input source. -func debugfAt(p token.Position, f string, a ...interface{}) { - if debugEnabled() { - fmt.Printf("%s:\n %s", p, fmt.Sprintf(f, a...)) - } -} - -// emit generates a line of code in the output file. -// -// emit is a wrapper around writing a formatted string to the output -// buffer. emit can be invoked in one of two ways: -// -// (1) emit("some string") -// When emit is called with a single string argument, it is simply copied to -// the output buffer without any further formatting. -// (2) emit(fmtString, args...) -// emit can also be invoked in a similar fashion to *Printf() functions, -// where the first argument is a format string. -// -// Calling emit with a single argument that is not a string will result in a -// panic, as the caller's intent is ambiguous. -func emit(out io.Writer, indent int, a ...interface{}) { - const spacesPerIndentLevel = 4 - - if len(a) < 1 { - panic("emit() called with no arguments") - } - - if indent > 0 { - if _, err := fmt.Fprint(out, strings.Repeat(" ", indent*spacesPerIndentLevel)); err != nil { - // Writing to the emit output should not fail. Typically the output - // is a byte.Buffer; writes to these never fail. - panic(err) - } - } - - first, ok := a[0].(string) - if !ok { - // First argument must be either the string to emit (case 1 from - // function-level comment), or a format string (case 2). - panic(fmt.Sprintf("First argument to emit() is not a string: %+v", a[0])) - } - - if len(a) == 1 { - // Single string argument. Assume no formatting requested. - if _, err := fmt.Fprint(out, first); err != nil { - // Writing to out should not fail. - panic(err) - } - return - - } - - // Formatting requested. - if _, err := fmt.Fprintf(out, first, a[1:]...); err != nil { - // Writing to out should not fail. - panic(err) - } -} - -// sourceBuffer represents fragments of generated go source code. -// -// sourceBuffer provides a convenient way to build up go souce fragments in -// memory. May be safely zero-value initialized. Not thread-safe. -type sourceBuffer struct { - // Current indentation level. - indent int - - // Memory buffer containing contents while they're being generated. - b bytes.Buffer -} - -func (b *sourceBuffer) reset() { - b.indent = 0 - b.b.Reset() -} - -func (b *sourceBuffer) incIndent() { - b.indent++ -} - -func (b *sourceBuffer) decIndent() { - if b.indent <= 0 { - panic("decIndent() without matching incIndent()") - } - b.indent-- -} - -func (b *sourceBuffer) emit(a ...interface{}) { - emit(&b.b, b.indent, a...) -} - -func (b *sourceBuffer) emitNoIndent(a ...interface{}) { - emit(&b.b, 0 /*indent*/, a...) -} - -func (b *sourceBuffer) inIndent(body func()) { - b.incIndent() - body() - b.decIndent() -} - -func (b *sourceBuffer) write(out io.Writer) error { - _, err := fmt.Fprint(out, b.b.String()) - return err -} - -// Write implements io.Writer.Write. -func (b *sourceBuffer) Write(buf []byte) (int, error) { - return (b.b.Write(buf)) -} - -// importStmt represents a single import statement. -type importStmt struct { - // Local name of the imported package. - name string - // Import path. - path string - // Indicates whether the local name is an alias, or simply the final - // component of the path. - aliased bool - // Indicates whether this import was referenced by generated code. - used bool - // AST node and file set representing the import statement, if any. These - // are only non-nil if the import statement originates from an input source - // file. - spec *ast.ImportSpec - fset *token.FileSet -} - -func newImport(p string) *importStmt { - name := path.Base(p) - return &importStmt{ - name: name, - path: p, - aliased: false, - } -} - -func newImportFromSpec(spec *ast.ImportSpec, f *token.FileSet) *importStmt { - p := spec.Path.Value[1 : len(spec.Path.Value)-1] // Strip the " quotes around path. - name := path.Base(p) - if name == "" || name == "/" || name == "." { - panic(fmt.Sprintf("Couldn't process local package name for import at %s, (processed as %s)", - f.Position(spec.Path.Pos()), name)) - } - if spec.Name != nil { - name = spec.Name.Name - } - return &importStmt{ - name: name, - path: p, - aliased: spec.Name != nil, - spec: spec, - fset: f, - } -} - -// String implements fmt.Stringer.String. This generates a string for the import -// statement appropriate for writing directly to generated code. -func (i *importStmt) String() string { - if i.aliased { - return fmt.Sprintf("%s %q", i.name, i.path) - } - return fmt.Sprintf("%q", i.path) -} - -// debugString returns a debug string representing an import statement. This -// representation is not valid golang code and is used for debugging output. -func (i *importStmt) debugString() string { - if i.spec != nil && i.fset != nil { - return fmt.Sprintf("%s: %s", i.fset.Position(i.spec.Path.Pos()), i) - } - return fmt.Sprintf("(go-marshal import): %s", i) -} - -func (i *importStmt) markUsed() { - i.used = true -} - -func (i *importStmt) equivalent(other *importStmt) bool { - return i.name == other.name && i.path == other.path && i.aliased == other.aliased -} - -// importTable represents a collection of importStmts. -// -// An importTable may contain multiple import statements referencing the same -// local name. All import statements aliasing to the same local name are -// technically ambiguous, as if such an import name is used in the generated -// code, it's not clear which import statement it refers to. We ignore any -// potential collisions until actually writing the import table to the generated -// source file. See importTable.write. -// -// Given the following import statements across all the files comprising a -// package marshalled: -// -// "sync" -// "pkg/sync" -// "pkg/sentry/kernel" -// ktime "pkg/sentry/kernel/time" -// -// An importTable representing them would look like this: -// -// importTable { -// is: map[string][]*importStmt { -// "sync": []*importStmt{ -// importStmt{name:"sync", path:"sync", aliased:false} -// importStmt{name:"sync", path:"pkg/sync", aliased:false} -// }, -// "kernel": []*importStmt{importStmt{ -// name: "kernel", -// path: "pkg/sentry/kernel", -// aliased: false -// }}, -// "ktime": []*importStmt{importStmt{ -// name: "ktime", -// path: "pkg/sentry/kernel/time", -// aliased: true, -// }}, -// } -// } -// -// Note that the local name "sync" is assigned to two different import -// statements. This is possible if the import statements are from different -// source files in the same package. -// -// Since go-marshal generates a single output file per package regardless of the -// number of input files, if "sync" is referenced by any generated code, it's -// unclear which import statement "sync" refers to. While it's theoretically -// possible to resolve this by assigning a unique local alias to each instance -// of the sync package, go-marshal currently aborts when it encounters such an -// ambiguity. -// -// TODO(b/151478251): importTable considers the final component of an import -// path to be the package name, but this is only a convention. The actual -// package name is determined by the package statement in the source files for -// the package. -type importTable struct { - // Map of imports and whether they should be copied to the output. - is map[string][]*importStmt -} - -func newImportTable() *importTable { - return &importTable{ - is: make(map[string][]*importStmt), - } -} - -// Merges import statements from other into i. -func (i *importTable) merge(other *importTable) { - for name, ims := range other.is { - i.is[name] = append(i.is[name], ims...) - } -} - -func (i *importTable) addStmt(s *importStmt) *importStmt { - i.is[s.name] = append(i.is[s.name], s) - return s -} - -func (i *importTable) add(s string) *importStmt { - n := newImport(s) - return i.addStmt(n) -} - -func (i *importTable) addFromSpec(spec *ast.ImportSpec, f *token.FileSet) *importStmt { - return i.addStmt(newImportFromSpec(spec, f)) -} - -// Marks the import named n as used. If no such import is in the table, returns -// false. -func (i *importTable) markUsed(n string) bool { - if ns, ok := i.is[n]; ok { - for _, n := range ns { - n.markUsed() - } - return true - } - return false -} - -func (i *importTable) clear() { - for _, is := range i.is { - for _, i := range is { - i.used = false - } - } -} - -func (i *importTable) write(out io.Writer) error { - if len(i.is) == 0 { - // Nothing to import, we're done. - return nil - } - - imports := make([]string, 0, len(i.is)) - for name, is := range i.is { - var lastUsed *importStmt - var ambiguous bool - - for _, i := range is { - if i.used { - if lastUsed != nil { - if !i.equivalent(lastUsed) { - ambiguous = true - } - } - lastUsed = i - } - } - - if ambiguous { - // We have two or more import statements across the different source - // files that share a local name, and at least one of these imports - // are used by the generated code. This ambiguity can't be resolved - // by go-marshal and requires the user intervention. Dump a list of - // the colliding import statements and let the user modify the input - // files as appropriate. - var b strings.Builder - fmt.Fprintf(&b, "The imported name %q is used by one of the types marked for marshalling, and which import statement the code refers to is ambiguous. Perhaps give the imports unique local names?\n\n", name) - fmt.Fprintf(&b, "The following %d import statements are ambiguous for the local name %q:\n", len(is), name) - // Note: len(is) is guaranteed to be 1 or greater or ambiguous can't - // be true. Therefore the slicing below is safe. - for _, i := range is[:len(is)-1] { - fmt.Fprintf(&b, " %v\n", i.debugString()) - } - fmt.Fprintf(&b, " %v", is[len(is)-1].debugString()) - panic(b.String()) - } - - if lastUsed != nil { - imports = append(imports, lastUsed.String()) - } - } - sort.Strings(imports) - - var b sourceBuffer - b.emit("import (\n") - b.incIndent() - for _, i := range imports { - b.emit("%s\n", i) - } - b.decIndent() - b.emit(")\n\n") - - return b.write(out) -} diff --git a/tools/go_marshal/main.go b/tools/go_marshal/main.go deleted file mode 100644 index 6e4a3e8c4..000000000 --- a/tools/go_marshal/main.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// go_marshal is a code generation utility for automatically generating code to -// marshal go data structures to memory. -// -// This binary is typically run as part of the build process, and is invoked by -// the go_marshal bazel rule defined in defs.bzl. -// -// See README.md. -package main - -import ( - "flag" - "fmt" - "os" - "strings" - - "gvisor.dev/gvisor/tools/go_marshal/gomarshal" -) - -var ( - pkg = flag.String("pkg", "", "output package") - output = flag.String("output", "", "output file") - outputTest = flag.String("output_test", "", "output file for tests") - outputTestUnconditional = flag.String("output_test_unconditional", "", "output file for unconditional tests") - imports = flag.String("imports", "", "comma-separated list of extra packages to import in generated code") -) - -func main() { - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage: %s <input go src files>\n", os.Args[0]) - flag.PrintDefaults() - } - flag.Parse() - if len(flag.Args()) == 0 { - flag.Usage() - os.Exit(1) - } - - if *pkg == "" { - flag.Usage() - fmt.Fprint(os.Stderr, "Flag -pkg must be provided.\n") - os.Exit(1) - } - - var extraImports []string - if len(*imports) > 0 { - // Note: strings.Split(s, sep) returns s if sep doesn't exist in s. Thus - // we check for an empty imports list to avoid emitting an empty string - // as an import. - extraImports = strings.Split(*imports, ",") - } - g, err := gomarshal.NewGenerator(flag.Args(), *output, *outputTest, *outputTestUnconditional, *pkg, extraImports) - if err != nil { - panic(err) - } - - if err := g.Run(); err != nil { - panic(err) - } -} diff --git a/tools/go_marshal/test/BUILD b/tools/go_marshal/test/BUILD deleted file mode 100644 index e872560a9..000000000 --- a/tools/go_marshal/test/BUILD +++ /dev/null @@ -1,52 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -licenses(["notice"]) - -package_group( - name = "gomarshal_test", - packages = [ - "//tools/go_marshal/test/...", - ], -) - -go_test( - name = "benchmark_test", - srcs = ["benchmark_test.go"], - deps = [ - ":test", - "//pkg/binary", - "//pkg/hostarch", - "//tools/go_marshal/analysis", - ], -) - -go_library( - name = "test", - testonly = 1, - srcs = [ - "dynamic.go", - "test.go", - ], - marshal = True, - visibility = ["//tools/go_marshal/test:__subpackages__"], - deps = [ - "//pkg/marshal/primitive", - "//tools/go_marshal/test/external", - ], -) - -go_test( - name = "marshal_test", - size = "small", - srcs = ["marshal_test.go"], - deps = [ - ":test", - "//pkg/hostarch", - "//pkg/marshal", - "//pkg/marshal/primitive", - "//pkg/syserror", - "//pkg/usermem", - "//tools/go_marshal/analysis", - "@com_github_google_go_cmp//cmp:go_default_library", - ], -) diff --git a/tools/go_marshal/test/benchmark_test.go b/tools/go_marshal/test/benchmark_test.go deleted file mode 100644 index 16f478ff7..000000000 --- a/tools/go_marshal/test/benchmark_test.go +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package benchmark_test - -import ( - "bytes" - encbin "encoding/binary" - "fmt" - "reflect" - "testing" - - "gvisor.dev/gvisor/pkg/binary" - "gvisor.dev/gvisor/pkg/hostarch" - "gvisor.dev/gvisor/tools/go_marshal/analysis" - "gvisor.dev/gvisor/tools/go_marshal/test" -) - -// Marshalling using the standard encoding/binary package. -func BenchmarkEncodingBinary(b *testing.B) { - var s1, s2 test.Stat - analysis.RandomizeValue(&s1) - - size := encbin.Size(&s1) - - b.ResetTimer() - - for n := 0; n < b.N; n++ { - buf := bytes.NewBuffer(make([]byte, size)) - buf.Reset() - if err := encbin.Write(buf, hostarch.ByteOrder, &s1); err != nil { - b.Error("Write:", err) - } - if err := encbin.Read(buf, hostarch.ByteOrder, &s2); err != nil { - b.Error("Read:", err) - } - } - - b.StopTimer() - - // Sanity check, make sure the values were preserved. - if !reflect.DeepEqual(s1, s2) { - panic(fmt.Sprintf("Data corruption across marshal/unmarshal cycle:\nBefore: %+v\nAfter: %+v\n", s1, s2)) - } -} - -// Marshalling using the sentry's binary.Marshal. -func BenchmarkBinary(b *testing.B) { - var s1, s2 test.Stat - analysis.RandomizeValue(&s1) - - size := binary.Size(s1) - - b.ResetTimer() - - for n := 0; n < b.N; n++ { - buf := make([]byte, 0, size) - buf = binary.Marshal(buf, hostarch.ByteOrder, &s1) - binary.Unmarshal(buf, hostarch.ByteOrder, &s2) - } - - b.StopTimer() - - // Sanity check, make sure the values were preserved. - if !reflect.DeepEqual(s1, s2) { - panic(fmt.Sprintf("Data corruption across marshal/unmarshal cycle:\nBefore: %+v\nAfter: %+v\n", s1, s2)) - } -} - -// Marshalling field-by-field with manually-written code. -func BenchmarkMarshalManual(b *testing.B) { - var s1, s2 test.Stat - analysis.RandomizeValue(&s1) - - b.ResetTimer() - - for n := 0; n < b.N; n++ { - buf := make([]byte, 0, s1.SizeBytes()) - - // Marshal - buf = binary.AppendUint64(buf, hostarch.ByteOrder, s1.Dev) - buf = binary.AppendUint64(buf, hostarch.ByteOrder, s1.Ino) - buf = binary.AppendUint64(buf, hostarch.ByteOrder, s1.Nlink) - buf = binary.AppendUint32(buf, hostarch.ByteOrder, s1.Mode) - buf = binary.AppendUint32(buf, hostarch.ByteOrder, s1.UID) - buf = binary.AppendUint32(buf, hostarch.ByteOrder, s1.GID) - buf = binary.AppendUint32(buf, hostarch.ByteOrder, 0) - buf = binary.AppendUint64(buf, hostarch.ByteOrder, s1.Rdev) - buf = binary.AppendUint64(buf, hostarch.ByteOrder, uint64(s1.Size)) - buf = binary.AppendUint64(buf, hostarch.ByteOrder, uint64(s1.Blksize)) - buf = binary.AppendUint64(buf, hostarch.ByteOrder, uint64(s1.Blocks)) - buf = binary.AppendUint64(buf, hostarch.ByteOrder, uint64(s1.ATime.Sec)) - buf = binary.AppendUint64(buf, hostarch.ByteOrder, uint64(s1.ATime.Nsec)) - buf = binary.AppendUint64(buf, hostarch.ByteOrder, uint64(s1.MTime.Sec)) - buf = binary.AppendUint64(buf, hostarch.ByteOrder, uint64(s1.MTime.Nsec)) - buf = binary.AppendUint64(buf, hostarch.ByteOrder, uint64(s1.CTime.Sec)) - buf = binary.AppendUint64(buf, hostarch.ByteOrder, uint64(s1.CTime.Nsec)) - - // Unmarshal - s2.Dev = hostarch.ByteOrder.Uint64(buf[0:8]) - s2.Ino = hostarch.ByteOrder.Uint64(buf[8:16]) - s2.Nlink = hostarch.ByteOrder.Uint64(buf[16:24]) - s2.Mode = hostarch.ByteOrder.Uint32(buf[24:28]) - s2.UID = hostarch.ByteOrder.Uint32(buf[28:32]) - s2.GID = hostarch.ByteOrder.Uint32(buf[32:36]) - // Padding: buf[36:40] - s2.Rdev = hostarch.ByteOrder.Uint64(buf[40:48]) - s2.Size = int64(hostarch.ByteOrder.Uint64(buf[48:56])) - s2.Blksize = int64(hostarch.ByteOrder.Uint64(buf[56:64])) - s2.Blocks = int64(hostarch.ByteOrder.Uint64(buf[64:72])) - s2.ATime.Sec = int64(hostarch.ByteOrder.Uint64(buf[72:80])) - s2.ATime.Nsec = int64(hostarch.ByteOrder.Uint64(buf[80:88])) - s2.MTime.Sec = int64(hostarch.ByteOrder.Uint64(buf[88:96])) - s2.MTime.Nsec = int64(hostarch.ByteOrder.Uint64(buf[96:104])) - s2.CTime.Sec = int64(hostarch.ByteOrder.Uint64(buf[104:112])) - s2.CTime.Nsec = int64(hostarch.ByteOrder.Uint64(buf[112:120])) - } - - b.StopTimer() - - // Sanity check, make sure the values were preserved. - if !reflect.DeepEqual(s1, s2) { - panic(fmt.Sprintf("Data corruption across marshal/unmarshal cycle:\nBefore: %+v\nAfter: %+v\n", s1, s2)) - } -} - -// Marshalling with the go_marshal safe API. -func BenchmarkGoMarshalSafe(b *testing.B) { - var s1, s2 test.Stat - analysis.RandomizeValue(&s1) - - b.ResetTimer() - - for n := 0; n < b.N; n++ { - buf := make([]byte, s1.SizeBytes()) - s1.MarshalBytes(buf) - s2.UnmarshalBytes(buf) - } - - b.StopTimer() - - // Sanity check, make sure the values were preserved. - if !reflect.DeepEqual(s1, s2) { - panic(fmt.Sprintf("Data corruption across marshal/unmarshal cycle:\nBefore: %+v\nAfter: %+v\n", s1, s2)) - } -} - -// Marshalling with the go_marshal unsafe API. -func BenchmarkGoMarshalUnsafe(b *testing.B) { - var s1, s2 test.Stat - analysis.RandomizeValue(&s1) - - b.ResetTimer() - - for n := 0; n < b.N; n++ { - buf := make([]byte, s1.SizeBytes()) - s1.MarshalUnsafe(buf) - s2.UnmarshalUnsafe(buf) - } - - b.StopTimer() - - // Sanity check, make sure the values were preserved. - if !reflect.DeepEqual(s1, s2) { - panic(fmt.Sprintf("Data corruption across marshal/unmarshal cycle:\nBefore: %+v\nAfter: %+v\n", s1, s2)) - } -} - -func BenchmarkBinarySlice(b *testing.B) { - var s1, s2 [64]test.Stat - analysis.RandomizeValue(&s1) - - size := binary.Size(s1) - - b.ResetTimer() - - for n := 0; n < b.N; n++ { - buf := make([]byte, 0, size) - buf = binary.Marshal(buf, hostarch.ByteOrder, &s1) - binary.Unmarshal(buf, hostarch.ByteOrder, &s2) - } - - b.StopTimer() - - // Sanity check, make sure the values were preserved. - if !reflect.DeepEqual(s1, s2) { - panic(fmt.Sprintf("Data corruption across marshal/unmarshal cycle:\nBefore: %+v\nAfter: %+v\n", s1, s2)) - } -} - -func BenchmarkGoMarshalUnsafeSlice(b *testing.B) { - var s1, s2 [64]test.Stat - analysis.RandomizeValue(&s1) - - b.ResetTimer() - - for n := 0; n < b.N; n++ { - buf := make([]byte, (*test.Stat)(nil).SizeBytes()*len(s1)) - test.MarshalUnsafeStatSlice(s1[:], buf) - test.UnmarshalUnsafeStatSlice(s2[:], buf) - } - - b.StopTimer() - - // Sanity check, make sure the values were preserved. - if !reflect.DeepEqual(s1, s2) { - panic(fmt.Sprintf("Data corruption across marshal/unmarshal cycle:\nBefore: %+v\nAfter: %+v\n", s1, s2)) - } -} diff --git a/tools/go_marshal/test/dynamic.go b/tools/go_marshal/test/dynamic.go deleted file mode 100644 index 9a812efe9..000000000 --- a/tools/go_marshal/test/dynamic.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2021 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import "gvisor.dev/gvisor/pkg/marshal/primitive" - -// Type12Dynamic is a dynamically sized struct which depends on the -// autogenerator to generate some Marshallable methods for it. -// -// +marshal dynamic -type Type12Dynamic struct { - X primitive.Int64 - Y []primitive.Int64 -} - -// SizeBytes implements marshal.Marshallable.SizeBytes. -func (t *Type12Dynamic) SizeBytes() int { - return (len(t.Y) * 8) + t.X.SizeBytes() -} - -// MarshalBytes implements marshal.Marshallable.MarshalBytes. -func (t *Type12Dynamic) MarshalBytes(dst []byte) { - t.X.MarshalBytes(dst) - dst = dst[t.X.SizeBytes():] - for i, x := range t.Y { - x.MarshalBytes(dst[i*8 : (i+1)*8]) - } -} - -// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. -func (t *Type12Dynamic) UnmarshalBytes(src []byte) { - t.X.UnmarshalBytes(src) - if t.Y != nil { - t.Y = t.Y[:0] - } - for i := t.X.SizeBytes(); i < len(src); i += 8 { - var x primitive.Int64 - x.UnmarshalBytes(src[i:]) - t.Y = append(t.Y, x) - } -} - -// Type13Dynamic is a dynamically sized struct which depends on the -// autogenerator to generate some Marshallable methods for it. -// -// It represents a string in memory which is preceded by a uint32 indicating -// the string size. -// -// +marshal dynamic -type Type13Dynamic string - -// SizeBytes implements marshal.Marshallable.SizeBytes. -func (t *Type13Dynamic) SizeBytes() int { - return (*primitive.Uint32)(nil).SizeBytes() + len(*t) -} - -// MarshalBytes implements marshal.Marshallable.MarshalBytes. -func (t *Type13Dynamic) MarshalBytes(dst []byte) { - strLen := primitive.Uint32(len(*t)) - strLen.MarshalBytes(dst) - dst = dst[strLen.SizeBytes():] - copy(dst[:strLen], *t) -} - -// UnmarshalBytes implements marshal.Marshallable.UnmarshalBytes. -func (t *Type13Dynamic) UnmarshalBytes(src []byte) { - var strLen primitive.Uint32 - strLen.UnmarshalBytes(src) - src = src[strLen.SizeBytes():] - *t = Type13Dynamic(src[:strLen]) -} diff --git a/tools/go_marshal/test/escape/BUILD b/tools/go_marshal/test/escape/BUILD deleted file mode 100644 index 62e0b4665..000000000 --- a/tools/go_marshal/test/escape/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -licenses(["notice"]) - -go_library( - name = "escape", - testonly = 1, - srcs = ["escape.go"], - deps = [ - "//pkg/hostarch", - "//pkg/marshal", - "//tools/go_marshal/test", - ], -) diff --git a/tools/go_marshal/test/escape/escape.go b/tools/go_marshal/test/escape/escape.go deleted file mode 100644 index 1ac606862..000000000 --- a/tools/go_marshal/test/escape/escape.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package escape contains test cases for escape analysis. -package escape - -import ( - "gvisor.dev/gvisor/pkg/hostarch" - "gvisor.dev/gvisor/pkg/marshal" - "gvisor.dev/gvisor/tools/go_marshal/test" -) - -// dummyCopyContext implements marshal.CopyContext. -type dummyCopyContext struct { -} - -func (*dummyCopyContext) CopyScratchBuffer(size int) []byte { - return make([]byte, size) -} - -func (*dummyCopyContext) CopyOutBytes(addr hostarch.Addr, b []byte) (int, error) { - return len(b), nil -} - -func (*dummyCopyContext) CopyInBytes(addr hostarch.Addr, b []byte) (int, error) { - return len(b), nil -} - -func (t *dummyCopyContext) MarshalBytes(addr hostarch.Addr, marshallable marshal.Marshallable) { - buf := t.CopyScratchBuffer(marshallable.SizeBytes()) - marshallable.MarshalBytes(buf) - t.CopyOutBytes(addr, buf) -} - -func (t *dummyCopyContext) MarshalUnsafe(addr hostarch.Addr, marshallable marshal.Marshallable) { - buf := t.CopyScratchBuffer(marshallable.SizeBytes()) - marshallable.MarshalUnsafe(buf) - t.CopyOutBytes(addr, buf) -} - -// +checkescape:all -//go:nosplit -func doCopyIn(t *dummyCopyContext) { - var stat test.Stat - stat.CopyIn(t, hostarch.Addr(0xf000ba12)) -} - -// +checkescape:all -//go:nosplit -func doCopyOut(t *dummyCopyContext) { - var stat test.Stat - stat.CopyOut(t, hostarch.Addr(0xf000ba12)) -} - -// +mustescape:builtin -// +mustescape:stack -//go:nosplit -func doMarshalBytesDirect(t *dummyCopyContext) { - var stat test.Stat - buf := t.CopyScratchBuffer(stat.SizeBytes()) - stat.MarshalBytes(buf) - t.CopyOutBytes(hostarch.Addr(0xf000ba12), buf) -} - -// +mustescape:builtin -// +mustescape:stack -//go:nosplit -func doMarshalUnsafeDirect(t *dummyCopyContext) { - var stat test.Stat - buf := t.CopyScratchBuffer(stat.SizeBytes()) - stat.MarshalUnsafe(buf) - t.CopyOutBytes(hostarch.Addr(0xf000ba12), buf) -} - -// +mustescape:local,heap -// +mustescape:stack -//go:nosplit -func doMarshalBytesViaMarshallable(t *dummyCopyContext) { - var stat test.Stat - t.MarshalBytes(hostarch.Addr(0xf000ba12), &stat) -} - -// +mustescape:local,heap -// +mustescape:stack -//go:nosplit -func doMarshalUnsafeViaMarshallable(t *dummyCopyContext) { - var stat test.Stat - t.MarshalUnsafe(hostarch.Addr(0xf000ba12), &stat) -} diff --git a/tools/go_marshal/test/external/BUILD b/tools/go_marshal/test/external/BUILD deleted file mode 100644 index 0cf6da603..000000000 --- a/tools/go_marshal/test/external/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -licenses(["notice"]) - -go_library( - name = "external", - testonly = 1, - srcs = ["external.go"], - marshal = True, - visibility = ["//tools/go_marshal/test:gomarshal_test"], -) diff --git a/tools/go_marshal/test/external/external.go b/tools/go_marshal/test/external/external.go deleted file mode 100644 index 26fe8e0c8..000000000 --- a/tools/go_marshal/test/external/external.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package external defines types we can import for testing. -package external - -// External is a public Marshallable type for use in testing. -// -// +marshal -type External struct { - j int64 -} - -// NotPacked is an unaligned Marshallable type for use in testing. -// -// +marshal -type NotPacked struct { - a int32 - b byte `marshal:"unaligned"` -} diff --git a/tools/go_marshal/test/marshal_test.go b/tools/go_marshal/test/marshal_test.go deleted file mode 100644 index 43bafbf96..000000000 --- a/tools/go_marshal/test/marshal_test.go +++ /dev/null @@ -1,554 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package marshal_test contains manual tests for the marshal interface. These -// are intended to test behaviour not covered by the automatically generated -// tests. -package marshal_test - -import ( - "bytes" - "encoding/binary" - "fmt" - "reflect" - "runtime" - "testing" - "unsafe" - - "github.com/google/go-cmp/cmp" - "gvisor.dev/gvisor/pkg/hostarch" - "gvisor.dev/gvisor/pkg/marshal" - "gvisor.dev/gvisor/pkg/marshal/primitive" - "gvisor.dev/gvisor/pkg/syserror" - "gvisor.dev/gvisor/pkg/usermem" - "gvisor.dev/gvisor/tools/go_marshal/analysis" - "gvisor.dev/gvisor/tools/go_marshal/test" -) - -var simulatedErr error = syserror.EFAULT - -// mockCopyContext implements marshal.CopyContext. -type mockCopyContext struct { - taskMem usermem.BytesIO -} - -// populate fills the task memory with the contents of val. -func (t *mockCopyContext) populate(val interface{}) { - var buf bytes.Buffer - // Use binary.Write so we aren't testing go-marshal against its own - // potentially buggy implementation. - if err := binary.Write(&buf, hostarch.ByteOrder, val); err != nil { - panic(err) - } - t.taskMem.Bytes = buf.Bytes() -} - -func (t *mockCopyContext) setLimit(n int) { - if len(t.taskMem.Bytes) < n { - grown := make([]byte, n) - copy(grown, t.taskMem.Bytes) - t.taskMem.Bytes = grown - return - } - t.taskMem.Bytes = t.taskMem.Bytes[:n] -} - -// CopyScratchBuffer implements marshal.CopyContext.CopyScratchBuffer. -func (t *mockCopyContext) CopyScratchBuffer(size int) []byte { - return make([]byte, size) -} - -// CopyOutBytes implements marshal.CopyContext.CopyOutBytes. The implementation -// completely ignores the target address and stores a copy of b in its -// internally buffer, overriding any previous contents. -func (t *mockCopyContext) CopyOutBytes(_ hostarch.Addr, b []byte) (int, error) { - return t.taskMem.CopyOut(nil, 0, b, usermem.IOOpts{}) -} - -// CopyInBytes implements marshal.CopyContext.CopyInBytes. The implementation -// completely ignores the source address and always fills b from the begining of -// its internal buffer. -func (t *mockCopyContext) CopyInBytes(_ hostarch.Addr, b []byte) (int, error) { - return t.taskMem.CopyIn(nil, 0, b, usermem.IOOpts{}) -} - -// unsafeMemory returns the underlying memory for m. The returned slice is only -// valid for the lifetime for m. The garbage collector isn't aware that the -// returned slice is related to m, the caller must ensure m lives long enough. -func unsafeMemory(m marshal.Marshallable) []byte { - if !m.Packed() { - // We can't return a slice pointing to the underlying memory - // since the layout isn't packed. Allocate a temporary buffer - // and marshal instead. - var buf bytes.Buffer - if err := binary.Write(&buf, hostarch.ByteOrder, m); err != nil { - panic(err) - } - return buf.Bytes() - } - - // reflect.ValueOf(m) - // .Elem() // Unwrap interface to inner concrete object - // .Addr() // Pointer value to object - // .Pointer() // Actual address from the pointer value - ptr := reflect.ValueOf(m).Elem().Addr().Pointer() - - size := m.SizeBytes() - - var mem []byte - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&mem)) - hdr.Data = ptr - hdr.Len = size - hdr.Cap = size - - return mem -} - -// unsafeMemorySlice returns the underlying memory for m. The returned slice is -// only valid for the lifetime for m. The garbage collector isn't aware that the -// returned slice is related to m, the caller must ensure m lives long enough. -// -// Precondition: m must be a slice. -func unsafeMemorySlice(m interface{}, elt marshal.Marshallable) []byte { - kind := reflect.TypeOf(m).Kind() - if kind != reflect.Slice { - panic("unsafeMemorySlice called on non-slice") - } - - if !elt.Packed() { - // We can't return a slice pointing to the underlying memory - // since the layout isn't packed. Allocate a temporary buffer - // and marshal instead. - var buf bytes.Buffer - if err := binary.Write(&buf, hostarch.ByteOrder, m); err != nil { - panic(err) - } - return buf.Bytes() - } - - v := reflect.ValueOf(m) - length := v.Len() * elt.SizeBytes() - - var mem []byte - hdr := (*reflect.SliceHeader)(unsafe.Pointer(&mem)) - hdr.Data = v.Pointer() // This is a pointer to the first elem for slices. - hdr.Len = length - hdr.Cap = length - - return mem -} - -func isZeroes(buf []byte) bool { - for _, b := range buf { - if b != 0 { - return false - } - } - return true -} - -// compareMemory compares the first n bytes of two chuncks of memory represented -// by expected and actual. -func compareMemory(t *testing.T, expected, actual []byte, n int) { - t.Logf("Expected (%d): %v (%d) + (%d) %v\n", len(expected), expected[:n], n, len(expected)-n, expected[n:]) - t.Logf("Actual (%d): %v (%d) + (%d) %v\n", len(actual), actual[:n], n, len(actual)-n, actual[n:]) - - if diff := cmp.Diff(expected[:n], actual[:n]); diff != "" { - t.Errorf("Memory buffers don't match:\n--- expected only\n+++ actual only\n%v", diff) - } -} - -// limitedCopyIn populates task memory with src, then unmarshals task memory to -// dst. The task signals an error at limit bytes during copy-in, which should -// result in a truncated unmarshalling. -func limitedCopyIn(t *testing.T, src, dst marshal.Marshallable, limit int) { - var cc mockCopyContext - cc.populate(src) - cc.setLimit(limit) - - n, err := dst.CopyIn(&cc, hostarch.Addr(0)) - if n != limit { - t.Errorf("CopyIn copied unexpected number of bytes, expected %d, got %d", limit, n) - } - if err != simulatedErr { - t.Errorf("CopyIn returned unexpected error, expected %v, got %v", simulatedErr, err) - } - - expectedMem := unsafeMemory(src) - defer runtime.KeepAlive(src) - actualMem := unsafeMemory(dst) - defer runtime.KeepAlive(dst) - - compareMemory(t, expectedMem, actualMem, n) - - // The last n bytes should be zero for actual, since actual was - // zero-initialized, and CopyIn shouldn't have touched those bytes. However - // we can only guarantee we didn't touch anything in the last n bytes if the - // layout is packed. - if dst.Packed() && !isZeroes(actualMem[n:]) { - t.Errorf("Expected the last %d bytes of copied in object to be zeroes, got %v\n", dst.SizeBytes()-n, actualMem) - } -} - -// limitedCopyOut marshals src to task memory. The task signals an error at -// limit bytes during copy-out, which should result in a truncated marshalling. -func limitedCopyOut(t *testing.T, src marshal.Marshallable, limit int) { - var cc mockCopyContext - cc.setLimit(limit) - - n, err := src.CopyOut(&cc, hostarch.Addr(0)) - if n != limit { - t.Errorf("CopyOut copied unexpected number of bytes, expected %d, got %d", limit, n) - } - if err != simulatedErr { - t.Errorf("CopyOut returned unexpected error, expected %v, got %v", simulatedErr, err) - } - - expectedMem := unsafeMemory(src) - defer runtime.KeepAlive(src) - actualMem := cc.taskMem.Bytes - - compareMemory(t, expectedMem, actualMem, n) -} - -// copyOutN marshals src to task memory, requesting the marshalling to be -// limited to limit bytes. -func copyOutN(t *testing.T, src marshal.Marshallable, limit int) { - var cc mockCopyContext - cc.setLimit(limit) - - n, err := src.CopyOutN(&cc, hostarch.Addr(0), limit) - if err != nil { - t.Errorf("CopyOut returned unexpected error: %v", err) - } - if n != limit { - t.Errorf("CopyOut copied unexpected number of bytes, expected %d, got %d", limit, n) - } - - expectedMem := unsafeMemory(src) - defer runtime.KeepAlive(src) - actualMem := cc.taskMem.Bytes - - t.Logf("Expected: %v + %v\n", expectedMem[:n], expectedMem[n:]) - t.Logf("Actual : %v + %v\n", actualMem[:n], actualMem[n:]) - - compareMemory(t, expectedMem, actualMem, n) -} - -// TestLimitedMarshalling verifies marshalling/unmarshalling succeeds when the -// underyling copy in/out operations partially succeed. -func TestLimitedMarshalling(t *testing.T) { - types := []reflect.Type{ - // Packed types. - reflect.TypeOf((*test.Type2)(nil)), - reflect.TypeOf((*test.Type3)(nil)), - reflect.TypeOf((*test.Timespec)(nil)), - reflect.TypeOf((*test.Stat)(nil)), - reflect.TypeOf((*test.InetAddr)(nil)), - reflect.TypeOf((*test.SignalSet)(nil)), - reflect.TypeOf((*test.SignalSetAlias)(nil)), - // Non-packed types. - reflect.TypeOf((*test.Type1)(nil)), - reflect.TypeOf((*test.Type4)(nil)), - reflect.TypeOf((*test.Type5)(nil)), - reflect.TypeOf((*test.Type6)(nil)), - reflect.TypeOf((*test.Type7)(nil)), - reflect.TypeOf((*test.Type8)(nil)), - } - - for _, tyPtr := range types { - // Remove one level of pointer-indirection from the type. We get this - // back when we pass the type to reflect.New. - ty := tyPtr.Elem() - - // Partial copy-in. - t.Run(fmt.Sprintf("PartialCopyIn_%v", ty), func(t *testing.T) { - expected := reflect.New(ty).Interface().(marshal.Marshallable) - actual := reflect.New(ty).Interface().(marshal.Marshallable) - analysis.RandomizeValue(expected) - - limitedCopyIn(t, expected, actual, expected.SizeBytes()/2) - }) - - // Partial copy-out. - t.Run(fmt.Sprintf("PartialCopyOut_%v", ty), func(t *testing.T) { - expected := reflect.New(ty).Interface().(marshal.Marshallable) - analysis.RandomizeValue(expected) - - limitedCopyOut(t, expected, expected.SizeBytes()/2) - }) - - // Explicitly request partial copy-out. - t.Run(fmt.Sprintf("PartialCopyOutN_%v", ty), func(t *testing.T) { - expected := reflect.New(ty).Interface().(marshal.Marshallable) - analysis.RandomizeValue(expected) - - copyOutN(t, expected, expected.SizeBytes()/2) - }) - } -} - -// TestLimitedMarshalling verifies marshalling/unmarshalling of slices of -// marshallable types succeed when the underyling copy in/out operations -// partially succeed. -func TestLimitedSliceMarshalling(t *testing.T) { - types := []struct { - arrayPtrType reflect.Type - copySliceIn func(cc marshal.CopyContext, addr hostarch.Addr, dstSlice interface{}) (int, error) - copySliceOut func(cc marshal.CopyContext, addr hostarch.Addr, srcSlice interface{}) (int, error) - unsafeMemory func(arrPtr interface{}) []byte - }{ - // Packed types. - { - reflect.TypeOf((*[20]test.Stat)(nil)), - func(cc marshal.CopyContext, addr hostarch.Addr, dst interface{}) (int, error) { - slice := dst.(*[20]test.Stat)[:] - return test.CopyStatSliceIn(cc, addr, slice) - }, - func(cc marshal.CopyContext, addr hostarch.Addr, src interface{}) (int, error) { - slice := src.(*[20]test.Stat)[:] - return test.CopyStatSliceOut(cc, addr, slice) - }, - func(a interface{}) []byte { - slice := a.(*[20]test.Stat)[:] - return unsafeMemorySlice(slice, &slice[0]) - }, - }, - { - reflect.TypeOf((*[1]test.Stat)(nil)), - func(cc marshal.CopyContext, addr hostarch.Addr, dst interface{}) (int, error) { - slice := dst.(*[1]test.Stat)[:] - return test.CopyStatSliceIn(cc, addr, slice) - }, - func(cc marshal.CopyContext, addr hostarch.Addr, src interface{}) (int, error) { - slice := src.(*[1]test.Stat)[:] - return test.CopyStatSliceOut(cc, addr, slice) - }, - func(a interface{}) []byte { - slice := a.(*[1]test.Stat)[:] - return unsafeMemorySlice(slice, &slice[0]) - }, - }, - { - reflect.TypeOf((*[5]test.SignalSetAlias)(nil)), - func(cc marshal.CopyContext, addr hostarch.Addr, dst interface{}) (int, error) { - slice := dst.(*[5]test.SignalSetAlias)[:] - return test.CopySignalSetAliasSliceIn(cc, addr, slice) - }, - func(cc marshal.CopyContext, addr hostarch.Addr, src interface{}) (int, error) { - slice := src.(*[5]test.SignalSetAlias)[:] - return test.CopySignalSetAliasSliceOut(cc, addr, slice) - }, - func(a interface{}) []byte { - slice := a.(*[5]test.SignalSetAlias)[:] - return unsafeMemorySlice(slice, &slice[0]) - }, - }, - // Non-packed types. - { - reflect.TypeOf((*[20]test.Type1)(nil)), - func(cc marshal.CopyContext, addr hostarch.Addr, dst interface{}) (int, error) { - slice := dst.(*[20]test.Type1)[:] - return test.CopyType1SliceIn(cc, addr, slice) - }, - func(cc marshal.CopyContext, addr hostarch.Addr, src interface{}) (int, error) { - slice := src.(*[20]test.Type1)[:] - return test.CopyType1SliceOut(cc, addr, slice) - }, - func(a interface{}) []byte { - slice := a.(*[20]test.Type1)[:] - return unsafeMemorySlice(slice, &slice[0]) - }, - }, - { - reflect.TypeOf((*[1]test.Type1)(nil)), - func(cc marshal.CopyContext, addr hostarch.Addr, dst interface{}) (int, error) { - slice := dst.(*[1]test.Type1)[:] - return test.CopyType1SliceIn(cc, addr, slice) - }, - func(cc marshal.CopyContext, addr hostarch.Addr, src interface{}) (int, error) { - slice := src.(*[1]test.Type1)[:] - return test.CopyType1SliceOut(cc, addr, slice) - }, - func(a interface{}) []byte { - slice := a.(*[1]test.Type1)[:] - return unsafeMemorySlice(slice, &slice[0]) - }, - }, - { - reflect.TypeOf((*[7]test.Type8)(nil)), - func(cc marshal.CopyContext, addr hostarch.Addr, dst interface{}) (int, error) { - slice := dst.(*[7]test.Type8)[:] - return test.CopyType8SliceIn(cc, addr, slice) - }, - func(cc marshal.CopyContext, addr hostarch.Addr, src interface{}) (int, error) { - slice := src.(*[7]test.Type8)[:] - return test.CopyType8SliceOut(cc, addr, slice) - }, - func(a interface{}) []byte { - slice := a.(*[7]test.Type8)[:] - return unsafeMemorySlice(slice, &slice[0]) - }, - }, - } - - for _, tt := range types { - // The body of this loop is generic over the type tt.arrayPtrType, with - // the help of reflection. To aid in readability, comments below show - // the equivalent go code assuming - // tt.arrayPtrType = typeof(*[20]test.Stat). - - // Equivalent: - // var x *[20]test.Stat - // arrayTy := reflect.TypeOf(*x) - arrayTy := tt.arrayPtrType.Elem() - - // Partial copy-in of slices. - t.Run(fmt.Sprintf("PartialCopySliceIn_%v", arrayTy), func(t *testing.T) { - // Equivalent: - // var x [20]test.Stat - // length := len(x) - length := arrayTy.Len() - if length < 1 { - panic("Test type can't be zero-length array") - } - // Equivalent: - // elem := new(test.Stat).(marshal.Marshallable) - elem := reflect.New(arrayTy.Elem()).Interface().(marshal.Marshallable) - - // Equivalent: - // var expected, actual interface{} - // expected = new([20]test.Stat) - // actual = new([20]test.Stat) - expected := reflect.New(arrayTy).Interface() - actual := reflect.New(arrayTy).Interface() - - analysis.RandomizeValue(expected) - - limit := (length * elem.SizeBytes()) / 2 - // Also make sure the limit is partially inside one of the elements. - limit += elem.SizeBytes() / 2 - analysis.RandomizeValue(expected) - - var cc mockCopyContext - cc.populate(expected) - cc.setLimit(limit) - - n, err := tt.copySliceIn(&cc, hostarch.Addr(0), actual) - if n != limit { - t.Errorf("CopyIn copied unexpected number of bytes, expected %d, got %d", limit, n) - } - if n < length*elem.SizeBytes() && err != simulatedErr { - t.Errorf("CopyIn returned unexpected error, expected %v, got %v", simulatedErr, err) - } - - expectedMem := tt.unsafeMemory(expected) - defer runtime.KeepAlive(expected) - actualMem := tt.unsafeMemory(actual) - defer runtime.KeepAlive(actual) - - compareMemory(t, expectedMem, actualMem, n) - - // The last n bytes should be zero for actual, since actual was - // zero-initialized, and CopyIn shouldn't have touched those bytes. However - // we can only guarantee we didn't touch anything in the last n bytes if the - // layout is packed. - if elem.Packed() && !isZeroes(actualMem[n:]) { - t.Errorf("Expected the last %d bytes of copied in object to be zeroes, got %v\n", (elem.SizeBytes()*length)-n, actualMem) - } - }) - - // Partial copy-out of slices. - t.Run(fmt.Sprintf("PartialCopySliceOut_%v", arrayTy), func(t *testing.T) { - // Equivalent: - // var x [20]test.Stat - // length := len(x) - length := arrayTy.Len() - if length < 1 { - panic("Test type can't be zero-length array") - } - // Equivalent: - // elem := new(test.Stat).(marshal.Marshallable) - elem := reflect.New(arrayTy.Elem()).Interface().(marshal.Marshallable) - - // Equivalent: - // var expected, actual interface{} - // expected = new([20]test.Stat) - // actual = new([20]test.Stat) - expected := reflect.New(arrayTy).Interface() - - analysis.RandomizeValue(expected) - - limit := (length * elem.SizeBytes()) / 2 - // Also make sure the limit is partially inside one of the elements. - limit += elem.SizeBytes() / 2 - analysis.RandomizeValue(expected) - - var cc mockCopyContext - cc.populate(expected) - cc.setLimit(limit) - - n, err := tt.copySliceOut(&cc, hostarch.Addr(0), expected) - if n != limit { - t.Errorf("CopyIn copied unexpected number of bytes, expected %d, got %d", limit, n) - } - if n < length*elem.SizeBytes() && err != simulatedErr { - t.Errorf("CopyIn returned unexpected error, expected %v, got %v", simulatedErr, err) - } - - expectedMem := tt.unsafeMemory(expected) - defer runtime.KeepAlive(expected) - actualMem := cc.taskMem.Bytes - - compareMemory(t, expectedMem, actualMem, n) - }) - } -} - -func TestDynamicTypeStruct(t *testing.T) { - t12 := test.Type12Dynamic{ - X: 32, - Y: []primitive.Int64{5, 6, 7}, - } - var cc mockCopyContext - cc.setLimit(t12.SizeBytes()) - - if _, err := t12.CopyOut(&cc, hostarch.Addr(0)); err != nil { - t.Fatalf("cc.CopyOut faile: %v", err) - } - - res := test.Type12Dynamic{ - Y: make([]primitive.Int64, len(t12.Y)), - } - res.CopyIn(&cc, hostarch.Addr(0)) - if !reflect.DeepEqual(t12, res) { - t.Errorf("dynamic type is not same after marshalling and unmarshalling: before = %+v, after = %+v", t12, res) - } -} - -func TestDynamicTypeIdentifier(t *testing.T) { - s := test.Type13Dynamic("go_marshal") - var cc mockCopyContext - cc.setLimit(s.SizeBytes()) - - if _, err := s.CopyOut(&cc, hostarch.Addr(0)); err != nil { - t.Fatalf("cc.CopyOut faile: %v", err) - } - - res := test.Type13Dynamic(make([]byte, len(s))) - res.CopyIn(&cc, hostarch.Addr(0)) - if res != s { - t.Errorf("dynamic type is not same after marshalling and unmarshalling: before = %s, after = %s", s, res) - } -} diff --git a/tools/go_marshal/test/test.go b/tools/go_marshal/test/test.go deleted file mode 100644 index e7e3ed74a..000000000 --- a/tools/go_marshal/test/test.go +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package test contains data structures for testing the go_marshal tool. -package test - -import ( - // We're intentionally using a package name alias here even though it's not - // necessary to test the code generator's ability to handle package aliases. - ex "gvisor.dev/gvisor/tools/go_marshal/test/external" -) - -// Type1 is a test data type. -// -// +marshal slice:Type1Slice -type Type1 struct { - a Type2 - x, y int64 // Multiple field names. - b byte `marshal:"unaligned"` // Short field. - c uint64 - _ uint32 // Unnamed scalar field. - _ [6]byte // Unnamed vector field, typical padding. - _ [2]byte - xs [8]int32 - as [10]Type2 `marshal:"unaligned"` // Array of Marshallable objects. - ss Type3 -} - -// Type2 is a test data type. -// -// +marshal -type Type2 struct { - n int64 - c byte - _ [7]byte - m int64 - a int64 -} - -// Type3 is a test data type. -// -// +marshal -type Type3 struct { - s int64 - x ex.External // Type defined in another package. -} - -// Type4 is a test data type. -// -// +marshal -type Type4 struct { - c byte - x int64 `marshal:"unaligned"` - d byte - _ [7]byte -} - -// Type5 is a test data type. -// -// +marshal -type Type5 struct { - n int64 - t Type4 - m int64 -} - -// Type6 is a test data type ends mid-word. -// -// +marshal -type Type6 struct { - a int64 - b int64 - // If c isn't marked unaligned, analysis fails (as it should, since - // the unsafe API corrupts Type7). - c byte `marshal:"unaligned"` -} - -// Type7 is a test data type that contains a child struct that ends -// mid-word. -// +marshal -type Type7 struct { - x Type6 - y int64 -} - -// Type8 is a test data type which contains an external non-packed field. -// -// +marshal slice:Type8Slice -type Type8 struct { - a int64 - np ex.NotPacked - b int64 -} - -// Timespec represents struct timespec in <time.h>. -// -// +marshal -type Timespec struct { - Sec int64 - Nsec int64 -} - -// Stat represents struct stat. -// -// +marshal slice:StatSlice -type Stat struct { - Dev uint64 - Ino uint64 - Nlink uint64 - Mode uint32 - UID uint32 - GID uint32 - _ int32 - Rdev uint64 - Size int64 - Blksize int64 - Blocks int64 - ATime Timespec - MTime Timespec - CTime Timespec - _ [3]int64 -} - -// InetAddr is an example marshallable newtype on an array. -// -// +marshal -type InetAddr [4]byte - -// SignalSet is an example marshallable newtype on a primitive. -// -// +marshal slice:SignalSetSlice:inner -type SignalSet uint64 - -// SignalSetAlias is an example newtype on another marshallable type. -// -// +marshal slice:SignalSetAliasSlice -type SignalSetAlias SignalSet - -const sizeA = 64 -const sizeB = 8 - -// TestArray is a test data structure on an array with a constant length. -// -// +marshal -type TestArray [sizeA]int32 - -// TestArray2 is a newtype on an array with a simple arithmetic expression of -// constants for the array length. -// -// +marshal -type TestArray2 [sizeA * sizeB]int32 - -// TestArray3 is a newtype on an array with a simple arithmetic expression of -// mixed constants and literals for the array length. -// -// +marshal -type TestArray3 [sizeA*sizeB + 12]int32 - -// Type9 is a test data type containing an array with a non-literal length. -// -// +marshal -type Type9 struct { - x int64 - y [sizeA]int32 -} - -// Type10Embed is a test data type which is be embedded into another type. -// -// +marshal -type Type10Embed struct { - x int64 -} - -// Type10 is a test data type which contains an embedded struct. -// -// +marshal -type Type10 struct { - Type10Embed - y int64 -} - -// Type11 is a test data type which contains an embedded struct from an external -// package. -// -// +marshal -type Type11 struct { - ex.External - y int64 -} diff --git a/tools/go_stateify/BUILD b/tools/go_stateify/BUILD deleted file mode 100644 index 913558b4e..000000000 --- a/tools/go_stateify/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -load("//tools:defs.bzl", "bzl_library", "go_binary") - -package(licenses = ["notice"]) - -go_binary( - name = "stateify", - srcs = ["main.go"], - visibility = ["//:sandbox"], - deps = ["//tools/tags"], -) - -bzl_library( - name = "defs_bzl", - srcs = ["defs.bzl"], - visibility = ["//visibility:private"], -) diff --git a/tools/go_stateify/defs.bzl b/tools/go_stateify/defs.bzl deleted file mode 100644 index 6a5e666f0..000000000 --- a/tools/go_stateify/defs.bzl +++ /dev/null @@ -1,60 +0,0 @@ -"""Stateify is a tool for generating state wrappers for Go types.""" - -def _go_stateify_impl(ctx): - """Implementation for the stateify tool.""" - output = ctx.outputs.out - - # Run the stateify command. - args = ["-output=%s" % output.path] - args.append("-fullpkg=%s" % ctx.attr.package) - if ctx.attr._statepkg: - args.append("-statepkg=%s" % ctx.attr._statepkg) - if ctx.attr.imports: - args.append("-imports=%s" % ",".join(ctx.attr.imports)) - args.append("--") - for src in ctx.attr.srcs: - args += [f.path for f in src.files.to_list()] - ctx.actions.run( - inputs = ctx.files.srcs, - outputs = [output], - mnemonic = "GoStateify", - progress_message = "Generating state library %s" % ctx.label, - arguments = args, - executable = ctx.executable._tool, - ) - -go_stateify = rule( - implementation = _go_stateify_impl, - doc = "Generates save and restore logic from a set of Go files.", - attrs = { - "srcs": attr.label_list( - doc = """ -The input source files. These files should include all structs in the package -that need to be saved. -""", - mandatory = True, - allow_files = True, - ), - "imports": attr.string_list( - doc = """ -An optional list of extra non-aliased, Go-style absolute import paths required -for statified types. -""", - mandatory = False, - ), - "package": attr.string( - doc = "The fully qualified package name for the input sources.", - mandatory = True, - ), - "out": attr.output( - doc = "Name of the generator output file.", - mandatory = True, - ), - "_tool": attr.label( - executable = True, - cfg = "host", - default = Label("//tools/go_stateify:stateify"), - ), - "_statepkg": attr.string(default = "gvisor.dev/gvisor/pkg/state"), - }, -) diff --git a/tools/go_stateify/main.go b/tools/go_stateify/main.go deleted file mode 100644 index 93022f504..000000000 --- a/tools/go_stateify/main.go +++ /dev/null @@ -1,472 +0,0 @@ -// Copyright 2018 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Stateify provides a simple way to generate Load/Save methods based on -// existing types and struct tags. -package main - -import ( - "flag" - "fmt" - "go/ast" - "go/parser" - "go/token" - "os" - "path/filepath" - "reflect" - "strings" - "sync" - - "gvisor.dev/gvisor/tools/tags" -) - -var ( - fullPkg = flag.String("fullpkg", "", "fully qualified output package") - imports = flag.String("imports", "", "extra imports for the output file") - output = flag.String("output", "", "output file") - statePkg = flag.String("statepkg", "", "state import package; defaults to empty") -) - -// resolveTypeName returns a qualified type name. -func resolveTypeName(typ ast.Expr) (field string, qualified string) { - for done := false; !done; { - // Resolve star expressions. - switch rs := typ.(type) { - case *ast.StarExpr: - qualified += "*" - typ = rs.X - case *ast.ArrayType: - if rs.Len == nil { - // Slice type declaration. - qualified += "[]" - } else { - // Array type declaration. - qualified += "[" + rs.Len.(*ast.BasicLit).Value + "]" - } - typ = rs.Elt - default: - // No more descent. - done = true - } - } - - // Resolve a package selector. - sel, ok := typ.(*ast.SelectorExpr) - if ok { - qualified = qualified + sel.X.(*ast.Ident).Name + "." - typ = sel.Sel - } - - // Figure out actual type name. - field = typ.(*ast.Ident).Name - qualified = qualified + field - return -} - -// extractStateTag pulls the relevant state tag. -func extractStateTag(tag *ast.BasicLit) string { - if tag == nil { - return "" - } - if len(tag.Value) < 2 { - return "" - } - return reflect.StructTag(tag.Value[1 : len(tag.Value)-1]).Get("state") -} - -// scanFunctions is a set of functions passed to scanFields. -type scanFunctions struct { - zerovalue func(name string) - normal func(name string) - wait func(name string) - value func(name, typName string) -} - -// scanFields scans the fields of a struct. -// -// Each provided function will be applied to appropriately tagged fields, or -// skipped if nil. -// -// Fields tagged nosave are skipped. -func scanFields(ss *ast.StructType, prefix string, fn scanFunctions) { - if ss.Fields.List == nil { - // No fields. - return - } - - // Scan all fields. - for _, field := range ss.Fields.List { - // Calculate the name. - name := "" - if field.Names != nil { - // It's a named field; override. - name = field.Names[0].Name - } else { - // Anonymous types can't be embedded, so we don't need - // to worry about providing a useful name here. - name, _ = resolveTypeName(field.Type) - } - - // Skip _ fields. - if name == "_" { - continue - } - - // Is this a anonymous struct? If yes, then continue the - // recursion with the given prefix. We don't pay attention to - // any tags on the top-level struct field. - tag := extractStateTag(field.Tag) - if anon, ok := field.Type.(*ast.StructType); ok && tag == "" { - scanFields(anon, name+".", fn) - continue - } - - switch tag { - case "zerovalue": - if fn.zerovalue != nil { - fn.zerovalue(name) - } - - case "": - if fn.normal != nil { - fn.normal(name) - } - - case "wait": - if fn.wait != nil { - fn.wait(name) - } - - case "manual", "nosave", "ignore": - // Do nothing. - - default: - if strings.HasPrefix(tag, ".(") && strings.HasSuffix(tag, ")") { - if fn.value != nil { - fn.value(name, tag[2:len(tag)-1]) - } - } - } - } -} - -func camelCased(name string) string { - return strings.ToUpper(name[:1]) + name[1:] -} - -func main() { - // Parse flags. - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage: %s [options]\n", os.Args[0]) - flag.PrintDefaults() - } - flag.Parse() - if len(flag.Args()) == 0 { - flag.Usage() - os.Exit(1) - } - if *fullPkg == "" { - fmt.Fprintf(os.Stderr, "Error: package required.") - os.Exit(1) - } - - // Open the output file. - var ( - outputFile *os.File - err error - ) - if *output == "" || *output == "-" { - outputFile = os.Stdout - } else { - outputFile, err = os.OpenFile(*output, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - fmt.Fprintf(os.Stderr, "Error opening output %q: %v", *output, err) - } - defer outputFile.Close() - } - - // Set the statePrefix for below, depending on the import. - statePrefix := "" - if *statePkg != "" { - parts := strings.Split(*statePkg, "/") - statePrefix = parts[len(parts)-1] + "." - } - - // initCalls is dumped at the end. - var initCalls []string - - // Common closures. - emitRegister := func(name string) { - initCalls = append(initCalls, fmt.Sprintf("%sRegister((*%s)(nil))", statePrefix, name)) - } - - // Automated warning. - fmt.Fprint(outputFile, "// automatically generated by stateify.\n\n") - - // Emit build tags. - if t := tags.Aggregate(flag.Args()); len(t) > 0 { - fmt.Fprintf(outputFile, "%s\n\n", strings.Join(t.Lines(), "\n")) - } - - // Emit the package name. - _, pkg := filepath.Split(*fullPkg) - fmt.Fprintf(outputFile, "package %s\n\n", pkg) - - // Emit the imports lazily. - var once sync.Once - maybeEmitImports := func() { - once.Do(func() { - // Emit the imports. - fmt.Fprint(outputFile, "import (\n") - if *statePkg != "" { - fmt.Fprintf(outputFile, " \"%s\"\n", *statePkg) - } - if *imports != "" { - for _, i := range strings.Split(*imports, ",") { - fmt.Fprintf(outputFile, " \"%s\"\n", i) - } - } - fmt.Fprint(outputFile, ")\n\n") - }) - } - - files := make([]*ast.File, 0, len(flag.Args())) - - // Parse the input files. - for _, filename := range flag.Args() { - // Parse the file. - fset := token.NewFileSet() - f, err := parser.ParseFile(fset, filename, nil, parser.ParseComments) - if err != nil { - // Not a valid input file? - fmt.Fprintf(os.Stderr, "Input %q can't be parsed: %v\n", filename, err) - os.Exit(1) - } - - files = append(files, f) - } - - type method struct { - typeName string - methodName string - } - - // Search for and add all method to a set. We auto-detecting several - // different methods (and insert them if we don't find them, in order - // to ensure that expectations match reality). - // - // While we do this, figure out the right receiver name. If there are - // multiple distinct receivers, then we will just pick the last one. - simpleMethods := make(map[method]struct{}) - receiverNames := make(map[string]string) - for _, f := range files { - // Go over all functions. - for _, decl := range f.Decls { - d, ok := decl.(*ast.FuncDecl) - if !ok { - continue - } - if d.Recv == nil || len(d.Recv.List) != 1 { - // Not a named method. - continue - } - - // Save the method and the receiver. - name, _ := resolveTypeName(d.Recv.List[0].Type) - simpleMethods[method{ - typeName: name, - methodName: d.Name.Name, - }] = struct{}{} - if len(d.Recv.List[0].Names) > 0 { - receiverNames[name] = d.Recv.List[0].Names[0].Name - } - } - } - - for _, f := range files { - // Go over all named types. - for _, decl := range f.Decls { - d, ok := decl.(*ast.GenDecl) - if !ok || d.Tok != token.TYPE { - continue - } - - // Only generate code for types marked "// +stateify - // savable" in one of the proceeding comment lines. If - // the line is marked "// +stateify type" then only - // generate type information and register the type. - if d.Doc == nil { - continue - } - var ( - generateTypeInfo = false - generateSaverLoader = false - ) - for _, l := range d.Doc.List { - if l.Text == "// +stateify savable" { - generateTypeInfo = true - generateSaverLoader = true - break - } - if l.Text == "// +stateify type" { - generateTypeInfo = true - } - } - if !generateTypeInfo && !generateSaverLoader { - continue - } - - for _, gs := range d.Specs { - ts := gs.(*ast.TypeSpec) - recv, ok := receiverNames[ts.Name.Name] - if !ok { - // Maybe no methods were defined? - recv = strings.ToLower(ts.Name.Name[:1]) - } - switch x := ts.Type.(type) { - case *ast.StructType: - maybeEmitImports() - - // Record the slot for each field. - fieldCount := 0 - fields := make(map[string]int) - emitField := func(name string) { - fmt.Fprintf(outputFile, " \"%s\",\n", name) - fields[name] = fieldCount - fieldCount++ - } - emitFieldValue := func(name string, _ string) { - emitField(name) - } - emitLoadValue := func(name, typName string) { - fmt.Fprintf(outputFile, " stateSourceObject.LoadValue(%d, new(%s), func(y interface{}) { %s.load%s(y.(%s)) })\n", fields[name], typName, recv, camelCased(name), typName) - } - emitLoad := func(name string) { - fmt.Fprintf(outputFile, " stateSourceObject.Load(%d, &%s.%s)\n", fields[name], recv, name) - } - emitLoadWait := func(name string) { - fmt.Fprintf(outputFile, " stateSourceObject.LoadWait(%d, &%s.%s)\n", fields[name], recv, name) - } - emitSaveValue := func(name, typName string) { - fmt.Fprintf(outputFile, " var %sValue %s = %s.save%s()\n", name, typName, recv, camelCased(name)) - fmt.Fprintf(outputFile, " stateSinkObject.SaveValue(%d, %sValue)\n", fields[name], name) - } - emitSave := func(name string) { - fmt.Fprintf(outputFile, " stateSinkObject.Save(%d, &%s.%s)\n", fields[name], recv, name) - } - emitZeroCheck := func(name string) { - fmt.Fprintf(outputFile, " if !%sIsZeroValue(&%s.%s) { %sFailf(\"%s is %%#v, expected zero\", &%s.%s) }\n", statePrefix, recv, name, statePrefix, name, recv, name) - } - - // Generate the type name method. - fmt.Fprintf(outputFile, "func (%s *%s) StateTypeName() string {\n", recv, ts.Name.Name) - fmt.Fprintf(outputFile, " return \"%s.%s\"\n", *fullPkg, ts.Name.Name) - fmt.Fprintf(outputFile, "}\n\n") - - // Generate the fields method. - fmt.Fprintf(outputFile, "func (%s *%s) StateFields() []string {\n", recv, ts.Name.Name) - fmt.Fprintf(outputFile, " return []string{\n") - scanFields(x, "", scanFunctions{ - normal: emitField, - wait: emitField, - value: emitFieldValue, - }) - fmt.Fprintf(outputFile, " }\n") - fmt.Fprintf(outputFile, "}\n\n") - - // Define beforeSave if a definition was not found. This prevents - // the code from compiling if a custom beforeSave was defined in a - // file not provided to this binary and prevents inherited methods - // from being called multiple times by overriding them. - if _, ok := simpleMethods[method{ - typeName: ts.Name.Name, - methodName: "beforeSave", - }]; !ok && generateSaverLoader { - fmt.Fprintf(outputFile, "func (%s *%s) beforeSave() {}\n\n", recv, ts.Name.Name) - } - - // Generate the save method. - // - // N.B. For historical reasons, we perform the value saves first, - // and perform the value loads last. There should be no dependency - // on this specific behavior, but the ability to specify slots - // allows a manual implementation to be order-dependent. - if generateSaverLoader { - fmt.Fprintf(outputFile, "// +checklocksignore\n") - fmt.Fprintf(outputFile, "func (%s *%s) StateSave(stateSinkObject %sSink) {\n", recv, ts.Name.Name, statePrefix) - fmt.Fprintf(outputFile, " %s.beforeSave()\n", recv) - scanFields(x, "", scanFunctions{zerovalue: emitZeroCheck}) - scanFields(x, "", scanFunctions{value: emitSaveValue}) - scanFields(x, "", scanFunctions{normal: emitSave, wait: emitSave}) - fmt.Fprintf(outputFile, "}\n\n") - } - - // Define afterLoad if a definition was not found. We do this for - // the same reason that we do it for beforeSave. - _, hasAfterLoad := simpleMethods[method{ - typeName: ts.Name.Name, - methodName: "afterLoad", - }] - if !hasAfterLoad && generateSaverLoader { - fmt.Fprintf(outputFile, "func (%s *%s) afterLoad() {}\n\n", recv, ts.Name.Name) - } - - // Generate the load method. - // - // N.B. See the comment above for the save method. - if generateSaverLoader { - fmt.Fprintf(outputFile, "// +checklocksignore\n") - fmt.Fprintf(outputFile, "func (%s *%s) StateLoad(stateSourceObject %sSource) {\n", recv, ts.Name.Name, statePrefix) - scanFields(x, "", scanFunctions{normal: emitLoad, wait: emitLoadWait}) - scanFields(x, "", scanFunctions{value: emitLoadValue}) - if hasAfterLoad { - // The call to afterLoad is made conditionally, because when - // AfterLoad is called, the object encodes a dependency on - // referred objects (i.e. fields). This means that afterLoad - // will not be called until the other afterLoads are called. - fmt.Fprintf(outputFile, " stateSourceObject.AfterLoad(%s.afterLoad)\n", recv) - } - fmt.Fprintf(outputFile, "}\n\n") - } - - // Add to our registration. - emitRegister(ts.Name.Name) - - case *ast.Ident, *ast.SelectorExpr, *ast.ArrayType: - maybeEmitImports() - - // Generate the info methods. - fmt.Fprintf(outputFile, "func (%s *%s) StateTypeName() string {\n", recv, ts.Name.Name) - fmt.Fprintf(outputFile, " return \"%s.%s\"\n", *fullPkg, ts.Name.Name) - fmt.Fprintf(outputFile, "}\n\n") - fmt.Fprintf(outputFile, "func (%s *%s) StateFields() []string {\n", recv, ts.Name.Name) - fmt.Fprintf(outputFile, " return nil\n") - fmt.Fprintf(outputFile, "}\n\n") - - // See above. - emitRegister(ts.Name.Name) - } - } - } - } - - if len(initCalls) > 0 { - // Emit the init() function. - fmt.Fprintf(outputFile, "func init() {\n") - for _, ic := range initCalls { - fmt.Fprintf(outputFile, " %s\n", ic) - } - fmt.Fprintf(outputFile, "}\n") - } -} diff --git a/tools/images.mk b/tools/images.mk deleted file mode 100644 index 2003da5bd..000000000 --- a/tools/images.mk +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/make -f - -# Copyright 2018 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## -## Docker image targets. -## -## Images used by the tests must also be built and available locally. -## The canonical test targets defined below will automatically load -## relevant images. These can be loaded or built manually via these -## targets. -## -## (*) Note that you may provide an ARCH parameter in order to build -## and load images from an alternate archiecture (using qemu). When -## bazel is run as a server, this has the effect of running an full -## cross-architecture chain, and can produce cross-compiled binaries. -## - -# ARCH is the architecture used for the build. This may be overriden at the -# command line in order to perform a cross-build (in a limited capacity). -ARCH := $(shell uname -m) -ifneq ($(ARCH),$(shell uname -m)) -DOCKER_PLATFORM_ARGS := --platform=$(ARCH) -else -DOCKER_PLATFORM_ARGS := -endif - -# Note that the image prefixes used here must match the image mangling in -# runsc/testutil.MangleImage. Names are mangled in this way to ensure that all -# tests are using locally-defined images (that are consistent and idempotent). -REMOTE_IMAGE_PREFIX ?= gcr.io/gvisor-presubmit -LOCAL_IMAGE_PREFIX ?= gvisor.dev/images -ALL_IMAGES := $(subst /,_,$(subst images/,,$(shell find images/ -name Dockerfile -o -name Dockerfile.$(ARCH) | xargs -n 1 dirname | uniq))) -SUB_IMAGES := $(foreach image,$(ALL_IMAGES),$(if $(findstring _,$(image)),$(image),)) -IMAGE_GROUPS := $(sort $(foreach image,$(SUB_IMAGES),$(firstword $(subst _, ,$(image))))) - -define expand_group = -load-$(1): $$(patsubst $(1)_%, load-$(1)_%, $$(filter $(1)_%,$$(ALL_IMAGES))) - @ -.PHONY: load-$(1) -push-$(1): $$(patsubst $(1)_%, push-$(1)_%, $$(filter $(1)_%,$$(ALL_IMAGES))) - @ -.PHONY: push-$(1) -endef -$(foreach group,$(IMAGE_GROUPS),$(eval $(call expand_group,$(group)))) - -list-all-images: ## List all images. - @for image in $(ALL_IMAGES); do echo $${image}; done -.PHONY: list-all-images - -load-all-images: ## Load all images. -load-all-images: $(patsubst %,load-%,$(ALL_IMAGES)) -.PHONY: load-all-images - -push-all-images: ## Push all images. -push-all-images: $(patsubst %,push-%,$(ALL_IMAGES)) -.PHONY: push-all-images - -# path and dockerfile are used to extract the relevant path and dockerfile -# (depending on what's available for the given architecture). -path = images/$(subst _,/,$(1)) -dockerfile = $$(if [ -f "$(call path,$(1))/Dockerfile.$(ARCH)" ]; then echo Dockerfile.$(ARCH); else echo Dockerfile; fi) - -# The tag construct is used to memoize the image generated (see README.md). -# This scheme is used to enable aggressive caching in a central repository, but -# ensuring that images will always be sourced using the local files. -tag = $(shell cd images && find $(subst _,/,$(1)) -type f | sort | xargs -n 1 sha256sum | sha256sum - | cut -c 1-16) -remote_image = $(REMOTE_IMAGE_PREFIX)/$(subst _,/,$(1))_$(ARCH) -local_image = $(LOCAL_IMAGE_PREFIX)/$(subst _,/,$(1)) - -# Include all existing images as targets here. -# -# Note that we use a _ for the tag separator, instead of :, as the latter is -# interpreted by Make, unfortunately. tag_expand expands the generic rules to -# tag-specific targets. These is needed to provide sensible targets for load -# below, with caching. Basically, if there is a rule generated here, then the -# load will be skipped. If there is no load generated here, then the default -# rule for load will kick in. -# -# Note that if this rule does not successfully rule, we will simply have -# additional Docker pull commands that run for all images that are already -# pulled. No real harm done. -EXISTING_IMAGES = $(shell docker images --format '{{.Repository}}_{{.Tag}}' | grep -v '<none>') -define existing_image_rule = -loaded0_$(1)=load-$$(1): tag-$$(1) # Already available. -loaded1_$(1)=.PHONY: load-$$(1) -endef -$(foreach image, $(EXISTING_IMAGES), $(eval $(call existing_image_rule,$(image)))) -define tag_expand_rule = -$(eval $(loaded0_$(call remote_image,$(1))_$(call tag,$(1)))) -$(eval $(loaded1_$(call remote_image,$(1))_$(call tag,$(1)))) -endef -$(foreach image, $(ALL_IMAGES), $(eval $(call tag_expand_rule,$(image)))) - -# tag tags a local image. This applies both the hash-based tag from above to -# ensure that caching works as expected, as well as the "latest" tag that is -# used by the tests. -local_tag = \ - docker tag $(call remote_image,$(1)):$(call tag,$(1)) $(call local_image,$(1)):$(call tag,$(1)) >&2 -latest_tag = \ - docker tag $(call local_image,$(1)):$(call tag,$(1)) $(call local_image,$(1)) >&2 -tag-%: ## Tag a local image. - @$(call header,TAG $*) - @$(call local_tag,$*) && $(call latest_tag,$*) - -# pull forces the image to be pulled. -pull = \ - $(call header,PULL $(1)) && \ - docker pull $(DOCKER_PLATFORM_ARGS) $(call remote_image,$(1)):$(call tag,$(1)) >&2 && \ - $(call local_tag,$(1)) && \ - $(call latest_tag,$(1)) -pull-%: register-cross ## Force a repull of the image. - @$(call pull,$*) - -# rebuild builds the image locally. Only the "remote" tag will be applied. Note -# we need to explicitly repull the base layer in order to ensure that the -# architecture is correct. Note that we use the term "rebuild" here to avoid -# conflicting with the bazel "build" terminology, which is used elsewhere. -rebuild = \ - $(call header,REBUILD $(1)) && \ - (T=$$(mktemp -d) && cp -a $(call path,$(1))/* $$T && \ - $(foreach image,$(shell grep FROM "$(call path,$(1))/$(call dockerfile,$(1))" 2>/dev/null | cut -d' ' -f2),docker pull $(DOCKER_PLATFORM_ARGS) $(image) >&2 &&) \ - docker build $(DOCKER_PLATFORM_ARGS) \ - -f "$$T/$(call dockerfile,$(1))" \ - -t "$(call remote_image,$(1)):$(call tag,$(1))" \ - $$T >&2 && \ - rm -rf $$T) && \ - $(call local_tag,$(1)) && \ - $(call latest_tag,$(1)) -rebuild-%: register-cross ## Force rebuild an image locally. - @$(call rebuild,$*) - -# load will either pull the "remote" or build it locally. This is the preferred -# entrypoint, as it should never fail. The local tag should always be set after -# this returns (either by the pull or the build). -load-%: register-cross ## Pull or build an image locally. - @($(call pull,$*)) || ($(call rebuild,$*)) - -# push pushes the remote image, after either pulling (to validate that the tag -# already exists) or building manually. Note that this generic rule will match -# the fully-expanded remote image tag. -push-%: load-% ## Push a given image. - @docker push $(call remote_image,$*):$(call tag,$*) >&2 - -# register-cross registers the necessary qemu binaries for cross-compilation. -# This may be used by any target that may execute containers that are not the -# native format. Note that this will only apply on the first execution. -register-cross: -ifneq ($(ARCH),$(shell uname -m)) -ifeq (,$(wildcard /proc/sys/fs/binfmt_misc/qemu-*)) - @docker run --rm --privileged multiarch/qemu-user-static --reset --persistent yes >&2 -else - @ -endif -else - @ -endif diff --git a/tools/installers/BUILD b/tools/installers/BUILD deleted file mode 100644 index d9f9c4c40..000000000 --- a/tools/installers/BUILD +++ /dev/null @@ -1,32 +0,0 @@ -# Installers for use by top-level scripts. - -package( - default_visibility = ["//:sandbox"], - licenses = ["notice"], -) - -sh_binary( - name = "head", - srcs = ["head.sh"], - data = [ - "//runsc", - ], -) - -sh_binary( - name = "master", - srcs = ["master.sh"], -) - -sh_binary( - name = "containerd", - srcs = ["containerd.sh"], -) - -sh_binary( - name = "shim", - srcs = ["shim.sh"], - data = [ - "//shim:containerd-shim-runsc-v1", - ], -) diff --git a/tools/installers/containerd.sh b/tools/installers/containerd.sh deleted file mode 100755 index e598bce89..000000000 --- a/tools/installers/containerd.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -xeo pipefail - -declare -r CONTAINERD_VERSION=${1:-1.3.0} -declare -r CONTAINERD_MAJOR="$(echo ${CONTAINERD_VERSION} | awk -F '.' '{ print $1; }')" -declare -r CONTAINERD_MINOR="$(echo ${CONTAINERD_VERSION} | awk -F '.' '{ print $2; }')" - -# Default to an older version for crictl for containerd <= 1.2. -if [[ "${CONTAINERD_MAJOR}" -eq 1 ]] && [[ "${CONTAINERD_MINOR}" -le 2 ]]; then - declare -r CRITOOLS_VERSION=${CRITOOLS_VERSION:-1.13.0} -else - declare -r CRITOOLS_VERSION=${CRITOOLS_VERSION:-1.18.0} -fi - -# Helper for Go packages below. -install_helper() { - PACKAGE="${1}" - TAG="${2}" - - # Clone the repository. - mkdir -p "${GOPATH}"/src/$(dirname "${PACKAGE}") && \ - git clone https://"${PACKAGE}" "${GOPATH}"/src/"${PACKAGE}" - - # Checkout and build the repository. - (cd "${GOPATH}"/src/"${PACKAGE}" && \ - git checkout "${TAG}" && \ - make && \ - make install) -} - -# Figure out were btrfs headers are. -# -# Ubuntu 16.04 has only btrfs-tools, while 18.04 has a transitional package, -# and later versions no longer have the transitional package. -source /etc/os-release -declare BTRFS_DEV -if [[ "${VERSION_ID%.*}" -le "18" ]]; then - BTRFS_DEV="btrfs-tools" -else - BTRFS_DEV="libbtrfs-dev" -fi -readonly BTRFS_DEV - -# Install dependencies for the crictl tests. -while true; do - if (apt-get update && apt-get install -y \ - "${BTRFS_DEV}" \ - libseccomp-dev); then - break - fi - result=$? - if [[ $result -ne 100 ]]; then - exit $result - fi -done - -# Install containerd & cri-tools. -declare -rx GOPATH=$(mktemp -d --tmpdir gopathXXXXX) -install_helper github.com/containerd/containerd "v${CONTAINERD_VERSION}" "${GOPATH}" -install_helper github.com/kubernetes-sigs/cri-tools "v${CRITOOLS_VERSION}" "${GOPATH}" - -# Configure containerd-shim. -declare -r shim_config_path=/etc/containerd/runsc/config.toml -mkdir -p $(dirname ${shim_config_path}) -cat > ${shim_config_path} <<-EOF -log_path = "/tmp/shim-logs/" -log_level = "debug" - -[runsc_config] - debug = "true" - debug-log = "/tmp/runsc-logs/" - strace = "true" - file-access = "shared" -EOF - -# Configure CNI. -(cd "${GOPATH}" && src/github.com/containerd/containerd/script/setup/install-cni) -cat <<EOF | sudo tee /etc/cni/net.d/10-bridge.conf -{ - "cniVersion": "0.3.1", - "name": "bridge", - "type": "bridge", - "bridge": "cnio0", - "isGateway": true, - "ipMasq": true, - "ipam": { - "type": "host-local", - "ranges": [ - [{"subnet": "10.200.0.0/24"}] - ], - "routes": [{"dst": "0.0.0.0/0"}] - } -} -EOF -cat <<EOF | sudo tee /etc/cni/net.d/99-loopback.conf -{ - "cniVersion": "0.3.1", - "type": "loopback" -} -EOF - -# Configure crictl. -cat <<EOF | sudo tee /etc/crictl.yaml -runtime-endpoint: unix:///run/containerd/containerd.sock -EOF - -# Cleanup. -rm -rf "${GOPATH}" diff --git a/tools/installers/head.sh b/tools/installers/head.sh deleted file mode 100755 index a613fcb5b..000000000 --- a/tools/installers/head.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Install our runtime. -runfiles=. -if [[ -d "$0.runfiles" ]]; then - runfiles="$0.runfiles" -fi -$(find -L "${runfiles}" -executable -type f -name runsc) install - -# Restart docker. -if service docker status 2>/dev/null; then - service docker restart -fi diff --git a/tools/installers/images.sh b/tools/installers/images.sh deleted file mode 100755 index 52e750f57..000000000 --- a/tools/installers/images.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# Copyright 2020 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -xeuo pipefail - -# Find the images directory. -for images in $(find . -type d -name images); do - if [[ -f "${images}"/Makefile ]]; then - make -C "${images}" load-all-images - fi -done diff --git a/tools/installers/master.sh b/tools/installers/master.sh deleted file mode 100755 index 2c6001c6c..000000000 --- a/tools/installers/master.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Install runsc from the master branch. -set -e - -curl -fsSL https://gvisor.dev/archive.key | sudo apt-key add - -add-apt-repository "deb https://storage.googleapis.com/gvisor/releases release main" - -while true; do - if (apt-get update && apt-get install -y runsc); then - break - fi - result=$? - if [[ $result -ne 100 ]]; then - exit $result - fi -done - -runsc install -service docker restart diff --git a/tools/installers/shim.sh b/tools/installers/shim.sh deleted file mode 100755 index 9af50b5c7..000000000 --- a/tools/installers/shim.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Install all the shims. -# -# Note that containerd looks at the current executable directory -# in order to find the shim binary. So we need to check in order -# of preference. The local containerd installer will install to -# /usr/local, so we use that first. -if [[ -x /usr/local/bin/containerd ]]; then - containerd_install_dir=/usr/local/bin -else - containerd_install_dir=/usr/bin -fi -runfiles=. -if [[ -d "$0.runfiles" ]]; then - runfiles="$0.runfiles" -fi -find -L "${runfiles}" -executable -type f -name containerd-shim-runsc-v1 -exec cp -L {} "${containerd_install_dir}" \; diff --git a/tools/make_apt.sh b/tools/make_apt.sh deleted file mode 100755 index 935c4db2d..000000000 --- a/tools/make_apt.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/bin/bash - -# Copyright 2018 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ "$#" -le 3 ]]; then - echo "usage: $0 <private-key> <suite> <root> <packages...>" - exit 1 -fi -declare private_key -declare suite -declare root -private_key="$(readlink -e "$1")" -suite="$2" -root="$(readlink -m "$3")" -readonly private_key -readonly suite -readonly root -shift; shift; shift # For "$@" below. - -# Ensure that we have the correct packages installed. -function apt_install() { - while true; do - sudo apt-get update && - sudo apt-get install -y "$@" && - true - result="${?}" - case $result in - 0) - break - ;; - 100) - # 100 is the error code that apt-get returns. - ;; - *) - exit $result - ;; - esac - done -} -dpkg-sig --help >/dev/null 2>&1 || apt_install dpkg-sig -apt-ftparchive --help >/dev/null 2>&1 || apt_install apt-utils -xz --help >/dev/null 2>&1 || apt_install xz-utils - -# Verbose from this point. -set -xeo pipefail - -# Create a directory for the release. -declare -r release="${root}/dists/${suite}" -mkdir -p "${release}" - -# Create a temporary keyring, and ensure it is cleaned up. -# Using separate homedir allows us to install apt repositories multiple times -# using the same key. This is a limitation in GnuPG pre-2.1. -declare keyring -declare homedir -declare gpg_opts -keyring="$(mktemp /tmp/keyringXXXXXX.gpg)" -homedir="$(mktemp -d /tmp/homedirXXXXXX)" -gpg_opts=("--no-default-keyring" "--secret-keyring" "${keyring}" "--homedir" "${homedir}") -readonly keyring -readonly homedir -readonly gpg_opts -cleanup() { - rm -rf "${keyring}" "${homedir}" -} -trap cleanup EXIT - -# We attempt the import twice because the first one will fail if the public key -# is not found. This isn't actually a failure for us, because we don't require -# the public key (this may be stored separately). The second import will succeed -# because, in reality, the first import succeeded and it's a no-op. -gpg "${gpg_opts[@]}" --import "${private_key}" || \ - gpg "${gpg_opts[@]}" --import "${private_key}" - -# Copy the packages into the root. -for pkg in "$@"; do - if ! [[ -f "${pkg}" ]]; then - continue - fi - ext=${pkg##*.} - if [[ "${ext}" != "deb" ]]; then - continue - fi - - # Extract package information. - name=$(basename "${pkg}" ".${ext}") - arch=$(dpkg --info "${pkg}" | grep 'Architecture:' | cut -d':' -f2) - version=$(dpkg --info "${pkg}" | grep 'Version:' | cut -d':' -f2) - arch=${arch// /} # Trim whitespace. - version=${version// /} # Ditto. - destdir="${root}/pool/${version}/binary-${arch}" - - # Copy & sign the package. - mkdir -p "${destdir}" - cp -a -L "$(dirname "${pkg}")/${name}.deb" "${destdir}" - cp -a -L "$(dirname "${pkg}")/${name}.changes" "${destdir}" - chmod 0644 "${destdir}"/"${name}".* - # Sign a package only if it isn't signed yet. - # We use [*] here to expand the gpg_opts array into a single shell-word. - dpkg-sig -g "${gpg_opts[*]}" --verify "${destdir}/${name}.deb" || - dpkg-sig -g "${gpg_opts[*]}" --sign builder "${destdir}/${name}.deb" -done - -# Build the package list. -declare arches=() -for dir in "${root}"/pool/*/binary-*; do - name=$(basename "${dir}") - arch=${name##binary-} - arches+=("${arch}") - repo_packages="${release}"/main/"${name}" - mkdir -p "${repo_packages}" - (cd "${root}" && apt-ftparchive packages "${dir##${root}/}" > "${repo_packages}"/Packages) - if ! [[ -s "${repo_packages}"/Packages ]]; then - echo "Packages file is size zero." >&2 - exit 1 - fi - (cd "${repo_packages}" && cat Packages | gzip > Packages.gz) - (cd "${repo_packages}" && cat Packages | xz > Packages.xz) -done - -# Build the release list. -cat > "${release}"/apt.conf <<EOF -APT { - FTPArchive { - Release { - Architectures "${arches[@]}"; - Suite "${suite}"; - Components "main"; - }; - }; -}; -EOF -(cd "${release}" && apt-ftparchive -c=apt.conf release . > Release) -rm "${release}"/apt.conf - -# Sign the release. -declare -r digest_opts=("--digest-algo" "SHA512" "--cert-digest-algo" "SHA512") -(cd "${release}" && rm -f Release.gpg InRelease) -(cd "${release}" && gpg "${gpg_opts[@]}" --clearsign "${digest_opts[@]}" -o InRelease Release) -(cd "${release}" && gpg "${gpg_opts[@]}" -abs "${digest_opts[@]}" -o Release.gpg Release) diff --git a/tools/make_release.sh b/tools/make_release.sh deleted file mode 100755 index 10742dd54..000000000 --- a/tools/make_release.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -# Copyright 2018 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [[ "$#" -le 2 ]]; then - echo "usage: $0 <private-key> <root> <binaries & packages...>" - echo "The environment variable NIGHTLY may be set to control" - echo "whether the nightly packages are produced or not." - exit 1 -fi - -set -xeo pipefail -declare -r private_key="$1"; shift -declare -r root="$1"; shift -declare -a binaries -declare -a pkgs - -# Collect binaries & packages. -for arg in "$@"; do - if [[ "${arg}" == *.deb ]] || [[ "${arg}" == *.changes ]]; then - pkgs+=("${arg}") - else - binaries+=("${arg}") - fi -done - -# install_raw installs raw artifacts. -install_raw() { - for binary in "${binaries[@]}"; do - # Copy the raw file & generate a sha512sum, sorted by architecture. - arch=$(file "${binary}" | cut -d',' -f2 | awk '{print $NF}' | tr '-' '_') - name=$(basename "${binary}") - mkdir -p "${root}/$1/${arch}" - cp -f "${binary}" "${root}/$1/${arch}" - (cd "${root}/$1/${arch}" && sha512sum "${name}" > "${name}.sha512") - done -} - -# install_apt installs an apt repository. -install_apt() { - tools/make_apt.sh "${private_key}" "$1" "${root}" "${pkgs[@]}" -} - -# If nightly, install only nightly artifacts. -if [[ "${NIGHTLY:-false}" == "true" ]]; then - # The "latest" directory and current date. - stamp="$(date -Idate)" - install_raw "nightly/latest" - install_raw "nightly/${stamp}" - install_apt "nightly" -else - # Is it a tagged release? Build that. - tags="$(git tag --points-at HEAD 2>/dev/null || true)" - if ! [[ -z "${tags}" ]]; then - # Note that a given commit can match any number of tags. We have to iterate - # through all possible tags and produce associated artifacts. - for tag in ${tags}; do - name=$(echo "${tag}" | cut -d'-' -f2) - base=$(echo "${name}" | cut -d'.' -f1) - install_raw "release/${name}" - install_raw "release/latest" - install_apt "release" - install_apt "${base}" - done - else - # Otherwise, assume it is a raw master commit. - install_raw "master/latest" - install_apt "master" - fi -fi diff --git a/tools/nogo/BUILD b/tools/nogo/BUILD deleted file mode 100644 index 6c6f604b5..000000000 --- a/tools/nogo/BUILD +++ /dev/null @@ -1,78 +0,0 @@ -load("//tools:defs.bzl", "bzl_library", "go_library", "select_goarch", "select_goos") -load("//tools/nogo:defs.bzl", "nogo_objdump_tool", "nogo_stdlib", "nogo_target") - -package(licenses = ["notice"]) - -exports_files(["config-schema.json"]) - -nogo_target( - name = "target", - goarch = select_goarch(), - goos = select_goos(), - visibility = ["//visibility:public"], -) - -nogo_objdump_tool( - name = "objdump_tool", - visibility = ["//visibility:public"], -) - -nogo_stdlib( - name = "stdlib", - visibility = ["//visibility:public"], -) - -go_library( - name = "nogo", - srcs = [ - "analyzers.go", - "build.go", - "config.go", - "findings.go", - "nogo.go", - ], - nogo = False, - visibility = ["//:sandbox"], - deps = [ - "//tools/checkescape", - "//tools/checklocks", - "//tools/checkunsafe", - "//tools/worker", - "@co_honnef_go_tools//staticcheck:go_default_library", - "@co_honnef_go_tools//stylecheck:go_default_library", - "@org_golang_x_tools//go/analysis:go_default_library", - "@org_golang_x_tools//go/analysis/internal/facts:go_default_library", - "@org_golang_x_tools//go/analysis/passes/asmdecl:go_default_library", - "@org_golang_x_tools//go/analysis/passes/assign:go_default_library", - "@org_golang_x_tools//go/analysis/passes/atomic:go_default_library", - "@org_golang_x_tools//go/analysis/passes/bools:go_default_library", - "@org_golang_x_tools//go/analysis/passes/buildtag:go_default_library", - "@org_golang_x_tools//go/analysis/passes/cgocall:go_default_library", - "@org_golang_x_tools//go/analysis/passes/composite:go_default_library", - "@org_golang_x_tools//go/analysis/passes/copylock:go_default_library", - "@org_golang_x_tools//go/analysis/passes/errorsas:go_default_library", - "@org_golang_x_tools//go/analysis/passes/httpresponse:go_default_library", - "@org_golang_x_tools//go/analysis/passes/loopclosure:go_default_library", - "@org_golang_x_tools//go/analysis/passes/lostcancel:go_default_library", - "@org_golang_x_tools//go/analysis/passes/nilfunc:go_default_library", - "@org_golang_x_tools//go/analysis/passes/nilness:go_default_library", - "@org_golang_x_tools//go/analysis/passes/printf:go_default_library", - "@org_golang_x_tools//go/analysis/passes/shadow:go_default_library", - "@org_golang_x_tools//go/analysis/passes/shift:go_default_library", - "@org_golang_x_tools//go/analysis/passes/stdmethods:go_default_library", - "@org_golang_x_tools//go/analysis/passes/stringintconv:go_default_library", - "@org_golang_x_tools//go/analysis/passes/structtag:go_default_library", - "@org_golang_x_tools//go/analysis/passes/tests:go_default_library", - "@org_golang_x_tools//go/analysis/passes/unmarshal:go_default_library", - "@org_golang_x_tools//go/analysis/passes/unreachable:go_default_library", - "@org_golang_x_tools//go/analysis/passes/unsafeptr:go_default_library", - "@org_golang_x_tools//go/analysis/passes/unusedresult:go_default_library", - "@org_golang_x_tools//go/gcexportdata:go_default_library", - ], -) - -bzl_library( - name = "defs_bzl", - srcs = ["defs.bzl"], - visibility = ["//visibility:private"], -) diff --git a/tools/nogo/README.md b/tools/nogo/README.md deleted file mode 100644 index 6e4db18de..000000000 --- a/tools/nogo/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Extended "nogo" analysis - -This package provides a build aspect that perform nogo analysis. This will be -automatically injected to all relevant libraries when using the default -`go_binary` and `go_library` rules. - -It exists for several reasons. - -* The default `nogo` provided by bazel is insufficient with respect to the - possibility of binary analysis. This package allows us to analyze the - generated binary in addition to using the standard analyzers. - -* The configuration provided in this package is much richer than the standard - `nogo` JSON blob. Specifically, it allows us to exclude specific structures - from the composite rules (such as the Ranges that are common with the set - types). - -* The bazel version of `nogo` is run directly against the `go_library` and - `go_binary` targets, meaning that any change to the configuration requires a - rebuild from scratch (for some reason included all C++ source files in the - process). Using an aspect is more efficient in this regard. - -* The checks supported by this package are exported as tests, which makes it - easier to reason about and plumb into the build system. - -* For uninteresting reasons, it is impossible to integrate the default `nogo` - analyzer provided by bazel with internal Google tooling. To provide a - consistent experience, this package allows those systems to be unified. - -To use this package, import `nogo_test` from `defs.bzl` and add a single -dependency which is a `go_binary` or `go_library` rule. diff --git a/tools/nogo/analyzers.go b/tools/nogo/analyzers.go deleted file mode 100644 index 2b3c03fec..000000000 --- a/tools/nogo/analyzers.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package nogo - -import ( - "encoding/gob" - - "golang.org/x/tools/go/analysis" - "golang.org/x/tools/go/analysis/passes/asmdecl" - "golang.org/x/tools/go/analysis/passes/assign" - "golang.org/x/tools/go/analysis/passes/atomic" - "golang.org/x/tools/go/analysis/passes/bools" - "golang.org/x/tools/go/analysis/passes/buildtag" - "golang.org/x/tools/go/analysis/passes/cgocall" - "golang.org/x/tools/go/analysis/passes/composite" - "golang.org/x/tools/go/analysis/passes/copylock" - "golang.org/x/tools/go/analysis/passes/errorsas" - "golang.org/x/tools/go/analysis/passes/httpresponse" - "golang.org/x/tools/go/analysis/passes/loopclosure" - "golang.org/x/tools/go/analysis/passes/lostcancel" - "golang.org/x/tools/go/analysis/passes/nilfunc" - "golang.org/x/tools/go/analysis/passes/nilness" - "golang.org/x/tools/go/analysis/passes/printf" - "golang.org/x/tools/go/analysis/passes/shadow" - "golang.org/x/tools/go/analysis/passes/shift" - "golang.org/x/tools/go/analysis/passes/stdmethods" - "golang.org/x/tools/go/analysis/passes/stringintconv" - "golang.org/x/tools/go/analysis/passes/structtag" - "golang.org/x/tools/go/analysis/passes/tests" - "golang.org/x/tools/go/analysis/passes/unmarshal" - "golang.org/x/tools/go/analysis/passes/unreachable" - "golang.org/x/tools/go/analysis/passes/unsafeptr" - "golang.org/x/tools/go/analysis/passes/unusedresult" - "honnef.co/go/tools/staticcheck" - "honnef.co/go/tools/stylecheck" - - "gvisor.dev/gvisor/tools/checkescape" - "gvisor.dev/gvisor/tools/checklocks" - "gvisor.dev/gvisor/tools/checkunsafe" -) - -// AllAnalyzers is a list of all available analyzers. -var AllAnalyzers = []*analysis.Analyzer{ - asmdecl.Analyzer, - assign.Analyzer, - atomic.Analyzer, - bools.Analyzer, - buildtag.Analyzer, - cgocall.Analyzer, - composite.Analyzer, - copylock.Analyzer, - errorsas.Analyzer, - httpresponse.Analyzer, - loopclosure.Analyzer, - lostcancel.Analyzer, - nilfunc.Analyzer, - nilness.Analyzer, - printf.Analyzer, - shift.Analyzer, - stdmethods.Analyzer, - stringintconv.Analyzer, - shadow.Analyzer, - structtag.Analyzer, - tests.Analyzer, - unmarshal.Analyzer, - unreachable.Analyzer, - unsafeptr.Analyzer, - unusedresult.Analyzer, - checkescape.Analyzer, - checkunsafe.Analyzer, - checklocks.Analyzer, -} - -func register(all []*analysis.Analyzer) { - // Register all fact types. - // - // N.B. This needs to be done recursively, because there may be - // analyzers in the Requires list that do not appear explicitly above. - registered := make(map[*analysis.Analyzer]struct{}) - var registerOne func(*analysis.Analyzer) - registerOne = func(a *analysis.Analyzer) { - if _, ok := registered[a]; ok { - return - } - - // Register dependencies. - for _, da := range a.Requires { - registerOne(da) - } - - // Register local facts. - for _, f := range a.FactTypes { - gob.Register(f) - } - - registered[a] = struct{}{} // Done. - } - for _, a := range all { - registerOne(a) - } -} - -func init() { - // Add all staticcheck analyzers. - for _, a := range staticcheck.Analyzers { - AllAnalyzers = append(AllAnalyzers, a) - } - // Add all stylecheck analyzers. - for _, a := range stylecheck.Analyzers { - AllAnalyzers = append(AllAnalyzers, a) - } - - // Register lists. - register(AllAnalyzers) -} diff --git a/tools/nogo/build.go b/tools/nogo/build.go deleted file mode 100644 index d173cff1f..000000000 --- a/tools/nogo/build.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package nogo - -import ( - "fmt" - "io" - "os" -) - -// findStdPkg needs to find the bundled standard library packages. -func findStdPkg(GOOS, GOARCH, path string) (io.ReadCloser, error) { - if path == "C" { - // Cgo builds cannot be analyzed. Skip. - return nil, ErrSkip - } - return os.Open(fmt.Sprintf("external/go_sdk/pkg/%s_%s/%s.a", GOOS, GOARCH, path)) -} diff --git a/tools/nogo/check/BUILD b/tools/nogo/check/BUILD deleted file mode 100644 index 666780dd3..000000000 --- a/tools/nogo/check/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -load("//tools:defs.bzl", "go_binary") - -package(licenses = ["notice"]) - -go_binary( - name = "check", - srcs = ["main.go"], - nogo = False, - visibility = ["//visibility:public"], - deps = [ - "//tools/nogo", - "//tools/worker", - ], -) diff --git a/tools/nogo/check/main.go b/tools/nogo/check/main.go deleted file mode 100644 index 3a6c3fb08..000000000 --- a/tools/nogo/check/main.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Binary check is the nogo entrypoint. -package main - -import ( - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "log" - "os" - - "gvisor.dev/gvisor/tools/nogo" - "gvisor.dev/gvisor/tools/worker" -) - -var ( - packageFile = flag.String("package", "", "package configuration file (in JSON format)") - stdlibFile = flag.String("stdlib", "", "stdlib configuration file (in JSON format)") - findingsOutput = flag.String("findings", "", "output file (or stdout, if not specified)") - factsOutput = flag.String("facts", "", "output file for facts (optional)") -) - -func loadConfig(file string, config interface{}) interface{} { - // Load the configuration. - f, err := os.Open(file) - if err != nil { - log.Fatalf("unable to open configuration %q: %v", file, err) - } - defer f.Close() - dec := json.NewDecoder(f) - dec.DisallowUnknownFields() - if err := dec.Decode(config); err != nil { - log.Fatalf("unable to decode configuration: %v", err) - } - return config -} - -func main() { - worker.Work(run) -} - -func run([]string) int { - var ( - findings []nogo.Finding - factData []byte - err error - ) - - // Check & load the configuration. - if *packageFile != "" && *stdlibFile != "" { - log.Fatalf("unable to perform stdlib and package analysis; provide only one!") - } - - // Run the configuration. - if *stdlibFile != "" { - // Perform stdlib analysis. - c := loadConfig(*stdlibFile, new(nogo.StdlibConfig)).(*nogo.StdlibConfig) - findings, factData, err = nogo.CheckStdlib(c, nogo.AllAnalyzers) - } else if *packageFile != "" { - // Perform standard analysis. - c := loadConfig(*packageFile, new(nogo.PackageConfig)).(*nogo.PackageConfig) - findings, factData, err = nogo.CheckPackage(c, nogo.AllAnalyzers, nil) - } else { - log.Fatalf("please provide at least one of package or stdlib!") - } - - // Check that analysis was successful. - if err != nil { - log.Fatalf("error performing analysis: %v", err) - } - - // Save facts. - if *factsOutput != "" { - if err := ioutil.WriteFile(*factsOutput, factData, 0644); err != nil { - log.Fatalf("error saving findings to %q: %v", *factsOutput, err) - } - } - - // Write all findings. - if *findingsOutput != "" { - w, err := os.OpenFile(*findingsOutput, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - log.Fatalf("error opening output file %q: %v", *findingsOutput, err) - } - if err := nogo.WriteFindingsTo(w, findings, false /* json */); err != nil { - log.Fatalf("error writing findings to %q: %v", *findingsOutput, err) - } - } else { - for _, finding := range findings { - fmt.Fprintf(os.Stdout, "%s\n", finding.String()) - } - } - - return 0 -} diff --git a/tools/nogo/config-schema.json b/tools/nogo/config-schema.json deleted file mode 100644 index 3c25fe221..000000000 --- a/tools/nogo/config-schema.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "definitions": { - "group": { - "type": "object", - "properties": { - "name": { - "description": "The name of the group.", - "type": "string" - }, - "regex": { - "description": "A regular expression for matching paths.", - "type": "string" - }, - "default": { - "description": "Whether the group is enabled by default.", - "type": "boolean" - } - }, - "required": [ - "name", - "regex", - "default" - ], - "additionalProperties": false - }, - "regexlist": { - "description": "A list of regular expressions.", - "oneOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "rule": { - "type": "object", - "properties": { - "exclude": { - "description": "A regular expression for paths to exclude.", - "$ref": "#/definitions/regexlist" - }, - "suppress": { - "description": "A regular expression for messages to suppress.", - "$ref": "#/definitions/regexlist" - } - }, - "additionalProperties": false - }, - "ruleList": { - "type": "object", - "additionalProperties": { - "oneOf": [ - { - "$ref": "#/definitions/rule" - }, - { - "type": "null" - } - ] - } - } - }, - "properties": { - "groups": { - "description": "A definition of all groups.", - "type": "array", - "items": { - "$ref": "#/definitions/group" - }, - "minItems": 1 - }, - "global": { - "description": "A global set of rules.", - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/rule" - } - }, - "analyzers": { - "description": "A definition of all groups.", - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/ruleList" - } - } - }, - "required": [ - "groups" - ], - "additionalProperties": false -} diff --git a/tools/nogo/config.go b/tools/nogo/config.go deleted file mode 100644 index 6436f9d34..000000000 --- a/tools/nogo/config.go +++ /dev/null @@ -1,311 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package nogo - -import ( - "fmt" - "regexp" -) - -// GroupName is a named group. -type GroupName string - -// AnalyzerName is a named analyzer. -type AnalyzerName string - -// Group represents a named collection of files. -type Group struct { - // Name is the short name for the group. - Name GroupName `yaml:"name"` - - // Regex matches all full paths in the group. - Regex string `yaml:"regex"` - regex *regexp.Regexp `yaml:"-"` - - // Default determines the default group behavior. - // - // If Default is true, all Analyzers are enabled for this - // group. Otherwise, Analyzers must be individually enabled - // by specifying a (possible empty) ItemConfig for the group - // in the AnalyzerConfig. - Default bool `yaml:"default"` -} - -func (g *Group) compile() error { - r, err := regexp.Compile(g.Regex) - if err != nil { - return err - } - g.regex = r - return nil -} - -// ItemConfig is an (Analyzer,Group) configuration. -type ItemConfig struct { - // Exclude are analyzer exclusions. - // - // Exclude is a list of regular expressions. If the corresponding - // Analyzer emits a Finding for which Finding.Position.String() - // matches a regular expression in Exclude, the finding will not - // be reported. - Exclude []string `yaml:"exclude,omitempty"` - exclude []*regexp.Regexp `yaml:"-"` - - // Suppress are analyzer suppressions. - // - // Suppress is a list of regular expressions. If the corresponding - // Analyzer emits a Finding for which Finding.Message matches a regular - // expression in Suppress, the finding will not be reported. - Suppress []string `yaml:"suppress,omitempty"` - suppress []*regexp.Regexp `yaml:"-"` -} - -func compileRegexps(ss []string, rs *[]*regexp.Regexp) error { - *rs = make([]*regexp.Regexp, len(ss)) - for i, s := range ss { - r, err := regexp.Compile(s) - if err != nil { - return err - } - (*rs)[i] = r - } - return nil -} - -// RegexpCount is used by AnalyzerConfig.RegexpCount. -func (i *ItemConfig) RegexpCount() int64 { - if i == nil { - // See compile. - return 0 - } - // Return the number of regular expressions compiled for these items. - // This is how the cache size of the configuration is measured. - return int64(len(i.exclude) + len(i.suppress)) -} - -func (i *ItemConfig) compile() error { - if i == nil { - // This may be nil if nothing is included in the - // item configuration. That's fine, there's nothing - // to compile and nothing to exclude & suppress. - return nil - } - if err := compileRegexps(i.Exclude, &i.exclude); err != nil { - return fmt.Errorf("in exclude: %w", err) - } - if err := compileRegexps(i.Suppress, &i.suppress); err != nil { - return fmt.Errorf("in suppress: %w", err) - } - return nil -} - -func merge(a, b []string) []string { - found := make(map[string]struct{}) - result := make([]string, 0, len(a)+len(b)) - for _, elem := range a { - found[elem] = struct{}{} - result = append(result, elem) - } - for _, elem := range b { - if _, ok := found[elem]; ok { - continue - } - result = append(result, elem) - } - return result -} - -func (i *ItemConfig) merge(other *ItemConfig) { - i.Exclude = merge(i.Exclude, other.Exclude) - i.Suppress = merge(i.Suppress, other.Suppress) -} - -func (i *ItemConfig) shouldReport(fullPos, msg string) bool { - if i == nil { - // See above. - return true - } - for _, r := range i.exclude { - if r.MatchString(fullPos) { - return false - } - } - for _, r := range i.suppress { - if r.MatchString(msg) { - return false - } - } - return true -} - -// AnalyzerConfig is the configuration for a single analyzers. -// -// This map is keyed by individual Group names, to allow for different -// configurations depending on what Group the file belongs to. -type AnalyzerConfig map[GroupName]*ItemConfig - -// RegexpCount is used by Config.Size. -func (a AnalyzerConfig) RegexpCount() int64 { - count := int64(0) - for _, gc := range a { - count += gc.RegexpCount() - } - return count -} - -func (a AnalyzerConfig) compile() error { - for name, gc := range a { - if err := gc.compile(); err != nil { - return fmt.Errorf("invalid group %q: %v", name, err) - } - } - return nil -} - -func (a AnalyzerConfig) merge(other AnalyzerConfig) { - // Merge all the groups. - for name, gc := range other { - old, ok := a[name] - if !ok || old == nil { - a[name] = gc // Not configured in a. - continue - } - old.merge(gc) - } -} - -func (a AnalyzerConfig) shouldReport(groupConfig *Group, fullPos, msg string) bool { - gc, ok := a[groupConfig.Name] - if !ok { - return groupConfig.Default - } - - // Note that if a section appears for a particular group - // for a particular analyzer, then it will now be enabled, - // and the group default no longer applies. - return gc.shouldReport(fullPos, msg) -} - -// Config is a nogo configuration. -type Config struct { - // Prefixes defines a set of regular expressions that - // are standard "prefixes", so that files can be grouped - // and specific rules applied to individual groups. - Groups []Group `yaml:"groups"` - - // Global is the global analyzer config. - Global AnalyzerConfig `yaml:"global"` - - // Analyzers are individual analyzer configurations. The - // key for each analyzer is the name of the analyzer. The - // value is either a boolean (enable/disable), or a map to - // the groups above. - Analyzers map[AnalyzerName]AnalyzerConfig `yaml:"analyzers"` -} - -// Size implements worker.Sizer.Size. -func (c *Config) Size() int64 { - count := c.Global.RegexpCount() - for _, config := range c.Analyzers { - count += config.RegexpCount() - } - // The size is measured as the number of regexps that are compiled - // here. We multiply by 1k to produce an estimate. - return 1024 * count -} - -// Merge merges two configurations. -func (c *Config) Merge(other *Config) { - // Merge all groups. - // - // Select the other first, as the order provided in the second will - // provide precendence over the same group defined in the first one. - seenGroups := make(map[GroupName]struct{}) - newGroups := make([]Group, 0, len(c.Groups)+len(other.Groups)) - for _, g := range other.Groups { - newGroups = append(newGroups, g) - seenGroups[g.Name] = struct{}{} - } - for _, g := range c.Groups { - if _, ok := seenGroups[g.Name]; ok { - continue - } - newGroups = append(newGroups, g) - } - c.Groups = newGroups - - // Merge global configurations. - c.Global.merge(other.Global) - - // Merge all analyzer configurations. - for name, ac := range other.Analyzers { - old, ok := c.Analyzers[name] - if !ok { - c.Analyzers[name] = ac // No analyzer in original config. - continue - } - old.merge(ac) - } -} - -// Compile compiles a configuration to make it useable. -func (c *Config) Compile() error { - for i := 0; i < len(c.Groups); i++ { - if err := c.Groups[i].compile(); err != nil { - return fmt.Errorf("invalid group %q: %w", c.Groups[i].Name, err) - } - } - if err := c.Global.compile(); err != nil { - return fmt.Errorf("invalid global: %w", err) - } - for name, ac := range c.Analyzers { - if err := ac.compile(); err != nil { - return fmt.Errorf("invalid analyzer %q: %w", name, err) - } - } - return nil -} - -// ShouldReport returns true iff the finding should match the Config. -func (c *Config) ShouldReport(finding Finding) bool { - fullPos := finding.Position.String() - - // Find the matching group. - var groupConfig *Group - for i := 0; i < len(c.Groups); i++ { - if c.Groups[i].regex.MatchString(fullPos) { - groupConfig = &c.Groups[i] - break - } - } - - // If there is no group matching this path, then - // we default to accept the finding. - if groupConfig == nil { - return true - } - - // Suppress via global rule? - if !c.Global.shouldReport(groupConfig, fullPos, finding.Message) { - return false - } - - // Try the analyzer config. - ac, ok := c.Analyzers[finding.Category] - if !ok { - return groupConfig.Default - } - return ac.shouldReport(groupConfig, fullPos, finding.Message) -} diff --git a/tools/nogo/defs.bzl b/tools/nogo/defs.bzl deleted file mode 100644 index ddf5816a6..000000000 --- a/tools/nogo/defs.bzl +++ /dev/null @@ -1,488 +0,0 @@ -"""Nogo rules.""" - -load("//tools/bazeldefs:go.bzl", "go_context", "go_embed_libraries", "go_importpath", "go_rule") - -NogoConfigInfo = provider( - "information about a nogo configuration", - fields = { - "srcs": "the collection of configuration files", - }, -) - -def _nogo_config_impl(ctx): - return [NogoConfigInfo( - srcs = ctx.files.srcs, - )] - -nogo_config = rule( - implementation = _nogo_config_impl, - attrs = { - "srcs": attr.label_list( - doc = "a list of yaml files (schema defined by tool/nogo/config.go).", - allow_files = True, - ), - }, -) - -NogoTargetInfo = provider( - "information about the Go target", - fields = { - "goarch": "the build architecture (GOARCH)", - "goos": "the build OS target (GOOS)", - "worker_debug": "transitive debugging", - }, -) - -def _nogo_target_impl(ctx): - return [NogoTargetInfo( - goarch = ctx.attr.goarch, - goos = ctx.attr.goos, - worker_debug = ctx.attr.worker_debug, - )] - -nogo_target = go_rule( - rule, - implementation = _nogo_target_impl, - attrs = { - "goarch": attr.string( - doc = "the Go build architecture (propagated to other rules).", - mandatory = True, - ), - "goos": attr.string( - doc = "the Go OS target (propagated to other rules).", - mandatory = True, - ), - "worker_debug": attr.bool( - doc = "whether worker debugging should be enabled.", - default = False, - ), - }, -) - -def _nogo_objdump_tool_impl(ctx): - # Construct the magic dump command. - # - # Note that in some cases, the input is being fed into the tool via stdin. - # Unfortunately, the Go objdump tool expects to see a seekable file [1], so - # we need the tool to handle this case by creating a temporary file. - # - # [1] https://github.com/golang/go/issues/41051 - nogo_target_info = ctx.attr._target[NogoTargetInfo] - go_ctx = go_context(ctx, goos = nogo_target_info.goos, goarch = nogo_target_info.goarch) - env_prefix = " ".join(["%s=%s" % (key, value) for (key, value) in go_ctx.env.items()]) - dumper = ctx.actions.declare_file(ctx.label.name) - ctx.actions.write(dumper, "\n".join([ - "#!/bin/bash", - "set -euo pipefail", - "if [[ $# -eq 0 ]]; then", - " T=$(mktemp -u -t libXXXXXX.a)", - " cat /dev/stdin > ${T}", - "else", - " T=$1;", - "fi", - "%s %s tool objdump ${T}" % ( - env_prefix, - go_ctx.go.path, - ), - "if [[ $# -eq 0 ]]; then", - " rm -rf ${T}", - "fi", - "", - ]), is_executable = True) - - # Include the full runfiles. - return [DefaultInfo( - runfiles = ctx.runfiles(files = go_ctx.runfiles.to_list()), - executable = dumper, - )] - -nogo_objdump_tool = go_rule( - rule, - implementation = _nogo_objdump_tool_impl, - attrs = { - "_target": attr.label( - default = "//tools/nogo:target", - cfg = "target", - ), - }, -) - -# NogoStdlibInfo is the set of standard library facts. -NogoStdlibInfo = provider( - "information for nogo analysis (standard library facts)", - fields = { - "facts": "serialized standard library facts", - "raw_findings": "raw package findings (if relevant)", - }, -) - -def _nogo_stdlib_impl(ctx): - # Build the standard library facts. - nogo_target_info = ctx.attr._target[NogoTargetInfo] - go_ctx = go_context(ctx, goos = nogo_target_info.goos, goarch = nogo_target_info.goarch) - facts = ctx.actions.declare_file(ctx.label.name + ".facts") - raw_findings = ctx.actions.declare_file(ctx.label.name + ".raw_findings") - config = struct( - Srcs = [f.path for f in go_ctx.stdlib_srcs], - GOOS = go_ctx.goos, - GOARCH = go_ctx.goarch, - Tags = go_ctx.gotags, - ) - config_file = ctx.actions.declare_file(ctx.label.name + ".cfg") - ctx.actions.write(config_file, config.to_json()) - args_file = ctx.actions.declare_file(ctx.label.name + "_args_file") - ctx.actions.write( - output = args_file, - content = "\n".join(go_ctx.nogo_args + [ - "-objdump_tool=%s" % ctx.files._objdump_tool[0].path, - "-stdlib=%s" % config_file.path, - "-findings=%s" % raw_findings.path, - "-facts=%s" % facts.path, - ]), - ) - ctx.actions.run( - inputs = [config_file] + go_ctx.stdlib_srcs + [args_file], - outputs = [facts, raw_findings], - tools = depset(go_ctx.runfiles.to_list() + ctx.files._objdump_tool), - executable = ctx.files._check[0], - mnemonic = "GoStandardLibraryAnalysis", - # Note that this does not support work execution currently. There is an - # issue with stdout pollution that is not yet resolved, so this is kept - # as a separate menomic. - progress_message = "Analyzing Go Standard Library", - arguments = [ - "--worker_debug=%s" % nogo_target_info.worker_debug, - "@%s" % args_file.path, - ], - ) - - # Return the stdlib facts as output. - return [NogoStdlibInfo( - facts = facts, - raw_findings = raw_findings, - )] - -nogo_stdlib = go_rule( - rule, - implementation = _nogo_stdlib_impl, - attrs = { - "_check": attr.label( - default = "//tools/nogo/check:check", - cfg = "host", - ), - "_objdump_tool": attr.label( - default = "//tools/nogo:objdump_tool", - cfg = "host", - ), - "_target": attr.label( - default = "//tools/nogo:target", - cfg = "target", - ), - }, -) - -# NogoInfo is the serialized set of package facts for a nogo analysis. -# -# Each go_library rule will generate a corresponding nogo rule, which will run -# with the source files as input. Note however, that the individual nogo rules -# are simply stubs that enter into the shadow dependency tree (the "aspect"). -NogoInfo = provider( - "information for nogo analysis", - fields = { - "facts": "serialized package facts", - "raw_findings": "raw package findings (if relevant)", - "importpath": "package import path", - "binaries": "package binary files", - "srcs": "srcs (for go_test support)", - "deps": "deps (for go_test support)", - }, -) - -def _nogo_aspect_impl(target, ctx): - # If this is a nogo rule itself (and not the shadow of a go_library or - # go_binary rule created by such a rule), then we simply return nothing. - # All work is done in the shadow properties for go rules. For a proto - # library, we simply skip the analysis portion but still need to return a - # valid NogoInfo to reference the generated binary. - # - # Note that we almost exclusively use go_library, not go_tool_library. - # This is because nogo is manually annotated, so the go_tool_library kind - # is not needed to avoid dependency loops. Unfortunately, bazel coverdata - # is exported *only* as a go_tool_library. This does not cause a problem, - # since there is guaranteed to be no conflict. However for consistency, - # we should not introduce new go_tool_library dependencies unless strictly - # necessary. - if ctx.rule.kind in ("go_library", "go_tool_library", "go_binary", "go_test"): - srcs = ctx.rule.files.srcs - deps = ctx.rule.attr.deps - elif ctx.rule.kind in ("go_proto_library", "go_wrap_cc"): - srcs = [] - deps = ctx.rule.attr.deps - else: - return [NogoInfo()] - - # If we're using the "library" attribute, then we need to aggregate the - # original library sources and dependencies into this target to perform - # proper type analysis. - for embed in go_embed_libraries(ctx.rule): - info = embed[NogoInfo] - if hasattr(info, "srcs"): - srcs = srcs + info.srcs - if hasattr(info, "deps"): - deps = deps + info.deps - - # Start with all target files and srcs as input. - inputs = target.files.to_list() + srcs - - # Generate a shell script that dumps the binary. Annoyingly, this seems - # necessary as the context in which a run_shell command runs does not seem - # to cleanly allow us redirect stdout to the actual output file. Perhaps - # I'm missing something here, but the intermediate script does work. - binaries = target.files.to_list() - objfiles = [f for f in binaries if f.path.endswith(".a")] - if len(objfiles) > 0: - # Prefer the .a files for go_library targets. - target_objfile = objfiles[0] - else: - # Use the raw binary for go_binary and go_test targets. - target_objfile = binaries[0] - inputs.append(target_objfile) - - # Extract the importpath for this package. - if ctx.rule.kind == "go_test": - # If this is a test, then it will not be imported by anything else. - # We can safely set the importapth to just "test". Note that this - # is necessary if the library also imports the core library (in - # addition to including the sources directly), which happens in - # some complex cases (seccomp_victim). - importpath = "test" - else: - importpath = go_importpath(target) - - # Collect all info from shadow dependencies. - fact_map = dict() - import_map = dict() - all_raw_findings = [] - for dep in deps: - # There will be no file attribute set for all transitive dependencies - # that are not go_library or go_binary rules, such as a proto rules. - # This is handled by the ctx.rule.kind check above. - info = dep[NogoInfo] - if not hasattr(info, "facts"): - continue - - # Configure where to find the binary & fact files. Note that this will - # use .x and .a regardless of whether this is a go_binary rule, since - # these dependencies must be go_library rules. - x_files = [f.path for f in info.binaries if f.path.endswith(".x")] - if not len(x_files): - x_files = [f.path for f in info.binaries if f.path.endswith(".a")] - import_map[info.importpath] = x_files[0] - fact_map[info.importpath] = info.facts.path - - # Collect all findings; duplicates are resolved at the end. - all_raw_findings.extend(info.raw_findings) - - # Ensure the above are available as inputs. - inputs.append(info.facts) - inputs += info.binaries - - # Add the standard library facts. - stdlib_info = ctx.attr._nogo_stdlib[NogoStdlibInfo] - stdlib_facts = stdlib_info.facts - inputs.append(stdlib_facts) - - # The nogo tool operates on a configuration serialized in JSON format. - nogo_target_info = ctx.attr._target[NogoTargetInfo] - go_ctx = go_context(ctx, goos = nogo_target_info.goos, goarch = nogo_target_info.goarch) - facts = ctx.actions.declare_file(target.label.name + ".facts") - raw_findings = ctx.actions.declare_file(target.label.name + ".raw_findings") - config = struct( - ImportPath = importpath, - GoFiles = [src.path for src in srcs if src.path.endswith(".go")], - NonGoFiles = [src.path for src in srcs if not src.path.endswith(".go")], - GOOS = go_ctx.goos, - GOARCH = go_ctx.goarch, - Tags = go_ctx.gotags, - FactMap = fact_map, - ImportMap = import_map, - StdlibFacts = stdlib_facts.path, - ) - config_file = ctx.actions.declare_file(target.label.name + ".cfg") - ctx.actions.write(config_file, config.to_json()) - inputs.append(config_file) - args_file = ctx.actions.declare_file(ctx.label.name + "_args_file") - ctx.actions.write( - output = args_file, - content = "\n".join(go_ctx.nogo_args + [ - "-binary=%s" % target_objfile.path, - "-objdump_tool=%s" % ctx.files._objdump_tool[0].path, - "-package=%s" % config_file.path, - "-findings=%s" % raw_findings.path, - "-facts=%s" % facts.path, - ]), - ) - ctx.actions.run( - inputs = inputs + [args_file], - outputs = [facts, raw_findings], - tools = depset(go_ctx.runfiles.to_list() + ctx.files._objdump_tool), - executable = ctx.files._check[0], - mnemonic = "GoStaticAnalysis", - progress_message = "Analyzing %s" % target.label, - execution_requirements = {"supports-workers": "1"}, - arguments = [ - "--worker_debug=%s" % nogo_target_info.worker_debug, - "@%s" % args_file.path, - ], - ) - - # Flatten all findings from all dependencies. - # - # This is done because all the filtering must be done at the - # top-level nogo_test to dynamically apply a configuration. - # This does not actually add any additional work here, but - # will simply propagate the full list of files. - all_raw_findings = [stdlib_info.raw_findings] + depset(all_raw_findings).to_list() + [raw_findings] - - # Return the package facts as output. - return [ - NogoInfo( - facts = facts, - raw_findings = all_raw_findings, - importpath = importpath, - binaries = binaries, - srcs = srcs, - deps = deps, - ), - ] - -nogo_aspect = go_rule( - aspect, - implementation = _nogo_aspect_impl, - attr_aspects = [ - "deps", - "library", - "embed", - ], - attrs = { - "_check": attr.label( - default = "//tools/nogo/check:check", - cfg = "host", - ), - "_objdump_tool": attr.label( - default = "//tools/nogo:objdump_tool", - cfg = "host", - ), - "_target": attr.label( - default = "//tools/nogo:target", - cfg = "target", - ), - # The name of this attribute must not be _stdlib, since that - # appears to be reserved for some internal bazel use. - "_nogo_stdlib": attr.label( - default = "//tools/nogo:stdlib", - cfg = "host", - ), - }, -) - -def _nogo_test_impl(ctx): - """Check nogo findings.""" - nogo_target_info = ctx.attr._target[NogoTargetInfo] - - # Ensure there's a single dependency. - if len(ctx.attr.deps) != 1: - fail("nogo_test requires exactly one dep.") - raw_findings = ctx.attr.deps[0][NogoInfo].raw_findings - - # Build a step that applies the configuration. - config_srcs = ctx.attr.config[NogoConfigInfo].srcs - findings = ctx.actions.declare_file(ctx.label.name + ".findings") - args_file = ctx.actions.declare_file(ctx.label.name + "_args_file") - ctx.actions.write( - output = args_file, - content = "\n".join( - ["-input=%s" % f.path for f in raw_findings] + - ["-config=%s" % f.path for f in config_srcs] + - ["-output=%s" % findings.path], - ), - ) - ctx.actions.run( - inputs = raw_findings + ctx.files.srcs + config_srcs + [args_file], - outputs = [findings], - tools = depset(ctx.files._filter), - executable = ctx.files._filter[0], - mnemonic = "GoStaticAnalysis", - progress_message = "Generating %s" % ctx.label, - execution_requirements = {"supports-workers": "1"}, - arguments = [ - "--worker_debug=%s" % nogo_target_info.worker_debug, - "@%s" % args_file.path, - ], - ) - - # Build a runner that checks the filtered facts. - # - # Note that this calls the filter binary without any configuration, so all - # findings will be included. But this is expected, since we've already - # filtered out everything that should not be included. - runner = ctx.actions.declare_file(ctx.label.name) - runner_content = [ - "#!/bin/bash", - "exec %s -check -input=%s" % (ctx.files._filter[0].short_path, findings.short_path), - "", - ] - ctx.actions.write(runner, "\n".join(runner_content), is_executable = True) - - return [DefaultInfo( - # The runner just executes the filter again, on the - # newly generated filtered findings. We still need - # the filter tool as part of our runfiles, however. - runfiles = ctx.runfiles(files = ctx.files._filter + [findings]), - executable = runner, - ), OutputGroupInfo( - # Propagate the filtered filters, for consumption by - # build tooling. Note that the build tooling typically - # pays attention to the mnemoic above, so this must be - # what is expected by the tooling. - nogo_findings = depset([findings]), - )] - -nogo_test = rule( - implementation = _nogo_test_impl, - attrs = { - "config": attr.label( - mandatory = True, - doc = "A rule of kind nogo_config.", - ), - "deps": attr.label_list( - aspects = [nogo_aspect], - doc = "Exactly one Go dependency to be analyzed.", - ), - "srcs": attr.label_list( - allow_files = True, - doc = "Relevant src files. This is ignored except to make the nogo_test directly affected by the files.", - ), - "_target": attr.label( - default = "//tools/nogo:target", - cfg = "target", - ), - "_filter": attr.label(default = "//tools/nogo/filter:filter"), - }, - test = True, -) - -def _nogo_aspect_tricorder_impl(target, ctx): - if ctx.rule.kind != "nogo_test" or OutputGroupInfo not in target: - return [] - if not hasattr(target[OutputGroupInfo], "nogo_findings"): - return [] - return [ - OutputGroupInfo(tricorder = target[OutputGroupInfo].nogo_findings), - ] - -# Trivial aspect that forwards the findings from a nogo_test rule to -# go/tricorder, which reads from the `tricorder` output group. -nogo_aspect_tricorder = aspect( - implementation = _nogo_aspect_tricorder_impl, -) diff --git a/tools/nogo/filter/BUILD b/tools/nogo/filter/BUILD deleted file mode 100644 index e3049521e..000000000 --- a/tools/nogo/filter/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -load("//tools:defs.bzl", "go_binary") - -package(licenses = ["notice"]) - -go_binary( - name = "filter", - srcs = ["main.go"], - nogo = False, - visibility = ["//visibility:public"], - deps = [ - "//tools/nogo", - "//tools/worker", - "@in_gopkg_yaml_v2//:go_default_library", - ], -) diff --git a/tools/nogo/filter/main.go b/tools/nogo/filter/main.go deleted file mode 100644 index d50336b9b..000000000 --- a/tools/nogo/filter/main.go +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Binary check is the nogo entrypoint. -package main - -import ( - "bytes" - "flag" - "fmt" - "io/ioutil" - "log" - "os" - "strings" - - yaml "gopkg.in/yaml.v2" - "gvisor.dev/gvisor/tools/nogo" - "gvisor.dev/gvisor/tools/worker" -) - -type stringList []string - -func (s *stringList) String() string { - return strings.Join(*s, ",") -} - -func (s *stringList) Set(value string) error { - *s = append(*s, value) - return nil -} - -var ( - inputFiles stringList - configFiles stringList - outputFile string - showConfig bool - check bool -) - -func init() { - flag.Var(&inputFiles, "input", "findings input files (gob format)") - flag.StringVar(&outputFile, "output", "", "findings output file (json format)") - flag.Var(&configFiles, "config", "findings configuration files") - flag.BoolVar(&showConfig, "show-config", false, "dump configuration only") - flag.BoolVar(&check, "check", false, "assume input is in json format") -} - -func main() { - worker.Work(run) -} - -var ( - cachedFindings = worker.NewCache("findings") // With nogo.FindingSet. - cachedFiltered = worker.NewCache("filtered") // With nogo.FindingSet. - cachedConfigs = worker.NewCache("configs") // With nogo.Config. - cachedFullConfigs = worker.NewCache("compiled") // With nogo.Config. -) - -func loadFindings(filename string) nogo.FindingSet { - return cachedFindings.Lookup([]string{filename}, func() worker.Sizer { - r, err := os.Open(filename) - if err != nil { - log.Fatalf("unable to open input %q: %v", filename, err) - } - inputFindings, err := nogo.ExtractFindingsFrom(r, check /* json */) - if err != nil { - log.Fatalf("unable to extract findings from %s: %v", filename, err) - } - return inputFindings - }).(nogo.FindingSet) -} - -func loadConfig(filename string) *nogo.Config { - return cachedConfigs.Lookup([]string{filename}, func() worker.Sizer { - content, err := ioutil.ReadFile(filename) - if err != nil { - log.Fatalf("unable to read %s: %v", filename, err) - } - var newConfig nogo.Config // For current file. - dec := yaml.NewDecoder(bytes.NewBuffer(content)) - dec.SetStrict(true) - if err := dec.Decode(&newConfig); err != nil { - log.Fatalf("unable to decode %s: %v", filename, err) - } - if showConfig { - content, err := yaml.Marshal(&newConfig) - if err != nil { - log.Fatalf("error marshalling config: %v", err) - } - fmt.Fprintf(os.Stdout, "Loaded configuration from %s:\n%s\n", filename, string(content)) - } - return &newConfig - }).(*nogo.Config) -} - -func loadConfigs(filenames []string) *nogo.Config { - return cachedFullConfigs.Lookup(filenames, func() worker.Sizer { - config := &nogo.Config{ - Global: make(nogo.AnalyzerConfig), - Analyzers: make(map[nogo.AnalyzerName]nogo.AnalyzerConfig), - } - for _, filename := range configFiles { - config.Merge(loadConfig(filename)) - if showConfig { - mergedBytes, err := yaml.Marshal(config) - if err != nil { - log.Fatalf("error marshalling config: %v", err) - } - fmt.Fprintf(os.Stdout, "Merged configuration:\n%s\n", string(mergedBytes)) - } - } - if err := config.Compile(); err != nil { - log.Fatalf("error compiling config: %v", err) - } - return config - }).(*nogo.Config) -} - -func run([]string) int { - // Open and merge all configuations. - config := loadConfigs(configFiles) - if showConfig { - return 0 - } - - // Load and filer available findings. - var filteredFindings []nogo.Finding - for _, filename := range inputFiles { - // Note that this applies a caching strategy to the filtered - // findings, because *this is by far the most expensive part of - // evaluation*. The set of findings is large and applying the - // configuration is complex. Therefore, we segment this cache - // on each individual raw findings input file and the - // configuration files. Note that this cache is keyed on all - // the configuration files and each individual raw findings, so - // is guaranteed to be safe. This allows us to reuse the same - // filter result many times over, because e.g. all standard - // library findings will be available to all packages. - filteredFindings = append(filteredFindings, - cachedFiltered.Lookup(append(configFiles, filename), func() worker.Sizer { - inputFindings := loadFindings(filename) - filteredFindings := make(nogo.FindingSet, 0, len(inputFindings)) - for _, finding := range inputFindings { - if ok := config.ShouldReport(finding); ok { - filteredFindings = append(filteredFindings, finding) - } - } - return filteredFindings - }).(nogo.FindingSet)...) - } - - // Write the output (if required). - // - // If the outputFile is specified, then we exit here. Otherwise, - // we continue to write to stdout and treat like a test. - // - // Note that the output of the filter is always json, which is - // human readable and the format that is consumed by tricorder. - if outputFile != "" { - w, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - log.Fatalf("unable to open output file %q: %v", outputFile, err) - } - if err := nogo.WriteFindingsTo(w, filteredFindings, true /* json */); err != nil { - log.Fatalf("unable to write findings: %v", err) - } - return 0 - } - - // Treat the run as a test. - if len(filteredFindings) == 0 { - fmt.Fprintf(os.Stdout, "PASS\n") - return 0 - } - for _, finding := range filteredFindings { - fmt.Fprintf(os.Stdout, "%s\n", finding.String()) - } - return 1 -} diff --git a/tools/nogo/findings.go b/tools/nogo/findings.go deleted file mode 100644 index 329a7062e..000000000 --- a/tools/nogo/findings.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package nogo - -import ( - "encoding/gob" - "encoding/json" - "fmt" - "go/token" - "io" - "os" - "reflect" - "sort" -) - -// Finding is a single finding. -type Finding struct { - Category AnalyzerName - Position token.Position - Message string -} - -// findingSize is the size of the finding struct itself. -var findingSize = int64(reflect.TypeOf(Finding{}).Size()) - -// Size implements worker.Sizer.Size. -func (f *Finding) Size() int64 { - return int64(len(f.Category)) + int64(len(f.Message)) + findingSize -} - -// String implements fmt.Stringer.String. -func (f *Finding) String() string { - return fmt.Sprintf("%s: %s: %s", f.Category, f.Position.String(), f.Message) -} - -// FindingSet is a collection of findings. -type FindingSet []Finding - -// Size implmements worker.Sizer.Size. -func (fs FindingSet) Size() int64 { - size := int64(0) - for _, finding := range fs { - size += finding.Size() - } - return size -} - -// Sort sorts all findings. -func (fs FindingSet) Sort() { - sort.Slice(fs, func(i, j int) bool { - switch { - case fs[i].Position.Filename < fs[j].Position.Filename: - return true - case fs[i].Position.Filename > fs[j].Position.Filename: - return false - case fs[i].Position.Line < fs[j].Position.Line: - return true - case fs[i].Position.Line > fs[j].Position.Line: - return false - case fs[i].Position.Column < fs[j].Position.Column: - return true - case fs[i].Position.Column > fs[j].Position.Column: - return false - case fs[i].Category < fs[j].Category: - return true - case fs[i].Category > fs[j].Category: - return false - case fs[i].Message < fs[j].Message: - return true - case fs[i].Message > fs[j].Message: - return false - default: - return false - } - }) -} - -// WriteFindingsTo serializes findings. -func WriteFindingsTo(w io.Writer, findings FindingSet, asJSON bool) error { - // N.B. Sort all the findings in order to maximize cacheability. - findings.Sort() - if asJSON { - enc := json.NewEncoder(w) - return enc.Encode(findings) - } - enc := gob.NewEncoder(w) - return enc.Encode(findings) -} - -// ExtractFindingsFromFile loads findings from a file. -func ExtractFindingsFromFile(filename string, asJSON bool) (FindingSet, error) { - r, err := os.Open(filename) - if err != nil { - return nil, err - } - defer r.Close() - return ExtractFindingsFrom(r, asJSON) -} - -// ExtractFindingsFromBytes loads findings from bytes. -func ExtractFindingsFrom(r io.Reader, asJSON bool) (findings FindingSet, err error) { - if asJSON { - dec := json.NewDecoder(r) - err = dec.Decode(&findings) - } else { - dec := gob.NewDecoder(r) - err = dec.Decode(&findings) - } - return findings, err -} - -func init() { - gob.Register((*Finding)(nil)) - gob.Register((*FindingSet)(nil)) -} diff --git a/tools/nogo/nogo.go b/tools/nogo/nogo.go deleted file mode 100644 index acee7c8bc..000000000 --- a/tools/nogo/nogo.go +++ /dev/null @@ -1,537 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package nogo implements binary analysis similar to bazel's nogo, -// or the unitchecker package. It exists in order to provide additional -// facilities for analysis, namely plumbing through the output from -// dumping the generated binary (to analyze actual produced code). -package nogo - -import ( - "bytes" - "encoding/gob" - "errors" - "fmt" - "go/ast" - "go/build" - "go/parser" - "go/token" - "go/types" - "io" - "io/ioutil" - "log" - "os" - "path" - "path/filepath" - "reflect" - "sort" - "strings" - - "golang.org/x/tools/go/analysis" - "golang.org/x/tools/go/analysis/internal/facts" - "golang.org/x/tools/go/gcexportdata" - - // Special case: flags live here and change overall behavior. - "gvisor.dev/gvisor/tools/checkescape" - "gvisor.dev/gvisor/tools/worker" -) - -// StdlibConfig is serialized as the configuration. -// -// This contains everything required for stdlib analysis. -type StdlibConfig struct { - Srcs []string - GOOS string - GOARCH string - Tags []string -} - -// PackageConfig is serialized as the configuration. -// -// This contains everything required for single package analysis. -type PackageConfig struct { - ImportPath string - GoFiles []string - NonGoFiles []string - Tags []string - GOOS string - GOARCH string - ImportMap map[string]string - FactMap map[string]string - StdlibFacts string -} - -// loader is a fact-loader function. -type loader func(string) ([]byte, error) - -// saver is a fact-saver function. -type saver func([]byte) error - -// stdlibFact is used for serialiation. -type stdlibFact struct { - Package string - Facts []byte -} - -// stdlibFacts is a set of standard library facts. -type stdlibFacts map[string][]byte - -// Size implements worker.Sizer.Size. -func (sf stdlibFacts) Size() int64 { - size := int64(0) - for filename, data := range sf { - size += int64(len(filename)) - size += int64(len(data)) - } - return size -} - -// EncodeTo serializes stdlibFacts. -func (sf stdlibFacts) EncodeTo(w io.Writer) error { - stdlibFactsSorted := make([]stdlibFact, 0, len(sf)) - for pkg, facts := range sf { - stdlibFactsSorted = append(stdlibFactsSorted, stdlibFact{ - Package: pkg, - Facts: facts, - }) - } - sort.Slice(stdlibFactsSorted, func(i, j int) bool { - return stdlibFactsSorted[i].Package < stdlibFactsSorted[j].Package - }) - enc := gob.NewEncoder(w) - if err := enc.Encode(stdlibFactsSorted); err != nil { - return err - } - return nil -} - -// DecodeFrom deserializes stdlibFacts. -func (sf stdlibFacts) DecodeFrom(r io.Reader) error { - var stdlibFactsSorted []stdlibFact - dec := gob.NewDecoder(r) - if err := dec.Decode(&stdlibFactsSorted); err != nil { - return err - } - for _, stdlibFact := range stdlibFactsSorted { - sf[stdlibFact.Package] = stdlibFact.Facts - } - return nil -} - -var ( - // cachedFacts caches by file (just byte data). - cachedFacts = worker.NewCache("facts") - - // stdlibCachedFacts caches the standard library (stdlibFacts). - stdlibCachedFacts = worker.NewCache("stdlib") -) - -// factLoader loads facts. -func (c *PackageConfig) factLoader(path string) (data []byte, err error) { - filename, ok := c.FactMap[path] - if ok { - cb := cachedFacts.Lookup([]string{filename}, func() worker.Sizer { - data, readErr := ioutil.ReadFile(filename) - if readErr != nil { - err = fmt.Errorf("error loading %q: %w", filename, readErr) - return nil - } - return worker.CacheBytes(data) - }) - if cb != nil { - return []byte(cb.(worker.CacheBytes)), err - } - return nil, err - } - cb := stdlibCachedFacts.Lookup([]string{c.StdlibFacts}, func() worker.Sizer { - r, openErr := os.Open(c.StdlibFacts) - if openErr != nil { - err = fmt.Errorf("error loading stdlib facts from %q: %w", c.StdlibFacts, openErr) - return nil - } - defer r.Close() - sf := make(stdlibFacts) - if readErr := sf.DecodeFrom(r); readErr != nil { - err = fmt.Errorf("error loading stdlib facts: %w", readErr) - return nil - } - return sf - }) - if cb != nil { - return (cb.(stdlibFacts))[path], err - } - return nil, err -} - -// shouldInclude indicates whether the file should be included. -// -// NOTE: This does only basic parsing of tags. -func (c *PackageConfig) shouldInclude(path string) (bool, error) { - ctx := build.Default - ctx.GOOS = c.GOOS - ctx.GOARCH = c.GOARCH - ctx.BuildTags = c.Tags - return ctx.MatchFile(filepath.Dir(path), filepath.Base(path)) -} - -// importer is an implementation of go/types.Importer. -// -// This wraps a configuration, which provides the map of package names to -// files, and the facts. Note that this importer implementation will always -// pass when a given package is not available. -type importer struct { - *PackageConfig - fset *token.FileSet - cache map[string]*types.Package - lastErr error - callback func(string) error -} - -// Import implements types.Importer.Import. -func (i *importer) Import(path string) (*types.Package, error) { - if path == "unsafe" { - // Special case: go/types has pre-defined type information for - // unsafe. We ensure that this package is correct, in case any - // analyzers are specifically looking for this. - return types.Unsafe, nil - } - - // Call the internal callback. This is used to resolve loading order - // for the standard library. See checkStdlib. - if i.callback != nil { - if err := i.callback(path); err != nil { - i.lastErr = err - return nil, err - } - } - - // Actually load the data. - realPath, ok := i.ImportMap[path] - var ( - rc io.ReadCloser - err error - ) - if !ok { - // Not found in the import path. Attempt to find the package - // via the standard library. - rc, err = findStdPkg(i.GOOS, i.GOARCH, path) - } else { - // Open the file. - rc, err = os.Open(realPath) - } - if err != nil { - i.lastErr = err - return nil, err - } - defer rc.Close() - - // Load all exported data. - r, err := gcexportdata.NewReader(rc) - if err != nil { - return nil, err - } - - return gcexportdata.Read(r, i.fset, i.cache, path) -} - -// ErrSkip indicates the package should be skipped. -var ErrSkip = errors.New("skipped") - -// CheckStdlib checks the standard library. -// -// This constructs a synthetic package configuration for each library in the -// standard library sources, and call CheckPackage repeatedly. -// -// Note that not all parts of the source are expected to build. We skip obvious -// test files, and cmd files, which should not be dependencies. -func CheckStdlib(config *StdlibConfig, analyzers []*analysis.Analyzer) (allFindings FindingSet, facts []byte, err error) { - if len(config.Srcs) == 0 { - return nil, nil, nil - } - - // Ensure all paths are normalized. - for i := 0; i < len(config.Srcs); i++ { - config.Srcs[i] = path.Clean(config.Srcs[i]) - } - - // Calculate the root source directory. This is always a directory - // named 'src', of which we simply take the first we find. This is a - // bit fragile, but works for all currently known Go source - // configurations. - // - // Note that there may be extra files outside of the root source - // directory; we simply ignore those. - rootSrcPrefix := "" - for _, file := range config.Srcs { - const src = "/src/" - i := strings.Index(file, src) - if i == -1 { - // Superfluous file. - continue - } - - // Index of first character after /src/. - i += len(src) - rootSrcPrefix = file[:i] - break - } - - // Aggregate all files by directory. - packages := make(map[string]*PackageConfig) - for _, file := range config.Srcs { - if !strings.HasPrefix(file, rootSrcPrefix) { - // Superflouous file. - continue - } - - d := path.Dir(file) - if len(rootSrcPrefix) >= len(d) { - continue // Not a file. - } - pkg := d[len(rootSrcPrefix):] - // Skip cmd packages and obvious test files: see above. - if strings.HasPrefix(pkg, "cmd/") || strings.HasSuffix(file, "_test.go") { - continue - } - c, ok := packages[pkg] - if !ok { - c = &PackageConfig{ - ImportPath: pkg, - GOOS: config.GOOS, - GOARCH: config.GOARCH, - Tags: config.Tags, - } - packages[pkg] = c - } - // Add the files appropriately. Note that they will be further - // filtered by architecture and build tags below, so this need - // not be done immediately. - if strings.HasSuffix(file, ".go") { - c.GoFiles = append(c.GoFiles, file) - } else { - c.NonGoFiles = append(c.NonGoFiles, file) - } - } - - // Closure to check a single package. - localStdlibFacts := make(stdlibFacts) - localStdlibErrs := make(map[string]error) - var checkOne func(pkg string) error // Recursive. - checkOne = func(pkg string) error { - // Is this already done? - if _, ok := localStdlibFacts[pkg]; ok { - return nil - } - // Did this fail previously? - if _, ok := localStdlibErrs[pkg]; ok { - return nil - } - - // Lookup the configuration. - config, ok := packages[pkg] - if !ok { - return nil // Not known. - } - - // Find the binary package, and provide to objdump. - rc, err := findStdPkg(config.GOOS, config.GOARCH, pkg) - if err != nil { - // If there's no binary for this package, it is likely - // not built with the distribution. That's fine, we can - // just skip analysis. - localStdlibErrs[pkg] = err - return nil - } - - // Provide the input. - oldReader := checkescape.Reader - checkescape.Reader = rc // For analysis. - defer func() { - rc.Close() - checkescape.Reader = oldReader // Restore. - }() - - // Run the analysis. - findings, factData, err := CheckPackage(config, analyzers, checkOne) - if err != nil { - // If we can't analyze a package from the standard library, - // then we skip it. It will simply not have any findings. - localStdlibErrs[pkg] = err - return nil - } - localStdlibFacts[pkg] = factData - allFindings = append(allFindings, findings...) - return nil - } - - // Check all packages. - // - // Note that this may call checkOne recursively, so it's not guaranteed - // to evaluate in the order provided here. We do ensure however, that - // all packages are evaluated. - for pkg := range packages { - if err := checkOne(pkg); err != nil { - return nil, nil, err - } - } - - // Sanity check. - if len(localStdlibFacts) == 0 { - return nil, nil, fmt.Errorf("no stdlib facts found: misconfiguration?") - } - - // Write out all findings. - buf := bytes.NewBuffer(nil) - if err := localStdlibFacts.EncodeTo(buf); err != nil { - return nil, nil, fmt.Errorf("error serialized stdlib facts: %v", err) - } - - // Write out all errors. - for pkg, err := range localStdlibErrs { - log.Printf("WARNING: error while processing %v: %v", pkg, err) - } - - // Return all findings. - return allFindings, buf.Bytes(), nil -} - -// CheckPackage runs all given analyzers. -// -// The implementation was adapted from [1], which was in turn adpated from [2]. -// This returns a list of matching analysis issues, or an error if the analysis -// could not be completed. -// -// [1] bazelbuid/rules_go/tools/builders/nogo_main.go -// [2] golang.org/x/tools/go/checker/internal/checker -func CheckPackage(config *PackageConfig, analyzers []*analysis.Analyzer, importCallback func(string) error) (findings []Finding, factData []byte, err error) { - imp := &importer{ - PackageConfig: config, - fset: token.NewFileSet(), - cache: make(map[string]*types.Package), - callback: importCallback, - } - - // Load all source files. - var syntax []*ast.File - for _, file := range config.GoFiles { - include, err := config.shouldInclude(file) - if err != nil { - return nil, nil, fmt.Errorf("error evaluating file %q: %v", file, err) - } - if !include { - continue - } - s, err := parser.ParseFile(imp.fset, file, nil, parser.ParseComments) - if err != nil { - return nil, nil, fmt.Errorf("error parsing file %q: %v", file, err) - } - syntax = append(syntax, s) - } - - // Check type information. - typesSizes := types.SizesFor("gc", config.GOARCH) - typeConfig := types.Config{Importer: imp} - typesInfo := &types.Info{ - Types: make(map[ast.Expr]types.TypeAndValue), - Uses: make(map[*ast.Ident]types.Object), - Defs: make(map[*ast.Ident]types.Object), - Implicits: make(map[ast.Node]types.Object), - Scopes: make(map[ast.Node]*types.Scope), - Selections: make(map[*ast.SelectorExpr]*types.Selection), - } - types, err := typeConfig.Check(config.ImportPath, imp.fset, syntax, typesInfo) - if err != nil && imp.lastErr != ErrSkip { - return nil, nil, fmt.Errorf("error checking types: %w", err) - } - - // Load all package facts. - facts, err := facts.Decode(types, config.factLoader) - if err != nil { - return nil, nil, fmt.Errorf("error decoding facts: %w", err) - } - - // Register fact types and establish dependencies between analyzers. - // The visit closure will execute recursively, and populate results - // will all required analysis results. - results := make(map[*analysis.Analyzer]interface{}) - var visit func(*analysis.Analyzer) error // For recursion. - visit = func(a *analysis.Analyzer) error { - if _, ok := results[a]; ok { - return nil - } - - // Run recursively for all dependencies. - for _, req := range a.Requires { - if err := visit(req); err != nil { - return err - } - } - - // Run the analysis. - factFilter := make(map[reflect.Type]bool) - for _, f := range a.FactTypes { - factFilter[reflect.TypeOf(f)] = true - } - p := &analysis.Pass{ - Analyzer: a, - Fset: imp.fset, - Files: syntax, - Pkg: types, - TypesInfo: typesInfo, - ResultOf: results, // All results. - Report: func(d analysis.Diagnostic) { - findings = append(findings, Finding{ - Category: AnalyzerName(a.Name), - Position: imp.fset.Position(d.Pos), - Message: d.Message, - }) - }, - ImportPackageFact: facts.ImportPackageFact, - ExportPackageFact: facts.ExportPackageFact, - ImportObjectFact: facts.ImportObjectFact, - ExportObjectFact: facts.ExportObjectFact, - AllPackageFacts: func() []analysis.PackageFact { return facts.AllPackageFacts(factFilter) }, - AllObjectFacts: func() []analysis.ObjectFact { return facts.AllObjectFacts(factFilter) }, - TypesSizes: typesSizes, - } - result, err := a.Run(p) - if err != nil { - return fmt.Errorf("error running analysis %s: %v", a, err) - } - - // Sanity check & save the result. - if got, want := reflect.TypeOf(result), a.ResultType; got != want { - return fmt.Errorf("error: analyzer %s returned a result of type %v, but declared ResultType %v", a, got, want) - } - results[a] = result - return nil // Success. - } - - // Visit all analyzers recursively. - for _, a := range analyzers { - if imp.lastErr == ErrSkip { - continue // No local analysis. - } - if err := visit(a); err != nil { - return nil, nil, err // Already has context. - } - } - - // Return all findings. - return findings, facts.Encode(), nil -} - -func init() { - gob.Register((*stdlibFact)(nil)) -} diff --git a/tools/parsers/BUILD b/tools/parsers/BUILD deleted file mode 100644 index 6932bba9a..000000000 --- a/tools/parsers/BUILD +++ /dev/null @@ -1,45 +0,0 @@ -load("//tools:defs.bzl", "go_binary", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_test( - name = "parsers_test", - size = "small", - srcs = ["go_parser_test.go"], - library = ":parsers", - nogo = False, - deps = [ - "//tools/bigquery", - "@com_github_google_go_cmp//cmp:go_default_library", - ], -) - -go_library( - name = "parsers", - testonly = 1, - srcs = [ - "go_parser.go", - ], - nogo = False, - visibility = ["//:sandbox"], - deps = [ - "//test/benchmarks/tools", - "//tools/bigquery", - ], -) - -go_binary( - name = "parser", - testonly = 1, - srcs = [ - "parser_main.go", - "version.go", - ], - nogo = False, - x_defs = {"main.version": "{STABLE_VERSION}"}, - deps = [ - ":parsers", - "//runsc/flag", - "//tools/bigquery", - ], -) diff --git a/tools/parsers/go_parser.go b/tools/parsers/go_parser.go deleted file mode 100644 index 57e538149..000000000 --- a/tools/parsers/go_parser.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package parsers holds parsers to parse Benchmark test output. -// -// Parsers parse Benchmark test output and place it in BigQuery -// structs for sending to BigQuery databases. -package parsers - -import ( - "fmt" - "strconv" - "strings" - - "gvisor.dev/gvisor/test/benchmarks/tools" - "gvisor.dev/gvisor/tools/bigquery" -) - -// ParseOutput expects golang benchmark output and returns a struct formatted -// for BigQuery. -func ParseOutput(output string, name string, official bool) (*bigquery.Suite, error) { - suite := bigquery.NewSuite(name, official) - lines := strings.Split(output, "\n") - for _, line := range lines { - bm, err := parseLine(line) - if err != nil { - return nil, fmt.Errorf("failed to parse line '%s': %v", line, err) - } - if bm != nil { - suite.Benchmarks = append(suite.Benchmarks, bm) - } - } - return suite, nil -} - -// parseLine handles parsing a benchmark line into a bigquery.Benchmark. -// -// Example: "BenchmarkRuby/server_threads.1-6 1 1397875880 ns/op 140 requests_per_second.QPS" -// -// This function will return the following benchmark: -// *bigquery.Benchmark{ -// Name: BenchmarkRuby -// []*bigquery.Condition{ -// {Name: GOMAXPROCS, 6} -// {Name: server_threads, 1} -// } -// []*bigquery.Metric{ -// {Name: ns/op, Unit: ns/op, Sample: 1397875880} -// {Name: requests_per_second, Unit: QPS, Sample: 140 } -// } -//} -func parseLine(line string) (*bigquery.Benchmark, error) { - fields := strings.Fields(line) - - // Check if this line is a Benchmark line. Otherwise ignore the line. - if len(fields) < 2 || !strings.HasPrefix(fields[0], "Benchmark") { - return nil, nil - } - - iters, err := strconv.Atoi(fields[1]) - if err != nil { - return nil, fmt.Errorf("expecting number of runs, got %s: %v", fields[1], err) - } - - name, params, err := parseNameParams(fields[0]) - if err != nil { - return nil, fmt.Errorf("parse name/params: %v", err) - } - - bm := bigquery.NewBenchmark(name, iters) - for _, p := range params { - bm.AddCondition(p.Name, p.Value) - } - - for i := 1; i < len(fields)/2; i++ { - value := fields[2*i] - metric := fields[2*i+1] - if err := makeMetric(bm, value, metric); err != nil { - return nil, fmt.Errorf("makeMetric on metric %q value: %s: %v", metric, value, err) - } - } - return bm, nil -} - -// parseNameParams parses the Name, GOMAXPROCS, and Params from the test. -// Field here should be of the format TESTNAME/PARAMS-GOMAXPROCS. -// Parameters will be separated by a "/" with individual params being -// "name.value". -func parseNameParams(field string) (string, []*tools.Parameter, error) { - var params []*tools.Parameter - // Remove GOMAXPROCS from end. - maxIndex := strings.LastIndex(field, "-") - if maxIndex < 0 { - return "", nil, fmt.Errorf("GOMAXPROCS not found: %s", field) - } - maxProcs := field[maxIndex+1:] - params = append(params, &tools.Parameter{ - Name: "GOMAXPROCS", - Value: maxProcs, - }) - - remainder := field[0:maxIndex] - index := strings.Index(remainder, "/") - if index == -1 { - return remainder, params, nil - } - - name := remainder[0:index] - p := remainder[index+1:] - - ps, err := tools.NameToParameters(p) - if err != nil { - return "", nil, fmt.Errorf("NameToParameters %s: %v", field, err) - } - params = append(params, ps...) - return name, params, nil -} - -// makeMetric parses metrics and adds them to the passed Benchmark. -func makeMetric(bm *bigquery.Benchmark, value, metric string) error { - switch metric { - // Ignore most output from golang benchmarks. - case "MB/s", "B/op", "allocs/op": - return nil - case "ns/op": - val, err := strconv.ParseFloat(value, 64) - if err != nil { - return fmt.Errorf("ParseFloat %s: %v", value, err) - } - bm.AddMetric(metric /*metric name*/, metric /*unit*/, val /*sample*/) - default: - m, err := tools.ParseCustomMetric(value, metric) - if err != nil { - return fmt.Errorf("ParseCustomMetric %s: %v ", metric, err) - } - bm.AddMetric(m.Name, m.Unit, m.Sample) - } - return nil -} diff --git a/tools/parsers/go_parser_test.go b/tools/parsers/go_parser_test.go deleted file mode 100644 index 39a13b4af..000000000 --- a/tools/parsers/go_parser_test.go +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package parsers - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - "gvisor.dev/gvisor/tools/bigquery" -) - -func TestParseLine(t *testing.T) { - testCases := []struct { - name string - data string - want *bigquery.Benchmark - }{ - { - name: "Iperf", - data: "BenchmarkIperf/Upload-6 1 11094914892 ns/op 4751711232 bandwidth.bytes_per_second", - want: &bigquery.Benchmark{ - Name: "BenchmarkIperf", - Condition: []*bigquery.Condition{ - { - Name: "iterations", - Value: "1", - }, - { - Name: "GOMAXPROCS", - Value: "6", - }, - { - Name: "Upload", - Value: "Upload", - }, - }, - Metric: []*bigquery.Metric{ - { - Name: "ns/op", - Unit: "ns/op", - Sample: 11094914892.0, - }, - { - Name: "bandwidth", - Unit: "bytes_per_second", - Sample: 4751711232.0, - }, - }, - }, - }, - { - name: "Ruby", - data: "BenchmarkRuby/server_threads.1-6 1 1397875880 ns/op 0.00710 average_latency.s 140 requests_per_second.QPS", - want: &bigquery.Benchmark{ - Name: "BenchmarkRuby", - Condition: []*bigquery.Condition{ - { - Name: "iterations", - Value: "1", - }, - { - Name: "GOMAXPROCS", - Value: "6", - }, - { - Name: "server_threads", - Value: "1", - }, - }, - Metric: []*bigquery.Metric{ - { - Name: "ns/op", - Unit: "ns/op", - Sample: 1397875880.0, - }, - { - Name: "average_latency", - Unit: "s", - Sample: 0.00710, - }, - { - Name: "requests_per_second", - Unit: "QPS", - Sample: 140.0, - }, - }, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - got, err := parseLine(tc.data) - if err != nil { - t.Fatalf("parseLine failed with: %v", err) - } - - if !cmp.Equal(tc.want, got, nil) { - for i := range got.Condition { - t.Logf("Metric: want: %+v got:%+v", got.Condition[i], tc.want.Condition[i]) - } - - for i := range got.Metric { - t.Logf("Metric: want: %+v got:%+v", got.Metric[i], tc.want.Metric[i]) - } - - t.Fatalf("Compare failed want: %+v got: %+v", tc.want, got) - } - }) - - } -} - -func TestParseOutput(t *testing.T) { - testCases := []struct { - name string - data string - numBenchmarks int - numMetrics int - numConditions int - }{ - { - name: "Startup", - data: ` - BenchmarkStartupEmpty - BenchmarkStartupEmpty-6 2 766377884 ns/op 1 allocs/op - BenchmarkStartupNode - BenchmarkStartupNode-6 1 1752158409 ns/op 1 allocs/op - `, - numBenchmarks: 2, - numMetrics: 1, - numConditions: 2, - }, - { - name: "Ruby", - data: `BenchmarkRuby -BenchmarkRuby/server_threads.1 -BenchmarkRuby/server_threads.1-6 1 1397875880 ns/op 0.00710 average_latency.s 140 requests_per_second.QPS -BenchmarkRuby/server_threads.5 -BenchmarkRuby/server_threads.5-6 1 1416003331 ns/op 0.00950 average_latency.s 465 requests_per_second.QPS`, - numBenchmarks: 2, - numMetrics: 3, - numConditions: 3, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - suite, err := ParseOutput(tc.data, "", false) - if err != nil { - t.Fatalf("parseOutput failed: %v", err) - } else if len(suite.Benchmarks) != tc.numBenchmarks { - t.Fatalf("NumBenchmarks failed want: %d got: %d %+v", tc.numBenchmarks, len(suite.Benchmarks), suite.Benchmarks) - } - - for _, bm := range suite.Benchmarks { - if len(bm.Metric) != tc.numMetrics { - t.Fatalf("NumMetrics failed want: %d got: %d %+v", tc.numMetrics, len(bm.Metric), bm.Metric) - } - - if len(bm.Condition) != tc.numConditions { - t.Fatalf("NumConditions failed want: %d got: %d %+v", tc.numConditions, len(bm.Condition), bm.Condition) - } - } - }) - } -} diff --git a/tools/parsers/parser_main.go b/tools/parsers/parser_main.go deleted file mode 100644 index 01396494a..000000000 --- a/tools/parsers/parser_main.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Binary parser parses Benchmark data from golang benchmarks, -// puts it into a Schema for BigQuery, and sends it to BigQuery. -// parser will also initialize a table with the Benchmarks BigQuery schema. -package main - -import ( - "context" - "fmt" - "io/ioutil" - "log" - "os" - - "gvisor.dev/gvisor/runsc/flag" - bq "gvisor.dev/gvisor/tools/bigquery" - "gvisor.dev/gvisor/tools/parsers" -) - -const ( - initString = "init" - initDescription = "initializes a new table with benchmarks schema" - parseString = "parse" - parseDescription = "parses given benchmarks file and sends it to BigQuery table." -) - -var ( - // The init command will create a new dataset/table in the given project and initialize - // the table with the schema in //tools/bigquery/bigquery.go. If the table/dataset exists - // or has been initialized, init has no effect and successfully returns. - initCmd = flag.NewFlagSet(initString, flag.ContinueOnError) - initProject = initCmd.String("project", "", "GCP project to send benchmarks.") - initDataset = initCmd.String("dataset", "", "dataset to send benchmarks data.") - initTable = initCmd.String("table", "", "table to send benchmarks data.") - - // The parse command parses benchmark data in `file` and sends it to the - // requested table. - parseCmd = flag.NewFlagSet(parseString, flag.ContinueOnError) - file = parseCmd.String("file", "", "file to parse for benchmarks") - name = parseCmd.String("suite_name", "", "name of the benchmark suite") - parseProject = parseCmd.String("project", "", "GCP project to send benchmarks.") - parseDataset = parseCmd.String("dataset", "", "dataset to send benchmarks data.") - parseTable = parseCmd.String("table", "", "table to send benchmarks data.") - official = parseCmd.Bool("official", false, "mark input data as official.") - runtime = parseCmd.String("runtime", "", "runtime used to run the benchmark") - debug = parseCmd.Bool("debug", false, "print debug logs") -) - -// initBenchmarks initializes a dataset/table in a BigQuery project. -func initBenchmarks(ctx context.Context) error { - return bq.InitBigQuery(ctx, *initProject, *initDataset, *initTable, nil) -} - -// parseBenchmarks parses the given file into the BigQuery schema, -// adds some custom data for the commit, and sends the data to BigQuery. -func parseBenchmarks(ctx context.Context) error { - debugLog("Reading file: %s", *file) - data, err := ioutil.ReadFile(*file) - if err != nil { - return fmt.Errorf("failed to read file %s: %v", *file, err) - } - debugLog("Parsing output: %s", string(data)) - suite, err := parsers.ParseOutput(string(data), *name, *official) - if err != nil { - return fmt.Errorf("failed parse data: %v", err) - } - debugLog("Parsed benchmarks: %d", len(suite.Benchmarks)) - if len(suite.Benchmarks) < 1 { - fmt.Fprintf(os.Stderr, "Failed to find benchmarks for file: %s", *file) - return nil - } - - extraConditions := []*bq.Condition{ - { - Name: "runtime", - Value: *runtime, - }, - { - Name: "version", - Value: version, - }, - } - - suite.Official = *official - suite.Conditions = append(suite.Conditions, extraConditions...) - debugLog("Sending benchmarks") - return bq.SendBenchmarks(ctx, suite, *parseProject, *parseDataset, *parseTable, nil) -} - -func main() { - ctx := context.Background() - switch { - // the "init" command - case len(os.Args) >= 2 && os.Args[1] == initString: - if err := initCmd.Parse(os.Args[2:]); err != nil { - log.Fatalf("Failed parse flags: %v\n", err) - os.Exit(1) - } - if err := initBenchmarks(ctx); err != nil { - failure := "failed to initialize project: %s dataset: %s table: %s: %v\n" - log.Fatalf(failure, *parseProject, *parseDataset, *parseTable, err) - os.Exit(1) - } - // the "parse" command. - case len(os.Args) >= 2 && os.Args[1] == parseString: - if err := parseCmd.Parse(os.Args[2:]); err != nil { - log.Fatalf("Failed parse flags: %v\n", err) - os.Exit(1) - } - if err := parseBenchmarks(ctx); err != nil { - log.Fatalf("Failed parse benchmarks: %v\n", err) - os.Exit(1) - } - default: - printUsage() - os.Exit(1) - } -} - -// printUsage prints the top level usage string. -func printUsage() { - usage := `Usage: parser <command> <flags> ... - -Available commands: - %s %s - %s %s -` - log.Printf(usage, initCmd.Name(), initDescription, parseCmd.Name(), parseDescription) -} - -func debugLog(msg string, args ...interface{}) { - if *debug { - log.Printf(msg, args...) - } -} diff --git a/tools/parsers/version.go b/tools/parsers/version.go deleted file mode 100644 index ab9194b9d..000000000 --- a/tools/parsers/version.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -// version is set during linking. -var version = "VERSION_MISSING" diff --git a/tools/rules_go_symbols.patch b/tools/rules_go_symbols.patch deleted file mode 100644 index 46767f169..000000000 --- a/tools/rules_go_symbols.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff --git a/go/private/rules/test.bzl b/go/private/rules/test.bzl -index 17516ad7..76b6c68c 100644 ---- a/go/private/rules/test.bzl -+++ b/go/private/rules/test.bzl -@@ -117,9 +117,6 @@ def _go_test_impl(ctx): - ) - - test_gc_linkopts = gc_linkopts(ctx) -- if not go.mode.debug: -- # Disable symbol table and DWARF generation for test binaries. -- test_gc_linkopts.extend(["-s", "-w"]) - - # Link in the run_dir global for bzltestutil - test_gc_linkopts.extend(["-X", "github.com/bazelbuild/rules_go/go/tools/bzltestutil.RunDir=" + run_dir]) diff --git a/tools/rules_go_visibility.patch b/tools/rules_go_visibility.patch deleted file mode 100644 index e5bb2e3d5..000000000 --- a/tools/rules_go_visibility.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/third_party/org_golang_x_tools-gazelle.patch b/third_party/org_golang_x_tools-gazelle.patch -index 7bdacff5..2fe9ce93 100644 ---- a/third_party/org_golang_x_tools-gazelle.patch -+++ b/third_party/org_golang_x_tools-gazelle.patch -@@ -2054,7 +2054,7 @@ diff -urN b/go/analysis/internal/facts/BUILD.bazel c/go/analysis/internal/facts/ - + "imports.go", - + ], - + importpath = "golang.org/x/tools/go/analysis/internal/facts", --+ visibility = ["//go/analysis:__subpackages__"], -++ visibility = ["//visibility:public"], - + deps = [ - + "//go/analysis", - + "//go/types/objectpath", -@@ -2078,7 +2078,7 @@ diff -urN b/go/analysis/internal/facts/BUILD.bazel c/go/analysis/internal/facts/ - +alias( - + name = "go_default_library", - + actual = ":facts", --+ visibility = ["//go/analysis:__subpackages__"], -++ visibility = ["//visibility:public"], - +) - + - +go_test( diff --git a/tools/tag_release.sh b/tools/tag_release.sh deleted file mode 100755 index 50378065e..000000000 --- a/tools/tag_release.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script will optionally map a PiperOrigin-RevId to a given commit, -# validate a provided release name, create a tag and push it. It must be -# run manually when a release is created. - -set -xeuo pipefail - -# Check arguments. -if [[ "$#" -ne 3 ]]; then - echo "usage: $0 <commit|revid> <release.rc> <message-file>" - exit 1 -fi - -declare -r target_commit="$1" -declare -r release="$2" -declare -r message_file="$3" - -if [[ -z "${target_commit}" ]]; then - echo "error: <commit|revid> is empty." -fi -if [[ -z "${release}" ]]; then - echo "error: <release.rc> is empty." -fi -if ! [[ -r "${message_file}" ]]; then - echo "error: message file '${message_file}' is not readable." - exit 1 -fi - -closest_commit() { - while read line; do - if [[ "$line" =~ ^"commit " ]]; then - current_commit="${line#commit }" - continue - elif [[ "$line" =~ "PiperOrigin-RevId: " ]]; then - revid="${line#PiperOrigin-RevId: }" - [[ "${revid}" -le "$1" ]] && break - fi - done - echo "${current_commit}" -} - -# Is the passed identifier a sha commit? -if ! git show "${target_commit}" &> /dev/null; then - # Extract the commit given a piper ID. - commit="$(set +o pipefail; \ - git log --first-parent | closest_commit "${target_commit}")" - declare -r commit -else - declare -r commit="${target_commit}" -fi -if ! git show "${commit}" &> /dev/null; then - echo "unknown commit: ${target_commit}" - exit 1 -fi - -# Is the release name sane? Must be a date with patch/rc. -if ! [[ "${release}" =~ ^20[0-9]{6}\.[0-9]+$ ]]; then - declare -r expected="$(date +%Y%m%d.0)" # Use today's date. - echo "unexpected release format: ${release}" - echo " ... expected like ${expected}" - exit 1 -fi - -# Tag the given commit (annotated, to record the committer). Note that the tag -# here is applied as a force, in case the tag already exists and is the same. -# The push will fail in this case (because it is not forced). -declare -r tag="release-${release}" -git tag -f -F "${message_file}" -a "${tag}" "${commit}" && \ - git push origin tag "${tag}" diff --git a/tools/tags/BUILD b/tools/tags/BUILD deleted file mode 100644 index 1c02e2c89..000000000 --- a/tools/tags/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -load("//tools:defs.bzl", "go_library") - -package(licenses = ["notice"]) - -go_library( - name = "tags", - srcs = ["tags.go"], - marshal = False, - stateify = False, - visibility = ["//tools:__subpackages__"], -) diff --git a/tools/tags/tags.go b/tools/tags/tags.go deleted file mode 100644 index f35904e0a..000000000 --- a/tools/tags/tags.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package tags is a utility for parsing build tags. -package tags - -import ( - "fmt" - "io/ioutil" - "strings" -) - -// OrSet is a set of tags on a single line. -// -// Note that tags may include ",", and we don't distinguish this case in the -// logic below. Ideally, this constraints can be split into separate top-level -// build tags in order to resolve any issues. -type OrSet []string - -// Line returns the line for this or. -func (or OrSet) Line() string { - return fmt.Sprintf("// +build %s", strings.Join([]string(or), " ")) -} - -// AndSet is the set of all OrSets. -type AndSet []OrSet - -// Lines returns the lines to be printed. -func (and AndSet) Lines() (ls []string) { - for _, or := range and { - ls = append(ls, or.Line()) - } - return -} - -// Join joins this AndSet with another. -func (and AndSet) Join(other AndSet) AndSet { - return append(and, other...) -} - -// Tags returns the unique set of +build tags. -// -// Derived form the runtime's canBuild. -func Tags(file string) (tags AndSet) { - data, err := ioutil.ReadFile(file) - if err != nil { - return nil - } - // Check file contents for // +build lines. - for _, p := range strings.Split(string(data), "\n") { - p = strings.TrimSpace(p) - if p == "" { - continue - } - if !strings.HasPrefix(p, "//") { - break - } - if !strings.Contains(p, "+build") { - continue - } - fields := strings.Fields(p[2:]) - if len(fields) < 1 || fields[0] != "+build" { - continue - } - tags = append(tags, OrSet(fields[1:])) - } - return tags -} - -// Aggregate aggregates all tags from a set of files. -// -// Note that these may be in conflict, in which case the build will fail. -func Aggregate(files []string) (tags AndSet) { - for _, file := range files { - tags = tags.Join(Tags(file)) - } - return tags -} diff --git a/tools/verity/BUILD b/tools/verity/BUILD deleted file mode 100644 index 77d16359c..000000000 --- a/tools/verity/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -load("//tools:defs.bzl", "go_binary") - -licenses(["notice"]) - -go_binary( - name = "measure_tool", - srcs = [ - "measure_tool.go", - "measure_tool_unsafe.go", - ], - pure = True, - deps = [ - "//pkg/abi/linux", - ], -) diff --git a/tools/verity/measure_tool.go b/tools/verity/measure_tool.go deleted file mode 100644 index 0d314ae70..000000000 --- a/tools/verity/measure_tool.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2021 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This binary can be used to run a measurement of the verity file system, -// generate the corresponding Merkle tree files, and return the root hash. -package main - -import ( - "flag" - "io/ioutil" - "log" - "os" - "syscall" - - "gvisor.dev/gvisor/pkg/abi/linux" -) - -var path = flag.String("path", "", "path to the verity file system.") - -const maxDigestSize = 64 - -type digest struct { - metadata linux.DigestMetadata - digest [maxDigestSize]byte -} - -func main() { - flag.Parse() - if *path == "" { - log.Fatalf("no path provided") - } - if err := enableDir(*path); err != nil { - log.Fatalf("Failed to enable file system %s: %v", *path, err) - } - // Print the root hash of the file system to stdout. - if err := measure(*path); err != nil { - log.Fatalf("Failed to measure file system %s: %v", *path, err) - } -} - -// enableDir enables verity features on all the files and sub-directories within -// path. -func enableDir(path string) error { - files, err := ioutil.ReadDir(path) - if err != nil { - return err - } - for _, file := range files { - if file.IsDir() { - // For directories, first enable its children. - if err := enableDir(path + "/" + file.Name()); err != nil { - return err - } - } else if file.Mode().IsRegular() { - // For regular files, open and enable verity feature. - f, err := os.Open(path + "/" + file.Name()) - if err != nil { - return err - } - var p uintptr - if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, uintptr(f.Fd()), uintptr(linux.FS_IOC_ENABLE_VERITY), p); err != 0 { - return err - } - } - } - // Once all children are enabled, enable the parent directory. - f, err := os.Open(path) - if err != nil { - return err - } - var p uintptr - if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, uintptr(f.Fd()), uintptr(linux.FS_IOC_ENABLE_VERITY), p); err != 0 { - return err - } - return nil -} diff --git a/tools/verity/measure_tool_unsafe.go b/tools/verity/measure_tool_unsafe.go deleted file mode 100644 index d4079be9e..000000000 --- a/tools/verity/measure_tool_unsafe.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2021 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package main - -import ( - "encoding/hex" - "fmt" - "os" - "syscall" - "unsafe" - - "gvisor.dev/gvisor/pkg/abi/linux" -) - -// measure prints the hash of path to stdout. -func measure(path string) error { - f, err := os.Open(path) - if err != nil { - return err - } - var digest digest - digest.metadata.DigestSize = maxDigestSize - if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, uintptr(f.Fd()), uintptr(linux.FS_IOC_MEASURE_VERITY), uintptr(unsafe.Pointer(&digest))); err != 0 { - return err - } - fmt.Fprintf(os.Stdout, "%s\n", hex.EncodeToString(digest.digest[:digest.metadata.DigestSize])) - return err -} diff --git a/tools/worker/BUILD b/tools/worker/BUILD deleted file mode 100644 index dc03ce11e..000000000 --- a/tools/worker/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -load("//tools:defs.bzl", "bazel_worker_proto", "go_library") - -package(licenses = ["notice"]) - -# For Google-tooling. -# @unused -glaze_ignore = [ - "worker.go", -] - -go_library( - name = "worker", - srcs = ["worker.go"], - visibility = ["//tools:__subpackages__"], - deps = [ - bazel_worker_proto, - "@org_golang_google_protobuf//encoding/protowire:go_default_library", - "@org_golang_google_protobuf//proto:go_default_library", - "@org_golang_x_sys//unix:go_default_library", - ], -) diff --git a/tools/worker/worker.go b/tools/worker/worker.go deleted file mode 100644 index 669a5f203..000000000 --- a/tools/worker/worker.go +++ /dev/null @@ -1,325 +0,0 @@ -// Copyright 2021 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package worker provides an implementation of the bazel worker protocol. -// -// Tools may be written as a normal command line utility, except the passed -// run function may be invoked multiple times. -package worker - -import ( - "bufio" - "bytes" - "flag" - "fmt" - "io" - "io/ioutil" - "log" - "net" - "net/http" - "os" - "path/filepath" - "sort" - "strings" - "time" - - _ "net/http/pprof" // For profiling. - - "golang.org/x/sys/unix" - "google.golang.org/protobuf/encoding/protowire" - "google.golang.org/protobuf/proto" - wpb "gvisor.dev/bazel/worker_protocol_go_proto" -) - -var ( - persistentWorker = flag.Bool("persistent_worker", false, "enable persistent worker.") - workerDebug = flag.Bool("worker_debug", false, "debug persistent workers.") - maximumCacheUsage = flag.Int64("maximum_cache_usage", 1024*1024*1024, "maximum cache size.") -) - -var ( - // inputFiles is the last set of input files. - // - // This is used for cache invalidation. The key is the *absolute* path - // name, and the value is the digest in the current run. - inputFiles = make(map[string]string) - - // activeCaches is the set of active caches. - activeCaches = make(map[*Cache]struct{}) - - // totalCacheUsage is the total usage of all caches. - totalCacheUsage int64 -) - -// mustAbs returns the absolute path of a filename or dies. -func mustAbs(filename string) string { - abs, err := filepath.Abs(filename) - if err != nil { - log.Fatalf("error getting absolute path: %v", err) - } - return abs -} - -// updateInputFiles creates an entry in inputFiles. -func updateInputFile(filename, digest string) { - inputFiles[mustAbs(filename)] = digest -} - -// Sizer returns a size. -type Sizer interface { - Size() int64 -} - -// CacheBytes is an example of a Sizer. -type CacheBytes []byte - -// Size implements Sizer.Size. -func (cb CacheBytes) Size() int64 { - return int64(len(cb)) -} - -// Cache is a worker cache. -// -// They can be created via NewCache. -type Cache struct { - name string - entries map[string]Sizer - size int64 - hits int64 - misses int64 -} - -// NewCache returns a new cache. -func NewCache(name string) *Cache { - return &Cache{ - name: name, - } -} - -// Lookup looks up an entry in the cache. -// -// It is a function of the given files. -func (c *Cache) Lookup(filenames []string, generate func() Sizer) Sizer { - digests := make([]string, 0, len(filenames)) - for _, filename := range filenames { - digest, ok := inputFiles[mustAbs(filename)] - if !ok { - // This is not a valid input. We may not be running as - // persistent worker in this cache. If that's the case, - // then the file's contents will not change across the - // run, and we just use the filename itself. - digest = filename - } - digests = append(digests, digest) - } - - // Attempt the lookup. - sort.Slice(digests, func(i, j int) bool { - return digests[i] < digests[j] - }) - cacheKey := strings.Join(digests, "+") - if c.entries == nil { - c.entries = make(map[string]Sizer) - activeCaches[c] = struct{}{} - } - entry, ok := c.entries[cacheKey] - if ok { - c.hits++ - return entry - } - - // Generate a new entry. - entry = generate() - c.misses++ - c.entries[cacheKey] = entry - if entry != nil { - sz := entry.Size() - c.size += sz - totalCacheUsage += sz - } - - // Check the capacity of all caches. If it greater than the maximum, we - // flush everything but still return this entry. - if totalCacheUsage > *maximumCacheUsage { - for entry, _ := range activeCaches { - // Drop all entries. - entry.size = 0 - entry.entries = nil - } - totalCacheUsage = 0 // Reset. - } - - return entry -} - -// allCacheStats returns stats for all caches. -func allCacheStats() string { - var sb strings.Builder - for entry, _ := range activeCaches { - ratio := float64(entry.hits) / float64(entry.hits+entry.misses) - fmt.Fprintf(&sb, - "% 10s: count: % 5d size: % 10d hits: % 7d misses: % 7d ratio: %2.2f\n", - entry.name, len(entry.entries), entry.size, entry.hits, entry.misses, ratio) - } - if len(activeCaches) > 0 { - fmt.Fprintf(&sb, "total: % 10d\n", totalCacheUsage) - } - return sb.String() -} - -// LookupDigest returns a digest for the given file. -func LookupDigest(filename string) (string, bool) { - digest, ok := inputFiles[filename] - return digest, ok -} - -// Work invokes the main function. -func Work(run func([]string) int) { - flag.CommandLine.Parse(os.Args[1:]) - if !*persistentWorker { - // Handle the argument file. - args := flag.CommandLine.Args() - if len(args) == 1 && len(args[0]) > 1 && args[0][0] == '@' { - content, err := ioutil.ReadFile(args[0][1:]) - if err != nil { - log.Fatalf("unable to parse args file: %v", err) - } - // Pull arguments from the file. - args = strings.Split(string(content), "\n") - flag.CommandLine.Parse(args) - args = flag.CommandLine.Args() - } - os.Exit(run(args)) - } - - var listenHeader string // Emitted always. - if *workerDebug { - // Bind a server for profiling. - listener, err := net.Listen("tcp", "localhost:0") - if err != nil { - log.Fatalf("unable to bind a server: %v", err) - } - // Construct the header for stats output, below. - listenHeader = fmt.Sprintf("Listening @ http://localhost:%d\n", listener.Addr().(*net.TCPAddr).Port) - go http.Serve(listener, nil) - } - - // Move stdout. This is done to prevent anything else from accidentally - // printing to stdout, which must contain only the valid WorkerResponse - // serialized protos. - newOutput, err := unix.Dup(1) - if err != nil { - log.Fatalf("unable to move stdout: %v", err) - } - // Stderr may be closed or may be a copy of stdout. We make sure that - // we have an output that is in a completely separate range. - for newOutput <= 2 { - newOutput, err = unix.Dup(newOutput) - if err != nil { - log.Fatalf("unable to move stdout: %v", err) - } - } - - // Best-effort: collect logs. - rPipe, wPipe, err := os.Pipe() - if err != nil { - log.Fatalf("unable to create pipe: %v", err) - } - if err := unix.Dup2(int(wPipe.Fd()), 1); err != nil { - log.Fatalf("error duping over stdout: %v", err) - } - if err := unix.Dup2(int(wPipe.Fd()), 2); err != nil { - log.Fatalf("error duping over stderr: %v", err) - } - wPipe.Close() - defer rPipe.Close() - - // Read requests from stdin. - input := bufio.NewReader(os.NewFile(0, "input")) - output := bufio.NewWriter(os.NewFile(uintptr(newOutput), "output")) - for { - szBuf, err := input.Peek(4) - if err != nil { - log.Fatalf("unabel to read header: %v", err) - } - - // Parse the size, and discard bits. - sz, szBytes := protowire.ConsumeVarint(szBuf) - if szBytes < 0 { - szBytes = 0 - } - if _, err := input.Discard(szBytes); err != nil { - log.Fatalf("error discarding size: %v", err) - } - - // Read a full message. - msg := make([]byte, int(sz)) - if _, err := io.ReadFull(input, msg); err != nil { - log.Fatalf("error reading worker request: %v", err) - } - var wreq wpb.WorkRequest - if err := proto.Unmarshal(msg, &wreq); err != nil { - log.Fatalf("error unmarshaling worker request: %v", err) - } - - // Flush relevant caches. - inputFiles = make(map[string]string) - for _, input := range wreq.GetInputs() { - updateInputFile(input.GetPath(), string(input.GetDigest())) - } - - // Prepare logging. - outputBuffer := bytes.NewBuffer(nil) - outputBuffer.WriteString(listenHeader) - log.SetOutput(outputBuffer) - - // Parse all arguments. - flag.CommandLine.Parse(wreq.GetArguments()) - var exitCode int - exitChan := make(chan int) - go func() { exitChan <- run(flag.CommandLine.Args()) }() - for running := true; running; { - select { - case exitCode = <-exitChan: - running = false - default: - } - // N.B. rPipe is given a read deadline of 1ms. We expect - // this to turn a copy error after 1ms, and we just keep - // flushing this buffer while the task is running. - rPipe.SetReadDeadline(time.Now().Add(time.Millisecond)) - outputBuffer.ReadFrom(rPipe) - } - - if *workerDebug { - // Attach all cache stats. - outputBuffer.WriteString(allCacheStats()) - } - - // Send the response. - var wresp wpb.WorkResponse - wresp.ExitCode = int32(exitCode) - wresp.Output = string(outputBuffer.Bytes()) - rmsg, err := proto.Marshal(&wresp) - if err != nil { - log.Fatalf("error marshaling response: %v", err) - } - if _, err := output.Write(append(protowire.AppendVarint(nil, uint64(len(rmsg))), rmsg...)); err != nil { - log.Fatalf("error sending worker response: %v", err) - } - if err := output.Flush(); err != nil { - log.Fatalf("error flushing output: %v", err) - } - } -} diff --git a/tools/workspace_status.sh b/tools/workspace_status.sh deleted file mode 100755 index 62d78ed3d..000000000 --- a/tools/workspace_status.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Copyright 2018 The gVisor Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The STABLE_ prefix will trigger a re-link if it changes. -echo STABLE_VERSION "$(git describe --always --tags --abbrev=12 --dirty 2>/dev/null || echo 0.0.0)" diff --git a/tools/yamltest/BUILD b/tools/yamltest/BUILD deleted file mode 100644 index 475b3badd..000000000 --- a/tools/yamltest/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -load("//tools:defs.bzl", "go_binary") - -package(licenses = ["notice"]) - -go_binary( - name = "yamltest", - srcs = ["main.go"], - visibility = ["//visibility:public"], - deps = [ - "@com_github_xeipuuv_gojsonschema//:go_default_library", - "@in_gopkg_yaml_v2//:go_default_library", - ], -) diff --git a/tools/yamltest/defs.bzl b/tools/yamltest/defs.bzl deleted file mode 100644 index fd04f947d..000000000 --- a/tools/yamltest/defs.bzl +++ /dev/null @@ -1,41 +0,0 @@ -"""Tools for testing yaml files against schemas.""" - -def _yaml_test_impl(ctx): - """Implementation for yaml_test.""" - runner = ctx.actions.declare_file(ctx.label.name) - ctx.actions.write(runner, "\n".join([ - "#!/bin/bash", - "set -euo pipefail", - "%s -schema=%s -- %s" % ( - ctx.files._tool[0].short_path, - ctx.files.schema[0].short_path, - " ".join([f.short_path for f in ctx.files.srcs]), - ), - ]), is_executable = True) - return [DefaultInfo( - runfiles = ctx.runfiles(files = ctx.files._tool + ctx.files.schema + ctx.files.srcs), - executable = runner, - )] - -yaml_test = rule( - implementation = _yaml_test_impl, - doc = "Tests a yaml file against a schema.", - attrs = { - "srcs": attr.label_list( - doc = "The input yaml files.", - mandatory = True, - allow_files = True, - ), - "schema": attr.label( - doc = "The schema file in JSON schema format.", - allow_single_file = True, - mandatory = True, - ), - "_tool": attr.label( - executable = True, - cfg = "host", - default = Label("//tools/yamltest:yamltest"), - ), - }, - test = True, -) diff --git a/tools/yamltest/main.go b/tools/yamltest/main.go deleted file mode 100644 index 88271fb66..000000000 --- a/tools/yamltest/main.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Binary yamltest does strict yaml parsing and validation. -package main - -import ( - "encoding/json" - "errors" - "flag" - "fmt" - "os" - - "github.com/xeipuuv/gojsonschema" - yaml "gopkg.in/yaml.v2" -) - -func fixup(v interface{}) (interface{}, error) { - switch x := v.(type) { - case map[interface{}]interface{}: - // Coerse into a string-based map, required for yaml. - strMap := make(map[string]interface{}) - for k, v := range x { - strK, ok := k.(string) - if !ok { - // This cannot be converted to JSON at all. - return nil, fmt.Errorf("invalid key %T in (%#v)", k, x) - } - fv, err := fixup(v) - if err != nil { - return nil, fmt.Errorf(".%s%w", strK, err) - } - strMap[strK] = fv - } - return strMap, nil - case []interface{}: - for i := range x { - fv, err := fixup(x[i]) - if err != nil { - return nil, fmt.Errorf("[%d]%w", i, err) - } - x[i] = fv - } - return x, nil - default: - return v, nil - } -} - -func loadFile(filename string) (gojsonschema.JSONLoader, error) { - f, err := os.Open(filename) - if err != nil { - return nil, err - } - defer f.Close() - dec := yaml.NewDecoder(f) - dec.SetStrict(true) - var object interface{} - if err := dec.Decode(&object); err != nil { - return nil, err - } - fixedObject, err := fixup(object) // For serialization. - if err != nil { - return nil, err - } - bytes, err := json.Marshal(fixedObject) - if err != nil { - return nil, err - } - return gojsonschema.NewStringLoader(string(bytes)), nil -} - -var schema = flag.String("schema", "", "path to JSON schema file.") - -func main() { - flag.Parse() - if *schema == "" || len(flag.Args()) == 0 { - flag.Usage() - os.Exit(2) - } - - // Construct our schema loader. - schemaLoader := gojsonschema.NewReferenceLoader(fmt.Sprintf("file://%s", *schema)) - - // Parse all documents. - allErrors := make(map[string][]error) - for _, filename := range flag.Args() { - // Record the filename with an empty slice for below, where - // we will emit all files (even those without any errors). - allErrors[filename] = nil - documentLoader, err := loadFile(filename) - if err != nil { - allErrors[filename] = append(allErrors[filename], err) - continue - } - result, err := gojsonschema.Validate(schemaLoader, documentLoader) - if err != nil { - allErrors[filename] = append(allErrors[filename], err) - continue - } - for _, desc := range result.Errors() { - allErrors[filename] = append(allErrors[filename], errors.New(desc.String())) - } - } - - // Print errors in yaml format. - totalErrors := 0 - for filename, errs := range allErrors { - totalErrors += len(errs) - if len(errs) == 0 { - fmt.Fprintf(os.Stderr, "%s: ✓\n", filename) - continue - } - fmt.Fprintf(os.Stderr, "%s:\n", filename) - for _, err := range errs { - fmt.Fprintf(os.Stderr, "- %s\n", err) - } - } - if totalErrors != 0 { - os.Exit(1) - } -} |