From 27500d529f7fb87eef8812278fd1bbca67bcba72 Mon Sep 17 00:00:00 2001 From: Ian Gudger Date: Thu, 9 Jan 2020 22:00:42 -0800 Subject: New sync package. * Rename syncutil to sync. * Add aliases to sync types. * Replace existing usage of standard library sync package. This will make it easier to swap out synchronization primitives. For example, this will allow us to use primitives from github.com/sasha-s/go-deadlock to check for lock ordering violations. Updates #1472 PiperOrigin-RevId: 289033387 --- pkg/metric/metric.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pkg/metric/metric.go') diff --git a/pkg/metric/metric.go b/pkg/metric/metric.go index eadde06e4..93d4f2b8c 100644 --- a/pkg/metric/metric.go +++ b/pkg/metric/metric.go @@ -18,12 +18,12 @@ package metric import ( "errors" "fmt" - "sync" "sync/atomic" "gvisor.dev/gvisor/pkg/eventchannel" "gvisor.dev/gvisor/pkg/log" pb "gvisor.dev/gvisor/pkg/metric/metric_go_proto" + "gvisor.dev/gvisor/pkg/sync" ) var ( -- cgit v1.2.3 From 1b6a12a768216a99a5e0428c42ea4faf79cf3b50 Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Wed, 5 Feb 2020 22:45:44 -0800 Subject: Add notes to relevant tests. These were out-of-band notes that can help provide additional context and simplify automated imports. PiperOrigin-RevId: 293525915 --- pkg/metric/metric.go | 1 - pkg/sentry/arch/arch_x86.go | 4 ++ pkg/sentry/arch/signal_amd64.go | 2 +- pkg/sentry/fs/file_overlay_test.go | 1 + pkg/sentry/fs/proc/README.md | 4 ++ pkg/sentry/kernel/BUILD | 1 + pkg/sentry/kernel/kernel.go | 3 ++ pkg/sentry/kernel/kernel_opts.go | 20 +++++++ pkg/sentry/socket/hostinet/BUILD | 1 + pkg/sentry/socket/hostinet/socket.go | 5 +- pkg/sentry/socket/hostinet/sockopt_impl.go | 27 ++++++++++ pkg/tcpip/transport/tcp/endpoint.go | 3 ++ runsc/boot/filter/BUILD | 1 + runsc/boot/filter/config.go | 13 ----- runsc/boot/filter/config_profile.go | 34 ++++++++++++ runsc/container/console_test.go | 5 +- runsc/dockerutil/dockerutil.go | 11 ++-- runsc/testutil/BUILD | 5 +- runsc/testutil/testutil.go | 54 ------------------- runsc/testutil/testutil_runfiles.go | 75 +++++++++++++++++++++++++++ test/image/image_test.go | 8 +-- test/syscalls/build_defs.bzl | 35 +++++++++++-- test/syscalls/linux/chroot.cc | 2 +- test/syscalls/linux/concurrency.cc | 3 +- test/syscalls/linux/exec_proc_exe_workload.cc | 6 +++ test/syscalls/linux/fork.cc | 5 +- test/syscalls/linux/mmap.cc | 8 +-- test/syscalls/linux/open_create.cc | 1 + test/syscalls/linux/preadv.cc | 1 + test/syscalls/linux/proc.cc | 46 +++++++++++++--- test/syscalls/linux/readv.cc | 4 +- test/syscalls/linux/rseq.cc | 2 +- test/syscalls/linux/select.cc | 2 +- test/syscalls/linux/shm.cc | 2 +- test/syscalls/linux/sigprocmask.cc | 2 +- test/syscalls/linux/socket_unix_non_stream.cc | 4 +- test/syscalls/linux/symlink.cc | 2 +- test/syscalls/linux/tcp_socket.cc | 3 +- test/syscalls/linux/time.cc | 1 + test/syscalls/linux/tkill.cc | 2 +- test/util/temp_path.cc | 1 + tools/build/tags.bzl | 4 ++ tools/defs.bzl | 17 +++++- 43 files changed, 318 insertions(+), 113 deletions(-) create mode 100644 pkg/sentry/kernel/kernel_opts.go create mode 100644 pkg/sentry/socket/hostinet/sockopt_impl.go create mode 100644 runsc/boot/filter/config_profile.go create mode 100644 runsc/testutil/testutil_runfiles.go (limited to 'pkg/metric/metric.go') diff --git a/pkg/metric/metric.go b/pkg/metric/metric.go index 93d4f2b8c..006fcd9ab 100644 --- a/pkg/metric/metric.go +++ b/pkg/metric/metric.go @@ -46,7 +46,6 @@ var ( // // TODO(b/67298402): Support non-cumulative metrics. // TODO(b/67298427): Support metric fields. -// type Uint64Metric struct { // value is the actual value of the metric. It must be accessed // atomically. diff --git a/pkg/sentry/arch/arch_x86.go b/pkg/sentry/arch/arch_x86.go index a18093155..3db8bd34b 100644 --- a/pkg/sentry/arch/arch_x86.go +++ b/pkg/sentry/arch/arch_x86.go @@ -114,6 +114,10 @@ func newX86FPStateSlice() []byte { size, align := cpuid.HostFeatureSet().ExtendedStateSize() capacity := size // Always use at least 4096 bytes. + // + // For the KVM platform, this state is a fixed 4096 bytes, so make sure + // that the underlying array is at _least_ that size otherwise we will + // corrupt random memory. This is not a pleasant thing to debug. if capacity < 4096 { capacity = 4096 } diff --git a/pkg/sentry/arch/signal_amd64.go b/pkg/sentry/arch/signal_amd64.go index 81b92bb43..6fb756f0e 100644 --- a/pkg/sentry/arch/signal_amd64.go +++ b/pkg/sentry/arch/signal_amd64.go @@ -55,7 +55,7 @@ type SignalContext64 struct { Trapno uint64 Oldmask linux.SignalSet Cr2 uint64 - // Pointer to a struct _fpstate. + // Pointer to a struct _fpstate. See b/33003106#comment8. Fpstate uint64 Reserved [8]uint64 } diff --git a/pkg/sentry/fs/file_overlay_test.go b/pkg/sentry/fs/file_overlay_test.go index 02538bb4f..a76d87e3a 100644 --- a/pkg/sentry/fs/file_overlay_test.go +++ b/pkg/sentry/fs/file_overlay_test.go @@ -177,6 +177,7 @@ func TestReaddirRevalidation(t *testing.T) { // TestReaddirOverlayFrozen tests that calling Readdir on an overlay file with // a frozen dirent tree does not make Readdir calls to the underlying files. +// This is a regression test for b/114808269. func TestReaddirOverlayFrozen(t *testing.T) { ctx := contexttest.Context(t) diff --git a/pkg/sentry/fs/proc/README.md b/pkg/sentry/fs/proc/README.md index 5d4ec6c7b..6667a0916 100644 --- a/pkg/sentry/fs/proc/README.md +++ b/pkg/sentry/fs/proc/README.md @@ -11,6 +11,8 @@ inconsistency, please file a bug. The following files are implemented: + + | File /proc/ | Content | | :------------------------ | :---------------------------------------------------- | | [cpuinfo](#cpuinfo) | Info about the CPU | @@ -22,6 +24,8 @@ The following files are implemented: | [uptime](#uptime) | Wall clock since boot, combined idle time of all cpus | | [version](#version) | Kernel version | + + ### cpuinfo ```bash diff --git a/pkg/sentry/kernel/BUILD b/pkg/sentry/kernel/BUILD index a27628c0a..2231d6973 100644 --- a/pkg/sentry/kernel/BUILD +++ b/pkg/sentry/kernel/BUILD @@ -91,6 +91,7 @@ go_library( "fs_context.go", "ipc_namespace.go", "kernel.go", + "kernel_opts.go", "kernel_state.go", "pending_signals.go", "pending_signals_list.go", diff --git a/pkg/sentry/kernel/kernel.go b/pkg/sentry/kernel/kernel.go index dcd6e91c4..3ee760ba2 100644 --- a/pkg/sentry/kernel/kernel.go +++ b/pkg/sentry/kernel/kernel.go @@ -235,6 +235,9 @@ type Kernel struct { // events. This is initialized lazily on the first unimplemented // syscall. unimplementedSyscallEmitter eventchannel.Emitter `state:"nosave"` + + // SpecialOpts contains special kernel options. + SpecialOpts } // InitKernelArgs holds arguments to Init. diff --git a/pkg/sentry/kernel/kernel_opts.go b/pkg/sentry/kernel/kernel_opts.go new file mode 100644 index 000000000..2e66ec587 --- /dev/null +++ b/pkg/sentry/kernel/kernel_opts.go @@ -0,0 +1,20 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kernel + +// SpecialOpts contains non-standard options for the kernel. +// +// +stateify savable +type SpecialOpts struct{} diff --git a/pkg/sentry/socket/hostinet/BUILD b/pkg/sentry/socket/hostinet/BUILD index 5a07d5d0e..023bad156 100644 --- a/pkg/sentry/socket/hostinet/BUILD +++ b/pkg/sentry/socket/hostinet/BUILD @@ -10,6 +10,7 @@ go_library( "save_restore.go", "socket.go", "socket_unsafe.go", + "sockopt_impl.go", "stack.go", ], visibility = ["//pkg/sentry:internal"], diff --git a/pkg/sentry/socket/hostinet/socket.go b/pkg/sentry/socket/hostinet/socket.go index 34f63986f..de76388ac 100644 --- a/pkg/sentry/socket/hostinet/socket.go +++ b/pkg/sentry/socket/hostinet/socket.go @@ -285,7 +285,7 @@ func (s *socketOperations) GetSockOpt(t *kernel.Task, level int, name int, outPt } // Whitelist options and constrain option length. - var optlen int + optlen := getSockOptLen(t, level, name) switch level { case linux.SOL_IP: switch name { @@ -330,7 +330,7 @@ func (s *socketOperations) GetSockOpt(t *kernel.Task, level int, name int, outPt // SetSockOpt implements socket.Socket.SetSockOpt. func (s *socketOperations) SetSockOpt(t *kernel.Task, level int, name int, opt []byte) *syserr.Error { // Whitelist options and constrain option length. - var optlen int + optlen := setSockOptLen(t, level, name) switch level { case linux.SOL_IP: switch name { @@ -353,6 +353,7 @@ func (s *socketOperations) SetSockOpt(t *kernel.Task, level int, name int, opt [ optlen = sizeofInt32 } } + if optlen == 0 { // Pretend to accept socket options we don't understand. This seems // dangerous, but it's what netstack does... diff --git a/pkg/sentry/socket/hostinet/sockopt_impl.go b/pkg/sentry/socket/hostinet/sockopt_impl.go new file mode 100644 index 000000000..8a783712e --- /dev/null +++ b/pkg/sentry/socket/hostinet/sockopt_impl.go @@ -0,0 +1,27 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostinet + +import ( + "gvisor.dev/gvisor/pkg/sentry/kernel" +) + +func getSockOptLen(t *kernel.Task, level, name int) int { + return 0 // No custom options. +} + +func setSockOptLen(t *kernel.Task, level, name int) int { + return 0 // No custom options. +} diff --git a/pkg/tcpip/transport/tcp/endpoint.go b/pkg/tcpip/transport/tcp/endpoint.go index e4a6b1b8b..f2be0e651 100644 --- a/pkg/tcpip/transport/tcp/endpoint.go +++ b/pkg/tcpip/transport/tcp/endpoint.go @@ -2166,6 +2166,9 @@ func (e *endpoint) listen(backlog int) *tcpip.Error { e.isRegistered = true e.setEndpointState(StateListen) + // The channel may be non-nil when we're restoring the endpoint, and it + // may be pre-populated with some previously accepted (but not Accepted) + // endpoints. if e.acceptedChan == nil { e.acceptedChan = make(chan *endpoint, backlog) } diff --git a/runsc/boot/filter/BUILD b/runsc/boot/filter/BUILD index ce30f6c53..ed18f0047 100644 --- a/runsc/boot/filter/BUILD +++ b/runsc/boot/filter/BUILD @@ -8,6 +8,7 @@ go_library( "config.go", "config_amd64.go", "config_arm64.go", + "config_profile.go", "extra_filters.go", "extra_filters_msan.go", "extra_filters_race.go", diff --git a/runsc/boot/filter/config.go b/runsc/boot/filter/config.go index f8d351c7b..c69f4c602 100644 --- a/runsc/boot/filter/config.go +++ b/runsc/boot/filter/config.go @@ -536,16 +536,3 @@ func controlServerFilters(fd int) seccomp.SyscallRules { }, } } - -// profileFilters returns extra syscalls made by runtime/pprof package. -func profileFilters() seccomp.SyscallRules { - return seccomp.SyscallRules{ - syscall.SYS_OPENAT: []seccomp.Rule{ - { - seccomp.AllowAny{}, - seccomp.AllowAny{}, - seccomp.AllowValue(syscall.O_RDONLY | syscall.O_LARGEFILE | syscall.O_CLOEXEC), - }, - }, - } -} diff --git a/runsc/boot/filter/config_profile.go b/runsc/boot/filter/config_profile.go new file mode 100644 index 000000000..194952a7b --- /dev/null +++ b/runsc/boot/filter/config_profile.go @@ -0,0 +1,34 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filter + +import ( + "syscall" + + "gvisor.dev/gvisor/pkg/seccomp" +) + +// profileFilters returns extra syscalls made by runtime/pprof package. +func profileFilters() seccomp.SyscallRules { + return seccomp.SyscallRules{ + syscall.SYS_OPENAT: []seccomp.Rule{ + { + seccomp.AllowAny{}, + seccomp.AllowAny{}, + seccomp.AllowValue(syscall.O_RDONLY | syscall.O_LARGEFILE | syscall.O_CLOEXEC), + }, + }, + } +} diff --git a/runsc/container/console_test.go b/runsc/container/console_test.go index 060b63bf3..c2518d52b 100644 --- a/runsc/container/console_test.go +++ b/runsc/container/console_test.go @@ -196,7 +196,10 @@ func TestJobControlSignalExec(t *testing.T) { defer ptyMaster.Close() defer ptySlave.Close() - // Exec bash and attach a terminal. + // Exec bash and attach a terminal. Note that occasionally /bin/sh + // may be a different shell or have a different configuration (such + // as disabling interactive mode and job control). Since we want to + // explicitly test interactive mode, use /bin/bash. See b/116981926. execArgs := &control.ExecArgs{ Filename: "/bin/bash", // Don't let bash execute from profile or rc files, otherwise diff --git a/runsc/dockerutil/dockerutil.go b/runsc/dockerutil/dockerutil.go index 9b6346ca2..1ff5e8cc3 100644 --- a/runsc/dockerutil/dockerutil.go +++ b/runsc/dockerutil/dockerutil.go @@ -143,8 +143,11 @@ func PrepareFiles(names ...string) (string, error) { return "", fmt.Errorf("os.Chmod(%q, 0777) failed: %v", dir, err) } for _, name := range names { - src := getLocalPath(name) - dst := path.Join(dir, name) + src, err := testutil.FindFile(name) + if err != nil { + return "", fmt.Errorf("testutil.Preparefiles(%q) failed: %v", name, err) + } + dst := path.Join(dir, path.Base(name)) if err := testutil.Copy(src, dst); err != nil { return "", fmt.Errorf("testutil.Copy(%q, %q) failed: %v", src, dst, err) } @@ -152,10 +155,6 @@ func PrepareFiles(names ...string) (string, error) { return dir, nil } -func getLocalPath(file string) string { - return path.Join(".", file) -} - // do executes docker command. func do(args ...string) (string, error) { log.Printf("Running: docker %s\n", args) diff --git a/runsc/testutil/BUILD b/runsc/testutil/BUILD index f845120b0..945405303 100644 --- a/runsc/testutil/BUILD +++ b/runsc/testutil/BUILD @@ -5,7 +5,10 @@ package(licenses = ["notice"]) go_library( name = "testutil", testonly = 1, - srcs = ["testutil.go"], + srcs = [ + "testutil.go", + "testutil_runfiles.go", + ], visibility = ["//:sandbox"], deps = [ "//pkg/log", diff --git a/runsc/testutil/testutil.go b/runsc/testutil/testutil.go index edf2e809a..80c2c9680 100644 --- a/runsc/testutil/testutil.go +++ b/runsc/testutil/testutil.go @@ -79,60 +79,6 @@ func ConfigureExePath() error { return nil } -// FindFile searchs for a file inside the test run environment. It returns the -// full path to the file. It fails if none or more than one file is found. -func FindFile(path string) (string, error) { - wd, err := os.Getwd() - if err != nil { - return "", err - } - - // The test root is demarcated by a path element called "__main__". Search for - // it backwards from the working directory. - root := wd - for { - dir, name := filepath.Split(root) - if name == "__main__" { - break - } - if len(dir) == 0 { - return "", fmt.Errorf("directory __main__ not found in %q", wd) - } - // Remove ending slash to loop around. - root = dir[:len(dir)-1] - } - - // Annoyingly, bazel adds the build type to the directory path for go - // binaries, but not for c++ binaries. We use two different patterns to - // to find our file. - patterns := []string{ - // Try the obvious path first. - filepath.Join(root, path), - // If it was a go binary, use a wildcard to match the build - // type. The pattern is: /test-path/__main__/directories/*/file. - filepath.Join(root, filepath.Dir(path), "*", filepath.Base(path)), - } - - for _, p := range patterns { - matches, err := filepath.Glob(p) - if err != nil { - // "The only possible returned error is ErrBadPattern, - // when pattern is malformed." -godoc - return "", fmt.Errorf("error globbing %q: %v", p, err) - } - switch len(matches) { - case 0: - // Try the next pattern. - case 1: - // We found it. - return matches[0], nil - default: - return "", fmt.Errorf("more than one match found for %q: %s", path, matches) - } - } - return "", fmt.Errorf("file %q not found", path) -} - // TestConfig returns the default configuration to use in tests. Note that // 'RootDir' must be set by caller if required. func TestConfig() *boot.Config { diff --git a/runsc/testutil/testutil_runfiles.go b/runsc/testutil/testutil_runfiles.go new file mode 100644 index 000000000..ece9ea9a1 --- /dev/null +++ b/runsc/testutil/testutil_runfiles.go @@ -0,0 +1,75 @@ +// Copyright 2018 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testutil + +import ( + "fmt" + "os" + "path/filepath" +) + +// FindFile searchs for a file inside the test run environment. It returns the +// full path to the file. It fails if none or more than one file is found. +func FindFile(path string) (string, error) { + wd, err := os.Getwd() + if err != nil { + return "", err + } + + // The test root is demarcated by a path element called "__main__". Search for + // it backwards from the working directory. + root := wd + for { + dir, name := filepath.Split(root) + if name == "__main__" { + break + } + if len(dir) == 0 { + return "", fmt.Errorf("directory __main__ not found in %q", wd) + } + // Remove ending slash to loop around. + root = dir[:len(dir)-1] + } + + // Annoyingly, bazel adds the build type to the directory path for go + // binaries, but not for c++ binaries. We use two different patterns to + // to find our file. + patterns := []string{ + // Try the obvious path first. + filepath.Join(root, path), + // If it was a go binary, use a wildcard to match the build + // type. The pattern is: /test-path/__main__/directories/*/file. + filepath.Join(root, filepath.Dir(path), "*", filepath.Base(path)), + } + + for _, p := range patterns { + matches, err := filepath.Glob(p) + if err != nil { + // "The only possible returned error is ErrBadPattern, + // when pattern is malformed." -godoc + return "", fmt.Errorf("error globbing %q: %v", p, err) + } + switch len(matches) { + case 0: + // Try the next pattern. + case 1: + // We found it. + return matches[0], nil + default: + return "", fmt.Errorf("more than one match found for %q: %s", path, matches) + } + } + return "", fmt.Errorf("file %q not found", path) +} diff --git a/test/image/image_test.go b/test/image/image_test.go index d0dcb1861..0a1e19d6f 100644 --- a/test/image/image_test.go +++ b/test/image/image_test.go @@ -107,7 +107,7 @@ func TestHttpd(t *testing.T) { } d := dockerutil.MakeDocker("http-test") - dir, err := dockerutil.PrepareFiles("latin10k.txt") + dir, err := dockerutil.PrepareFiles("test/image/latin10k.txt") if err != nil { t.Fatalf("PrepareFiles() failed: %v", err) } @@ -139,7 +139,7 @@ func TestNginx(t *testing.T) { } d := dockerutil.MakeDocker("net-test") - dir, err := dockerutil.PrepareFiles("latin10k.txt") + dir, err := dockerutil.PrepareFiles("test/image/latin10k.txt") if err != nil { t.Fatalf("PrepareFiles() failed: %v", err) } @@ -183,7 +183,7 @@ func TestMysql(t *testing.T) { } client := dockerutil.MakeDocker("mysql-client-test") - dir, err := dockerutil.PrepareFiles("mysql.sql") + dir, err := dockerutil.PrepareFiles("test/image/mysql.sql") if err != nil { t.Fatalf("PrepareFiles() failed: %v", err) } @@ -283,7 +283,7 @@ func TestRuby(t *testing.T) { } d := dockerutil.MakeDocker("ruby-test") - dir, err := dockerutil.PrepareFiles("ruby.rb", "ruby.sh") + dir, err := dockerutil.PrepareFiles("test/image/ruby.rb", "test/image/ruby.sh") if err != nil { t.Fatalf("PrepareFiles() failed: %v", err) } diff --git a/test/syscalls/build_defs.bzl b/test/syscalls/build_defs.bzl index 1df761dd0..cbab85ef7 100644 --- a/test/syscalls/build_defs.bzl +++ b/test/syscalls/build_defs.bzl @@ -2,8 +2,6 @@ load("//tools:defs.bzl", "loopback") -# syscall_test is a macro that will create targets to run the given test target -# on the host (native) and runsc. def syscall_test( test, shard_count = 5, @@ -13,6 +11,19 @@ def syscall_test( add_uds_tree = False, add_hostinet = False, tags = None): + """syscall_test is a macro that will create targets for all platforms. + + Args: + test: the test target. + shard_count: shards for defined tests. + size: the defined test size. + use_tmpfs: use tmpfs in the defined tests. + add_overlay: add an overlay test. + add_uds_tree: add a UDS test. + add_hostinet: add a hostinet test. + tags: starting test tags. + """ + _syscall_test( test = test, shard_count = shard_count, @@ -111,6 +122,19 @@ def _syscall_test( # all the tests on a specific flavor. Use --test_tag_filters=ptrace,file_shared. tags += [full_platform, "file_" + file_access] + # Hash this target into one of 15 buckets. This can be used to + # randomly split targets between different workflows. + hash15 = hash(native.package_name() + name) % 15 + tags.append("hash15:" + str(hash15)) + + # TODO(b/139838000): Tests using hostinet must be disabled on Guitar until + # we figure out how to request ipv4 sockets on Guitar machines. + if network == "host": + tags.append("noguitar") + + # Disable off-host networking. + tags.append("requires-net:loopback") + # Add tag to prevent the tests from running in a Bazel sandbox. # TODO(b/120560048): Make the tests run without this tag. tags.append("no-sandbox") @@ -118,8 +142,11 @@ def _syscall_test( # TODO(b/112165693): KVM tests are tagged "manual" to until the platform is # more stable. if platform == "kvm": - tags += ["manual"] - tags += ["requires-kvm"] + tags.append("manual") + tags.append("requires-kvm") + + # TODO(b/112165693): Remove when tests pass reliably. + tags.append("notap") args = [ # Arguments are passed directly to syscall_test_runner binary. diff --git a/test/syscalls/linux/chroot.cc b/test/syscalls/linux/chroot.cc index 0a2d44a2c..85ec013d5 100644 --- a/test/syscalls/linux/chroot.cc +++ b/test/syscalls/linux/chroot.cc @@ -167,7 +167,7 @@ TEST(ChrootTest, DotDotFromOpenFD) { } // Test that link resolution in a chroot can escape the root by following an -// open proc fd. +// open proc fd. Regression test for b/32316719. TEST(ChrootTest, ProcFdLinkResolutionInChroot) { SKIP_IF(!ASSERT_NO_ERRNO_AND_VALUE(HaveCapability(CAP_SYS_CHROOT))); diff --git a/test/syscalls/linux/concurrency.cc b/test/syscalls/linux/concurrency.cc index f41f99900..7cd6a75bd 100644 --- a/test/syscalls/linux/concurrency.cc +++ b/test/syscalls/linux/concurrency.cc @@ -46,7 +46,8 @@ TEST(ConcurrencyTest, SingleProcessMultithreaded) { } // Test that multiple threads in this process continue to execute in parallel, -// even if an unrelated second process is spawned. +// even if an unrelated second process is spawned. Regression test for +// b/32119508. TEST(ConcurrencyTest, MultiProcessMultithreaded) { // In PID 1, start TIDs 1 and 2, and put both to sleep. // diff --git a/test/syscalls/linux/exec_proc_exe_workload.cc b/test/syscalls/linux/exec_proc_exe_workload.cc index b790fe5be..2989379b7 100644 --- a/test/syscalls/linux/exec_proc_exe_workload.cc +++ b/test/syscalls/linux/exec_proc_exe_workload.cc @@ -21,6 +21,12 @@ #include "test/util/posix_error.h" int main(int argc, char** argv, char** envp) { + // This is annoying. Because remote build systems may put these binaries + // in a content-addressable-store, you may wind up with /proc/self/exe + // pointing to some random path (but with a sensible argv[0]). + // + // Therefore, this test simply checks that the /proc/self/exe + // is absolute and *doesn't* match argv[1]. std::string exe = gvisor::testing::ProcessExePath(getpid()).ValueOrDie(); if (exe[0] != '/') { diff --git a/test/syscalls/linux/fork.cc b/test/syscalls/linux/fork.cc index 906f3358d..ff8bdfeb0 100644 --- a/test/syscalls/linux/fork.cc +++ b/test/syscalls/linux/fork.cc @@ -271,7 +271,7 @@ TEST_F(ForkTest, Alarm) { EXPECT_EQ(0, alarmed); } -// Child cannot affect parent private memory. +// Child cannot affect parent private memory. Regression test for b/24137240. TEST_F(ForkTest, PrivateMemory) { std::atomic local(0); @@ -298,6 +298,9 @@ TEST_F(ForkTest, PrivateMemory) { } // Kernel-accessed buffers should remain coherent across COW. +// +// The buffer must be >= usermem.ZeroCopyMinBytes, as UnsafeAccess operates +// differently. Regression test for b/33811887. TEST_F(ForkTest, COWSegment) { constexpr int kBufSize = 1024; char* read_buf = private_; diff --git a/test/syscalls/linux/mmap.cc b/test/syscalls/linux/mmap.cc index 1c4d9f1c7..11fb1b457 100644 --- a/test/syscalls/linux/mmap.cc +++ b/test/syscalls/linux/mmap.cc @@ -1418,7 +1418,7 @@ TEST_P(MMapFileParamTest, NoSigBusOnPageContainingEOF) { // // On most platforms this is trivial, but when the file is mapped via the sentry // page cache (which does not yet support writing to shared mappings), a bug -// caused reads to fail unnecessarily on such mappings. +// caused reads to fail unnecessarily on such mappings. See b/28913513. TEST_F(MMapFileTest, ReadingWritableSharedFilePageSucceeds) { uintptr_t addr; size_t len = strlen(kFileContents); @@ -1435,7 +1435,7 @@ TEST_F(MMapFileTest, ReadingWritableSharedFilePageSucceeds) { // Tests that EFAULT is returned when invoking a syscall that requires the OS to // read past end of file (resulting in a fault in sentry context in the gVisor -// case). +// case). See b/28913513. TEST_F(MMapFileTest, InternalSigBus) { uintptr_t addr; ASSERT_THAT(addr = Map(0, 2 * kPageSize, PROT_READ | PROT_WRITE, MAP_PRIVATE, @@ -1578,7 +1578,7 @@ TEST_F(MMapFileTest, Bug38498194) { } // Tests that reading from a file to a memory mapping of the same file does not -// deadlock. +// deadlock. See b/34813270. TEST_F(MMapFileTest, SelfRead) { uintptr_t addr; ASSERT_THAT(addr = Map(0, kPageSize, PROT_READ | PROT_WRITE, MAP_SHARED, @@ -1590,7 +1590,7 @@ TEST_F(MMapFileTest, SelfRead) { } // Tests that writing to a file from a memory mapping of the same file does not -// deadlock. +// deadlock. Regression test for b/34813270. TEST_F(MMapFileTest, SelfWrite) { uintptr_t addr; ASSERT_THAT(addr = Map(0, kPageSize, PROT_READ, MAP_SHARED, fd_.get(), 0), diff --git a/test/syscalls/linux/open_create.cc b/test/syscalls/linux/open_create.cc index 431733dbe..902d0a0dc 100644 --- a/test/syscalls/linux/open_create.cc +++ b/test/syscalls/linux/open_create.cc @@ -132,6 +132,7 @@ TEST(CreateTest, CreateFailsOnDirWithoutWritePerms) { } // A file originally created RW, but opened RO can later be opened RW. +// Regression test for b/65385065. TEST(CreateTest, OpenCreateROThenRW) { TempPath file(NewTempAbsPath()); diff --git a/test/syscalls/linux/preadv.cc b/test/syscalls/linux/preadv.cc index f7ea44054..5b0743fe9 100644 --- a/test/syscalls/linux/preadv.cc +++ b/test/syscalls/linux/preadv.cc @@ -37,6 +37,7 @@ namespace testing { namespace { +// Stress copy-on-write. Attempts to reproduce b/38430174. TEST(PreadvTest, MMConcurrencyStress) { // Fill a one-page file with zeroes (the contents don't really matter). const auto f = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateFileWith( diff --git a/test/syscalls/linux/proc.cc b/test/syscalls/linux/proc.cc index 169b723eb..a23fdb58d 100644 --- a/test/syscalls/linux/proc.cc +++ b/test/syscalls/linux/proc.cc @@ -1352,13 +1352,19 @@ TEST(ProcPidSymlink, SubprocessZombied) { // FIXME(gvisor.dev/issue/164): Inconsistent behavior between gVisor and linux // on proc files. - // 4.17 & gVisor: Syscall succeeds and returns 1 + // + // ~4.3: Syscall fails with EACCES. + // 4.17 & gVisor: Syscall succeeds and returns 1. + // // EXPECT_THAT(ReadlinkWhileZombied("ns/pid", buf, sizeof(buf)), // SyscallFailsWithErrno(EACCES)); // FIXME(gvisor.dev/issue/164): Inconsistent behavior between gVisor and linux // on proc files. - // 4.17 & gVisor: Syscall succeeds and returns 1. + // + // ~4.3: Syscall fails with EACCES. + // 4.17 & gVisor: Syscall succeeds and returns 1. + // // EXPECT_THAT(ReadlinkWhileZombied("ns/user", buf, sizeof(buf)), // SyscallFailsWithErrno(EACCES)); } @@ -1431,8 +1437,12 @@ TEST(ProcPidFile, SubprocessRunning) { TEST(ProcPidFile, SubprocessZombie) { char buf[1]; - // 4.17: Succeeds and returns 1 - // gVisor: Succeeds and returns 0 + // FIXME(gvisor.dev/issue/164): Loosen requirement due to inconsistent + // behavior on different kernels. + // + // ~4.3: Succeds and returns 0. + // 4.17: Succeeds and returns 1. + // gVisor: Succeeds and returns 0. EXPECT_THAT(ReadWhileZombied("auxv", buf, sizeof(buf)), SyscallSucceeds()); EXPECT_THAT(ReadWhileZombied("cmdline", buf, sizeof(buf)), @@ -1458,7 +1468,10 @@ TEST(ProcPidFile, SubprocessZombie) { // FIXME(gvisor.dev/issue/164): Inconsistent behavior between gVisor and linux // on proc files. + // + // ~4.3: Fails and returns EACCES. // gVisor & 4.17: Succeeds and returns 1. + // // EXPECT_THAT(ReadWhileZombied("io", buf, sizeof(buf)), // SyscallFailsWithErrno(EACCES)); } @@ -1467,9 +1480,12 @@ TEST(ProcPidFile, SubprocessZombie) { TEST(ProcPidFile, SubprocessExited) { char buf[1]; - // FIXME(gvisor.dev/issue/164): Inconsistent behavior between kernels + // FIXME(gvisor.dev/issue/164): Inconsistent behavior between kernels. + // + // ~4.3: Fails and returns ESRCH. // gVisor: Fails with ESRCH. // 4.17: Succeeds and returns 1. + // // EXPECT_THAT(ReadWhileExited("auxv", buf, sizeof(buf)), // SyscallFailsWithErrno(ESRCH)); @@ -1641,7 +1657,7 @@ TEST(ProcTask, KilledThreadsDisappear) { EXPECT_NO_ERRNO(DirContainsExactly("/proc/self/task", TaskFiles(initial, {child1.Tid()}))); - // Stat child1's task file. + // Stat child1's task file. Regression test for b/32097707. struct stat statbuf; const std::string child1_task_file = absl::StrCat("/proc/self/task/", child1.Tid()); @@ -1669,7 +1685,7 @@ TEST(ProcTask, KilledThreadsDisappear) { EXPECT_NO_ERRNO(EventuallyDirContainsExactly( "/proc/self/task", TaskFiles(initial, {child3.Tid(), child5.Tid()}))); - // Stat child1's task file again. This time it should fail. + // Stat child1's task file again. This time it should fail. See b/32097707. EXPECT_THAT(stat(child1_task_file.c_str(), &statbuf), SyscallFailsWithErrno(ENOENT)); @@ -1824,7 +1840,7 @@ TEST(ProcSysVmOvercommitMemory, HasNumericValue) { } // Check that link for proc fd entries point the target node, not the -// symlink itself. +// symlink itself. Regression test for b/31155070. TEST(ProcTaskFd, FstatatFollowsSymlink) { const TempPath file = ASSERT_NO_ERRNO_AND_VALUE(TempPath::CreateFile()); const FileDescriptor fd = @@ -1883,6 +1899,20 @@ TEST(ProcMounts, IsSymlink) { EXPECT_EQ(link, "self/mounts"); } +TEST(ProcSelfMountinfo, RequiredFieldsArePresent) { + auto mountinfo = + ASSERT_NO_ERRNO_AND_VALUE(GetContents("/proc/self/mountinfo")); + EXPECT_THAT( + mountinfo, + AllOf( + // Root mount. + ContainsRegex( + R"([0-9]+ [0-9]+ [0-9]+:[0-9]+ / / (rw|ro).*- \S+ \S+ (rw|ro)\S*)"), + // Proc mount - always rw. + ContainsRegex( + R"([0-9]+ [0-9]+ [0-9]+:[0-9]+ / /proc rw.*- \S+ \S+ rw\S*)"))); +} + // Check that /proc/self/mounts looks something like a real mounts file. TEST(ProcSelfMounts, RequiredFieldsArePresent) { auto mounts = ASSERT_NO_ERRNO_AND_VALUE(GetContents("/proc/self/mounts")); diff --git a/test/syscalls/linux/readv.cc b/test/syscalls/linux/readv.cc index 4069cbc7e..baaf9f757 100644 --- a/test/syscalls/linux/readv.cc +++ b/test/syscalls/linux/readv.cc @@ -254,7 +254,9 @@ TEST_F(ReadvTest, IovecOutsideTaskAddressRangeInNonemptyArray) { // This test depends on the maximum extent of a single readv() syscall, so // we can't tolerate interruption from saving. TEST(ReadvTestNoFixture, TruncatedAtMax_NoRandomSave) { - // Ensure that we won't be interrupted by ITIMER_PROF. + // Ensure that we won't be interrupted by ITIMER_PROF. This is particularly + // important in environments where automated profiling tools may start + // ITIMER_PROF automatically. struct itimerval itv = {}; auto const cleanup_itimer = ASSERT_NO_ERRNO_AND_VALUE(ScopedItimer(ITIMER_PROF, itv)); diff --git a/test/syscalls/linux/rseq.cc b/test/syscalls/linux/rseq.cc index 106c045e3..4bfb1ff56 100644 --- a/test/syscalls/linux/rseq.cc +++ b/test/syscalls/linux/rseq.cc @@ -36,7 +36,7 @@ namespace { // We must be very careful about how these tests are written. Each thread may // only have one struct rseq registration, which may be done automatically at // thread start (as of 2019-11-13, glibc does *not* support rseq and thus does -// not do so). +// not do so, but other libraries do). // // Testing of rseq is thus done primarily in a child process with no // registration. This means exec'ing a nostdlib binary, as rseq registration can diff --git a/test/syscalls/linux/select.cc b/test/syscalls/linux/select.cc index 424e2a67f..be2364fb8 100644 --- a/test/syscalls/linux/select.cc +++ b/test/syscalls/linux/select.cc @@ -146,7 +146,7 @@ TEST_F(SelectTest, IgnoreBitsAboveNfds) { // This test illustrates Linux's behavior of 'select' calls passing after // setrlimit RLIMIT_NOFILE is called. In particular, versions of sshd rely on -// this behavior. +// this behavior. See b/122318458. TEST_F(SelectTest, SetrlimitCallNOFILE) { fd_set read_set; FD_ZERO(&read_set); diff --git a/test/syscalls/linux/shm.cc b/test/syscalls/linux/shm.cc index 7ba752599..c7fdbb924 100644 --- a/test/syscalls/linux/shm.cc +++ b/test/syscalls/linux/shm.cc @@ -473,7 +473,7 @@ TEST(ShmTest, PartialUnmap) { } // Check that sentry does not panic when asked for a zero-length private shm -// segment. +// segment. Regression test for b/110694797. TEST(ShmTest, GracefullyFailOnZeroLenSegmentCreation) { EXPECT_THAT(Shmget(IPC_PRIVATE, 0, 0), PosixErrorIs(EINVAL, _)); } diff --git a/test/syscalls/linux/sigprocmask.cc b/test/syscalls/linux/sigprocmask.cc index 654c6a47f..a603fc1d1 100644 --- a/test/syscalls/linux/sigprocmask.cc +++ b/test/syscalls/linux/sigprocmask.cc @@ -237,7 +237,7 @@ TEST_F(SigProcMaskTest, SignalHandler) { } // Check that sigprocmask correctly handles aliasing of the set and oldset -// pointers. +// pointers. Regression test for b/30502311. TEST_F(SigProcMaskTest, AliasedSets) { sigset_t mask; diff --git a/test/syscalls/linux/socket_unix_non_stream.cc b/test/syscalls/linux/socket_unix_non_stream.cc index 276a94eb8..884319e1d 100644 --- a/test/syscalls/linux/socket_unix_non_stream.cc +++ b/test/syscalls/linux/socket_unix_non_stream.cc @@ -109,7 +109,7 @@ PosixErrorOr> CreateFragmentedRegion(const int size, } // A contiguous iov that is heavily fragmented in FileMem can still be sent -// successfully. +// successfully. See b/115833655. TEST_P(UnixNonStreamSocketPairTest, FragmentedSendMsg) { auto sockets = ASSERT_NO_ERRNO_AND_VALUE(NewSocketPair()); @@ -165,7 +165,7 @@ TEST_P(UnixNonStreamSocketPairTest, FragmentedSendMsg) { } // A contiguous iov that is heavily fragmented in FileMem can still be received -// into successfully. +// into successfully. Regression test for b/115833655. TEST_P(UnixNonStreamSocketPairTest, FragmentedRecvMsg) { auto sockets = ASSERT_NO_ERRNO_AND_VALUE(NewSocketPair()); diff --git a/test/syscalls/linux/symlink.cc b/test/syscalls/linux/symlink.cc index b249ff91f..03ee1250d 100644 --- a/test/syscalls/linux/symlink.cc +++ b/test/syscalls/linux/symlink.cc @@ -38,7 +38,7 @@ mode_t FilePermission(const std::string& path) { } // Test that name collisions are checked on the new link path, not the source -// path. +// path. Regression test for b/31782115. TEST(SymlinkTest, CanCreateSymlinkWithCachedSourceDirent) { const std::string srcname = NewTempAbsPath(); const std::string newname = NewTempAbsPath(); diff --git a/test/syscalls/linux/tcp_socket.cc b/test/syscalls/linux/tcp_socket.cc index 8a8b68e75..c4591a3b9 100644 --- a/test/syscalls/linux/tcp_socket.cc +++ b/test/syscalls/linux/tcp_socket.cc @@ -244,7 +244,8 @@ TEST_P(TcpSocketTest, ZeroWriteAllowed) { } // Test that a non-blocking write with a buffer that is larger than the send -// buffer size will not actually write the whole thing at once. +// buffer size will not actually write the whole thing at once. Regression test +// for b/64438887. TEST_P(TcpSocketTest, NonblockingLargeWrite) { // Set the FD to O_NONBLOCK. int opts; diff --git a/test/syscalls/linux/time.cc b/test/syscalls/linux/time.cc index c7eead17e..1ccb95733 100644 --- a/test/syscalls/linux/time.cc +++ b/test/syscalls/linux/time.cc @@ -62,6 +62,7 @@ TEST(TimeTest, VsyscallTime_InvalidAddressSIGSEGV) { ::testing::KilledBySignal(SIGSEGV), ""); } +// Mimics the gettimeofday(2) wrapper from the Go runtime <= 1.2. int vsyscall_gettimeofday(struct timeval* tv, struct timezone* tz) { constexpr uint64_t kVsyscallGettimeofdayEntry = 0xffffffffff600000; return reinterpret_cast( diff --git a/test/syscalls/linux/tkill.cc b/test/syscalls/linux/tkill.cc index bae377c69..8d8ebbb24 100644 --- a/test/syscalls/linux/tkill.cc +++ b/test/syscalls/linux/tkill.cc @@ -54,7 +54,7 @@ void SigHandler(int sig, siginfo_t* info, void* context) { TEST_CHECK(info->si_code == SI_TKILL); } -// Test with a real signal. +// Test with a real signal. Regression test for b/24790092. TEST(TkillTest, ValidTIDAndRealSignal) { struct sigaction sa; sa.sa_sigaction = SigHandler; diff --git a/test/util/temp_path.cc b/test/util/temp_path.cc index 35aacb172..9c10b6674 100644 --- a/test/util/temp_path.cc +++ b/test/util/temp_path.cc @@ -77,6 +77,7 @@ std::string NewTempAbsPath() { std::string NewTempRelPath() { return NextTempBasename(); } std::string GetAbsoluteTestTmpdir() { + // Note that TEST_TMPDIR is guaranteed to be set. char* env_tmpdir = getenv("TEST_TMPDIR"); std::string tmp_dir = env_tmpdir != nullptr ? std::string(env_tmpdir) : "/tmp"; diff --git a/tools/build/tags.bzl b/tools/build/tags.bzl index e99c87f81..a6db44e47 100644 --- a/tools/build/tags.bzl +++ b/tools/build/tags.bzl @@ -33,4 +33,8 @@ go_suffixes = [ "_wasm_unsafe", "_linux", "_linux_unsafe", + "_opts", + "_opts_unsafe", + "_impl", + "_impl_unsafe", ] diff --git a/tools/defs.bzl b/tools/defs.bzl index 5d5fa134a..c03b557ae 100644 --- a/tools/defs.bzl +++ b/tools/defs.bzl @@ -73,6 +73,16 @@ def calculate_sets(srcs): result[target].append(file) return result +def go_imports(name, src, out): + """Simplify a single Go source file by eliminating unused imports.""" + native.genrule( + name = name, + srcs = [src], + outs = [out], + tools = ["@org_golang_x_tools//cmd/goimports:goimports"], + cmd = ("$(location @org_golang_x_tools//cmd/goimports:goimports) $(SRCS) > $@"), + ) + def go_library(name, srcs, deps = [], imports = [], stateify = True, marshal = False, **kwargs): """Wraps the standard go_library and does stateification and marshalling. @@ -107,10 +117,15 @@ def go_library(name, srcs, deps = [], imports = [], stateify = True, marshal = F state_sets = calculate_sets(srcs) for (suffix, srcs) in state_sets.items(): go_stateify( - name = name + suffix + "_state_autogen", + name = name + suffix + "_state_autogen_with_imports", srcs = srcs, imports = imports, package = name, + out = name + suffix + "_state_autogen_with_imports.go", + ) + go_imports( + name = name + suffix + "_state_autogen", + src = name + suffix + "_state_autogen_with_imports.go", out = name + suffix + "_state_autogen.go", ) all_srcs = all_srcs + [ -- cgit v1.2.3 From c615aafa219e8d9783b9c9a25252e4973de57d4a Mon Sep 17 00:00:00 2001 From: Adin Scannell Date: Mon, 20 Apr 2020 20:57:02 -0700 Subject: Add internal nogo analysis & checkescape tool. See tools/nogo/README.md. The checkescape tool is able to perform recursive escape analysis, using the actual generated binary to confirm the results produced by the compiler itself. As an initial use case, this replaces the manual escape analysis tests used for go_marshal, and validates that the CopyIn and CopyOut paths will not require any allocation or stack splits. Updates #2243 PiperOrigin-RevId: 307532986 --- BUILD | 33 - WORKSPACE | 33 +- pkg/metric/metric.go | 2 +- tools/BUILD | 2 - tools/bazeldefs/defs.bzl | 41 +- tools/checkescape/BUILD | 16 + tools/checkescape/checkescape.go | 726 +++++++++++++++++++++ tools/checkescape/test1/BUILD | 9 + tools/checkescape/test1/test1.go | 195 ++++++ tools/checkescape/test2/BUILD | 9 + tools/checkescape/test2/test2.go | 94 +++ tools/checkunsafe/BUILD | 7 +- tools/defs.bzl | 12 +- .../generator_interfaces_array_newtype.go | 9 +- .../generator_interfaces_primitive_newtype.go | 15 +- .../gomarshal/generator_interfaces_struct.go | 20 +- tools/go_marshal/test/BUILD | 15 +- tools/go_marshal/test/escape.go | 114 ---- tools/go_marshal/test/escape/BUILD | 14 + tools/go_marshal/test/escape/escape.go | 95 +++ tools/nogo.json | 39 -- tools/nogo/BUILD | 49 ++ tools/nogo/README.md | 31 + tools/nogo/build.go | 36 + tools/nogo/check/BUILD | 12 + tools/nogo/check/main.go | 24 + tools/nogo/config.go | 113 ++++ tools/nogo/data/BUILD | 10 + tools/nogo/data/data.go | 21 + tools/nogo/defs.bzl | 172 +++++ tools/nogo/io_bazel_rules_go-visibility.patch | 25 + tools/nogo/matchers.go | 138 ++++ tools/nogo/nogo.go | 316 +++++++++ tools/nogo/register.go | 64 ++ 34 files changed, 2269 insertions(+), 242 deletions(-) create mode 100644 tools/checkescape/BUILD create mode 100644 tools/checkescape/checkescape.go create mode 100644 tools/checkescape/test1/BUILD create mode 100644 tools/checkescape/test1/test1.go create mode 100644 tools/checkescape/test2/BUILD create mode 100644 tools/checkescape/test2/test2.go delete mode 100644 tools/go_marshal/test/escape.go create mode 100644 tools/go_marshal/test/escape/BUILD create mode 100644 tools/go_marshal/test/escape/escape.go delete mode 100644 tools/nogo.json create mode 100644 tools/nogo/BUILD create mode 100644 tools/nogo/README.md create mode 100644 tools/nogo/build.go create mode 100644 tools/nogo/check/BUILD create mode 100644 tools/nogo/check/main.go create mode 100644 tools/nogo/config.go create mode 100644 tools/nogo/data/BUILD create mode 100644 tools/nogo/data/data.go create mode 100644 tools/nogo/defs.bzl create mode 100644 tools/nogo/io_bazel_rules_go-visibility.patch create mode 100644 tools/nogo/matchers.go create mode 100644 tools/nogo/nogo.go create mode 100644 tools/nogo/register.go (limited to 'pkg/metric/metric.go') diff --git a/BUILD b/BUILD index a709a9816..c010e2131 100644 --- a/BUILD +++ b/BUILD @@ -44,39 +44,6 @@ go_path( # bazel run //:gazelle -- update-repos -from_file=go.mod gazelle(name = "gazelle") -# nogo applies checks to all Go source in this repository, enforcing code -# guidelines and restrictions. Note that the tool libraries themselves should -# live in the tools subdirectory (unless they are standard). -nogo( - name = "nogo", - config = "//tools:nogo.json", - visibility = ["//visibility:public"], - deps = [ - "//tools/checkunsafe", - "@org_golang_x_tools//go/analysis/passes/asmdecl:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/assign:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/atomic:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/atomicalign:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/bools:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/buildtag:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/cgocall:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/copylock:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/deepequalerrors:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/loopclosure:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/lostcancel:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/nilfunc:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/nilness:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/printf:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/shift:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/stdmethods:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/structtag:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/tests:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/unmarshal:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/unsafeptr:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/unusedresult:go_tool_library", - ], -) - # We need to define a bazel platform and toolchain to specify dockerPrivileged # and dockerRunAsRoot options, they are required to run tests on the RBE # cluster in Kokoro. diff --git a/WORKSPACE b/WORKSPACE index c40e03ad2..b895647fb 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -2,8 +2,16 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") # Load go bazel rules and gazelle. +# +# Note that this repository actually patches some other Go repositories as it +# loads it, in order to limit visibility. We hack this process by patching the +# patch used by the Go rules, turning the trick against itself. http_archive( name = "io_bazel_rules_go", + patch_args = ["-p1"], + patches = [ + "//tools/nogo:io_bazel_rules_go-visibility.patch", + ], sha256 = "db2b2d35293f405430f553bc7a865a8749a8ef60c30287e90d2b278c32771afe", urls = [ "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.22.3/rules_go-v0.22.3.tar.gz", @@ -24,10 +32,7 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe go_rules_dependencies() -go_register_toolchains( - go_version = "1.14.2", - nogo = "@//:nogo", -) +go_register_toolchains(go_version = "1.14.2") load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") @@ -170,9 +175,13 @@ http_archive( "https://github.com/grpc/grpc/archive/v1.26.0.tar.gz", ], ) + load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") + grpc_deps() + load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps") + grpc_extra_deps() # External repositories, in sorted order. @@ -221,8 +230,8 @@ go_repository( go_repository( name = "com_github_imdario_mergo", importpath = "github.com/imdario/mergo", - version = "v0.3.8", sum = "h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=", + version = "v0.3.8", ) go_repository( @@ -248,8 +257,8 @@ go_repository( go_repository( name = "com_github_mohae_deepcopy", - importpath = "github.com/mohae/deepcopy", commit = "c48cc78d482608239f6c4c92a4abd87eb8761c90", + importpath = "github.com/mohae/deepcopy", ) go_repository( @@ -298,8 +307,8 @@ go_repository( go_repository( name = "org_golang_x_crypto", importpath = "golang.org/x/crypto", - sum = "h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=", - version = "v0.0.0-20191011191535-87dc89f01550", + sum = "h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=", + version = "v0.0.0-20190308221718-c2843e01d9a2", ) go_repository( @@ -340,15 +349,15 @@ go_repository( go_repository( name = "org_golang_x_tools", importpath = "golang.org/x/tools", - sum = "h1:aZzprAO9/8oim3qStq3wc1Xuxx4QmAGriC4VU4ojemQ=", - version = "v0.0.0-20191119224855-298f0cb1881e", + sum = "h1:Uglradbb4KfUWaYasZhlsDsGRwHHvRsHoNAEONef0W8=", + version = "v0.0.0-20200131233409-575de47986ce", ) go_repository( name = "org_golang_x_xerrors", importpath = "golang.org/x/xerrors", - sum = "h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=", - version = "v0.0.0-20191204190536-9bdfabe68543", + sum = "h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=", + version = "v0.0.0-20190717185122-a985d3407aa7", ) go_repository( diff --git a/pkg/metric/metric.go b/pkg/metric/metric.go index 006fcd9ab..895253625 100644 --- a/pkg/metric/metric.go +++ b/pkg/metric/metric.go @@ -244,6 +244,6 @@ func EmitMetricUpdate() { return } - log.Debugf("Emitting metrics: %v", m) + log.Debugf("Emitting metrics: %v", &m) eventchannel.Emit(&m) } diff --git a/tools/BUILD b/tools/BUILD index ba3506c04..34b950644 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -1,3 +1 @@ package(licenses = ["notice"]) - -exports_files(["nogo.json"]) diff --git a/tools/bazeldefs/defs.bzl b/tools/bazeldefs/defs.bzl index 0a74370a6..2207b9b34 100644 --- a/tools/bazeldefs/defs.bzl +++ b/tools/bazeldefs/defs.bzl @@ -1,7 +1,7 @@ """Bazel implementations of standard rules.""" load("@bazel_tools//tools/cpp:cc_flags_supplier.bzl", _cc_flags_supplier = "cc_flags_supplier") -load("@io_bazel_rules_go//go:def.bzl", _go_binary = "go_binary", _go_embed_data = "go_embed_data", _go_library = "go_library", _go_test = "go_test", _go_tool_library = "go_tool_library") +load("@io_bazel_rules_go//go:def.bzl", "GoLibrary", _go_binary = "go_binary", _go_context = "go_context", _go_embed_data = "go_embed_data", _go_library = "go_library", _go_test = "go_test") load("@io_bazel_rules_go//proto:def.bzl", _go_grpc_library = "go_grpc_library", _go_proto_library = "go_proto_library") load("@rules_cc//cc:defs.bzl", _cc_binary = "cc_binary", _cc_library = "cc_library", _cc_proto_library = "cc_proto_library", _cc_test = "cc_test") load("@rules_pkg//:pkg.bzl", _pkg_deb = "pkg_deb", _pkg_tar = "pkg_tar") @@ -99,6 +99,10 @@ def go_binary(name, static = False, pure = False, **kwargs): **kwargs ) +def go_importpath(target): + """Returns the importpath for the target.""" + return target[GoLibrary].importpath + def go_library(name, **kwargs): _go_library( name = name, @@ -106,13 +110,6 @@ def go_library(name, **kwargs): **kwargs ) -def go_tool_library(name, **kwargs): - _go_tool_library( - name = name, - importpath = "gvisor.dev/gvisor/" + native.package_name(), - **kwargs - ) - def go_test(name, pure = False, library = None, **kwargs): """Build a go test. @@ -131,6 +128,34 @@ def go_test(name, pure = False, library = None, **kwargs): **kwargs ) +def go_rule(rule, implementation, **kwargs): + """Wraps a rule definition with Go attributes. + + Args: + rule: rule function (typically rule or aspect). + implementation: implementation function. + **kwargs: other arguments to pass to rule. + + Returns: + The result of invoking the rule. + """ + attrs = kwargs.pop("attrs", []) + attrs["_go_context_data"] = attr.label(default = "@io_bazel_rules_go//:go_context_data") + attrs["_stdlib"] = attr.label(default = "@io_bazel_rules_go//:stdlib") + toolchains = kwargs.get("toolchains", []) + ["@io_bazel_rules_go//go:toolchain"] + return rule(implementation, attrs = attrs, toolchains = toolchains, **kwargs) + +def go_context(ctx): + go_ctx = _go_context(ctx) + return struct( + go = go_ctx.go, + env = go_ctx.env, + runfiles = depset([go_ctx.go] + go_ctx.sdk.tools + go_ctx.stdlib.libs), + goos = go_ctx.sdk.goos, + goarch = go_ctx.sdk.goarch, + tags = go_ctx.tags, + ) + def py_requirement(name, direct = True): return _py_requirement(name) diff --git a/tools/checkescape/BUILD b/tools/checkescape/BUILD new file mode 100644 index 000000000..b8c3ddf44 --- /dev/null +++ b/tools/checkescape/BUILD @@ -0,0 +1,16 @@ +load("//tools:defs.bzl", "go_library") + +package(licenses = ["notice"]) + +go_library( + name = "checkescape", + srcs = ["checkescape.go"], + nogo = False, + visibility = ["//tools/nogo:__subpackages__"], + deps = [ + "//tools/nogo/data", + "@org_golang_x_tools//go/analysis:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/buildssa:go_tool_library", + "@org_golang_x_tools//go/ssa:go_tool_library", + ], +) diff --git a/tools/checkescape/checkescape.go b/tools/checkescape/checkescape.go new file mode 100644 index 000000000..571e9a6e6 --- /dev/null +++ b/tools/checkescape/checkescape.go @@ -0,0 +1,726 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package checkescape allows recursive escape analysis for hot paths. +// +// The analysis tracks multiple types of escapes, in two categories. First, +// 'hard' escapes are explicit allocations. Second, 'soft' escapes are +// interface dispatches or dynamic function dispatches; these don't necessarily +// escape but they *may* escape. The analysis is capable of making assertions +// recursively: soft escapes cannot be analyzed in this way, and therefore +// count as escapes for recursive purposes. +// +// The different types of escapes are as follows, with the category in +// parentheses: +// +// heap: A direct allocation is made on the heap (hard). +// builtin: A call is made to a built-in allocation function (hard). +// stack: A stack split as part of a function preamble (soft). +// interface: A call is made via an interface whicy *may* escape (soft). +// dynamic: A dynamic function is dispatched which *may* escape (soft). +// +// To the use the package, annotate a function-level comment with either the +// line "// +checkescape" or "// +checkescape:OPTION[,OPTION]". In the second +// case, the OPTION field is either a type above, or one of: +// +// local: Escape analysis is limited to local hard escapes only. +// all: All the escapes are included. +// hard: All hard escapes are included. +// +// If the "// +checkescape" annotation is provided, this is equivalent to +// provided the local and hard options. +// +// Some examples of this syntax are: +// +// +checkescape:all - Analyzes for all escapes in this function and all calls. +// +checkescape:local - Analyzes only for default local hard escapes. +// +checkescape:heap - Only analyzes for heap escapes. +// +checkescape:interface,dynamic - Only checks for dynamic calls and interface calls. +// +checkescape - Does the same as +checkescape:local,hard. +// +// Note that all of the above can be inverted by using +mustescape. The +// +checkescape keyword will ensure failure if the class of escape occurs, +// whereas +mustescape will fail if the given class of escape does not occur. +// +// Local exemptions can be made by a comment of the form "// escapes: reason." +// This must appear on the line of the escape and will also apply to callers of +// the function as well (for non-local escape analysis). +package checkescape + +import ( + "bufio" + "bytes" + "fmt" + "go/ast" + "go/token" + "go/types" + "io" + "os" + "path/filepath" + "strconv" + "strings" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/buildssa" + "golang.org/x/tools/go/ssa" + "gvisor.dev/gvisor/tools/nogo/data" +) + +const ( + // magic is the magic annotation. + magic = "// +checkescape" + + // magicParams is the magic annotation with specific parameters. + magicParams = magic + ":" + + // testMagic is the test magic annotation (parameters required). + testMagic = "// +mustescape:" + + // exempt is the exemption annotation. + exempt = "// escapes:" +) + +// escapingBuiltins are builtins known to escape. +// +// These are lowered at an earlier stage of compilation to explicit function +// calls, but are not available for recursive analysis. +var escapingBuiltins = []string{ + "append", + "makemap", + "newobject", + "mallocgc", +} + +// Analyzer defines the entrypoint. +var Analyzer = &analysis.Analyzer{ + Name: "checkescape", + Doc: "surfaces recursive escape analysis results", + Run: run, + Requires: []*analysis.Analyzer{buildssa.Analyzer}, + FactTypes: []analysis.Fact{(*packageEscapeFacts)(nil)}, +} + +// packageEscapeFacts is the set of all functions in a package, and whether or +// not they recursively pass escape analysis. +// +// All the type names for receivers are encoded in the full key. The key +// represents the fully qualified package and type name used at link time. +type packageEscapeFacts struct { + Funcs map[string][]Escape +} + +// AFact implements analysis.Fact.AFact. +func (*packageEscapeFacts) AFact() {} + +// CallSite is a single call site. +// +// These can be chained. +type CallSite struct { + LocalPos token.Pos + Resolved LinePosition +} + +// Escape is a single escape instance. +type Escape struct { + Reason EscapeReason + Detail string + Chain []CallSite +} + +// LinePosition is a low-resolution token.Position. +// +// This is used to match against possible exemptions placed in the source. +type LinePosition struct { + Filename string + Line int +} + +// String implements fmt.Stringer.String. +func (e *LinePosition) String() string { + return fmt.Sprintf("%s:%d", e.Filename, e.Line) +} + +// String implements fmt.Stringer.String. +// +// Note that this string will contain new lines. +func (e *Escape) String() string { + var b bytes.Buffer + fmt.Fprintf(&b, "%s", e.Reason.String()) + for i, cs := range e.Chain { + if i == len(e.Chain)-1 { + fmt.Fprintf(&b, "\n @ %s → %s", cs.Resolved.String(), e.Detail) + } else { + fmt.Fprintf(&b, "\n + %s", cs.Resolved.String()) + } + } + return b.String() +} + +// EscapeReason is an escape reason. +// +// This is a simple enum. +type EscapeReason int + +const ( + interfaceInvoke EscapeReason = iota + unknownPackage + allocation + builtin + dynamicCall + stackSplit + reasonCount // Count for below. +) + +// String returns the string for the EscapeReason. +// +// Note that this also implicitly defines the reverse string -> EscapeReason +// mapping, which is the word before the colon (computed below). +func (e EscapeReason) String() string { + switch e { + case interfaceInvoke: + return "interface: function invocation via interface" + case unknownPackage: + return "unknown: no package information available" + case allocation: + return "heap: call to runtime heap allocation" + case builtin: + return "builtin: call to runtime builtin" + case dynamicCall: + return "dynamic: call via dynamic function" + case stackSplit: + return "stack: stack split on function entry" + default: + panic(fmt.Sprintf("unknown reason: %d", e)) + } +} + +var hardReasons = []EscapeReason{ + allocation, + builtin, +} + +var softReasons = []EscapeReason{ + interfaceInvoke, + unknownPackage, + dynamicCall, + stackSplit, +} + +var allReasons = append(hardReasons, softReasons...) + +var escapeTypes = func() map[string]EscapeReason { + result := make(map[string]EscapeReason) + for _, r := range allReasons { + parts := strings.Split(r.String(), ":") + result[parts[0]] = r // Key before ':'. + } + return result +}() + +// EscapeCount counts escapes. +// +// It is used to avoid accumulating too many escapes for the same reason, for +// the same function. We limit each class to 3 instances (arbitrarily). +type EscapeCount struct { + byReason [reasonCount]uint32 +} + +// maxRecordsPerReason is the number of explicit records. +// +// See EscapeCount (and usage), and Record implementation. +const maxRecordsPerReason = 5 + +// Record records the reason or returns false if it should not be added. +func (ec *EscapeCount) Record(reason EscapeReason) bool { + ec.byReason[reason]++ + if ec.byReason[reason] > maxRecordsPerReason { + return false + } + return true +} + +// loadObjdump reads the objdump output. +// +// This records if there is a call any function for every source line. It is +// used only to remove false positives for escape analysis. The call will be +// elided if escape analysis is able to put the object on the heap exclusively. +func loadObjdump() (map[LinePosition]string, error) { + f, err := os.Open(data.Objdump) + if err != nil { + return nil, err + } + defer f.Close() + + // Build the map. + m := make(map[LinePosition]string) + r := bufio.NewReader(f) + var ( + lastField string + lastPos LinePosition + ) + for { + line, err := r.ReadString('\n') + if err != nil && err != io.EOF { + return nil, err + } + + // We recognize lines corresponding to actual code (not the + // symbol name or other metadata) and annotate them if they + // correspond to an explicit CALL instruction. We assume that + // the lack of a CALL for a given line is evidence that escape + // analysis has eliminated an allocation. + // + // Lines look like this (including the first space): + // gohacks_unsafe.go:33 0xa39 488b442408 MOVQ 0x8(SP), AX + if len(line) > 0 && line[0] == ' ' { + fields := strings.Fields(line) + if !strings.Contains(fields[3], "CALL") { + continue + } + + // Ignore strings containing duffzero, which is just + // used by stack allocations for types that are large + // enough to warrant Duff's device. + if strings.Contains(line, "runtime.duffzero") { + continue + } + + // Ignore the racefuncenter call, which is used for + // race builds. This does not escape. + if strings.Contains(line, "runtime.racefuncenter") { + continue + } + + // Calculate the filename and line. Note that per the + // example above, the filename is not a fully qualified + // base, just the basename (what we require). + if fields[0] != lastField { + parts := strings.SplitN(fields[0], ":", 2) + lineNum, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, err + } + lastPos = LinePosition{ + Filename: parts[0], + Line: int(lineNum), + } + lastField = fields[0] + } + if _, ok := m[lastPos]; ok { + continue // Already marked. + } + + // Save the actual call for the detail. + m[lastPos] = strings.Join(fields[3:], " ") + } + if err == io.EOF { + break + } + } + + return m, nil +} + +// poser is a type that implements Pos. +type poser interface { + Pos() token.Pos +} + +// run performs the analysis. +func run(pass *analysis.Pass) (interface{}, error) { + calls, err := loadObjdump() + if err != nil { + return nil, err + } + pef := packageEscapeFacts{ + Funcs: make(map[string][]Escape), + } + linePosition := func(inst, parent poser) LinePosition { + p := pass.Fset.Position(inst.Pos()) + if (p.Filename == "" || p.Line == 0) && parent != nil { + p = pass.Fset.Position(parent.Pos()) + } + return LinePosition{ + Filename: filepath.Base(p.Filename), + Line: p.Line, + } + } + hasCall := func(inst poser) (string, bool) { + p := linePosition(inst, nil) + s, ok := calls[p] + return s, ok + } + callSite := func(inst ssa.Instruction) CallSite { + return CallSite{ + LocalPos: inst.Pos(), + Resolved: linePosition(inst, inst.Parent()), + } + } + escapes := func(reason EscapeReason, detail string, inst ssa.Instruction, ec *EscapeCount) []Escape { + if !ec.Record(reason) { + return nil // Skip. + } + es := Escape{ + Reason: reason, + Detail: detail, + Chain: []CallSite{callSite(inst)}, + } + return []Escape{es} + } + resolve := func(sub []Escape, inst ssa.Instruction, ec *EscapeCount) (es []Escape) { + for _, e := range sub { + if !ec.Record(e.Reason) { + continue // Skip. + } + es = append(es, Escape{ + Reason: e.Reason, + Detail: e.Detail, + Chain: append([]CallSite{callSite(inst)}, e.Chain...), + }) + } + return es + } + state := pass.ResultOf[buildssa.Analyzer].(*buildssa.SSA) + + var loadFunc func(*ssa.Function) []Escape // Used below. + + analyzeInstruction := func(inst ssa.Instruction, ec *EscapeCount) []Escape { + switch x := inst.(type) { + case *ssa.Call: + if x.Call.IsInvoke() { + // This is an interface dispatch. There is no + // way to know if this is actually escaping or + // not, since we don't know the underlying + // type. + call, _ := hasCall(inst) + return escapes(interfaceInvoke, call, inst, ec) + } + switch x := x.Call.Value.(type) { + case *ssa.Function: + if x.Pkg == nil { + // Can't resolve the package. + return escapes(unknownPackage, "no package", inst, ec) + } + + // Atomic functions are instrinics. We can + // assume that they don't escape. + if x.Pkg.Pkg.Name() == "atomic" { + return nil + } + + // Is this a local function? If yes, call the + // function to load the local function. The + // local escapes are the escapes found in the + // local function. + if x.Pkg.Pkg == pass.Pkg { + return resolve(loadFunc(x), inst, ec) + } + + // Recursively collect information from + // the other analyzers. + var imp packageEscapeFacts + if !pass.ImportPackageFact(x.Pkg.Pkg, &imp) { + // Unable to import the dependency; we must + // declare these as escaping. + return escapes(unknownPackage, "no analysis", inst, ec) + } + + // The escapes of this instruction are the + // escapes of the called function directly. + return resolve(imp.Funcs[x.RelString(x.Pkg.Pkg)], inst, ec) + case *ssa.Builtin: + // Ignore elided escapes. + if _, has := hasCall(inst); !has { + return nil + } + + // Check if the builtin is escaping. + for _, name := range escapingBuiltins { + if x.Name() == name { + return escapes(builtin, name, inst, ec) + } + } + default: + // All dynamic calls are counted as soft + // escapes. They are similar to interface + // dispatches. We cannot actually look up what + // this refers to using static analysis alone. + call, _ := hasCall(inst) + return escapes(dynamicCall, call, inst, ec) + } + case *ssa.Alloc: + // Ignore non-heap allocations. + if !x.Heap { + return nil + } + + // Ignore elided escapes. + call, has := hasCall(inst) + if !has { + return nil + } + + // This is a real heap allocation. + return escapes(allocation, call, inst, ec) + case *ssa.MakeMap: + return escapes(builtin, "makemap", inst, ec) + case *ssa.MakeSlice: + return escapes(builtin, "makeslice", inst, ec) + case *ssa.MakeClosure: + return escapes(builtin, "makeclosure", inst, ec) + case *ssa.MakeChan: + return escapes(builtin, "makechan", inst, ec) + } + return nil // No escapes. + } + + var analyzeBasicBlock func(*ssa.BasicBlock, *EscapeCount) []Escape // Recursive. + analyzeBasicBlock = func(block *ssa.BasicBlock, ec *EscapeCount) (rval []Escape) { + for _, inst := range block.Instrs { + rval = append(rval, analyzeInstruction(inst, ec)...) + } + return rval // N.B. may be empty. + } + + loadFunc = func(fn *ssa.Function) []Escape { + // Is this already available? + name := fn.RelString(pass.Pkg) + if es, ok := pef.Funcs[name]; ok { + return es + } + + // In the case of a true cycle, we assume that the current + // function itself has no escapes until the rest of the + // analysis is complete. This will trip the above in the case + // of a cycle of any kind. + pef.Funcs[name] = nil + + // Perform the basic analysis. + var ( + es []Escape + ec EscapeCount + ) + if fn.Recover != nil { + es = append(es, analyzeBasicBlock(fn.Recover, &ec)...) + } + for _, block := range fn.Blocks { + es = append(es, analyzeBasicBlock(block, &ec)...) + } + + // Check for a stack split. + if call, has := hasCall(fn); has { + es = append(es, Escape{ + Reason: stackSplit, + Detail: call, + Chain: []CallSite{CallSite{ + LocalPos: fn.Pos(), + Resolved: linePosition(fn, fn.Parent()), + }}, + }) + } + + // Save the result and return. + pef.Funcs[name] = es + return es + } + + // Complete all local functions. + for _, fn := range state.SrcFuncs { + loadFunc(fn) + } + + // Build the exception list. + exemptions := make(map[LinePosition]string) + for _, f := range pass.Files { + for _, cg := range f.Comments { + for _, c := range cg.List { + p := pass.Fset.Position(c.Slash) + if strings.HasPrefix(c.Text, exempt) { + exemptions[LinePosition{ + Filename: filepath.Base(p.Filename), + Line: p.Line, + }] = c.Text[len(exempt):] + } + } + } + } + + // Delete everything matching the excemtions. + // + // This has the implication that exceptions are applied recursively, + // since this now modified set is what will be saved. + for name, escapes := range pef.Funcs { + var newEscapes []Escape + for _, escape := range escapes { + isExempt := false + for line, _ := range exemptions { + // Note that an exemption applies if it is + // marked as an exemption anywhere in the call + // chain. It need not be marked as escapes in + // the function itself, nor in the top-level + // caller. + for _, callSite := range escape.Chain { + if callSite.Resolved == line { + isExempt = true + break + } + } + if isExempt { + break + } + } + if !isExempt { + // Record this escape; not an exception. + newEscapes = append(newEscapes, escape) + } + } + pef.Funcs[name] = newEscapes // Update. + } + + // Export all findings for future packages. + pass.ExportPackageFact(&pef) + + // Scan all functions for violations. + for _, f := range pass.Files { + // Scan all declarations. + for _, decl := range f.Decls { + fdecl, ok := decl.(*ast.FuncDecl) + // Function declaration? + if !ok { + continue + } + // Is there a comment? + if fdecl.Doc == nil { + continue + } + var ( + reasons []EscapeReason + found bool + local bool + testReasons = make(map[EscapeReason]bool) // reason -> local? + ) + // Does the comment contain a +checkescape line? + for _, c := range fdecl.Doc.List { + if !strings.HasPrefix(c.Text, magic) && !strings.HasPrefix(c.Text, testMagic) { + continue + } + if c.Text == magic { + // Default: hard reasons, local only. + reasons = hardReasons + local = true + } else if strings.HasPrefix(c.Text, magicParams) { + // Extract specific reasons. + types := strings.Split(c.Text[len(magicParams):], ",") + found = true // For below. + for i := 0; i < len(types); i++ { + if types[i] == "local" { + // Limit search to local escapes. + local = true + } else if types[i] == "all" { + // Append all reasons. + reasons = append(reasons, allReasons...) + } else if types[i] == "hard" { + // Append all hard reasons. + reasons = append(reasons, hardReasons...) + } else { + r, ok := escapeTypes[types[i]] + if !ok { + // This is not a valid escape reason. + pass.Reportf(fdecl.Pos(), "unknown reason: %v", types[i]) + continue + } + reasons = append(reasons, r) + } + } + } else if strings.HasPrefix(c.Text, testMagic) { + types := strings.Split(c.Text[len(testMagic):], ",") + local := false + for i := 0; i < len(types); i++ { + if types[i] == "local" { + local = true + } else { + r, ok := escapeTypes[types[i]] + if !ok { + // This is not a valid escape reason. + pass.Reportf(fdecl.Pos(), "unknown reason: %v", types[i]) + continue + } + if v, ok := testReasons[r]; ok && v { + // Already registered as local. + continue + } + testReasons[r] = local + } + } + } + } + if len(reasons) == 0 && found { + // A magic annotation was provided, but no reasons. + pass.Reportf(fdecl.Pos(), "no reasons provided") + continue + } + + // Scan for matches. + fn := pass.TypesInfo.Defs[fdecl.Name].(*types.Func) + name := state.Pkg.Prog.FuncValue(fn).RelString(pass.Pkg) + es, ok := pef.Funcs[name] + if !ok { + pass.Reportf(fdecl.Pos(), "internal error: function %s not found.", name) + continue + } + for _, e := range es { + for _, r := range reasons { + // Is does meet our local requirement? + if local && len(e.Chain) > 1 { + continue + } + // Does this match the reason? Emit + // with a full stack trace that + // explains why this violates our + // constraints. + if e.Reason == r { + pass.Reportf(e.Chain[0].LocalPos, "%s", e.String()) + } + } + } + + // Scan for test (required) matches. + testReasonsFound := make(map[EscapeReason]bool) + for _, e := range es { + // Is this local? + local, ok := testReasons[e.Reason] + wantLocal := len(e.Chain) == 1 + testReasonsFound[e.Reason] = wantLocal + if !ok { + continue + } + if local == wantLocal { + delete(testReasons, e.Reason) + } + } + for reason, local := range testReasons { + // We didn't find the escapes we wanted. + pass.Reportf(fdecl.Pos(), fmt.Sprintf("testescapes not found: reason=%s, local=%t", reason, local)) + } + if len(testReasons) > 0 { + // Dump all reasons found to help in debugging. + for _, e := range es { + pass.Reportf(e.Chain[0].LocalPos, "escape found: %s", e.String()) + } + } + } + } + + return nil, nil +} diff --git a/tools/checkescape/test1/BUILD b/tools/checkescape/test1/BUILD new file mode 100644 index 000000000..783403247 --- /dev/null +++ b/tools/checkescape/test1/BUILD @@ -0,0 +1,9 @@ +load("//tools:defs.bzl", "go_library") + +package(licenses = ["notice"]) + +go_library( + name = "test1", + srcs = ["test1.go"], + visibility = ["//tools/checkescape/test2:__pkg__"], +) diff --git a/tools/checkescape/test1/test1.go b/tools/checkescape/test1/test1.go new file mode 100644 index 000000000..68d3f72cc --- /dev/null +++ b/tools/checkescape/test1/test1.go @@ -0,0 +1,195 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package test1 is a test package. +package test1 + +import ( + "fmt" + "reflect" +) + +// Interface is a generic interface. +type Interface interface { + Foo() +} + +// Type is a concrete implementation of Interface. +type Type struct { + A uint64 + B uint64 +} + +// Foo implements Interface.Foo. +//go:nosplit +func (t Type) Foo() { + fmt.Printf("%v", t) // Never executed. +} + +// +checkescape:all,hard +//go:nosplit +func InterfaceFunction(i Interface) { + // Do nothing; exported for tests. +} + +// +checkesacape:all,hard +//go:nosplit +func TypeFunction(t *Type) { +} + +// +mustescape:local,builtin +//go:noinline +//go:nosplit +func BuiltinMap(x int) map[string]bool { + return make(map[string]bool) +} + +// +mustescape:builtin +//go:noinline +//go:nosplit +func builtinMapRec(x int) map[string]bool { + return BuiltinMap(x) +} + +// +temustescapestescape:local,builtin +//go:noinline +//go:nosplit +func BuiltinClosure(x int) func() { + return func() { + fmt.Printf("%v", x) + } +} + +// +mustescape:builtin +//go:noinline +//go:nosplit +func builtinClosureRec(x int) func() { + return BuiltinClosure(x) +} + +// +mustescape:local,builtin +//go:noinline +//go:nosplit +func BuiltinMakeSlice(x int) []byte { + return make([]byte, x) +} + +// +mustescape:builtin +//go:noinline +//go:nosplit +func builtinMakeSliceRec(x int) []byte { + return BuiltinMakeSlice(x) +} + +// +mustescape:local,builtin +//go:noinline +//go:nosplit +func BuiltinAppend(x []byte) []byte { + return append(x, 0) +} + +// +mustescape:builtin +//go:noinline +//go:nosplit +func builtinAppendRec() []byte { + return BuiltinAppend(nil) +} + +// +mustescape:local,builtin +//go:noinline +//go:nosplit +func BuiltinChan() chan int { + return make(chan int) +} + +// +mustescape:builtin +//go:noinline +//go:nosplit +func builtinChanRec() chan int { + return BuiltinChan() +} + +// +mustescape:local,heap +//go:noinline +//go:nosplit +func Heap() *Type { + var t Type + return &t +} + +// +mustescape:heap +//go:noinline +//go:nosplit +func heapRec() *Type { + return Heap() +} + +// +mustescape:local,interface +//go:noinline +//go:nosplit +func Dispatch(i Interface) { + i.Foo() +} + +// +mustescape:interface +//go:noinline +//go:nosplit +func dispatchRec(i Interface) { + Dispatch(i) +} + +// +mustescape:local,dynamic +//go:noinline +//go:nosplit +func Dynamic(f func()) { + f() +} + +// +mustescape:dynamic +//go:noinline +//go:nosplit +func dynamicRec(f func()) { + Dynamic(f) +} + +// +mustescape:local,unknown +//go:noinline +//go:nosplit +func Unknown() { + _ = reflect.TypeOf((*Type)(nil)) // Does not actually escape. +} + +// +mustescape:unknown +//go:noinline +//go:nosplit +func unknownRec() { + Unknown() +} + +//go:noinline +//go:nosplit +func internalFunc() { +} + +// +mustescape:local,stack +//go:noinline +func Split() { + internalFunc() +} + +// +mustescape:stack +//go:noinline +func splitRec() { + Split() +} diff --git a/tools/checkescape/test2/BUILD b/tools/checkescape/test2/BUILD new file mode 100644 index 000000000..5a11e4b43 --- /dev/null +++ b/tools/checkescape/test2/BUILD @@ -0,0 +1,9 @@ +load("//tools:defs.bzl", "go_library") + +package(licenses = ["notice"]) + +go_library( + name = "test2", + srcs = ["test2.go"], + deps = ["//tools/checkescape/test1"], +) diff --git a/tools/checkescape/test2/test2.go b/tools/checkescape/test2/test2.go new file mode 100644 index 000000000..7fce3e3be --- /dev/null +++ b/tools/checkescape/test2/test2.go @@ -0,0 +1,94 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package test2 is a test package that imports test1. +package test2 + +import ( + "gvisor.dev/gvisor/tools/checkescape/test1" +) + +// +checkescape:all +//go:nosplit +func interfaceFunctionCrossPkg() { + var i test1.Interface + test1.InterfaceFunction(i) +} + +// +checkesacape:all +//go:nosplit +func typeFunctionCrossPkg() { + var t test1.Type + test1.TypeFunction(&t) +} + +// +mustescape:builtin +//go:noinline +func builtinMapCrossPkg(x int) map[string]bool { + return test1.BuiltinMap(x) +} + +// +mustescape:builtin +//go:noinline +func builtinClosureCrossPkg(x int) func() { + return test1.BuiltinClosure(x) +} + +// +mustescape:builtin +//go:noinline +func builtinMakeSliceCrossPkg(x int) []byte { + return test1.BuiltinMakeSlice(x) +} + +// +mustescape:builtin +//go:noinline +func builtinAppendCrossPkg() []byte { + return test1.BuiltinAppend(nil) +} + +// +mustescape:builtin +//go:noinline +func builtinChanCrossPkg() chan int { + return test1.BuiltinChan() +} + +// +mustescape:heap +//go:noinline +func heapCrossPkg() *test1.Type { + return test1.Heap() +} + +// +mustescape:interface +//go:noinline +func dispatchCrossPkg(i test1.Interface) { + test1.Dispatch(i) +} + +// +mustescape:dynamic +//go:noinline +func dynamicCrossPkg(f func()) { + test1.Dynamic(f) +} + +// +mustescape:unknown +//go:noinline +func unknownCrossPkg() { + test1.Unknown() +} + +// +mustescape:stack +//go:noinline +func splitCrosssPkt() { + test1.Split() +} diff --git a/tools/checkunsafe/BUILD b/tools/checkunsafe/BUILD index 4f1a31a6d..0c264151b 100644 --- a/tools/checkunsafe/BUILD +++ b/tools/checkunsafe/BUILD @@ -1,11 +1,12 @@ -load("//tools:defs.bzl", "go_tool_library") +load("//tools:defs.bzl", "go_library") package(licenses = ["notice"]) -go_tool_library( +go_library( name = "checkunsafe", srcs = ["check_unsafe.go"], - visibility = ["//:sandbox"], + nogo = False, + visibility = ["//tools/nogo:__subpackages__"], deps = [ "@org_golang_x_tools//go/analysis:go_tool_library", ], diff --git a/tools/defs.bzl b/tools/defs.bzl index 91d689a82..6a224d7d5 100644 --- a/tools/defs.bzl +++ b/tools/defs.bzl @@ -7,9 +7,10 @@ change for Google-internal and bazel-compatible rules. load("//tools/go_stateify:defs.bzl", "go_stateify") load("//tools/go_marshal:defs.bzl", "go_marshal", "marshal_deps", "marshal_test_deps") -load("//tools/bazeldefs:defs.bzl", _cc_binary = "cc_binary", _cc_flags_supplier = "cc_flags_supplier", _cc_grpc_library = "cc_grpc_library", _cc_library = "cc_library", _cc_proto_library = "cc_proto_library", _cc_test = "cc_test", _cc_toolchain = "cc_toolchain", _container_image = "container_image", _default_installer = "default_installer", _default_net_util = "default_net_util", _gbenchmark = "gbenchmark", _go_binary = "go_binary", _go_embed_data = "go_embed_data", _go_grpc_and_proto_libraries = "go_grpc_and_proto_libraries", _go_image = "go_image", _go_library = "go_library", _go_proto_library = "go_proto_library", _go_test = "go_test", _go_tool_library = "go_tool_library", _grpcpp = "grpcpp", _gtest = "gtest", _loopback = "loopback", _pkg_deb = "pkg_deb", _pkg_tar = "pkg_tar", _proto_library = "proto_library", _py_binary = "py_binary", _py_library = "py_library", _py_requirement = "py_requirement", _py_test = "py_test", _select_arch = "select_arch", _select_system = "select_system") +load("//tools/bazeldefs:defs.bzl", _cc_binary = "cc_binary", _cc_flags_supplier = "cc_flags_supplier", _cc_grpc_library = "cc_grpc_library", _cc_library = "cc_library", _cc_proto_library = "cc_proto_library", _cc_test = "cc_test", _cc_toolchain = "cc_toolchain", _container_image = "container_image", _default_installer = "default_installer", _default_net_util = "default_net_util", _gbenchmark = "gbenchmark", _go_binary = "go_binary", _go_embed_data = "go_embed_data", _go_grpc_and_proto_libraries = "go_grpc_and_proto_libraries", _go_image = "go_image", _go_library = "go_library", _go_proto_library = "go_proto_library", _go_test = "go_test", _grpcpp = "grpcpp", _gtest = "gtest", _loopback = "loopback", _pkg_deb = "pkg_deb", _pkg_tar = "pkg_tar", _proto_library = "proto_library", _py_binary = "py_binary", _py_library = "py_library", _py_requirement = "py_requirement", _py_test = "py_test", _select_arch = "select_arch", _select_system = "select_system") load("//tools/bazeldefs:platforms.bzl", _default_platform = "default_platform", _platforms = "platforms") load("//tools/bazeldefs:tags.bzl", "go_suffixes") +load("//tools/nogo:defs.bzl", "nogo_test") # Delegate directly. cc_binary = _cc_binary @@ -25,7 +26,6 @@ gbenchmark = _gbenchmark go_embed_data = _go_embed_data go_image = _go_image go_test = _go_test -go_tool_library = _go_tool_library gtest = _gtest grpcpp = _grpcpp loopback = _loopback @@ -38,6 +38,7 @@ py_test = _py_test select_arch = _select_arch select_system = _select_system +# Platform options. default_platform = _default_platform platforms = _platforms @@ -91,7 +92,7 @@ def go_imports(name, src, out): cmd = ("$(location @org_golang_x_tools//cmd/goimports:goimports) $(SRCS) > $@"), ) -def go_library(name, srcs, deps = [], imports = [], stateify = True, marshal = False, marshal_debug = False, **kwargs): +def go_library(name, srcs, deps = [], imports = [], stateify = True, marshal = False, marshal_debug = False, nogo = True, **kwargs): """Wraps the standard go_library and does stateification and marshalling. The recommended way is to use this rule with mostly identical configuration as the native @@ -177,6 +178,11 @@ def go_library(name, srcs, deps = [], imports = [], stateify = True, marshal = F deps = all_deps, **kwargs ) + if nogo: + nogo_test( + name = name + "_nogo", + deps = [":" + name], + ) if marshal: # Ignore importpath for go_test. diff --git a/tools/go_marshal/gomarshal/generator_interfaces_array_newtype.go b/tools/go_marshal/gomarshal/generator_interfaces_array_newtype.go index 8d6f102d5..72ef03a22 100644 --- a/tools/go_marshal/gomarshal/generator_interfaces_array_newtype.go +++ b/tools/go_marshal/gomarshal/generator_interfaces_array_newtype.go @@ -44,6 +44,7 @@ func (g *interfaceGenerator) emitMarshallableForArrayNewtype(n *ast.Ident, a *as lenExpr := g.arrayLenExpr(a) g.emit("// SizeBytes implements marshal.Marshallable.SizeBytes.\n") + g.emit("//go:nosplit\n") g.emit("func (%s *%s) SizeBytes() int {\n", g.r, g.typeName()) g.inIndent(func() { if size, dynamic := g.scalarSize(elt); !dynamic { @@ -77,6 +78,7 @@ func (g *interfaceGenerator) emitMarshallableForArrayNewtype(n *ast.Ident, a *as g.emit("}\n\n") g.emit("// Packed implements marshal.Marshallable.Packed.\n") + g.emit("//go:nosplit\n") g.emit("func (%s *%s) Packed() bool {\n", g.r, g.typeName()) g.inIndent(func() { g.emit("// Array newtypes are always packed.\n") @@ -99,17 +101,19 @@ func (g *interfaceGenerator) emitMarshallableForArrayNewtype(n *ast.Ident, a *as g.emit("}\n\n") g.emit("// CopyOutN implements marshal.Marshallable.CopyOutN.\n") + g.emit("//go:nosplit\n") g.emit("func (%s *%s) CopyOutN(task marshal.Task, addr usermem.Addr, limit int) (int, error) {\n", g.r, g.typeName()) g.inIndent(func() { g.emitCastToByteSlice(g.r, "buf", fmt.Sprintf("%s.SizeBytes()", g.r)) - g.emit("length, err := task.CopyOutBytes(addr, buf[:limit])\n") + g.emit("length, err := task.CopyOutBytes(addr, buf[:limit]) // escapes: okay.\n") g.emitKeepAlive(g.r) g.emit("return length, err\n") }) g.emit("}\n\n") g.emit("// CopyOut implements marshal.Marshallable.CopyOut.\n") + g.emit("//go:nosplit\n") g.emit("func (%s *%s) CopyOut(task marshal.Task, addr usermem.Addr) (int, error) {\n", g.r, g.typeName()) g.inIndent(func() { g.emit("return %s.CopyOutN(task, addr, %s.SizeBytes())\n", g.r, g.r) @@ -117,11 +121,12 @@ func (g *interfaceGenerator) emitMarshallableForArrayNewtype(n *ast.Ident, a *as g.emit("}\n\n") g.emit("// CopyIn implements marshal.Marshallable.CopyIn.\n") + g.emit("//go:nosplit\n") g.emit("func (%s *%s) CopyIn(task marshal.Task, addr usermem.Addr) (int, error) {\n", g.r, g.typeName()) g.inIndent(func() { g.emitCastToByteSlice(g.r, "buf", fmt.Sprintf("%s.SizeBytes()", g.r)) - g.emit("length, err := task.CopyInBytes(addr, buf)\n") + g.emit("length, err := task.CopyInBytes(addr, buf) // escapes: okay.\n") g.emitKeepAlive(g.r) g.emit("return length, err\n") }) diff --git a/tools/go_marshal/gomarshal/generator_interfaces_primitive_newtype.go b/tools/go_marshal/gomarshal/generator_interfaces_primitive_newtype.go index ef9bb903d..39f654ea8 100644 --- a/tools/go_marshal/gomarshal/generator_interfaces_primitive_newtype.go +++ b/tools/go_marshal/gomarshal/generator_interfaces_primitive_newtype.go @@ -104,6 +104,7 @@ func (g *interfaceGenerator) emitMarshallableForPrimitiveNewtype(nt *ast.Ident) g.recordUsedImport("usermem") g.emit("// SizeBytes implements marshal.Marshallable.SizeBytes.\n") + g.emit("//go:nosplit\n") g.emit("func (%s *%s) SizeBytes() int {\n", g.r, g.typeName()) g.inIndent(func() { if size, dynamic := g.scalarSize(nt); !dynamic { @@ -129,6 +130,7 @@ func (g *interfaceGenerator) emitMarshallableForPrimitiveNewtype(nt *ast.Ident) g.emit("}\n\n") g.emit("// Packed implements marshal.Marshallable.Packed.\n") + g.emit("//go:nosplit\n") g.emit("func (%s *%s) Packed() bool {\n", g.r, g.typeName()) g.inIndent(func() { g.emit("// Scalar newtypes are always packed.\n") @@ -151,17 +153,19 @@ func (g *interfaceGenerator) emitMarshallableForPrimitiveNewtype(nt *ast.Ident) g.emit("}\n\n") g.emit("// CopyOutN implements marshal.Marshallable.CopyOutN.\n") + g.emit("//go:nosplit\n") g.emit("func (%s *%s) CopyOutN(task marshal.Task, addr usermem.Addr, limit int) (int, error) {\n", g.r, g.typeName()) g.inIndent(func() { g.emitCastToByteSlice(g.r, "buf", fmt.Sprintf("%s.SizeBytes()", g.r)) - g.emit("length, err := task.CopyOutBytes(addr, buf[:limit])\n") + g.emit("length, err := task.CopyOutBytes(addr, buf[:limit]) // escapes: okay.\n") g.emitKeepAlive(g.r) g.emit("return length, err\n") }) g.emit("}\n\n") g.emit("// CopyOut implements marshal.Marshallable.CopyOut.\n") + g.emit("//go:nosplit\n") g.emit("func (%s *%s) CopyOut(task marshal.Task, addr usermem.Addr) (int, error) {\n", g.r, g.typeName()) g.inIndent(func() { g.emit("return %s.CopyOutN(task, addr, %s.SizeBytes())\n", g.r, g.r) @@ -169,11 +173,12 @@ func (g *interfaceGenerator) emitMarshallableForPrimitiveNewtype(nt *ast.Ident) g.emit("}\n\n") g.emit("// CopyIn implements marshal.Marshallable.CopyIn.\n") + g.emit("//go:nosplit\n") g.emit("func (%s *%s) CopyIn(task marshal.Task, addr usermem.Addr) (int, error) {\n", g.r, g.typeName()) g.inIndent(func() { g.emitCastToByteSlice(g.r, "buf", fmt.Sprintf("%s.SizeBytes()", g.r)) - g.emit("length, err := task.CopyInBytes(addr, buf)\n") + g.emit("length, err := task.CopyInBytes(addr, buf) // escapes: okay.\n") g.emitKeepAlive(g.r) g.emit("return length, err\n") }) @@ -205,6 +210,7 @@ func (g *interfaceGenerator) emitMarshallableSliceForPrimitiveNewtype(nt *ast.Id } g.emit("// Copy%sIn copies in a slice of %s objects from the task's memory.\n", slice.ident, eltType) + g.emit("//go:nosplit\n") g.emit("func Copy%sIn(task marshal.Task, addr usermem.Addr, dst []%s) (int, error) {\n", slice.ident, eltType) g.inIndent(func() { g.emit("count := len(dst)\n") @@ -217,13 +223,14 @@ func (g *interfaceGenerator) emitMarshallableSliceForPrimitiveNewtype(nt *ast.Id g.emitCastSliceToByteSlice("&dst", "buf", "size * count") - g.emit("length, err := task.CopyInBytes(addr, buf)\n") + g.emit("length, err := task.CopyInBytes(addr, buf) // escapes: okay.\n") g.emitKeepAlive("dst") g.emit("return length, err\n") }) g.emit("}\n\n") g.emit("// Copy%sOut copies a slice of %s objects to the task's memory.\n", slice.ident, eltType) + g.emit("//go:nosplit\n") g.emit("func Copy%sOut(task marshal.Task, addr usermem.Addr, src []%s) (int, error) {\n", slice.ident, eltType) g.inIndent(func() { g.emit("count := len(src)\n") @@ -236,7 +243,7 @@ func (g *interfaceGenerator) emitMarshallableSliceForPrimitiveNewtype(nt *ast.Id g.emitCastSliceToByteSlice("&src", "buf", "size * count") - g.emit("length, err := task.CopyOutBytes(addr, buf)\n") + g.emit("length, err := task.CopyOutBytes(addr, buf) // escapes: okay.\n") g.emitKeepAlive("src") g.emit("return length, err\n") }) diff --git a/tools/go_marshal/gomarshal/generator_interfaces_struct.go b/tools/go_marshal/gomarshal/generator_interfaces_struct.go index 4236e978e..9cd3c9579 100644 --- a/tools/go_marshal/gomarshal/generator_interfaces_struct.go +++ b/tools/go_marshal/gomarshal/generator_interfaces_struct.go @@ -249,6 +249,7 @@ func (g *interfaceGenerator) emitMarshallableForStruct(st *ast.StructType) { g.emit("}\n\n") g.emit("// Packed implements marshal.Marshallable.Packed.\n") + g.emit("//go:nosplit\n") g.emit("func (%s *%s) Packed() bool {\n", g.r, g.typeName()) g.inIndent(func() { expr, fieldsMaybePacked := g.areFieldsPackedExpression() @@ -317,15 +318,16 @@ func (g *interfaceGenerator) emitMarshallableForStruct(st *ast.StructType) { g.emit("}\n\n") g.emit("// CopyOutN implements marshal.Marshallable.CopyOutN.\n") + g.emit("//go:nosplit\n") g.recordUsedImport("marshal") g.recordUsedImport("usermem") g.emit("func (%s *%s) CopyOutN(task marshal.Task, addr usermem.Addr, limit int) (int, error) {\n", g.r, g.typeName()) g.inIndent(func() { fallback := func() { g.emit("// Type %s doesn't have a packed layout in memory, fall back to MarshalBytes.\n", g.typeName()) - g.emit("buf := task.CopyScratchBuffer(%s.SizeBytes())\n", g.r) - g.emit("%s.MarshalBytes(buf)\n", g.r) - g.emit("return task.CopyOutBytes(addr, buf[:limit])\n") + g.emit("buf := task.CopyScratchBuffer(%s.SizeBytes()) // escapes: okay.\n", g.r) + g.emit("%s.MarshalBytes(buf) // escapes: fallback.\n", g.r) + g.emit("return task.CopyOutBytes(addr, buf[:limit]) // escapes: okay.\n") } if thisPacked { g.recordUsedImport("reflect") @@ -339,7 +341,7 @@ func (g *interfaceGenerator) emitMarshallableForStruct(st *ast.StructType) { // Fast serialization. g.emitCastToByteSlice(g.r, "buf", fmt.Sprintf("%s.SizeBytes()", g.r)) - g.emit("length, err := task.CopyOutBytes(addr, buf[:limit])\n") + g.emit("length, err := task.CopyOutBytes(addr, buf[:limit]) // escapes: okay.\n") g.emitKeepAlive(g.r) g.emit("return length, err\n") } else { @@ -349,6 +351,7 @@ func (g *interfaceGenerator) emitMarshallableForStruct(st *ast.StructType) { g.emit("}\n\n") g.emit("// CopyOut implements marshal.Marshallable.CopyOut.\n") + g.emit("//go:nosplit\n") g.recordUsedImport("marshal") g.recordUsedImport("usermem") g.emit("func (%s *%s) CopyOut(task marshal.Task, addr usermem.Addr) (int, error) {\n", g.r, g.typeName()) @@ -358,17 +361,18 @@ func (g *interfaceGenerator) emitMarshallableForStruct(st *ast.StructType) { g.emit("}\n\n") g.emit("// CopyIn implements marshal.Marshallable.CopyIn.\n") + g.emit("//go:nosplit\n") g.recordUsedImport("marshal") g.recordUsedImport("usermem") g.emit("func (%s *%s) CopyIn(task marshal.Task, addr usermem.Addr) (int, error) {\n", g.r, g.typeName()) g.inIndent(func() { fallback := func() { g.emit("// Type %s doesn't have a packed layout in memory, fall back to UnmarshalBytes.\n", g.typeName()) - g.emit("buf := task.CopyScratchBuffer(%s.SizeBytes())\n", g.r) - g.emit("length, err := task.CopyInBytes(addr, buf)\n") + g.emit("buf := task.CopyScratchBuffer(%s.SizeBytes()) // escapes: okay.\n", g.r) + g.emit("length, err := task.CopyInBytes(addr, buf) // escapes: okay.\n") g.emit("// Unmarshal unconditionally. If we had a short copy-in, this results in a\n") g.emit("// partially unmarshalled struct.\n") - g.emit("%s.UnmarshalBytes(buf)\n", g.r) + g.emit("%s.UnmarshalBytes(buf) // escapes: fallback.\n", g.r) g.emit("return length, err\n") } if thisPacked { @@ -383,7 +387,7 @@ func (g *interfaceGenerator) emitMarshallableForStruct(st *ast.StructType) { // Fast deserialization. g.emitCastToByteSlice(g.r, "buf", fmt.Sprintf("%s.SizeBytes()", g.r)) - g.emit("length, err := task.CopyInBytes(addr, buf)\n") + g.emit("length, err := task.CopyInBytes(addr, buf) // escapes: okay.\n") g.emitKeepAlive(g.r) g.emit("return length, err\n") } else { diff --git a/tools/go_marshal/test/BUILD b/tools/go_marshal/test/BUILD index 3b839799d..2fbcc8a03 100644 --- a/tools/go_marshal/test/BUILD +++ b/tools/go_marshal/test/BUILD @@ -1,4 +1,4 @@ -load("//tools:defs.bzl", "go_binary", "go_library", "go_test") +load("//tools:defs.bzl", "go_library", "go_test") licenses(["notice"]) @@ -25,21 +25,10 @@ go_library( testonly = 1, srcs = ["test.go"], marshal = True, + visibility = ["//tools/go_marshal/test:__subpackages__"], deps = ["//tools/go_marshal/test/external"], ) -go_binary( - name = "escape", - testonly = 1, - srcs = ["escape.go"], - gc_goopts = ["-m"], - deps = [ - ":test", - "//pkg/usermem", - "//tools/go_marshal/marshal", - ], -) - go_test( name = "marshal_test", size = "small", diff --git a/tools/go_marshal/test/escape.go b/tools/go_marshal/test/escape.go deleted file mode 100644 index 184f05ea3..000000000 --- a/tools/go_marshal/test/escape.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2020 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This binary provides a convienient target for analyzing how the go-marshal -// API causes its various arguments to escape to the heap. To use, build and -// observe the output from the go compiler's escape analysis: -// -// $ bazel build :escape -// ... -// escape.go:67:2: moved to heap: task -// escape.go:77:31: make([]byte, size) escapes to heap -// escape.go:87:31: make([]byte, size) escapes to heap -// escape.go:96:6: moved to heap: stat -// ... -// -// This is not an automated test, but simply a minimal binary for easy analysis. -package main - -import ( - "gvisor.dev/gvisor/pkg/usermem" - "gvisor.dev/gvisor/tools/go_marshal/marshal" - "gvisor.dev/gvisor/tools/go_marshal/test" -) - -// dummyTask implements marshal.Task. -type dummyTask struct { -} - -func (*dummyTask) CopyScratchBuffer(size int) []byte { - return make([]byte, size) -} - -func (*dummyTask) CopyOutBytes(addr usermem.Addr, b []byte) (int, error) { - return len(b), nil -} - -func (*dummyTask) CopyInBytes(addr usermem.Addr, b []byte) (int, error) { - return len(b), nil -} - -func (task *dummyTask) MarshalBytes(addr usermem.Addr, marshallable marshal.Marshallable) { - buf := task.CopyScratchBuffer(marshallable.SizeBytes()) - marshallable.MarshalBytes(buf) - task.CopyOutBytes(addr, buf) -} - -func (task *dummyTask) MarshalUnsafe(addr usermem.Addr, marshallable marshal.Marshallable) { - buf := task.CopyScratchBuffer(marshallable.SizeBytes()) - marshallable.MarshalUnsafe(buf) - task.CopyOutBytes(addr, buf) -} - -// Expected escapes: -// - task: passed to marshal.Marshallable.CopyOut as the marshal.Task interface. -func doCopyOut() { - task := dummyTask{} - var stat test.Stat - stat.CopyOut(&task, usermem.Addr(0xf000ba12)) -} - -// Expected escapes: -// - buf: make allocates on the heap. -func doMarshalBytesDirect() { - task := dummyTask{} - var stat test.Stat - buf := task.CopyScratchBuffer(stat.SizeBytes()) - stat.MarshalBytes(buf) - task.CopyOutBytes(usermem.Addr(0xf000ba12), buf) -} - -// Expected escapes: -// - buf: make allocates on the heap. -func doMarshalUnsafeDirect() { - task := dummyTask{} - var stat test.Stat - buf := task.CopyScratchBuffer(stat.SizeBytes()) - stat.MarshalUnsafe(buf) - task.CopyOutBytes(usermem.Addr(0xf000ba12), buf) -} - -// Expected escapes: -// - stat: passed to dummyTask.MarshalBytes as the marshal.Marshallable interface. -func doMarshalBytesViaMarshallable() { - task := dummyTask{} - var stat test.Stat - task.MarshalBytes(usermem.Addr(0xf000ba12), &stat) -} - -// Expected escapes: -// - stat: passed to dummyTask.MarshalUnsafe as the marshal.Marshallable interface. -func doMarshalUnsafeViaMarshallable() { - task := dummyTask{} - var stat test.Stat - task.MarshalUnsafe(usermem.Addr(0xf000ba12), &stat) -} - -func main() { - doCopyOut() - doMarshalBytesDirect() - doMarshalUnsafeDirect() - doMarshalBytesViaMarshallable() - doMarshalUnsafeViaMarshallable() -} diff --git a/tools/go_marshal/test/escape/BUILD b/tools/go_marshal/test/escape/BUILD new file mode 100644 index 000000000..f74e6ffae --- /dev/null +++ b/tools/go_marshal/test/escape/BUILD @@ -0,0 +1,14 @@ +load("//tools:defs.bzl", "go_library") + +licenses(["notice"]) + +go_library( + name = "escape", + testonly = 1, + srcs = ["escape.go"], + deps = [ + "//pkg/usermem", + "//tools/go_marshal/marshal", + "//tools/go_marshal/test", + ], +) diff --git a/tools/go_marshal/test/escape/escape.go b/tools/go_marshal/test/escape/escape.go new file mode 100644 index 000000000..6a46ddbf8 --- /dev/null +++ b/tools/go_marshal/test/escape/escape.go @@ -0,0 +1,95 @@ +// Copyright 2020 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package escape + +import ( + "gvisor.dev/gvisor/pkg/usermem" + "gvisor.dev/gvisor/tools/go_marshal/marshal" + "gvisor.dev/gvisor/tools/go_marshal/test" +) + +// dummyTask implements marshal.Task. +type dummyTask struct { +} + +func (*dummyTask) CopyScratchBuffer(size int) []byte { + return make([]byte, size) +} + +func (*dummyTask) CopyOutBytes(addr usermem.Addr, b []byte) (int, error) { + return len(b), nil +} + +func (*dummyTask) CopyInBytes(addr usermem.Addr, b []byte) (int, error) { + return len(b), nil +} + +func (t *dummyTask) MarshalBytes(addr usermem.Addr, marshallable marshal.Marshallable) { + buf := t.CopyScratchBuffer(marshallable.SizeBytes()) + marshallable.MarshalBytes(buf) + t.CopyOutBytes(addr, buf) +} + +func (t *dummyTask) MarshalUnsafe(addr usermem.Addr, marshallable marshal.Marshallable) { + buf := t.CopyScratchBuffer(marshallable.SizeBytes()) + marshallable.MarshalUnsafe(buf) + t.CopyOutBytes(addr, buf) +} + +// +checkescape:all +//go:nosplit +func doCopyIn(t *dummyTask) { + var stat test.Stat + stat.CopyIn(t, usermem.Addr(0xf000ba12)) +} + +// +checkescape:all +//go:nosplit +func doCopyOut(t *dummyTask) { + var stat test.Stat + stat.CopyOut(t, usermem.Addr(0xf000ba12)) +} + +// +mustescape:builtin +// +mustescape:stack +func doMarshalBytesDirect(t *dummyTask) { + var stat test.Stat + buf := t.CopyScratchBuffer(stat.SizeBytes()) + stat.MarshalBytes(buf) + t.CopyOutBytes(usermem.Addr(0xf000ba12), buf) +} + +// +mustescape:builtin +// +mustescape:stack +func doMarshalUnsafeDirect(t *dummyTask) { + var stat test.Stat + buf := t.CopyScratchBuffer(stat.SizeBytes()) + stat.MarshalUnsafe(buf) + t.CopyOutBytes(usermem.Addr(0xf000ba12), buf) +} + +// +mustescape:local,heap +// +mustescape:stack +func doMarshalBytesViaMarshallable(t *dummyTask) { + var stat test.Stat + t.MarshalBytes(usermem.Addr(0xf000ba12), &stat) +} + +// +mustescape:local,heap +// +mustescape:stack +func doMarshalUnsafeViaMarshallable(t *dummyTask) { + var stat test.Stat + t.MarshalUnsafe(usermem.Addr(0xf000ba12), &stat) +} diff --git a/tools/nogo.json b/tools/nogo.json deleted file mode 100644 index ae969409e..000000000 --- a/tools/nogo.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "assign": { - "exclude_files": { - "/external/bazel_gazelle/walk/walk.go": "allowed: false positive" - } - }, - "checkunsafe": { - "exclude_files": { - "/external/": "allowed: not subject to unsafe naming rules" - } - }, - "nilness": { - "exclude_files": { - "/com_github_vishvananda_netlink/route_linux.go": "allowed: false positive", - "/external/bazel_gazelle/cmd/gazelle/.*": "allowed: false positive", - "/org_golang_x_tools/go/packages/golist.go": "allowed: runtime internals", - "/pkg/sentry/platform/kvm/kvm_test.go": "allowed: intentional", - "/tools/bigquery/bigquery.go": "allowed: false positive", - "/external/io_opencensus_go/tag/map_codec.go": "allowed: false positive" - } - }, - "structtag": { - "exclude_files": { - "/external/": "allowed: may use arbitrary tags" - } - }, - "unsafeptr": { - "exclude_files": { - ".*_test.go": "allowed: exclude tests", - "/pkg/flipcall/flipcall_unsafe.go": "allowed: special case", - "/pkg/gohacks/gohacks_unsafe.go": "allowed: special case", - "/pkg/sentry/fs/fsutil/host_file_mapper_unsafe.go": "allowed: special case", - "/pkg/sentry/platform/kvm/(bluepill|machine)_unsafe.go": "allowed: special case", - "/pkg/sentry/platform/ring0/pagetables/allocator_unsafe.go": "allowed: special case", - "/pkg/sentry/platform/safecopy/safecopy_unsafe.go": "allowed: special case", - "/pkg/sentry/vfs/mount_unsafe.go": "allowed: special case" - } - } -} diff --git a/tools/nogo/BUILD b/tools/nogo/BUILD new file mode 100644 index 000000000..c21b09511 --- /dev/null +++ b/tools/nogo/BUILD @@ -0,0 +1,49 @@ +load("//tools:defs.bzl", "go_library") + +package(licenses = ["notice"]) + +go_library( + name = "nogo", + srcs = [ + "build.go", + "config.go", + "matchers.go", + "nogo.go", + "register.go", + ], + nogo = False, + visibility = ["//:sandbox"], + deps = [ + "//tools/checkescape", + "//tools/checkunsafe", + "//tools/nogo/data", + "@org_golang_x_tools//go/analysis:go_tool_library", + "@org_golang_x_tools//go/analysis/internal/facts:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/asmdecl:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/assign:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/atomic:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/bools:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/buildtag:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/cgocall:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/composite:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/copylock:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/errorsas:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/httpresponse:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/loopclosure:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/lostcancel:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/nilfunc:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/nilness:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/printf:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/shadow:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/shift:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/stdmethods:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/stringintconv:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/structtag:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/tests:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/unmarshal:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/unreachable:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/unsafeptr:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/unusedresult:go_tool_library", + "@org_golang_x_tools//go/gcexportdata:go_tool_library", + ], +) diff --git a/tools/nogo/README.md b/tools/nogo/README.md new file mode 100644 index 000000000..6e4db18de --- /dev/null +++ b/tools/nogo/README.md @@ -0,0 +1,31 @@ +# Extended "nogo" analysis + +This package provides a build aspect that perform nogo analysis. This will be +automatically injected to all relevant libraries when using the default +`go_binary` and `go_library` rules. + +It exists for several reasons. + +* The default `nogo` provided by bazel is insufficient with respect to the + possibility of binary analysis. This package allows us to analyze the + generated binary in addition to using the standard analyzers. + +* The configuration provided in this package is much richer than the standard + `nogo` JSON blob. Specifically, it allows us to exclude specific structures + from the composite rules (such as the Ranges that are common with the set + types). + +* The bazel version of `nogo` is run directly against the `go_library` and + `go_binary` targets, meaning that any change to the configuration requires a + rebuild from scratch (for some reason included all C++ source files in the + process). Using an aspect is more efficient in this regard. + +* The checks supported by this package are exported as tests, which makes it + easier to reason about and plumb into the build system. + +* For uninteresting reasons, it is impossible to integrate the default `nogo` + analyzer provided by bazel with internal Google tooling. To provide a + consistent experience, this package allows those systems to be unified. + +To use this package, import `nogo_test` from `defs.bzl` and add a single +dependency which is a `go_binary` or `go_library` rule. diff --git a/tools/nogo/build.go b/tools/nogo/build.go new file mode 100644 index 000000000..1c0d08661 --- /dev/null +++ b/tools/nogo/build.go @@ -0,0 +1,36 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nogo + +import ( + "fmt" + "io" + "os" +) + +var ( + // internalPrefix is the internal path prefix. Note that this is not + // special, as paths should be passed relative to the repository root + // and should not have any special prefix applied. + internalPrefix = fmt.Sprintf("^") + + // externalPrefix is external workspace packages. + externalPrefix = "^external/" +) + +// findStdPkg needs to find the bundled standard library packages. +func findStdPkg(path, GOOS, GOARCH string) (io.ReadCloser, error) { + return os.Open(fmt.Sprintf("external/go_sdk/pkg/%s_%s/%s.a", GOOS, GOARCH, path)) +} diff --git a/tools/nogo/check/BUILD b/tools/nogo/check/BUILD new file mode 100644 index 000000000..e2d76cd5c --- /dev/null +++ b/tools/nogo/check/BUILD @@ -0,0 +1,12 @@ +load("//tools:defs.bzl", "go_binary") + +package(licenses = ["notice"]) + +# Note that the check binary must be public, since an aspect may be applied +# across lots of different rules in different repositories. +go_binary( + name = "check", + srcs = ["main.go"], + visibility = ["//visibility:public"], + deps = ["//tools/nogo"], +) diff --git a/tools/nogo/check/main.go b/tools/nogo/check/main.go new file mode 100644 index 000000000..3828edf3a --- /dev/null +++ b/tools/nogo/check/main.go @@ -0,0 +1,24 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Binary check is the nogo entrypoint. +package main + +import ( + "gvisor.dev/gvisor/tools/nogo" +) + +func main() { + nogo.Main() +} diff --git a/tools/nogo/config.go b/tools/nogo/config.go new file mode 100644 index 000000000..0c4b7dd40 --- /dev/null +++ b/tools/nogo/config.go @@ -0,0 +1,113 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nogo + +import ( + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/asmdecl" + "golang.org/x/tools/go/analysis/passes/assign" + "golang.org/x/tools/go/analysis/passes/atomic" + "golang.org/x/tools/go/analysis/passes/bools" + "golang.org/x/tools/go/analysis/passes/buildtag" + "golang.org/x/tools/go/analysis/passes/cgocall" + "golang.org/x/tools/go/analysis/passes/composite" + "golang.org/x/tools/go/analysis/passes/copylock" + "golang.org/x/tools/go/analysis/passes/errorsas" + "golang.org/x/tools/go/analysis/passes/httpresponse" + "golang.org/x/tools/go/analysis/passes/loopclosure" + "golang.org/x/tools/go/analysis/passes/lostcancel" + "golang.org/x/tools/go/analysis/passes/nilfunc" + "golang.org/x/tools/go/analysis/passes/nilness" + "golang.org/x/tools/go/analysis/passes/printf" + "golang.org/x/tools/go/analysis/passes/shadow" + "golang.org/x/tools/go/analysis/passes/shift" + "golang.org/x/tools/go/analysis/passes/stdmethods" + "golang.org/x/tools/go/analysis/passes/stringintconv" + "golang.org/x/tools/go/analysis/passes/structtag" + "golang.org/x/tools/go/analysis/passes/tests" + "golang.org/x/tools/go/analysis/passes/unmarshal" + "golang.org/x/tools/go/analysis/passes/unreachable" + "golang.org/x/tools/go/analysis/passes/unsafeptr" + "golang.org/x/tools/go/analysis/passes/unusedresult" + + "gvisor.dev/gvisor/tools/checkescape" + "gvisor.dev/gvisor/tools/checkunsafe" +) + +var analyzerConfig = map[*analysis.Analyzer]matcher{ + // Standard analyzers. + asmdecl.Analyzer: alwaysMatches(), + assign.Analyzer: externalExcluded( + ".*gazelle/walk/walk.go", // False positive. + ), + atomic.Analyzer: alwaysMatches(), + bools.Analyzer: alwaysMatches(), + buildtag.Analyzer: alwaysMatches(), + cgocall.Analyzer: alwaysMatches(), + composite.Analyzer: and( + disableMatches(), // Disabled for now. + resultExcluded{ + "Object_", + "Range{", + }, + ), + copylock.Analyzer: internalMatches(), // Common external issues (e.g. protos). + errorsas.Analyzer: alwaysMatches(), + httpresponse.Analyzer: alwaysMatches(), + loopclosure.Analyzer: alwaysMatches(), + lostcancel.Analyzer: internalMatches(), // Common external issues. + nilfunc.Analyzer: alwaysMatches(), + nilness.Analyzer: and( + internalMatches(), // Common "tautological checks". + internalExcluded( + "pkg/sentry/platform/kvm/kvm_test.go", // Intentional. + "tools/bigquery/bigquery.go", // False positive. + ), + ), + printf.Analyzer: alwaysMatches(), + shift.Analyzer: alwaysMatches(), + stdmethods.Analyzer: internalMatches(), // Common external issues (e.g. methods named "Write"). + stringintconv.Analyzer: and( + internalExcluded(), + externalExcluded( + ".*protobuf/.*.go", // Bad conversions. + ".*flate/huffman_bit_writer.go", // Bad conversion. + ), + ), + shadow.Analyzer: disableMatches(), // Disabled for now. + structtag.Analyzer: internalMatches(), // External not subject to rules. + tests.Analyzer: alwaysMatches(), + unmarshal.Analyzer: alwaysMatches(), + unreachable.Analyzer: internalMatches(), + unsafeptr.Analyzer: and( + internalMatches(), + internalExcluded( + ".*_test.go", // Exclude tests. + "pkg/flipcall/.*_unsafe.go", // Special case. + "pkg/gohacks/gohacks_unsafe.go", // Special case. + "pkg/sentry/fs/fsutil/host_file_mapper_unsafe.go", // Special case. + "pkg/sentry/platform/kvm/bluepill_unsafe.go", // Special case. + "pkg/sentry/platform/kvm/machine_unsafe.go", // Special case. + "pkg/sentry/platform/ring0/pagetables/allocator_unsafe.go", // Special case. + "pkg/sentry/platform/safecopy/safecopy_unsafe.go", // Special case. + "pkg/sentry/vfs/mount_unsafe.go", // Special case. + ), + ), + unusedresult.Analyzer: alwaysMatches(), + + // Internal analyzers: external packages not subject. + checkescape.Analyzer: internalMatches(), + checkunsafe.Analyzer: internalMatches(), +} diff --git a/tools/nogo/data/BUILD b/tools/nogo/data/BUILD new file mode 100644 index 000000000..b7564cc44 --- /dev/null +++ b/tools/nogo/data/BUILD @@ -0,0 +1,10 @@ +load("//tools:defs.bzl", "go_library") + +package(licenses = ["notice"]) + +go_library( + name = "data", + srcs = ["data.go"], + nogo = False, + visibility = ["//tools:__subpackages__"], +) diff --git a/tools/nogo/data/data.go b/tools/nogo/data/data.go new file mode 100644 index 000000000..eb84d0d27 --- /dev/null +++ b/tools/nogo/data/data.go @@ -0,0 +1,21 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package data contains shared data for nogo analysis. +// +// This is used to break a dependency cycle. +package data + +// Objdump is the dumped binary under analysis. +var Objdump string diff --git a/tools/nogo/defs.bzl b/tools/nogo/defs.bzl new file mode 100644 index 000000000..6560b57c8 --- /dev/null +++ b/tools/nogo/defs.bzl @@ -0,0 +1,172 @@ +"""Nogo rules.""" + +load("//tools/bazeldefs:defs.bzl", "go_context", "go_importpath", "go_rule") + +# NogoInfo is the serialized set of package facts for a nogo analysis. +# +# Each go_library rule will generate a corresponding nogo rule, which will run +# with the source files as input. Note however, that the individual nogo rules +# are simply stubs that enter into the shadow dependency tree (the "aspect"). +NogoInfo = provider( + fields = { + "facts": "serialized package facts", + "importpath": "package import path", + "binaries": "package binary files", + }, +) + +def _nogo_aspect_impl(target, ctx): + # If this is a nogo rule itself (and not the shadow of a go_library or + # go_binary rule created by such a rule), then we simply return nothing. + # All work is done in the shadow properties for go rules. For a proto + # library, we simply skip the analysis portion but still need to return a + # valid NogoInfo to reference the generated binary. + if ctx.rule.kind == "go_library": + srcs = ctx.rule.files.srcs + elif ctx.rule.kind == "go_proto_library" or ctx.rule.kind == "go_wrap_cc": + srcs = [] + else: + return [NogoInfo()] + + # Construct the Go environment from the go_context.env dictionary. + env_prefix = " ".join(["%s=%s" % (key, value) for (key, value) in go_context(ctx).env.items()]) + + # Start with all target files and srcs as input. + inputs = target.files.to_list() + srcs + + # Generate a shell script that dumps the binary. Annoyingly, this seems + # necessary as the context in which a run_shell command runs does not seem + # to cleanly allow us redirect stdout to the actual output file. Perhaps + # I'm missing something here, but the intermediate script does work. + binaries = target.files.to_list() + disasm_file = ctx.actions.declare_file(target.label.name + ".out") + dumper = ctx.actions.declare_file("%s-dumper" % ctx.label.name) + ctx.actions.write(dumper, "\n".join([ + "#!/bin/bash", + "%s %s tool objdump %s > %s\n" % ( + env_prefix, + go_context(ctx).go.path, + [f.path for f in binaries if f.path.endswith(".a")][0], + disasm_file.path, + ), + ]), is_executable = True) + ctx.actions.run( + inputs = binaries, + outputs = [disasm_file], + tools = go_context(ctx).runfiles, + mnemonic = "GoObjdump", + progress_message = "Objdump %s" % target.label, + executable = dumper, + ) + inputs.append(disasm_file) + + # Extract the importpath for this package. + importpath = go_importpath(target) + + # The nogo tool requires a configfile serialized in JSON format to do its + # work. This must line up with the nogo.Config fields. + facts = ctx.actions.declare_file(target.label.name + ".facts") + config = struct( + ImportPath = importpath, + GoFiles = [src.path for src in srcs if src.path.endswith(".go")], + NonGoFiles = [src.path for src in srcs if not src.path.endswith(".go")], + GOOS = go_context(ctx).goos, + GOARCH = go_context(ctx).goarch, + Tags = go_context(ctx).tags, + FactMap = {}, # Constructed below. + ImportMap = {}, # Constructed below. + FactOutput = facts.path, + Objdump = disasm_file.path, + ) + + # Collect all info from shadow dependencies. + for dep in ctx.rule.attr.deps: + # There will be no file attribute set for all transitive dependencies + # that are not go_library or go_binary rules, such as a proto rules. + # This is handled by the ctx.rule.kind check above. + info = dep[NogoInfo] + if not hasattr(info, "facts"): + continue + + # Configure where to find the binary & fact files. Note that this will + # use .x and .a regardless of whether this is a go_binary rule, since + # these dependencies must be go_library rules. + x_files = [f.path for f in info.binaries if f.path.endswith(".x")] + if not len(x_files): + x_files = [f.path for f in info.binaries if f.path.endswith(".a")] + config.ImportMap[info.importpath] = x_files[0] + config.FactMap[info.importpath] = info.facts.path + + # Ensure the above are available as inputs. + inputs.append(info.facts) + inputs += info.binaries + + # Write the configuration and run the tool. + config_file = ctx.actions.declare_file(target.label.name + ".cfg") + ctx.actions.write(config_file, config.to_json()) + inputs.append(config_file) + + # Run the nogo tool itself. + ctx.actions.run( + inputs = inputs, + outputs = [facts], + tools = go_context(ctx).runfiles, + executable = ctx.files._nogo[0], + mnemonic = "GoStaticAnalysis", + progress_message = "Analyzing %s" % target.label, + arguments = ["-config=%s" % config_file.path], + ) + + # Return the package facts as output. + return [NogoInfo( + facts = facts, + importpath = importpath, + binaries = binaries, + )] + +nogo_aspect = go_rule( + aspect, + implementation = _nogo_aspect_impl, + attr_aspects = ["deps"], + attrs = { + "_nogo": attr.label( + default = "//tools/nogo/check:check", + allow_single_file = True, + ), + }, +) + +def _nogo_test_impl(ctx): + """Check nogo findings.""" + + # Build a runner that checks for the existence of the facts file. Note that + # the actual build will fail in the case of a broken analysis. We things + # this way so that any test applied is effectively pushed down to all + # upstream dependencies through the aspect. + inputs = [] + runner = ctx.actions.declare_file("%s-executer" % ctx.label.name) + runner_content = ["#!/bin/bash"] + for dep in ctx.attr.deps: + info = dep[NogoInfo] + inputs.append(info.facts) + + # Draw a sweet unicode checkmark with the package name (in green). + runner_content.append("echo -e \"\\033[0;32m\\xE2\\x9C\\x94\\033[0;31m\\033[0m %s\"" % info.importpath) + runner_content.append("exit 0\n") + ctx.actions.write(runner, "\n".join(runner_content), is_executable = True) + return [DefaultInfo( + runfiles = ctx.runfiles(files = inputs), + executable = runner, + )] + +_nogo_test = rule( + implementation = _nogo_test_impl, + attrs = { + "deps": attr.label_list(aspects = [nogo_aspect]), + }, + test = True, +) + +def nogo_test(**kwargs): + tags = kwargs.pop("tags", []) + ["nogo"] + _nogo_test(tags = tags, **kwargs) diff --git a/tools/nogo/io_bazel_rules_go-visibility.patch b/tools/nogo/io_bazel_rules_go-visibility.patch new file mode 100644 index 000000000..6b64b2e85 --- /dev/null +++ b/tools/nogo/io_bazel_rules_go-visibility.patch @@ -0,0 +1,25 @@ +diff --git a/third_party/org_golang_x_tools-extras.patch b/third_party/org_golang_x_tools-extras.patch +index 133fbccc..5f0d9a47 100644 +--- a/third_party/org_golang_x_tools-extras.patch ++++ b/third_party/org_golang_x_tools-extras.patch +@@ -32,7 +32,7 @@ diff -urN c/go/analysis/internal/facts/BUILD.bazel d/go/analysis/internal/facts/ + + go_library( + name = "go_default_library", +-@@ -14,6 +14,23 @@ ++@@ -14,6 +14,20 @@ + ], + ) + +@@ -43,10 +43,7 @@ diff -urN c/go/analysis/internal/facts/BUILD.bazel d/go/analysis/internal/facts/ + + "imports.go", + + ], + + importpath = "golang.org/x/tools/go/analysis/internal/facts", +-+ visibility = [ +-+ "//go/analysis:__subpackages__", +-+ "@io_bazel_rules_go//go/tools/builders:__pkg__", +-+ ], +++ visibility = ["//visibility:public"], + + deps = [ + + "//go/analysis:go_tool_library", + + "//go/types/objectpath:go_tool_library", diff --git a/tools/nogo/matchers.go b/tools/nogo/matchers.go new file mode 100644 index 000000000..bc5772303 --- /dev/null +++ b/tools/nogo/matchers.go @@ -0,0 +1,138 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nogo + +import ( + "go/token" + "path/filepath" + "regexp" + "strings" + + "golang.org/x/tools/go/analysis" +) + +type matcher interface { + ShouldReport(d analysis.Diagnostic, fs *token.FileSet) bool +} + +// pathRegexps excludes explicit paths. +type pathRegexps struct { + expr []*regexp.Regexp + whitelist bool +} + +// buildRegexps builds a list of regular expressions. +// +// This will panic on error. +func buildRegexps(prefix string, args ...string) []*regexp.Regexp { + result := make([]*regexp.Regexp, 0, len(args)) + for _, arg := range args { + result = append(result, regexp.MustCompile(filepath.Join(prefix, arg))) + } + return result +} + +// ShouldReport implements matcher.ShouldReport. +func (p *pathRegexps) ShouldReport(d analysis.Diagnostic, fs *token.FileSet) bool { + fullPos := fs.Position(d.Pos).String() + for _, path := range p.expr { + if path.MatchString(fullPos) { + return p.whitelist + } + } + return !p.whitelist +} + +// internalExcluded excludes specific internal paths. +func internalExcluded(paths ...string) *pathRegexps { + return &pathRegexps{ + expr: buildRegexps(internalPrefix, paths...), + whitelist: false, + } +} + +// excludedExcluded excludes specific external paths. +func externalExcluded(paths ...string) *pathRegexps { + return &pathRegexps{ + expr: buildRegexps(externalPrefix, paths...), + whitelist: false, + } +} + +// internalMatches returns a path matcher for internal packages. +func internalMatches() *pathRegexps { + return &pathRegexps{ + expr: buildRegexps(internalPrefix, ".*"), + whitelist: true, + } +} + +// resultExcluded excludes explicit message contents. +type resultExcluded []string + +// ShouldReport implements matcher.ShouldReport. +func (r resultExcluded) ShouldReport(d analysis.Diagnostic, _ *token.FileSet) bool { + for _, str := range r { + if strings.Contains(d.Message, str) { + return false + } + } + return true // Not blacklisted. +} + +// andMatcher is a composite matcher. +type andMatcher struct { + first matcher + second matcher +} + +// ShouldReport implements matcher.ShouldReport. +func (a *andMatcher) ShouldReport(d analysis.Diagnostic, fs *token.FileSet) bool { + return a.first.ShouldReport(d, fs) && a.second.ShouldReport(d, fs) +} + +// and is a syntactic convension for andMatcher. +func and(first matcher, second matcher) *andMatcher { + return &andMatcher{ + first: first, + second: second, + } +} + +// anyMatcher matches everything. +type anyMatcher struct{} + +// ShouldReport implements matcher.ShouldReport. +func (anyMatcher) ShouldReport(analysis.Diagnostic, *token.FileSet) bool { + return true +} + +// alwaysMatches returns an anyMatcher instance. +func alwaysMatches() anyMatcher { + return anyMatcher{} +} + +// neverMatcher will never match. +type neverMatcher struct{} + +// ShouldReport implements matcher.ShouldReport. +func (neverMatcher) ShouldReport(analysis.Diagnostic, *token.FileSet) bool { + return false +} + +// disableMatches returns a neverMatcher instance. +func disableMatches() neverMatcher { + return neverMatcher{} +} diff --git a/tools/nogo/nogo.go b/tools/nogo/nogo.go new file mode 100644 index 000000000..203cdf688 --- /dev/null +++ b/tools/nogo/nogo.go @@ -0,0 +1,316 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package nogo implements binary analysis similar to bazel's nogo, +// or the unitchecker package. It exists in order to provide additional +// facilities for analysis, namely plumbing through the output from +// dumping the generated binary (to analyze actual produced code). +package nogo + +import ( + "encoding/json" + "flag" + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "go/types" + "io" + "io/ioutil" + "log" + "os" + "path/filepath" + "reflect" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/internal/facts" + "golang.org/x/tools/go/gcexportdata" + "gvisor.dev/gvisor/tools/nogo/data" +) + +// pkgConfig is serialized as the configuration. +// +// This contains everything required for the analysis. +type pkgConfig struct { + ImportPath string + GoFiles []string + NonGoFiles []string + Tags []string + GOOS string + GOARCH string + ImportMap map[string]string + FactMap map[string]string + FactOutput string + Objdump string +} + +// loadFacts finds and loads facts per FactMap. +func (c *pkgConfig) loadFacts(path string) ([]byte, error) { + realPath, ok := c.FactMap[path] + if !ok { + return nil, nil // No facts available. + } + + // Read the files file. + data, err := ioutil.ReadFile(realPath) + if err != nil { + return nil, err + } + return data, nil +} + +// shouldInclude indicates whether the file should be included. +// +// NOTE: This does only basic parsing of tags. +func (c *pkgConfig) shouldInclude(path string) (bool, error) { + ctx := build.Default + ctx.GOOS = c.GOOS + ctx.GOARCH = c.GOARCH + ctx.BuildTags = c.Tags + return ctx.MatchFile(filepath.Dir(path), filepath.Base(path)) +} + +// importer is an implementation of go/types.Importer. +// +// This wraps a configuration, which provides the map of package names to +// files, and the facts. Note that this importer implementation will always +// pass when a given package is not available. +type importer struct { + pkgConfig + fset *token.FileSet + cache map[string]*types.Package +} + +// Import implements types.Importer.Import. +func (i *importer) Import(path string) (*types.Package, error) { + if path == "unsafe" { + // Special case: go/types has pre-defined type information for + // unsafe. We ensure that this package is correct, in case any + // analyzers are specifically looking for this. + return types.Unsafe, nil + } + realPath, ok := i.ImportMap[path] + var ( + rc io.ReadCloser + err error + ) + if !ok { + // Not found in the import path. Attempt to find the package + // via the standard library. + rc, err = findStdPkg(path, i.GOOS, i.GOARCH) + } else { + // Open the file. + rc, err = os.Open(realPath) + } + if err != nil { + return nil, err + } + defer rc.Close() + + // Load all exported data. + r, err := gcexportdata.NewReader(rc) + if err != nil { + return nil, err + } + + return gcexportdata.Read(r, i.fset, i.cache, path) +} + +// checkPackage runs all analyzers. +// +// The implementation was adapted from [1], which was in turn adpated from [2]. +// This returns a list of matching analysis issues, or an error if the analysis +// could not be completed. +// +// [1] bazelbuid/rules_go/tools/builders/nogo_main.go +// [2] golang.org/x/tools/go/checker/internal/checker +func checkPackage(config pkgConfig) ([]string, error) { + imp := &importer{ + pkgConfig: config, + fset: token.NewFileSet(), + cache: make(map[string]*types.Package), + } + + // Load all source files. + var syntax []*ast.File + for _, file := range config.GoFiles { + include, err := config.shouldInclude(file) + if err != nil { + return nil, fmt.Errorf("error evaluating file %q: %v", file, err) + } + if !include { + continue + } + s, err := parser.ParseFile(imp.fset, file, nil, parser.ParseComments) + if err != nil { + return nil, fmt.Errorf("error parsing file %q: %v", file, err) + } + syntax = append(syntax, s) + } + + // Check type information. + typesSizes := types.SizesFor("gc", config.GOARCH) + typeConfig := types.Config{Importer: imp} + typesInfo := &types.Info{ + Types: make(map[ast.Expr]types.TypeAndValue), + Uses: make(map[*ast.Ident]types.Object), + Defs: make(map[*ast.Ident]types.Object), + Implicits: make(map[ast.Node]types.Object), + Scopes: make(map[ast.Node]*types.Scope), + Selections: make(map[*ast.SelectorExpr]*types.Selection), + } + types, err := typeConfig.Check(config.ImportPath, imp.fset, syntax, typesInfo) + if err != nil { + return nil, fmt.Errorf("error checking types: %v", err) + } + + // Load all package facts. + facts, err := facts.Decode(types, config.loadFacts) + if err != nil { + return nil, fmt.Errorf("error decoding facts: %v", err) + } + + // Set the binary global for use. + data.Objdump = config.Objdump + + // Register fact types and establish dependencies between analyzers. + // The visit closure will execute recursively, and populate results + // will all required analysis results. + diagnostics := make(map[*analysis.Analyzer][]analysis.Diagnostic) + results := make(map[*analysis.Analyzer]interface{}) + var visit func(*analysis.Analyzer) error // For recursion. + visit = func(a *analysis.Analyzer) error { + if _, ok := results[a]; ok { + return nil + } + + // Run recursively for all dependencies. + for _, req := range a.Requires { + if err := visit(req); err != nil { + return err + } + } + + // Prepare the matcher. + m := analyzerConfig[a] + report := func(d analysis.Diagnostic) { + if m.ShouldReport(d, imp.fset) { + diagnostics[a] = append(diagnostics[a], d) + } + } + + // Run the analysis. + factFilter := make(map[reflect.Type]bool) + for _, f := range a.FactTypes { + factFilter[reflect.TypeOf(f)] = true + } + p := &analysis.Pass{ + Analyzer: a, + Fset: imp.fset, + Files: syntax, + Pkg: types, + TypesInfo: typesInfo, + ResultOf: results, // All results. + Report: report, + ImportPackageFact: facts.ImportPackageFact, + ExportPackageFact: facts.ExportPackageFact, + ImportObjectFact: facts.ImportObjectFact, + ExportObjectFact: facts.ExportObjectFact, + AllPackageFacts: func() []analysis.PackageFact { return facts.AllPackageFacts(factFilter) }, + AllObjectFacts: func() []analysis.ObjectFact { return facts.AllObjectFacts(factFilter) }, + TypesSizes: typesSizes, + } + result, err := a.Run(p) + if err != nil { + return fmt.Errorf("error running analysis %s: %v", a, err) + } + + // Sanity check & save the result. + if got, want := reflect.TypeOf(result), a.ResultType; got != want { + return fmt.Errorf("error: analyzer %s returned a result of type %v, but declared ResultType %v", a, got, want) + } + results[a] = result + return nil // Success. + } + + // Visit all analysis recursively. + for a, _ := range analyzerConfig { + if err := visit(a); err != nil { + return nil, err // Already has context. + } + } + + // Write the output file. + if config.FactOutput != "" { + factData := facts.Encode() + if err := ioutil.WriteFile(config.FactOutput, factData, 0644); err != nil { + return nil, fmt.Errorf("error: unable to open facts output %q: %v", config.FactOutput, err) + } + } + + // Convert all diagnostics to strings. + findings := make([]string, 0, len(diagnostics)) + for a, ds := range diagnostics { + for _, d := range ds { + // Include the anlyzer name for debugability and configuration. + findings = append(findings, fmt.Sprintf("%s: %s: %s", a.Name, imp.fset.Position(d.Pos), d.Message)) + } + } + + // Return all findings. + return findings, nil +} + +var ( + configFile = flag.String("config", "", "configuration file (in JSON format)") +) + +// Main is the entrypoint; it should be called directly from main. +// +// N.B. This package registers it's own flags. +func Main() { + // Parse all flags. + flag.Parse() + + // Load the configuration. + f, err := os.Open(*configFile) + if err != nil { + log.Fatalf("unable to open configuration %q: %v", *configFile, err) + } + defer f.Close() + config := new(pkgConfig) + dec := json.NewDecoder(f) + dec.DisallowUnknownFields() + if err := dec.Decode(config); err != nil { + log.Fatalf("unable to decode configuration: %v", err) + } + + // Process the package. + findings, err := checkPackage(*config) + if err != nil { + log.Fatalf("error checking package: %v", err) + } + + // No findings? + if len(findings) == 0 { + os.Exit(0) + } + + // Print findings and exit with non-zero code. + for _, finding := range findings { + fmt.Fprintf(os.Stdout, "%s\n", finding) + } + os.Exit(1) +} diff --git a/tools/nogo/register.go b/tools/nogo/register.go new file mode 100644 index 000000000..62b499661 --- /dev/null +++ b/tools/nogo/register.go @@ -0,0 +1,64 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nogo + +import ( + "encoding/gob" + "log" + + "golang.org/x/tools/go/analysis" +) + +// analyzers returns all configured analyzers. +func analyzers() (all []*analysis.Analyzer) { + for a, _ := range analyzerConfig { + all = append(all, a) + } + return all +} + +func init() { + // Validate basic configuration. + if err := analysis.Validate(analyzers()); err != nil { + log.Fatalf("unable to validate analyzer: %v", err) + } + + // Register all fact types. + // + // N.B. This needs to be done recursively, because there may be + // analyzers in the Requires list that do not appear explicitly above. + registered := make(map[*analysis.Analyzer]struct{}) + var register func(*analysis.Analyzer) + register = func(a *analysis.Analyzer) { + if _, ok := registered[a]; ok { + return + } + + // Regsiter dependencies. + for _, da := range a.Requires { + register(da) + } + + // Register local facts. + for _, f := range a.FactTypes { + gob.Register(f) + } + + registered[a] = struct{}{} // Done. + } + for _, a := range analyzers() { + register(a) + } +} -- cgit v1.2.3 From 7c0f3bc8576addbec001095d754a756691d26df3 Mon Sep 17 00:00:00 2001 From: Dave Bailey Date: Tue, 21 Apr 2020 09:34:42 -0700 Subject: Sentry metrics updates. Sentry metrics with nanoseconds units are labeled as such, and non-cumulative sentry metrics are supported. PiperOrigin-RevId: 307621080 --- pkg/metric/metric.go | 41 +++++++++++++++++----------------- pkg/metric/metric.proto | 10 ++++++++- pkg/metric/metric_test.go | 22 +++++++++++------- pkg/sentry/fs/file.go | 2 +- pkg/sentry/fs/gofer/file.go | 4 ++-- pkg/sentry/fs/tmpfs/inode_file.go | 2 +- pkg/sentry/socket/netstack/netstack.go | 14 ++++++++---- 7 files changed, 58 insertions(+), 37 deletions(-) (limited to 'pkg/metric/metric.go') diff --git a/pkg/metric/metric.go b/pkg/metric/metric.go index 895253625..64aa365ce 100644 --- a/pkg/metric/metric.go +++ b/pkg/metric/metric.go @@ -39,16 +39,11 @@ var ( // Uint64Metric encapsulates a uint64 that represents some kind of metric to be // monitored. // -// All metrics must be cumulative, meaning that their values will only increase -// over time. -// // Metrics are not saved across save/restore and thus reset to zero on restore. // -// TODO(b/67298402): Support non-cumulative metrics. // TODO(b/67298427): Support metric fields. type Uint64Metric struct { - // value is the actual value of the metric. It must be accessed - // atomically. + // value is the actual value of the metric. It must be accessed atomically. value uint64 } @@ -110,13 +105,10 @@ type customUint64Metric struct { // Register must only be called at init and will return and error if called // after Initialized. // -// All metrics must be cumulative, meaning that the return values of value must -// only increase over time. -// // Preconditions: // * name must be globally unique. // * Initialize/Disable have not been called. -func RegisterCustomUint64Metric(name string, sync bool, description string, value func() uint64) error { +func RegisterCustomUint64Metric(name string, cumulative, sync bool, units pb.MetricMetadata_Units, description string, value func() uint64) error { if initialized { return ErrInitializationDone } @@ -129,9 +121,10 @@ func RegisterCustomUint64Metric(name string, sync bool, description string, valu metadata: &pb.MetricMetadata{ Name: name, Description: description, - Cumulative: true, + Cumulative: cumulative, Sync: sync, - Type: pb.MetricMetadata_UINT64, + Type: pb.MetricMetadata_TYPE_UINT64, + Units: units, }, value: value, } @@ -140,24 +133,32 @@ func RegisterCustomUint64Metric(name string, sync bool, description string, valu // MustRegisterCustomUint64Metric calls RegisterCustomUint64Metric and panics // if it returns an error. -func MustRegisterCustomUint64Metric(name string, sync bool, description string, value func() uint64) { - if err := RegisterCustomUint64Metric(name, sync, description, value); err != nil { +func MustRegisterCustomUint64Metric(name string, cumulative, sync bool, description string, value func() uint64) { + if err := RegisterCustomUint64Metric(name, cumulative, sync, pb.MetricMetadata_UNITS_NONE, description, value); err != nil { panic(fmt.Sprintf("Unable to register metric %q: %v", name, err)) } } -// NewUint64Metric creates and registers a new metric with the given name. +// NewUint64Metric creates and registers a new cumulative metric with the given name. // // Metrics must be statically defined (i.e., at init). -func NewUint64Metric(name string, sync bool, description string) (*Uint64Metric, error) { +func NewUint64Metric(name string, sync bool, units pb.MetricMetadata_Units, description string) (*Uint64Metric, error) { var m Uint64Metric - return &m, RegisterCustomUint64Metric(name, sync, description, m.Value) + return &m, RegisterCustomUint64Metric(name, true /* cumulative */, sync, units, description, m.Value) } -// MustCreateNewUint64Metric calls NewUint64Metric and panics if it returns an -// error. +// MustCreateNewUint64Metric calls NewUint64Metric and panics if it returns an error. func MustCreateNewUint64Metric(name string, sync bool, description string) *Uint64Metric { - m, err := NewUint64Metric(name, sync, description) + m, err := NewUint64Metric(name, sync, pb.MetricMetadata_UNITS_NONE, description) + if err != nil { + panic(fmt.Sprintf("Unable to create metric %q: %v", name, err)) + } + return m +} + +// MustCreateNewUint64NanosecondsMetric calls NewUint64Metric and panics if it returns an error. +func MustCreateNewUint64NanosecondsMetric(name string, sync bool, description string) *Uint64Metric { + m, err := NewUint64Metric(name, sync, pb.MetricMetadata_UNITS_NANOSECONDS, description) if err != nil { panic(fmt.Sprintf("Unable to create metric %q: %v", name, err)) } diff --git a/pkg/metric/metric.proto b/pkg/metric/metric.proto index a2c2bd1ba..3cc89047d 100644 --- a/pkg/metric/metric.proto +++ b/pkg/metric/metric.proto @@ -36,10 +36,18 @@ message MetricMetadata { // the monitoring system. bool sync = 4; - enum Type { UINT64 = 0; } + enum Type { TYPE_UINT64 = 0; } // type is the type of the metric value. Type type = 5; + + enum Units { + UNITS_NONE = 0; + UNITS_NANOSECONDS = 1; + } + + // units is the units of the metric value. + Units units = 6; } // MetricRegistration contains the metadata for all metrics that will be in diff --git a/pkg/metric/metric_test.go b/pkg/metric/metric_test.go index 34969385a..c425ea532 100644 --- a/pkg/metric/metric_test.go +++ b/pkg/metric/metric_test.go @@ -66,12 +66,12 @@ const ( func TestInitialize(t *testing.T) { defer reset() - _, err := NewUint64Metric("/foo", false, fooDescription) + _, err := NewUint64Metric("/foo", false, pb.MetricMetadata_UNITS_NONE, fooDescription) if err != nil { t.Fatalf("NewUint64Metric got err %v want nil", err) } - _, err = NewUint64Metric("/bar", true, barDescription) + _, err = NewUint64Metric("/bar", true, pb.MetricMetadata_UNITS_NANOSECONDS, barDescription) if err != nil { t.Fatalf("NewUint64Metric got err %v want nil", err) } @@ -94,8 +94,8 @@ func TestInitialize(t *testing.T) { foundFoo := false foundBar := false for _, m := range mr.Metrics { - if m.Type != pb.MetricMetadata_UINT64 { - t.Errorf("Metadata %+v Type got %v want %v", m, m.Type, pb.MetricMetadata_UINT64) + if m.Type != pb.MetricMetadata_TYPE_UINT64 { + t.Errorf("Metadata %+v Type got %v want %v", m, m.Type, pb.MetricMetadata_TYPE_UINT64) } if !m.Cumulative { t.Errorf("Metadata %+v Cumulative got false want true", m) @@ -110,6 +110,9 @@ func TestInitialize(t *testing.T) { if m.Sync { t.Errorf("/foo %+v Sync got true want false", m) } + if m.Units != pb.MetricMetadata_UNITS_NONE { + t.Errorf("/foo %+v Units got %v want %v", m, m.Units, pb.MetricMetadata_UNITS_NONE) + } case "/bar": foundBar = true if m.Description != barDescription { @@ -118,6 +121,9 @@ func TestInitialize(t *testing.T) { if !m.Sync { t.Errorf("/bar %+v Sync got true want false", m) } + if m.Units != pb.MetricMetadata_UNITS_NANOSECONDS { + t.Errorf("/bar %+v Units got %v want %v", m, m.Units, pb.MetricMetadata_UNITS_NANOSECONDS) + } } } @@ -132,12 +138,12 @@ func TestInitialize(t *testing.T) { func TestDisable(t *testing.T) { defer reset() - _, err := NewUint64Metric("/foo", false, fooDescription) + _, err := NewUint64Metric("/foo", false, pb.MetricMetadata_UNITS_NONE, fooDescription) if err != nil { t.Fatalf("NewUint64Metric got err %v want nil", err) } - _, err = NewUint64Metric("/bar", true, barDescription) + _, err = NewUint64Metric("/bar", true, pb.MetricMetadata_UNITS_NONE, barDescription) if err != nil { t.Fatalf("NewUint64Metric got err %v want nil", err) } @@ -161,12 +167,12 @@ func TestDisable(t *testing.T) { func TestEmitMetricUpdate(t *testing.T) { defer reset() - foo, err := NewUint64Metric("/foo", false, fooDescription) + foo, err := NewUint64Metric("/foo", false, pb.MetricMetadata_UNITS_NONE, fooDescription) if err != nil { t.Fatalf("NewUint64Metric got err %v want nil", err) } - _, err = NewUint64Metric("/bar", true, barDescription) + _, err = NewUint64Metric("/bar", true, pb.MetricMetadata_UNITS_NONE, barDescription) if err != nil { t.Fatalf("NewUint64Metric got err %v want nil", err) } diff --git a/pkg/sentry/fs/file.go b/pkg/sentry/fs/file.go index 78100e448..846252c89 100644 --- a/pkg/sentry/fs/file.go +++ b/pkg/sentry/fs/file.go @@ -44,7 +44,7 @@ var ( RecordWaitTime = false reads = metric.MustCreateNewUint64Metric("/fs/reads", false /* sync */, "Number of file reads.") - readWait = metric.MustCreateNewUint64Metric("/fs/read_wait", false /* sync */, "Time waiting on file reads, in nanoseconds.") + readWait = metric.MustCreateNewUint64NanosecondsMetric("/fs/read_wait", false /* sync */, "Time waiting on file reads, in nanoseconds.") ) // IncrementWait increments the given wait time metric, if enabled. diff --git a/pkg/sentry/fs/gofer/file.go b/pkg/sentry/fs/gofer/file.go index 23296f246..b2fcab127 100644 --- a/pkg/sentry/fs/gofer/file.go +++ b/pkg/sentry/fs/gofer/file.go @@ -37,9 +37,9 @@ var ( opens9P = metric.MustCreateNewUint64Metric("/gofer/opens_9p", false /* sync */, "Number of times a 9P file was opened from a gofer.") opensHost = metric.MustCreateNewUint64Metric("/gofer/opens_host", false /* sync */, "Number of times a host file was opened from a gofer.") reads9P = metric.MustCreateNewUint64Metric("/gofer/reads_9p", false /* sync */, "Number of 9P file reads from a gofer.") - readWait9P = metric.MustCreateNewUint64Metric("/gofer/read_wait_9p", false /* sync */, "Time waiting on 9P file reads from a gofer, in nanoseconds.") + readWait9P = metric.MustCreateNewUint64NanosecondsMetric("/gofer/read_wait_9p", false /* sync */, "Time waiting on 9P file reads from a gofer, in nanoseconds.") readsHost = metric.MustCreateNewUint64Metric("/gofer/reads_host", false /* sync */, "Number of host file reads from a gofer.") - readWaitHost = metric.MustCreateNewUint64Metric("/gofer/read_wait_host", false /* sync */, "Time waiting on host file reads from a gofer, in nanoseconds.") + readWaitHost = metric.MustCreateNewUint64NanosecondsMetric("/gofer/read_wait_host", false /* sync */, "Time waiting on host file reads from a gofer, in nanoseconds.") ) // fileOperations implements fs.FileOperations for a remote file system. diff --git a/pkg/sentry/fs/tmpfs/inode_file.go b/pkg/sentry/fs/tmpfs/inode_file.go index 25abbc151..1dc75291d 100644 --- a/pkg/sentry/fs/tmpfs/inode_file.go +++ b/pkg/sentry/fs/tmpfs/inode_file.go @@ -39,7 +39,7 @@ var ( opensRO = metric.MustCreateNewUint64Metric("/in_memory_file/opens_ro", false /* sync */, "Number of times an in-memory file was opened in read-only mode.") opensW = metric.MustCreateNewUint64Metric("/in_memory_file/opens_w", false /* sync */, "Number of times an in-memory file was opened in write mode.") reads = metric.MustCreateNewUint64Metric("/in_memory_file/reads", false /* sync */, "Number of in-memory file reads.") - readWait = metric.MustCreateNewUint64Metric("/in_memory_file/read_wait", false /* sync */, "Time waiting on in-memory file reads, in nanoseconds.") + readWait = metric.MustCreateNewUint64NanosecondsMetric("/in_memory_file/read_wait", false /* sync */, "Time waiting on in-memory file reads, in nanoseconds.") ) // fileInodeOperations implements fs.InodeOperations for a regular tmpfs file. diff --git a/pkg/sentry/socket/netstack/netstack.go b/pkg/sentry/socket/netstack/netstack.go index 7ac38764d..d5879c10f 100644 --- a/pkg/sentry/socket/netstack/netstack.go +++ b/pkg/sentry/socket/netstack/netstack.go @@ -63,7 +63,13 @@ import ( func mustCreateMetric(name, description string) *tcpip.StatCounter { var cm tcpip.StatCounter - metric.MustRegisterCustomUint64Metric(name, false /* sync */, description, cm.Value) + metric.MustRegisterCustomUint64Metric(name, true /* cumulative */, false /* sync */, description, cm.Value) + return &cm +} + +func mustCreateGauge(name, description string) *tcpip.StatCounter { + var cm tcpip.StatCounter + metric.MustRegisterCustomUint64Metric(name, false /* cumulative */, false /* sync */, description, cm.Value) return &cm } @@ -151,10 +157,10 @@ var Metrics = tcpip.Stats{ TCP: tcpip.TCPStats{ ActiveConnectionOpenings: mustCreateMetric("/netstack/tcp/active_connection_openings", "Number of connections opened successfully via Connect."), PassiveConnectionOpenings: mustCreateMetric("/netstack/tcp/passive_connection_openings", "Number of connections opened successfully via Listen."), - CurrentEstablished: mustCreateMetric("/netstack/tcp/current_established", "Number of connections in ESTABLISHED state now."), - CurrentConnected: mustCreateMetric("/netstack/tcp/current_open", "Number of connections that are in connected state."), + CurrentEstablished: mustCreateGauge("/netstack/tcp/current_established", "Number of connections in ESTABLISHED state now."), + CurrentConnected: mustCreateGauge("/netstack/tcp/current_open", "Number of connections that are in connected state."), EstablishedResets: mustCreateMetric("/netstack/tcp/established_resets", "Number of times TCP connections have made a direct transition to the CLOSED state from either the ESTABLISHED state or the CLOSE-WAIT state"), - EstablishedClosed: mustCreateMetric("/netstack/tcp/established_closed", "number of times established TCP connections made a transition to CLOSED state."), + EstablishedClosed: mustCreateMetric("/netstack/tcp/established_closed", "Number of times established TCP connections made a transition to CLOSED state."), EstablishedTimedout: mustCreateMetric("/netstack/tcp/established_timedout", "Number of times an established connection was reset because of keep-alive time out."), ListenOverflowSynDrop: mustCreateMetric("/netstack/tcp/listen_overflow_syn_drop", "Number of times the listen queue overflowed and a SYN was dropped."), ListenOverflowAckDrop: mustCreateMetric("/netstack/tcp/listen_overflow_ack_drop", "Number of times the listen queue overflowed and the final ACK in the handshake was dropped."), -- cgit v1.2.3