diff options
Diffstat (limited to 'tools')
37 files changed, 1315 insertions, 409 deletions
diff --git a/tools/BUILD b/tools/BUILD index da83877b1..faf310676 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -5,5 +5,7 @@ package(licenses = ["notice"]) bzl_library( name = "defs_bzl", srcs = ["defs.bzl"], - visibility = ["//visibility:private"], + visibility = [ + "//:sandbox", + ], ) diff --git a/tools/bazel.mk b/tools/bazel.mk index 5cc1cdea2..25575c02c 100644 --- a/tools/bazel.mk +++ b/tools/bazel.mk @@ -19,6 +19,7 @@ SHELL=/bin/bash -o pipefail BRANCH_NAME := $(shell (git branch --show-current 2>/dev/null || \ git rev-parse --abbrev-ref HEAD 2>/dev/null) | \ xargs -n 1 basename 2>/dev/null) +BUILD_ROOT := $(CURDIR)/bazel-bin/ # Bazel container configuration (see below). USER ?= gvisor @@ -131,7 +132,7 @@ bazel-server-start: bazel-image ## Starts the bazel server. --workdir "$(CURDIR)" \ $(FULL_DOCKER_RUN_OPTIONS) \ $(BUILDER_IMAGE) \ - sh -c "tail -f --pid=\$$($(BAZEL) info server_pid)" + sh -c "tail -f --pid=\$$($(BAZEL) info server_pid) /dev/null" .PHONY: bazel-server-start bazel-shutdown: ## Shuts down a running bazel server. @@ -151,8 +152,9 @@ build_cmd = docker exec $(FULL_DOCKER_EXEC_OPTIONS) $(DOCKER_NAME) sh -o pipefai build_paths = $(build_cmd) 2>&1 \ | tee /proc/self/fd/2 \ - | grep -E "^ bazel-bin/" \ - | tr -d '\r' \ + | grep " bazel-bin/" \ + | sed "s/ /\n/g" \ + | strings -n 10 \ | awk '{$$1=$$1};1' \ | xargs -n 1 -I {} sh -c "$(1)" diff --git a/tools/bazeldefs/defs.bzl b/tools/bazeldefs/defs.bzl index dad5fc3b2..cf5b1dc0d 100644 --- a/tools/bazeldefs/defs.bzl +++ b/tools/bazeldefs/defs.bzl @@ -191,3 +191,6 @@ def default_installer(): def default_net_util(): return [] # Nothing needed. + +def coreutil(): + return [] # Nothing needed. diff --git a/tools/checkescape/checkescape.go b/tools/checkescape/checkescape.go index d98f5c3a1..523a42692 100644 --- a/tools/checkescape/checkescape.go +++ b/tools/checkescape/checkescape.go @@ -102,8 +102,8 @@ var ( // This may be set instead of Binary. Reader io.Reader - // Tool is the tool used to dump a binary. - tool = flag.String("dump_tool", "", "tool used to dump a binary") + // objdumpTool is the tool used to dump a binary. + objdumpTool = flag.String("objdump_tool", "", "tool used to dump a binary") ) // EscapeReason is an escape reason. @@ -387,7 +387,7 @@ func loadObjdump() (map[string][]string, error) { } // Construct our command. - cmd := exec.Command(*tool, args...) + cmd := exec.Command(*objdumpTool, args...) cmd.Stdin = stdin cmd.Stderr = os.Stderr out, err := cmd.StdoutPipe() @@ -398,7 +398,37 @@ func loadObjdump() (map[string][]string, error) { return nil, err } + // Identify calls by address or name. Note that this is also + // constructed dynamically below, as we encounted the addresses. + // This is because some of the functions (duffzero) may have + // jump targets in the middle of the function itself. + funcsAllowed := map[string]struct{}{ + "runtime.duffzero": struct{}{}, + "runtime.duffcopy": struct{}{}, + "runtime.racefuncenter": struct{}{}, + "runtime.gcWriteBarrier": struct{}{}, + "runtime.retpolineAX": struct{}{}, + "runtime.retpolineBP": struct{}{}, + "runtime.retpolineBX": struct{}{}, + "runtime.retpolineCX": struct{}{}, + "runtime.retpolineDI": struct{}{}, + "runtime.retpolineDX": struct{}{}, + "runtime.retpolineR10": struct{}{}, + "runtime.retpolineR11": struct{}{}, + "runtime.retpolineR12": struct{}{}, + "runtime.retpolineR13": struct{}{}, + "runtime.retpolineR14": struct{}{}, + "runtime.retpolineR15": struct{}{}, + "runtime.retpolineR8": struct{}{}, + "runtime.retpolineR9": struct{}{}, + "runtime.retpolineSI": struct{}{}, + "runtime.stackcheck": struct{}{}, + "runtime.settls": struct{}{}, + } + addrsAllowed := make(map[string]struct{}) + // Build the map. + nextFunc := "" // For funcsAllowed. m := make(map[string][]string) r := bufio.NewReader(out) NextLine: @@ -407,6 +437,19 @@ NextLine: if err != nil && err != io.EOF { return nil, err } + fields := strings.Fields(line) + + // Is this an "allowed" function definition? + if len(fields) >= 2 && fields[0] == "TEXT" { + nextFunc = strings.TrimSuffix(fields[1], "(SB)") + if _, ok := funcsAllowed[nextFunc]; !ok { + nextFunc = "" // Don't record addresses. + } + } + if nextFunc != "" && len(fields) > 2 { + // Save the given address (in hex form, as it appears). + addrsAllowed[fields[1]] = struct{}{} + } // We recognize lines corresponding to actual code (not the // symbol name or other metadata) and annotate them if they @@ -416,53 +459,31 @@ NextLine: // // Lines look like this (including the first space): // gohacks_unsafe.go:33 0xa39 488b442408 MOVQ 0x8(SP), AX - if len(line) > 0 && line[0] == ' ' { - fields := strings.Fields(line) + if len(fields) >= 5 && line[0] == ' ' { if !strings.Contains(fields[3], "CALL") { continue } - site := strings.TrimSpace(fields[0]) - var callStr string // Friendly string. - if len(fields) > 5 { - callStr = strings.Join(fields[5:], " ") - } - if len(callStr) == 0 { - // Just a raw call? is this asm? - callStr = strings.Join(fields[3:], " ") - } - - // Ignore strings containing duffzero, which is just - // used by stack allocations for types that are large - // enough to warrant Duff's device. - if strings.Contains(callStr, "runtime.duffzero") || - strings.Contains(callStr, "runtime.duffcopy") { - continue - } - - // Ignore the racefuncenter call, which is used for - // race builds. This does not escape. - if strings.Contains(callStr, "runtime.racefuncenter") { - continue - } + site := fields[0] + target := strings.TrimSuffix(fields[4], "(SB)") - // Ignore the write barriers. - if strings.Contains(callStr, "runtime.gcWriteBarrier") { + // Ignore strings containing allowed functions. + if _, ok := funcsAllowed[target]; ok { continue } - - // Ignore retpolines. - if strings.Contains(callStr, "runtime.retpoline") { + if _, ok := addrsAllowed[target]; ok { continue } - - // Ignore stack sanity check (does not split). - if strings.Contains(callStr, "runtime.stackcheck") { - continue - } - - // Ignore tls functions. - if strings.Contains(callStr, "runtime.settls") { - continue + if len(fields) > 5 { + // This may be a future relocation. Some + // objdump versions describe this differently. + // If it contains any of the functions allowed + // above as a string, we let it go. + softTarget := strings.Join(fields[5:], " ") + for name := range funcsAllowed { + if strings.Contains(softTarget, name) { + continue NextLine + } + } } // Does this exist already? @@ -471,11 +492,11 @@ NextLine: existing = make([]string, 0, 1) } for _, other := range existing { - if callStr == other { + if target == other { continue NextLine } } - existing = append(existing, callStr) + existing = append(existing, target) m[site] = existing // Update. } if err == io.EOF { @@ -483,12 +504,25 @@ NextLine: } } + // Zap any accidental false positives. + final := make(map[string][]string) + for site, calls := range m { + filteredCalls := make([]string, 0, len(calls)) + for _, call := range calls { + if _, ok := addrsAllowed[call]; ok { + continue // Omit this call. + } + filteredCalls = append(filteredCalls, call) + } + final[site] = filteredCalls + } + // Wait for the dump to finish. if err := cmd.Wait(); err != nil { return nil, err } - return m, nil + return final, nil } // poser is a type that implements Pos. diff --git a/tools/defs.bzl b/tools/defs.bzl index 290d564f2..e2c72a1f6 100644 --- a/tools/defs.bzl +++ b/tools/defs.bzl @@ -7,7 +7,7 @@ change for Google-internal and bazel-compatible rules. load("//tools/go_stateify:defs.bzl", "go_stateify") load("//tools/go_marshal:defs.bzl", "go_marshal", "marshal_deps", "marshal_test_deps") -load("//tools/bazeldefs:defs.bzl", _build_test = "build_test", _bzl_library = "bzl_library", _cc_binary = "cc_binary", _cc_flags_supplier = "cc_flags_supplier", _cc_grpc_library = "cc_grpc_library", _cc_library = "cc_library", _cc_proto_library = "cc_proto_library", _cc_test = "cc_test", _cc_toolchain = "cc_toolchain", _default_installer = "default_installer", _default_net_util = "default_net_util", _gazelle = "gazelle", _gbenchmark = "gbenchmark", _go_binary = "go_binary", _go_embed_data = "go_embed_data", _go_grpc_and_proto_libraries = "go_grpc_and_proto_libraries", _go_library = "go_library", _go_path = "go_path", _go_proto_library = "go_proto_library", _go_test = "go_test", _grpcpp = "grpcpp", _gtest = "gtest", _loopback = "loopback", _pkg_deb = "pkg_deb", _pkg_tar = "pkg_tar", _proto_library = "proto_library", _py_binary = "py_binary", _rbe_platform = "rbe_platform", _rbe_toolchain = "rbe_toolchain", _select_arch = "select_arch", _select_system = "select_system", _short_path = "short_path", _vdso_linker_option = "vdso_linker_option") +load("//tools/bazeldefs:defs.bzl", _build_test = "build_test", _bzl_library = "bzl_library", _cc_binary = "cc_binary", _cc_flags_supplier = "cc_flags_supplier", _cc_grpc_library = "cc_grpc_library", _cc_library = "cc_library", _cc_proto_library = "cc_proto_library", _cc_test = "cc_test", _cc_toolchain = "cc_toolchain", _coreutil = "coreutil", _default_installer = "default_installer", _default_net_util = "default_net_util", _gazelle = "gazelle", _gbenchmark = "gbenchmark", _go_binary = "go_binary", _go_embed_data = "go_embed_data", _go_grpc_and_proto_libraries = "go_grpc_and_proto_libraries", _go_library = "go_library", _go_path = "go_path", _go_proto_library = "go_proto_library", _go_test = "go_test", _grpcpp = "grpcpp", _gtest = "gtest", _loopback = "loopback", _pkg_deb = "pkg_deb", _pkg_tar = "pkg_tar", _proto_library = "proto_library", _py_binary = "py_binary", _rbe_platform = "rbe_platform", _rbe_toolchain = "rbe_toolchain", _select_arch = "select_arch", _select_system = "select_system", _short_path = "short_path", _vdso_linker_option = "vdso_linker_option") load("//tools/bazeldefs:platforms.bzl", _default_platform = "default_platform", _platforms = "platforms") load("//tools/bazeldefs:tags.bzl", "go_suffixes") load("//tools/nogo:defs.bzl", "nogo_test") @@ -39,6 +39,7 @@ short_path = _short_path rbe_platform = _rbe_platform rbe_toolchain = _rbe_toolchain vdso_linker_option = _vdso_linker_option +coreutil = _coreutil # Platform options. default_platform = _default_platform @@ -71,7 +72,8 @@ def go_binary(name, nogo = True, pure = False, static = False, x_defs = None, ** ) nogo_test( name = name + "_nogo", - deps = [":" + name + "_nogo_library"], + srcs = kwargs.get("srcs", []), + library = ":" + name + "_nogo_library", ) def calculate_sets(srcs): @@ -202,7 +204,8 @@ def go_library(name, srcs, deps = [], imports = [], stateify = True, marshal = F if nogo: nogo_test( name = name + "_nogo", - deps = [":" + name], + srcs = all_srcs, + library = ":" + name, ) if marshal: @@ -214,7 +217,10 @@ def go_library(name, srcs, deps = [], imports = [], stateify = True, marshal = F for (suffix, _) in marshal_sets.items(): _go_test( name = name + suffix + "_abi_autogen_test", - srcs = [name + suffix + "_abi_autogen_test.go"], + srcs = [ + name + suffix + "_abi_autogen_test.go", + name + suffix + "_abi_autogen_unconditional_test.go", + ], library = ":" + name, deps = marshal_test_deps, **kwargs @@ -235,7 +241,8 @@ def go_test(name, nogo = True, **kwargs): if nogo: nogo_test( name = name + "_nogo", - deps = [":" + name], + srcs = kwargs.get("srcs", []), + library = ":" + name, ) def proto_library(name, srcs, deps = None, has_services = 0, **kwargs): diff --git a/tools/github/BUILD b/tools/github/BUILD new file mode 100644 index 000000000..aad088d13 --- /dev/null +++ b/tools/github/BUILD @@ -0,0 +1,15 @@ +load("//tools:defs.bzl", "go_binary") + +package(licenses = ["notice"]) + +go_binary( + name = "github", + srcs = ["main.go"], + nogo = False, + deps = [ + "//tools/github/nogo", + "//tools/github/reviver", + "@com_github_google_go_github_v28//github:go_default_library", + "@org_golang_x_oauth2//:go_default_library", + ], +) diff --git a/tools/github/main.go b/tools/github/main.go new file mode 100644 index 000000000..7a74dc033 --- /dev/null +++ b/tools/github/main.go @@ -0,0 +1,162 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Binary github is the entry point for GitHub utilities. +package main + +import ( + "context" + "flag" + "fmt" + "io/ioutil" + "os" + "os/exec" + "strings" + + "github.com/google/go-github/github" + "golang.org/x/oauth2" + "gvisor.dev/gvisor/tools/github/nogo" + "gvisor.dev/gvisor/tools/github/reviver" +) + +var ( + owner string + repo string + tokenFile string + path string + commit string + dryRun bool +) + +// Keep the options simple for now. Supports only a single path and repo. +func init() { + flag.StringVar(&owner, "owner", "", "GitHub project org/owner (required, except nogo dry-run)") + flag.StringVar(&repo, "repo", "", "GitHub repo (required, except nogo dry-run)") + flag.StringVar(&tokenFile, "oauth-token-file", "", "file containing the GitHub token (or GITHUB_TOKEN is set)") + flag.StringVar(&path, "path", ".", "path to scan (required for revive and nogo)") + flag.StringVar(&commit, "commit", "", "commit to associated (required for nogo, except dry-run)") + flag.BoolVar(&dryRun, "dry-run", false, "just print changes to be made") +} + +func main() { + // Set defaults from the environment. + repository := os.Getenv("GITHUB_REPOSITORY") + if parts := strings.SplitN(repository, "/", 2); len(parts) == 2 { + owner = parts[0] + repo = parts[1] + } + + // Parse flags. + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "usage: %s [options] <command>\n", os.Args[0]) + fmt.Fprintf(flag.CommandLine.Output(), "commands: revive, nogo\n") + flag.PrintDefaults() + } + flag.Parse() + args := flag.Args() + if len(args) != 1 { + fmt.Fprintf(flag.CommandLine.Output(), "extra arguments: %s\n", strings.Join(args[1:], ", ")) + flag.Usage() + os.Exit(1) + } + + // Check for mandatory parameters. + command := args[0] + if len(owner) == 0 && (command != "nogo" || !dryRun) { + fmt.Fprintln(flag.CommandLine.Output(), "missing --owner option.") + flag.Usage() + os.Exit(1) + } + if len(repo) == 0 && (command != "nogo" || !dryRun) { + fmt.Fprintln(flag.CommandLine.Output(), "missing --repo option.") + flag.Usage() + os.Exit(1) + } + if len(path) == 0 { + fmt.Fprintln(flag.CommandLine.Output(), "missing --path option.") + flag.Usage() + os.Exit(1) + } + + // The access token may be passed as a file so it doesn't show up in + // command line arguments. It also may be provided through the + // environment to faciliate use through GitHub's CI system. + token := os.Getenv("GITHUB_TOKEN") + if len(tokenFile) != 0 { + bytes, err := ioutil.ReadFile(tokenFile) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + token = string(bytes) + } + var client *github.Client + if len(token) == 0 { + // Client is unauthenticated. + client = github.NewClient(nil) + } else { + // Using the above token. + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + tc := oauth2.NewClient(context.Background(), ts) + client = github.NewClient(tc) + } + + switch command { + case "revive": + // Load existing GitHub bugs. + bugger, err := reviver.NewGitHubBugger(client, owner, repo, dryRun) + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting github issues: %v\n", err) + os.Exit(1) + } + // Scan the provided path. + rev := reviver.New([]string{path}, []reviver.Bugger{bugger}) + if errs := rev.Run(); len(errs) > 0 { + fmt.Fprintf(os.Stderr, "Encountered %d errors:\n", len(errs)) + for _, err := range errs { + fmt.Fprintf(os.Stderr, "\t%v\n", err) + } + os.Exit(1) + } + case "nogo": + // Did we get a commit? Try to extract one. + if len(commit) == 0 && !dryRun { + cmd := exec.Command("git", "rev-parse", "HEAD") + revBytes, err := cmd.Output() + if err != nil { + fmt.Fprintf(flag.CommandLine.Output(), "missing --commit option, unable to infer: %v\n", err) + flag.Usage() + os.Exit(1) + } + commit = strings.TrimSpace(string(revBytes)) + } + // Scan all findings. + poster := nogo.NewFindingsPoster(client, owner, repo, commit, dryRun) + if err := poster.Walk(path); err != nil { + fmt.Fprintln(os.Stderr, "Error finding nogo findings:", err) + os.Exit(1) + } + // Post to GitHub. + if err := poster.Post(); err != nil { + fmt.Fprintln(os.Stderr, "Error posting nogo findings:", err) + } + default: + // Not a known command. + fmt.Fprintf(flag.CommandLine.Output(), "unknown command: %s\n", command) + flag.Usage() + os.Exit(1) + } +} diff --git a/tools/github/nogo/BUILD b/tools/github/nogo/BUILD new file mode 100644 index 000000000..0633eaf19 --- /dev/null +++ b/tools/github/nogo/BUILD @@ -0,0 +1,16 @@ +load("//tools:defs.bzl", "go_library") + +package(licenses = ["notice"]) + +go_library( + name = "nogo", + srcs = ["nogo.go"], + nogo = False, + visibility = [ + "//tools/github:__subpackages__", + ], + deps = [ + "//tools/nogo/util", + "@com_github_google_go_github_v28//github:go_default_library", + ], +) diff --git a/tools/github/nogo/nogo.go b/tools/github/nogo/nogo.go new file mode 100644 index 000000000..b70dfe63b --- /dev/null +++ b/tools/github/nogo/nogo.go @@ -0,0 +1,126 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package nogo provides nogo-related utilities. +package nogo + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/google/go-github/github" + "gvisor.dev/gvisor/tools/nogo/util" +) + +// FindingsPoster is a simple wrapper around the GitHub api. +type FindingsPoster struct { + owner string + repo string + commit string + dryRun bool + startTime time.Time + + findings map[util.Finding]struct{} + client *github.Client +} + +// NewFindingsPoster returns a object that can post findings. +func NewFindingsPoster(client *github.Client, owner, repo, commit string, dryRun bool) *FindingsPoster { + return &FindingsPoster{ + owner: owner, + repo: repo, + commit: commit, + dryRun: dryRun, + startTime: time.Now(), + findings: make(map[util.Finding]struct{}), + client: client, + } +} + +// Walk walks the given path tree for findings files. +func (p *FindingsPoster) Walk(path string) error { + return filepath.Walk(path, func(filename string, info os.FileInfo, err error) error { + if err != nil { + return err + } + // Skip any directories or files not ending in .findings. + if !strings.HasSuffix(filename, ".findings") || info.IsDir() { + return nil + } + findings, err := util.ExtractFindingsFromFile(filename) + if err != nil { + return err + } + // Add all findings to the list. We use a map to ensure + // that each finding is unique. + for _, finding := range findings { + p.findings[finding] = struct{}{} + } + return nil + }) +} + +// Post posts all results to the GitHub API as a check run. +func (p *FindingsPoster) Post() error { + // Just show results? + if p.dryRun { + for finding, _ := range p.findings { + // Pretty print, so that this is useful for debugging. + fmt.Printf("%s: (%s+%d) %s\n", finding.Category, finding.Path, finding.Line, finding.Message) + } + return nil + } + + // Construct the message. + title := "nogo" + count := len(p.findings) + status := "completed" + conclusion := "success" + if count > 0 { + conclusion = "failure" // Contains errors. + } + summary := fmt.Sprintf("%d findings.", count) + opts := github.CreateCheckRunOptions{ + Name: title, + HeadSHA: p.commit, + Status: &status, + Conclusion: &conclusion, + StartedAt: &github.Timestamp{p.startTime}, + CompletedAt: &github.Timestamp{time.Now()}, + Output: &github.CheckRunOutput{ + Title: &title, + Summary: &summary, + AnnotationsCount: &count, + }, + } + annotationLevel := "failure" // Always. + for finding, _ := range p.findings { + opts.Output.Annotations = append(opts.Output.Annotations, &github.CheckRunAnnotation{ + Path: &finding.Path, + StartLine: &finding.Line, + EndLine: &finding.Line, + Message: &finding.Message, + Title: &finding.Category, + AnnotationLevel: &annotationLevel, + }) + } + + // Post to GitHub. + _, _, err := p.client.Checks.CreateCheckRun(context.Background(), p.owner, p.repo, opts) + return err +} diff --git a/tools/github/reviver/BUILD b/tools/github/reviver/BUILD new file mode 100644 index 000000000..7d78480a7 --- /dev/null +++ b/tools/github/reviver/BUILD @@ -0,0 +1,27 @@ +load("//tools:defs.bzl", "go_library", "go_test") + +package(licenses = ["notice"]) + +go_library( + name = "reviver", + srcs = [ + "github.go", + "reviver.go", + ], + nogo = False, + visibility = [ + "//tools/github:__subpackages__", + ], + deps = ["@com_github_google_go_github_v28//github:go_default_library"], +) + +go_test( + name = "reviver_test", + size = "small", + srcs = [ + "github_test.go", + "reviver_test.go", + ], + library = ":reviver", + nogo = False, +) diff --git a/tools/issue_reviver/github/github.go b/tools/github/reviver/github.go index 8ffd7e606..a95df0fb6 100644 --- a/tools/issue_reviver/github/github.go +++ b/tools/github/reviver/github.go @@ -12,8 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package github implements reviver.Bugger interface on top of Github issues. -package github +package reviver import ( "context" @@ -23,12 +22,10 @@ import ( "time" "github.com/google/go-github/github" - "golang.org/x/oauth2" - "gvisor.dev/gvisor/tools/issue_reviver/reviver" ) -// Bugger implements reviver.Bugger interface for github issues. -type Bugger struct { +// GitHubBugger implements Bugger interface for github issues. +type GitHubBugger struct { owner string repo string dryRun bool @@ -37,36 +34,25 @@ type Bugger struct { issues map[int]*github.Issue } -// NewBugger creates a new Bugger. -func NewBugger(token, owner, repo string, dryRun bool) (*Bugger, error) { - b := &Bugger{ +// NewGitHubBugger creates a new GitHubBugger. +func NewGitHubBugger(client *github.Client, owner, repo string, dryRun bool) (*GitHubBugger, error) { + b := &GitHubBugger{ owner: owner, repo: repo, dryRun: dryRun, issues: map[int]*github.Issue{}, + client: client, } - if err := b.load(token); err != nil { + if err := b.load(); err != nil { return nil, err } return b, nil } -func (b *Bugger) load(token string) error { - ctx := context.Background() - if len(token) == 0 { - fmt.Print("No OAUTH token provided, using unauthenticated account.\n") - b.client = github.NewClient(nil) - } else { - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - tc := oauth2.NewClient(ctx, ts) - b.client = github.NewClient(tc) - } - +func (b *GitHubBugger) load() error { err := processAllPages(func(listOpts github.ListOptions) (*github.Response, error) { opts := &github.IssueListByRepoOptions{State: "open", ListOptions: listOpts} - tmps, resp, err := b.client.Issues.ListByRepo(ctx, b.owner, b.repo, opts) + tmps, resp, err := b.client.Issues.ListByRepo(context.Background(), b.owner, b.repo, opts) if err != nil { return resp, err } @@ -83,8 +69,8 @@ func (b *Bugger) load(token string) error { return nil } -// Activate implements reviver.Bugger. -func (b *Bugger) Activate(todo *reviver.Todo) (bool, error) { +// Activate implements Bugger.Activate. +func (b *GitHubBugger) Activate(todo *Todo) (bool, error) { id, err := parseIssueNo(todo.Issue) if err != nil { return true, err diff --git a/tools/issue_reviver/github/github_test.go b/tools/github/reviver/github_test.go index a78b230ef..5df7e3624 100644 --- a/tools/issue_reviver/github/github_test.go +++ b/tools/github/reviver/github_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package github +package reviver import ( "testing" diff --git a/tools/issue_reviver/reviver/reviver.go b/tools/github/reviver/reviver.go index 2af7f0d59..2af7f0d59 100644 --- a/tools/issue_reviver/reviver/reviver.go +++ b/tools/github/reviver/reviver.go diff --git a/tools/issue_reviver/reviver/reviver_test.go b/tools/github/reviver/reviver_test.go index a9fb1f9f1..a9fb1f9f1 100644 --- a/tools/issue_reviver/reviver/reviver_test.go +++ b/tools/github/reviver/reviver_test.go diff --git a/tools/go_generics/go_merge/main.go b/tools/go_generics/go_merge/main.go index f6a331123..e0345500f 100644 --- a/tools/go_generics/go_merge/main.go +++ b/tools/go_generics/go_merge/main.go @@ -77,6 +77,7 @@ func main() { // Create a new declaration slice with all imports at the top, merging any // redundant imports. imports := make(map[string]*ast.ImportSpec) + var importNames []string // Keep imports in the original order to get deterministic output. var anonImports []*ast.ImportSpec for _, d := range f.Decls { if g, ok := d.(*ast.GenDecl); ok && g.Tok == token.IMPORT { @@ -98,6 +99,7 @@ func main() { } } else { imports[n] = i + importNames = append(importNames, n) } } } @@ -112,8 +114,8 @@ func main() { Lparen: token.NoPos + 1, Specs: make([]ast.Spec, 0, l), } - for _, i := range imports { - d.Specs = append(d.Specs, i) + for _, i := range importNames { + d.Specs = append(d.Specs, imports[i]) } for _, i := range anonImports { d.Specs = append(d.Specs, i) diff --git a/tools/go_generics/imports.go b/tools/go_generics/imports.go index 148dc7216..90d3aa1e0 100644 --- a/tools/go_generics/imports.go +++ b/tools/go_generics/imports.go @@ -21,6 +21,7 @@ import ( "go/format" "go/parser" "go/token" + "sort" "strconv" "gvisor.dev/gvisor/tools/go_generics/globals" @@ -132,10 +133,17 @@ func updateImports(maps []mapValue, imports mapValue) (ast.Decl, error) { if len(importsUsed) == 0 { return nil, nil } + var names []string + for n := range importsUsed { + names = append(names, n) + } + // Sort the new imports for deterministic build outputs. + sort.Strings(names) // Create spec array for each new import. specs := make([]ast.Spec, 0, len(importsUsed)) - for _, i := range importsUsed { + for _, n := range names { + i := importsUsed[n] specs = append(specs, &ast.ImportSpec{ Name: &ast.Ident{Name: i.newName}, Path: &ast.BasicLit{Value: i.path}, diff --git a/tools/go_marshal/README.md b/tools/go_marshal/README.md index 75e5c7888..d8045c295 100644 --- a/tools/go_marshal/README.md +++ b/tools/go_marshal/README.md @@ -113,3 +113,18 @@ The following are some guidelines for modifying the `go_marshal` tool: - No runtime reflection in the code generated for the marshallable interface. The entire point of the tool is to avoid runtime reflection. The generated tests may use reflection. + +## Debugging + +To enable debugging output from the go-marshal tool, use one of the following +options, depending on how go-marshal is being invoked: + +- Pass `--define gomarshal=verbose` to the bazel command. Note that this can + generate a lot of output depending on what's being compiled, as this will + enable debugging for all packages built by the command. + +- Set `marshal_debug = True` on the top-level `go_library` BUILD rule. + +- Set `debug = True` on the `go_marshal` BUILD rule. + +- Pass `-debug` to the go-marshal tool invocation. diff --git a/tools/go_marshal/defs.bzl b/tools/go_marshal/defs.bzl index ba98f3599..f44f83eab 100644 --- a/tools/go_marshal/defs.bzl +++ b/tools/go_marshal/defs.bzl @@ -4,11 +4,13 @@ def _go_marshal_impl(ctx): """Execute the go_marshal tool.""" output = ctx.outputs.lib output_test = ctx.outputs.test + output_test_unconditional = ctx.outputs.test_unconditional # Run the marshal command. args = ["-output=%s" % output.path] - args += ["-pkg=%s" % ctx.attr.package] - args += ["-output_test=%s" % output_test.path] + args.append("-pkg=%s" % ctx.attr.package) + args.append("-output_test=%s" % output_test.path) + args.append("-output_test_unconditional=%s" % output_test_unconditional.path) if ctx.attr.debug: args += ["-debug"] @@ -18,7 +20,7 @@ def _go_marshal_impl(ctx): args += [f.path for f in src.files.to_list()] ctx.actions.run( inputs = ctx.files.srcs, - outputs = [output, output_test], + outputs = [output, output_test, output_test_unconditional], mnemonic = "GoMarshal", progress_message = "go_marshal: %s" % ctx.label, arguments = args, @@ -48,6 +50,7 @@ go_marshal = rule( outputs = { "lib": "%{name}_unsafe.go", "test": "%{name}_test.go", + "test_unconditional": "%{name}_unconditional_test.go", }, ) diff --git a/tools/go_marshal/gomarshal/generator.go b/tools/go_marshal/gomarshal/generator.go index 72ed6d109..4a53d25be 100644 --- a/tools/go_marshal/gomarshal/generator.go +++ b/tools/go_marshal/gomarshal/generator.go @@ -68,6 +68,8 @@ type Generator struct { output *os.File // Output file to write generated tests. outputTest *os.File + // Output file to write unconditionally generated tests. + outputTestUC *os.File // Package name for the generated file. pkg string // Set of extra packages to import in the generated file. @@ -75,21 +77,26 @@ type Generator struct { } // NewGenerator creates a new code Generator. -func NewGenerator(srcs []string, out, outTest, pkg string, imports []string) (*Generator, error) { +func NewGenerator(srcs []string, out, outTest, outTestUnconditional, pkg string, imports []string) (*Generator, error) { f, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) if err != nil { - return nil, fmt.Errorf("Couldn't open output file %q: %v", out, err) + return nil, fmt.Errorf("couldn't open output file %q: %w", out, err) } fTest, err := os.OpenFile(outTest, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) if err != nil { - return nil, fmt.Errorf("Couldn't open test output file %q: %v", out, err) + return nil, fmt.Errorf("couldn't open test output file %q: %w", out, err) + } + fTestUC, err := os.OpenFile(outTestUnconditional, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return nil, fmt.Errorf("couldn't open unconditional test output file %q: %w", out, err) } g := Generator{ - inputs: srcs, - output: f, - outputTest: fTest, - pkg: pkg, - imports: newImportTable(), + inputs: srcs, + output: f, + outputTest: fTest, + outputTestUC: fTestUC, + pkg: pkg, + imports: newImportTable(), } for _, i := range imports { // All imports on the extra imports list are unconditionally marked as @@ -174,7 +181,7 @@ func (g *Generator) parse() ([]*ast.File, []*token.FileSet, error) { f, err := parser.ParseFile(fset, path, nil, parser.ParseComments) if err != nil { // Not a valid input file? - return nil, nil, fmt.Errorf("Input %q can't be parsed: %v", path, err) + return nil, nil, fmt.Errorf("input %q can't be parsed: %w", path, err) } if debugEnabled() { @@ -454,6 +461,46 @@ func (g *Generator) Run() error { // source file. func (g *Generator) writeTests(ts []*testGenerator) error { var b sourceBuffer + + // Write the unconditional test file. This file is always compiled, + // regardless of what build tags were specified on the original input + // files. We use this file to guarantee we never end up with an empty test + // file, as that causes the build to fail with "no tests/benchmarks/examples + // found". + // + // There's no easy way to determine ahead of time if we'll end up with an + // empty build file since build constraints can arbitrarily cause some of + // the original types to be not defined. We also have no way to tell bazel + // to omit the entire test suite since the output files are already defined + // before go-marshal is called. + b.emit("// Automatically generated marshal tests. See tools/go_marshal.\n\n") + b.emit("package %s\n\n", g.pkg) + b.emit("func Example() {\n") + b.inIndent(func() { + b.emit("// This example is intentionally empty, and ensures this package contains at\n") + b.emit("// least one testable entity. go-marshal is forced to emit a test package if the\n") + b.emit("// input package is marked marshallable, but emitting no testable entities \n") + b.emit("// results in a build failure.\n") + }) + b.emit("}\n") + if err := b.write(g.outputTestUC); err != nil { + return err + } + + // Now generate the real test file that contains the real types we + // processed. These need to be conditionally compiled according to the build + // tags, as the original types may not be defined under all build + // configurations. + + b.reset() + b.emit("// Automatically generated marshal tests. See tools/go_marshal.\n\n") + + // Emit build tags. + if t := tags.Aggregate(g.inputs); len(t) > 0 { + b.emit(strings.Join(t.Lines(), "\n")) + b.emit("\n\n") + } + b.emit("package %s\n\n", g.pkg) if err := b.write(g.outputTest); err != nil { return err @@ -470,26 +517,6 @@ func (g *Generator) writeTests(ts []*testGenerator) error { } // Write test functions. - - // If we didn't generate any Marshallable implementations, we can't just - // emit an empty test file, since that causes the build to fail with "no - // tests/benchmarks/examples found". Unfortunately we can't signal bazel to - // omit the entire package since the outputs are already defined before - // go-marshal is called. If we'd otherwise emit an empty test suite, emit an - // empty example instead. - if len(ts) == 0 { - b.reset() - b.emit("func Example() {\n") - b.inIndent(func() { - b.emit("// This example is intentionally empty to ensure this file contains at least\n") - b.emit("// one testable entity. go-marshal is forced to emit a test file if a package\n") - b.emit("// is marked marshallable, but emitting a test file with no entities results\n") - b.emit("// in a build failure.\n") - }) - b.emit("}\n") - return b.write(g.outputTest) - } - for _, t := range ts { if err := t.write(g.outputTest); err != nil { return err diff --git a/tools/go_marshal/gomarshal/generator_interfaces.go b/tools/go_marshal/gomarshal/generator_interfaces.go index cf76b5241..36447b86b 100644 --- a/tools/go_marshal/gomarshal/generator_interfaces.go +++ b/tools/go_marshal/gomarshal/generator_interfaces.go @@ -43,8 +43,8 @@ type interfaceGenerator struct { // of t's interfaces. ms map[string]struct{} - // as records embedded fields in t that are potentially not packed. The key - // is the accessor for the field. + // as records fields in t that are potentially not packed. The key is the + // accessor for the field. as map[string]struct{} } diff --git a/tools/go_marshal/gomarshal/generator_interfaces_struct.go b/tools/go_marshal/gomarshal/generator_interfaces_struct.go index d3fc1c1c6..fe76d3785 100644 --- a/tools/go_marshal/gomarshal/generator_interfaces_struct.go +++ b/tools/go_marshal/gomarshal/generator_interfaces_struct.go @@ -20,6 +20,7 @@ package gomarshal import ( "fmt" "go/ast" + "sort" "strings" ) @@ -40,6 +41,8 @@ func (g *interfaceGenerator) areFieldsPackedExpression() (string, bool) { for accessor, _ := range g.as { cs = append(cs, fmt.Sprintf("%s.Packed()", accessor)) } + // Sort expressions for determinstic build outputs. + sort.Strings(cs) return strings.Join(cs, " && "), true } @@ -48,12 +51,6 @@ func (g *interfaceGenerator) areFieldsPackedExpression() (string, bool) { // later. func (g *interfaceGenerator) validateStruct(ts *ast.TypeSpec, st *ast.StructType) { forEachStructField(st, func(f *ast.Field) { - if len(f.Names) == 0 { - g.abortAt(f.Pos(), "Cannot marshal structs with embedded fields, give the field a name; use '_' for anonymous fields such as padding fields") - } - }) - - forEachStructField(st, func(f *ast.Field) { fieldDispatcher{ primitive: func(_, t *ast.Ident) { g.validatePrimitiveNewtype(t) @@ -98,7 +95,7 @@ func (g *interfaceGenerator) emitMarshallableForStruct(st *ast.StructType) { var dynamicSizeTerms []string forEachStructField(st, fieldDispatcher{ - primitive: func(n, t *ast.Ident) { + primitive: func(_, t *ast.Ident) { if size, dynamic := g.scalarSize(t); !dynamic { primitiveSize += size } else { @@ -106,13 +103,13 @@ func (g *interfaceGenerator) emitMarshallableForStruct(st *ast.StructType) { dynamicSizeTerms = append(dynamicSizeTerms, fmt.Sprintf("(*%s)(nil).SizeBytes()", t.Name)) } }, - selector: func(n, tX, tSel *ast.Ident) { + selector: func(_, tX, tSel *ast.Ident) { tName := fmt.Sprintf("%s.%s", tX.Name, tSel.Name) g.recordUsedImport(tX.Name) g.recordUsedMarshallable(tName) dynamicSizeTerms = append(dynamicSizeTerms, fmt.Sprintf("(*%s)(nil).SizeBytes()", tName)) }, - array: func(n *ast.Ident, a *ast.ArrayType, t *ast.Ident) { + array: func(_ *ast.Ident, a *ast.ArrayType, t *ast.Ident) { lenExpr := g.arrayLenExpr(a) if size, dynamic := g.scalarSize(t); !dynamic { dynamicSizeTerms = append(dynamicSizeTerms, fmt.Sprintf("%d*%s", size, lenExpr)) diff --git a/tools/go_marshal/gomarshal/util.go b/tools/go_marshal/gomarshal/util.go index d94314302..6a42691cd 100644 --- a/tools/go_marshal/gomarshal/util.go +++ b/tools/go_marshal/gomarshal/util.go @@ -79,7 +79,7 @@ type fieldDispatcher struct { } // Precondition: All dispatch callbacks that will be invoked must be -// provided. Embedded fields are not allowed, len(f.Names) >= 1. +// provided. func (fd fieldDispatcher) dispatch(f *ast.Field) { // Each field declaration may actually be multiple declarations of the same // type. For example, consider: @@ -88,12 +88,24 @@ func (fd fieldDispatcher) dispatch(f *ast.Field) { // x, y, z int // } // - // We invoke the call-backs once per such instance. Embedded fields are not - // allowed, and results in a panic. + // We invoke the call-backs once per such instance. + + // Handle embedded fields. Embedded fields have no names, but can be + // referenced by the type name. if len(f.Names) < 1 { - panic("Precondition not met: attempted to dispatch on embedded field") + switch v := f.Type.(type) { + case *ast.Ident: + fd.primitive(v, v) + case *ast.SelectorExpr: + fd.selector(v.Sel, v.X.(*ast.Ident), v.Sel) + default: + // Note: Arrays can't be embedded, which is handled here. + panic(fmt.Sprintf("Attempted to dispatch on embedded field of unsupported kind: %#v", f.Type)) + } + return } + // Non-embedded field. for _, name := range f.Names { switch v := f.Type.(type) { case *ast.Ident: diff --git a/tools/go_marshal/main.go b/tools/go_marshal/main.go index f74be5c29..6e4a3e8c4 100644 --- a/tools/go_marshal/main.go +++ b/tools/go_marshal/main.go @@ -31,10 +31,11 @@ import ( ) var ( - pkg = flag.String("pkg", "", "output package") - output = flag.String("output", "", "output file") - outputTest = flag.String("output_test", "", "output file for tests") - imports = flag.String("imports", "", "comma-separated list of extra packages to import in generated code") + pkg = flag.String("pkg", "", "output package") + output = flag.String("output", "", "output file") + outputTest = flag.String("output_test", "", "output file for tests") + outputTestUnconditional = flag.String("output_test_unconditional", "", "output file for unconditional tests") + imports = flag.String("imports", "", "comma-separated list of extra packages to import in generated code") ) func main() { @@ -61,7 +62,7 @@ func main() { // as an import. extraImports = strings.Split(*imports, ",") } - g, err := gomarshal.NewGenerator(flag.Args(), *output, *outputTest, *pkg, extraImports) + g, err := gomarshal.NewGenerator(flag.Args(), *output, *outputTest, *outputTestUnconditional, *pkg, extraImports) if err != nil { panic(err) } diff --git a/tools/go_marshal/test/test.go b/tools/go_marshal/test/test.go index f75ca1b7f..d9e9f341b 100644 --- a/tools/go_marshal/test/test.go +++ b/tools/go_marshal/test/test.go @@ -174,3 +174,27 @@ type Type9 struct { x int64 y [sizeA]int32 } + +// Type10Embed is a test data type which is be embedded into another type. +// +// +marshal +type Type10Embed struct { + x int64 +} + +// Type10 is a test data type which contains an embedded struct. +// +// +marshal +type Type10 struct { + Type10Embed + y int64 +} + +// Type11 is a test data type which contains an embedded struct from an external +// package. +// +// +marshal +type Type11 struct { + ex.External + y int64 +} diff --git a/tools/go_stateify/main.go b/tools/go_stateify/main.go index 4f6ed208a..e1de12e25 100644 --- a/tools/go_stateify/main.go +++ b/tools/go_stateify/main.go @@ -39,7 +39,7 @@ var ( ) // resolveTypeName returns a qualified type name. -func resolveTypeName(name string, typ ast.Expr) (field string, qualified string) { +func resolveTypeName(typ ast.Expr) (field string, qualified string) { for done := false; !done; { // Resolve star expressions. switch rs := typ.(type) { @@ -69,11 +69,7 @@ func resolveTypeName(name string, typ ast.Expr) (field string, qualified string) } // Figure out actual type name. - ident, ok := typ.(*ast.Ident) - if !ok { - panic(fmt.Sprintf("type not supported: %s (involves anonymous types?)", name)) - } - field = ident.Name + field = typ.(*ast.Ident).Name qualified = qualified + field return } @@ -119,7 +115,7 @@ func scanFields(ss *ast.StructType, prefix string, fn scanFunctions) { } else { // Anonymous types can't be embedded, so we don't need // to worry about providing a useful name here. - name, _ = resolveTypeName("", field.Type) + name, _ = resolveTypeName(field.Type) } // Skip _ fields. @@ -214,9 +210,6 @@ func main() { emitRegister := func(name string) { initCalls = append(initCalls, fmt.Sprintf("%sRegister((*%s)(nil))", statePrefix, name)) } - emitZeroCheck := func(name string) { - fmt.Fprintf(outputFile, " if !%sIsZeroValue(&x.%s) { %sFailf(\"%s is %%#v, expected zero\", &x.%s) }\n", statePrefix, name, statePrefix, name, name) - } // Automated warning. fmt.Fprint(outputFile, "// automatically generated by stateify.\n\n") @@ -265,52 +258,39 @@ func main() { } type method struct { - receiver string - name string + typeName string + methodName string } - // Search for and add all methods with a pointer receiver and no other - // arguments to a set. We support auto-detecting the existence of - // several different methods with this signature. - simpleMethods := map[method]struct{}{} + // Search for and add all method to a set. We auto-detecting several + // different methods (and insert them if we don't find them, in order + // to ensure that expectations match reality). + // + // While we do this, figure out the right receiver name. If there are + // multiple distinct receivers, then we will just pick the last one. + simpleMethods := make(map[method]struct{}) + receiverNames := make(map[string]string) for _, f := range files { - // Go over all functions. for _, decl := range f.Decls { d, ok := decl.(*ast.FuncDecl) if !ok { continue } - if d.Name == nil || d.Recv == nil || d.Type == nil { + if d.Recv == nil || len(d.Recv.List) != 1 { // Not a named method. continue } - if len(d.Recv.List) != 1 { - // Wrong number of receivers? - continue - } - if d.Type.Params != nil && len(d.Type.Params.List) != 0 { - // Has argument(s). - continue - } - if d.Type.Results != nil && len(d.Type.Results.List) != 0 { - // Has return(s). - continue - } - pt, ok := d.Recv.List[0].Type.(*ast.StarExpr) - if !ok { - // Not a pointer receiver. - continue + // Save the method and the receiver. + name, _ := resolveTypeName(d.Recv.List[0].Type) + simpleMethods[method{ + typeName: name, + methodName: d.Name.Name, + }] = struct{}{} + if len(d.Recv.List[0].Names) > 0 { + receiverNames[name] = d.Recv.List[0].Names[0].Name } - - t, ok := pt.X.(*ast.Ident) - if !ok { - // This shouldn't happen with valid Go. - continue - } - - simpleMethods[method{t.Name, d.Name.Name}] = struct{}{} } } @@ -349,6 +329,11 @@ func main() { for _, gs := range d.Specs { ts := gs.(*ast.TypeSpec) + recv, ok := receiverNames[ts.Name.Name] + if !ok { + // Maybe no methods were defined? + recv = strings.ToLower(ts.Name.Name[:1]) + } switch x := ts.Type.(type) { case *ast.StructType: maybeEmitImports() @@ -365,29 +350,32 @@ func main() { emitField(name) } emitLoadValue := func(name, typName string) { - fmt.Fprintf(outputFile, " m.LoadValue(%d, new(%s), func(y interface{}) { x.load%s(y.(%s)) })\n", fields[name], typName, camelCased(name), typName) + fmt.Fprintf(outputFile, " stateSourceObject.LoadValue(%d, new(%s), func(y interface{}) { %s.load%s(y.(%s)) })\n", fields[name], typName, recv, camelCased(name), typName) } emitLoad := func(name string) { - fmt.Fprintf(outputFile, " m.Load(%d, &x.%s)\n", fields[name], name) + fmt.Fprintf(outputFile, " stateSourceObject.Load(%d, &%s.%s)\n", fields[name], recv, name) } emitLoadWait := func(name string) { - fmt.Fprintf(outputFile, " m.LoadWait(%d, &x.%s)\n", fields[name], name) + fmt.Fprintf(outputFile, " stateSourceObject.LoadWait(%d, &%s.%s)\n", fields[name], recv, name) } emitSaveValue := func(name, typName string) { - fmt.Fprintf(outputFile, " var %s %s = x.save%s()\n", name, typName, camelCased(name)) - fmt.Fprintf(outputFile, " m.SaveValue(%d, %s)\n", fields[name], name) + fmt.Fprintf(outputFile, " var %sValue %s = %s.save%s()\n", name, typName, recv, camelCased(name)) + fmt.Fprintf(outputFile, " stateSinkObject.SaveValue(%d, %sValue)\n", fields[name], name) } emitSave := func(name string) { - fmt.Fprintf(outputFile, " m.Save(%d, &x.%s)\n", fields[name], name) + fmt.Fprintf(outputFile, " stateSinkObject.Save(%d, &%s.%s)\n", fields[name], recv, name) + } + emitZeroCheck := func(name string) { + fmt.Fprintf(outputFile, " if !%sIsZeroValue(&%s.%s) { %sFailf(\"%s is %%#v, expected zero\", &%s.%s) }\n", statePrefix, recv, name, statePrefix, name, recv, name) } // Generate the type name method. - fmt.Fprintf(outputFile, "func (x *%s) StateTypeName() string {\n", ts.Name.Name) + fmt.Fprintf(outputFile, "func (%s *%s) StateTypeName() string {\n", recv, ts.Name.Name) fmt.Fprintf(outputFile, " return \"%s.%s\"\n", *fullPkg, ts.Name.Name) fmt.Fprintf(outputFile, "}\n\n") // Generate the fields method. - fmt.Fprintf(outputFile, "func (x *%s) StateFields() []string {\n", ts.Name.Name) + fmt.Fprintf(outputFile, "func (%s *%s) StateFields() []string {\n", recv, ts.Name.Name) fmt.Fprintf(outputFile, " return []string{\n") scanFields(x, "", scanFunctions{ normal: emitField, @@ -401,8 +389,11 @@ func main() { // the code from compiling if a custom beforeSave was defined in a // file not provided to this binary and prevents inherited methods // from being called multiple times by overriding them. - if _, ok := simpleMethods[method{ts.Name.Name, "beforeSave"}]; !ok && generateSaverLoader { - fmt.Fprintf(outputFile, "func (x *%s) beforeSave() {}\n\n", ts.Name.Name) + if _, ok := simpleMethods[method{ + typeName: ts.Name.Name, + methodName: "beforeSave", + }]; !ok && generateSaverLoader { + fmt.Fprintf(outputFile, "func (%s *%s) beforeSave() {}\n\n", recv, ts.Name.Name) } // Generate the save method. @@ -412,8 +403,8 @@ func main() { // on this specific behavior, but the ability to specify slots // allows a manual implementation to be order-dependent. if generateSaverLoader { - fmt.Fprintf(outputFile, "func (x *%s) StateSave(m %sSink) {\n", ts.Name.Name, statePrefix) - fmt.Fprintf(outputFile, " x.beforeSave()\n") + fmt.Fprintf(outputFile, "func (%s *%s) StateSave(stateSinkObject %sSink) {\n", recv, ts.Name.Name, statePrefix) + fmt.Fprintf(outputFile, " %s.beforeSave()\n", recv) scanFields(x, "", scanFunctions{zerovalue: emitZeroCheck}) scanFields(x, "", scanFunctions{value: emitSaveValue}) scanFields(x, "", scanFunctions{normal: emitSave, wait: emitSave}) @@ -422,16 +413,19 @@ func main() { // Define afterLoad if a definition was not found. We do this for // the same reason that we do it for beforeSave. - _, hasAfterLoad := simpleMethods[method{ts.Name.Name, "afterLoad"}] + _, hasAfterLoad := simpleMethods[method{ + typeName: ts.Name.Name, + methodName: "afterLoad", + }] if !hasAfterLoad && generateSaverLoader { - fmt.Fprintf(outputFile, "func (x *%s) afterLoad() {}\n\n", ts.Name.Name) + fmt.Fprintf(outputFile, "func (%s *%s) afterLoad() {}\n\n", recv, ts.Name.Name) } // Generate the load method. // // N.B. See the comment above for the save method. if generateSaverLoader { - fmt.Fprintf(outputFile, "func (x *%s) StateLoad(m %sSource) {\n", ts.Name.Name, statePrefix) + fmt.Fprintf(outputFile, "func (%s *%s) StateLoad(stateSourceObject %sSource) {\n", recv, ts.Name.Name, statePrefix) scanFields(x, "", scanFunctions{normal: emitLoad, wait: emitLoadWait}) scanFields(x, "", scanFunctions{value: emitLoadValue}) if hasAfterLoad { @@ -439,7 +433,7 @@ func main() { // AfterLoad is called, the object encodes a dependency on // referred objects (i.e. fields). This means that afterLoad // will not be called until the other afterLoads are called. - fmt.Fprintf(outputFile, " m.AfterLoad(x.afterLoad)\n") + fmt.Fprintf(outputFile, " stateSourceObject.AfterLoad(%s.afterLoad)\n", recv) } fmt.Fprintf(outputFile, "}\n\n") } @@ -451,10 +445,10 @@ func main() { maybeEmitImports() // Generate the info methods. - fmt.Fprintf(outputFile, "func (x *%s) StateTypeName() string {\n", ts.Name.Name) + fmt.Fprintf(outputFile, "func (%s *%s) StateTypeName() string {\n", recv, ts.Name.Name) fmt.Fprintf(outputFile, " return \"%s.%s\"\n", *fullPkg, ts.Name.Name) fmt.Fprintf(outputFile, "}\n\n") - fmt.Fprintf(outputFile, "func (x *%s) StateFields() []string {\n", ts.Name.Name) + fmt.Fprintf(outputFile, "func (%s *%s) StateFields() []string {\n", recv, ts.Name.Name) fmt.Fprintf(outputFile, " return nil\n") fmt.Fprintf(outputFile, "}\n\n") diff --git a/tools/issue_reviver/BUILD b/tools/issue_reviver/BUILD deleted file mode 100644 index 35b0111ca..000000000 --- a/tools/issue_reviver/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -load("//tools:defs.bzl", "go_binary") - -package(licenses = ["notice"]) - -go_binary( - name = "issue_reviver", - srcs = ["main.go"], - nogo = False, - deps = [ - "//tools/issue_reviver/github", - "//tools/issue_reviver/reviver", - ], -) diff --git a/tools/issue_reviver/github/BUILD b/tools/issue_reviver/github/BUILD deleted file mode 100644 index 555abd296..000000000 --- a/tools/issue_reviver/github/BUILD +++ /dev/null @@ -1,25 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "github", - srcs = ["github.go"], - nogo = False, - visibility = [ - "//tools/issue_reviver:__subpackages__", - ], - deps = [ - "//tools/issue_reviver/reviver", - "@com_github_google_go_github_v28//github:go_default_library", - "@org_golang_x_oauth2//:go_default_library", - ], -) - -go_test( - name = "github_test", - size = "small", - srcs = ["github_test.go"], - library = ":github", - nogo = False, -) diff --git a/tools/issue_reviver/main.go b/tools/issue_reviver/main.go deleted file mode 100644 index 47c796b8a..000000000 --- a/tools/issue_reviver/main.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2019 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package main is the entry point for issue_reviver. -package main - -import ( - "flag" - "fmt" - "io/ioutil" - "os" - "strings" - - "gvisor.dev/gvisor/tools/issue_reviver/github" - "gvisor.dev/gvisor/tools/issue_reviver/reviver" -) - -var ( - owner string - repo string - tokenFile string - path string - dryRun bool -) - -// Keep the options simple for now. Supports only a single path and repo. -func init() { - flag.StringVar(&owner, "owner", "", "Github project org/owner to look for issues") - flag.StringVar(&repo, "repo", "", "Github repo to look for issues") - flag.StringVar(&tokenFile, "oauth-token-file", "", "Path to file containing the OAUTH token to be used as credential to github") - flag.StringVar(&path, "path", ".", "Path to scan for TODOs") - flag.BoolVar(&dryRun, "dry-run", false, "If set to true, no changes are made to issues") -} - -func main() { - // Set defaults from the environment. - repository := os.Getenv("GITHUB_REPOSITORY") - if parts := strings.SplitN(repository, "/", 2); len(parts) == 2 { - owner = parts[0] - repo = parts[1] - } - - // Parse flags. - flag.Parse() - - // Check for mandatory parameters. - if len(owner) == 0 { - fmt.Println("missing --owner option.") - flag.Usage() - os.Exit(1) - } - if len(repo) == 0 { - fmt.Println("missing --repo option.") - flag.Usage() - os.Exit(1) - } - if len(path) == 0 { - fmt.Println("missing --path option.") - flag.Usage() - os.Exit(1) - } - - // The access token may be passed as a file so it doesn't show up in - // command line arguments. It also may be provided through the - // environment to faciliate use through GitHub's CI system. - token := os.Getenv("GITHUB_TOKEN") - if len(tokenFile) != 0 { - bytes, err := ioutil.ReadFile(tokenFile) - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - token = string(bytes) - } - - bugger, err := github.NewBugger(token, owner, repo, dryRun) - if err != nil { - fmt.Fprintln(os.Stderr, "Error getting github issues:", err) - os.Exit(1) - } - rev := reviver.New([]string{path}, []reviver.Bugger{bugger}) - if errs := rev.Run(); len(errs) > 0 { - fmt.Fprintf(os.Stderr, "Encountered %d errors:\n", len(errs)) - for _, err := range errs { - fmt.Fprintf(os.Stderr, "\t%v\n", err) - } - os.Exit(1) - } -} diff --git a/tools/issue_reviver/reviver/BUILD b/tools/issue_reviver/reviver/BUILD deleted file mode 100644 index d262932bd..000000000 --- a/tools/issue_reviver/reviver/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -load("//tools:defs.bzl", "go_library", "go_test") - -package(licenses = ["notice"]) - -go_library( - name = "reviver", - srcs = ["reviver.go"], - visibility = [ - "//tools/issue_reviver:__subpackages__", - ], -) - -go_test( - name = "reviver_test", - size = "small", - srcs = ["reviver_test.go"], - library = ":reviver", -) diff --git a/tools/nogo/BUILD b/tools/nogo/BUILD index 9f1fcd9c7..dd4b46f58 100644 --- a/tools/nogo/BUILD +++ b/tools/nogo/BUILD @@ -1,10 +1,10 @@ load("//tools:defs.bzl", "bzl_library", "go_library") -load("//tools/nogo:defs.bzl", "nogo_dump_tool", "nogo_stdlib") +load("//tools/nogo:defs.bzl", "nogo_objdump_tool", "nogo_stdlib") package(licenses = ["notice"]) -nogo_dump_tool( - name = "dump_tool", +nogo_objdump_tool( + name = "objdump_tool", visibility = ["//visibility:public"], ) @@ -13,6 +13,12 @@ nogo_stdlib( visibility = ["//visibility:public"], ) +sh_binary( + name = "gentest", + srcs = ["gentest.sh"], + visibility = ["//visibility:public"], +) + go_library( name = "nogo", srcs = [ @@ -27,6 +33,8 @@ go_library( deps = [ "//tools/checkescape", "//tools/checkunsafe", + "@co_honnef_go_tools//staticcheck:go_default_library", + "@co_honnef_go_tools//stylecheck:go_default_library", "@org_golang_x_tools//go/analysis:go_tool_library", "@org_golang_x_tools//go/analysis/internal/facts:go_tool_library", "@org_golang_x_tools//go/analysis/passes/asmdecl:go_tool_library", diff --git a/tools/nogo/build.go b/tools/nogo/build.go index 37947b5c3..55d34760f 100644 --- a/tools/nogo/build.go +++ b/tools/nogo/build.go @@ -26,6 +26,12 @@ var ( // and should not have any special prefix applied. internalPrefix = fmt.Sprintf("^") + // internalDefault is applied when no paths are provided. + internalDefault = fmt.Sprintf("%s/.*", notPath("external")) + + // generatedPrefix is a regex for generated files. + generatedPrefix = "^(.*/)?(bazel-genfiles|bazel-out|bazel-bin)/" + // externalPrefix is external workspace packages. externalPrefix = "^external/" ) diff --git a/tools/nogo/config.go b/tools/nogo/config.go index cfe7b4aa4..8079618ab 100644 --- a/tools/nogo/config.go +++ b/tools/nogo/config.go @@ -41,6 +41,8 @@ import ( "golang.org/x/tools/go/analysis/passes/unreachable" "golang.org/x/tools/go/analysis/passes/unsafeptr" "golang.org/x/tools/go/analysis/passes/unusedresult" + "honnef.co/go/tools/staticcheck" + "honnef.co/go/tools/stylecheck" "gvisor.dev/gvisor/tools/checkescape" "gvisor.dev/gvisor/tools/checkunsafe" @@ -123,6 +125,432 @@ var analyzerConfig = map[*analysis.Analyzer]matcher{ checkunsafe.Analyzer: internalMatches(), } +func init() { + staticMatcher := and( + // Only match internal, non-generated files. + internalMatches(), + generatedExcluded(), + + // We use ALL_CAPS for system definitions, + // which are common enough in the code base + // that we shouldn't annotate exceptions. + // + // Same story for underscores. + resultExcluded([]string{ + "should not use ALL_CAPS in Go names", + "should not use underscores in Go names", + }), + + // Exclude existing matches. + internalExcluded( + "pkg/abi/linux/fuse.go:22", + "pkg/abi/linux/fuse.go:25", + "pkg/abi/linux/socket.go:113", + "pkg/abi/linux/tty.go:73", + "pkg/bpf/decoder.go:112", + "pkg/cpuid/cpuid_x86.go:675", + "pkg/eventchannel/event.go:193", + "pkg/eventchannel/event.go:27", + "pkg/eventchannel/event_test.go:22", + "pkg/eventchannel/rate.go:19", + "pkg/gohacks/gohacks_unsafe.go:33", + "pkg/log/json.go:30", + "pkg/log/log.go:359", + "pkg/merkletree/merkletree.go:230", + "pkg/merkletree/merkletree.go:243", + "pkg/merkletree/merkletree.go:249", + "pkg/merkletree/merkletree.go:266", + "pkg/merkletree/merkletree.go:355", + "pkg/merkletree/merkletree.go:369", + "pkg/metric/metric_test.go:20", + "pkg/p9/p9test/client_test.go:687", + "pkg/p9/transport_test.go:196", + "pkg/pool/pool.go:15", + "pkg/refs/refcounter.go:510", + "pkg/refs/refcounter_test.go:169", + "pkg/refs_vfs2/refs.go:16", + "pkg/safemem/block_unsafe.go:89", + "pkg/seccomp/seccomp.go:82", + "pkg/segment/test/set_functions.go:15", + "pkg/sentry/arch/signal.go:166", + "pkg/sentry/arch/signal.go:171", + "pkg/sentry/control/pprof.go:196", + "pkg/sentry/devices/memdev/full.go:58", + "pkg/sentry/devices/memdev/null.go:59", + "pkg/sentry/devices/memdev/random.go:68", + "pkg/sentry/devices/memdev/zero.go:86", + "pkg/sentry/fdimport/fdimport.go:15", + "pkg/sentry/fs/attr.go:257", + "pkg/sentry/fsbridge/fs.go:116", + "pkg/sentry/fsbridge/vfs.go:124", + "pkg/sentry/fsbridge/vfs.go:70", + "pkg/sentry/fs/copy_up.go:365", + "pkg/sentry/fs/copy_up_test.go:65", + "pkg/sentry/fs/dev/net_tun.go:161", + "pkg/sentry/fs/dev/net_tun.go:63", + "pkg/sentry/fs/dev/null.go:97", + "pkg/sentry/fs/dirent_cache.go:64", + "pkg/sentry/fs/file_overlay.go:327", + "pkg/sentry/fs/file_overlay.go:524", + "pkg/sentry/fs/filetest/filetest.go:55", + "pkg/sentry/fs/filetest/filetest.go:60", + "pkg/sentry/fs/fs.go:77", + "pkg/sentry/fs/fsutil/file.go:290", + "pkg/sentry/fs/fsutil/file.go:346", + "pkg/sentry/fs/fsutil/host_file_mapper.go:105", + "pkg/sentry/fs/fsutil/inode_cached.go:676", + "pkg/sentry/fs/fsutil/inode_cached.go:772", + "pkg/sentry/fs/gofer/attr.go:120", + "pkg/sentry/fs/gofer/fifo.go:33", + "pkg/sentry/fs/gofer/inode.go:410", + "pkg/sentry/fsimpl/devpts/devpts.go:110", + "pkg/sentry/fsimpl/devpts/devpts.go:246", + "pkg/sentry/fsimpl/devpts/devpts.go:50", + "pkg/sentry/fsimpl/devpts/master.go:110", + "pkg/sentry/fsimpl/devpts/master.go:55", + "pkg/sentry/fsimpl/devpts/replica.go:113", + "pkg/sentry/fsimpl/devpts/replica.go:57", + "pkg/sentry/fsimpl/devtmpfs/devtmpfs.go:54", + "pkg/sentry/fsimpl/ext/disklayout/superblock_64.go:97", + "pkg/sentry/fsimpl/ext/disklayout/superblock_old.go:92", + "pkg/sentry/fsimpl/ext/disklayout/block_group_32.go:44", + "pkg/sentry/fsimpl/ext/disklayout/inode_new.go:91", + "pkg/sentry/fsimpl/ext/disklayout/inode_old.go:93", + "pkg/sentry/fsimpl/ext/disklayout/superblock_32.go:66", + "pkg/sentry/fsimpl/ext/disklayout/block_group_64.go:53", + "pkg/sentry/fsimpl/eventfd/eventfd.go:268", + "pkg/sentry/fsimpl/ext/directory.go:163", + "pkg/sentry/fsimpl/ext/directory.go:164", + "pkg/sentry/fsimpl/ext/extent_file.go:142", + "pkg/sentry/fsimpl/ext/extent_file.go:143", + "pkg/sentry/fsimpl/ext/ext.go:105", + "pkg/sentry/fsimpl/ext/filesystem.go:287", + "pkg/sentry/fsimpl/ext/regular_file.go:153", + "pkg/sentry/fsimpl/ext/symlink.go:113", + "pkg/sentry/fsimpl/fuse/connection_control.go:194", + "pkg/sentry/fsimpl/fuse/dev.go:387", + "pkg/sentry/fsimpl/fuse/dev_test.go:318", + "pkg/sentry/fsimpl/fuse/fusefs.go:102", + "pkg/sentry/fsimpl/fuse/read_write.go:129", + "pkg/sentry/fsimpl/fuse/request_response.go:71", + "pkg/sentry/fsimpl/gofer/directory.go:135", + "pkg/sentry/fsimpl/gofer/filesystem.go:679", + "pkg/sentry/fsimpl/gofer/gofer.go:1694", + "pkg/sentry/fsimpl/gofer/gofer.go:276", + "pkg/sentry/fsimpl/gofer/regular_file.go:81", + "pkg/sentry/fsimpl/gofer/special_file.go:141", + "pkg/sentry/fsimpl/host/host.go:184", + "pkg/sentry/fsimpl/kernfs/dynamic_bytes_file.go:50", + "pkg/sentry/fsimpl/kernfs/dynamic_bytes_file.go:90", + "pkg/sentry/fsimpl/kernfs/fd_impl_util.go:273", + "pkg/sentry/fsimpl/kernfs/filesystem.go:247", + "pkg/sentry/fsimpl/kernfs/inode_impl_util.go:320", + "pkg/sentry/fsimpl/kernfs/inode_impl_util.go:497", + "pkg/sentry/fsimpl/kernfs/synthetic_directory.go:52", + "pkg/sentry/fsimpl/overlay/directory.go:119", + "pkg/sentry/fsimpl/overlay/filesystem.go:527", + "pkg/sentry/fsimpl/overlay/non_directory.go:152", + "pkg/sentry/fsimpl/overlay/overlay.go:115", + "pkg/sentry/fsimpl/overlay/overlay.go:719", + "pkg/sentry/fsimpl/pipefs/pipefs.go:74", + "pkg/sentry/fsimpl/proc/filesystem.go:52", + "pkg/sentry/fsimpl/proc/filesystem.go:81", + "pkg/sentry/fsimpl/proc/subtasks.go:126", + "pkg/sentry/fsimpl/proc/subtasks.go:189", + "pkg/sentry/fsimpl/proc/task_fds.go:168", + "pkg/sentry/fsimpl/proc/task_fds.go:228", + "pkg/sentry/fsimpl/proc/task_fds.go:301", + "pkg/sentry/fsimpl/proc/task_fds.go:318", + "pkg/sentry/fsimpl/proc/task_fds.go:67", + "pkg/sentry/fsimpl/proc/task_files.go:112", + "pkg/sentry/fsimpl/proc/task_files.go:158", + "pkg/sentry/fsimpl/proc/task_files.go:259", + "pkg/sentry/fsimpl/proc/task_files.go:285", + "pkg/sentry/fsimpl/proc/task_files.go:305", + "pkg/sentry/fsimpl/proc/task_files.go:384", + "pkg/sentry/fsimpl/proc/task_files.go:403", + "pkg/sentry/fsimpl/proc/task_files.go:428", + "pkg/sentry/fsimpl/proc/task_files.go:691", + "pkg/sentry/fsimpl/proc/task_files.go:770", + "pkg/sentry/fsimpl/proc/task_files.go:797", + "pkg/sentry/fsimpl/proc/task_files.go:828", + "pkg/sentry/fsimpl/proc/task_files.go:879", + "pkg/sentry/fsimpl/proc/task_files.go:910", + "pkg/sentry/fsimpl/proc/task_files.go:961", + "pkg/sentry/fsimpl/proc/task.go:127", + "pkg/sentry/fsimpl/proc/task.go:193", + "pkg/sentry/fsimpl/proc/task_net.go:134", + "pkg/sentry/fsimpl/proc/task_net.go:475", + "pkg/sentry/fsimpl/proc/task_net.go:491", + "pkg/sentry/fsimpl/proc/task_net.go:508", + "pkg/sentry/fsimpl/proc/task_net.go:665", + "pkg/sentry/fsimpl/proc/task_net.go:715", + "pkg/sentry/fsimpl/proc/task_net.go:779", + "pkg/sentry/fsimpl/proc/tasks_files.go:113", + "pkg/sentry/fsimpl/proc/tasks_files.go:388", + "pkg/sentry/fsimpl/proc/tasks.go:232", + "pkg/sentry/fsimpl/proc/tasks_sys.go:145", + "pkg/sentry/fsimpl/proc/tasks_sys.go:181", + "pkg/sentry/fsimpl/proc/tasks_sys.go:239", + "pkg/sentry/fsimpl/proc/tasks_sys.go:291", + "pkg/sentry/fsimpl/proc/tasks_sys.go:375", + "pkg/sentry/fsimpl/signalfd/signalfd.go:124", + "pkg/sentry/fsimpl/signalfd/signalfd.go:15", + "pkg/sentry/fsimpl/signalfd/signalfd.go:126", + "pkg/sentry/fsimpl/sockfs/sockfs.go:36", + "pkg/sentry/fsimpl/sockfs/sockfs.go:79", + "pkg/sentry/fsimpl/sys/kcov.go:49", + "pkg/sentry/fsimpl/sys/kcov.go:99", + "pkg/sentry/fsimpl/sys/sys.go:118", + "pkg/sentry/fsimpl/sys/sys.go:56", + "pkg/sentry/fsimpl/testutil/testutil.go:257", + "pkg/sentry/fsimpl/testutil/testutil.go:260", + "pkg/sentry/fsimpl/timerfd/timerfd.go:87", + "pkg/sentry/fsimpl/tmpfs/directory.go:112", + "pkg/sentry/fsimpl/tmpfs/filesystem.go:195", + "pkg/sentry/fsimpl/tmpfs/regular_file.go:226", + "pkg/sentry/fsimpl/tmpfs/regular_file.go:346", + "pkg/sentry/fsimpl/tmpfs/tmpfs.go:103", + "pkg/sentry/fsimpl/tmpfs/tmpfs.go:733", + "pkg/sentry/fsimpl/verity/filesystem.go:490", + "pkg/sentry/fsimpl/verity/verity.go:156", + "pkg/sentry/fsimpl/verity/verity.go:629", + "pkg/sentry/fsimpl/verity/verity.go:672", + "pkg/sentry/fs/mount.go:162", + "pkg/sentry/fs/mount.go:256", + "pkg/sentry/fs/mount_overlay.go:144", + "pkg/sentry/fs/mounts.go:432", + "pkg/sentry/fs/proc/exec_args.go:104", + "pkg/sentry/fs/proc/exec_args.go:73", + "pkg/sentry/fs/proc/fds.go:269", + "pkg/sentry/fs/proc/loadavg.go:33", + "pkg/sentry/fs/proc/meminfo.go:39", + "pkg/sentry/fs/proc/mounts.go:193", + "pkg/sentry/fs/proc/mounts.go:84", + "pkg/sentry/fs/proc/net.go:125", + "pkg/sentry/fs/proc/proc.go:146", + "pkg/sentry/fs/proc/proc.go:204", + "pkg/sentry/fs/proc/seqfile/seqfile.go:210", + "pkg/sentry/fs/proc/sys.go:146", + "pkg/sentry/fs/proc/sys.go:43", + "pkg/sentry/fs/proc/sys_net.go:113", + "pkg/sentry/fs/proc/sys_net.go:205", + "pkg/sentry/fs/proc/sys_net.go:233", + "pkg/sentry/fs/proc/sys_net.go:307", + "pkg/sentry/fs/proc/sys_net.go:335", + "pkg/sentry/fs/proc/sys_net.go:446", + "pkg/sentry/fs/proc/sys_net.go:456", + "pkg/sentry/fs/proc/sys_net.go:89", + "pkg/sentry/fs/proc/task.go:170", + "pkg/sentry/fs/proc/task.go:322", + "pkg/sentry/fs/proc/task.go:427", + "pkg/sentry/fs/proc/task.go:467", + "pkg/sentry/fs/proc/task.go:500", + "pkg/sentry/fs/proc/task.go:784", + "pkg/sentry/fs/proc/task.go:839", + "pkg/sentry/fs/proc/task.go:920", + "pkg/sentry/fs/proc/uid_gid_map.go:108", + "pkg/sentry/fs/proc/uid_gid_map.go:79", + "pkg/sentry/fs/proc/uptime.go:75", + "pkg/sentry/fs/ramfs/dir.go:447", + "pkg/sentry/fs/tmpfs/inode_file.go:436", + "pkg/sentry/fs/tmpfs/inode_file.go:537", + "pkg/sentry/fs/tty/dir.go:313", + "pkg/sentry/fs/tty/master.go:131", + "pkg/sentry/fs/tty/master.go:91", + "pkg/sentry/fs/tty/replica.go:116", + "pkg/sentry/fs/tty/replica.go:88", + "pkg/sentry/kernel/auth/id_map.go:269", + "pkg/sentry/kernel/fasync/fasync.go:67", + "pkg/sentry/kernel/kcov.go:209", + "pkg/sentry/kernel/kcov.go:223", + "pkg/sentry/kernel/kernel.go:343", + "pkg/sentry/kernel/kernel.go:368", + "pkg/sentry/kernel/pipe/node_test.go:112", + "pkg/sentry/kernel/pipe/node_test.go:119", + "pkg/sentry/kernel/pipe/node_test.go:130", + "pkg/sentry/kernel/pipe/node_test.go:137", + "pkg/sentry/kernel/pipe/node_test.go:149", + "pkg/sentry/kernel/pipe/node_test.go:150", + "pkg/sentry/kernel/pipe/node_test.go:158", + "pkg/sentry/kernel/pipe/node_test.go:174", + "pkg/sentry/kernel/pipe/node_test.go:180", + "pkg/sentry/kernel/pipe/node_test.go:193", + "pkg/sentry/kernel/pipe/node_test.go:202", + "pkg/sentry/kernel/pipe/node_test.go:205", + "pkg/sentry/kernel/pipe/node_test.go:216", + "pkg/sentry/kernel/pipe/node_test.go:219", + "pkg/sentry/kernel/pipe/node_test.go:271", + "pkg/sentry/kernel/pipe/node_test.go:290", + "pkg/sentry/kernel/pipe/pipe_test.go:93", + "pkg/sentry/kernel/pipe/reader_writer.go:65", + "pkg/sentry/kernel/posixtimer.go:157", + "pkg/sentry/kernel/ptrace.go:218", + "pkg/sentry/kernel/semaphore/semaphore.go:323", + "pkg/sentry/kernel/sessions.go:123", + "pkg/sentry/kernel/sessions.go:508", + "pkg/sentry/kernel/signal_handlers.go:57", + "pkg/sentry/kernel/task_context.go:72", + "pkg/sentry/kernel/task_exit.go:67", + "pkg/sentry/kernel/task_sched.go:255", + "pkg/sentry/kernel/task_sched.go:280", + "pkg/sentry/kernel/task_sched.go:323", + "pkg/sentry/kernel/task_stop.go:192", + "pkg/sentry/kernel/thread_group.go:530", + "pkg/sentry/kernel/timekeeper.go:316", + "pkg/sentry/kernel/vdso.go:106", + "pkg/sentry/kernel/vdso.go:118", + "pkg/sentry/memmap/memmap.go:103", + "pkg/sentry/memmap/memmap.go:163", + "pkg/sentry/mm/address_space.go:42", + "pkg/sentry/mm/address_space.go:42", + "pkg/sentry/mm/aio_context.go:208", + "pkg/sentry/mm/aio_context.go:288", + "pkg/sentry/mm/pma.go:683", + "pkg/sentry/mm/special_mappable.go:80", + "pkg/sentry/platform/systrap/subprocess.go:370", + "pkg/sentry/platform/systrap/usertrap/usertrap_amd64.go:124", + "pkg/sentry/socket/control/control.go:260", + "pkg/sentry/socket/control/control.go:94", + "pkg/sentry/socket/control/control_vfs2.go:37", + "pkg/sentry/socket/hostinet/stack.go:433", + "pkg/sentry/socket/hostinet/stack.go:438", + "pkg/sentry/socket/hostinet/stack.go:444", + "pkg/sentry/socket/hostinet/stack.go:460", + "pkg/sentry/socket/netfilter/tcp_matcher.go:74", + "pkg/sentry/socket/netfilter/udp_matcher.go:71", + "pkg/sentry/socket/netlink/route/protocol.go:38", + "pkg/sentry/socket/socket.go:332", + "pkg/sentry/socket/unix/transport/connectioned.go:394", + "pkg/sentry/socket/unix/transport/connectionless.go:152", + "pkg/sentry/socket/unix/transport/unix.go:436", + "pkg/sentry/socket/unix/transport/unix.go:490", + "pkg/sentry/socket/unix/transport/unix.go:685", + "pkg/sentry/socket/unix/transport/unix.go:795", + "pkg/sentry/syscalls/linux/sys_sem.go:62", + "pkg/sentry/syscalls/linux/sys_time.go:189", + "pkg/sentry/usage/cpu.go:42", + "pkg/sentry/vfs/anonfs.go:302", + "pkg/sentry/vfs/anonfs.go:99", + "pkg/sentry/vfs/dentry.go:214", + "pkg/sentry/vfs/epoll.go:168", + "pkg/sentry/vfs/epoll.go:314", + "pkg/sentry/vfs/file_description.go:549", + "pkg/sentry/vfs/file_description_impl_util.go:304", + "pkg/sentry/vfs/file_description_impl_util.go:412", + "pkg/sentry/vfs/filesystem.go:76", + "pkg/sentry/vfs/lock.go:15", + "pkg/sentry/vfs/lock.go:47", + "pkg/sentry/vfs/memxattr/xattr.go:37", + "pkg/sentry/vfs/mount.go:510", + "pkg/sentry/vfs/mount.go:667", + "pkg/sentry/vfs/mount_test.go:106", + "pkg/sentry/vfs/mount_test.go:160", + "pkg/sentry/vfs/mount_test.go:215", + "pkg/sentry/vfs/mount_unsafe.go:153", + "pkg/sentry/vfs/resolving_path.go:228", + "pkg/sentry/vfs/vfs.go:897", + "pkg/shim/runsc/runsc.go:16", + "pkg/shim/runsc/utils.go:16", + "pkg/shim/v1/proc/deleted_state.go:16", + "pkg/shim/v1/proc/exec.go:16", + "pkg/shim/v1/proc/exec_state.go:16", + "pkg/shim/v1/proc/init.go:16", + "pkg/shim/v1/proc/init_state.go:16", + "pkg/shim/v1/proc/io.go:16", + "pkg/shim/v1/proc/process.go:16", + "pkg/shim/v1/proc/types.go:16", + "pkg/shim/v1/proc/utils.go:16", + "pkg/shim/v1/shim/api.go:16", + "pkg/shim/v1/shim/platform.go:16", + "pkg/shim/v1/shim/service.go:16", + "pkg/shim/v1/utils/annotations.go:15", + "pkg/shim/v1/utils/utils.go:15", + "pkg/shim/v1/utils/volumes.go:15", + "pkg/shim/v2/api.go:16", + "pkg/shim/v2/epoll.go:18", + "pkg/shim/v2/options/options.go:15", + "pkg/shim/v2/options/options.go:24", + "pkg/shim/v2/options/options.go:26", + "pkg/shim/v2/runtimeoptions/runtimeoptions.go:16", + "pkg/shim/v2/runtimeoptions/runtimeoptions_test.go:22", + "pkg/shim/v2/service.go:15", + "pkg/shim/v2/service_linux.go:18", + "pkg/state/tests/integer_test.go:23", + "pkg/state/tests/integer_test.go:28", + "pkg/sync/rwmutex_test.go:105", + "pkg/syserr/host_linux.go:35", + "pkg/tcpip/adapters/gonet/gonet_test.go:144", + "pkg/tcpip/adapters/gonet/gonet_test.go:415", + "pkg/tcpip/adapters/gonet/gonet_test.go:99", + "pkg/tcpip/buffer/view.go:238", + "pkg/tcpip/buffer/view.go:238", + "pkg/tcpip/buffer/view.go:246", + "pkg/tcpip/header/tcp.go:151", + "pkg/tcpip/link/sharedmem/pipe/pipe_test.go:493", + "pkg/tcpip/stack/iptables.go:293", + "pkg/tcpip/stack/iptables_types.go:277", + "pkg/tcpip/stack/stack.go:553", + "pkg/tcpip/stack/transport_test.go:30", + "pkg/tcpip/transport/packet/endpoint.go:126", + "pkg/tcpip/transport/raw/endpoint.go:145", + "pkg/tcpip/transport/tcp/sack_scoreboard.go:167", + "pkg/unet/unet_test.go:634", + "pkg/unet/unet_test.go:662", + "pkg/unet/unet_test.go:703", + "pkg/unet/unet_test.go:98", + "pkg/usermem/addr.go:34", + "pkg/usermem/usermem.go:171", + "pkg/usermem/usermem.go:170", + "runsc/boot/compat.go:22", + "runsc/boot/compat.go:56", + "runsc/boot/loader.go:1115", + "runsc/boot/loader.go:1120", + "runsc/cmd/checkpoint.go:151", + "runsc/config/flags.go:32", + "runsc/container/container.go:641", + "runsc/container/container.go:988", + "runsc/specutils/specutils.go:172", + "runsc/specutils/specutils.go:428", + "runsc/specutils/specutils.go:436", + "runsc/specutils/specutils.go:442", + "runsc/specutils/specutils.go:447", + "runsc/specutils/specutils.go:454", + "test/cmd/test_app/fds.go:171", + "test/iptables/filter_output.go:251", + "test/packetimpact/testbench/connections.go:77", + "tools/bigquery/bigquery.go:106", + "tools/checkescape/test1/test1.go:108", + "tools/checkescape/test1/test1.go:122", + "tools/checkescape/test1/test1.go:137", + "tools/checkescape/test1/test1.go:151", + "tools/checkescape/test1/test1.go:170", + "tools/checkescape/test1/test1.go:39", + "tools/checkescape/test1/test1.go:45", + "tools/checkescape/test1/test1.go:50", + "tools/checkescape/test1/test1.go:64", + "tools/checkescape/test1/test1.go:80", + "tools/checkescape/test1/test1.go:94", + "tools/go_generics/imports.go:51", + "tools/go_generics/imports.go:75", + "tools/go_marshal/gomarshal/generator.go:177", + "tools/go_marshal/gomarshal/generator.go:81", + "tools/go_marshal/gomarshal/generator.go:85", + "tools/go_marshal/test/escape/escape.go:15", + "tools/go_marshal/test/test.go:164", + ), + ) + + // Add all staticcheck analyzers; internal only. + for _, a := range staticcheck.Analyzers { + analyzerConfig[a] = staticMatcher + } + // Add all stylecheck analyzers; internal only. + for _, a := range stylecheck.Analyzers { + analyzerConfig[a] = staticMatcher + } +} + var escapesConfig = map[*analysis.Analyzer]matcher{ // Informational only: include all packages. checkescape.EscapeAnalyzer: alwaysMatches(), diff --git a/tools/nogo/defs.bzl b/tools/nogo/defs.bzl index 480438047..c6fcfd402 100644 --- a/tools/nogo/defs.bzl +++ b/tools/nogo/defs.bzl @@ -2,8 +2,7 @@ load("//tools/bazeldefs:defs.bzl", "go_context", "go_importpath", "go_rule", "go_test_library") -def _nogo_dump_tool_impl(ctx): - # Extract the Go context. +def _nogo_objdump_tool_impl(ctx): go_ctx = go_context(ctx) # Construct the magic dump command. @@ -40,9 +39,9 @@ def _nogo_dump_tool_impl(ctx): executable = dumper, )] -nogo_dump_tool = go_rule( +nogo_objdump_tool = go_rule( rule, - implementation = _nogo_dump_tool_impl, + implementation = _nogo_objdump_tool_impl, ) # NogoStdlibInfo is the set of standard library facts. @@ -55,7 +54,6 @@ NogoStdlibInfo = provider( ) def _nogo_stdlib_impl(ctx): - # Extract the Go context. go_ctx = go_context(ctx) # Build the standard library facts. @@ -72,12 +70,12 @@ def _nogo_stdlib_impl(ctx): ctx.actions.run( inputs = [config_file] + go_ctx.stdlib_srcs, outputs = [facts, findings], - tools = depset(go_ctx.runfiles.to_list() + ctx.files._dump_tool), + tools = depset(go_ctx.runfiles.to_list() + ctx.files._objdump_tool), executable = ctx.files._nogo[0], mnemonic = "GoStandardLibraryAnalysis", progress_message = "Analyzing Go Standard Library", arguments = go_ctx.nogo_args + [ - "-dump_tool=%s" % ctx.files._dump_tool[0].path, + "-objdump_tool=%s" % ctx.files._objdump_tool[0].path, "-stdlib=%s" % config_file.path, "-findings=%s" % findings.path, "-facts=%s" % facts.path, @@ -97,8 +95,8 @@ nogo_stdlib = go_rule( "_nogo": attr.label( default = "//tools/nogo/check:check", ), - "_dump_tool": attr.label( - default = "//tools/nogo:dump_tool", + "_objdump_tool": attr.label( + default = "//tools/nogo:objdump_tool", ), }, ) @@ -121,6 +119,8 @@ NogoInfo = provider( ) def _nogo_aspect_impl(target, ctx): + go_ctx = go_context(ctx) + # If this is a nogo rule itself (and not the shadow of a go_library or # go_binary rule created by such a rule), then we simply return nothing. # All work is done in the shadow properties for go rules. For a proto @@ -135,9 +135,6 @@ def _nogo_aspect_impl(target, ctx): else: return [NogoInfo()] - # Extract the Go context. - go_ctx = go_context(ctx) - # If we're using the "library" attribute, then we need to aggregate the # original library sources and dependencies into this target to perform # proper type analysis. @@ -227,13 +224,13 @@ def _nogo_aspect_impl(target, ctx): ctx.actions.run( inputs = inputs, outputs = [facts, findings, escapes], - tools = depset(go_ctx.runfiles.to_list() + ctx.files._dump_tool), + tools = depset(go_ctx.runfiles.to_list() + ctx.files._objdump_tool), executable = ctx.files._nogo[0], mnemonic = "GoStaticAnalysis", progress_message = "Analyzing %s" % target.label, arguments = go_ctx.nogo_args + [ "-binary=%s" % target_objfile.path, - "-dump_tool=%s" % ctx.files._dump_tool[0].path, + "-objdump_tool=%s" % ctx.files._objdump_tool[0].path, "-package=%s" % config_file.path, "-findings=%s" % findings.path, "-facts=%s" % facts.path, @@ -271,58 +268,48 @@ nogo_aspect = go_rule( attrs = { "_nogo": attr.label(default = "//tools/nogo/check:check"), "_nogo_stdlib": attr.label(default = "//tools/nogo:stdlib"), - "_dump_tool": attr.label(default = "//tools/nogo:dump_tool"), + "_objdump_tool": attr.label(default = "//tools/nogo:objdump_tool"), }, ) def _nogo_test_impl(ctx): """Check nogo findings.""" - # Build a runner that checks for the existence of the facts file. Note that - # the actual build will fail in the case of a broken analysis. We things - # this way so that any test applied is effectively pushed down to all - # upstream dependencies through the aspect. - inputs = [] - findings = [] - runner = ctx.actions.declare_file("%s-executer" % ctx.label.name) - runner_content = ["#!/bin/bash"] - for dep in ctx.attr.deps: - # Extract the findings. - info = dep[NogoInfo] - inputs.append(info.findings) - findings.append(info.findings) - - # Include all source files, transitively. This will make this target - # "directly affected" for the purpose of build analysis. - inputs += info.srcs - - # If there are findings, dump them and fail. - runner_content.append("if [[ -s \"%s\" ]]; then cat \"%s\" && exit 1; fi" % ( - info.findings.short_path, - info.findings.short_path, - )) - - # Otherwise, draw a sweet unicode checkmark with the package name (in green). - runner_content.append("echo -e \"\\033[0;32m\\xE2\\x9C\\x94\\033[0;31m\\033[0m %s\"" % info.importpath) - runner_content.append("exit 0\n") - ctx.actions.write(runner, "\n".join(runner_content), is_executable = True) + # Build a runner that checks the facts files. + findings = [dep[NogoInfo].findings for dep in ctx.attr.deps] + runner = ctx.actions.declare_file(ctx.label.name) + ctx.actions.run( + inputs = findings + ctx.files.srcs, + outputs = [runner], + tools = depset(ctx.files._gentest), + executable = ctx.files._gentest[0], + mnemonic = "Gentest", + progress_message = "Generating %s" % ctx.label, + arguments = [runner.path] + [f.path for f in findings], + ) return [DefaultInfo( - runfiles = ctx.runfiles(files = inputs), executable = runner, )] _nogo_test = rule( implementation = _nogo_test_impl, attrs = { + # deps should have only a single element. "deps": attr.label_list(aspects = [nogo_aspect]), + # srcs exist here only to ensure that this target is + # directly affected by changes to the source files. + "srcs": attr.label_list(allow_files = True), + "_gentest": attr.label(default = "//tools/nogo:gentest"), }, test = True, ) -def nogo_test(name, **kwargs): +def nogo_test(name, srcs, library, **kwargs): tags = kwargs.pop("tags", []) + ["nogo"] _nogo_test( name = name, + srcs = srcs, + deps = [library], tags = tags, **kwargs ) diff --git a/tools/nogo/gentest.sh b/tools/nogo/gentest.sh new file mode 100755 index 000000000..033da11ad --- /dev/null +++ b/tools/nogo/gentest.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright 2019 The gVisor Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +if [[ "$#" -lt 2 ]]; then + echo "usage: $0 <output> <findings...>" + exit 2 +fi +declare violations=0 +declare output=$1 +shift + +# Start the script. +echo "#!/bin/sh" > "${output}" + +# Read a list of findings files. +declare filename +declare line +for filename in "$@"; do + if [[ -z "${filename}" ]]; then + continue + fi + while read -r line; do + violations=$((${violations}+1)); + echo "echo -e '\\033[0;31m${line}\\033[0;31m\\033[0m'" >> "${output}" + done < "${filename}" +done + +# Show violations. +if [[ "${violations}" -eq 0 ]]; then + echo "echo -e '\\033[0;32mPASS\\033[0;31m\\033[0m'" >> "${output}" +else + echo "exit 1" >> "${output}" +fi diff --git a/tools/nogo/matchers.go b/tools/nogo/matchers.go index 57a250501..b7b73fa27 100644 --- a/tools/nogo/matchers.go +++ b/tools/nogo/matchers.go @@ -16,7 +16,6 @@ package nogo import ( "go/token" - "path/filepath" "regexp" "strings" @@ -44,11 +43,30 @@ type pathRegexps struct { func buildRegexps(prefix string, args ...string) []*regexp.Regexp { result := make([]*regexp.Regexp, 0, len(args)) for _, arg := range args { - result = append(result, regexp.MustCompile(filepath.Join(prefix, arg))) + result = append(result, regexp.MustCompile(prefix+arg)) } return result } +// notPath works around the lack of backtracking. +// +// It is used to construct a regular expression for non-matching components. +func notPath(name string) string { + sb := strings.Builder{} + sb.WriteString("(") + for i := range name { + if i > 0 { + sb.WriteString("|") + } + sb.WriteString(name[:i]) + sb.WriteString("[^") + sb.WriteByte(name[i]) + sb.WriteString("/][^/]*") + } + sb.WriteString(")") + return sb.String() +} + // ShouldReport implements matcher.ShouldReport. func (p *pathRegexps) ShouldReport(d analysis.Diagnostic, fs *token.FileSet) bool { fullPos := fs.Position(d.Pos).String() @@ -79,11 +97,19 @@ func externalExcluded(paths ...string) *pathRegexps { // internalMatches returns a path matcher for internal packages. func internalMatches() *pathRegexps { return &pathRegexps{ - expr: buildRegexps(internalPrefix, ".*"), + expr: buildRegexps(internalPrefix, internalDefault), include: true, } } +// generatedExcluded excludes all generated code. +func generatedExcluded() *pathRegexps { + return &pathRegexps{ + expr: buildRegexps(generatedPrefix, ".*"), + include: false, + } +} + // resultExcluded excludes explicit message contents. type resultExcluded []string @@ -99,20 +125,23 @@ func (r resultExcluded) ShouldReport(d analysis.Diagnostic, _ *token.FileSet) bo // andMatcher is a composite matcher. type andMatcher struct { - first matcher - second matcher + all []matcher } // ShouldReport implements matcher.ShouldReport. func (a *andMatcher) ShouldReport(d analysis.Diagnostic, fs *token.FileSet) bool { - return a.first.ShouldReport(d, fs) && a.second.ShouldReport(d, fs) + for _, m := range a.all { + if !m.ShouldReport(d, fs) { + return false + } + } + return true } // and is a syntactic convension for andMatcher. -func and(first matcher, second matcher) *andMatcher { +func and(ms ...matcher) *andMatcher { return &andMatcher{ - first: first, - second: second, + all: ms, } } diff --git a/tools/nogo/util/BUILD b/tools/nogo/util/BUILD new file mode 100644 index 000000000..7ab340b51 --- /dev/null +++ b/tools/nogo/util/BUILD @@ -0,0 +1,9 @@ +load("//tools:defs.bzl", "go_library") + +package(licenses = ["notice"]) + +go_library( + name = "util", + srcs = ["util.go"], + visibility = ["//visibility:public"], +) diff --git a/tools/nogo/util/util.go b/tools/nogo/util/util.go new file mode 100644 index 000000000..919fec799 --- /dev/null +++ b/tools/nogo/util/util.go @@ -0,0 +1,85 @@ +// Copyright 2019 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package util contains nogo-related utilities. +package util + +import ( + "fmt" + "io/ioutil" + "regexp" + "strconv" + "strings" +) + +// findingRegexp is used to parse findings. +var findingRegexp = regexp.MustCompile(`([a-zA-Z0-9_\/\.-]+): (-|([a-zA-Z0-9_\/\.-]+):([0-9]+)(:([0-9]+))?): (.*)`) + +const ( + categoryIndex = 1 + fullPathAndLineIndex = 2 + fullPathIndex = 3 + lineIndex = 4 + messageIndex = 7 +) + +// Finding is a single finding. +type Finding struct { + Category string + Path string + Line int + Message string +} + +// ExtractFindingsFromFile loads findings from a file. +func ExtractFindingsFromFile(filename string) ([]Finding, error) { + content, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + return ExtractFindingsFromBytes(content) +} + +// ExtractFindingsFromBytes loads findings from bytes. +func ExtractFindingsFromBytes(content []byte) (findings []Finding, err error) { + lines := strings.Split(string(content), "\n") + for _, singleLine := range lines { + // Skip blank lines. + singleLine = strings.TrimSpace(singleLine) + if singleLine == "" { + continue + } + m := findingRegexp.FindStringSubmatch(singleLine) + if m == nil { + // We shouldn't see findings like this. + return findings, fmt.Errorf("poorly formated line: %v", singleLine) + } + if m[fullPathAndLineIndex] == "-" { + continue // No source file available. + } + // Cleanup the message. + message := m[messageIndex] + message = strings.Replace(message, " → ", "\n → ", -1) + message = strings.Replace(message, " or ", "\n or ", -1) + // Construct a new annotation. + lineNumber, _ := strconv.ParseUint(m[lineIndex], 10, 32) + findings = append(findings, Finding{ + Category: m[categoryIndex], + Path: m[fullPathIndex], + Line: int(lineNumber), + Message: message, + }) + } + return findings, nil +} |