Github bug reviver

For everyone's joy, this is a tool that reopens issues that
have been closed, but are still referenced by TODOs in the
code. The idea is to run it in Kokoro nightly. Kokoro changes
are coming up next.

PiperOrigin-RevId: 288789560

7 files changed, 581 insertions, 0 deletions

diff --git a/tools/issue_reviver/BUILD b/tools/issue_reviver/BUILD
new file mode 100644
index 000000000..ee7ea11fd
--- /dev/null
+++ b/tools/issue_reviver/BUILD
@@ -0,0 +1,12 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary")
+
+package(licenses = ["notice"])
+
+go_binary(
+    name = "issue_reviver",
+    srcs = ["main.go"],
+    deps = [
+        "//tools/issue_reviver/github",
+        "//tools/issue_reviver/reviver",
+    ],
+)
diff --git a/tools/issue_reviver/github/BUILD b/tools/issue_reviver/github/BUILD
new file mode 100644
index 000000000..6da22ba1c
--- /dev/null
+++ b/tools/issue_reviver/github/BUILD
@@ -0,0 +1,17 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+package(licenses = ["notice"])
+
+go_library(
+    name = "github",
+    srcs = ["github.go"],
+    importpath = "gvisor.dev/gvisor/tools/issue_reviver/github",
+    visibility = [
+        "//tools/issue_reviver:__subpackages__",
+    ],
+    deps = [
+        "//tools/issue_reviver/reviver",
+        "@com_github_google_go-github//github:go_default_library",
+        "@org_golang_x_oauth2//:go_default_library",
+    ],
+)
diff --git a/tools/issue_reviver/github/github.go b/tools/issue_reviver/github/github.go
new file mode 100644
index 000000000..e07949c8f
--- /dev/null
+++ b/tools/issue_reviver/github/github.go
@@ -0,0 +1,164 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package github implements reviver.Bugger interface on top of Github issues.
+package github
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/google/go-github/github"
+	"golang.org/x/oauth2"
+	"gvisor.dev/gvisor/tools/issue_reviver/reviver"
+)
+
+// Bugger implements reviver.Bugger interface for github issues.
+type Bugger struct {
+	owner  string
+	repo   string
+	dryRun bool
+
+	client *github.Client
+	issues map[int]*github.Issue
+}
+
+// NewBugger creates a new Bugger.
+func NewBugger(token, owner, repo string, dryRun bool) (*Bugger, error) {
+	b := &Bugger{
+		owner:  owner,
+		repo:   repo,
+		dryRun: dryRun,
+		issues: map[int]*github.Issue{},
+	}
+	if err := b.load(token); err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+func (b *Bugger) load(token string) error {
+	ctx := context.Background()
+	if len(token) == 0 {
+		fmt.Print("No OAUTH token provided, using unauthenticated account.\n")
+		b.client = github.NewClient(nil)
+	} else {
+		ts := oauth2.StaticTokenSource(
+			&oauth2.Token{AccessToken: token},
+		)
+		tc := oauth2.NewClient(ctx, ts)
+		b.client = github.NewClient(tc)
+	}
+
+	err := processAllPages(func(listOpts github.ListOptions) (*github.Response, error) {
+		opts := &github.IssueListByRepoOptions{State: "open", ListOptions: listOpts}
+		tmps, resp, err := b.client.Issues.ListByRepo(ctx, b.owner, b.repo, opts)
+		if err != nil {
+			return resp, err
+		}
+		for _, issue := range tmps {
+			b.issues[issue.GetNumber()] = issue
+		}
+		return resp, nil
+	})
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("Loaded %d issues from github.com/%s/%s\n", len(b.issues), b.owner, b.repo)
+	return nil
+}
+
+// Activate implements reviver.Bugger.
+func (b *Bugger) Activate(todo *reviver.Todo) (bool, error) {
+	const prefix = "gvisor.dev/issue/"
+
+	// First check if I can handle the TODO.
+	idStr := strings.TrimPrefix(todo.Issue, prefix)
+	if len(todo.Issue) == len(idStr) {
+		return false, nil
+	}
+
+	id, err := strconv.Atoi(idStr)
+	if err != nil {
+		return true, err
+	}
+
+	// Check against active issues cache.
+	if _, ok := b.issues[id]; ok {
+		fmt.Printf("%q is active: OK\n", todo.Issue)
+		return true, nil
+	}
+
+	fmt.Printf("%q is not active: reopening issue %d\n", todo.Issue, id)
+
+	// Format comment with TODO locations and search link.
+	comment := strings.Builder{}
+	fmt.Fprintln(&comment, "There are TODOs still referencing this issue:")
+	for _, l := range todo.Locations {
+		fmt.Fprintf(&comment,
+			"1. [%s:%d](https://github.com/%s/%s/blob/HEAD/%s#%d): %s\n",
+			l.File, l.Line, b.owner, b.repo, l.File, l.Line, l.Comment)
+	}
+	fmt.Fprintf(&comment,
+		"\n\nSearch [TODO](https://github.com/%s/%s/search?q=%%22%s%d%%22)", b.owner, b.repo, prefix, id)
+
+	if b.dryRun {
+		fmt.Printf("[dry-run: skipping change to issue %d]\n%s\n=======================\n", id, comment.String())
+		return true, nil
+	}
+
+	ctx := context.Background()
+	req := &github.IssueRequest{State: github.String("open")}
+	_, _, err = b.client.Issues.Edit(ctx, b.owner, b.repo, id, req)
+	if err != nil {
+		return true, fmt.Errorf("failed to reactivate issue %d: %v", id, err)
+	}
+
+	cmt := &github.IssueComment{
+		Body:      github.String(comment.String()),
+		Reactions: &github.Reactions{Confused: github.Int(1)},
+	}
+	if _, _, err := b.client.Issues.CreateComment(ctx, b.owner, b.repo, id, cmt); err != nil {
+		return true, fmt.Errorf("failed to add comment to issue %d: %v", id, err)
+	}
+
+	return true, nil
+}
+
+func processAllPages(fn func(github.ListOptions) (*github.Response, error)) error {
+	opts := github.ListOptions{PerPage: 1000}
+	for {
+		resp, err := fn(opts)
+		if err != nil {
+			if rateErr, ok := err.(*github.RateLimitError); ok {
+				duration := rateErr.Rate.Reset.Sub(time.Now())
+				if duration > 5*time.Minute {
+					return fmt.Errorf("Rate limited for too long: %v", duration)
+				}
+				fmt.Printf("Rate limited, sleeping for: %v\n", duration)
+				time.Sleep(duration)
+				continue
+			}
+			return err
+		}
+		if resp.NextPage == 0 {
+			return nil
+		}
+		opts.Page = resp.NextPage
+	}
+}
diff --git a/tools/issue_reviver/main.go b/tools/issue_reviver/main.go
new file mode 100644
index 000000000..4256f5a6c
--- /dev/null
+++ b/tools/issue_reviver/main.go
@@ -0,0 +1,89 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package main is the entry point for issue_reviver.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"gvisor.dev/gvisor/tools/issue_reviver/github"
+	"gvisor.dev/gvisor/tools/issue_reviver/reviver"
+)
+
+var (
+	owner     string
+	repo      string
+	tokenFile string
+	path      string
+	dryRun    bool
+)
+
+// Keep the options simple for now. Supports only a single path and repo.
+func init() {
+	flag.StringVar(&owner, "owner", "google", "Github project org/owner to look for issues")
+	flag.StringVar(&repo, "repo", "gvisor", "Github repo to look for issues")
+	flag.StringVar(&tokenFile, "oauth-token-file", "", "Path to file containing the OAUTH token to be used as credential to github")
+	flag.StringVar(&path, "path", "", "Path to scan for TODOs")
+	flag.BoolVar(&dryRun, "dry-run", false, "If set to true, no changes are made to issues")
+}
+
+func main() {
+	flag.Parse()
+
+	// Check for mandatory parameters.
+	if len(owner) == 0 {
+		fmt.Println("missing --owner option.")
+		flag.Usage()
+		os.Exit(1)
+	}
+	if len(repo) == 0 {
+		fmt.Println("missing --repo option.")
+		flag.Usage()
+		os.Exit(1)
+	}
+	if len(path) == 0 {
+		fmt.Println("missing --path option.")
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	// Token is passed as a file so it doesn't show up in command line arguments.
+	var token string
+	if len(tokenFile) != 0 {
+		bytes, err := ioutil.ReadFile(tokenFile)
+		if err != nil {
+			fmt.Println(err.Error())
+			os.Exit(1)
+		}
+		token = string(bytes)
+	}
+
+	bugger, err := github.NewBugger(token, owner, repo, dryRun)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Error getting github issues:", err)
+		os.Exit(1)
+	}
+	rev := reviver.New([]string{path}, []reviver.Bugger{bugger})
+	if errs := rev.Run(); len(errs) > 0 {
+		fmt.Fprintf(os.Stderr, "Encountered %d errors:\n", len(errs))
+		for _, err := range errs {
+			fmt.Fprintf(os.Stderr, "\t%v\n", err)
+		}
+		os.Exit(1)
+	}
+}
diff --git a/tools/issue_reviver/reviver/BUILD b/tools/issue_reviver/reviver/BUILD
new file mode 100644
index 000000000..2c3675977
--- /dev/null
+++ b/tools/issue_reviver/reviver/BUILD
@@ -0,0 +1,19 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+package(licenses = ["notice"])
+
+go_library(
+    name = "reviver",
+    srcs = ["reviver.go"],
+    importpath = "gvisor.dev/gvisor/tools/issue_reviver/reviver",
+    visibility = [
+        "//tools/issue_reviver:__subpackages__",
+    ],
+)
+
+go_test(
+    name = "reviver_test",
+    size = "small",
+    srcs = ["reviver_test.go"],
+    embed = [":reviver"],
+)
diff --git a/tools/issue_reviver/reviver/reviver.go b/tools/issue_reviver/reviver/reviver.go
new file mode 100644
index 000000000..682db0c01
--- /dev/null
+++ b/tools/issue_reviver/reviver/reviver.go
@@ -0,0 +1,192 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package reviver scans the code looking for TODOs and pass them to registered
+// Buggers to ensure TODOs point to active issues.
+package reviver
+
+import (
+	"bufio"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sync"
+)
+
+// This is how a TODO looks like.
+var regexTodo = regexp.MustCompile(`(\/\/|#)\s*(TODO|FIXME)\(([a-zA-Z0-9.\/]+)\):\s*(.+)`)
+
+// Bugger interface is called for every TODO found in the code. If it can handle
+// the TODO, it must return true. If it returns false, the next Bugger is
+// called. If no Bugger handles the TODO, it's dropped on the floor.
+type Bugger interface {
+	Activate(todo *Todo) (bool, error)
+}
+
+// Location saves the location where the TODO was found.
+type Location struct {
+	Comment string
+	File    string
+	Line    uint
+}
+
+// Todo represents a unique TODO. There can be several TODOs pointing to the
+// same issue in the code. They are all grouped together.
+type Todo struct {
+	Issue     string
+	Locations []Location
+}
+
+// Reviver scans the given paths for TODOs and calls Buggers to handle them.
+type Reviver struct {
+	paths   []string
+	buggers []Bugger
+
+	mu    sync.Mutex
+	todos map[string]*Todo
+	errs  []error
+}
+
+// New create a new Reviver.
+func New(paths []string, buggers []Bugger) *Reviver {
+	return &Reviver{
+		paths:   paths,
+		buggers: buggers,
+		todos:   map[string]*Todo{},
+	}
+}
+
+// Run runs. It returns all errors found during processing, it doesn't stop
+// on errors.
+func (r *Reviver) Run() []error {
+	// Process each directory in parallel.
+	wg := sync.WaitGroup{}
+	for _, path := range r.paths {
+		wg.Add(1)
+		go func(path string) {
+			defer wg.Done()
+			r.processPath(path, &wg)
+		}(path)
+	}
+
+	wg.Wait()
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	fmt.Printf("Processing %d TODOs (%d errors)...\n", len(r.todos), len(r.errs))
+	dropped := 0
+	for _, todo := range r.todos {
+		ok, err := r.processTodo(todo)
+		if err != nil {
+			r.errs = append(r.errs, err)
+		}
+		if !ok {
+			dropped++
+		}
+	}
+	fmt.Printf("Processed %d TODOs, %d were skipped (%d errors)\n", len(r.todos)-dropped, dropped, len(r.errs))
+
+	return r.errs
+}
+
+func (r *Reviver) processPath(path string, wg *sync.WaitGroup) {
+	fmt.Printf("Processing dir %q\n", path)
+	fis, err := ioutil.ReadDir(path)
+	if err != nil {
+		r.addErr(fmt.Errorf("error processing dir %q: %v", path, err))
+		return
+	}
+
+	for _, fi := range fis {
+		childPath := filepath.Join(path, fi.Name())
+		switch {
+		case fi.Mode().IsDir():
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				r.processPath(childPath, wg)
+			}()
+
+		case fi.Mode().IsRegular():
+			file, err := os.Open(childPath)
+			if err != nil {
+				r.addErr(err)
+				continue
+			}
+
+			scanner := bufio.NewScanner(file)
+			lineno := uint(0)
+			for scanner.Scan() {
+				lineno++
+				line := scanner.Text()
+				if todo := r.processLine(line, childPath, lineno); todo != nil {
+					r.addTodo(todo)
+				}
+			}
+		}
+	}
+}
+
+func (r *Reviver) processLine(line, path string, lineno uint) *Todo {
+	matches := regexTodo.FindStringSubmatch(line)
+	if matches == nil {
+		return nil
+	}
+	if len(matches) != 5 {
+		panic(fmt.Sprintf("regex returned wrong matches for %q: %v", line, matches))
+	}
+	return &Todo{
+		Issue: matches[3],
+		Locations: []Location{
+			{
+				File:    path,
+				Line:    lineno,
+				Comment: matches[4],
+			},
+		},
+	}
+}
+
+func (r *Reviver) addTodo(newTodo *Todo) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if todo := r.todos[newTodo.Issue]; todo == nil {
+		r.todos[newTodo.Issue] = newTodo
+	} else {
+		todo.Locations = append(todo.Locations, newTodo.Locations...)
+	}
+}
+
+func (r *Reviver) addErr(err error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.errs = append(r.errs, err)
+}
+
+func (r *Reviver) processTodo(todo *Todo) (bool, error) {
+	for _, bugger := range r.buggers {
+		ok, err := bugger.Activate(todo)
+		if err != nil {
+			return false, err
+		}
+		if ok {
+			return true, nil
+		}
+	}
+	return false, nil
+}
diff --git a/tools/issue_reviver/reviver/reviver_test.go b/tools/issue_reviver/reviver/reviver_test.go
new file mode 100644
index 000000000..a9fb1f9f1
--- /dev/null
+++ b/tools/issue_reviver/reviver/reviver_test.go
@@ -0,0 +1,88 @@
+// Copyright 2019 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package reviver
+
+import (
+	"testing"
+)
+
+func TestProcessLine(t *testing.T) {
+	for _, tc := range []struct {
+		line string
+		want *Todo
+	}{
+		{
+			line: "// TODO(foobar.com/issue/123): comment, bla. blabla.",
+			want: &Todo{
+				Issue: "foobar.com/issue/123",
+				Locations: []Location{
+					{Comment: "comment, bla. blabla."},
+				},
+			},
+		},
+		{
+			line: "// FIXME(b/123): internal bug",
+			want: &Todo{
+				Issue: "b/123",
+				Locations: []Location{
+					{Comment: "internal bug"},
+				},
+			},
+		},
+		{
+			line: "TODO(issue): not todo",
+		},
+		{
+			line: "FIXME(issue): not todo",
+		},
+		{
+			line: "// TODO (issue): not todo",
+		},
+		{
+			line: "// TODO(issue) not todo",
+		},
+		{
+			line: "// todo(issue): not todo",
+		},
+		{
+			line: "// TODO(issue):",
+		},
+	} {
+		t.Logf("Testing: %s", tc.line)
+		r := Reviver{}
+		got := r.processLine(tc.line, "test", 0)
+		if got == nil {
+			if tc.want != nil {
+				t.Errorf("failed to process line, want: %+v", tc.want)
+			}
+		} else {
+			if tc.want == nil {
+				t.Errorf("expected error, got: %+v", got)
+				continue
+			}
+			if got.Issue != tc.want.Issue {
+				t.Errorf("wrong issue, got: %v, want: %v", got.Issue, tc.want.Issue)
+			}
+			if len(got.Locations) != len(tc.want.Locations) {
+				t.Errorf("wrong number of locations, got: %v, want: %v, locations: %+v", len(got.Locations), len(tc.want.Locations), got.Locations)
+			}
+			for i, wantLoc := range tc.want.Locations {
+				if got.Locations[i].Comment != wantLoc.Comment {
+					t.Errorf("wrong comment, got: %v, want: %v", got.Locations[i].Comment, wantLoc.Comment)
+				}
+			}
+		}
+	}
+}