internal/expect: fork go/expect

Almost no-one outside x/tools uses it, so we'd like to evolve
it for our needs, and tag and delete the public package.

Updates golang/go#70229

Change-Id: I77c7923881efdf772a1ad53134483ad0078c941d
Reviewed-on: https://go-review.googlesource.com/c/tools/+/625918
Auto-Submit: Alan Donovan <adonovan@google.com>
Reviewed-by: Robert Findley <rfindley@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>

internal/expect/expect.go

@@ -0,0 +1,125 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+Package expect provides support for interpreting structured comments in Go
+source code (including go.mod and go.work files) as test expectations.
+
+This is primarily intended for writing tests of things that process Go source
+files, although it does not directly depend on the testing package.
+
+Collect notes with the Extract or Parse functions, and use the
+MatchBefore function to find matches within the lines the comments were on.
+
+The interpretation of the notes depends on the application.
+For example, the test suite for a static checking tool might
+use a @diag note to indicate an expected diagnostic:
+
+	fmt.Printf("%s", 1) //@ diag("%s wants a string, got int")
+
+By contrast, the test suite for a source code navigation tool
+might use notes to indicate the positions of features of
+interest, the actions to be performed by the test,
+and their expected outcomes:
+
+	var x = 1 //@ x_decl
+	...
+	print(x) //@ definition("x", x_decl)
+	print(x) //@ typeof("x", "int")
+
+# Note comment syntax
+
+Note comments always start with the special marker @, which must be the
+very first character after the comment opening pair, so //@ or /*@ with no
+spaces.
+
+This is followed by a comma separated list of notes.
+
+A note always starts with an identifier, which is optionally followed by an
+argument list. The argument list is surrounded with parentheses and contains a
+comma-separated list of arguments.
+The empty parameter list and the missing parameter list are distinguishable if
+needed; they result in a nil or an empty list in the Args parameter respectively.
+
+Arguments are either identifiers or literals.
+The literals supported are the basic value literals, of string, float, integer
+true, false or nil. All the literals match the standard go conventions, with
+all bases of integers, and both quote and backtick strings.
+There is one extra literal type, which is a string literal preceded by the
+identifier "re" which is compiled to a regular expression.
+*/
+package expect
+
+import (
+	"bytes"
+	"fmt"
+	"go/token"
+	"regexp"
+)
+
+// Note is a parsed note from an expect comment.
+// It knows the position of the start of the comment, and the name and
+// arguments that make up the note.
+type Note struct {
+	Pos  token.Pos     // The position at which the note identifier appears
+	Name string        // the name associated with the note
+	Args []interface{} // the arguments for the note
+}
+
+// ReadFile is the type of a function that can provide file contents for a
+// given filename.
+// This is used in MatchBefore to look up the content of the file in order to
+// find the line to match the pattern against.
+type ReadFile func(filename string) ([]byte, error)
+
+// MatchBefore attempts to match a pattern in the line before the supplied pos.
+// It uses the FileSet and the ReadFile to work out the contents of the line
+// that end is part of, and then matches the pattern against the content of the
+// start of that line up to the supplied position.
+// The pattern may be either a simple string, []byte or a *regexp.Regexp.
+// MatchBefore returns the range of the line that matched the pattern, and
+// invalid positions if there was no match, or an error if the line could not be
+// found.
+func MatchBefore(fset *token.FileSet, readFile ReadFile, end token.Pos, pattern interface{}) (token.Pos, token.Pos, error) {
+	f := fset.File(end)
+	content, err := readFile(f.Name())
+	if err != nil {
+		return token.NoPos, token.NoPos, fmt.Errorf("invalid file: %v", err)
+	}
+	position := f.Position(end)
+	startOffset := f.Offset(f.LineStart(position.Line))
+	endOffset := f.Offset(end)
+	line := content[startOffset:endOffset]
+	matchStart, matchEnd := -1, -1
+	switch pattern := pattern.(type) {
+	case string:
+		bytePattern := []byte(pattern)
+		matchStart = bytes.Index(line, bytePattern)
+		if matchStart >= 0 {
+			matchEnd = matchStart + len(bytePattern)
+		}
+	case []byte:
+		matchStart = bytes.Index(line, pattern)
+		if matchStart >= 0 {
+			matchEnd = matchStart + len(pattern)
+		}
+	case *regexp.Regexp:
+		match := pattern.FindIndex(line)
+		if len(match) > 0 {
+			matchStart = match[0]
+			matchEnd = match[1]
+		}
+	}
+	if matchStart < 0 {
+		return token.NoPos, token.NoPos, nil
+	}
+	return f.Pos(startOffset + matchStart), f.Pos(startOffset + matchEnd), nil
+}
+
+func lineEnd(f *token.File, line int) token.Pos {
+	if line >= f.LineCount() {
+		return token.Pos(f.Base() + f.Size())
+	}
+	return f.LineStart(line + 1)
+}

internal/expect/expect_test.go

@@ -0,0 +1,164 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package expect_test
+
+import (
+	"bytes"
+	"go/token"
+	"os"
+	"testing"
+
+	"golang.org/x/tools/internal/expect"
+)
+
+func TestMarker(t *testing.T) {
+	for _, tt := range []struct {
+		filename      string
+		expectNotes   int
+		expectMarkers map[string]string
+		expectChecks  map[string][]interface{}
+	}{
+		{
+			filename:    "testdata/test.go",
+			expectNotes: 13,
+			expectMarkers: map[string]string{
+				"αSimpleMarker": "α",
+				"OffsetMarker":  "β",
+				"RegexMarker":   "γ",
+				"εMultiple":     "ε",
+				"ζMarkers":      "ζ",
+				"ηBlockMarker":  "η",
+				"Declared":      "η",
+				"Comment":       "ι",
+				"LineComment":   "someFunc",
+				"NonIdentifier": "+",
+				"StringMarker":  "\"hello\"",
+			},
+			expectChecks: map[string][]interface{}{
+				"αSimpleMarker": nil,
+				"StringAndInt":  {"Number %d", int64(12)},
+				"Bool":          {true},
+			},
+		},
+		{
+			filename:    "testdata/go.fake.mod",
+			expectNotes: 2,
+			expectMarkers: map[string]string{
+				"αMarker": "αfake1α",
+				"βMarker": "require golang.org/modfile v0.0.0",
+			},
+		},
+		{
+			filename:    "testdata/go.fake.work",
+			expectNotes: 2,
+			expectMarkers: map[string]string{
+				"αMarker": "1.23.0",
+				"βMarker": "αβ",
+			},
+		},
+	} {
+		t.Run(tt.filename, func(t *testing.T) {
+			content, err := os.ReadFile(tt.filename)
+			if err != nil {
+				t.Fatal(err)
+			}
+			readFile := func(string) ([]byte, error) { return content, nil }
+
+			markers := make(map[string]token.Pos)
+			for name, tok := range tt.expectMarkers {
+				offset := bytes.Index(content, []byte(tok))
+				markers[name] = token.Pos(offset + 1)
+				end := bytes.Index(content[offset:], []byte(tok))
+				if end > 0 {
+					markers[name+"@"] = token.Pos(offset + end + 2)
+				}
+			}
+
+			fset := token.NewFileSet()
+			notes, err := expect.Parse(fset, tt.filename, content)
+			if err != nil {
+				t.Fatalf("Failed to extract notes: %v", err)
+			}
+			if len(notes) != tt.expectNotes {
+				t.Errorf("Expected %v notes, got %v", tt.expectNotes, len(notes))
+			}
+			for _, n := range notes {
+				switch {
+				case n.Args == nil:
+					// A //@foo note associates the name foo with the position of the
+					// first match of "foo" on the current line.
+					checkMarker(t, fset, readFile, markers, n.Pos, n.Name, n.Name)
+				case n.Name == "mark":
+					// A //@mark(name, "pattern") note associates the specified name
+					// with the position on the first match of pattern on the current line.
+					if len(n.Args) != 2 {
+						t.Errorf("%v: expected 2 args to mark, got %v", fset.Position(n.Pos), len(n.Args))
+						continue
+					}
+					ident, ok := n.Args[0].(expect.Identifier)
+					if !ok {
+						t.Errorf("%v: identifier, got %T", fset.Position(n.Pos), n.Args[0])
+						continue
+					}
+					checkMarker(t, fset, readFile, markers, n.Pos, string(ident), n.Args[1])
+
+				case n.Name == "check":
+					// A //@check(args, ...) note specifies some hypothetical action to
+					// be taken by the test driver and its expected outcome.
+					// In this test, the action is to compare the arguments
+					// against expectChecks.
+					if len(n.Args) < 1 {
+						t.Errorf("%v: expected 1 args to check, got %v", fset.Position(n.Pos), len(n.Args))
+						continue
+					}
+					ident, ok := n.Args[0].(expect.Identifier)
+					if !ok {
+						t.Errorf("%v: identifier, got %T", fset.Position(n.Pos), n.Args[0])
+						continue
+					}
+					args, ok := tt.expectChecks[string(ident)]
+					if !ok {
+						t.Errorf("%v: unexpected check %v", fset.Position(n.Pos), ident)
+						continue
+					}
+					if len(n.Args) != len(args)+1 {
+						t.Errorf("%v: expected %v args to check, got %v", fset.Position(n.Pos), len(args)+1, len(n.Args))
+						continue
+					}
+					for i, got := range n.Args[1:] {
+						if args[i] != got {
+							t.Errorf("%v: arg %d expected %v, got %v", fset.Position(n.Pos), i, args[i], got)
+						}
+					}
+				default:
+					t.Errorf("Unexpected note %v at %v", n.Name, fset.Position(n.Pos))
+				}
+			}
+		})
+	}
+}
+
+func checkMarker(t *testing.T, fset *token.FileSet, readFile expect.ReadFile, markers map[string]token.Pos, pos token.Pos, name string, pattern interface{}) {
+	start, end, err := expect.MatchBefore(fset, readFile, pos, pattern)
+	if err != nil {
+		t.Errorf("%v: MatchBefore failed: %v", fset.Position(pos), err)
+		return
+	}
+	if start == token.NoPos {
+		t.Errorf("%v: Pattern %v did not match", fset.Position(pos), pattern)
+		return
+	}
+	expectStart, ok := markers[name]
+	if !ok {
+		t.Errorf("%v: unexpected marker %v", fset.Position(pos), name)
+		return
+	}
+	if start != expectStart {
+		t.Errorf("%v: Expected %v got %v", fset.Position(pos), fset.Position(expectStart), fset.Position(start))
+	}
+	if expectEnd, ok := markers[name+"@"]; ok && end != expectEnd {
+		t.Errorf("%v: Expected end %v got %v", fset.Position(pos), fset.Position(expectEnd), fset.Position(end))
+	}
+}