Task collection improvements (#109)

vendor/golang.org/x/tools/go/internal/packagesdriver/sizes.go

@@ -81,13 +81,13 @@ func GetSizesGolist(ctx context.Context, buildFlags, env []string, dir string, u
 	args := []string{"list", "-f", "{{context.GOARCH}} {{context.Compiler}}"}
 	args = append(args, buildFlags...)
 	args = append(args, "--", "unsafe")
-	stdout, err := InvokeGo(ctx, env, dir, usesExportData, args...)
+	stdout, stderr, err := invokeGo(ctx, env, dir, usesExportData, args...)
 	var goarch, compiler string
 	if err != nil {
 		if strings.Contains(err.Error(), "cannot find main module") {
 			// User's running outside of a module. All bets are off. Get GOARCH and guess compiler is gc.
 			// TODO(matloob): Is this a problem in practice?
-			envout, enverr := InvokeGo(ctx, env, dir, usesExportData, "env", "GOARCH")
+			envout, _, enverr := invokeGo(ctx, env, dir, usesExportData, "env", "GOARCH")
 			if enverr != nil {
 				return nil, err
 			}
@@ -99,7 +99,8 @@ func GetSizesGolist(ctx context.Context, buildFlags, env []string, dir string, u
 	} else {
 		fields := strings.Fields(stdout.String())
 		if len(fields) < 2 {
-			return nil, fmt.Errorf("could not determine GOARCH and Go compiler")
+			return nil, fmt.Errorf("could not parse GOARCH and Go compiler in format \"<GOARCH> <compiler>\" from stdout of go command:\n%s\ndir: %s\nstdout: <<%s>>\nstderr: <<%s>>",
+				cmdDebugStr(env, args...), dir, stdout.String(), stderr.String())
 		}
 		goarch = fields[0]
 		compiler = fields[1]
@@ -107,8 +108,8 @@ func GetSizesGolist(ctx context.Context, buildFlags, env []string, dir string, u
 	return types.SizesFor(compiler, goarch), nil
 }
 
-// InvokeGo returns the stdout of a go command invocation.
-func InvokeGo(ctx context.Context, env []string, dir string, usesExportData bool, args ...string) (*bytes.Buffer, error) {
+// invokeGo returns the stdout and stderr of a go command invocation.
+func invokeGo(ctx context.Context, env []string, dir string, usesExportData bool, args ...string) (*bytes.Buffer, *bytes.Buffer, error) {
 	if debug {
 		defer func(start time.Time) { log.Printf("%s for %v", time.Since(start), cmdDebugStr(env, args...)) }(time.Now())
 	}
@@ -131,7 +132,7 @@ func InvokeGo(ctx context.Context, env []string, dir string, usesExportData bool
 			// Catastrophic error:
 			// - executable not found
 			// - context cancellation
-			return nil, fmt.Errorf("couldn't exec 'go %v': %s %T", args, err, err)
+			return nil, nil, fmt.Errorf("couldn't exec 'go %v': %s %T", args, err, err)
 		}
 
 		// Export mode entails a build.
@@ -139,7 +140,7 @@ func InvokeGo(ctx context.Context, env []string, dir string, usesExportData bool
 		// (despite the -e flag) and the Export field is blank.
 		// Do not fail in that case.
 		if !usesExportData {
-			return nil, fmt.Errorf("go %v: %s: %s", args, exitErr, stderr)
+			return nil, nil, fmt.Errorf("go %v: %s: %s", args, exitErr, stderr)
 		}
 	}
 
@@ -158,7 +159,7 @@ func InvokeGo(ctx context.Context, env []string, dir string, usesExportData bool
 		fmt.Fprintf(os.Stderr, "%s stdout: <<%s>>\n", cmdDebugStr(env, args...), stdout)
 	}
 
-	return stdout, nil
+	return stdout, stderr, nil
 }
 
 func cmdDebugStr(envlist []string, args ...string) string {

vendor/golang.org/x/tools/go/packages/external.go

@@ -12,6 +12,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"os"
 	"os/exec"
 	"strings"
 )
@@ -76,12 +77,17 @@ func findExternalDriver(cfg *Config) driver {
 		}
 
 		buf := new(bytes.Buffer)
+		stderr := new(bytes.Buffer)
 		cmd := exec.CommandContext(cfg.Context, tool, words...)
 		cmd.Dir = cfg.Dir
 		cmd.Env = cfg.Env
 		cmd.Stdin = bytes.NewReader(req)
 		cmd.Stdout = buf
-		cmd.Stderr = new(bytes.Buffer)
+		cmd.Stderr = stderr
+		if len(stderr.Bytes()) != 0 && os.Getenv("GOPACKAGESPRINTDRIVERERRORS") != "" {
+			fmt.Fprintf(os.Stderr, "%s stderr: <<%s>>\n", cmdDebugStr(cmd, words...), stderr)
+		}
+
 		if err := cmd.Run(); err != nil {
 			return nil, fmt.Errorf("%v: %v: %s", tool, err, cmd.Stderr)
 		}

vendor/golang.org/x/tools/go/packages/golist.go

@@ -110,6 +110,7 @@ func goListDriver(cfg *Config, patterns ...string) (*driverResponse, error) {
 			sizeswg.Done()
 		}()
 	}
+	defer sizeswg.Wait()
 
 	// start fetching rootDirs
 	var info goInfo
@@ -128,6 +129,10 @@ func goListDriver(cfg *Config, patterns ...string) (*driverResponse, error) {
 		return &info
 	}
 
+	// Ensure that we don't leak goroutines: Load is synchronous, so callers will
+	// not expect it to access the fields of cfg after the call returns.
+	defer getGoInfo()
+
 	// always pass getGoInfo to golistDriver
 	golistDriver := func(cfg *Config, patterns ...string) (*driverResponse, error) {
 		return golistDriver(cfg, getGoInfo, patterns...)
@@ -265,10 +270,7 @@ func addNeededOverlayPackages(cfg *Config, driver driver, response *responseDedu
 	if err != nil {
 		return err
 	}
-	if err := addNeededOverlayPackages(cfg, driver, response, needPkgs, getGoInfo); err != nil {
-		return err
-	}
-	return nil
+	return addNeededOverlayPackages(cfg, driver, response, needPkgs, getGoInfo)
 }
 
 func runContainsQueries(cfg *Config, driver driver, response *responseDeduper, queries []string, goInfo func() *goInfo) error {
@@ -368,7 +370,8 @@ func adHocPackage(cfg *Config, driver driver, pattern, query string) (*driverRes
 	// Special case to handle issue #33482:
 	// If this is a file= query for ad-hoc packages where the file only exists on an overlay,
 	// and exists outside of a module, add the file in for the package.
-	if len(dirResponse.Packages) == 1 && (dirResponse.Packages[0].ID == "command-line-arguments" || dirResponse.Packages[0].PkgPath == filepath.ToSlash(query)) {
+	if len(dirResponse.Packages) == 1 && (dirResponse.Packages[0].ID == "command-line-arguments" ||
+		filepath.ToSlash(dirResponse.Packages[0].PkgPath) == filepath.ToSlash(query)) {
 		if len(dirResponse.Packages[0].GoFiles) == 0 {
 			filename := filepath.Join(pattern, filepath.Base(query)) // avoid recomputing abspath
 			// TODO(matloob): check if the file is outside of a root dir?
@@ -734,9 +737,9 @@ func golistDriver(cfg *Config, rootsDirs func() *goInfo, words ...string) (*driv
 	// go list uses the following identifiers in ImportPath and Imports:
 	//
 	// 	"p"			-- importable package or main (command)
-	//      "q.test"		-- q's test executable
+	// 	"q.test"		-- q's test executable
 	// 	"p [q.test]"		-- variant of p as built for q's test executable
-	//	"q_test [q.test]"	-- q's external test package
+	// 	"q_test [q.test]"	-- q's external test package
 	//
 	// The packages p that are built differently for a test q.test
 	// are q itself, plus any helpers used by the external test q_test,
@@ -1061,7 +1064,14 @@ func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) {
 				// TODO(matloob): command-line-arguments isn't correct here.
 				"command-line-arguments", strings.Trim(stderr.String(), "\n"))
 			return bytes.NewBufferString(output), nil
+		}
 
+		// Another variation of the previous error
+		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "outside module root") {
+			output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
+				// TODO(matloob): command-line-arguments isn't correct here.
+				"command-line-arguments", strings.Trim(stderr.String(), "\n"))
+			return bytes.NewBufferString(output), nil
 		}
 
 		// Workaround for an instance of golang.org/issue/26755: go list -e  will return a non-zero exit

vendor/golang.org/x/tools/go/packages/loadmode_string.go

@@ -0,0 +1,57 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packages
+
+import (
+	"fmt"
+	"strings"
+)
+
+var allModes = []LoadMode{
+	NeedName,
+	NeedFiles,
+	NeedCompiledGoFiles,
+	NeedImports,
+	NeedDeps,
+	NeedExportsFile,
+	NeedTypes,
+	NeedSyntax,
+	NeedTypesInfo,
+	NeedTypesSizes,
+}
+
+var modeStrings = []string{
+	"NeedName",
+	"NeedFiles",
+	"NeedCompiledGoFiles",
+	"NeedImports",
+	"NeedDeps",
+	"NeedExportsFile",
+	"NeedTypes",
+	"NeedSyntax",
+	"NeedTypesInfo",
+	"NeedTypesSizes",
+}
+
+func (mod LoadMode) String() string {
+	m := mod
+	if m == 0 {
+		return fmt.Sprintf("LoadMode(0)")
+	}
+	var out []string
+	for i, x := range allModes {
+		if x > m {
+			break
+		}
+		if (m & x) != 0 {
+			out = append(out, modeStrings[i])
+			m = m ^ x
+		}
+	}
+	if m != 0 {
+		out = append(out, "Unknown")
+	}
+	return fmt.Sprintf("LoadMode(%s)", strings.Join(out, "|"))
+}

vendor/golang.org/x/tools/go/packages/packages.go

@@ -467,7 +467,7 @@ func newLoader(cfg *Config) *loader {
 	ld.requestedMode = ld.Mode
 	ld.Mode = impliedLoadMode(ld.Mode)
 
-	if ld.Mode&NeedTypes != 0 {
+	if ld.Mode&NeedTypes != 0 || ld.Mode&NeedSyntax != 0 {
 		if ld.Fset == nil {
 			ld.Fset = token.NewFileSet()
 		}
@@ -609,9 +609,9 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
 			}
 		}
 	}
-	// Load type data if needed, starting at
+	// Load type data and syntax if needed, starting at
 	// the initial packages (roots of the import DAG).
-	if ld.Mode&NeedTypes != 0 {
+	if ld.Mode&NeedTypes != 0 || ld.Mode&NeedSyntax != 0 {
 		var wg sync.WaitGroup
 		for _, lpkg := range initial {
 			wg.Add(1)
@@ -770,7 +770,7 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) {
 		lpkg.Errors = append(lpkg.Errors, errs...)
 	}
 
-	if len(lpkg.CompiledGoFiles) == 0 && lpkg.ExportFile != "" {
+	if ld.Config.Mode&NeedTypes != 0 && len(lpkg.CompiledGoFiles) == 0 && lpkg.ExportFile != "" {
 		// The config requested loading sources and types, but sources are missing.
 		// Add an error to the package and fall back to loading from export data.
 		appendError(Error{"-", fmt.Sprintf("sources missing for package %s", lpkg.ID), ParseError})
@@ -784,6 +784,9 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) {
 	}
 
 	lpkg.Syntax = files
+	if ld.Config.Mode&NeedTypes == 0 {
+		return
+	}
 
 	lpkg.TypesInfo = &types.Info{
 		Types:      make(map[ast.Expr]types.TypeAndValue),

vendor/golang.org/x/tools/internal/span/token.go

@@ -13,9 +13,10 @@ import (
 // It also carries the FileSet that produced the positions, so that it is
 // self contained.
 type Range struct {
-	FileSet *token.FileSet
-	Start   token.Pos
-	End     token.Pos
+	FileSet   *token.FileSet
+	Start     token.Pos
+	End       token.Pos
+	Converter Converter
 }
 
 // TokenConverter is a Converter backed by a token file set and file.
@@ -64,23 +65,56 @@ func (r Range) Span() (Span, error) {
 	if f == nil {
 		return Span{}, fmt.Errorf("file not found in FileSet")
 	}
-	s := Span{v: span{URI: FileURI(f.Name())}}
+	var s Span
 	var err error
-	s.v.Start.Offset, err = offset(f, r.Start)
+	var startFilename string
+	startFilename, s.v.Start.Line, s.v.Start.Column, err = position(f, r.Start)
 	if err != nil {
 		return Span{}, err
 	}
+	s.v.URI = FileURI(startFilename)
 	if r.End.IsValid() {
-		s.v.End.Offset, err = offset(f, r.End)
+		var endFilename string
+		endFilename, s.v.End.Line, s.v.End.Column, err = position(f, r.End)
 		if err != nil {
 			return Span{}, err
 		}
+		// In the presence of line directives, a single File can have sections from
+		// multiple file names.
+		if endFilename != startFilename {
+			return Span{}, fmt.Errorf("span begins in file %q but ends in %q", startFilename, endFilename)
+		}
 	}
 	s.v.Start.clean()
 	s.v.End.clean()
 	s.v.clean()
-	converter := NewTokenConverter(r.FileSet, f)
-	return s.WithPosition(converter)
+	if r.Converter != nil {
+		return s.WithOffset(r.Converter)
+	}
+	if startFilename != f.Name() {
+		return Span{}, fmt.Errorf("must supply Converter for file %q containing lines from %q", f.Name(), startFilename)
+	}
+	return s.WithOffset(NewTokenConverter(r.FileSet, f))
+}
+
+func position(f *token.File, pos token.Pos) (string, int, int, error) {
+	off, err := offset(f, pos)
+	if err != nil {
+		return "", 0, 0, err
+	}
+	return positionFromOffset(f, off)
+}
+
+func positionFromOffset(f *token.File, offset int) (string, int, int, error) {
+	if offset > f.Size() {
+		return "", 0, 0, fmt.Errorf("offset %v is past the end of the file %v", offset, f.Size())
+	}
+	pos := f.Pos(offset)
+	p := f.Position(pos)
+	if offset == f.Size() {
+		return p.Filename, p.Line + 1, 1, nil
+	}
+	return p.Filename, p.Line, p.Column, nil
 }
 
 // offset is a copy of the Offset function in go/token, but with the adjustment
@@ -108,22 +142,16 @@ func (s Span) Range(converter *TokenConverter) (Range, error) {
 		return Range{}, fmt.Errorf("end offset %v is past the end of the file %v", s.End(), converter.file.Size())
 	}
 	return Range{
-		FileSet: converter.fset,
-		Start:   converter.file.Pos(s.Start().Offset()),
-		End:     converter.file.Pos(s.End().Offset()),
+		FileSet:   converter.fset,
+		Start:     converter.file.Pos(s.Start().Offset()),
+		End:       converter.file.Pos(s.End().Offset()),
+		Converter: converter,
 	}, nil
 }
 
 func (l *TokenConverter) ToPosition(offset int) (int, int, error) {
-	if offset > l.file.Size() {
-		return 0, 0, fmt.Errorf("offset %v is past the end of the file %v", offset, l.file.Size())
-	}
-	pos := l.file.Pos(offset)
-	p := l.fset.Position(pos)
-	if offset == l.file.Size() {
-		return p.Line + 1, 1, nil
-	}
-	return p.Line, p.Column, nil
+	_, line, col, err := positionFromOffset(l.file, offset)
+	return line, col, err
 }
 
 func (l *TokenConverter) ToOffset(line, col int) (int, error) {