libgo/go/cmd/go/internal/modfetch/codehost/codehost.go

   1 // Copyright 2018 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Package codehost defines the interface implemented by a code hosting source,
   6 // along with support code for use by implementations.
   7 package codehost
   8
   9 import (
  10         "bytes"
  11         "crypto/sha256"
  12         "fmt"
  13         exec "internal/execabs"
  14         "io"
  15         "io/fs"
  16         "os"
  17         "path/filepath"
  18         "strings"
  19         "sync"
  20         "time"
  21
  22         "cmd/go/internal/cfg"
  23         "cmd/go/internal/lockedfile"
  24         "cmd/go/internal/str"
  25 )
  26
  27 // Downloaded size limits.
  28 const (
  29         MaxGoMod   = 16 << 20  // maximum size of go.mod file
  30         MaxLICENSE = 16 << 20  // maximum size of LICENSE file
  31         MaxZipFile = 500 << 20 // maximum size of downloaded zip file
  32 )
  33
  34 // A Repo represents a code hosting source.
  35 // Typical implementations include local version control repositories,
  36 // remote version control servers, and code hosting sites.
  37 // A Repo must be safe for simultaneous use by multiple goroutines.
  38 type Repo interface {
  39         // List lists all tags with the given prefix.
  40         Tags(prefix string) (tags []string, err error)
  41
  42         // Stat returns information about the revision rev.
  43         // A revision can be any identifier known to the underlying service:
  44         // commit hash, branch, tag, and so on.
  45         Stat(rev string) (*RevInfo, error)
  46
  47         // Latest returns the latest revision on the default branch,
  48         // whatever that means in the underlying implementation.
  49         Latest() (*RevInfo, error)
  50
  51         // ReadFile reads the given file in the file tree corresponding to revision rev.
  52         // It should refuse to read more than maxSize bytes.
  53         //
  54         // If the requested file does not exist it should return an error for which
  55         // os.IsNotExist(err) returns true.
  56         ReadFile(rev, file string, maxSize int64) (data []byte, err error)
  57
  58         // ReadZip downloads a zip file for the subdir subdirectory
  59         // of the given revision to a new file in a given temporary directory.
  60         // It should refuse to read more than maxSize bytes.
  61         // It returns a ReadCloser for a streamed copy of the zip file.
  62         // All files in the zip file are expected to be
  63         // nested in a single top-level directory, whose name is not specified.
  64         ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error)
  65
  66         // RecentTag returns the most recent tag on rev or one of its predecessors
  67         // with the given prefix. allowed may be used to filter out unwanted versions.
  68         RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error)
  69
  70         // DescendsFrom reports whether rev or any of its ancestors has the given tag.
  71         //
  72         // DescendsFrom must return true for any tag returned by RecentTag for the
  73         // same revision.
  74         DescendsFrom(rev, tag string) (bool, error)
  75 }
  76
  77 // A Rev describes a single revision in a source code repository.
  78 type RevInfo struct {
  79         Name    string    // complete ID in underlying repository
  80         Short   string    // shortened ID, for use in pseudo-version
  81         Version string    // version used in lookup
  82         Time    time.Time // commit time
  83         Tags    []string  // known tags for commit
  84 }
  85
  86 // A FileRev describes the result of reading a file at a given revision.
  87 type FileRev struct {
  88         Rev  string // requested revision
  89         Data []byte // file data
  90         Err  error  // error if any; os.IsNotExist(Err)==true if rev exists but file does not exist in that rev
  91 }
  92
  93 // UnknownRevisionError is an error equivalent to fs.ErrNotExist, but for a
  94 // revision rather than a file.
  95 type UnknownRevisionError struct {
  96         Rev string
  97 }
  98
  99 func (e *UnknownRevisionError) Error() string {
 100         return "unknown revision " + e.Rev
 101 }
 102 func (UnknownRevisionError) Is(err error) bool {
 103         return err == fs.ErrNotExist
 104 }
 105
 106 // ErrNoCommits is an error equivalent to fs.ErrNotExist indicating that a given
 107 // repository or module contains no commits.
 108 var ErrNoCommits error = noCommitsError{}
 109
 110 type noCommitsError struct{}
 111
 112 func (noCommitsError) Error() string {
 113         return "no commits"
 114 }
 115 func (noCommitsError) Is(err error) bool {
 116         return err == fs.ErrNotExist
 117 }
 118
 119 // AllHex reports whether the revision rev is entirely lower-case hexadecimal digits.
 120 func AllHex(rev string) bool {
 121         for i := 0; i < len(rev); i++ {
 122                 c := rev[i]
 123                 if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' {
 124                         continue
 125                 }
 126                 return false
 127         }
 128         return true
 129 }
 130
 131 // ShortenSHA1 shortens a SHA1 hash (40 hex digits) to the canonical length
 132 // used in pseudo-versions (12 hex digits).
 133 func ShortenSHA1(rev string) string {
 134         if AllHex(rev) && len(rev) == 40 {
 135                 return rev[:12]
 136         }
 137         return rev
 138 }
 139
 140 // WorkDir returns the name of the cached work directory to use for the
 141 // given repository type and name.
 142 func WorkDir(typ, name string) (dir, lockfile string, err error) {
 143         if cfg.GOMODCACHE == "" {
 144                 return "", "", fmt.Errorf("neither GOPATH nor GOMODCACHE are set")
 145         }
 146
 147         // We name the work directory for the SHA256 hash of the type and name.
 148         // We intentionally avoid the actual name both because of possible
 149         // conflicts with valid file system paths and because we want to ensure
 150         // that one checkout is never nested inside another. That nesting has
 151         // led to security problems in the past.
 152         if strings.Contains(typ, ":") {
 153                 return "", "", fmt.Errorf("codehost.WorkDir: type cannot contain colon")
 154         }
 155         key := typ + ":" + name
 156         dir = filepath.Join(cfg.GOMODCACHE, "cache/vcs", fmt.Sprintf("%x", sha256.Sum256([]byte(key))))
 157
 158         if cfg.BuildX {
 159                 fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", filepath.Dir(dir), typ, name)
 160         }
 161         if err := os.MkdirAll(filepath.Dir(dir), 0777); err != nil {
 162                 return "", "", err
 163         }
 164
 165         lockfile = dir + ".lock"
 166         if cfg.BuildX {
 167                 fmt.Fprintf(os.Stderr, "# lock %s", lockfile)
 168         }
 169
 170         unlock, err := lockedfile.MutexAt(lockfile).Lock()
 171         if err != nil {
 172                 return "", "", fmt.Errorf("codehost.WorkDir: can't find or create lock file: %v", err)
 173         }
 174         defer unlock()
 175
 176         data, err := os.ReadFile(dir + ".info")
 177         info, err2 := os.Stat(dir)
 178         if err == nil && err2 == nil && info.IsDir() {
 179                 // Info file and directory both already exist: reuse.
 180                 have := strings.TrimSuffix(string(data), "\n")
 181                 if have != key {
 182                         return "", "", fmt.Errorf("%s exists with wrong content (have %q want %q)", dir+".info", have, key)
 183                 }
 184                 if cfg.BuildX {
 185                         fmt.Fprintf(os.Stderr, "# %s for %s %s\n", dir, typ, name)
 186                 }
 187                 return dir, lockfile, nil
 188         }
 189
 190         // Info file or directory missing. Start from scratch.
 191         if cfg.BuildX {
 192                 fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", dir, typ, name)
 193         }
 194         os.RemoveAll(dir)
 195         if err := os.MkdirAll(dir, 0777); err != nil {
 196                 return "", "", err
 197         }
 198         if err := os.WriteFile(dir+".info", []byte(key), 0666); err != nil {
 199                 os.RemoveAll(dir)
 200                 return "", "", err
 201         }
 202         return dir, lockfile, nil
 203 }
 204
 205 type RunError struct {
 206         Cmd      string
 207         Err      error
 208         Stderr   []byte
 209         HelpText string
 210 }
 211
 212 func (e *RunError) Error() string {
 213         text := e.Cmd + ": " + e.Err.Error()
 214         stderr := bytes.TrimRight(e.Stderr, "\n")
 215         if len(stderr) > 0 {
 216                 text += ":\n\t" + strings.ReplaceAll(string(stderr), "\n", "\n\t")
 217         }
 218         if len(e.HelpText) > 0 {
 219                 text += "\n" + e.HelpText
 220         }
 221         return text
 222 }
 223
 224 var dirLock sync.Map
 225
 226 // Run runs the command line in the given directory
 227 // (an empty dir means the current directory).
 228 // It returns the standard output and, for a non-zero exit,
 229 // a *RunError indicating the command, exit status, and standard error.
 230 // Standard error is unavailable for commands that exit successfully.
 231 func Run(dir string, cmdline ...any) ([]byte, error) {
 232         return RunWithStdin(dir, nil, cmdline...)
 233 }
 234
 235 // bashQuoter escapes characters that have special meaning in double-quoted strings in the bash shell.
 236 // See https://www.gnu.org/software/bash/manual/html_node/Double-Quotes.html.
 237 var bashQuoter = strings.NewReplacer(`"`, `\"`, `$`, `\$`, "`", "\\`", `\`, `\\`)
 238
 239 func RunWithStdin(dir string, stdin io.Reader, cmdline ...any) ([]byte, error) {
 240         if dir != "" {
 241                 muIface, ok := dirLock.Load(dir)
 242                 if !ok {
 243                         muIface, _ = dirLock.LoadOrStore(dir, new(sync.Mutex))
 244                 }
 245                 mu := muIface.(*sync.Mutex)
 246                 mu.Lock()
 247                 defer mu.Unlock()
 248         }
 249
 250         cmd := str.StringList(cmdline...)
 251         if os.Getenv("TESTGOVCS") == "panic" {
 252                 panic(fmt.Sprintf("use of vcs: %v", cmd))
 253         }
 254         if cfg.BuildX {
 255                 text := new(strings.Builder)
 256                 if dir != "" {
 257                         text.WriteString("cd ")
 258                         text.WriteString(dir)
 259                         text.WriteString("; ")
 260                 }
 261                 for i, arg := range cmd {
 262                         if i > 0 {
 263                                 text.WriteByte(' ')
 264                         }
 265                         switch {
 266                         case strings.ContainsAny(arg, "'"):
 267                                 // Quote args that could be mistaken for quoted args.
 268                                 text.WriteByte('"')
 269                                 text.WriteString(bashQuoter.Replace(arg))
 270                                 text.WriteByte('"')
 271                         case strings.ContainsAny(arg, "$`\\*?[\"\t\n\v\f\r \u0085\u00a0"):
 272                                 // Quote args that contain special characters, glob patterns, or spaces.
 273                                 text.WriteByte('\'')
 274                                 text.WriteString(arg)
 275                                 text.WriteByte('\'')
 276                         default:
 277                                 text.WriteString(arg)
 278                         }
 279                 }
 280                 fmt.Fprintf(os.Stderr, "%s\n", text)
 281                 start := time.Now()
 282                 defer func() {
 283                         fmt.Fprintf(os.Stderr, "%.3fs # %s\n", time.Since(start).Seconds(), text)
 284                 }()
 285         }
 286         // TODO: Impose limits on command output size.
 287         // TODO: Set environment to get English error messages.
 288         var stderr bytes.Buffer
 289         var stdout bytes.Buffer
 290         c := exec.Command(cmd[0], cmd[1:]...)
 291         c.Dir = dir
 292         c.Stdin = stdin
 293         c.Stderr = &stderr
 294         c.Stdout = &stdout
 295         err := c.Run()
 296         if err != nil {
 297                 err = &RunError{Cmd: strings.Join(cmd, " ") + " in " + dir, Stderr: stderr.Bytes(), Err: err}
 298         }
 299         return stdout.Bytes(), err
 300 }