vcweb.go

Documentation: cmd/go/internal/vcweb

     1  // Copyright 2022 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package vcweb serves version control repos for testing the go command.
     6  //
     7  // It is loosely derived from golang.org/x/build/vcs-test/vcweb,
     8  // which ran as a service hosted at vcs-test.golang.org.
     9  //
    10  // When a repository URL is first requested, the vcweb [Server] dynamically
    11  // regenerates the repository using a script interpreted by a [script.Engine].
    12  // The script produces the server's contents for a corresponding root URL and
    13  // all subdirectories of that URL, which are then cached: subsequent requests
    14  // for any URL generated by the script will serve the script's previous output
    15  // until the script is modified.
    16  //
    17  // The script engine includes all of the engine's default commands and
    18  // conditions, as well as commands for each supported VCS binary (bzr, fossil,
    19  // git, hg, and svn), a "handle" command that informs the script which protocol
    20  // or handler to use to serve the request, and utilities "at" (which sets
    21  // environment variables for Git timestamps) and "unquote" (which unquotes its
    22  // argument as if it were a Go string literal).
    23  //
    24  // The server's "/" endpoint provides a summary of the available scripts,
    25  // and "/help" provides documentation for the script environment.
    26  //
    27  // To run a standalone server based on the vcweb engine, use:
    28  //
    29  //	go test cmd/go/internal/vcweb/vcstest -v --port=0
    30  package vcweb
    31  
    32  import (
    33  	"bufio"
    34  	"cmd/go/internal/script"
    35  	"context"
    36  	"crypto/sha256"
    37  	"errors"
    38  	"fmt"
    39  	"io"
    40  	"io/fs"
    41  	"log"
    42  	"net/http"
    43  	"os"
    44  	"os/exec"
    45  	"path"
    46  	"path/filepath"
    47  	"runtime/debug"
    48  	"strings"
    49  	"sync"
    50  	"text/tabwriter"
    51  	"time"
    52  )
    53  
    54  // A Server serves cached, dynamically-generated version control repositories.
    55  type Server struct {
    56  	env    []string
    57  	logger *log.Logger
    58  
    59  	scriptDir string
    60  	workDir   string
    61  	homeDir   string // $workdir/home
    62  	engine    *script.Engine
    63  
    64  	scriptCache sync.Map // script path → *scriptResult
    65  
    66  	vcsHandlers map[string]vcsHandler
    67  }
    68  
    69  // A vcsHandler serves repositories over HTTP for a known version-control tool.
    70  type vcsHandler interface {
    71  	Available() bool
    72  	Handler(dir string, env []string, logger *log.Logger) (http.Handler, error)
    73  }
    74  
    75  // A scriptResult describes the cached result of executing a vcweb script.
    76  type scriptResult struct {
    77  	mu sync.RWMutex
    78  
    79  	hash     [sha256.Size]byte // hash of the script file, for cache invalidation
    80  	hashTime time.Time         // timestamp at which the script was run, for diagnostics
    81  
    82  	handler http.Handler // HTTP handler configured by the script
    83  	err     error        // error from executing the script, if any
    84  }
    85  
    86  // NewServer returns a Server that generates and serves repositories in workDir
    87  // using the scripts found in scriptDir and its subdirectories.
    88  //
    89  // A request for the path /foo/bar/baz will be handled by the first script along
    90  // that path that exists: $scriptDir/foo.txt, $scriptDir/foo/bar.txt, or
    91  // $scriptDir/foo/bar/baz.txt.
    92  func NewServer(scriptDir, workDir string, logger *log.Logger) (*Server, error) {
    93  	if scriptDir == "" {
    94  		panic("vcweb.NewServer: scriptDir is required")
    95  	}
    96  	var err error
    97  	scriptDir, err = filepath.Abs(scriptDir)
    98  	if err != nil {
    99  		return nil, err
   100  	}
   101  
   102  	if workDir == "" {
   103  		workDir, err = os.MkdirTemp("", "vcweb-*")
   104  		if err != nil {
   105  			return nil, err
   106  		}
   107  		logger.Printf("vcweb work directory: %s", workDir)
   108  	} else {
   109  		workDir, err = filepath.Abs(workDir)
   110  		if err != nil {
   111  			return nil, err
   112  		}
   113  	}
   114  
   115  	homeDir := filepath.Join(workDir, "home")
   116  	if err := os.MkdirAll(homeDir, 0755); err != nil {
   117  		return nil, err
   118  	}
   119  
   120  	env := scriptEnviron(homeDir)
   121  
   122  	s := &Server{
   123  		env:       env,
   124  		logger:    logger,
   125  		scriptDir: scriptDir,
   126  		workDir:   workDir,
   127  		homeDir:   homeDir,
   128  		engine:    newScriptEngine(),
   129  		vcsHandlers: map[string]vcsHandler{
   130  			"auth":     new(authHandler),
   131  			"dir":      new(dirHandler),
   132  			"bzr":      new(bzrHandler),
   133  			"fossil":   new(fossilHandler),
   134  			"git":      new(gitHandler),
   135  			"hg":       new(hgHandler),
   136  			"insecure": new(insecureHandler),
   137  			"svn":      &svnHandler{svnRoot: workDir, logger: logger},
   138  		},
   139  	}
   140  
   141  	if err := os.WriteFile(filepath.Join(s.homeDir, ".gitconfig"), []byte(gitConfig), 0644); err != nil {
   142  		return nil, err
   143  	}
   144  	gitConfigDir := filepath.Join(s.homeDir, ".config", "git")
   145  	if err := os.MkdirAll(gitConfigDir, 0755); err != nil {
   146  		return nil, err
   147  	}
   148  	if err := os.WriteFile(filepath.Join(gitConfigDir, "ignore"), []byte(""), 0644); err != nil {
   149  		return nil, err
   150  	}
   151  
   152  	if err := os.WriteFile(filepath.Join(s.homeDir, ".hgrc"), []byte(hgrc), 0644); err != nil {
   153  		return nil, err
   154  	}
   155  
   156  	return s, nil
   157  }
   158  
   159  func (s *Server) Close() error {
   160  	var firstErr error
   161  	for _, h := range s.vcsHandlers {
   162  		if c, ok := h.(io.Closer); ok {
   163  			if closeErr := c.Close(); firstErr == nil {
   164  				firstErr = closeErr
   165  			}
   166  		}
   167  	}
   168  	return firstErr
   169  }
   170  
   171  // gitConfig contains a ~/.gitconfg file that attempts to provide
   172  // deterministic, platform-agnostic behavior for the 'git' command.
   173  var gitConfig = `
   174  [user]
   175  	name = Go Gopher
   176  	email = gopher@golang.org
   177  [init]
   178  	defaultBranch = main
   179  [core]
   180  	eol = lf
   181  [gui]
   182  	encoding = utf-8
   183  `[1:]
   184  
   185  // hgrc contains a ~/.hgrc file that attempts to provide
   186  // deterministic, platform-agnostic behavior for the 'hg' command.
   187  var hgrc = `
   188  [ui]
   189  username=Go Gopher <gopher@golang.org>
   190  [phases]
   191  new-commit=public
   192  [extensions]
   193  convert=
   194  `[1:]
   195  
   196  // ServeHTTP implements [http.Handler] for version-control repositories.
   197  func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
   198  	s.logger.Printf("serving %s", req.URL)
   199  
   200  	defer func() {
   201  		if v := recover(); v != nil {
   202  			debug.PrintStack()
   203  			s.logger.Fatal(v)
   204  		}
   205  	}()
   206  
   207  	urlPath := req.URL.Path
   208  	if !strings.HasPrefix(urlPath, "/") {
   209  		urlPath = "/" + urlPath
   210  	}
   211  	clean := path.Clean(urlPath)[1:]
   212  	if clean == "" {
   213  		s.overview(w, req)
   214  		return
   215  	}
   216  	if clean == "help" {
   217  		s.help(w, req)
   218  		return
   219  	}
   220  
   221  	// Locate the script that generates the requested path.
   222  	// We follow directories all the way to the end, then look for a ".txt" file
   223  	// matching the first component that doesn't exist. That guarantees
   224  	// uniqueness: if a path exists as a directory, then it cannot exist as a
   225  	// ".txt" script (because the search would ignore that file).
   226  	scriptPath := "."
   227  	for _, part := range strings.Split(clean, "/") {
   228  		scriptPath = filepath.Join(scriptPath, part)
   229  		dir := filepath.Join(s.scriptDir, scriptPath)
   230  		if _, err := os.Stat(dir); err != nil {
   231  			if !os.IsNotExist(err) {
   232  				http.Error(w, err.Error(), http.StatusInternalServerError)
   233  				return
   234  			}
   235  			// scriptPath does not exist as a directory, so it either is the script
   236  			// location or the script doesn't exist.
   237  			break
   238  		}
   239  	}
   240  	scriptPath += ".txt"
   241  
   242  	err := s.HandleScript(scriptPath, s.logger, func(handler http.Handler) {
   243  		handler.ServeHTTP(w, req)
   244  	})
   245  	if err != nil {
   246  		s.logger.Print(err)
   247  		if notFound := (ScriptNotFoundError{}); errors.As(err, &notFound) {
   248  			http.NotFound(w, req)
   249  		} else if notInstalled := (ServerNotInstalledError{}); errors.As(err, &notInstalled) || errors.Is(err, exec.ErrNotFound) {
   250  			http.Error(w, err.Error(), http.StatusNotImplemented)
   251  		} else {
   252  			http.Error(w, err.Error(), http.StatusInternalServerError)
   253  		}
   254  	}
   255  }
   256  
   257  // A ScriptNotFoundError indicates that the requested script file does not exist.
   258  // (It typically wraps a "stat" error for the script file.)
   259  type ScriptNotFoundError struct{ err error }
   260  
   261  func (e ScriptNotFoundError) Error() string { return e.err.Error() }
   262  func (e ScriptNotFoundError) Unwrap() error { return e.err }
   263  
   264  // A ServerNotInstalledError indicates that the server binary required for the
   265  // indicated VCS does not exist.
   266  type ServerNotInstalledError struct{ name string }
   267  
   268  func (v ServerNotInstalledError) Error() string {
   269  	return fmt.Sprintf("server for %#q VCS is not installed", v.name)
   270  }
   271  
   272  // HandleScript ensures that the script at scriptRelPath has been evaluated
   273  // with its current contents.
   274  //
   275  // If the script completed successfully, HandleScript invokes f on the handler
   276  // with the script's result still read-locked, and waits for it to return. (That
   277  // ensures that cache invalidation does not race with an in-flight handler.)
   278  //
   279  // Otherwise, HandleScript returns the (cached) error from executing the script.
   280  func (s *Server) HandleScript(scriptRelPath string, logger *log.Logger, f func(http.Handler)) error {
   281  	ri, ok := s.scriptCache.Load(scriptRelPath)
   282  	if !ok {
   283  		ri, _ = s.scriptCache.LoadOrStore(scriptRelPath, new(scriptResult))
   284  	}
   285  	r := ri.(*scriptResult)
   286  
   287  	relDir := strings.TrimSuffix(scriptRelPath, filepath.Ext(scriptRelPath))
   288  	workDir := filepath.Join(s.workDir, relDir)
   289  	prefix := path.Join("/", filepath.ToSlash(relDir))
   290  
   291  	r.mu.RLock()
   292  	defer r.mu.RUnlock()
   293  	for {
   294  		// For efficiency, we cache the script's output (in the work directory)
   295  		// across invocations. However, to allow for rapid iteration, we hash the
   296  		// script's contents and regenerate its output if the contents change.
   297  		//
   298  		// That way, one can use 'go run main.go' in this directory to stand up a
   299  		// server and see the output of the test script in order to fine-tune it.
   300  		content, err := os.ReadFile(filepath.Join(s.scriptDir, scriptRelPath))
   301  		if err != nil {
   302  			if !os.IsNotExist(err) {
   303  				return err
   304  			}
   305  			return ScriptNotFoundError{err}
   306  		}
   307  
   308  		hash := sha256.Sum256(content)
   309  		if prevHash := r.hash; prevHash != hash {
   310  			// The script's hash has changed, so regenerate its output.
   311  			func() {
   312  				r.mu.RUnlock()
   313  				r.mu.Lock()
   314  				defer func() {
   315  					r.mu.Unlock()
   316  					r.mu.RLock()
   317  				}()
   318  				if r.hash != prevHash {
   319  					// The cached result changed while we were waiting on the lock.
   320  					// It may have been updated to our hash or something even newer,
   321  					// so don't overwrite it.
   322  					return
   323  				}
   324  
   325  				r.hash = hash
   326  				r.hashTime = time.Now()
   327  				r.handler, r.err = nil, nil
   328  
   329  				if err := os.RemoveAll(workDir); err != nil {
   330  					r.err = err
   331  					return
   332  				}
   333  
   334  				// Note: we use context.Background here instead of req.Context() so that we
   335  				// don't cache a spurious error (and lose work) if the request is canceled
   336  				// while the script is still running.
   337  				scriptHandler, err := s.loadScript(context.Background(), logger, scriptRelPath, content, workDir)
   338  				if err != nil {
   339  					r.err = err
   340  					return
   341  				}
   342  				r.handler = http.StripPrefix(prefix, scriptHandler)
   343  			}()
   344  		}
   345  
   346  		if r.hash != hash {
   347  			continue // Raced with an update from another handler; try again.
   348  		}
   349  
   350  		if r.err != nil {
   351  			return r.err
   352  		}
   353  		f(r.handler)
   354  		return nil
   355  	}
   356  }
   357  
   358  // overview serves an HTML summary of the status of the scripts in the server's
   359  // script directory.
   360  func (s *Server) overview(w http.ResponseWriter, r *http.Request) {
   361  	fmt.Fprintf(w, "<html>\n")
   362  	fmt.Fprintf(w, "<title>vcweb</title>\n<pre>\n")
   363  	fmt.Fprintf(w, "<b>vcweb</b>\n\n")
   364  	fmt.Fprintf(w, "This server serves various version control repos for testing the go command.\n\n")
   365  	fmt.Fprintf(w, "For an overview of the script language, see <a href=\"/help\">/help</a>.\n\n")
   366  
   367  	fmt.Fprintf(w, "<b>cache</b>\n")
   368  
   369  	tw := tabwriter.NewWriter(w, 1, 8, 1, '\t', 0)
   370  	err := filepath.WalkDir(s.scriptDir, func(path string, d fs.DirEntry, err error) error {
   371  		if err != nil {
   372  			return err
   373  		}
   374  		if filepath.Ext(path) != ".txt" {
   375  			return nil
   376  		}
   377  
   378  		rel, err := filepath.Rel(s.scriptDir, path)
   379  		if err != nil {
   380  			return err
   381  		}
   382  		hashTime := "(not loaded)"
   383  		status := ""
   384  		if ri, ok := s.scriptCache.Load(rel); ok {
   385  			r := ri.(*scriptResult)
   386  			r.mu.RLock()
   387  			defer r.mu.RUnlock()
   388  
   389  			if !r.hashTime.IsZero() {
   390  				hashTime = r.hashTime.Format(time.RFC3339)
   391  			}
   392  			if r.err == nil {
   393  				status = "ok"
   394  			} else {
   395  				status = r.err.Error()
   396  			}
   397  		}
   398  		fmt.Fprintf(tw, "%s\t%s\t%s\n", rel, hashTime, status)
   399  		return nil
   400  	})
   401  	tw.Flush()
   402  
   403  	if err != nil {
   404  		fmt.Fprintln(w, err)
   405  	}
   406  }
   407  
   408  // help serves a plain-text summary of the server's supported script language.
   409  func (s *Server) help(w http.ResponseWriter, req *http.Request) {
   410  	st, err := s.newState(req.Context(), s.workDir)
   411  	if err != nil {
   412  		http.Error(w, err.Error(), http.StatusInternalServerError)
   413  		return
   414  	}
   415  
   416  	scriptLog := new(strings.Builder)
   417  	err = s.engine.Execute(st, "help", bufio.NewReader(strings.NewReader("help")), scriptLog)
   418  	if err != nil {
   419  		http.Error(w, err.Error(), http.StatusInternalServerError)
   420  		return
   421  	}
   422  
   423  	w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
   424  	io.WriteString(w, scriptLog.String())
   425  }
   426
View as plain text