summercms/internal/build/source.go

package build

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"slices"
	"sort"
	"strings"

	"git.golem15.com/golem15/summercms/modules/bonfire"
	"git.golem15.com/golem15/summercms/modules/kiosk"
)

// cliGitProtocols is GIT_ALLOW_PROTOCOL for the git commands of
// plugin:install and plugin:update, and the URL schemes they accept.
// Tests add "file" for fixture repositories; nothing else changes it.
var cliGitProtocols = "https:ssh"

var (
	scpLikeRe   = regexp.MustCompile(`^[A-Za-z0-9._-]+@[A-Za-z0-9.-]+:`)
	repoNameRe  = regexp.MustCompile(`^[a-z][a-z0-9]*$`)
	installTemp = ".summer-install-"
)

// allowedGitProtocol reports whether cliGitProtocols lists name.
func allowedGitProtocol(name string) bool {
	return slices.Contains(strings.Split(cliGitProtocols, ":"), name)
}

// runCLIGit runs git in dir for the CLI. It keeps the user's git config
// (credential helpers, insteadOf), allows only cliGitProtocols, and stops
// git from prompting when no person can answer. Callers put "--" before
// URLs and paths.
func runCLIGit(ctx context.Context, dir string, interactive bool, args ...string) ([]byte, error) {
	cmd := exec.CommandContext(ctx, "git", args...)
	cmd.Dir = dir
	env := make([]string, 0, len(os.Environ())+2)
	for _, e := range os.Environ() {
		if strings.HasPrefix(e, "GIT_ALLOW_PROTOCOL=") || strings.HasPrefix(e, "GIT_TERMINAL_PROMPT=") {
			continue
		}
		env = append(env, e)
	}
	env = append(env, "GIT_ALLOW_PROTOCOL="+cliGitProtocols)
	if !interactive {
		env = append(env, "GIT_TERMINAL_PROMPT=0")
	}
	cmd.Env = env
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	out, err := cmd.Output()
	if err != nil {
		msg := strings.TrimSpace(stderr.String())
		if msg == "" {
			msg = err.Error()
		}
		return out, fmt.Errorf("git %s: %s", args[0], msg)
	}
	return out, nil
}

// isGitURL reports whether arg is meant as a repository address rather
// than a plugin ID: a scheme URL, a scp-like user@host:path or a
// transport::address form (which validateGitURL then refuses).
func isGitURL(arg string) bool {
	return strings.Contains(arg, "://") || strings.Contains(arg, "::") || scpLikeRe.MatchString(arg)
}

// validateGitURL accepts https://, ssh:// and scp-like user@host:path
// addresses (file:// only when cliGitProtocols allows it, in tests). It
// refuses a leading "-", transport::address forms, http, credentials in
// the URL and control characters.
func validateGitURL(raw string) error {
	switch {
	case raw == "":
		return errors.New("git URL is empty")
	case strings.HasPrefix(raw, "-"):
		return fmt.Errorf("git URL %q must not start with -", raw)
	case strings.ContainsFunc(raw, func(r rune) bool { return r <= ' ' || r == 0x7f }):
		return fmt.Errorf("git URL %q contains spaces or control characters", raw)
	case strings.Contains(raw, "::"):
		return fmt.Errorf("git URL %q uses a transport:: form; use https://, ssh:// or user@host:path", raw)
	}
	if !strings.Contains(raw, "://") {
		if !scpLikeRe.MatchString(raw) {
			return fmt.Errorf("%q is not a git URL; use https://, ssh:// or user@host:path", raw)
		}
		if !allowedGitProtocol("ssh") {
			return fmt.Errorf("git URL %q: ssh is not allowed", raw)
		}
		_, p, _ := strings.Cut(raw, ":")
		if p == "" || strings.HasPrefix(p, "-") {
			return fmt.Errorf("git URL %q has an invalid path", raw)
		}
		return nil
	}
	u, err := url.Parse(raw)
	if err != nil {
		return fmt.Errorf("git URL %q: %w", raw, err)
	}
	scheme := strings.ToLower(u.Scheme)
	if !allowedGitProtocol(scheme) {
		return fmt.Errorf("git URL %q: only https://, ssh:// and user@host:path are accepted", raw)
	}
	if scheme != "file" && u.Host == "" {
		return fmt.Errorf("git URL %q has no host", raw)
	}
	if _, hasPassword := u.User.Password(); hasPassword {
		return fmt.Errorf("git URL must not carry a password; use a git credential helper")
	}
	if strings.HasPrefix(u.Path, "/-") || u.Path == "" || u.Path == "/" {
		return fmt.Errorf("git URL %q has an invalid path", raw)
	}
	return nil
}

// validateRepositoryURL checks a repository URL the registry sent: it must
// be https without user info (file:// only in tests).
func validateRepositoryURL(id, raw string) error {
	if raw == "" {
		return fmt.Errorf("build: registry entry %s has no repository URL; refused", id)
	}
	if err := validateGitURL(raw); err != nil {
		return fmt.Errorf("build: registry entry %s: %w; refused", id, err)
	}
	u, err := url.Parse(raw)
	if err != nil || !strings.Contains(raw, "://") {
		return fmt.Errorf("build: registry entry %s: repository %q is not an https URL; refused", id, raw)
	}
	if s := strings.ToLower(u.Scheme); s != "https" && !(s == "file" && allowedGitProtocol("file")) {
		return fmt.Errorf("build: registry entry %s: repository %q is not an https URL; refused", id, raw)
	}
	if u.User != nil {
		return fmt.Errorf("build: registry entry %s: repository URL carries user info; refused", id)
	}
	return nil
}

// requireGitRepo refuses submodule installs in an application that is not
// a git repository.
func requireGitRepo(ctx context.Context, appDir string) error {
	if _, err := gitOutput(ctx, appDir, "rev-parse", "--show-toplevel"); err != nil {
		return bonfire.WithHint(fmt.Errorf("build: %s is not a git repository; a plugin installed from source is a git submodule, which needs one", appDir),
			"--source and git URL installs add the plugin as a git submodule of the application. Initialise a repository first, or install the plugin by ID as a Go module.",
			"git init && git add -A && git commit -m init")
	}
	return nil
}

// cloneInto clones url into a new temporary directory under
// <app>/plugins, so the later move to plugins/<vendor>/<name> stays on one
// file system.
func cloneInto(ctx context.Context, appDir, rawURL string, interactive bool) (string, error) {
	parent := filepath.Join(appDir, "plugins")
	if err := os.MkdirAll(parent, 0o755); err != nil {
		return "", fmt.Errorf("build: create %s: %w", parent, err)
	}
	tmp, err := os.MkdirTemp(parent, installTemp)
	if err != nil {
		return "", fmt.Errorf("build: %w", err)
	}
	if _, err := runCLIGit(ctx, appDir, interactive, "clone", "--quiet", "--", rawURL, tmp); err != nil {
		_ = os.RemoveAll(tmp)
		return "", fmt.Errorf("build: clone %s: %w", rawURL, err)
	}
	return tmp, nil
}

// releaseTags returns the vX.Y.Z tags of the repository in dir, newest
// first.
func releaseTags(ctx context.Context, dir string) ([]string, error) {
	out, err := runCLIGit(ctx, dir, false, "tag", "--list", "v*")
	if err != nil {
		return nil, fmt.Errorf("build: %w", err)
	}
	var tags []string
	for _, t := range strings.Fields(string(out)) {
		if kiosk.ValidVersion(t) {
			tags = append(tags, t)
		}
	}
	sort.Slice(tags, func(i, j int) bool { return kiosk.CompareVersions(tags[i], tags[j]) > 0 })
	return tags, nil
}

// manifestAt reads <rel>/plugin.yaml at a git ref; nil when the file does
// not exist there.
func manifestAt(ctx context.Context, dir, ref, rel string) (*kiosk.Manifest, error) {
	p := kiosk.FileName
	if rel != "" {
		p = path.Join(rel, kiosk.FileName)
	}
	out, err := runCLIGit(ctx, dir, false, "show", ref+":"+p)
	if err != nil {
		return nil, nil
	}
	m, err := kiosk.Parse(out)
	if err != nil {
		return nil, fmt.Errorf("build: %s at %s: %w", p, ref, err)
	}
	return &m, nil
}

// checkout moves the work tree in dir to ref (a release tag).
func checkout(ctx context.Context, dir, ref string) error {
	if _, err := runCLIGit(ctx, dir, false, "checkout", "--quiet", "--detach", ref); err != nil {
		return fmt.Errorf("build: check out %s: %w", ref, err)
	}
	return nil
}

// planGitURL clones the repository at rawURL, picks the newest release
// tag whose plugin.yaml framework minimum the application satisfies
// (falling back to the default branch head with a warning) and adds the
// plugin and its requirements to the plan. The plugin itself is never
// looked up in the registry.
func (in *installer) planGitURL(rawURL string) error {
	if err := validateGitURL(rawURL); err != nil {
		return fmt.Errorf("build: %w", err)
	}
	if err := requireGitRepo(in.ctx, in.appDir); err != nil {
		return err
	}
	if dir := submoduleWithURL(in.ctx, in.appDir, rawURL); dir != "" {
		return bonfire.WithHint(fmt.Errorf("build: %s is already checked out at %s", rawURL, dir),
			"plugin:install does not clone a repository the application already has as a submodule. Update the plugins in it with plugin:update.",
			"{bin} plugin:update")
	}
	fmt.Fprintf(in.out, "Cloning %s\n", rawURL)
	tmp, err := cloneInto(in.ctx, in.appDir, rawURL, in.opts.Interactive)
	if err != nil {
		return err
	}
	in.plan.temps = append(in.plan.temps, tmp)

	tags, err := releaseTags(in.ctx, tmp)
	if err != nil {
		return err
	}
	ref, version := "", ""
	var pm *kiosk.Manifest
	newest := ""
	for _, tag := range tags {
		m, err := manifestAt(in.ctx, tmp, tag, "")
		if err != nil {
			fmt.Fprintf(in.out, "warning: %v; tag skipped\n", err)
			continue
		}
		if m == nil {
			continue
		}
		if newest == "" {
			newest = tag + " needs framework " + m.Framework
		}
		if kiosk.Satisfies(in.fw, m.Framework) {
			ref, version, pm = tag, tag, m
			break
		}
	}
	if ref == "" {
		why := "no release tag carries a plugin.yaml"
		if newest != "" {
			why = "no release tag fits framework " + displayVersion(in.fw) + " (" + newest + ")"
		}
		fmt.Fprintf(in.out, "warning: %s; installing the default branch head of %s\n", why, rawURL)
	} else if err := checkout(in.ctx, tmp, ref); err != nil {
		return err
	}

	step := InstallStep{Version: version, Repository: rawURL, Mode: ModeSubmodule, clone: tmp}
	var requires kiosk.Requires
	if pm == nil {
		pm, err = readCheckoutManifest(tmp, "")
		if err != nil {
			return err
		}
	}
	if pm != nil {
		step.ID, step.Module, step.Package, requires = pm.ID, pm.Module, pm.Package, pm.Requires
	} else {
		// No plugin.yaml at all: read the identity from the sources.
		id, err := kiosk.ReadIdentity(in.ctx, tmp)
		if id.ID == "" {
			return fmt.Errorf("build: %s has no plugin.yaml and no readable plugin.go at its root: %v", rawURL, err)
		}
		mod, err := readModulePath(filepath.Join(tmp, "go.mod"))
		if err != nil {
			return err
		}
		step.ID, step.Module, step.Package = id.ID, mod, mod
		requires = kiosk.Requires{}
		for _, r := range id.Requires {
			requires[r] = ""
		}
	}
	if err := ValidatePluginID(step.ID); err != nil {
		return fmt.Errorf("build: %s: %w", rawURL, err)
	}
	if err := in.requireAbsent(step.ID); err != nil {
		return err
	}
	if err := validateStep(step); err != nil {
		return err
	}
	if im, ok := in.installed[step.Module]; ok {
		return bonfire.WithHint(fmt.Errorf("build: %s is module %s, which the application already has (for %s)", rawURL, step.Module, im.by),
			"plugin:install does not move plugins the application already has, and a second checkout of one module breaks the workspace. Update the module with plugin:update.",
			"{bin} plugin:update "+im.by)
	}
	if _, err := submoduleTarget(in.ctx, in.appDir, tmp, step.ID, rawURL); err != nil {
		return err
	}
	in.moduleVersion[step.Module] = step.Version
	in.visiting = append(in.visiting, step.ID)
	if err := in.resolveRequires(step.ID, step.Version, requires); err != nil {
		return err
	}
	in.visiting = in.visiting[:len(in.visiting)-1]
	in.planned[step.ID] = step
	in.plan.Steps = append(in.plan.Steps, step)
	return nil
}

// submoduleWithURL returns the path of the application's submodule whose
// .gitmodules URL is rawURL (ignoring a trailing slash or .git), or "".
func submoduleWithURL(ctx context.Context, appDir, rawURL string) string {
	out, err := gitOutput(ctx, appDir, "config", "--file", ".gitmodules", "--get-regexp", `^submodule\..*\.url$`)
	if err != nil {
		return ""
	}
	norm := func(u string) string {
		return strings.TrimSuffix(strings.TrimRight(u, "/"), ".git")
	}
	want := norm(rawURL)
	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
		key, val, ok := strings.Cut(line, " ")
		if !ok || norm(strings.TrimSpace(val)) != want {
			continue
		}
		name := strings.TrimSuffix(strings.TrimPrefix(key, "submodule."), ".url")
		if p, err := gitOutput(ctx, appDir, "config", "--file", ".gitmodules", "submodule."+name+".path"); err == nil {
			return strings.TrimSpace(string(p))
		}
		return name
	}
	return ""
}

// readCheckoutManifest reads <dir>/<rel>/plugin.yaml from a work tree; nil
// when it does not exist.
func readCheckoutManifest(dir, rel string) (*kiosk.Manifest, error) {
	p := filepath.Join(dir, filepath.FromSlash(rel), kiosk.FileName)
	raw, err := os.ReadFile(p)
	if errors.Is(err, os.ErrNotExist) {
		return nil, nil
	}
	if err != nil {
		return nil, fmt.Errorf("build: read %s: %w", p, err)
	}
	m, err := kiosk.Parse(raw)
	if err != nil {
		return nil, fmt.Errorf("build: %s: %w", p, err)
	}
	return &m, nil
}

// submoduleTarget returns plugins/<vendor>/<name> for a checkout: the ID
// in the repository root's plugin.yaml (or plugin.go), otherwise the
// repository name without sm- and -plugin under the vendor of stepID.
func submoduleTarget(ctx context.Context, appDir, cloneDir, stepID, repoURL string) (string, error) {
	id := ""
	if pm, err := readCheckoutManifest(cloneDir, ""); err == nil && pm != nil {
		id = pm.ID
	} else if ident, _ := kiosk.ReadIdentity(ctx, cloneDir); ident.ID != "" {
		id = ident.ID
	}
	var dir string
	if id != "" {
		if err := ValidatePluginID(id); err != nil {
			return "", fmt.Errorf("build: %s: %w", repoURL, err)
		}
		dir = filepath.Join(appDir, "plugins", pluginVendor(id), pluginName(id))
	} else {
		name := strings.TrimSuffix(path.Base(strings.TrimRight(repoURL, "/")), ".git")
		if i := strings.LastIndex(name, ":"); i >= 0 {
			name = name[i+1:]
		}
		name = strings.TrimSuffix(strings.TrimPrefix(name, "sm-"), "-plugin")
		if !repoNameRe.MatchString(name) {
			return "", fmt.Errorf("build: cannot name a plugin directory after repository %s", repoURL)
		}
		dir = filepath.Join(appDir, "plugins", pluginVendor(stepID), name)
	}
	if !underRoot(appDir, dir) {
		return "", fmt.Errorf("build: plugin path %q escapes the app root", dir)
	}
	if _, err := os.Stat(dir); err == nil {
		return "", fmt.Errorf("build: %s already exists; remove it or register it with plugin:add", dir)
	}
	return dir, nil
}

// applySubmodule installs one submodule step: clone and check out the tag
// (a git URL step was cloned while planning), check that the plugin.yaml
// in the checkout describes the planned plugin, move the checkout to
// plugins/<vendor>/<name>, add it as a submodule and register it in go.mod
// and go.work with the tag as the require version.
func applySubmodule(ctx context.Context, plan *InstallPlan, s *InstallStep, interactive bool, out io.Writer) error {
	appDir := plan.AppDir
	dir := s.clone
	if dir == "" {
		fmt.Fprintf(out, "Cloning %s at %s\n", s.Repository, s.Version)
		tmp, err := cloneInto(ctx, appDir, s.Repository, interactive)
		if err != nil {
			return err
		}
		plan.temps = append(plan.temps, tmp)
		if err := checkout(ctx, tmp, s.Version); err != nil {
			return err
		}
		dir = tmp
	}
	rel := strings.TrimPrefix(strings.TrimPrefix(s.Package, s.Module), "/")
	pm, err := readCheckoutManifest(dir, rel)
	if err != nil {
		return err
	}
	if pm != nil && (pm.ID != s.ID || pm.Module != s.Module) {
		return fmt.Errorf("build: %s at %s describes %s (%s), not %s (%s); refused", s.Repository, displayRef(s.Version), pm.ID, pm.Module, s.ID, s.Module)
	}
	if pm == nil && s.clone == "" {
		return fmt.Errorf("build: %s at %s has no %s for %s; refused", s.Repository, s.Version, path.Join(rel, kiosk.FileName), s.ID)
	}
	mod, err := readModulePath(filepath.Join(dir, "go.mod"))
	if err != nil {
		return err
	}
	if mod != s.Module {
		return fmt.Errorf("build: %s is module %s, not %s; refused", s.Repository, mod, s.Module)
	}
	target, err := submoduleTarget(ctx, appDir, dir, s.ID, s.Repository)
	if err != nil {
		return err
	}
	if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
		return fmt.Errorf("build: %w", err)
	}
	if err := os.Rename(dir, target); err != nil {
		return fmt.Errorf("build: move checkout to %s: %w", target, err)
	}
	// From here on a failure must take the checkout away again.
	plan.snap.addedCheckout(target)
	relTarget, err := filepath.Rel(appDir, target)
	if err != nil {
		return fmt.Errorf("build: %w", err)
	}
	relTarget = filepath.ToSlash(relTarget)
	if _, err := runCLIGit(ctx, appDir, interactive, "submodule", "add", "--quiet", "--", s.Repository, relTarget); err != nil {
		return fmt.Errorf("build: %w", err)
	}
	if _, err := runCLIGit(ctx, appDir, interactive, "submodule", "absorbgitdirs", "--", relTarget); err != nil {
		return fmt.Errorf("build: %w", err)
	}
	version := s.Version
	if version == "" {
		version = "v0.0.0"
	}
	if err := editAppModule(ctx, appDir, target, s.Module, version); err != nil {
		return err
	}
	m, err := LoadManifest(filepath.Join(appDir, manifestName))
	if err != nil {
		return err
	}
	if err := ensureGoWork(ctx, appDir, m, target); err != nil {
		return err
	}
	fmt.Fprintf(out, "Added submodule %s (%s)\n", relTarget, displayRef(s.Version))
	return nil
}

func displayRef(version string) string {
	if version == "" {
		return "default branch head"
	}
	return version
}

// refreshWorkOverrides adds the go.work replace overrides a workspace of
// submodule plugins needs (research F2). A plugin repository's go.mod
// often replaces the framework or sibling plugins with paths of its own
// development layout; in the application's workspace those paths do not
// exist, and different modules replacing one path differently is an
// error. For every local (directory) replace in the go.mod of a module go.work uses
// (other than the app), whose path go.work does not replace yet, it adds:
// the app's own replace of that path when the app's go.mod has one; the
// workspace module's directory, at the version required, when the path is
// a workspace module (a whole-path replace of a workspace module is an
// error); otherwise the path at the version the app's go.mod (or the
// plugin's) requires. A pin of the last kind that no longer matches the
// app's require is moved to it.
func refreshWorkOverrides(ctx context.Context, appDir string, out io.Writer) error {
	gw := findGoWork(appDir)
	if gw == "" {
		return nil
	}
	workDir := filepath.Dir(gw)
	work, err := editJSON(ctx, workDir, "work", "edit", "-json", gw)
	if err != nil {
		return fmt.Errorf("build: %w", err)
	}
	app, err := editJSON(ctx, appDir, "mod", "edit", "-json", filepath.Join(appDir, "go.mod"))
	if err != nil {
		return fmt.Errorf("build: %w", err)
	}
	appRequire := map[string]string{}
	for _, r := range app.Require {
		appRequire[r.Path] = r.Version
	}
	replaced := map[string]bool{}
	var edits []string
	for _, r := range work.Replace {
		replaced[r.Old.Path] = true
		// A pin of a path to one of its own versions follows the app.
		if r.New.Path == r.Old.Path && r.New.Version != "" {
			if want := appRequire[r.Old.Path]; want != "" && want != r.New.Version {
				edits = append(edits, "-replace="+r.Old.Path+"="+r.Old.Path+"@"+want)
			}
		}
	}
	workspace := map[string]string{} // module path -> directory
	var useDirs []string
	for _, u := range work.Use {
		dir := u.DiskPath
		if !filepath.IsAbs(dir) {
			dir = filepath.Join(workDir, dir)
		}
		dir = filepath.Clean(dir)
		mod, err := readModulePath(filepath.Join(dir, "go.mod"))
		if err != nil {
			continue
		}
		workspace[mod] = dir
		if !samePath(dir, appDir) {
			useDirs = append(useDirs, dir)
		}
	}
	sort.Strings(useDirs)
	for _, dir := range useDirs {
		mod, err := editJSON(ctx, dir, "mod", "edit", "-json", filepath.Join(dir, "go.mod"))
		if err != nil {
			return fmt.Errorf("build: %w", err)
		}
		pluginRequire := map[string]string{}
		for _, r := range mod.Require {
			pluginRequire[r.Path] = r.Version
		}
		for _, r := range mod.Replace {
			p := r.Old.Path
			if r.New.Version != "" || replaced[p] {
				continue
			}
			version := appRequire[p]
			if version == "" {
				version = pluginRequire[p]
			}
			var edit string
			switch {
			case workspace[p] != "":
				if version == "" {
					fmt.Fprintf(out, "warning: cannot pin %s in go.work: no module requires a version of it\n", p)
					continue
				}
				target, err := relModulePath(workDir, workspace[p])
				if err != nil {
					return fmt.Errorf("build: %w", err)
				}
				edit = "-replace=" + p + "@" + version + "=" + target
			case appReplaceTarget(app, p, appDir) != "":
				target := appReplaceTarget(app, p, appDir)
				if !strings.Contains(target, "@") {
					if rel, err := relModulePath(workDir, target); err == nil {
						target = rel
					}
				}
				edit = "-replace=" + p + "=" + target
			default:
				if version == "" {
					fmt.Fprintf(out, "warning: cannot pin %s in go.work: no module requires a version of it\n", p)
					continue
				}
				edit = "-replace=" + p + "=" + p + "@" + version
			}
			replaced[p] = true
			edits = append(edits, edit)
		}
	}
	if len(edits) == 0 {
		return nil
	}
	cmd := exec.CommandContext(ctx, "go", append(append([]string{"work", "edit"}, edits...), gw)...)
	cmd.Dir = workDir
	applyGoWorkFile(cmd, gw)
	if b, err := cmd.CombinedOutput(); err != nil {
		return fmt.Errorf("build: go work edit: %w\n%s", err, b)
	}
	for _, e := range edits {
		fmt.Fprintf(out, "go.work: replace %s\n", strings.TrimPrefix(e, "-replace="))
	}
	return ensureToolchain(gw)
}

// appReplaceTarget returns what the app's go.mod replaces path with: an
// absolute directory, or path@version; "" when it does not replace it.
func appReplaceTarget(app goModJSON, p, appDir string) string {
	for _, r := range app.Replace {
		if r.Old.Path != p {
			continue
		}
		if r.New.Version != "" {
			return r.New.Path + "@" + r.New.Version
		}
		if filepath.IsAbs(r.New.Path) {
			return r.New.Path
		}
		return filepath.Join(appDir, r.New.Path)
	}
	return ""
}