package build import ( "bytes" "context" "errors" "fmt" "io" "net/url" "os" "os/exec" "path" "path/filepath" "regexp" "slices" "sort" "strings" "git.golem15.com/golem15/summercms/modules/bonfire" "git.golem15.com/golem15/summercms/modules/kiosk" ) // cliGitProtocols is GIT_ALLOW_PROTOCOL for the git commands of // plugin:install and plugin:update, and the URL schemes they accept. // Tests add "file" for fixture repositories; nothing else changes it. var cliGitProtocols = "https:ssh" var ( scpLikeRe = regexp.MustCompile(`^[A-Za-z0-9._-]+@[A-Za-z0-9.-]+:`) repoNameRe = regexp.MustCompile(`^[a-z][a-z0-9]*$`) installTemp = ".summer-install-" ) // allowedGitProtocol reports whether cliGitProtocols lists name. func allowedGitProtocol(name string) bool { return slices.Contains(strings.Split(cliGitProtocols, ":"), name) } // runCLIGit runs git in dir for the CLI. It keeps the user's git config // (credential helpers, insteadOf), allows only cliGitProtocols, and stops // git from prompting when no person can answer. Callers put "--" before // URLs and paths. func runCLIGit(ctx context.Context, dir string, interactive bool, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, "git", args...) cmd.Dir = dir env := make([]string, 0, len(os.Environ())+2) for _, e := range os.Environ() { if strings.HasPrefix(e, "GIT_ALLOW_PROTOCOL=") || strings.HasPrefix(e, "GIT_TERMINAL_PROMPT=") { continue } env = append(env, e) } env = append(env, "GIT_ALLOW_PROTOCOL="+cliGitProtocols) if !interactive { env = append(env, "GIT_TERMINAL_PROMPT=0") } cmd.Env = env var stderr bytes.Buffer cmd.Stderr = &stderr out, err := cmd.Output() if err != nil { msg := strings.TrimSpace(stderr.String()) if msg == "" { msg = err.Error() } return out, fmt.Errorf("git %s: %s", args[0], msg) } return out, nil } // isGitURL reports whether arg is meant as a repository address rather // than a plugin ID: a scheme URL, a scp-like user@host:path or a // transport::address form (which validateGitURL then refuses). func isGitURL(arg string) bool { return strings.Contains(arg, "://") || strings.Contains(arg, "::") || scpLikeRe.MatchString(arg) } // validateGitURL accepts https://, ssh:// and scp-like user@host:path // addresses (file:// only when cliGitProtocols allows it, in tests). It // refuses a leading "-", transport::address forms, http, credentials in // the URL and control characters. func validateGitURL(raw string) error { switch { case raw == "": return errors.New("git URL is empty") case strings.HasPrefix(raw, "-"): return fmt.Errorf("git URL %q must not start with -", raw) case strings.ContainsFunc(raw, func(r rune) bool { return r <= ' ' || r == 0x7f }): return fmt.Errorf("git URL %q contains spaces or control characters", raw) case strings.Contains(raw, "::"): return fmt.Errorf("git URL %q uses a transport:: form; use https://, ssh:// or user@host:path", raw) } if !strings.Contains(raw, "://") { if !scpLikeRe.MatchString(raw) { return fmt.Errorf("%q is not a git URL; use https://, ssh:// or user@host:path", raw) } if !allowedGitProtocol("ssh") { return fmt.Errorf("git URL %q: ssh is not allowed", raw) } _, p, _ := strings.Cut(raw, ":") if p == "" || strings.HasPrefix(p, "-") { return fmt.Errorf("git URL %q has an invalid path", raw) } return nil } u, err := url.Parse(raw) if err != nil { return fmt.Errorf("git URL %q: %w", raw, err) } scheme := strings.ToLower(u.Scheme) if !allowedGitProtocol(scheme) { return fmt.Errorf("git URL %q: only https://, ssh:// and user@host:path are accepted", raw) } if scheme != "file" && u.Host == "" { return fmt.Errorf("git URL %q has no host", raw) } if _, hasPassword := u.User.Password(); hasPassword { return fmt.Errorf("git URL must not carry a password; use a git credential helper") } if strings.HasPrefix(u.Path, "/-") || u.Path == "" || u.Path == "/" { return fmt.Errorf("git URL %q has an invalid path", raw) } return nil } // validateRepositoryURL checks a repository URL the registry sent: it must // be https without user info (file:// only in tests). func validateRepositoryURL(id, raw string) error { if raw == "" { return fmt.Errorf("build: registry entry %s has no repository URL; refused", id) } if err := validateGitURL(raw); err != nil { return fmt.Errorf("build: registry entry %s: %w; refused", id, err) } u, err := url.Parse(raw) if err != nil || !strings.Contains(raw, "://") { return fmt.Errorf("build: registry entry %s: repository %q is not an https URL; refused", id, raw) } if s := strings.ToLower(u.Scheme); s != "https" && !(s == "file" && allowedGitProtocol("file")) { return fmt.Errorf("build: registry entry %s: repository %q is not an https URL; refused", id, raw) } if u.User != nil { return fmt.Errorf("build: registry entry %s: repository URL carries user info; refused", id) } return nil } // requireGitRepo refuses submodule installs in an application that is not // a git repository. func requireGitRepo(ctx context.Context, appDir string) error { if _, err := gitOutput(ctx, appDir, "rev-parse", "--show-toplevel"); err != nil { return bonfire.WithHint(fmt.Errorf("build: %s is not a git repository; a plugin installed from source is a git submodule, which needs one", appDir), "--source and git URL installs add the plugin as a git submodule of the application. Initialise a repository first, or install the plugin by ID as a Go module.", "git init && git add -A && git commit -m init") } return nil } // cloneInto clones url into a new temporary directory under // /plugins, so the later move to plugins// stays on one // file system. func cloneInto(ctx context.Context, appDir, rawURL string, interactive bool) (string, error) { parent := filepath.Join(appDir, "plugins") if err := os.MkdirAll(parent, 0o755); err != nil { return "", fmt.Errorf("build: create %s: %w", parent, err) } tmp, err := os.MkdirTemp(parent, installTemp) if err != nil { return "", fmt.Errorf("build: %w", err) } if _, err := runCLIGit(ctx, appDir, interactive, "clone", "--quiet", "--", rawURL, tmp); err != nil { _ = os.RemoveAll(tmp) return "", fmt.Errorf("build: clone %s: %w", rawURL, err) } return tmp, nil } // releaseTags returns the vX.Y.Z tags of the repository in dir, newest // first. func releaseTags(ctx context.Context, dir string) ([]string, error) { out, err := runCLIGit(ctx, dir, false, "tag", "--list", "v*") if err != nil { return nil, fmt.Errorf("build: %w", err) } var tags []string for _, t := range strings.Fields(string(out)) { if kiosk.ValidVersion(t) { tags = append(tags, t) } } sort.Slice(tags, func(i, j int) bool { return kiosk.CompareVersions(tags[i], tags[j]) > 0 }) return tags, nil } // manifestAt reads /plugin.yaml at a git ref; nil when the file does // not exist there. func manifestAt(ctx context.Context, dir, ref, rel string) (*kiosk.Manifest, error) { p := kiosk.FileName if rel != "" { p = path.Join(rel, kiosk.FileName) } out, err := runCLIGit(ctx, dir, false, "show", ref+":"+p) if err != nil { return nil, nil } m, err := kiosk.Parse(out) if err != nil { return nil, fmt.Errorf("build: %s at %s: %w", p, ref, err) } return &m, nil } // checkout moves the work tree in dir to ref (a release tag). func checkout(ctx context.Context, dir, ref string) error { if _, err := runCLIGit(ctx, dir, false, "checkout", "--quiet", "--detach", ref); err != nil { return fmt.Errorf("build: check out %s: %w", ref, err) } return nil } // planGitURL clones the repository at rawURL, picks the newest release // tag whose plugin.yaml framework minimum the application satisfies // (falling back to the default branch head with a warning) and adds the // plugin and its requirements to the plan. The plugin itself is never // looked up in the registry. func (in *installer) planGitURL(rawURL string) error { if err := validateGitURL(rawURL); err != nil { return fmt.Errorf("build: %w", err) } if err := requireGitRepo(in.ctx, in.appDir); err != nil { return err } if dir := submoduleWithURL(in.ctx, in.appDir, rawURL); dir != "" { return bonfire.WithHint(fmt.Errorf("build: %s is already checked out at %s", rawURL, dir), "plugin:install does not clone a repository the application already has as a submodule. Update the plugins in it with plugin:update.", "{bin} plugin:update") } fmt.Fprintf(in.out, "Cloning %s\n", rawURL) tmp, err := cloneInto(in.ctx, in.appDir, rawURL, in.opts.Interactive) if err != nil { return err } in.plan.temps = append(in.plan.temps, tmp) tags, err := releaseTags(in.ctx, tmp) if err != nil { return err } ref, version := "", "" var pm *kiosk.Manifest newest := "" for _, tag := range tags { m, err := manifestAt(in.ctx, tmp, tag, "") if err != nil { fmt.Fprintf(in.out, "warning: %v; tag skipped\n", err) continue } if m == nil { continue } if newest == "" { newest = tag + " needs framework " + m.Framework } if kiosk.Satisfies(in.fw, m.Framework) { ref, version, pm = tag, tag, m break } } if ref == "" { why := "no release tag carries a plugin.yaml" if newest != "" { why = "no release tag fits framework " + displayVersion(in.fw) + " (" + newest + ")" } fmt.Fprintf(in.out, "warning: %s; installing the default branch head of %s\n", why, rawURL) } else if err := checkout(in.ctx, tmp, ref); err != nil { return err } step := InstallStep{Version: version, Repository: rawURL, Mode: ModeSubmodule, clone: tmp} var requires kiosk.Requires if pm == nil { pm, err = readCheckoutManifest(tmp, "") if err != nil { return err } } if pm != nil { step.ID, step.Module, step.Package, requires = pm.ID, pm.Module, pm.Package, pm.Requires } else { // No plugin.yaml at all: read the identity from the sources. id, err := kiosk.ReadIdentity(in.ctx, tmp) if id.ID == "" { return fmt.Errorf("build: %s has no plugin.yaml and no readable plugin.go at its root: %v", rawURL, err) } mod, err := readModulePath(filepath.Join(tmp, "go.mod")) if err != nil { return err } step.ID, step.Module, step.Package = id.ID, mod, mod requires = kiosk.Requires{} for _, r := range id.Requires { requires[r] = "" } } if err := ValidatePluginID(step.ID); err != nil { return fmt.Errorf("build: %s: %w", rawURL, err) } if err := in.requireAbsent(step.ID); err != nil { return err } if err := validateStep(step); err != nil { return err } if im, ok := in.installed[step.Module]; ok { return bonfire.WithHint(fmt.Errorf("build: %s is module %s, which the application already has (for %s)", rawURL, step.Module, im.by), "plugin:install does not move plugins the application already has, and a second checkout of one module breaks the workspace. Update the module with plugin:update.", "{bin} plugin:update "+im.by) } if _, err := submoduleTarget(in.ctx, in.appDir, tmp, step.ID, rawURL); err != nil { return err } in.moduleVersion[step.Module] = step.Version in.visiting = append(in.visiting, step.ID) if err := in.resolveRequires(step.ID, step.Version, requires); err != nil { return err } in.visiting = in.visiting[:len(in.visiting)-1] in.planned[step.ID] = step in.plan.Steps = append(in.plan.Steps, step) return nil } // submoduleWithURL returns the path of the application's submodule whose // .gitmodules URL is rawURL (ignoring a trailing slash or .git), or "". func submoduleWithURL(ctx context.Context, appDir, rawURL string) string { out, err := gitOutput(ctx, appDir, "config", "--file", ".gitmodules", "--get-regexp", `^submodule\..*\.url$`) if err != nil { return "" } norm := func(u string) string { return strings.TrimSuffix(strings.TrimRight(u, "/"), ".git") } want := norm(rawURL) for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { key, val, ok := strings.Cut(line, " ") if !ok || norm(strings.TrimSpace(val)) != want { continue } name := strings.TrimSuffix(strings.TrimPrefix(key, "submodule."), ".url") if p, err := gitOutput(ctx, appDir, "config", "--file", ".gitmodules", "submodule."+name+".path"); err == nil { return strings.TrimSpace(string(p)) } return name } return "" } // readCheckoutManifest reads //plugin.yaml from a work tree; nil // when it does not exist. func readCheckoutManifest(dir, rel string) (*kiosk.Manifest, error) { p := filepath.Join(dir, filepath.FromSlash(rel), kiosk.FileName) raw, err := os.ReadFile(p) if errors.Is(err, os.ErrNotExist) { return nil, nil } if err != nil { return nil, fmt.Errorf("build: read %s: %w", p, err) } m, err := kiosk.Parse(raw) if err != nil { return nil, fmt.Errorf("build: %s: %w", p, err) } return &m, nil } // submoduleTarget returns plugins// for a checkout: the ID // in the repository root's plugin.yaml (or plugin.go), otherwise the // repository name without sm- and -plugin under the vendor of stepID. func submoduleTarget(ctx context.Context, appDir, cloneDir, stepID, repoURL string) (string, error) { id := "" if pm, err := readCheckoutManifest(cloneDir, ""); err == nil && pm != nil { id = pm.ID } else if ident, _ := kiosk.ReadIdentity(ctx, cloneDir); ident.ID != "" { id = ident.ID } var dir string if id != "" { if err := ValidatePluginID(id); err != nil { return "", fmt.Errorf("build: %s: %w", repoURL, err) } dir = filepath.Join(appDir, "plugins", pluginVendor(id), pluginName(id)) } else { name := strings.TrimSuffix(path.Base(strings.TrimRight(repoURL, "/")), ".git") if i := strings.LastIndex(name, ":"); i >= 0 { name = name[i+1:] } name = strings.TrimSuffix(strings.TrimPrefix(name, "sm-"), "-plugin") if !repoNameRe.MatchString(name) { return "", fmt.Errorf("build: cannot name a plugin directory after repository %s", repoURL) } dir = filepath.Join(appDir, "plugins", pluginVendor(stepID), name) } if !underRoot(appDir, dir) { return "", fmt.Errorf("build: plugin path %q escapes the app root", dir) } if _, err := os.Stat(dir); err == nil { return "", fmt.Errorf("build: %s already exists; remove it or register it with plugin:add", dir) } return dir, nil } // applySubmodule installs one submodule step: clone and check out the tag // (a git URL step was cloned while planning), check that the plugin.yaml // in the checkout describes the planned plugin, move the checkout to // plugins//, add it as a submodule and register it in go.mod // and go.work with the tag as the require version. func applySubmodule(ctx context.Context, plan *InstallPlan, s *InstallStep, interactive bool, out io.Writer) error { appDir := plan.AppDir dir := s.clone if dir == "" { fmt.Fprintf(out, "Cloning %s at %s\n", s.Repository, s.Version) tmp, err := cloneInto(ctx, appDir, s.Repository, interactive) if err != nil { return err } plan.temps = append(plan.temps, tmp) if err := checkout(ctx, tmp, s.Version); err != nil { return err } dir = tmp } rel := strings.TrimPrefix(strings.TrimPrefix(s.Package, s.Module), "/") pm, err := readCheckoutManifest(dir, rel) if err != nil { return err } if pm != nil && (pm.ID != s.ID || pm.Module != s.Module) { return fmt.Errorf("build: %s at %s describes %s (%s), not %s (%s); refused", s.Repository, displayRef(s.Version), pm.ID, pm.Module, s.ID, s.Module) } if pm == nil && s.clone == "" { return fmt.Errorf("build: %s at %s has no %s for %s; refused", s.Repository, s.Version, path.Join(rel, kiosk.FileName), s.ID) } mod, err := readModulePath(filepath.Join(dir, "go.mod")) if err != nil { return err } if mod != s.Module { return fmt.Errorf("build: %s is module %s, not %s; refused", s.Repository, mod, s.Module) } target, err := submoduleTarget(ctx, appDir, dir, s.ID, s.Repository) if err != nil { return err } if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { return fmt.Errorf("build: %w", err) } if err := os.Rename(dir, target); err != nil { return fmt.Errorf("build: move checkout to %s: %w", target, err) } relTarget, err := filepath.Rel(appDir, target) if err != nil { return fmt.Errorf("build: %w", err) } relTarget = filepath.ToSlash(relTarget) if _, err := runCLIGit(ctx, appDir, interactive, "submodule", "add", "--quiet", "--", s.Repository, relTarget); err != nil { return fmt.Errorf("build: %w", err) } if _, err := runCLIGit(ctx, appDir, interactive, "submodule", "absorbgitdirs", "--", relTarget); err != nil { return fmt.Errorf("build: %w", err) } version := s.Version if version == "" { version = "v0.0.0" } if err := editAppModule(ctx, appDir, target, s.Module, version); err != nil { return err } m, err := LoadManifest(filepath.Join(appDir, manifestName)) if err != nil { return err } if err := ensureGoWork(ctx, appDir, m, target); err != nil { return err } fmt.Fprintf(out, "Added submodule %s (%s)\n", relTarget, displayRef(s.Version)) return nil } func displayRef(version string) string { if version == "" { return "default branch head" } return version } // refreshWorkOverrides adds the go.work replace overrides a workspace of // submodule plugins needs (research F2). A plugin repository's go.mod // often replaces the framework or sibling plugins with paths of its own // development layout; in the application's workspace those paths do not // exist, and different modules replacing one path differently is an // error. For every local (directory) replace in the go.mod of a module go.work uses // (other than the app), whose path go.work does not replace yet, it adds: // the app's own replace of that path when the app's go.mod has one; the // workspace module's directory, at the version required, when the path is // a workspace module (a whole-path replace of a workspace module is an // error); otherwise the path at the version the app's go.mod (or the // plugin's) requires. A pin of the last kind that no longer matches the // app's require is moved to it. func refreshWorkOverrides(ctx context.Context, appDir string, out io.Writer) error { gw := findGoWork(appDir) if gw == "" { return nil } workDir := filepath.Dir(gw) work, err := editJSON(ctx, workDir, "work", "edit", "-json", gw) if err != nil { return fmt.Errorf("build: %w", err) } app, err := editJSON(ctx, appDir, "mod", "edit", "-json", filepath.Join(appDir, "go.mod")) if err != nil { return fmt.Errorf("build: %w", err) } appRequire := map[string]string{} for _, r := range app.Require { appRequire[r.Path] = r.Version } replaced := map[string]bool{} var edits []string for _, r := range work.Replace { replaced[r.Old.Path] = true // A pin of a path to one of its own versions follows the app. if r.New.Path == r.Old.Path && r.New.Version != "" { if want := appRequire[r.Old.Path]; want != "" && want != r.New.Version { edits = append(edits, "-replace="+r.Old.Path+"="+r.Old.Path+"@"+want) } } } workspace := map[string]string{} // module path -> directory var useDirs []string for _, u := range work.Use { dir := u.DiskPath if !filepath.IsAbs(dir) { dir = filepath.Join(workDir, dir) } dir = filepath.Clean(dir) mod, err := readModulePath(filepath.Join(dir, "go.mod")) if err != nil { continue } workspace[mod] = dir if !samePath(dir, appDir) { useDirs = append(useDirs, dir) } } sort.Strings(useDirs) for _, dir := range useDirs { mod, err := editJSON(ctx, dir, "mod", "edit", "-json", filepath.Join(dir, "go.mod")) if err != nil { return fmt.Errorf("build: %w", err) } pluginRequire := map[string]string{} for _, r := range mod.Require { pluginRequire[r.Path] = r.Version } for _, r := range mod.Replace { p := r.Old.Path if r.New.Version != "" || replaced[p] { continue } version := appRequire[p] if version == "" { version = pluginRequire[p] } var edit string switch { case workspace[p] != "": if version == "" { fmt.Fprintf(out, "warning: cannot pin %s in go.work: no module requires a version of it\n", p) continue } target, err := relModulePath(workDir, workspace[p]) if err != nil { return fmt.Errorf("build: %w", err) } edit = "-replace=" + p + "@" + version + "=" + target case appReplaceTarget(app, p, appDir) != "": target := appReplaceTarget(app, p, appDir) if !strings.Contains(target, "@") { if rel, err := relModulePath(workDir, target); err == nil { target = rel } } edit = "-replace=" + p + "=" + target default: if version == "" { fmt.Fprintf(out, "warning: cannot pin %s in go.work: no module requires a version of it\n", p) continue } edit = "-replace=" + p + "=" + p + "@" + version } replaced[p] = true edits = append(edits, edit) } } if len(edits) == 0 { return nil } cmd := exec.CommandContext(ctx, "go", append(append([]string{"work", "edit"}, edits...), gw)...) cmd.Dir = workDir applyGoWorkFile(cmd, gw) if b, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("build: go work edit: %w\n%s", err, b) } for _, e := range edits { fmt.Fprintf(out, "go.work: replace %s\n", strings.TrimPrefix(e, "-replace=")) } return ensureToolchain(gw) } // appReplaceTarget returns what the app's go.mod replaces path with: an // absolute directory, or path@version; "" when it does not replace it. func appReplaceTarget(app goModJSON, p, appDir string) string { for _, r := range app.Replace { if r.Old.Path != p { continue } if r.New.Version != "" { return r.New.Path + "@" + r.New.Version } if filepath.IsAbs(r.New.Path) { return r.New.Path } return filepath.Join(appDir, r.New.Path) } return "" }