diff --git a/cmd/summer/main.go b/cmd/summer/main.go index 8d392dce..c020b736 100644 --- a/cmd/summer/main.go +++ b/cmd/summer/main.go @@ -549,6 +549,18 @@ names the framework version to move to; it never changes the framework. The plugin is installed as a Go module with go get and listed in summer.yaml. --source installs it as a git submodule under plugins// instead. +Required plugins are installed with it, in the same mode, each at the newest +compatible release that meets its minimum; one the application already has +must be at least that minimum. Submodule installs require the tag in go.mod, +add the plugin to go.work and add the go.work replaces its own go.mod needs. +--source and git URLs need the application to be a git repository. + +A git URL is cloned with your own git credentials and never sent to the +registry. The newest release tag whose plugin.yaml framework minimum fits is +checked out (the default branch head, with a warning, when none fits), and +the plugin is added as a submodule. Its required plugins are looked up in the +registry; with --offline they must be installed already. + The plugins to add are printed as id@version (mode) and confirmed with a prompt. --yes skips the prompt; a run without a terminal needs --yes. Run summer build afterwards. --offline makes no registry call, so it works only diff --git a/docs/architecture/go-modules-and-workspaces.md b/docs/architecture/go-modules-and-workspaces.md index 2aeb1dde..8045cdb1 100644 --- a/docs/architecture/go-modules-and-workspaces.md +++ b/docs/architecture/go-modules-and-workspaces.md @@ -93,7 +93,7 @@ build: The generated `main` publishes both with `party.Describe` before activating the plugins, so the admin can show the installed plugins with `app.Lookup[party.Installed]()`. When the metadata cannot be resolved, `summer build` prints a warning and bakes the IDs only; it never fails a build. A submodule bump or a new framework tag therefore changes `plugins.gen.go`: commit the regenerated file with the bump. -`build.tags` lists Go build tags that `summer build` (and `summer dev`) passes to `go build` as `-tags=,`, printing them as `build tags: …`. Each tag must consist of lower-case letters, digits and underscores; a duplicate is ignored. Framework features that pull in large dependencies sit behind a tag, such as `summercms_s3` for S3 uploads buckets (see [Storage](../services/storage.md#s3-and-s3-compatible-buckets)). `summer plugin:add` keeps the list when it rewrites the manifest. A script that runs its own `go build` must pass the same tags. +`build.tags` lists Go build tags that `summer build` (and `summer dev`) passes to `go build` as `-tags=,`, printing them as `build tags: …`. Each tag must consist of lower-case letters, digits and underscores; a duplicate is ignored. Framework features that pull in large dependencies sit behind a tag, such as `summercms_s3` for S3 uploads buckets (see [Storage](../services/storage.md#s3-and-s3-compatible-buckets)). `summer plugin:add` and `summer plugin:install` append their entries to the `plugins:` list in place, so the rest of the file, comments and blank lines included, stays as it was. A script that runs its own `go build` must pass the same tags. ## Local plugins in a workspace @@ -111,6 +111,18 @@ summer plugin:add plugins/acme/blog summer build ``` +## Plugins installed as submodules + +`summer plugin:install --source` and `summer plugin:install ` check a plugin's repository out at a release tag as a git submodule in `plugins//` and register it like `summer plugin:add`, with one difference: the `require` in `go.mod` names the tag (`v0.3.1`), not `v0.0.0`, so a build with `GOWORK=off` and the version checks of later installs see the real release. See [Installing plugins](../plugins/marketplace.md#installing-plugins). + +A shared plugin's own `go.mod` usually replaces the framework, and sometimes sibling plugins, with paths of its development layout, such as `replace git.golem15.com/golem15/summercms => ../summercms.go`. Inside the application's workspace those paths do not exist, and two workspace modules that replace one module with different paths make Go stop with "conflicting replacements". A `replace` in `go.work` wins over the replaces in the workspace modules' `go.mod` files, so the installer adds one for every directory replace it finds in a workspace module's `go.mod` that `go.work` does not replace yet: + +- when the application's `go.mod` replaces the same module (a local framework checkout, for example), `go.work` gets the same target; +- when the module is itself in the workspace (another plugin installed as a submodule), `go.work` replaces that module at its required version with its directory, such as `replace git.example.com/acme/translate v0.1.0 => ./plugins/acme/translate` (Go refuses a replace of a workspace module at all versions); +- otherwise `go.work` pins the module to the version the application requires, such as `replace git.golem15.com/golem15/summercms => git.golem15.com/golem15/summercms v0.6.0`. + +A pin of the last kind follows the application: when the framework requirement in `go.mod` moves, the next submodule install moves the pin to the new version. Replaces you wrote into `go.work` yourself are left alone. + ## Replacing and forking a plugin WinterCMS lets you replace a plugin by overriding its classes. In SummerCMS you fork the plugin's module and point the application at your fork with a `replace` directive. The plugin ID and the import path stay the same, so nothing else in the application changes: diff --git a/docs/plugins/marketplace.md b/docs/plugins/marketplace.md index 689c5ef6..f0dde829 100644 --- a/docs/plugins/marketplace.md +++ b/docs/plugins/marketplace.md @@ -114,7 +114,7 @@ A required plugin the application already has is not touched, but its version mu ### Confirmation -Before changing anything, the command prints each plugin it is about to add, one line per plugin: +Before changing anything, the command prints each plugin it is about to add, one line per plugin, with `module` or `submodule` as the mode: ```text Installing 2 plugins into /srv/acme (framework v0.6.2): @@ -124,6 +124,34 @@ Installing 2 plugins into /srv/acme (framework v0.6.2): Then it asks for confirmation, with "no" as the default. `--yes` skips the question. A run that cannot ask (no terminal, `-n` or `-q`) stops with an error naming `--yes` unless it is given. Declining changes nothing. +### Installing from source + +`--source` installs the plugin, and every required plugin it brings, as git submodules instead of Go modules: + +```sh +summer plugin:install acme.blog --source +``` + +Each repository (the `repository` the registry lists, which must be an `https` URL) is cloned, checked out at the chosen release tag and added as a submodule in `plugins//`, named after the ID in the repository root's `plugin.yaml`. A plugin in a subdirectory of its repository lives inside that directory. The `plugin.yaml` in the checkout must describe the plugin the registry listed (same ID and module), or the install is refused. The plugin is then registered like `summer plugin:add` does it, with the tag as the version `go.mod` requires, and `go.work` gets the replace overrides the plugin's own `go.mod` needs to build inside the application (see [Plugins installed as submodules](../architecture/go-modules-and-workspaces.md#plugins-installed-as-submodules)). Commit `.gitmodules`, the submodule, `go.mod`, `go.work` and `summer.yaml` together. + +A submodule needs a git repository: in an application that is not one, `--source` stops before cloning anything. + +### Installing from a git URL + +A plugin that is not listed, such as a private one, installs from its repository: + +```sh +summer plugin:install https://git.example.com/acme/sm-blog-plugin.git +summer plugin:install git@git.example.com:acme/sm-blog-plugin.git +summer plugin:install ssh://git@git.example.com/acme/sm-blog-plugin.git +``` + +The repository is cloned with your own git setup (credential helpers, SSH keys, `insteadOf` rules) and never sent to the registry. The command reads `plugin.yaml` at the repository's release tags, newest first, and checks out the newest tag whose framework minimum the application meets. When no tag fits, it takes the default branch head and prints a warning. A git URL install is always a submodule install, with the same layout and `go.work` overrides as `--source`. + +The plugin's required plugins are looked up in the registry by ID and installed as submodules too. With `--offline` no registry call is made, so every required plugin must already be in `summer.yaml`; a missing one stops the install and is named. + +Only `https://`, `ssh://` and scp-like `user@host:path` addresses are accepted. Git runs with `GIT_ALLOW_PROTOCOL=https:ssh`, so `ext::` and `file::` transports, `file://` and plain `http://` are refused, as is an address that starts with `-` or carries a password (use a credential helper instead). + ### The registry setting The registry is `https://summercms.io` unless the application's `plugins.registry_url` setting names another one: diff --git a/internal/build/fixtures_test.go b/internal/build/fixtures_test.go index 71e81e2a..4e58b21e 100644 --- a/internal/build/fixtures_test.go +++ b/internal/build/fixtures_test.go @@ -264,3 +264,135 @@ func helloCopy(t *testing.T) string { t.Helper() return copyHelloApp(t) } + +// fixtureRelease is one tagged commit of a fixture plugin repository. +type fixtureRelease struct { + Tag string + Framework string + // Requires maps required plugin IDs to {module, minimum}. + Requires map[string][2]string +} + +// pluginRepoFiles is a plugin repository at one release: go.mod in the +// development layout of the shared plugin repos (the framework and every +// required plugin replaced with sibling paths that do not exist in an +// application), plugin.go and the generated plugin.yaml. +func pluginRepoFiles(t *testing.T, id, module string, rel fixtureRelease) map[string]string { + t.Helper() + var gomod strings.Builder + gomod.WriteString("module " + module + "\n\ngo 1.27.0\n\nrequire (\n\t" + frameworkModule + " " + rel.Framework + "\n") + reqIDs := make([]string, 0, len(rel.Requires)) + for rid := range rel.Requires { + reqIDs = append(reqIDs, rid) + } + sort.Strings(reqIDs) + for _, rid := range reqIDs { + gomod.WriteString("\t" + rel.Requires[rid][0] + " " + rel.Requires[rid][1] + "\n") + } + gomod.WriteString(")\n\nreplace " + frameworkModule + " => ../summercms.go\n") + var imports, reqList strings.Builder + requires := kiosk.Requires{} + for _, rid := range reqIDs { + gomod.WriteString("\nreplace " + rel.Requires[rid][0] + " => ../sm-" + pluginName(rid) + "-plugin\n") + imports.WriteString("\t_ " + fmt.Sprintf("%q", rel.Requires[rid][0]) + "\n") + reqList.WriteString(fmt.Sprintf("%q, ", rid)) + requires[rid] = rel.Requires[rid][1] + } + pkg := pluginName(id) + src := fmt.Sprintf(`package %s + +import ( +%s + "git.golem15.com/golem15/summercms/modules/backpack" + "git.golem15.com/golem15/summercms/modules/party" +) + +// Plugin is the %s fixture plugin. +type Plugin struct{} + +func (p *Plugin) ID() string { return %q } +func (p *Plugin) Requires() []string { return []string{%s} } +func (p *Plugin) Register(*backpack.App) error { return nil } +func (p *Plugin) Boot(*backpack.App) error { return nil } + +func init() { party.Register(&Plugin{}) } +`, pkg, imports.String(), id, id, reqList.String()) + m := kiosk.Manifest{ + Schema: kiosk.SchemaVersion, + ID: id, + Module: module, + Package: module, + Name: "Fixture " + pkg, + Description: "A fixture plugin for install tests.", + Framework: rel.Framework, + } + if len(requires) > 0 { + m.Requires = requires + } + yml, err := kiosk.Marshal(m) + if err != nil { + t.Fatal(err) + } + return map[string]string{"go.mod": gomod.String(), "plugin.go": src, kiosk.FileName: string(yml)} +} + +// pluginRepo creates a git repository with one tagged commit per release +// and returns its file:// URL. head, when set, adds an untagged commit on +// top with that framework minimum. +func pluginRepo(t *testing.T, id, module string, releases []fixtureRelease, head *fixtureRelease) string { + t.Helper() + dir := t.TempDir() + runTestGit(t, dir, "init", "--quiet") + commit := func(rel fixtureRelease, tag bool) { + for name, body := range pluginRepoFiles(t, id, module, rel) { + write(t, filepath.Join(dir, name), body) + } + write(t, filepath.Join(dir, "CHANGES"), "release "+rel.Tag+"\n") + runTestGit(t, dir, "add", "-A") + runTestGit(t, dir, "commit", "--quiet", "-m", "release "+rel.Tag) + if tag { + runTestGit(t, dir, "tag", rel.Tag) + } + } + for _, rel := range releases { + commit(rel, true) + } + if head != nil { + commit(*head, false) + } + return "file://" + filepath.ToSlash(dir) +} + +// gitTestEnv isolates git from the developer's config, lets the CLI git +// runner use file:// fixture repositories and keeps the go commands off +// the network. +func gitTestEnv(t *testing.T) { + t.Helper() + isolateGit(t) + old := cliGitProtocols + cliGitProtocols = "https:ssh:file" + t.Cleanup(func() { cliGitProtocols = old }) + t.Setenv("GOPROXY", "off") + t.Setenv("GOFLAGS", "") + t.Setenv("GOTOOLCHAIN", "local") +} + +// gitApp is a helloCopy committed to a fresh git repository. +func gitApp(t *testing.T) string { + t.Helper() + app := helloCopy(t) + runTestGit(t, app, "init", "--quiet") + runTestGit(t, app, "add", "-A") + runTestGit(t, app, "commit", "--quiet", "-m", "app") + return app +} + +// installTemps lists leftover temporary checkouts under plugins/. +func installTemps(t *testing.T, app string) []string { + t.Helper() + matches, err := filepath.Glob(filepath.Join(app, "plugins", installTemp+"*")) + if err != nil { + t.Fatal(err) + } + return matches +} diff --git a/internal/build/install.go b/internal/build/install.go index 37c40a45..cbe8afe9 100644 --- a/internal/build/install.go +++ b/internal/build/install.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "os" "path/filepath" "slices" "sort" @@ -64,6 +65,11 @@ type InstallPlan struct { Framework string // Steps are in dependency order: a plugin's requirements come first. Steps []InstallStep + + // temps are checkouts under plugins/ made while planning or applying; + // whatever is left of them is removed at the end. + temps []string + interactive bool } // InstallStep is one plugin to add. @@ -74,6 +80,9 @@ type InstallStep struct { Package string Repository string Mode string + + // clone is the checkout a git URL step was planned from. + clone string } // installer carries the state of one PlanInstall run. @@ -93,6 +102,9 @@ type installer struct { planned map[string]InstallStep visiting []string moduleVersion map[string]string + // source installs registry plugins as submodules: --source, or the + // requirements of a git URL install. + source bool } // PlanInstall resolves what plugin:install would add: the requested @@ -129,27 +141,57 @@ func PlanInstall(ctx context.Context, opts InstallOptions) (*InstallPlan, error) in.planned = map[string]InstallStep{} in.moduleVersion = map[string]string{} + in.plan.interactive = opts.Interactive + in.source = opts.Source + ok := false + defer func() { + if !ok { + in.plan.removeTemps() + } + }() - id, pin, err := parseInstallArg(opts.Arg) - if err != nil { - return nil, err - } - if err := in.requireAbsent(id); err != nil { - return nil, err - } - if err := in.requireRegistry(id); err != nil { - return nil, err - } - if err := in.resolve(id, pin, ""); err != nil { - return nil, err + arg := strings.TrimSpace(opts.Arg) + if isGitURL(arg) { + in.source = true + if err := in.planGitURL(arg); err != nil { + return nil, err + } + } else { + id, pin, err := parseInstallArg(arg) + if err != nil { + return nil, err + } + if err := in.requireAbsent(id); err != nil { + return nil, err + } + if err := in.requireRegistry(id); err != nil { + return nil, err + } + if in.source { + if err := requireGitRepo(ctx, appDir); err != nil { + return nil, err + } + } + if err := in.resolve(id, pin, ""); err != nil { + return nil, err + } } if err := in.confirm(); err != nil { return nil, err } + ok = true return in.plan, nil } +// removeTemps deletes the leftover temporary checkouts of the plan. +func (p *InstallPlan) removeTemps() { + for _, dir := range p.temps { + _ = os.RemoveAll(dir) + } + p.temps = nil +} + // parseInstallArg splits vendor.name[@vX.Y.Z]. func parseInstallArg(arg string) (id, pin string, err error) { arg = strings.TrimSpace(arg) @@ -222,17 +264,26 @@ func (in *installer) resolve(id, pin, min string) error { } in.moduleVersion[step.Module] = step.Version in.visiting = append(in.visiting, id) + if err := in.resolveRequires(id, step.Version, requires); err != nil { + return err + } + in.visiting = in.visiting[:len(in.visiting)-1] + in.planned[id] = step + in.plan.Steps = append(in.plan.Steps, step) + return nil +} + +// resolveRequires resolves the requirements of id (at version), in ID +// order. +func (in *installer) resolveRequires(id, version string, requires kiosk.Requires) error { for _, dep := range sortedKeys(requires) { if err := ValidatePluginID(dep); err != nil { - return fmt.Errorf("build: registry entry %s %s requires %w; refused", id, step.Version, err) + return fmt.Errorf("build: %s %s requires %w; refused", id, displayRef(version), err) } if err := in.resolve(dep, "", requires[dep]); err != nil { return err } } - in.visiting = in.visiting[:len(in.visiting)-1] - in.planned[id] = step - in.plan.Steps = append(in.plan.Steps, step) return nil } @@ -416,6 +467,19 @@ func (in *installer) choose(id, pin, min string) (InstallStep, kiosk.Requires, e Package: chosen.Package, Mode: ModeModule, } + if in.source { + detail, err := in.opts.Registry.Plugin(in.ctx, id) + if err != nil { + return InstallStep{}, nil, fmt.Errorf("build: %w", err) + } + if err := validateRepositoryURL(id, detail.Repository); err != nil { + return InstallStep{}, nil, err + } + step.Mode, step.Repository = ModeSubmodule, detail.Repository + if dir := filepath.Join(in.appDir, "plugins", pluginVendor(id), pluginName(id)); dirExists(dir) { + return InstallStep{}, nil, fmt.Errorf("build: %s already exists; remove it or register it with plugin:add", dir) + } + } if err := validateStep(step); err != nil { return InstallStep{}, nil, err } @@ -451,7 +515,10 @@ func newestOf(versions []kiosk.RegistryVersion) kiosk.RegistryVersion { // the version must be a strict vX.Y.Z and the module and package must be // plain module paths, the package inside the module. func validateStep(s InstallStep) error { - if !kiosk.ValidVersion(s.Version) { + // A git URL install without a fitting release tag takes the default + // branch head, which has no version. + headOfClone := s.Version == "" && s.clone != "" + if !kiosk.ValidVersion(s.Version) && !headOfClone { return fmt.Errorf("build: registry entry %s: version %q is not a vX.Y.Z release; refused", s.ID, s.Version) } if err := validateModulePath(s.Module); err != nil { @@ -463,6 +530,15 @@ func validateStep(s InstallStep) error { if s.Package != s.Module && !strings.HasPrefix(s.Package, s.Module+"/") { return fmt.Errorf("build: registry entry %s: package %s is not inside module %s; refused", s.ID, s.Package, s.Module) } + switch s.Mode { + case ModeModule: + case ModeSubmodule: + if err := validateGitURL(s.Repository); err != nil { + return fmt.Errorf("build: %s: %w; refused", s.ID, err) + } + default: + return fmt.Errorf("build: %s: unknown install mode %q", s.ID, s.Mode) + } return nil } @@ -470,7 +546,7 @@ func validateStep(s InstallStep) error { func (p *InstallPlan) summary() string { var b strings.Builder for _, s := range p.Steps { - fmt.Fprintf(&b, " %s@%s (%s)\n", s.ID, s.Version, s.Mode) + fmt.Fprintf(&b, " %s@%s (%s)\n", s.ID, displayRef(s.Version), s.Mode) } return b.String() } @@ -508,6 +584,11 @@ func askToProceed(yes, interactive bool, confirm func(string) (bool, error), que return nil } +func dirExists(dir string) bool { + _, err := os.Stat(dir) + return err == nil +} + func displayVersion(v string) string { if v == "" { return "unknown" @@ -515,28 +596,58 @@ func displayVersion(v string) string { return v } -// ApplyInstall carries out a plan from PlanInstall: one go get for every -// module step, so minimal version selection sees the whole set, then the -// summer.yaml entries in plan order. It does not run go mod tidy: until +// ApplyInstall carries out a plan from PlanInstall. Submodule steps are +// cloned at their tag, added as git submodules under +// plugins// and registered in go.mod (require at the tag +// plus a local replace) and go.work, followed by the go.work replace +// overrides the workspace needs. Module steps go through one go get, so +// minimal version selection sees the whole set. The summer.yaml entries +// are appended last, in plan order. It does not run go mod tidy: until // summer build imports the new packages, tidy would drop them. func ApplyInstall(ctx context.Context, plan *InstallPlan, out io.Writer) error { if plan == nil || len(plan.Steps) == 0 { return fmt.Errorf("build: nothing to install") } + defer plan.removeTemps() if out == nil { out = io.Discard } var getArgs []string var entries []Plugin + submodules := false for _, s := range plan.Steps { if err := validateStep(s); err != nil { return err } - if s.Mode == ModeModule && !slices.Contains(getArgs, s.Module+"@"+s.Version) { - getArgs = append(getArgs, s.Module+"@"+s.Version) + switch s.Mode { + case ModeModule: + if !slices.Contains(getArgs, s.Module+"@"+s.Version) { + getArgs = append(getArgs, s.Module+"@"+s.Version) + } + case ModeSubmodule: + submodules = true } entries = append(entries, Plugin{ID: s.ID, Module: s.Package}) } + if submodules { + if err := requireGitRepo(ctx, plan.AppDir); err != nil { + return err + } + done := map[string]bool{} + for i := range plan.Steps { + s := &plan.Steps[i] + if s.Mode != ModeSubmodule || done[s.Module] { + continue + } + if err := applySubmodule(ctx, plan, s, plan.interactive, out); err != nil { + return err + } + done[s.Module] = true + } + if err := refreshWorkOverrides(ctx, plan.AppDir, out); err != nil { + return err + } + } if len(getArgs) > 0 { fmt.Fprintf(out, "go get %s\n", strings.Join(getArgs, " ")) if err := runGo(ctx, plan.AppDir, append([]string{"get", "--"}, getArgs...)...); err != nil { @@ -548,7 +659,7 @@ func ApplyInstall(ctx context.Context, plan *InstallPlan, out io.Writer) error { } ids := make([]string, len(plan.Steps)) for i, s := range plan.Steps { - ids[i] = s.ID + "@" + s.Version + ids[i] = s.ID + "@" + displayRef(s.Version) } fmt.Fprintf(out, "Installed %s; run summer build\n", strings.Join(ids, ", ")) return nil diff --git a/internal/build/install_smoke_test.go b/internal/build/install_smoke_test.go index 6229a7c6..b6a2a2ff 100644 --- a/internal/build/install_smoke_test.go +++ b/internal/build/install_smoke_test.go @@ -353,6 +353,268 @@ func TestInstallSmoke(t *testing.T) { } }) + // Source installs: fixture repositories over file://, submodules in a + // git copy of examples/hello, builds with GOPROXY=off. + translateReq := map[string][2]string{"acme.translate": {translateModule, "v0.1.0"}} + sourceRegistry := func(t *testing.T, blogRepo, translateRepo string) *fakeReg { + return fakeRegistry(t, map[string]fakePlugin{ + "acme.blog": regPlugin("acme.blog", blogRepo, + regVersion("v0.2.0", "v9.0.0", blogModule, blogModule, kiosk.Requires{"acme.translate": "v0.1.0"}), + regVersion("v0.1.0", "v0.5.0", blogModule, blogModule, kiosk.Requires{"acme.translate": "v0.1.0"})), + "acme.translate": regPlugin("acme.translate", translateRepo, + regVersion("v0.1.0", "v0.5.0", translateModule, translateModule, nil)), + }) + } + assertSubmodule := func(t *testing.T, app, rel, tag string) { + t.Helper() + gm, err := os.ReadFile(filepath.Join(app, ".gitmodules")) + if err != nil || !bytes.Contains(gm, []byte("path = "+rel)) { + t.Fatalf(".gitmodules lacks %s: %v\n%s", rel, err, gm) + } + if status := gitOut(t, app, "submodule", "status", "--", rel); !strings.Contains(status, "("+tag+")") { + t.Fatalf("submodule status = %q, want %s", status, tag) + } + if st, err := os.Stat(filepath.Join(app, rel, ".git")); err != nil || st.IsDir() { + t.Fatalf("%s/.git should be a file after absorbgitdirs: %v", rel, err) + } + } + buildApp := func(t *testing.T, app string) { + t.Helper() + var log bytes.Buffer + if err := App(t.Context(), app, &log); err != nil { + work, _ := os.ReadFile(filepath.Join(app, "go.work")) + t.Fatalf("build: %v\n%s\ngo.work:\n%s", err, log.String(), work) + } + } + + t.Run("source", func(t *testing.T) { + if testing.Short() { + t.Skip("clones fixture repositories and builds the app") + } + gitTestEnv(t) + blogRepo := pluginRepo(t, "acme.blog", blogModule, []fixtureRelease{{Tag: "v0.1.0", Framework: "v0.5.0"}, {Tag: "v0.2.0", Framework: "v9.0.0"}}, nil) + app := gitApp(t) + plantComment(t, app) + reg := fakeRegistry(t, map[string]fakePlugin{ + "acme.blog": regPlugin("acme.blog", blogRepo, + regVersion("v0.2.0", "v9.0.0", blogModule, blogModule, nil), + regVersion("v0.1.0", "v0.5.0", blogModule, blogModule, nil)), + }) + var out bytes.Buffer + plan, err := PlanInstall(t.Context(), InstallOptions{StartDir: app, Arg: "acme.blog", Source: true, Yes: true, Registry: reg.Client, Out: &out}) + if err != nil { + t.Fatalf("plan: %v\n%s", err, out.String()) + } + if got := stepList(plan); got != "acme.blog@v0.1.0/submodule" { + t.Fatalf("steps = %s", got) + } + if err := ApplyInstall(t.Context(), plan, &out); err != nil { + t.Fatalf("apply: %v\n%s", err, out.String()) + } + assertSubmodule(t, app, "plugins/acme/blog", "v0.1.0") + gomod, _ := os.ReadFile(filepath.Join(app, "go.mod")) + if !bytes.Contains(gomod, []byte(blogModule+" v0.1.0")) || !bytes.Contains(gomod, []byte(blogModule+" => ./plugins/acme/blog")) { + t.Fatalf("go.mod lacks the require at the tag or the local replace:\n%s", gomod) + } + work, _ := os.ReadFile(filepath.Join(app, "go.work")) + if !bytes.Contains(work, []byte("./plugins/acme/blog")) || !bytes.Contains(work, []byte("replace "+frameworkModule)) { + t.Fatalf("go.work lacks the use or the framework override:\n%s", work) + } + assertManifestKeepsComment(t, app) + if len(installTemps(t, app)) != 0 { + t.Fatalf("leftover checkouts: %v", installTemps(t, app)) + } + buildApp(t, app) + }) + + t.Run("source_deps", func(t *testing.T) { + if testing.Short() { + t.Skip("clones fixture repositories and builds the app") + } + gitTestEnv(t) + translateRepo := pluginRepo(t, "acme.translate", translateModule, []fixtureRelease{{Tag: "v0.1.0", Framework: "v0.5.0"}}, nil) + blogRepo := pluginRepo(t, "acme.blog", blogModule, []fixtureRelease{{Tag: "v0.1.0", Framework: "v0.5.0", Requires: translateReq}}, nil) + app := gitApp(t) + reg := sourceRegistry(t, blogRepo, translateRepo) + var out bytes.Buffer + plan, err := PlanInstall(t.Context(), InstallOptions{StartDir: app, Arg: "acme.blog", Source: true, Yes: true, Registry: reg.Client, Out: &out}) + if err != nil { + t.Fatalf("plan: %v\n%s", err, out.String()) + } + if got := stepList(plan); got != "acme.translate@v0.1.0/submodule,acme.blog@v0.1.0/submodule" { + t.Fatalf("steps = %s", got) + } + if err := ApplyInstall(t.Context(), plan, &out); err != nil { + t.Fatalf("apply: %v\n%s", err, out.String()) + } + assertSubmodule(t, app, "plugins/acme/translate", "v0.1.0") + assertSubmodule(t, app, "plugins/acme/blog", "v0.1.0") + work, _ := os.ReadFile(filepath.Join(app, "go.work")) + if !bytes.Contains(work, []byte(translateModule+" v0.1.0 => ./plugins/acme/translate")) { + t.Fatalf("go.work lacks the versioned override for the workspace module:\n%s", work) + } + buildApp(t, app) + }) + + t.Run("giturl", func(t *testing.T) { + if testing.Short() { + t.Skip("clones fixture repositories and builds the app") + } + gitTestEnv(t) + translateRepo := pluginRepo(t, "acme.translate", translateModule, []fixtureRelease{{Tag: "v0.1.0", Framework: "v0.5.0"}}, nil) + blogRepo := pluginRepo(t, "acme.blog", blogModule, []fixtureRelease{ + {Tag: "v0.1.0", Framework: "v0.5.0", Requires: translateReq}, + {Tag: "v0.2.0", Framework: "v9.0.0", Requires: translateReq}, + }, nil) + app := gitApp(t) + reg := fakeRegistry(t, map[string]fakePlugin{ + "acme.translate": regPlugin("acme.translate", translateRepo, regVersion("v0.1.0", "v0.5.0", translateModule, translateModule, nil)), + }) + var out bytes.Buffer + plan, err := PlanInstall(t.Context(), InstallOptions{StartDir: app, Arg: blogRepo, Yes: true, Registry: reg.Client, Out: &out}) + if err != nil { + t.Fatalf("plan: %v\n%s", err, out.String()) + } + if got := stepList(plan); got != "acme.translate@v0.1.0/submodule,acme.blog@v0.1.0/submodule" { + t.Fatalf("steps = %s", got) + } + if err := ApplyInstall(t.Context(), plan, &out); err != nil { + t.Fatalf("apply: %v\n%s", err, out.String()) + } + for _, r := range reg.Requests() { + if strings.Contains(r, "acme.blog") { + t.Fatalf("the git URL plugin was looked up in the registry: %v", reg.Requests()) + } + } + assertSubmodule(t, app, "plugins/acme/blog", "v0.1.0") + buildApp(t, app) + + // No tag fits: the default branch head is planned with a warning, + // and a declined prompt leaves no checkout behind. + headRepo := pluginRepo(t, "acme.news", "example.com/acme/news", []fixtureRelease{{Tag: "v0.1.0", Framework: "v9.0.0"}}, &fixtureRelease{Tag: "head", Framework: "v9.0.0"}) + out.Reset() + _, err = PlanInstall(t.Context(), InstallOptions{StartDir: app, Arg: headRepo, Registry: reg.Client, Out: &out, Interactive: true, + Confirm: func(string) (bool, error) { return false, nil }}) + if !errors.Is(err, ErrCancelled) { + t.Fatalf("head plan: err = %v\n%s", err, out.String()) + } + if !strings.Contains(out.String(), "warning: no release tag fits framework") || !strings.Contains(out.String(), "acme.news@default branch head (submodule)") { + t.Fatalf("head plan output:\n%s", out.String()) + } + if left := installTemps(t, app); len(left) != 0 { + t.Fatalf("declined plan left checkouts: %v", left) + } + }) + + t.Run("giturl_offline", func(t *testing.T) { + if testing.Short() { + t.Skip("clones fixture repositories") + } + gitTestEnv(t) + blogRepo := pluginRepo(t, "acme.blog", blogModule, []fixtureRelease{{Tag: "v0.1.0", Framework: "v0.5.0", Requires: translateReq}}, nil) + app := gitApp(t) + reg := fakeRegistry(t, map[string]fakePlugin{}) + _, err := PlanInstall(t.Context(), InstallOptions{StartDir: app, Arg: blogRepo, Yes: true, Offline: true, Registry: reg.Client}) + if err == nil || !strings.Contains(err.Error(), "acme.translate") || !strings.Contains(err.Error(), "--offline") { + t.Fatalf("missing requirement offline: err = %v", err) + } + if left := installTemps(t, app); len(left) != 0 { + t.Fatalf("failed plan left checkouts: %v", left) + } + if err := appendManifestPlugins(filepath.Join(app, manifestName), []Plugin{{ID: "acme.translate", Module: translateModule}}); err != nil { + t.Fatal(err) + } + if err := runGo(t.Context(), app, "mod", "edit", "-require="+translateModule+"@v0.1.0"); err != nil { + t.Fatal(err) + } + var out bytes.Buffer + plan, err := PlanInstall(t.Context(), InstallOptions{StartDir: app, Arg: blogRepo, Yes: true, Offline: true, Registry: reg.Client, Out: &out}) + if err != nil { + t.Fatalf("plan: %v\n%s", err, out.String()) + } + if err := ApplyInstall(t.Context(), plan, &out); err != nil { + t.Fatalf("apply: %v\n%s", err, out.String()) + } + if n := len(reg.Requests()); n != 0 { + t.Fatalf("offline install made %d registry requests: %v", n, reg.Requests()) + } + work, _ := os.ReadFile(filepath.Join(app, "go.work")) + if !bytes.Contains(work, []byte(translateModule+" => "+translateModule+" v0.1.0")) { + t.Fatalf("go.work lacks the version pin for the replaced requirement:\n%s", work) + } + }) + + t.Run("nogit", func(t *testing.T) { + gitTestEnv(t) + app := helloCopy(t) + reg := depsRegistry(t) + _, err := PlanInstall(t.Context(), InstallOptions{StartDir: app, Arg: "acme.blog", Source: true, Yes: true, Registry: reg.Client}) + if err == nil || !strings.Contains(err.Error(), "not a git repository") || !strings.Contains(err.Error(), "submodule") { + t.Fatalf("--source outside git: err = %v", err) + } + _, err = PlanInstall(t.Context(), InstallOptions{StartDir: app, Arg: "https://git.example.com/acme/sm-blog-plugin.git", Yes: true, Registry: reg.Client}) + if err == nil || !strings.Contains(err.Error(), "not a git repository") { + t.Fatalf("git URL outside git: err = %v", err) + } + if left := installTemps(t, app); len(left) != 0 { + t.Fatalf("refused install left checkouts: %v", left) + } + }) + + t.Run("urls", func(t *testing.T) { + accept := []string{ + "https://git.example.com/acme/sm-blog-plugin.git", + "ssh://git@git.example.com/acme/blog.git", + "git@git.example.com:acme/blog.git", + } + refuse := []string{ + "-uhttps://git.example.com/x.git", + "--upload-pack=touch /tmp/x", + "ext::sh -c touch% /tmp/pwned", + "file::/tmp/x", + "file:///tmp/repo", + "http://git.example.com/acme/blog.git", + "git://git.example.com/acme/blog.git", + "https://user:secret@git.example.com/acme/blog.git", + "https://git.example.com/-x", + "git@git.example.com:-oProxyCommand=x", + "https://git.example.com/a b.git", + } + for _, u := range accept { + if !isGitURL(u) { + t.Errorf("isGitURL(%q) = false", u) + } + if err := validateGitURL(u); err != nil { + t.Errorf("validateGitURL(%q) = %v", u, err) + } + } + for _, u := range refuse { + if err := validateGitURL(u); err == nil { + t.Errorf("validateGitURL(%q) accepted", u) + } + } + for _, id := range []string{"acme.blog", "acme.blog@v0.1.0"} { + if isGitURL(id) { + t.Errorf("isGitURL(%q) = true", id) + } + } + if err := validateRepositoryURL("acme.blog", "ssh://git@git.example.com/acme/blog.git"); err == nil { + t.Error("registry repository over ssh accepted") + } + if err := validateRepositoryURL("acme.blog", "https://git.example.com/acme/blog.git"); err != nil { + t.Errorf("registry https repository: %v", err) + } + old := cliGitProtocols + cliGitProtocols = "https:ssh:file" + defer func() { cliGitProtocols = old }() + if err := validateGitURL("file:///tmp/repo"); err != nil { + t.Errorf("file:// with the test protocols: %v", err) + } + if err := validateGitURL("file::/tmp/repo"); err == nil { + t.Error("file:: accepted with the test protocols") + } + }) + t.Run("append_manifest", func(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, manifestName) diff --git a/internal/build/scaffold.go b/internal/build/scaffold.go index 9b762c09..1872480a 100644 --- a/internal/build/scaffold.go +++ b/internal/build/scaffold.go @@ -195,7 +195,7 @@ func AddPlugin(ctx context.Context, startDir, pluginDir string) error { m.Plugins = append(m.Plugins, Plugin{ID: id, Module: modPath}) } - if err := editAppModule(ctx, appDir, pluginDir, modPath); err != nil { + if err := editAppModule(ctx, appDir, pluginDir, modPath, "v0.0.0"); err != nil { return err } if err := ensureGoWork(ctx, appDir, m, pluginDir); err != nil { @@ -369,8 +369,12 @@ func pluginIDFromGo(path string) (string, error) { return "", fmt.Errorf("build: %s does not return a plugin id: %w", path, err) } -func editAppModule(ctx context.Context, appDir, pluginDir, modPath string) error { - if err := runGo(ctx, appDir, "mod", "edit", "-require="+modPath+"@v0.0.0"); err != nil { +// editAppModule requires modPath at version in the app's go.mod and +// replaces it with the local pluginDir. plugin:add passes v0.0.0; a +// submodule install passes the tag it checked out, so a GOWORK=off build +// and version checks see the real release. +func editAppModule(ctx context.Context, appDir, pluginDir, modPath, version string) error { + if err := runGo(ctx, appDir, "mod", "edit", "-require="+modPath+"@"+version); err != nil { return err } rel, err := relModulePath(appDir, pluginDir) diff --git a/internal/build/source.go b/internal/build/source.go new file mode 100644 index 00000000..c22deb2c --- /dev/null +++ b/internal/build/source.go @@ -0,0 +1,601 @@ +package build + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/url" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "slices" + "sort" + "strings" + + "git.golem15.com/golem15/summercms/modules/bonfire" + "git.golem15.com/golem15/summercms/modules/kiosk" +) + +// cliGitProtocols is GIT_ALLOW_PROTOCOL for the git commands of +// plugin:install and plugin:update, and the URL schemes they accept. +// Tests add "file" for fixture repositories; nothing else changes it. +var cliGitProtocols = "https:ssh" + +var ( + scpLikeRe = regexp.MustCompile(`^[A-Za-z0-9._-]+@[A-Za-z0-9.-]+:`) + repoNameRe = regexp.MustCompile(`^[a-z][a-z0-9]*$`) + installTemp = ".summer-install-" +) + +// allowedGitProtocol reports whether cliGitProtocols lists name. +func allowedGitProtocol(name string) bool { + return slices.Contains(strings.Split(cliGitProtocols, ":"), name) +} + +// runCLIGit runs git in dir for the CLI. It keeps the user's git config +// (credential helpers, insteadOf), allows only cliGitProtocols, and stops +// git from prompting when no person can answer. Callers put "--" before +// URLs and paths. +func runCLIGit(ctx context.Context, dir string, interactive bool, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Dir = dir + env := make([]string, 0, len(os.Environ())+2) + for _, e := range os.Environ() { + if strings.HasPrefix(e, "GIT_ALLOW_PROTOCOL=") || strings.HasPrefix(e, "GIT_TERMINAL_PROMPT=") { + continue + } + env = append(env, e) + } + env = append(env, "GIT_ALLOW_PROTOCOL="+cliGitProtocols) + if !interactive { + env = append(env, "GIT_TERMINAL_PROMPT=0") + } + cmd.Env = env + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + msg := strings.TrimSpace(stderr.String()) + if msg == "" { + msg = err.Error() + } + return out, fmt.Errorf("git %s: %s", args[0], msg) + } + return out, nil +} + +// isGitURL reports whether arg is meant as a repository address rather +// than a plugin ID: a scheme URL, a scp-like user@host:path or a +// transport::address form (which validateGitURL then refuses). +func isGitURL(arg string) bool { + return strings.Contains(arg, "://") || strings.Contains(arg, "::") || scpLikeRe.MatchString(arg) +} + +// validateGitURL accepts https://, ssh:// and scp-like user@host:path +// addresses (file:// only when cliGitProtocols allows it, in tests). It +// refuses a leading "-", transport::address forms, http, credentials in +// the URL and control characters. +func validateGitURL(raw string) error { + switch { + case raw == "": + return errors.New("git URL is empty") + case strings.HasPrefix(raw, "-"): + return fmt.Errorf("git URL %q must not start with -", raw) + case strings.ContainsFunc(raw, func(r rune) bool { return r <= ' ' || r == 0x7f }): + return fmt.Errorf("git URL %q contains spaces or control characters", raw) + case strings.Contains(raw, "::"): + return fmt.Errorf("git URL %q uses a transport:: form; use https://, ssh:// or user@host:path", raw) + } + if !strings.Contains(raw, "://") { + if !scpLikeRe.MatchString(raw) { + return fmt.Errorf("%q is not a git URL; use https://, ssh:// or user@host:path", raw) + } + if !allowedGitProtocol("ssh") { + return fmt.Errorf("git URL %q: ssh is not allowed", raw) + } + _, p, _ := strings.Cut(raw, ":") + if p == "" || strings.HasPrefix(p, "-") { + return fmt.Errorf("git URL %q has an invalid path", raw) + } + return nil + } + u, err := url.Parse(raw) + if err != nil { + return fmt.Errorf("git URL %q: %w", raw, err) + } + scheme := strings.ToLower(u.Scheme) + if !allowedGitProtocol(scheme) { + return fmt.Errorf("git URL %q: only https://, ssh:// and user@host:path are accepted", raw) + } + if scheme != "file" && u.Host == "" { + return fmt.Errorf("git URL %q has no host", raw) + } + if _, hasPassword := u.User.Password(); hasPassword { + return fmt.Errorf("git URL must not carry a password; use a git credential helper") + } + if strings.HasPrefix(u.Path, "/-") || u.Path == "" || u.Path == "/" { + return fmt.Errorf("git URL %q has an invalid path", raw) + } + return nil +} + +// validateRepositoryURL checks a repository URL the registry sent: it must +// be https without user info (file:// only in tests). +func validateRepositoryURL(id, raw string) error { + if raw == "" { + return fmt.Errorf("build: registry entry %s has no repository URL; refused", id) + } + if err := validateGitURL(raw); err != nil { + return fmt.Errorf("build: registry entry %s: %w; refused", id, err) + } + u, err := url.Parse(raw) + if err != nil || !strings.Contains(raw, "://") { + return fmt.Errorf("build: registry entry %s: repository %q is not an https URL; refused", id, raw) + } + if s := strings.ToLower(u.Scheme); s != "https" && !(s == "file" && allowedGitProtocol("file")) { + return fmt.Errorf("build: registry entry %s: repository %q is not an https URL; refused", id, raw) + } + if u.User != nil { + return fmt.Errorf("build: registry entry %s: repository URL carries user info; refused", id) + } + return nil +} + +// requireGitRepo refuses submodule installs in an application that is not +// a git repository. +func requireGitRepo(ctx context.Context, appDir string) error { + if _, err := gitOutput(ctx, appDir, "rev-parse", "--show-toplevel"); err != nil { + return bonfire.WithHint(fmt.Errorf("build: %s is not a git repository; a plugin installed from source is a git submodule, which needs one", appDir), + "--source and git URL installs add the plugin as a git submodule of the application. Initialise a repository first, or install the plugin by ID as a Go module.", + "git init && git add -A && git commit -m init") + } + return nil +} + +// cloneInto clones url into a new temporary directory under +// /plugins, so the later move to plugins// stays on one +// file system. +func cloneInto(ctx context.Context, appDir, rawURL string, interactive bool) (string, error) { + parent := filepath.Join(appDir, "plugins") + if err := os.MkdirAll(parent, 0o755); err != nil { + return "", fmt.Errorf("build: create %s: %w", parent, err) + } + tmp, err := os.MkdirTemp(parent, installTemp) + if err != nil { + return "", fmt.Errorf("build: %w", err) + } + if _, err := runCLIGit(ctx, appDir, interactive, "clone", "--quiet", "--", rawURL, tmp); err != nil { + _ = os.RemoveAll(tmp) + return "", fmt.Errorf("build: clone %s: %w", rawURL, err) + } + return tmp, nil +} + +// releaseTags returns the vX.Y.Z tags of the repository in dir, newest +// first. +func releaseTags(ctx context.Context, dir string) ([]string, error) { + out, err := runCLIGit(ctx, dir, false, "tag", "--list", "v*") + if err != nil { + return nil, fmt.Errorf("build: %w", err) + } + var tags []string + for _, t := range strings.Fields(string(out)) { + if kiosk.ValidVersion(t) { + tags = append(tags, t) + } + } + sort.Slice(tags, func(i, j int) bool { return kiosk.CompareVersions(tags[i], tags[j]) > 0 }) + return tags, nil +} + +// manifestAt reads /plugin.yaml at a git ref; nil when the file does +// not exist there. +func manifestAt(ctx context.Context, dir, ref, rel string) (*kiosk.Manifest, error) { + p := kiosk.FileName + if rel != "" { + p = path.Join(rel, kiosk.FileName) + } + out, err := runCLIGit(ctx, dir, false, "show", ref+":"+p) + if err != nil { + return nil, nil + } + m, err := kiosk.Parse(out) + if err != nil { + return nil, fmt.Errorf("build: %s at %s: %w", p, ref, err) + } + return &m, nil +} + +// checkout moves the work tree in dir to ref (a release tag). +func checkout(ctx context.Context, dir, ref string) error { + if _, err := runCLIGit(ctx, dir, false, "checkout", "--quiet", "--detach", ref); err != nil { + return fmt.Errorf("build: check out %s: %w", ref, err) + } + return nil +} + +// planGitURL clones the repository at rawURL, picks the newest release +// tag whose plugin.yaml framework minimum the application satisfies +// (falling back to the default branch head with a warning) and adds the +// plugin and its requirements to the plan. The plugin itself is never +// looked up in the registry. +func (in *installer) planGitURL(rawURL string) error { + if err := validateGitURL(rawURL); err != nil { + return fmt.Errorf("build: %w", err) + } + if err := requireGitRepo(in.ctx, in.appDir); err != nil { + return err + } + fmt.Fprintf(in.out, "Cloning %s\n", rawURL) + tmp, err := cloneInto(in.ctx, in.appDir, rawURL, in.opts.Interactive) + if err != nil { + return err + } + in.plan.temps = append(in.plan.temps, tmp) + + tags, err := releaseTags(in.ctx, tmp) + if err != nil { + return err + } + ref, version := "", "" + var pm *kiosk.Manifest + newest := "" + for _, tag := range tags { + m, err := manifestAt(in.ctx, tmp, tag, "") + if err != nil { + fmt.Fprintf(in.out, "warning: %v; tag skipped\n", err) + continue + } + if m == nil { + continue + } + if newest == "" { + newest = tag + " needs framework " + m.Framework + } + if kiosk.Satisfies(in.fw, m.Framework) { + ref, version, pm = tag, tag, m + break + } + } + if ref == "" { + why := "no release tag carries a plugin.yaml" + if newest != "" { + why = "no release tag fits framework " + displayVersion(in.fw) + " (" + newest + ")" + } + fmt.Fprintf(in.out, "warning: %s; installing the default branch head of %s\n", why, rawURL) + } else if err := checkout(in.ctx, tmp, ref); err != nil { + return err + } + + step := InstallStep{Version: version, Repository: rawURL, Mode: ModeSubmodule, clone: tmp} + var requires kiosk.Requires + if pm == nil { + pm, err = readCheckoutManifest(tmp, "") + if err != nil { + return err + } + } + if pm != nil { + step.ID, step.Module, step.Package, requires = pm.ID, pm.Module, pm.Package, pm.Requires + } else { + // No plugin.yaml at all: read the identity from the sources. + id, err := kiosk.ReadIdentity(in.ctx, tmp) + if id.ID == "" { + return fmt.Errorf("build: %s has no plugin.yaml and no readable plugin.go at its root: %v", rawURL, err) + } + mod, err := readModulePath(filepath.Join(tmp, "go.mod")) + if err != nil { + return err + } + step.ID, step.Module, step.Package = id.ID, mod, mod + requires = kiosk.Requires{} + for _, r := range id.Requires { + requires[r] = "" + } + } + if err := ValidatePluginID(step.ID); err != nil { + return fmt.Errorf("build: %s: %w", rawURL, err) + } + if err := in.requireAbsent(step.ID); err != nil { + return err + } + if err := validateStep(step); err != nil { + return err + } + if _, err := submoduleTarget(in.ctx, in.appDir, tmp, step.ID, rawURL); err != nil { + return err + } + in.moduleVersion[step.Module] = step.Version + in.visiting = append(in.visiting, step.ID) + if err := in.resolveRequires(step.ID, step.Version, requires); err != nil { + return err + } + in.visiting = in.visiting[:len(in.visiting)-1] + in.planned[step.ID] = step + in.plan.Steps = append(in.plan.Steps, step) + return nil +} + +// readCheckoutManifest reads //plugin.yaml from a work tree; nil +// when it does not exist. +func readCheckoutManifest(dir, rel string) (*kiosk.Manifest, error) { + p := filepath.Join(dir, filepath.FromSlash(rel), kiosk.FileName) + raw, err := os.ReadFile(p) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("build: read %s: %w", p, err) + } + m, err := kiosk.Parse(raw) + if err != nil { + return nil, fmt.Errorf("build: %s: %w", p, err) + } + return &m, nil +} + +// submoduleTarget returns plugins// for a checkout: the ID +// in the repository root's plugin.yaml (or plugin.go), otherwise the +// repository name without sm- and -plugin under the vendor of stepID. +func submoduleTarget(ctx context.Context, appDir, cloneDir, stepID, repoURL string) (string, error) { + id := "" + if pm, err := readCheckoutManifest(cloneDir, ""); err == nil && pm != nil { + id = pm.ID + } else if ident, _ := kiosk.ReadIdentity(ctx, cloneDir); ident.ID != "" { + id = ident.ID + } + var dir string + if id != "" { + if err := ValidatePluginID(id); err != nil { + return "", fmt.Errorf("build: %s: %w", repoURL, err) + } + dir = filepath.Join(appDir, "plugins", pluginVendor(id), pluginName(id)) + } else { + name := strings.TrimSuffix(path.Base(strings.TrimRight(repoURL, "/")), ".git") + if i := strings.LastIndex(name, ":"); i >= 0 { + name = name[i+1:] + } + name = strings.TrimSuffix(strings.TrimPrefix(name, "sm-"), "-plugin") + if !repoNameRe.MatchString(name) { + return "", fmt.Errorf("build: cannot name a plugin directory after repository %s", repoURL) + } + dir = filepath.Join(appDir, "plugins", pluginVendor(stepID), name) + } + if !underRoot(appDir, dir) { + return "", fmt.Errorf("build: plugin path %q escapes the app root", dir) + } + if _, err := os.Stat(dir); err == nil { + return "", fmt.Errorf("build: %s already exists; remove it or register it with plugin:add", dir) + } + return dir, nil +} + +// applySubmodule installs one submodule step: clone and check out the tag +// (a git URL step was cloned while planning), check that the plugin.yaml +// in the checkout describes the planned plugin, move the checkout to +// plugins//, add it as a submodule and register it in go.mod +// and go.work with the tag as the require version. +func applySubmodule(ctx context.Context, plan *InstallPlan, s *InstallStep, interactive bool, out io.Writer) error { + appDir := plan.AppDir + dir := s.clone + if dir == "" { + fmt.Fprintf(out, "Cloning %s at %s\n", s.Repository, s.Version) + tmp, err := cloneInto(ctx, appDir, s.Repository, interactive) + if err != nil { + return err + } + plan.temps = append(plan.temps, tmp) + if err := checkout(ctx, tmp, s.Version); err != nil { + return err + } + dir = tmp + } + rel := strings.TrimPrefix(strings.TrimPrefix(s.Package, s.Module), "/") + pm, err := readCheckoutManifest(dir, rel) + if err != nil { + return err + } + if pm != nil && (pm.ID != s.ID || pm.Module != s.Module) { + return fmt.Errorf("build: %s at %s describes %s (%s), not %s (%s); refused", s.Repository, displayRef(s.Version), pm.ID, pm.Module, s.ID, s.Module) + } + if pm == nil && s.clone == "" { + return fmt.Errorf("build: %s at %s has no %s for %s; refused", s.Repository, s.Version, path.Join(rel, kiosk.FileName), s.ID) + } + mod, err := readModulePath(filepath.Join(dir, "go.mod")) + if err != nil { + return err + } + if mod != s.Module { + return fmt.Errorf("build: %s is module %s, not %s; refused", s.Repository, mod, s.Module) + } + target, err := submoduleTarget(ctx, appDir, dir, s.ID, s.Repository) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return fmt.Errorf("build: %w", err) + } + if err := os.Rename(dir, target); err != nil { + return fmt.Errorf("build: move checkout to %s: %w", target, err) + } + relTarget, err := filepath.Rel(appDir, target) + if err != nil { + return fmt.Errorf("build: %w", err) + } + relTarget = filepath.ToSlash(relTarget) + if _, err := runCLIGit(ctx, appDir, interactive, "submodule", "add", "--quiet", "--", s.Repository, relTarget); err != nil { + return fmt.Errorf("build: %w", err) + } + if _, err := runCLIGit(ctx, appDir, interactive, "submodule", "absorbgitdirs", "--", relTarget); err != nil { + return fmt.Errorf("build: %w", err) + } + version := s.Version + if version == "" { + version = "v0.0.0" + } + if err := editAppModule(ctx, appDir, target, s.Module, version); err != nil { + return err + } + m, err := LoadManifest(filepath.Join(appDir, manifestName)) + if err != nil { + return err + } + if err := ensureGoWork(ctx, appDir, m, target); err != nil { + return err + } + fmt.Fprintf(out, "Added submodule %s (%s)\n", relTarget, displayRef(s.Version)) + return nil +} + +func displayRef(version string) string { + if version == "" { + return "default branch head" + } + return version +} + +// refreshWorkOverrides adds the go.work replace overrides a workspace of +// submodule plugins needs (research F2). A plugin repository's go.mod +// often replaces the framework or sibling plugins with paths of its own +// development layout; in the application's workspace those paths do not +// exist, and different modules replacing one path differently is an +// error. For every local (directory) replace in the go.mod of a module go.work uses +// (other than the app), whose path go.work does not replace yet, it adds: +// the app's own replace of that path when the app's go.mod has one; the +// workspace module's directory, at the version required, when the path is +// a workspace module (a whole-path replace of a workspace module is an +// error); otherwise the path at the version the app's go.mod (or the +// plugin's) requires. A pin of the last kind that no longer matches the +// app's require is moved to it. +func refreshWorkOverrides(ctx context.Context, appDir string, out io.Writer) error { + gw := findGoWork(appDir) + if gw == "" { + return nil + } + workDir := filepath.Dir(gw) + work, err := editJSON(ctx, workDir, "work", "edit", "-json", gw) + if err != nil { + return fmt.Errorf("build: %w", err) + } + app, err := editJSON(ctx, appDir, "mod", "edit", "-json", filepath.Join(appDir, "go.mod")) + if err != nil { + return fmt.Errorf("build: %w", err) + } + appRequire := map[string]string{} + for _, r := range app.Require { + appRequire[r.Path] = r.Version + } + replaced := map[string]bool{} + var edits []string + for _, r := range work.Replace { + replaced[r.Old.Path] = true + // A pin of a path to one of its own versions follows the app. + if r.New.Path == r.Old.Path && r.New.Version != "" { + if want := appRequire[r.Old.Path]; want != "" && want != r.New.Version { + edits = append(edits, "-replace="+r.Old.Path+"="+r.Old.Path+"@"+want) + } + } + } + workspace := map[string]string{} // module path -> directory + var useDirs []string + for _, u := range work.Use { + dir := u.DiskPath + if !filepath.IsAbs(dir) { + dir = filepath.Join(workDir, dir) + } + dir = filepath.Clean(dir) + mod, err := readModulePath(filepath.Join(dir, "go.mod")) + if err != nil { + continue + } + workspace[mod] = dir + if !samePath(dir, appDir) { + useDirs = append(useDirs, dir) + } + } + sort.Strings(useDirs) + for _, dir := range useDirs { + mod, err := editJSON(ctx, dir, "mod", "edit", "-json", filepath.Join(dir, "go.mod")) + if err != nil { + return fmt.Errorf("build: %w", err) + } + pluginRequire := map[string]string{} + for _, r := range mod.Require { + pluginRequire[r.Path] = r.Version + } + for _, r := range mod.Replace { + p := r.Old.Path + if r.New.Version != "" || replaced[p] { + continue + } + version := appRequire[p] + if version == "" { + version = pluginRequire[p] + } + var edit string + switch { + case workspace[p] != "": + if version == "" { + fmt.Fprintf(out, "warning: cannot pin %s in go.work: no module requires a version of it\n", p) + continue + } + target, err := relModulePath(workDir, workspace[p]) + if err != nil { + return fmt.Errorf("build: %w", err) + } + edit = "-replace=" + p + "@" + version + "=" + target + case appReplaceTarget(app, p, appDir) != "": + target := appReplaceTarget(app, p, appDir) + if !strings.Contains(target, "@") { + if rel, err := relModulePath(workDir, target); err == nil { + target = rel + } + } + edit = "-replace=" + p + "=" + target + default: + if version == "" { + fmt.Fprintf(out, "warning: cannot pin %s in go.work: no module requires a version of it\n", p) + continue + } + edit = "-replace=" + p + "=" + p + "@" + version + } + replaced[p] = true + edits = append(edits, edit) + } + } + if len(edits) == 0 { + return nil + } + cmd := exec.CommandContext(ctx, "go", append(append([]string{"work", "edit"}, edits...), gw)...) + cmd.Dir = workDir + applyGoWorkFile(cmd, gw) + if b, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("build: go work edit: %w\n%s", err, b) + } + for _, e := range edits { + fmt.Fprintf(out, "go.work: replace %s\n", strings.TrimPrefix(e, "-replace=")) + } + return ensureToolchain(gw) +} + +// appReplaceTarget returns what the app's go.mod replaces path with: an +// absolute directory, or path@version; "" when it does not replace it. +func appReplaceTarget(app goModJSON, p, appDir string) string { + for _, r := range app.Replace { + if r.Old.Path != p { + continue + } + if r.New.Version != "" { + return r.New.Path + "@" + r.New.Version + } + if filepath.IsAbs(r.New.Path) { + return r.New.Path + } + return filepath.Join(appDir, r.New.Path) + } + return "" +}