sm-seo-plugin/linking_test.go

package seo_test

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strconv"
	"strings"
	"testing"

	"git.golem15.com/golem15/sm-seo-plugin/classes"
	"git.golem15.com/golem15/sm-seo-plugin/models"
	"git.golem15.com/golem15/summercms/modules/lagoon"
	"gorm.io/gorm"
)

// autolinksOf reads the resolve answer's autolinks as phrase → url, and
// its autolink_max (-1 when absent).
func autolinksOf(t *testing.T, got map[string]any) (map[string]string, int) {
	t.Helper()
	out := map[string]string{}
	if list, ok := got["autolinks"].([]any); ok {
		for _, item := range list {
			m, _ := item.(map[string]any)
			phrase, _ := m["phrase"].(string)
			url, _ := m["url"].(string)
			out[phrase] = url
		}
	}
	most := -1
	if n, ok := got["autolink_max"].(float64); ok {
		most = int(n)
	}
	return out, most
}

// numID is a decoded JSON id as text.
func numID(v any) string {
	if n, ok := v.(float64); ok {
		return strconv.FormatFloat(n, 'f', 0, 64)
	}
	if n, ok := v.(json.Number); ok {
		return n.String()
	}
	return ""
}

// saveAutolink writes an auto-link through the service, as the screen and
// the API do.
func (e *testEnv) saveAutolink(t *testing.T, a models.Autolink) models.Autolink {
	t.Helper()
	svc := e.seoPlugin(t).Service()
	err := lagoon.Transaction(t.Context(), e.db, func(ctx context.Context, tx *gorm.DB) error {
		return svc.SaveAutolink(classes.WithTx(ctx, tx), tx, &a)
	})
	if err != nil {
		t.Fatalf("save auto-link %q: %v", a.Phrase, err)
	}
	return a
}

// TestAutolinksTracer is the tracer of D-82's auto-links: an editor adds
// the phrase "lwia grzywa" for every locale, and the next page that
// resolves carries it with autolink_max; the page it links to does not,
// nor does a disabled phrase or a phrase of another locale; autolink_max 0
// answers none.
func TestAutolinksTracer(t *testing.T) {
	env := bootEnv(t)
	token := env.admin(t, true)
	env.expect(t, http.StatusCreated, http.MethodPost, "/_seo/api/v1/pages", token, map[string]any{"url": "/grzyby/lwia-grzywa", "title": "Lwia grzywa"})
	env.createPage(t, token, map[string]any{"title": "Other", "slug": "other", "published": true})

	env.saveAutolink(t, models.Autolink{Phrase: " lwia   grzywa ", URL: "/grzyby/lwia-grzywa", Enabled: true})
	env.saveAutolink(t, models.Autolink{Phrase: "boczniak", URL: "/grzyby/boczniak", Enabled: false})
	env.saveAutolink(t, models.Autolink{Phrase: "lion's mane", URL: "/lions-mane", Locale: "en", Enabled: true})

	links, most := autolinksOf(t, env.resolve(t, "/pl/pages/other"))
	if most != 3 || len(links) != 1 || links["lwia grzywa"] != "/grzyby/lwia-grzywa" {
		t.Fatalf("pl page autolinks = %v, autolink_max %d; want only lwia grzywa and 3", links, most)
	}
	links, _ = autolinksOf(t, env.resolve(t, "/pages/other"))
	if len(links) != 2 || links["lion's mane"] != "/lions-mane" || links["lwia grzywa"] == "" {
		t.Fatalf("en page autolinks = %v, want lwia grzywa and the en phrase", links)
	}
	got := env.resolve(t, "/grzyby/lwia-grzywa")
	if got["action"] != "ok" {
		t.Fatalf("registry page resolves %v", got)
	}
	if links, _ = autolinksOf(t, got); links["lwia grzywa"] != "" {
		t.Fatalf("the target page carries its own phrase: %v", links)
	}
	if got := env.resolve(t, "/pages/missing"); got["autolinks"] != nil || got["autolink_max"] != nil {
		t.Fatalf("a not_found answer carries auto-links: %v", got)
	}

	var revisions int64
	env.db.Raw(`SELECT COUNT(*) FROM golem15_seo_revisions WHERE subject_kind = 'autolink'`).Scan(&revisions)
	if revisions != 3 {
		t.Fatalf("auto-link history rows = %d, want 3", revisions)
	}

	env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{"autolink_max": 0})
	got = env.resolve(t, "/pl/pages/other")
	if links, most = autolinksOf(t, got); most != 0 || len(links) != 0 {
		raw, _ := json.Marshal(got)
		t.Fatalf("autolink_max 0 answers %s", raw)
	}
	rec := env.call(t, http.MethodPut, "/_seo/api/v1/settings", token, map[string]any{"autolink_max": 21})
	if rec.Code != http.StatusUnprocessableEntity {
		t.Fatalf("autolink_max 21 = %d %s, want 422", rec.Code, rec.Body.String())
	}
}

// autolinksAPI is the Auto-links admin controller's API.
var autolinksAPI = adminAPI("/golem15/seo/autolinks")

// TestAutolinks walks the Auto-links screen and its API (UI-SPEC C22,
// D-82, T-33-50): the refusals (short phrase, a phrase the locale has
// naming its address, javascript:, data: and protocol-relative
// addresses), the list by phrase with the disabled row state, labels and
// filters, history, Add focus keyphrases twice (the second run adds
// nothing), the MCP origin, update and delete.
func TestAutolinks(t *testing.T) {
	env := bootEnv(t)
	token := env.admin(t, true)

	refused := func(rec *httptest.ResponseRecorder, field, text string) {
		t.Helper()
		if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), field) || !strings.Contains(rec.Body.String(), text) {
			t.Fatalf("refusal = %d %s, want 422 on %s with %q", rec.Code, rec.Body.String(), field, text)
		}
	}
	refused(env.call(t, http.MethodPost, autolinksAPI, token, map[string]any{"phrase": "ab", "url": "/x"}), "phrase", "Use at least 3 characters.")
	for _, bad := range []string{"javascript:alert(1)", "data:text/html,x", "//evil.example/x", "ftp://example.com/x", "/a b"} {
		refused(env.call(t, http.MethodPost, autolinksAPI, token, map[string]any{"phrase": "boczniak", "url": bad}), "url", "Use a path starting with / or a full http(s) address.")
	}
	env.expect(t, http.StatusCreated, http.MethodPost, autolinksAPI, token, map[string]any{"phrase": "Lwia grzywa", "url": "/grzyby/lwia-grzywa", "locale": "", "enabled": true})
	refused(env.call(t, http.MethodPost, autolinksAPI, token, map[string]any{"phrase": "lwia GRZYWA", "url": "/inna", "locale": "pl", "enabled": true}), "phrase", "This phrase already links to “/grzyby/lwia-grzywa” in this language.")
	env.expect(t, http.StatusCreated, http.MethodPost, autolinksAPI, token, map[string]any{"phrase": "boczniak", "url": "https://example.com/boczniak", "locale": "pl", "enabled": false, "note": "later"})

	list := env.adminList(t, autolinksAPI, token)
	rows := map[string]map[string]any{}
	for _, row := range list.Data {
		rows[row["phrase"].(string)] = row
	}
	if len(list.Data) != 2 || rows["boczniak"] == nil || rows["Lwia grzywa"] == nil {
		t.Fatalf("list = %v, want boczniak and Lwia grzywa", list.Data)
	}
	if rows["boczniak"]["locale"] != "PL" || rows["Lwia grzywa"]["locale"] != "All languages" || rows["boczniak"]["origin"] != "Manual" {
		t.Fatalf("list labels = %v", list.Data)
	}
	off := numID(rows["boczniak"]["id"])
	if states := list.Meta.RowStates[off]; len(states) != 1 || states[0] != "disabled" {
		t.Fatalf("row states = %v, want %s disabled", list.Meta.RowStates, off)
	}
	if states := list.Meta.RowStates[numID(rows["Lwia grzywa"]["id"])]; len(states) != 0 {
		t.Fatalf("the active row has states %v", states)
	}
	if got := env.adminList(t, autolinksAPI+"?filter[active]=true", token); len(got.Data) != 1 || got.Data[0]["phrase"] != "Lwia grzywa" {
		t.Fatalf("active filter = %v", got.Data)
	}
	if got := env.adminList(t, autolinksAPI+"?filter[locale]=all", token); len(got.Data) != 1 || got.Data[0]["phrase"] != "Lwia grzywa" {
		t.Fatalf("locale all filter = %v", got.Data)
	}
	var revisions int64
	env.db.Raw(`SELECT COUNT(*) FROM golem15_seo_revisions WHERE subject_kind = 'autolink'`).Scan(&revisions)
	if revisions != 2 {
		t.Fatalf("history rows = %d, want 2", revisions)
	}

	// The admin form: create through the screen, refused like the API.
	rec := env.call(t, http.MethodPost, autolinksAPI, token, map[string]any{"phrase": "xy", "url": "/x"})
	if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "Use at least 3 characters.") {
		t.Fatalf("admin refusal = %d %s", rec.Code, rec.Body.String())
	}

	// Focus keyphrases: two records with one, one shared with a phrase.
	env.createPage(t, token, map[string]any{"title": "Shiitake", "slug": "shiitake", "published": true})
	env.createPage(t, token, map[string]any{"title": "Lion", "slug": "lion", "published": true})
	env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/seo", token, map[string]any{"url": "/pages/shiitake", "values": map[string]any{"keyphrase": "twardnik japoński"}})
	env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/seo", token, map[string]any{"url": "/pages/lion", "values": map[string]any{"keyphrase": "lwia grzywa"}})
	rec = env.expect(t, http.StatusOK, http.MethodPost, autolinksAPI+"/toolbar/from_keyphrases", token, map[string]any{})
	if !strings.Contains(rec.Body.String(), "1 phrase added from focus keyphrases.") {
		t.Fatalf("from_keyphrases = %s", rec.Body.String())
	}
	rec = env.expect(t, http.StatusOK, http.MethodPost, autolinksAPI+"/toolbar/from_keyphrases", token, map[string]any{})
	if !strings.Contains(rec.Body.String(), "No new focus keyphrases to add.") {
		t.Fatalf("second from_keyphrases = %s", rec.Body.String())
	}
	got := env.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/autolinks?origin=keyphrase", token, nil).Body.String()
	if !jsonContains(got, `"total":1`) || !jsonContains(got, `"phrase":"twardnik japoński"`) || !jsonContains(got, `"url":"/pages/shiitake"`) || !jsonContains(got, `"locale":"en"`) {
		t.Fatalf("keyphrase rows = %s", got)
	}
	if out := env.expect(t, http.StatusOK, http.MethodPost, "/_seo/api/v1/autolinks/from-keyphrases", token, map[string]any{}).Body.String(); !jsonContains(out, `"added":0`) {
		t.Fatalf("API from-keyphrases again = %s", out)
	}

	// MCP: the origin is mcp; update and delete.
	agent := env.principal(t, env.admin(t, false, "golem15.seo.manage"))
	created := env.mcpExpect(t, http.StatusCreated, http.MethodPost, "/_seo/api/v1/autolinks", agent, map[string]any{"phrase": "maitake", "url": "/maitake"})
	data, _ := created["data"].(map[string]any)
	if data["origin"] != "mcp" || data["enabled"] != true || data["locale"] != "" {
		t.Fatalf("MCP create = %v", created)
	}
	id := numID(data["id"])
	updated := env.mcpExpect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/autolinks/"+id, agent, map[string]any{"enabled": false})
	if d, _ := updated["data"].(map[string]any); d["enabled"] != false || d["phrase"] != "maitake" {
		t.Fatalf("MCP update = %v", updated)
	}
	env.mcpExpect(t, http.StatusOK, http.MethodDelete, "/_seo/api/v1/autolinks?ids="+id+","+off, agent, nil)
	if got := env.adminList(t, autolinksAPI, token); len(got.Data) != 2 {
		t.Fatalf("after delete = %v", got.Data)
	}
	env.db.Raw(`SELECT COUNT(*) FROM golem15_seo_revisions WHERE subject_kind = 'autolink'`).Scan(&revisions)
	if revisions != 7 {
		t.Fatalf("history rows = %d, want 7 (2 created, 1 from keyphrases, mcp create, update, 2 deletes)", revisions)
	}
	var vias []string
	env.db.Raw(`SELECT DISTINCT via FROM golem15_seo_revisions WHERE subject_kind = 'autolink' ORDER BY via`).Scan(&vias)
	if strings.Join(vias, ",") != "admin,mcp" {
		t.Fatalf("history vias = %v", vias)
	}
}

// linkReport is GET /_seo/api/v1/links and the seo_links action's answer.
type linkReport struct {
	CrawledAt *string `json:"crawledAt"`
	Inbound   *int    `json:"inbound"`
	Outbound  *int    `json:"outbound"`
	Orphan    bool    `json:"orphan"`
	Out       []struct {
		Phrase string `json:"phrase"`
		Title  string `json:"title"`
		URL    string `json:"url"`
	} `json:"out"`
	In []struct {
		Phrase   string `json:"phrase"`
		Title    string `json:"title"`
		AdminURL string `json:"adminUrl"`
	} `json:"in"`
}

func (e *testEnv) links(t *testing.T, token, path string) linkReport {
	t.Helper()
	var out linkReport
	if err := json.Unmarshal(e.expect(t, http.StatusOK, http.MethodGet, "/_seo/api/v1/links?url="+urlQueryEscape(path), token, nil).Body.Bytes(), &out); err != nil {
		t.Fatal(err)
	}
	return out
}

// TestLinkSuggestions covers E10's answer (UI-SPEC C19, D-82): before the
// first crawl the counts are null and the lists empty; after it, the
// inbound and outbound counts, the orphan state, the pages this text
// mentions by H1 or focus keyphrase and does not link to, and the pages
// whose text mentions this page's title or focus keyphrase and do not link
// here, with their admin forms; the SEO tab's seo_links widget answers the
// same, and E10 is in the SEO tab after the graph preview.
func TestLinkSuggestions(t *testing.T) {
	env := bootCrawlEnv(t)
	token := env.admin(t, true)
	hericium := env.createPage(t, token, map[string]any{"title": "Hericium", "slug": "hericium", "published": true})
	env.createPage(t, token, map[string]any{"title": "Shiitake", "slug": "shiitake", "published": true})
	oyster := env.createPage(t, token, map[string]any{"title": "Oyster", "slug": "oyster", "published": true})
	env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/seo", token, map[string]any{"url": "/pages/hericium", "values": map[string]any{"keyphrase": "lwia grzywa"}})
	env.expect(t, http.StatusOK, http.MethodPut, "/_seo/api/v1/seo", token, map[string]any{"url": "/pages/oyster", "values": map[string]any{"keyphrase": "boczniak"}})

	before := env.links(t, token, "/pages/hericium")
	if before.CrawledAt != nil || before.Inbound != nil || before.Outbound != nil || before.Orphan || len(before.Out) != 0 || len(before.In) != 0 {
		t.Fatalf("before the first crawl = %+v", before)
	}

	site := newFakeSite()
	site.sitemap("/", "/pages/hericium", "/pages/shiitake", "/pages/oyster")
	site.plant(
		doc{path: "/", title: "Home", links: []string{"/pages/shiitake", "/pages/oyster"}},
		doc{path: "/pages/hericium", title: "Hericium", body: "<p>Hericium grows near shiitake and boczniak.</p>"},
		doc{path: "/pages/shiitake", title: "Shiitake", body: "<p>Shiitake differs from lwia grzywa.</p>", links: []string{"/pages/oyster"}},
		doc{path: "/pages/oyster", title: "Oyster", body: "<p>Oyster tastes almost like Hericium.</p>", links: []string{"/pages/shiitake"}},
	)
	env.crawlOnce(t, token, siteCtx(t, site))

	got := env.links(t, token, siteURL+"/pages/hericium")
	if got.CrawledAt == nil || got.Inbound == nil || *got.Inbound != 0 || got.Outbound == nil || *got.Outbound != 0 || !got.Orphan {
		t.Fatalf("hericium counts = %+v", got)
	}
	outs := map[string]string{}
	for _, o := range got.Out {
		outs[o.URL] = o.Phrase + "|" + o.Title
	}
	if len(got.Out) != 2 || outs[siteURL+"/pages/shiitake"] != "Shiitake|Shiitake" || outs[siteURL+"/pages/oyster"] != "boczniak|Oyster" {
		t.Fatalf("hericium out = %+v", got.Out)
	}
	ins := map[string]string{}
	for _, i := range got.In {
		ins[i.Title] = i.Phrase + "|" + i.AdminURL
	}
	if len(got.In) != 2 || ins["Shiitake"] == "" || !strings.HasPrefix(ins["Shiitake"], "lwia grzywa|/backend/acme/pages/pages/update/") || ins["Oyster"] != "Hericium|/backend/acme/pages/pages/update/"+oyster {
		t.Fatalf("hericium in = %+v", got.In)
	}

	// Shiitake: linked from home and oyster, links to oyster (not
	// suggested again although its text names no other page).
	sh := env.links(t, token, "/pages/shiitake")
	if *sh.Inbound != 2 || *sh.Outbound != 1 || sh.Orphan {
		t.Fatalf("shiitake counts = %+v", sh)
	}
	if len(sh.Out) != 1 || sh.Out[0].URL != siteURL+"/pages/hericium" || sh.Out[0].Phrase != "lwia grzywa" {
		t.Fatalf("shiitake out = %+v", sh.Out)
	}
	if len(sh.In) != 1 || sh.In[0].Title != "Hericium" || sh.In[0].AdminURL != "/backend/acme/pages/pages/update/"+hericium {
		t.Fatalf("shiitake in = %+v", sh.In)
	}
	if rec := env.call(t, http.MethodGet, "/_seo/api/v1/links?url=javascript:x", token, nil); rec.Code != http.StatusUnprocessableEntity {
		t.Fatalf("a bad url = %d %s", rec.Code, rec.Body.String())
	}

	// The SEO tab: E10 after the graph preview, its action answers the
	// same report.
	findField(t, env.formSchema(t, token, "acme/pages/pages"), "seo_links")
	n, _ := strconv.Atoi(hericium)
	rec := env.expect(t, http.StatusOK, http.MethodPost, adminAPI("/acme/pages/pages/widgets/seo_links"), token, map[string]any{"record_id": n, "payload": map[string]any{"op": "links", "locale": "en"}})
	var widget struct {
		Data struct {
			Data linkReport `json:"data"`
		} `json:"data"`
	}
	if err := json.Unmarshal(rec.Body.Bytes(), &widget); err != nil {
		t.Fatal(err)
	}
	if w := widget.Data.Data; !w.Orphan || len(w.Out) != 2 || len(w.In) != 2 || w.Inbound == nil {
		t.Fatalf("widget answer = %s", rec.Body.String())
	}
	if rec := env.call(t, http.MethodPost, adminAPI("/acme/pages/pages/widgets/seo_links"), token, map[string]any{"record_id": n, "payload": map[string]any{"op": "graph"}}); rec.Code != http.StatusUnprocessableEntity {
		t.Fatalf("unknown op = %d %s", rec.Code, rec.Body.String())
	}
}