package api import ( "context" "net/http" "strings" "time" "git.golem15.com/golem15/sm-sitemap-plugin/classes" "git.golem15.com/golem15/sm-sitemap-plugin/models" "git.golem15.com/golem15/summercms/modules/backpack" "git.golem15.com/golem15/summercms/modules/lagoon" "git.golem15.com/golem15/summercms/modules/phrasebook" "git.golem15.com/golem15/summercms/modules/signpost" "gorm.io/gorm" ) // errUnknownType is the refusal of a types entry for a key no content type // has. const errUnknownType = "golem15.sitemap::lang.errors.type_unknown" // TypeInput is the settings of one content type in UpdateSettingsInput: // only the fields sent change. type TypeInput struct { Include *bool `json:"include,omitempty" jsonschema:"list the type's records in the sitemaps"` Slug *string `json:"slug,omitempty" jsonschema:"the file name part of the type's files (sitemap--.xml): lowercase letters, digits and hyphens; empty for the default"` Titles map[string]string `json:"titles,omitempty" jsonschema:"the type's heading in the HTML sitemap per locale"` } // UpdateSettingsInput is the body of PUT /_sitemap/api/v1/settings: only // the fields sent change, with the checks of the settings screen. type UpdateSettingsInput struct { SiteURL *string `json:"site_url,omitempty" jsonschema:"the public site address every sitemap address starts with, e.g. https://www.example.com; empty for the config value"` FileSize *int `json:"file_size,omitempty" jsonschema:"addresses per sitemap file, 1 to 50000; a change renumbers every file"` ExtraURLs *[]models.ExtraURL `json:"extra_urls,omitempty" jsonschema:"the additional addresses (url: a path or an absolute URL on the site; lastmod: optional YYYY-MM-DD); replaces the list"` ImagesEnabled *bool `json:"images_enabled,omitempty" jsonschema:"list the pages' images in the sitemap-images files"` NewsEnabled *bool `json:"news_enabled,omitempty" jsonschema:"serve the Google News sitemap sitemap-news.xml (only for sites accepted in Google News)"` NewsPublication *string `json:"news_publication,omitempty" jsonschema:"the publication name as registered in Google News; empty for the site name"` NewsSources *[]string `json:"news_sources,omitempty" jsonschema:"the article types and categories of the News sitemap, values of news_source_options (type: or category::); empty for every article type"` IndexNowEnabled *bool `json:"indexnow_enabled,omitempty" jsonschema:"notify IndexNow engines of new, changed and removed addresses after each rebuild (production only); turning it on generates the key"` IndexNowNewKey *bool `json:"indexnow_new_key,omitempty" jsonschema:"generate a new IndexNow key; the old key file stops working at once"` HTMLEnabled *bool `json:"html_enabled,omitempty" jsonschema:"serve the HTML sitemap's data to the site"` HTMLPaths map[string]string `json:"html_paths,omitempty" jsonschema:"the site's HTML sitemap address per locale, e.g. {\"pl\":\"/mapa-strony\"}; a blank value removes the locale"` HTMLPageSize *int `json:"html_page_size,omitempty" jsonschema:"links per HTML sitemap page, 50 to 500"` Types map[string]TypeInput `json:"types,omitempty" jsonschema:"content type settings by type key"` } // settingsValues is the settings part of an answer. func settingsValues(s models.Settings) map[string]any { return map[string]any{ "site_url": models.Text(s.SiteURL), "file_size": s.EffectiveFileSize(), "default_locale": models.Text(s.DefaultLocale), "extra_urls": nonNil(s.Extra()), "images_enabled": s.ImagesEnabled, "news_enabled": s.NewsEnabled, "news_publication": models.Text(s.NewsPublication), "news_sources": nonNil(s.Sources()), "indexnow_enabled": s.IndexNowEnabled, "html_enabled": s.HTMLEnabled, "html_paths": s.EffectiveHTMLPaths(), "html_page_size": s.EffectiveHTMLPageSize(), } } func nonNil[T any](s []T) []T { if s == nil { return []T{} } return s } // fileJSON is one file in an answer, with its public address. type fileJSON struct { Name string `json:"name"` URL string `json:"url"` Count int `json:"count"` LastMod *time.Time `json:"lastmod"` } // settingsBody is the answer of the settings routes: the settings, the // content types (label and settings by key), the files of the index, the // addresses left out by reason, the rebuild state, the IndexNow state and // the values news_sources takes. func settingsBody(ctx context.Context, app *backpack.App, db *gorm.DB) (map[string]any, error) { s, err := classes.LoadSettings(db) if err != nil { return nil, err } site := classes.SiteURL(app, s) reg, _ := app.Lookup[*signpost.Registry]() tr, _ := app.Lookup[*phrasebook.Translator]() types := map[string]any{} if reg != nil { rows, err := classes.LoadTypeSettings(db) if err != nil { return nil, err } for _, typ := range reg.ContentTypes() { cfg := classes.EffectiveType(typ, rows) label := typ.Label if tr != nil { label = tr.Get(ctx, typ.Label, nil) } titles := cfg.Titles if titles == nil { titles = map[string]string{} } types[typ.Key] = map[string]any{"label": label, "include": cfg.Include, "slug": cfg.Slug, "titles": titles} } } list, err := classes.SitemapFiles(db, app, s, time.Now().UTC()) if err != nil { return nil, err } files := make([]fileJSON, 0, len(list)) for _, f := range list { files = append(files, fileJSON{Name: f.Name, URL: site + "/" + f.Name, Count: f.Count, LastMod: f.LastMod}) } leftOut, err := classes.LeftOut(db) if err != nil { return nil, err } sources, err := classes.NewsSources(db, reg, models.Text(s.DefaultLocale)) if err != nil { return nil, err } index := "" if site != "" { index = site + "/sitemap.xml" } return map[string]any{ "settings": settingsValues(s), "types": types, "index_url": index, "files": files, "left_out": leftOut, "built_at": s.BuiltAt, "building": classes.Building(s.RebuildRequestedAt, s.BuiltAt, time.Now()), "indexnow": classes.LoadIndexNowStatus(db, app, s), "news_source_options": sources, }, nil } // database returns the application's database bound to ctx. func database(ctx context.Context, app *backpack.App) (*gorm.DB, error) { db, ok := app.Lookup[*gorm.DB]() if !ok || db == nil { return nil, classes.ErrNoDatabase } return db.WithContext(ctx), nil } // ShowSettings serves GET /_sitemap/api/v1/settings: the sitemap settings, // the content types, the files with their address counts, the left-out // counts, the IndexNow key and last notification (settingsBody). func ShowSettings(app func() *backpack.App) http.HandlerFunc { return manage(app, func(ctx context.Context, a *backpack.App, _ *http.Request) (int, any, error) { db, err := database(ctx, a) if err != nil { return 0, nil, err } body, err := settingsBody(ctx, a, db) if err != nil { return 0, nil, err } return http.StatusOK, map[string]any{"data": body}, nil }) } // UpdateSettings serves PUT /_sitemap/api/v1/settings: the fields sent // change with the checks of the settings screen (422 validation_failed // names the refused fields, types.. for a content type), the // IndexNow key is generated as on the screen and a rebuild is queued once // the change commits. It answers like ShowSettings. func UpdateSettings(app func() *backpack.App) http.HandlerFunc { return manage(app, func(ctx context.Context, a *backpack.App, r *http.Request) (int, any, error) { var in UpdateSettingsInput if err := decode(r, &in); err != nil { return 0, nil, err } gdb, err := database(ctx, a) if err != nil { return 0, nil, err } err = lagoon.Transaction(ctx, gdb, func(ctx context.Context, tx *gorm.DB) error { return applySettings(ctx, a, tx, in) }) if err != nil { return 0, nil, err } body, err := settingsBody(ctx, a, gdb) if err != nil { return 0, nil, err } return http.StatusOK, map[string]any{"data": body}, nil }) } // applySettings writes in inside tx: the settings columns sent, the HTML // sitemap addresses merged per locale, the content types, the IndexNow // key, then queues a rebuild after the commit. func applySettings(ctx context.Context, app *backpack.App, tx *gorm.DB, in UpdateSettingsInput) error { if err := tx.Exec(`INSERT INTO golem15_sitemap_settings (id) VALUES (1) ON CONFLICT (id) DO NOTHING`).Error; err != nil { return err } if err := tx.Exec(`SELECT id FROM golem15_sitemap_settings WHERE id = 1 FOR UPDATE`).Error; err != nil { return err } s, err := classes.LoadSettings(tx) if err != nil { return err } columns := map[string]any{} if in.SiteURL != nil { v := strings.TrimSpace(*in.SiteURL) s.SiteURL = &v columns["site_url"] = v } if in.FileSize != nil { s.FileSize = *in.FileSize columns["file_size"] = *in.FileSize } if in.ExtraURLs != nil { s.ExtraURLs = lagoon.Jsonable[[]models.ExtraURL]{Data: nonNil(*in.ExtraURLs), Valid: true} columns["extra_urls"] = s.ExtraURLs } if in.ImagesEnabled != nil { s.ImagesEnabled = *in.ImagesEnabled columns["images_enabled"] = *in.ImagesEnabled } if in.NewsEnabled != nil { s.NewsEnabled = *in.NewsEnabled columns["news_enabled"] = *in.NewsEnabled } if in.NewsPublication != nil { v := strings.TrimSpace(*in.NewsPublication) s.NewsPublication = &v columns["news_publication"] = v } if in.NewsSources != nil { list := []string{} for _, v := range *in.NewsSources { if v = strings.TrimSpace(v); v != "" { list = append(list, v) } } s.NewsSources = lagoon.Jsonable[[]string]{Data: list, Valid: true} columns["news_sources"] = s.NewsSources } if in.IndexNowEnabled != nil { s.IndexNowEnabled = *in.IndexNowEnabled columns["indexnow_enabled"] = *in.IndexNowEnabled } if in.HTMLEnabled != nil { s.HTMLEnabled = *in.HTMLEnabled columns["html_enabled"] = *in.HTMLEnabled } if in.HTMLPageSize != nil { s.HTMLPageSize = *in.HTMLPageSize columns["html_page_size"] = *in.HTMLPageSize } problems := s.Problems() delete(problems, "html_path") var paths map[string]string if in.HTMLPaths != nil { merged, bad := classes.MergeHTMLPaths(tx, in.HTMLPaths) if bad { problems["html_paths"] = []string{models.HTMLPathProblem} } paths = merged } var reg *signpost.Registry if app != nil { reg, _ = app.Lookup[*signpost.Registry]() } values, names := map[string]any{}, map[string]string{} for key, t := range in.Types { if reg == nil { problems["types."+key] = []string{errUnknownType} continue } if _, ok := reg.ContentType(key); !ok { problems["types."+key] = []string{errUnknownType} continue } base := classes.TypeFieldBase(key) names[base+"include"], names[base+"slug"], names[base+"title"] = "types."+key+".include", "types."+key+".slug", "types."+key+".titles" if t.Include != nil { values[base+"include"] = *t.Include } if t.Slug != nil { values[base+"slug"] = *t.Slug } if t.Titles != nil { values[base+"title"] = t.Titles } } if len(problems) > 0 { return &ProblemsError{Problems: problems} } if len(columns) > 0 { if err := tx.Model(&models.Settings{}).Where("id = ?", 1).Updates(columns).Error; err != nil { return err } } if paths != nil { if err := classes.StoreHTMLPaths(tx, paths); err != nil { return err } } if reg != nil && len(values) > 0 { refusals, err := classes.SaveTypeSettings(tx, reg.ContentTypes(), values) if err != nil { return err } if len(refusals) > 0 { out := map[string][]string{} for field, keys := range refusals { if name, ok := names[field]; ok { field = name } out[field] = keys } return &ProblemsError{Problems: out} } } if _, err := classes.EnsureKey(tx, s.IndexNowEnabled, in.IndexNowNewKey != nil && *in.IndexNowNewKey); err != nil { return err } return classes.RequestRebuild(ctx, app, tx) } // Rebuild serves POST /_sitemap/api/v1/rebuild: it queues a rebuild of the // sitemap files now (the settings screen then shows files.building) and // answers 202 {"data":{"queued":true}}. func Rebuild(app func() *backpack.App) http.HandlerFunc { return manage(app, func(ctx context.Context, a *backpack.App, _ *http.Request) (int, any, error) { gdb, err := database(ctx, a) if err != nil { return 0, nil, err } if err := lagoon.Transaction(ctx, gdb, func(ctx context.Context, tx *gorm.DB) error { return classes.RequestRebuild(ctx, a, tx) }); err != nil { return 0, nil, err } return http.StatusAccepted, map[string]any{"data": map[string]any{"queued": true}}, nil }) }