sm-sitemap-plugin/console/import.go

package console

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"

	"git.golem15.com/golem15/sm-sitemap-plugin/classes"
	"git.golem15.com/golem15/sm-sitemap-plugin/models"
	"git.golem15.com/golem15/summercms/modules/backpack"
	"git.golem15.com/golem15/summercms/modules/bonfire"
	"gorm.io/gorm"
)

// Import is sitemap:import FILE [--theme=] [--append], the PHP
// ImportCommand: it reads a JSON array of URL entries ({"url": ...,
// "changefreq": ..., "priority": ...}; changefreq defaults to monthly,
// priority to 0.5) into the sitemap definition. Without --append the
// definition keeps its non-url items first and the imported URLs replace
// its url items; with --append the URLs are added after every existing item.
// The definition is always the one /sitemap.xml serves (theme default);
// --theme is accepted for PHP compatibility and a theme other than default
// is reported. An entry whose url is not a path or an http(s) URL
// (classes.AllowedItemURL) is refused, as the admin form refuses it.
func Import(app *backpack.App) bonfire.Command {
	return bonfire.Command{
		Name:        "sitemap:import",
		Description: "Import sitemap URLs from a JSON manifest file",
		Long: `Reads a JSON array of URL entries ({"url": "/path", "changefreq":
"monthly", "priority": "0.5"}; changefreq and priority are optional) and puts
them into the sitemap definition that /sitemap.xml serves, in one
transaction. Every entry needs a url that is a path or an http(s) URL; one
bad entry stops the import before anything is written.

By default the import replaces the definition's url items and keeps its
dynamic items (pages, posts and other generated types). --append adds the
URLs and keeps every existing item. --theme is accepted for the PHP command's
sake: only the default definition is served, so the URLs always go there.
It ends with "Imported N URLs into sitemap definition for theme 'default'."`,
		Args: []bonfire.Arg{
			{Name: "file", Description: "Path to the JSON manifest (relative to the working directory or absolute)", Required: true},
		},
		Flags: []bonfire.Flag{
			{Name: "theme", Description: "Theme code of the PHP plugin; /sitemap.xml serves only the default definition"},
			{Name: "append", Description: "Add URLs without removing existing url-type items", Bare: true},
		},
		Examples: []bonfire.Example{
			{Command: "sitemap:import urls.json", Description: "Replace the url items with the manifest's URLs"},
			{Command: "sitemap:import urls.json --append", Description: "Add the URLs and keep every existing item"},
		},
		Run: func(ctx context.Context, in bonfire.Input, out bonfire.Output) error {
			file, _ := in.Argument("file")
			db, done, err := lookupDB(ctx, app)
			if err != nil {
				return err
			}
			defer done()
			return ImportFile(ctx, db, file, flagValue(in, "theme"), flagOn(in, "append"), out)
		},
	}
}

// ImportFile runs sitemap:import against db.
func ImportFile(ctx context.Context, db *gorm.DB, file, theme string, appendMode bool, out bonfire.Output) error {
	raw, err := os.ReadFile(file)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return fmt.Errorf("File not found: %s", file)
		}
		return fmt.Errorf("Cannot read %s: %w", file, err)
	}
	entries, labels, ok := orderedEntries(raw)
	if !ok {
		return errors.New("Invalid JSON: expected an array of URL entries")
	}
	newItems := make([]models.DefinitionItem, 0, len(entries))
	for i, entry := range entries {
		var fields map[string]json.RawMessage
		_ = json.Unmarshal(entry, &fields)
		url, err := models.DecodeText(fields["url"])
		if fields["url"] == nil || err != nil || phpEmpty(fields["url"]) {
			return fmt.Errorf("Entry %s is missing a 'url' field", labels[i])
		}
		if !classes.AllowedItemURL(models.Text(url)) {
			return fmt.Errorf("Entry %s has a url that is not a path or an http(s) URL", labels[i])
		}
		item := models.DefinitionItem{Type: models.ItemTypeURL, URL: url}
		item.Changefreq = textOr(fields["changefreq"], "monthly")
		item.Priority = textOr(fields["priority"], "0.5")
		newItems = append(newItems, item)
	}
	var kept int
	err = db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		def, err := singletonDefinition(tx)
		if err != nil {
			return err
		}
		existing := def.Items()
		if !appendMode {
			var dynamic []models.DefinitionItem
			for _, item := range existing {
				if item.Type != models.ItemTypeURL {
					dynamic = append(dynamic, item)
				}
			}
			existing = dynamic
		}
		kept = len(existing)
		def.SetItems(append(append([]models.DefinitionItem{}, existing...), newItems...))
		return tx.Save(&def).Error
	})
	if err != nil {
		return err
	}
	out.Success(fmt.Sprintf("Imported %d URLs into sitemap definition for theme '%s'.", len(newItems), models.DefinitionTheme))
	if kept > 0 {
		label := "existing non-URL (dynamic)"
		if appendMode {
			label = "existing"
		}
		out.Info(fmt.Sprintf("Kept %d %s items.", kept, label))
	}
	if theme != "" && theme != models.DefinitionTheme {
		out.Warning(fmt.Sprintf("Theme '%s' was given, but /sitemap.xml serves only the '%s' definition, so the URLs were imported there.", theme, models.DefinitionTheme))
	}
	return nil
}

// orderedEntries splits a JSON array, or a JSON object in key order (a PHP
// array with keys), into its values with their PHP labels (index or key).
func orderedEntries(raw []byte) ([]json.RawMessage, []string, bool) {
	raw = bytes.TrimSpace(raw)
	if len(raw) == 0 {
		return nil, nil, false
	}
	switch raw[0] {
	case '[':
		var list []json.RawMessage
		if err := json.Unmarshal(raw, &list); err != nil {
			return nil, nil, false
		}
		labels := make([]string, len(list))
		for i := range list {
			labels[i] = fmt.Sprint(i)
		}
		return list, labels, true
	case '{':
		dec := json.NewDecoder(bytes.NewReader(raw))
		if _, err := dec.Token(); err != nil {
			return nil, nil, false
		}
		var list []json.RawMessage
		var labels []string
		for dec.More() {
			tok, err := dec.Token()
			if err != nil {
				return nil, nil, false
			}
			key, _ := tok.(string)
			var v json.RawMessage
			if err := dec.Decode(&v); err != nil {
				return nil, nil, false
			}
			list = append(list, v)
			labels = append(labels, key)
		}
		return list, labels, true
	}
	return nil, nil, false
}

// phpEmpty is PHP empty() of a decoded JSON value.
func phpEmpty(raw json.RawMessage) bool {
	var v any
	if err := json.Unmarshal(raw, &v); err != nil {
		return true
	}
	switch t := v.(type) {
	case nil:
		return true
	case string:
		return t == "" || t == "0"
	case bool:
		return !t
	case float64:
		return t == 0
	case []any:
		return len(t) == 0
	case map[string]any:
		return len(t) == 0
	}
	return false
}

// textOr is PHP `$entry[key] ?? fallback` as text: a missing or null value
// is the fallback.
func textOr(raw json.RawMessage, fallback string) *string {
	if raw != nil {
		if s, err := models.DecodeText(raw); err == nil && s != nil {
			return s
		}
	}
	return &fallback
}