fix: the definitions migration keeps and logs every url item
Migration 202610280002 converts the Phase 17 url items through models.ConvertLegacyItems: a relative path gets its leading slash, spaces and the bytes " < > \ are percent-encoded, and a malformed item no longer drops the whole definition. Each url item that cannot be copied is logged at warn level with its label, value and reason.
This commit is contained in:
Jakub Zych
101
models/legacy.go
101
models/legacy.go
@@ -4,8 +4,11 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Text returns *p, "" for nil.
|
||||
@@ -175,3 +178,101 @@ func phpNumberString(n json.Number) string {
|
||||
}
|
||||
return strconv.FormatFloat(f, 'f', -1, 64)
|
||||
}
|
||||
|
||||
// DroppedItem is a url item of a Phase 17 definition that ConvertLegacyItems
|
||||
// could not copy: its PHP label (index or key; "" for a definition that is
|
||||
// not a list of items), the value as stored and the reason.
|
||||
type DroppedItem struct {
|
||||
Label string
|
||||
URL string
|
||||
Reason string
|
||||
}
|
||||
|
||||
// legacyScheme matches a value that starts with a URL scheme.
|
||||
var legacyScheme = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.\-]*:`)
|
||||
|
||||
// ConvertLegacyItems turns the url items of a Phase 17 definition into
|
||||
// additional addresses (D-32, P33-66; pure). The entries are parsed one by
|
||||
// one: a malformed item is dropped with its parse error and the others are
|
||||
// kept; items of another type (records the content types list) are neither
|
||||
// kept nor reported. A value without a scheme and without a leading / gets
|
||||
// one, as the Phase 17 sitemap resolved it against the site root; spaces
|
||||
// and the bytes " < > \ are percent-encoded. The result must pass
|
||||
// AllowedAddress; one that does not is dropped with the reason (an
|
||||
// absolute URL with & among them). A duplicate is kept once and not
|
||||
// reported.
|
||||
func ConvertLegacyItems(data []byte) ([]ExtraURL, []DroppedItem) {
|
||||
rows := []ExtraURL{}
|
||||
var dropped []DroppedItem
|
||||
data = LegacyJSON(data)
|
||||
if len(data) == 0 || bytes.Equal(data, []byte("null")) {
|
||||
return rows, nil
|
||||
}
|
||||
raws, labels, ok := LegacyEntries(data)
|
||||
if !ok {
|
||||
return rows, []DroppedItem{{Reason: "the definition data is not a list of items"}}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for i, raw := range raws {
|
||||
item, err := ParseLegacyItem(raw)
|
||||
if err != nil {
|
||||
dropped = append(dropped, DroppedItem{Label: labels[i], Reason: err.Error()})
|
||||
continue
|
||||
}
|
||||
if item.Type != LegacyItemURL {
|
||||
continue
|
||||
}
|
||||
u, reason := convertLegacyURL(item.URL)
|
||||
if reason != "" {
|
||||
dropped = append(dropped, DroppedItem{Label: labels[i], URL: item.URL, Reason: reason})
|
||||
continue
|
||||
}
|
||||
if seen[u] {
|
||||
continue
|
||||
}
|
||||
seen[u] = true
|
||||
rows = append(rows, ExtraURL{URL: u})
|
||||
}
|
||||
return rows, dropped
|
||||
}
|
||||
|
||||
// convertLegacyURL is one url item's value as an additional address, or
|
||||
// the reason it cannot be one.
|
||||
func convertLegacyURL(raw string) (string, string) {
|
||||
u := strings.TrimSpace(raw)
|
||||
if u == "" {
|
||||
return "", "the item has no address"
|
||||
}
|
||||
if !utf8.ValidString(u) {
|
||||
return "", "the address is not valid UTF-8"
|
||||
}
|
||||
if !legacyScheme.MatchString(u) && !strings.HasPrefix(u, "/") {
|
||||
u = "/" + u
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range u {
|
||||
switch {
|
||||
case r == '"' || r == '<' || r == '>' || r == '\\' || unicode.IsSpace(r):
|
||||
for _, c := range []byte(string(r)) {
|
||||
fmt.Fprintf(&b, "%%%02X", c)
|
||||
}
|
||||
default:
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
u = b.String()
|
||||
if AllowedAddress(u) {
|
||||
return u, ""
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(u, "//"):
|
||||
return "", "an address starting with // has no scheme"
|
||||
case strings.HasPrefix(u, "/"):
|
||||
return "", "the path holds a control character"
|
||||
case strings.Contains(u, "&"):
|
||||
return "", "an absolute address may not hold &"
|
||||
case strings.ContainsFunc(u, unicode.IsControl):
|
||||
return "", "the address holds a control character"
|
||||
}
|
||||
return "", "only a path or an absolute http(s) address can be an additional address"
|
||||
}
|
||||
|
||||
60
models/legacy_test.go
Normal file
60
models/legacy_test.go
Normal file
@@ -0,0 +1,60 @@
|
||||
package models_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.golem15.com/golem15/sm-sitemap-plugin/models"
|
||||
)
|
||||
|
||||
// TestConvertLegacyItems pins L2 WR-04 (D-32, P33-66): every url item the
|
||||
// Phase 17 sitemap served is copied, a relative path with a leading slash
|
||||
// and spaces and unsafe bytes percent-encoded; a duplicate is kept once
|
||||
// and not reported; an item that still is no allowed address, and a
|
||||
// malformed one, is dropped with its reason while the others are kept;
|
||||
// items of another type are neither kept nor reported.
|
||||
func TestConvertLegacyItems(t *testing.T) {
|
||||
data := `[{"type":"url","url":"o-nas"},` +
|
||||
`{"type":"url","url":"/blog/zażółć gęślą"},` +
|
||||
`{"type":"url","url":"/o-nas"},` +
|
||||
`{"type":"url","url":"https://site.test/x?a=1&b=2"},` +
|
||||
`{"type":"url","url":["not", "text"]},` +
|
||||
`{"type":"acme-post","reference":"7"},` +
|
||||
`{"url":"/a\"b<c>d\\e"},` +
|
||||
`{"type":"url","url":"https://site.test/kontakt"},` +
|
||||
`{"type":"url","url":"javascript:alert(1)"}]`
|
||||
rows, dropped := models.ConvertLegacyItems([]byte(data))
|
||||
var got []string
|
||||
for _, r := range rows {
|
||||
got = append(got, r.URL)
|
||||
if r.LastMod != "" {
|
||||
t.Errorf("a copied address has a lastmod: %+v", r)
|
||||
}
|
||||
}
|
||||
want := "/o-nas /blog/zażółć%20gęślą /a%22b%3Cc%3Ed%5Ce https://site.test/kontakt"
|
||||
if strings.Join(got, " ") != want {
|
||||
t.Fatalf("rows = %q, want %q", strings.Join(got, " "), want)
|
||||
}
|
||||
if len(dropped) != 3 {
|
||||
t.Fatalf("dropped = %+v, want the & URL, the malformed item and javascript:", dropped)
|
||||
}
|
||||
if d := dropped[0]; d.Label != "3" || d.URL != "https://site.test/x?a=1&b=2" || !strings.Contains(d.Reason, "&") {
|
||||
t.Errorf("dropped & URL = %+v", d)
|
||||
}
|
||||
if d := dropped[1]; d.Label != "4" || d.URL != "" || !strings.Contains(d.Reason, "url") {
|
||||
t.Errorf("dropped malformed item = %+v", d)
|
||||
}
|
||||
if d := dropped[2]; d.Label != "8" || d.URL != "javascript:alert(1)" || d.Reason == "" {
|
||||
t.Errorf("dropped javascript: = %+v", d)
|
||||
}
|
||||
|
||||
if rows, dropped := models.ConvertLegacyItems([]byte(`{"a":{"url":"/x"},"b":{"url":"/x"}}`)); len(rows) != 1 || rows[0].URL != "/x" || len(dropped) != 0 {
|
||||
t.Errorf("keyed object = %+v %+v", rows, dropped)
|
||||
}
|
||||
if rows, dropped := models.ConvertLegacyItems([]byte(`"not a list"`)); len(rows) != 0 || len(dropped) != 1 || dropped[0].Label != "" {
|
||||
t.Errorf("not a list = %+v %+v", rows, dropped)
|
||||
}
|
||||
if rows, dropped := models.ConvertLegacyItems(nil); len(rows) != 0 || dropped != nil {
|
||||
t.Errorf("empty = %+v %+v", rows, dropped)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user