diff --git a/examples/hello/plugins/base/lang/en/lang.yaml b/examples/hello/plugins/base/lang/en/lang.yaml index 31faeb4..b172f04 100644 --- a/examples/hello/plugins/base/lang/en/lang.yaml +++ b/examples/hello/plugins/base/lang/en/lang.yaml @@ -1,3 +1,8 @@ greeting: Hello labels: pluginName: Hello plugin +albums: + one: ":count album" + other: ":count albums" +items: "{0} No items|{1} One item|[2,*] :count items" +helloName: "hi :name :Name :NAME" diff --git a/examples/hello/plugins/base/lang/pl/lang.yaml b/examples/hello/plugins/base/lang/pl/lang.yaml index 978bcf5..86e6fb4 100644 --- a/examples/hello/plugins/base/lang/pl/lang.yaml +++ b/examples/hello/plugins/base/lang/pl/lang.yaml @@ -1,3 +1,10 @@ greeting: Cześć labels: pluginName: Wtyczka Hello +albums: + one: ":count album" + few: ":count albumy" + many: ":count albumów" + other: ":count albumu" +posts: ":count wpis|:count wpisy|:count wpisów|:count wpisu" +helloName: "cześć :name :Name :NAME" diff --git a/go.mod b/go.mod index 5c7b1f4..0e09420 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,6 @@ module git.golem15.com/golem15/summercms go 1.27.0 -toolchain go1.27.0 require ( github.com/fsnotify/fsnotify v1.10.1 @@ -14,10 +13,12 @@ require ( github.com/knadh/koanf/providers/env/v2 v2.0.1 github.com/knadh/koanf/providers/file v1.2.1 github.com/knadh/koanf/v2 v2.3.6 + github.com/nicksnyder/go-i18n/v2 v2.6.1 github.com/spf13/cobra v1.10.2 github.com/testcontainers/testcontainers-go v0.44.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 golang.org/x/term v0.46.0 + golang.org/x/text v0.40.0 gorm.io/driver/postgres v1.6.3 gorm.io/gorm v1.31.2 ) @@ -86,6 +87,5 @@ require ( golang.org/x/crypto v0.54.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.48.0 // indirect - golang.org/x/text v0.40.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 688d1ef..d687a63 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8af github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -122,6 +124,8 @@ github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/nicksnyder/go-i18n/v2 v2.6.1 h1:JDEJraFsQE17Dut9HFDHzCoAWGEQJom5s0TRd17NIEQ= +github.com/nicksnyder/go-i18n/v2 v2.6.1/go.mod h1:Vee0/9RD3Quc/NmwEjzzD7VTZ+Ir7QbXocrkhOzmUKA= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= diff --git a/phrasebook/loader.go b/phrasebook/loader.go index efef037..190bfc8 100644 --- a/phrasebook/loader.go +++ b/phrasebook/loader.go @@ -4,7 +4,9 @@ import ( "fmt" "io/fs" "path" + "regexp" "sort" + "strconv" "strings" "unicode" @@ -20,9 +22,30 @@ type Catalog struct { } type entry struct { - text string + text string + plurals map[string]string + pipes []pipePart } +type pipePart struct { + exact *float64 + lo *float64 + hi *float64 + text string + cat string +} + +var cldrNames = map[string]struct{}{ + "zero": {}, + "one": {}, + "two": {}, + "few": {}, + "many": {}, + "other": {}, +} + +var cldrOrder = []string{"zero", "one", "two", "few", "many", "other"} + // NewCatalog returns an empty translation catalog. func NewCatalog() *Catalog { return &Catalog{ @@ -105,11 +128,14 @@ func (c *Catalog) loadFile(pluginID, file, locale, group string, raw []byte) err func flatten(c *Catalog, pluginID, file, locale, prefix string, v any) error { switch val := v.(type) { case string: - return c.put(pluginID, file, locale, prefix, val) + return c.putString(pluginID, file, locale, prefix, val) case map[string]any: if len(val) == 0 { return nil } + if isPluralMap(val) { + return c.putPluralMap(pluginID, file, locale, prefix, val) + } keys := make([]string, 0, len(val)) for k := range val { keys = append(keys, k) @@ -135,7 +161,58 @@ func flatten(c *Catalog, pluginID, file, locale, prefix string, v any) error { } } -func (c *Catalog) put(pluginID, file, locale, key, text string) error { +func isPluralMap(m map[string]any) bool { + if len(m) == 0 { + return false + } + for k := range m { + if _, ok := cldrNames[k]; !ok { + return false + } + } + return true +} + +func (c *Catalog) putString(pluginID, file, locale, key, text string) error { + e := entry{text: text} + if strings.Contains(text, "|") { + parts, err := parsePipe(text, locale) + if err != nil { + return catalogErrorf(pluginID, file, key, "%s", err.Error()) + } + e.pipes = parts + } + return c.put(pluginID, file, locale, key, e) +} + +func (c *Catalog) putPluralMap(pluginID, file, locale, key string, raw map[string]any) error { + if _, ok := raw["other"]; !ok { + return catalogErrorf(pluginID, file, key, "plural map requires other") + } + allowed, err := cldrCategorySet(locale) + if err != nil { + return catalogErrorf(pluginID, file, key, "%s", err.Error()) + } + plurals := make(map[string]string, len(raw)) + keys := make([]string, 0, len(raw)) + for k := range raw { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if _, ok := allowed[k]; !ok { + return catalogErrorf(pluginID, file, key, "invalid plural category %q for locale %s", k, locale) + } + s, ok := raw[k].(string) + if !ok { + return catalogErrorf(pluginID, file, joinKey(key, k), "non-string leaf (%T)", raw[k]) + } + plurals[k] = s + } + return c.put(pluginID, file, locale, key, entry{plurals: plurals}) +} + +func (c *Catalog) put(pluginID, file, locale, key string, e entry) error { if key == "" || strings.HasSuffix(key, "::") || strings.HasSuffix(key, ".") { return catalogErrorf(pluginID, file, key, "empty key") } @@ -147,7 +224,7 @@ func (c *Catalog) put(pluginID, file, locale, key, text string) error { if _, exists := byKey[key]; exists { return catalogErrorf(pluginID, file, key, "duplicate key") } - byKey[key] = entry{text: text} + byKey[key] = e return nil } @@ -247,6 +324,87 @@ func anyMapToStringMap(v map[any]any) (map[string]any, error) { return out, nil } +var ( + pipeExact = regexp.MustCompile(`^\{(-?\d+)\}\s*(.*)$`) + pipeRange = regexp.MustCompile(`^\[(-?\d+),(-?\d+|\*)\]\s*(.*)$`) +) + +func parsePipe(text, locale string) ([]pipePart, error) { + raw := strings.Split(text, "|") + if len(raw) < 2 { + return nil, fmt.Errorf("malformed pipe") + } + cats, _, err := cldrCategories(locale) + if err != nil { + return nil, err + } + parts := make([]pipePart, 0, len(raw)) + var forms []pipePart + for _, seg := range raw { + seg = strings.TrimSpace(seg) + if seg == "" { + return nil, fmt.Errorf("malformed pipe") + } + part, err := parsePipeSegment(seg) + if err != nil { + return nil, err + } + parts = append(parts, part) + if part.exact == nil && part.lo == nil { + forms = append(forms, part) + } + } + if len(forms) > 0 && len(forms) != len(cats) { + return nil, fmt.Errorf("incomplete pipe") + } + fi := 0 + for i := range parts { + if parts[i].exact == nil && parts[i].lo == nil { + parts[i].cat = cats[fi] + fi++ + } + } + return parts, nil +} + +func parsePipeSegment(seg string) (pipePart, error) { + if strings.HasPrefix(seg, "{") { + m := pipeExact.FindStringSubmatch(seg) + if m == nil { + return pipePart{}, fmt.Errorf("malformed pipe") + } + n, err := strconv.ParseFloat(m[1], 64) + if err != nil { + return pipePart{}, fmt.Errorf("malformed pipe") + } + return pipePart{exact: &n, text: m[2]}, nil + } + if strings.HasPrefix(seg, "[") { + m := pipeRange.FindStringSubmatch(seg) + if m == nil { + return pipePart{}, fmt.Errorf("malformed pipe") + } + lo, err := strconv.ParseFloat(m[1], 64) + if err != nil { + return pipePart{}, fmt.Errorf("malformed pipe") + } + part := pipePart{lo: &lo, text: m[3]} + if m[2] == "*" { + return part, nil + } + hi, err := strconv.ParseFloat(m[2], 64) + if err != nil { + return pipePart{}, fmt.Errorf("malformed pipe") + } + if lo > hi { + return pipePart{}, fmt.Errorf("malformed pipe") + } + part.hi = &hi + return part, nil + } + return pipePart{text: seg}, nil +} + func catalogErrorf(pluginID, file, key, format string, args ...any) error { msg := fmt.Sprintf(format, args...) switch { diff --git a/phrasebook/translator.go b/phrasebook/translator.go index 7bc615a..0087917 100644 --- a/phrasebook/translator.go +++ b/phrasebook/translator.go @@ -3,13 +3,18 @@ package phrasebook import ( "context" "fmt" + "strconv" "strings" + "sync" "unicode" "unicode/utf8" "git.golem15.com/golem15/summercms/backpack" "git.golem15.com/golem15/summercms/pact" "git.golem15.com/golem15/summercms/towel" + "github.com/nicksnyder/go-i18n/v2/i18n" + "github.com/nicksnyder/go-i18n/v2/i18n/template" + "golang.org/x/text/language" ) const ( @@ -17,6 +22,26 @@ const ( defaultFallback = "en" ) +var ( + bundleCache sync.Map // locale -> *i18n.Bundle + categoryCache sync.Map // locale -> []string + identityParse = template.IdentityParser{} + pluralSamples = []any{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 20, 21, 22, 23, 24, 25, 100, 101, 102, 111, 1000, 1000000, + "0.0", "0.1", "0.5", "1.0", "1.1", "1.5", "2.0", "2.5", "10.0", "100.0", + } + categoryMessage = &i18n.Message{ + ID: "phrasebook.plural.category", + Zero: "zero", + One: "one", + Two: "two", + Few: "few", + Many: "many", + Other: "other", + } +) + // Options configures an app-scoped translator. type Options struct { Locale string @@ -101,19 +126,216 @@ func (t *Translator) GetIn(locale, key string, params map[string]string) string if !ok { return key } - return interpolate(e.text, params) + text := e.text + if text == "" && e.plurals != nil { + text = e.plurals["other"] + } + return interpolate(text, params) } -// Choice selects a translation for key. Plural selection is added in a later -// task; a non-plural value is returned as Get would. +// Choice selects a plural form for key using CLDR rules or Laravel pipe conditions. func (t *Translator) Choice(ctx context.Context, key string, count any, params map[string]string) string { return t.ChoiceIn(t.localeFrom(ctx), key, count, params) } // ChoiceIn is the explicit-locale form of Choice. func (t *Translator) ChoiceIn(locale, key string, count any, params map[string]string) string { - _ = count - return t.GetIn(locale, key, params) + if t == nil { + return key + } + e, ok := t.cat.lookup(locale, key) + if !ok { + return key + } + text := e.text + switch { + case len(e.pipes) > 0: + text = selectPipe(e.pipes, locale, count) + case e.plurals != nil: + text = selectMap(e.plurals, locale, count) + } + return interpolate(text, withCount(params, count)) +} + +func selectMap(plurals map[string]string, locale string, count any) string { + cat, err := cldrCategory(locale, count) + if err != nil { + return plurals["other"] + } + if s, ok := plurals[cat]; ok { + return s + } + return plurals["other"] +} + +func selectPipe(parts []pipePart, locale string, count any) string { + if n, ok := asFloat(count); ok { + for _, p := range parts { + if p.exact != nil && n == *p.exact { + return p.text + } + if p.lo != nil && n >= *p.lo && (p.hi == nil || n <= *p.hi) { + return p.text + } + } + } + cat, err := cldrCategory(locale, count) + if err != nil { + cat = "other" + } + var other, last string + for _, p := range parts { + last = p.text + if p.cat == cat { + return p.text + } + if p.cat == "other" { + other = p.text + } + } + if other != "" { + return other + } + return last +} + +func cldrCategory(locale string, count any) (string, error) { + if strings.TrimSpace(locale) == "" { + locale = defaultLocale + } + loc := i18n.NewLocalizer(bundleFor(locale), locale) + return loc.Localize(&i18n.LocalizeConfig{ + DefaultMessage: categoryMessage, + PluralCount: coercePluralCount(count), + TemplateParser: identityParse, + }) +} + +func bundleFor(locale string) *i18n.Bundle { + if v, ok := bundleCache.Load(locale); ok { + return v.(*i18n.Bundle) + } + tag, err := language.Parse(locale) + if err != nil { + tag = language.English + } + b := i18n.NewBundle(tag) + actual, _ := bundleCache.LoadOrStore(locale, b) + return actual.(*i18n.Bundle) +} + +func coercePluralCount(count any) any { + switch count.(type) { + case float32, float64: + return formatCount(count) + default: + return count + } +} + +func cldrCategories(locale string) ([]string, map[string]struct{}, error) { + if v, ok := categoryCache.Load(locale); ok { + ordered := v.([]string) + return ordered, setFrom(ordered), nil + } + seen := make(map[string]struct{}) + for _, n := range pluralSamples { + cat, err := cldrCategory(locale, n) + if err != nil { + return nil, nil, err + } + seen[cat] = struct{}{} + } + seen["other"] = struct{}{} + ordered := make([]string, 0, len(cldrOrder)) + for _, cat := range cldrOrder { + if _, ok := seen[cat]; ok { + ordered = append(ordered, cat) + } + } + categoryCache.Store(locale, ordered) + return ordered, seen, nil +} + +func cldrCategorySet(locale string) (map[string]struct{}, error) { + _, set, err := cldrCategories(locale) + return set, err +} + +func setFrom(vals []string) map[string]struct{} { + out := make(map[string]struct{}, len(vals)) + for _, v := range vals { + out[v] = struct{}{} + } + return out +} + +func withCount(params map[string]string, count any) map[string]string { + out := make(map[string]string, len(params)+1) + for k, v := range params { + out[k] = v + } + if _, ok := out["count"]; !ok { + out["count"] = formatCount(count) + } + return out +} + +func formatCount(count any) string { + switch v := count.(type) { + case int: + return strconv.Itoa(v) + case int8: + return strconv.FormatInt(int64(v), 10) + case int16: + return strconv.FormatInt(int64(v), 10) + case int32: + return strconv.FormatInt(int64(v), 10) + case int64: + return strconv.FormatInt(v, 10) + case uint: + return strconv.FormatUint(uint64(v), 10) + case uint64: + return strconv.FormatUint(v, 10) + case float32: + return formatFloat(float64(v)) + case float64: + return formatFloat(v) + default: + return fmt.Sprint(v) + } +} + +func formatFloat(v float64) string { + if v == float64(int64(v)) { + return strconv.FormatInt(int64(v), 10) + } + return strconv.FormatFloat(v, 'f', -1, 64) +} + +func asFloat(count any) (float64, bool) { + switch v := count.(type) { + case int: + return float64(v), true + case int8: + return float64(v), true + case int16: + return float64(v), true + case int32: + return float64(v), true + case int64: + return float64(v), true + case uint: + return float64(v), true + case uint64: + return float64(v), true + case float32: + return float64(v), true + case float64: + return v, true + default: + return 0, false + } } func (t *Translator) localeFrom(ctx context.Context) string { diff --git a/phrasebook/translator_test.go b/phrasebook/translator_test.go index d866f14..4fc4224 100644 --- a/phrasebook/translator_test.go +++ b/phrasebook/translator_test.go @@ -96,3 +96,96 @@ func (p *langPlugin) Requires() []string { return nil } func (p *langPlugin) Register(*backpack.App) error { return nil } func (p *langPlugin) Boot(*backpack.App) error { return nil } func (p *langPlugin) LangFS() fs.FS { return p.fsys } + +func TestPluralSmoke(t *testing.T) { + t.Parallel() + + fsys := fstest.MapFS{ + "lang/en/lang.yaml": {Data: []byte("" + + "albums:\n one: \":count album\"\n other: \":count albums\"\n" + + "items: \"{0} No items|{1} One item|[2,*] :count items\"\n" + + "helloName: \"hi :name :Name :NAME\"\n" + + "raw:\n other: \"{{.Count}} bottles\"\n")}, + "lang/pl/lang.yaml": {Data: []byte("" + + "albums:\n one: \":count album\"\n few: \":count albumy\"\n many: \":count albumów\"\n other: \":count albumu\"\n" + + "posts: \":count wpis|:count wpisy|:count wpisów|:count wpisu\"\n")}, + } + cat := NewCatalog() + if err := cat.Load("golem15.hello", fsys); err != nil { + t.Fatalf("Load: %v", err) + } + tr := NewTranslator(cat, Options{}) + + pl := map[any]string{1: "1 album", 2: "2 albumy", 5: "5 albumów", 22: "22 albumy", 0: "0 albumów", 1.5: "1.5 albumu"} + for n, want := range pl { + if got := tr.ChoiceIn("pl", "golem15.hello::lang.albums", n, nil); got != want { + t.Fatalf("pl albums %v = %q, want %q", n, got, want) + } + } + en := map[any]string{1: "1 album", 2: "2 albums", 0: "0 albums", 1.5: "1.5 albums"} + for n, want := range en { + if got := tr.ChoiceIn("en", "golem15.hello::lang.albums", n, nil); got != want { + t.Fatalf("en albums %v = %q, want %q", n, got, want) + } + } + if got := tr.ChoiceIn("pl", "golem15.hello::lang.posts", 1, nil); got != "1 wpis" { + t.Fatalf("pl posts 1 = %q", got) + } + if got := tr.ChoiceIn("pl", "golem15.hello::lang.posts", 2, nil); got != "2 wpisy" { + t.Fatalf("pl posts 2 = %q", got) + } + if got := tr.ChoiceIn("pl", "golem15.hello::lang.posts", 5, nil); got != "5 wpisów" { + t.Fatalf("pl posts 5 = %q", got) + } + if got := tr.ChoiceIn("pl", "golem15.hello::lang.posts", 22, nil); got != "22 wpisy" { + t.Fatalf("pl posts 22 = %q", got) + } + if got := tr.ChoiceIn("en", "golem15.hello::lang.items", 0, nil); got != "No items" { + t.Fatalf("items 0 = %q", got) + } + if got := tr.ChoiceIn("en", "golem15.hello::lang.items", 1, nil); got != "One item" { + t.Fatalf("items 1 = %q", got) + } + if got := tr.ChoiceIn("en", "golem15.hello::lang.items", 5, nil); got != "5 items" { + t.Fatalf("items 5 = %q", got) + } + if got := tr.ChoiceIn("en", "golem15.hello::lang.helloName", 1, map[string]string{"name": "alice"}); got != "hi alice Alice ALICE" { + t.Fatalf("params = %q", got) + } + if got := tr.ChoiceIn("en", "golem15.hello::lang.raw", 2, nil); got != "{{.Count}} bottles" { + t.Fatalf("go-i18n must not evaluate YAML as a template, got %q", got) + } + + t.Run("invalid category for locale", func(t *testing.T) { + err := NewCatalog().Load("golem15.hello", fstest.MapFS{ + "lang/en/lang.yaml": {Data: []byte("albums:\n few: x\n other: y\n")}, + }) + if err == nil || !strings.Contains(err.Error(), "invalid plural category") { + t.Fatalf("want invalid category, got %v", err) + } + }) + t.Run("missing other", func(t *testing.T) { + err := NewCatalog().Load("golem15.hello", fstest.MapFS{ + "lang/en/lang.yaml": {Data: []byte("albums:\n one: x\n")}, + }) + if err == nil || !strings.Contains(err.Error(), "requires other") { + t.Fatalf("want missing other, got %v", err) + } + }) + t.Run("malformed pipe", func(t *testing.T) { + err := NewCatalog().Load("golem15.hello", fstest.MapFS{ + "lang/en/lang.yaml": {Data: []byte("items: \"{x} nope|{1} one\"\n")}, + }) + if err == nil || !strings.Contains(err.Error(), "malformed pipe") { + t.Fatalf("want malformed pipe, got %v", err) + } + }) + t.Run("incomplete pipe", func(t *testing.T) { + err := NewCatalog().Load("golem15.hello", fstest.MapFS{ + "lang/en/lang.yaml": {Data: []byte("items: \":count post|:count posts|:count extra\"\n")}, + }) + if err == nil || !strings.Contains(err.Error(), "incomplete pipe") { + t.Fatalf("want incomplete pipe, got %v", err) + } + }) +}