import { el, emit, fmt, icon, labelsFor, num, uid, uiLocaleOf, type Labels } from './labels' export const ROBOTSTXT_TAG = 'golem15-seo-robotstxt' /** The largest robots.txt the server stores: Google reads 500 KiB. */ export const MAX_ROBOTS_BYTES = 500 * 1024 /** The findings shown before "and :count more". */ export const MAX_FINDINGS = 10 /** The directives the lint knows, as written in the suggestion. */ const DIRECTIVES = ['User-agent', 'Allow', 'Disallow', 'Sitemap', 'Crawl-delay', 'Host', 'Clean-param'] export type FindingKind = 'unknown' | 'unknownSuggest' | 'noGroup' | 'badPath' | 'noColon' | 'ownSitemap' | 'crawlDelay' export interface Finding { kind: FindingKind line: number word?: string suggestion?: string } /** Levenshtein distance of two short strings. */ function distance(a: string, b: string): number { const row = Array.from({ length: b.length + 1 }, (_, i) => i) for (let i = 1; i <= a.length; i++) { let prev = row[0] row[0] = i for (let j = 1; j <= b.length; j++) { const tmp = row[j] row[j] = Math.min(row[j] + 1, row[j - 1] + 1, prev + (a[i - 1] === b[j - 1] ? 0 : 1)) prev = tmp } } return row[b.length] } /** The closest known directive within two edits, or ''. */ function suggest(word: string): string { const lower = word.toLowerCase() let best = '' let bestDistance = 3 for (const d of DIRECTIVES) { const dist = distance(lower, d.toLowerCase()) if (dist < bestDistance) { best = d bestDistance = dist } } return best } /** * Lints a robots.txt (UI-SPEC E4): every finding is a warning that never * blocks a save. Lines are 1-based; comments (#) and blank lines are * skipped. */ export function lintRobots(text: string): Finding[] { const out: Finding[] = [] let group = false text.split(/\r?\n/).forEach((raw, index) => { const line = index + 1 const hash = raw.indexOf('#') const body = (hash >= 0 ? raw.slice(0, hash) : raw).trim() if (body === '') { return } const colon = body.indexOf(':') if (colon < 0) { out.push({ kind: 'noColon', line }) return } const word = body.slice(0, colon).trim() const value = body.slice(colon + 1).trim() switch (word.toLowerCase()) { case 'user-agent': group = true return case 'allow': case 'disallow': if (!group) { out.push({ kind: 'noGroup', line }) } else if (value !== '' && !value.startsWith('/') && !value.startsWith('*')) { out.push({ kind: 'badPath', line }) } return case 'sitemap': out.push({ kind: 'ownSitemap', line }) return case 'crawl-delay': out.push({ kind: 'crawlDelay', line }) return case 'host': case 'clean-param': return } const suggestion = suggest(word) out.push(suggestion ? { kind: 'unknownSuggest', line, word, suggestion } : { kind: 'unknown', line, word }) }) return out } /** The UTF-8 size of text in bytes. */ export function byteSize(text: string): number { return new TextEncoder().encode(text).length } /** The text of a finding in the element's language. */ export function findingText(labels: Labels, f: Finding): string { return fmt(labels[f.kind], { line: f.line, word: f.word ?? '', suggestion: f.suggestion ?? '' }) } /** * E4 golem15-seo-robotstxt (UI-SPEC C10): the robots.txt editor, a * monospaced textarea with a live lint panel (at most ten findings) and a * size meter. Findings never block a save; only a file over 500 KB is * refused, by the field type's Decode. The value is a plain string; the * element emits summer-change on every input. */ export class SeoRobotsTxtElement extends HTMLElement { static observedAttributes = ['disabled', 'invalid', 'ui-locale'] private current = '' private control: HTMLTextAreaElement | null = null private panel: HTMLDivElement | null = null private meter: HTMLDivElement | null = null private builtFor = '' get value(): string { return this.current } set value(v: unknown) { this.current = typeof v === 'string' ? v : v == null ? '' : String(v) if (this.control && this.control.value !== this.current) { this.control.value = this.current } this.update() } connectedCallback(): void { this.build() } attributeChangedCallback(name: string): void { if (!this.isConnected) { return } if (name === 'ui-locale') { this.build() return } this.sync() } private labels(): Labels { return labelsFor(uiLocaleOf(this)) } private build(): void { const key = uiLocaleOf(this) if (this.builtFor === key && this.control) { this.sync() return } this.builtFor = key this.replaceChildren() const panelId = uid('seo-robotstxt-lint') const control = el('textarea', 'seo-robotstxt__control') control.spellcheck = false control.setAttribute('wrap', 'off') control.value = this.current control.setAttribute('aria-describedby', panelId) const label = this.getAttribute('field-name') if (label) { control.setAttribute('aria-label', label) } control.addEventListener('input', () => { if (this.hasAttribute('disabled')) { return } this.current = control.value this.update() emit(this, 'summer-change', { value: this.current }) }) const panel = el('div', 'seo-robotstxt__lint') panel.id = panelId panel.setAttribute('role', 'status') const meter = el('div', 'seo-robotstxt__size') this.control = control this.panel = panel this.meter = meter this.append(control, panel, meter) this.sync() } private sync(): void { if (!this.control) { return } this.control.readOnly = this.hasAttribute('disabled') this.classList.toggle('seo-robotstxt--invalid', this.hasAttribute('invalid')) this.control.setAttribute('aria-invalid', this.hasAttribute('invalid') ? 'true' : 'false') this.update() } private update(): void { if (!this.panel || !this.meter) { return } const labels = this.labels() const locale = uiLocaleOf(this) const findings = lintRobots(this.current) const items: HTMLElement[] = [] if (findings.length === 0) { const item = el('div', 'seo-robotstxt__finding seo-robotstxt__finding--clean') item.append(icon('circle-check'), el('span', '', labels.clean)) items.push(item) } else { for (const f of findings.slice(0, MAX_FINDINGS)) { const item = el('div', 'seo-robotstxt__finding') item.append(icon('triangle-alert'), el('span', '', findingText(labels, f))) items.push(item) } if (findings.length > MAX_FINDINGS) { items.push(el('div', 'seo-robotstxt__more', fmt(labels.moreFindings, { count: num(findings.length - MAX_FINDINGS, locale) }))) } } this.panel.replaceChildren(...items) const bytes = byteSize(this.current) const kb = new Intl.NumberFormat(locale, { maximumFractionDigits: 1 }).format(bytes / 1024) + ' KB' this.meter.replaceChildren(el('span', '', fmt(labels.size, { size: kb }))) const over = bytes > MAX_ROBOTS_BYTES this.classList.toggle('seo-robotstxt--over', over) if (over) { this.meter.append(el('span', 'seo-robotstxt__over', labels.tooLarge)) } } }