sm-seo-plugin/assets/src/elements/robotstxt.ts

import { el, emit, fmt, icon, labelsFor, num, uid, uiLocaleOf, type Labels } from './labels'

export const ROBOTSTXT_TAG = 'golem15-seo-robotstxt'

/** The largest robots.txt the server stores: Google reads 500 KiB. */
export const MAX_ROBOTS_BYTES = 500 * 1024

/** The findings shown before "and :count more". */
export const MAX_FINDINGS = 10

/** The directives the lint knows, as written in the suggestion. */
const DIRECTIVES = ['User-agent', 'Allow', 'Disallow', 'Sitemap', 'Crawl-delay', 'Host', 'Clean-param']

export type FindingKind = 'unknown' | 'unknownSuggest' | 'noGroup' | 'badPath' | 'noColon' | 'ownSitemap' | 'crawlDelay'

export interface Finding {
  kind: FindingKind
  line: number
  word?: string
  suggestion?: string
}

/** Levenshtein distance of two short strings. */
function distance(a: string, b: string): number {
  const row = Array.from({ length: b.length + 1 }, (_, i) => i)
  for (let i = 1; i <= a.length; i++) {
    let prev = row[0]
    row[0] = i
    for (let j = 1; j <= b.length; j++) {
      const tmp = row[j]
      row[j] = Math.min(row[j] + 1, row[j - 1] + 1, prev + (a[i - 1] === b[j - 1] ? 0 : 1))
      prev = tmp
    }
  }
  return row[b.length]
}

/** The closest known directive within two edits, or ''. */
function suggest(word: string): string {
  const lower = word.toLowerCase()
  let best = ''
  let bestDistance = 3
  for (const d of DIRECTIVES) {
    const dist = distance(lower, d.toLowerCase())
    if (dist < bestDistance) {
      best = d
      bestDistance = dist
    }
  }
  return best
}

/**
 * Lints a robots.txt (UI-SPEC E4): every finding is a warning that never
 * blocks a save. Lines are 1-based; comments (#) and blank lines are
 * skipped.
 */
export function lintRobots(text: string): Finding[] {
  const out: Finding[] = []
  let group = false
  text.split(/\r?\n/).forEach((raw, index) => {
    const line = index + 1
    const hash = raw.indexOf('#')
    const body = (hash >= 0 ? raw.slice(0, hash) : raw).trim()
    if (body === '') {
      return
    }
    const colon = body.indexOf(':')
    if (colon < 0) {
      out.push({ kind: 'noColon', line })
      return
    }
    const word = body.slice(0, colon).trim()
    const value = body.slice(colon + 1).trim()
    switch (word.toLowerCase()) {
      case 'user-agent':
        group = true
        return
      case 'allow':
      case 'disallow':
        if (!group) {
          out.push({ kind: 'noGroup', line })
        } else if (value !== '' && !value.startsWith('/') && !value.startsWith('*')) {
          out.push({ kind: 'badPath', line })
        }
        return
      case 'sitemap':
        out.push({ kind: 'ownSitemap', line })
        return
      case 'crawl-delay':
        out.push({ kind: 'crawlDelay', line })
        return
      case 'host':
      case 'clean-param':
        return
    }
    const suggestion = suggest(word)
    out.push(suggestion ? { kind: 'unknownSuggest', line, word, suggestion } : { kind: 'unknown', line, word })
  })
  return out
}

/** The UTF-8 size of text in bytes. */
export function byteSize(text: string): number {
  return new TextEncoder().encode(text).length
}

/** The text of a finding in the element's language. */
export function findingText(labels: Labels, f: Finding): string {
  return fmt(labels[f.kind], { line: f.line, word: f.word ?? '', suggestion: f.suggestion ?? '' })
}

/**
 * E4 golem15-seo-robotstxt (UI-SPEC C10): the robots.txt editor, a
 * monospaced textarea with a live lint panel (at most ten findings) and a
 * size meter. Findings never block a save; only a file over 500 KB is
 * refused, by the field type's Decode. The value is a plain string; the
 * element emits summer-change on every input.
 */
export class SeoRobotsTxtElement extends HTMLElement {
  static observedAttributes = ['disabled', 'invalid', 'ui-locale']

  private current = ''
  private control: HTMLTextAreaElement | null = null
  private panel: HTMLDivElement | null = null
  private meter: HTMLDivElement | null = null
  private builtFor = ''

  get value(): string {
    return this.current
  }

  set value(v: unknown) {
    this.current = typeof v === 'string' ? v : v == null ? '' : String(v)
    if (this.control && this.control.value !== this.current) {
      this.control.value = this.current
    }
    this.update()
  }

  connectedCallback(): void {
    this.build()
  }

  attributeChangedCallback(name: string): void {
    if (!this.isConnected) {
      return
    }
    if (name === 'ui-locale') {
      this.build()
      return
    }
    this.sync()
  }

  private labels(): Labels {
    return labelsFor(uiLocaleOf(this))
  }

  private build(): void {
    const key = uiLocaleOf(this)
    if (this.builtFor === key && this.control) {
      this.sync()
      return
    }
    this.builtFor = key
    this.replaceChildren()
    const panelId = uid('seo-robotstxt-lint')
    const control = el('textarea', 'seo-robotstxt__control')
    control.spellcheck = false
    control.setAttribute('wrap', 'off')
    control.value = this.current
    control.setAttribute('aria-describedby', panelId)
    const label = this.getAttribute('field-name')
    if (label) {
      control.setAttribute('aria-label', label)
    }
    control.addEventListener('input', () => {
      if (this.hasAttribute('disabled')) {
        return
      }
      this.current = control.value
      this.update()
      emit(this, 'summer-change', { value: this.current })
    })
    const panel = el('div', 'seo-robotstxt__lint')
    panel.id = panelId
    panel.setAttribute('role', 'status')
    const meter = el('div', 'seo-robotstxt__size')
    this.control = control
    this.panel = panel
    this.meter = meter
    this.append(control, panel, meter)
    this.sync()
  }

  private sync(): void {
    if (!this.control) {
      return
    }
    this.control.readOnly = this.hasAttribute('disabled')
    this.classList.toggle('seo-robotstxt--invalid', this.hasAttribute('invalid'))
    this.control.setAttribute('aria-invalid', this.hasAttribute('invalid') ? 'true' : 'false')
    this.update()
  }

  private update(): void {
    if (!this.panel || !this.meter) {
      return
    }
    const labels = this.labels()
    const locale = uiLocaleOf(this)
    const findings = lintRobots(this.current)
    const items: HTMLElement[] = []
    if (findings.length === 0) {
      const item = el('div', 'seo-robotstxt__finding seo-robotstxt__finding--clean')
      item.append(icon('circle-check'), el('span', '', labels.clean))
      items.push(item)
    } else {
      for (const f of findings.slice(0, MAX_FINDINGS)) {
        const item = el('div', 'seo-robotstxt__finding')
        item.append(icon('triangle-alert'), el('span', '', findingText(labels, f)))
        items.push(item)
      }
      if (findings.length > MAX_FINDINGS) {
        items.push(el('div', 'seo-robotstxt__more', fmt(labels.moreFindings, { count: num(findings.length - MAX_FINDINGS, locale) })))
      }
    }
    this.panel.replaceChildren(...items)
    const bytes = byteSize(this.current)
    const kb = new Intl.NumberFormat(locale, { maximumFractionDigits: 1 }).format(bytes / 1024) + ' KB'
    this.meter.replaceChildren(el('span', '', fmt(labels.size, { size: kb })))
    const over = bytes > MAX_ROBOTS_BYTES
    this.classList.toggle('seo-robotstxt--over', over)
    if (over) {
      this.meter.append(el('span', 'seo-robotstxt__over', labels.tooLarge))
    }
  }
}