sludge.js

/**
 * Client-side Sludge: inference, decoding, provision mapping and remediation, in the browser.
 *
 * Mirrors sludge/model.py (SludgeDetector._decode), sludge/provisions.py (ProvisionMapper.map)
 * and the measured-value half of sludge/remediate.py. The feature vectors come from
 * featurizer.js, which is checked against the Python implementation by
 * scripts/check_web_parity.py.
 *
 * Nothing here decides legality. Every finding is "this provision may be implicated, here is
 * the element that is the factual basis" — the same contract as the CLI.
 */
import { Featurizer, contrastRatio, tokenize } from "./featurizer.js";

const ORT_CDN = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.19.2/dist/";

export class SludgeWeb {
  constructor() {
    this.ready = false;
  }

  async load(base = "assets", onProgress = () => {}) {
    // onProgress(message, fraction|null)
    onProgress("loading metadata");
    const j = async (f) => (await fetch(`${base}/${f}`)).json();
    const [lex, meta, vocab, provisions, examples] = await Promise.all([
      j("lexicons.json"), j("meta.json"), j("vocab.json"),
      j("provisions.json"), j("examples.json"),
    ]);
    this.meta = meta;
    this.provisions = provisions;
    this.examples = examples;
    this.patterns = new Map(meta.patterns.map((p) => [p.id, p]));
    this.fz = new Featurizer(lex, meta, vocab);

    onProgress("loading runtime");
    const ort = globalThis.ort;
    if (!ort) throw new Error("onnxruntime-web did not load");
    ort.env.wasm.wasmPaths = ORT_CDN;
    ort.env.wasm.numThreads = 1;

    // The graph ships in fp32. int8 was measured to flip about a quarter of this model's
    // screen-level decisions and fp16 is rejected by onnxruntime-web's WASM build, so the
    // honest choice is the larger download — reported here rather than hidden behind a spinner.
    const res = await fetch(`${base}/model.onnx`);
    const total = Number(res.headers.get("content-length")) || 0;
    const chunks = [];
    let got = 0;
    const reader = res.body?.getReader();
    if (reader) {
      for (;;) {
        const { done, value } = await reader.read();
        if (done) break;
        chunks.push(value);
        got += value.length;
        onProgress(total
          ? `downloading model — ${(got / 1e6).toFixed(1)} of ${(total / 1e6).toFixed(0)} MB`
          : `downloading model — ${(got / 1e6).toFixed(1)} MB`,
          total ? got / total : null);
      }
    }
    const bytes = new Uint8Array(got);
    let off = 0;
    for (const c of chunks) { bytes.set(c, off); off += c.length; }
    const buf = reader ? bytes.buffer : await res.arrayBuffer();
    onProgress("starting session");
    this.session = await ort.InferenceSession.create(buf, {
      executionProviders: ["wasm"], graphOptimizationLevel: "all",
    });
    this.ort = ort;
    this.ready = true;
    return this;
  }

  /**
   * Run the model on one tree.
   *
   * The graph has a fixed node axis (`max_nodes`), so inputs are padded and masked here. That
   * is verified output-preserving: padded positions are removed from attention by the key
   * padding mask, and the screen-level pooling below uses only the real nodes, so the result
   * matches an unpadded PyTorch run to float noise.
   */
  async predict(rawTree) {
    const f = this.fz.featurize(rawTree);
    const { Tensor } = this.ort;
    const N = this.meta.model.max_nodes, T = f.T, F = f.F;
    const n = Math.min(f.n, N);
    if (f.n > N) {
      console.warn(`tree has ${f.n} elements; the graph takes ${N}, so the tail is ignored`);
    }
    const pad = (src, per, Ctor) => {
      const dst = new Ctor(N * per);
      dst.set(src.subarray(0, n * per));
      return dst;
    };
    const feeds = {
      tokens: new Tensor("int64", pad(f.tokens, T, BigInt64Array), [1, N, T]),
      token_mask: new Tensor("float32", pad(f.tokenMask, T, Float32Array), [1, N, T]),
      roles: new Tensor("int64", pad(f.roles, 1, BigInt64Array), [1, N]),
      numeric: new Tensor("float32", pad(f.numeric, F, Float32Array), [1, N, F]),
      screen: new Tensor("int64", pad(f.screen, 1, BigInt64Array), [1, N]),
      node_mask: new Tensor("float32",
        Float32Array.from({ length: N }, (_, i) => (i < n ? 1 : 0)), [1, N]),
    };
    const t0 = performance.now();
    const out = await this.session.run(feeds);
    const ms = performance.now() - t0;

    const P = this.meta.pattern_ids.length;
    const nodeLogits = out.node_logits.data;   // (1, N, P) - only the first n rows are real
    const flowLogits = out.flow_logits.data;   // (1, P)

    // screen_logits = logsumexp over nodes - log(n) + flow, exactly as PatternDetectorNet does
    const screenProb = new Float32Array(P);
    for (let k = 0; k < P; k++) {
      let mx = -Infinity;
      for (let i = 0; i < n; i++) mx = Math.max(mx, nodeLogits[i * P + k]);
      let sum = 0;
      for (let i = 0; i < n; i++) sum += Math.exp(nodeLogits[i * P + k] - mx);
      const lse = mx + Math.log(sum) - Math.log(Math.max(1, n));
      screenProb[k] = 1 / (1 + Math.exp(-(lse + flowLogits[k])));
    }
    const nodeProb = new Float32Array(n * P);
    for (let i = 0; i < n * P; i++) nodeProb[i] = 1 / (1 + Math.exp(-nodeLogits[i]));
    return { f, screenProb, nodeProb, P, n, ms };
  }

  /** Mirrors SludgeDetector._decode: class above threshold, evidence from the per-node head. */
  decode(pred, minScore = null) {
    const { f, screenProb, nodeProb, P, n } = pred;
    const th = this.meta.thresholds;
    const dets = [];
    for (let k = 0; k < P; k++) {
      const pid = this.meta.pattern_ids[k];
      const thr = minScore === null ? (th.screen[pid] ?? 0.5) : minScore;
      const score = screenProb[k];
      if (score < thr) continue;
      const idx = Array.from({ length: n }, (_, i) => i)
        .sort((a, b) => nodeProb[b * P + k] - nodeProb[a * P + k])
        .slice(0, th.top_k_evidence)
        .filter((i) => nodeProb[i * P + k] >= th.node);
      const picked = idx.length ? idx
        : [Array.from({ length: n }, (_, i) => i)
            .reduce((a, b) => (nodeProb[b * P + k] > nodeProb[a * P + k] ? b : a), 0)];
      dets.push({
        pattern_id: pid, score,
        evidence: picked.map((i) => f.nodeIds[i]),
        scope: this.patterns.get(pid)?.scope || "element",
        screen: f.nodes[picked[0]]?.screen ?? 0,
      });
    }
    dets.sort((a, b) => b.score - a.score);
    return dets;
  }

  /** Mirrors ProvisionMapper.map. Merges by provision, carries evidence through unchanged. */
  mapProvisions(dets, jurisdiction) {
    const table = this.provisions[jurisdiction];
    if (!table) return [];
    const acc = new Map();
    for (const d of dets) {
      for (const prov of table.provisions) {
        const trig = prov.triggers[d.pattern_id];
        if (!trig) continue;
        if (!acc.has(prov.key)) {
          acc.set(prov.key, { prov, elements: new Map(), evidence: [], patterns: [],
                              rationale: [], score: 0 });
        }
        const slot = acc.get(prov.key);
        for (const eid of trig.elements) {
          const el = prov.elements.find((e) => e.id === eid);
          if (el) slot.elements.set(eid, el.requirement);
        }
        for (const nid of d.evidence) if (!slot.evidence.includes(nid)) slot.evidence.push(nid);
        if (!slot.patterns.includes(d.pattern_id)) {
          slot.patterns.push(d.pattern_id);
          slot.rationale.push(`${d.pattern_id}: ${trig.rationale
            || this.patterns.get(d.pattern_id)?.definition || ""}`);
        }
        slot.score = Math.max(slot.score, d.score);
      }
    }
    return [...acc.values()]
      .map((s) => ({
        provision_key: s.prov.key, instrument: s.prov.instrument, citation: s.prov.citation,
        title: s.prov.title, status: s.prov.status, scope_note: s.prov.scope_note,
        status_note: s.prov.status_note,
        elements: [...s.elements].map(([id, requirement]) => ({ id, requirement })),
        evidence: s.evidence, patterns: s.patterns, rationale: s.rationale, score: s.score,
        determination: "may_be_implicated",
      }))
      .sort((a, b) => b.score - a.score || a.provision_key.localeCompare(b.provision_key));
  }

  /**
   * The measured half of RemediationSuggester: advice whose target values are read off this
   * screen. Classes whose fix needs new copy are reported as advisory, never as a mechanical
   * change this page could apply for you.
   */
  remediate(tree, dets) {
    const byId = new Map(tree.nodes.map((n) => [n.node_id, n]));
    const fz = this.fz;
    const label = (n) => fz.label(n);
    const siblings = (n) => tree.nodes.filter(
      (o) => o.parent_id && o.parent_id === n.parent_id && o.node_id !== n.node_id);
    const peerOf = (n) => {
      const s = siblings(n).filter((x) => fz.isInteractive(x));
      const pool = s.length ? s : tree.nodes.filter(
        (x) => x.screen === n.screen && fz.isInteractive(x) && x.node_id !== n.node_id);
      return pool.length ? pool.reduce((a, b) => (fz.area(b) > fz.area(a) ? b : a)) : null;
    };
    const contrastColour = (fg, bg, target) => {
      if (contrastRatio(fg, bg) >= target) return fg;
      const bgLight = (bg[0] + bg[1] + bg[2]) / 765 > 0.5;
      const best = bgLight ? [0, 0, 0] : [255, 255, 255];
      for (let t = 1; t <= 20; t++) {
        const k = t / 20;
        const cand = fg.map((v, i) => Math.round(v + (best[i] - v) * k));
        if (contrastRatio(cand, bg) >= target) return cand;
      }
      return best;
    };

    const out = [];
    for (const d of dets) {
      const n = d.evidence.map((e) => byId.get(e)).find(Boolean);
      if (!n) continue;
      const pid = d.pattern_id;
      let s = null;
      if (pid === "false_hierarchy") {
        const p = peerOf(n);
        if (p) s = { action: "equalise_control_prominence", mechanical: true,
          text: `Render “${label(n)}” at ${p.bbox[2].toFixed(0)}×${p.bbox[3].toFixed(0)} px with `
              + `${p.font_size.toFixed(0)} px type, matching “${label(p)}” (currently `
              + `${n.bbox[2].toFixed(0)}×${n.bbox[3].toFixed(0)} px, ${n.font_size.toFixed(0)} px).`,
          from: [p.node_id] };
      } else if (pid === "visual_interference") {
        const t = contrastColour(n.color, n.background, 4.5);
        s = { action: "restore_text_contrast", mechanical: true,
          text: `Raise “${label(n).slice(0, 44)}” to at least 4.5:1 against its background `
              + `(currently ${contrastRatio(n.color, n.background).toFixed(2)}:1). `
              + `rgb(${t.join(", ")}) reaches `
              + `${contrastRatio(t, n.background).toFixed(2)}:1.` };
      } else if (pid === "hidden_subscription") {
        const prices = tree.nodes.filter(
          (x) => x.screen === n.screen && fz.test("PRICE_RE", label(x)));
        const p = prices.length
          ? prices.reduce((a, b) => (b.font_size > a.font_size ? b : a)) : null;
        if (p) s = { action: "match_recurrence_to_price_prominence", mechanical: true,
          text: `Render the renewal terms at ${p.font_size.toFixed(0)} px weight `
              + `${p.font_weight}, matching the headline price “${label(p)}” (currently `
              + `${n.font_size.toFixed(0)} px weight ${n.font_weight}).`,
          from: [p.node_id] };
      } else if (pid === "preselection" || pid === "pressured_selling") {
        s = { action: "clear_default_selection", mechanical: true,
          text: `Ship “${label(n).slice(0, 48)}” unselected, so the user's own act is what sets it.` };
      } else if (pid === "confirmshaming") {
        s = { action: "neutralise_decline_wording", mechanical: true,
          text: `Replace “${label(n)}” with a neutral label — “No thanks” or “Not now” — `
              + `stating the choice without judging it.` };
      } else if (pid === "hidden_costs") {
        s = { action: "disclose_all_price_components_up_front", mechanical: true,
          text: `Show “${label(n)}” on the first screen where a price appears (step 0), not at `
              + `step ${n.screen}, and include it in the displayed total.` };
      } else if (pid === "dead_end") {
        s = { action: "make_the_path_resolve", mechanical: true,
          text: `“${label(n)}” promises a destination but resolves nowhere. Point it at a screen `
              + `that contains the control it names.` };
      } else if (pid === "hard_to_cancel") {
        const signup = Number(tree.meta?.signup_steps || 2);
        s = { action: "match_cancellation_effort_to_signup", mechanical: false,
          text: `Cancellation takes ${Math.max(1, n.step_cost || 1)} steps against ${signup} to `
              + `sign up. Reduce it to at most ${signup}, in the channel the user signed up in.` };
      } else if (pid === "forced_enrollment") {
        const p = peerOf(n);
        s = { action: "restore_an_equal_decline_path", mechanical: false,
          text: `Offer a working decline control beside “${label(n)}”`
              + (p ? `, at ${p.bbox[2].toFixed(0)}×${p.bbox[3].toFixed(0)} px` : "")
              + `, that completes the user's original task.` };
      } else {
        const advice = {
          countdown_timer: "Replace the timer with the date and time the offer actually ends, and make it survive a reload. If there is no such deadline, remove it.",
          limited_time_message: "State the date the offer ends, or remove the claim.",
          low_stock_message: "State the inventory figure and where it comes from, or remove the claim.",
          high_demand_message: "State the figure and the window it was measured over, or remove the claim.",
          activity_message: "Derive the message from real, time-bounded events, or remove it.",
          testimonials_uncertain_origin: "Show who wrote the review and whether the purchase was verified, next to the quote.",
          trick_question: "Rewrite as one positive statement of what selecting it does; remove the double negation.",
          disguised_ad: "Label the promotional block “Sponsored” inside the ad container, at the ad body's own size and contrast.",
          sneak_into_basket: "Remove the line from the basket unless the user adds it; if offered, offer it unselected.",
          bait_and_switch: "Either point the control at what its label says, or relabel it to name what its target does.",
          hidden_information: "Show the text at the same level as the control it qualifies, not inside a collapsed container.",
          price_comparison_prevention: "Show a per-unit price on every option in the group, on one screen.",
          intermediate_currency: "Show the real-currency price beside the virtual one at the point of purchase.",
          privacy_maze: "Present all privacy controls on one screen reachable in a single step.",
          privacy_zuckering: "Ship every sharing option unselected and put the granular controls beside “Accept all”, at equal prominence.",
          forced_registration: "Offer “Continue as guest” beside account creation for a task that does not require an account.",
          nagging: "Stop re-showing the prompt after a decline; if it must reappear, give it a dismiss control and a long interval.",
        }[pid];
        if (advice) s = { action: "advisory", mechanical: false, text: advice };
      }
      if (s) out.push({ pattern_id: pid, node_id: n.node_id, ...s });
    }
    return out;
  }
}

/** DOM-based port of uitree.tree_from_html, for pasted markup. */
export function treeFromHtml(html) {
  const ROLE_BY_TAG = {
    a: "link", button: "button", input: "textbox", select: "combobox", textarea: "textbox",
    h1: "heading", h2: "heading", h3: "heading", h4: "heading", h5: "heading", h6: "heading",
    img: "image", label: "text", p: "text", span: "text", li: "listitem", ul: "list",
    ol: "list", form: "form", dialog: "dialog", table: "table", strong: "text", em: "text",
    small: "text",
  };
  const INPUT_ROLE = { checkbox: "checkbox", radio: "radio", submit: "button",
                       button: "button", range: "slider" };
  const NAMED = { black: [0,0,0], white: [255,255,255], red: [255,0,0], green: [0,128,0],
    blue: [0,0,255], gray: [128,128,128], grey: [128,128,128], silver: [192,192,192],
    lightgray: [211,211,211], lightgrey: [211,211,211], darkgray: [169,169,169],
    darkgrey: [169,169,169], transparent: [255,255,255], orange: [255,165,0], navy: [0,0,128],
    teal: [0,128,128], purple: [128,0,128], maroon: [128,0,0] };

  function parseColor(v, fallback) {
    if (!v) return fallback;
    const s = String(v).trim().toLowerCase();
    if (NAMED[s]) return NAMED[s];
    let m = /^#([0-9a-f]{3})$/.exec(s);
    if (m) return [...m[1]].map((c) => parseInt(c + c, 16));
    m = /^#([0-9a-f]{6})$/.exec(s);
    if (m) return [0, 2, 4].map((i) => parseInt(m[1].slice(i, i + 2), 16));
    m = /^rgba?\(([^)]*)\)$/.exec(s);
    if (m) {
      const parts = m[1].replace(/\//g, ",").split(",").map((x) => x.trim()).filter(Boolean);
      const pct = parts.slice(0, 3).some((p) => p.endsWith("%"));
      const vals = parts.slice(0, 3).map((p) => parseFloat(p));
      if (vals.every(Number.isFinite)) {
        return pct ? vals.map((x) => Math.round(x * 255 / 100)) : vals.map(Math.round);
      }
    }
    return fallback;
  }
  const decls = (style) => Object.fromEntries(
    (style || "").split(";").filter((d) => d.includes(":"))
      .map((d) => { const i = d.indexOf(":");
                    return [d.slice(0, i).trim().toLowerCase(), d.slice(i + 1).trim()]; }));
  function fontPx(v, inherited) {
    if (!v) return inherited;
    const m = /([\d.]+)\s*(px|pt|rem|em|%)?/.exec(v.trim());
    if (!m) return inherited;
    const n = parseFloat(m[1]);
    return { px: n, pt: n * 4 / 3, rem: n * 16, em: n * inherited,
             "%": inherited * n / 100 }[m[2] || "px"];
  }

  const doc = new DOMParser().parseFromString(html, "text/html");
  const nodes = [];
  let counter = 0, y = 0;

  function walk(el, parentId, depth, inh) {
    if (!el || el.nodeType !== 1) return;
    const tag = el.tagName.toLowerCase();
    if (["script", "style", "head", "meta", "title"].includes(tag)) return;
    const d = decls(el.getAttribute("style"));
    const fontSize = fontPx(d["font-size"], inh.font_size);
    const color = parseColor(d.color, inh.color);
    const background = parseColor(d["background-color"] || d.background, inh.background);
    const wRaw = d["font-weight"] || "";
    const weight = ["bold", "bolder"].includes(wRaw) ? 700
      : (/^\d+$/.test(wRaw) ? parseInt(wRaw, 10)
        : (["strong", "b", "h1", "h2", "h3"].includes(tag) ? 700 : inh.font_weight));

    let role = ROLE_BY_TAG[tag] || "generic";
    if (tag === "input") role = INPUT_ROLE[(el.getAttribute("type") || "text").toLowerCase()]
                                || "textbox";
    role = (el.getAttribute("role") || role).toLowerCase();

    counter += 1;
    const nid = el.id || `n${counter}`;
    const ownText = [...el.childNodes].filter((c) => c.nodeType === 3)
      .map((c) => c.textContent).join("").replace(/\s+/g, " ").trim();

    let width = d.width && d.width.endsWith("px") ? parseFloat(d.width) : 0;
    let height = d.height && d.height.endsWith("px") ? parseFloat(d.height) : 0;
    if (!width) width = Math.max(40, Math.min(360, ownText.length * fontSize * 0.55 + 24));
    if (!height) height = fontSize * (["button", "link"].includes(role) ? 2.6 : 1.4);
    const pad = /([\d.]+)px/.exec(d.padding || "");
    if (pad) height += 2 * parseFloat(pad[1]);

    nodes.push({
      node_id: nid, role, text: ownText, aria_label: el.getAttribute("aria-label") || "",
      bbox: [0, y, width, height], font_size: fontSize, font_weight: weight,
      color, background, depth, screen: 0, parent_id: parentId, children: [],
      default_selected: el.hasAttribute("checked") || el.hasAttribute("selected")
                        || el.getAttribute("aria-checked") === "true",
      disabled: el.hasAttribute("disabled") || el.getAttribute("aria-disabled") === "true",
      hidden: el.hasAttribute("hidden") || d.display === "none" || d.visibility === "hidden",
      aria_hidden: el.getAttribute("aria-hidden") === "true",
      focusable: ["button", "link", "checkbox", "radio", "textbox", "combobox"].includes(role),
      live_region: !!el.getAttribute("aria-live"),
      // A <button> with no href is not a dead end - buttons act, they do not navigate. Only
      // an <a> with no href is a control that promises a destination and has none.
      action: el.getAttribute("href") || el.getAttribute("formaction")
              || el.getAttribute("data-action")
              || (tag === "button" || role === "button" ? "#in-page" : null),
      step_cost: 0, attrs: {},
    });
    y += height;
    const childInh = { font_size: fontSize, color, background, font_weight: weight };
    for (const c of el.children) walk(c, nid, depth + 1, childInh);
  }

  const base = { font_size: 16, color: [17, 17, 17], background: [255, 255, 255],
                 font_weight: 400 };
  for (const c of (doc.body || doc).children) walk(c, null, 0, base);

  const byId = new Map(nodes.map((n) => [n.node_id, n]));
  for (const n of nodes) if (n.parent_id && byId.has(n.parent_id)) {
    byId.get(n.parent_id).children.push(n.node_id);
  }
  return { nodes, screens: [{ index: 0, step_from_entry: 0, goal: "", name: "" }],
           meta: { source: "html" } };
}