#!/usr/bin/env node
/* research/cup-weights/verify.mjs — attack the tool at public/tools/cup-weights/.
 *
 * Seven families, and only the first is a unit test:
 *
 *   1  the engine on arithmetic whose answer is known by hand: fractions, spoon ratios,
 *      the spread, the verdict, the recipe line
 *   2  the shipped data module is whole: every claim names a shipped ingredient and a
 *      fetched source, carries a positive figure and a verbatim quote, states a cup
 *      only where the source stated one, and every cup system carries its document
 *   3  a SECOND derivation of every USDA figure from the raw federal files, by a CSV
 *      reader written separately from the one the extractor used, compared row by row.
 *      This is the family that could catch the whole pipeline being wrong
 *   4  the page says what the data says: the figures its own head names exist in the
 *      table, its self-references are absolute, its JSON-LD parses, and the engine it
 *      imports is the engine this file tested
 *   5  the published dataset agrees with the shipped module, claim for claim
 *   6  negative controls: each family shown failing on purpose, so a green run means
 *      something
 *   7  the page prints this suite's own count, and it has to match
 *
 *   node research/cup-weights/verify.mjs             # families 1-5, 7
 *   node research/cup-weights/verify.mjs --fetch     # adds a live re-read of every quote
 *   node research/cup-weights/verify.mjs --selftest  # family 6
 */
import fs from 'node:fs';
import { execFileSync } from 'node:child_process';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import * as E from '../../public/tools/cup-weights/engine.mjs';
import * as D from '../../public/tools/cup-weights/data.mjs';

const HERE = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.resolve(HERE, '../..');
const TOOL = path.join(ROOT, 'public/tools/cup-weights');
const FETCH = process.argv.includes('--fetch');
const SELFTEST = process.argv.includes('--selftest');

let pass = 0, fail = 0;
const failures = [];
const log = [];
const section = (t) => log.push(`\n  ${t}`);
const ok = (cond, msg) => { if (cond) { pass++; log.push(`    ok   ${msg}`); } else { fail++; failures.push(msg); log.push(`    FAIL ${msg}`); } return !!cond; };
const eq = (a, b, msg) => ok(a === b, `${msg}  (got ${JSON.stringify(a)}, want ${JSON.stringify(b)})`);
const near = (a, b, tol, msg) => ok(Math.abs(a - b) <= tol, `${msg}  (got ${a}, want ${b} ± ${tol})`);

/* ================= 1. the engine ================= */
function familyEngine() {
  section('1 - the engine');
  const S = E.SYSTEMS;
  for (const [s, v] of [['1', 1], ['1.5', 1.5], ['.5', 0.5], ['1/2', 0.5], ['1 1/2', 1.5], ['1-1/2', 1.5], ['1½', 1.5], ['½', 0.5], ['2⅓', 2 + 1 / 3], ['1 and 3/4', 1.75]]) near(E.parseAmount(s), v, 1e-9, `parseAmount("${s}")`);
  for (const s of ['0', 'abc', '', '3/0', '-1', null]) eq(E.parseAmount(s), null, `parseAmount(${JSON.stringify(s)}) refuses`);
  eq(E.formatAmount(0.5), '½', 'formatAmount(0.5)');
  eq(E.formatAmount(1 + 1 / 3), '1⅓', 'formatAmount(4/3)');
  eq(E.formatAmount(2), '2', 'formatAmount(2)');
  eq(E.formatAmount(0.7), '0.7', 'formatAmount(0.7) does not round to a glyph it is not');
  eq(E.formatGrams(124.6), '125', 'formatGrams whole above 20');
  eq(E.formatGrams(12.34), '12.5', 'formatGrams halves between 5 and 20');
  eq(E.formatGrams(4.26), '4.3', 'formatGrams tenths below 5');
  eq(E.formatGrams(1234.5), '1,235', 'formatGrams pins en-US grouping');
  near(S['us-customary'].cup, 236.5882365, 1e-9, 'US customary cup is exactly 8 × 29.5735295625 mL');
  near(S['us-customary'].tbsp * 16, S['us-customary'].cup, 1e-9, '16 US tablespoons make a US cup');
  near(S['us-customary'].tsp * 48, S['us-customary'].cup, 1e-9, '48 US teaspoons make a US cup');
  eq(S['australian'].tbsp, 20, 'Australian tablespoon is 20 mL');
  eq(S['australian'].cup, 250, 'Australian cup is 250 mL');
  eq(S['japanese'].cup, 200, 'Japanese cup is 200 mL');
  eq(S['us-legal'].cup, 240, 'US legal cup is 240 mL');
  eq(E.cupRelation(250, 'metric'), 'same', 'a stated 250 mL cup is the metric cup');
  eq(E.cupRelation(250, 'australian'), 'same', 'a stated 250 mL cup is also the Australian cup');
  eq(E.cupRelation(237, 'us-customary'), 'same', 'a source printing 237 mL means the US customary cup');
  eq(E.cupRelation(236, 'us-customary'), 'same', 'a source printing 236 mL means the US customary cup');
  eq(E.cupRelation(240, 'us-customary'), 'different', '240 mL is not the US customary cup');
  eq(E.cupRelation(250, 'japanese'), 'different', '250 mL is not the Japanese cup');
  eq(E.cupRelation(null, 'metric'), 'none', 'a source that states no cup is never counted as yours');
  eq(E.cupRelation(240, 'nowhere'), 'none', 'an unknown system relates to nothing');
  near(E.amountIn(1, 'cup', 'tsp', S['australian']), 50, 1e-9, 'an Australian cup is 50 teaspoons');
  near(E.amountIn(3, 'tbsp', 'cup', S['australian']), 0.24, 1e-9, 'three Australian tablespoons are 0.24 of a cup');
  near(E.amountIn(3, 'tbsp', 'cup', S['us-customary']), 3 / 16, 1e-9, 'three US tablespoons are 3/16 of a cup');
  const claims = [{ source: 'a', grams_per_unit: 120, cup_ml: 236.59, method: 'spooned' }, { source: 'b', grams_per_unit: 142, cup_ml: null, method: 'dipped' }, { source: 'c', grams_per_unit: 125, cup_ml: null, method: null }];
  const r = E.convert({ amount: 2, unit: 'cup', system: S['us-customary'], claims });
  ok(r.ok, 'convert runs');
  eq(r.rows.map((x) => x.grams).join(','), '240,284,250', 'convert multiplies each figure as printed and never rescales');
  eq(r.verdict, 'disagree', '240 to 284 is a disagreement');
  near(r.spread.spreadPct, (284 - 240) / 250 * 100, 1e-9, 'spread is (max − min) / median'); // em-dash-ok: a minus sign in a formula
  const a = E.convert({ amount: 1, unit: 'cup', system: S['metric'], claims: [{ source: 'x', grams_per_unit: 200 }, { source: 'y', grams_per_unit: 210 }] });
  eq(a.verdict, 'agree', '200 and 210 agree within the stated tolerance');
  eq(E.convert({ amount: 1, unit: 'cup', system: S['metric'], claims: [{ source: 'x', grams_per_unit: 200 }] }).verdict, 'single', 'one claim is single, not agreement');
  eq(E.AGREE_PCT, 8, 'the agreement tolerance is the one the page states');
  const tsp = E.convert({ amount: 2, unit: 'tsp', system: S['us-customary'], claims: [{ source: 'u', grams_per_unit: 4.6 }], base: 'tsp' });
  near(tsp.rows[0].grams, 9.2, 1e-9, 'a per-teaspoon claim converts per teaspoon');
  const v = E.toVolume({ grams: 250, unit: 'cup', system: S['metric'], claims: [{ source: 'x', grams_per_unit: 125 }] });
  near(v.rows[0].amount, 2, 1e-9, '250 g at 125 g a cup is two cups');
  const v2 = E.toVolume({ grams: 125, unit: 'tbsp', system: S['australian'], claims: [{ source: 'x', grams_per_unit: 125 }] });
  near(v2.rows[0].amount, 12.5, 1e-9, '125 g at 125 g a cup is 12.5 Australian tablespoons');
  eq(E.convert({ amount: 0, unit: 'cup', system: S['metric'], claims }).ok, false, 'convert refuses a zero amount');
  const line = E.parseRecipeLine('2 cups all-purpose flour, sifted');
  ok(line && line.amount === 2 && line.unit === 'cup' && line.modifiers.sifted, 'parseRecipeLine reads "2 cups all-purpose flour, sifted"');
  eq(E.parseRecipeLine('- ½ cup packed brown sugar').modifiers.packed, true, 'parseRecipeLine reads a bullet, a vulgar fraction and "packed"');
  eq(E.parseRecipeLine('1 cup (120 g) flour').gramsPrinted, 120, 'parseRecipeLine keeps a gram figure the recipe already printed');
  eq(E.parseRecipeLine('3 large eggs'), null, 'parseRecipeLine leaves a line with no volume unit alone');
  eq(E.parseRecipeLine('salt to taste'), null, 'parseRecipeLine leaves a line with no amount alone');
  eq(E.normaliseUnit('Tbsp.'), 'tbsp', 'normaliseUnit reads "Tbsp."');
  const ings = [{ id: 'flour-ap', aliases: ['all-purpose flour', 'flour'] }, { id: 'flour-ww', aliases: ['whole wheat flour'] }];
  eq(E.matchIngredient('whole wheat flour, sifted', ings).ingredient.id, 'flour-ww', 'the longest alias wins');
  eq(E.matchIngredient('flour', ings).ingredient.id, 'flour-ap', 'a bare word matches its alias');
  eq(E.matchIngredient('eggs', ings), null, 'no alias, no match, no guess');
  eq(E.claimsForModifiers([{ method: 'packed' }, { method: 'loose' }], { packed: true }).length, 1, 'a packed line keeps only the packed claim');
  eq(E.claimsForModifiers([{ method: null }], { packed: true }).length, 1, 'with no packed claim, nothing is dropped');
  eq(E.systemForLocale('en-AU'), 'australian', 'en-AU guesses Australian spoons');
  eq(E.systemForLocale('en-US'), 'us-customary', 'en-US guesses US customary');
  eq(E.systemForLocale('ja-JP'), 'japanese', 'ja guesses the Japanese cup');
}

/* ================= 2. the data module ================= */
const METHODS = new Set(['spooned', 'scooped', 'dipped', 'sifted', 'unsifted', 'packed', 'loose', 'measured']);
const KINDS = new Set(['government', 'test-kitchen', 'mill', 'maker', 'publisher', 'blog', 'measurement', 'standard', 'document', 'forum']);
function familyData(d = D) {
  section('2 - the shipped data module');
  const ING = new Map(d.INGREDIENTS.map((i) => [i.id, i]));
  const SRC = new Map(d.SOURCES.map((s) => [s.id, s]));
  ok(d.INGREDIENTS.length >= 60, `${d.INGREDIENTS.length} ingredients are shipped`);
  ok(d.CLAIMS.length >= d.INGREDIENTS.length, `${d.CLAIMS.length} claims cover them`);
  eq(new Set(d.INGREDIENTS.map((i) => i.id)).size, d.INGREDIENTS.length, 'ingredient ids are unique');
  eq(new Set(d.CLAIMS.map((c) => c.id)).size, d.CLAIMS.length, 'claim ids are unique');
  eq(new Set(d.SOURCES.map((s) => s.id)).size, d.SOURCES.length, 'source ids are unique');
  let bad = 0;
  for (const c of d.CLAIMS) {
    const s = SRC.get(c.source);
    const i = ING.get(c.ingredient);
    if (!s || !i) { bad++; continue; }
    if (!(c.grams_per_unit > 0) || !c.printed || !c.url) bad++;
    if ((i.unit || 'cup') !== c.unit) bad++;
    if (c.method !== null && !METHODS.has(c.method)) bad++;
  }
  eq(bad, 0, 'every claim names a shipped ingredient and source, a positive figure, a quote, a URL, its unit and an allowed method');
  const claimed = new Set(d.CLAIMS.map((c) => c.ingredient));
  eq(d.INGREDIENTS.filter((i) => !claimed.has(i.id)).length, 0, 'no shipped ingredient is without a claim (the page would only refuse it)');
  let srcBad = 0, silentCup = 0;
  for (const s of d.SOURCES) {
    if (!s.id || !s.title || !s.publisher || !s.url || !s.accessed || !KINDS.has(s.kind)) srcBad++;
    if (!/^\d{4}-\d{2}-\d{2}$/.test(String(s.accessed))) srcBad++;
    if (s.cup_ml && !s.cup_quote && s.kind !== 'government') silentCup++;
    if (s.kind !== 'government' && s.kind !== 'document' && s.kind !== 'standard' && s.http_status !== 200) srcBad++;
  }
  eq(srcBad, 0, 'every source has an id, title, publisher, URL, a dated read, an allowed kind, and a 200 where it was fetched');
  eq(silentCup, 0, 'no chart is given a cup volume it did not state in words');
  const usdaSrc = d.SOURCES.filter((s) => /^usda-(sr-legacy|foundation)$/.test(s.id));
  eq(usdaSrc.map((s) => s.cup_ml).filter(Boolean).length, 0, 'USDA sources carry no cup volume, because the files state none');
  eq(d.CLAIMS.filter((c) => /^usda-/.test(c.source) && c.method && c.method !== 'packed' && c.method !== 'loose' && c.method !== 'sifted' && c.method !== 'unsifted').length, 0, 'a USDA claim carries a method only where USDA\'s own modifier says packed, unpacked, sifted or unsifted');
  for (const id of E.SYSTEM_ORDER) {
    const docs = d.SYSTEM_SOURCES.filter((x) => x.system === id);
    ok(docs.length >= 1 && docs.every((x) => x.quote && x.url && x.accessed && x.publisher), `the ${E.SYSTEMS[id].label} cup carries a defining document with a quote (${docs.length})`);
  }
  ok(d.SAMPLES.length >= 100, `${d.SAMPLES.length} USDA sample rows are shipped`);
  eq(d.SAMPLES.filter((s) => !claimed.has(s.ingredient) || !(s.grams_per_cup > 0) || !s.url).length, 0, 'every sample row belongs to a shipped ingredient and carries a figure and a URL');
  const flour = d.CLAIMS.filter((c) => c.ingredient === 'flour-ap');
  ok(flour.length >= 5, `all-purpose flour has ${flour.length} claims`);
  ok(new Set(flour.map((c) => c.source)).size >= 10, `all-purpose flour has figures from ${new Set(flour.map((c) => c.source)).size} distinct sources`);
  eq(new Set(d.CLAIMS.map((c) => `${c.source}|${c.ingredient}|${c.method || ''}|${c.id}`)).size, d.CLAIMS.length, 'no source states the same figure twice for one ingredient and method');
  const measured = d.CLAIMS.filter((c) => c.method === 'measured');
  ok(measured.length >= 2, `${measured.length} claims come from a source that weighed the cup itself`);
  eq(d.USDA_RELEASES.length, 2, 'both USDA releases are named with their digests');
  ok(d.USDA_RELEASES.every((r) => /^[0-9a-f]{64}$/.test(r.sha256)), 'each release digest is a sha256');
}

/* ================= 3. the second derivation ================= */
// A deliberately different CSV reader: line-oriented with a regex, where lib.mjs is a
// character state machine. If both read the same gram weights the number is USDA's.
function readCsvSeparately(file) {
  const text = fs.readFileSync(file, 'utf8').replace(/\r\n?/g, '\n');
  const lines = [];
  let buf = '';
  let open = false;
  for (const line of text.split('\n')) {
    buf = buf ? buf + '\n' + line : line;
    const quotes = (buf.match(/"/g) || []).length;
    open = quotes % 2 === 1;
    if (!open) { lines.push(buf); buf = ''; }
  }
  const cell = /("((?:[^"]|"")*)"|[^,]*)(,|$)/g;
  const rows = lines.filter((l) => l.length).map((l) => {
    const out = [];
    cell.lastIndex = 0;
    let m;
    while ((m = cell.exec(l)) !== null) {
      out.push(m[2] !== undefined ? m[2].replace(/""/g, '"') : m[1]);
      if (m[3] === '') break;
    }
    return out;
  });
  const head = rows[0];
  return rows.slice(1).map((r) => Object.fromEntries(head.map((h, i) => [h, r[i] ?? ''])));
}

function familyUsda() {
  section('3 - a second derivation from the raw federal files');
  const sources = JSON.parse(fs.readFileSync(path.join(HERE, 'sources.json'), 'utf8'));
  const dirs = sources.releases.map((r) => path.join(HERE, 'data', r.dir));
  if (!dirs.every((d) => fs.existsSync(path.join(d, 'food_portion.csv')))) {
    log.push('    skip the raw USDA files are not in research/cup-weights/data/; run fetch.mjs to enable this family');
    return false;
  }
  const jsonl = fs.readFileSync(path.join(HERE, 'out/usda-portions.jsonl'), 'utf8').trim().split('\n').map((l) => JSON.parse(l));
  const derived = [];
  {
    const dir = dirs[0];
    const food = new Map(readCsvSeparately(path.join(dir, 'food.csv')).map((r) => [r.fdc_id, r]));
    for (const p of readCsvSeparately(path.join(dir, 'food_portion.csv'))) {
      const m = String(p.modifier).trim().toLowerCase();
      const unit = /^cups?\b/.test(m) ? 'cup' : /^(tbsps?|tablespoons?)\b/.test(m) ? 'tbsp' : /^(tsps?|teaspoons?)\b/.test(m) ? 'tsp' : null;
      if (!unit || !food.has(p.fdc_id)) continue;
      const amount = Number(p.amount), grams = Number(p.gram_weight);
      if (!(amount > 0) || !(grams > 0)) continue;
      derived.push(`sr_legacy|${p.fdc_id}|${unit}|${amount}|${p.modifier}|${grams}`);
    }
  }
  {
    const dir = dirs[1];
    const food = new Map(readCsvSeparately(path.join(dir, 'food.csv')).map((r) => [r.fdc_id, r]));
    const units = { 1000: 'cup', 1001: 'tbsp', 1002: 'tsp' };
    for (const p of readCsvSeparately(path.join(dir, 'food_portion.csv'))) {
      const unit = units[p.measure_unit_id];
      const f = food.get(p.fdc_id);
      if (!unit || !f || (f.data_type !== 'foundation_food' && f.data_type !== 'market_acquisition')) continue;
      const amount = Number(p.amount), grams = Number(p.gram_weight);
      if (!(amount > 0) || !(grams > 0)) continue;
      derived.push(`foundation|${p.fdc_id}|${unit}|${amount}|${p.modifier || ''}|${grams}`);
    }
  }
  const shipped = jsonl.map((r) => `${r.release}|${r.fdc_id}|${r.unit}|${r.amount}|${r.modifier || ''}|${r.gram_weight}`);
  eq(derived.length, shipped.length, `the second reader finds the same number of cup, tablespoon and teaspoon rows (${derived.length})`);
  const count = (arr) => { const m = new Map(); for (const k of arr) m.set(k, (m.get(k) || 0) + 1); return m; };
  const a = count(derived), b = count(shipped);
  let mismatch = 0;
  for (const [k, n] of a) if (b.get(k) !== n) mismatch++;
  for (const [k, n] of b) if (a.get(k) !== n) mismatch++;
  eq(mismatch, 0, 'every (release, food, unit, amount, modifier, gram weight) agrees between the two readers, as a multiset');
  // every shipped USDA claim and sample is a real row of the raw file
  const raw = new Set(derived.map((k) => k.split('|').slice(0, 3).concat(k.split('|').slice(5)).join('|')));
  let missing = 0;
  for (const c of D.CLAIMS.filter((x) => /^usda-/.test(x.source))) {
    const rel = c.source === 'usda-sr-legacy' ? 'sr_legacy' : 'foundation';
    const amountMatch = derived.some((k) => { const [r, f, u, amt, , g] = k.split('|'); return r === rel && Number(f) === c.fdc_id && u === c.unit && Math.abs(Number(g) / Number(amt) - c.grams_per_unit) < 1e-6; });
    if (!amountMatch) missing++;
    for (const also of c.also || []) {
      const ok2 = derived.some((k) => { const [r, f, u, amt, , g] = k.split('|'); return r === rel && Number(f) === also.fdc_id && u === c.unit && Math.abs(Number(g) / Number(amt) - also.grams_per_unit) < 1e-6; });
      if (!ok2) missing++;
    }
  }
  eq(missing, 0, 'every USDA claim on the page, and every "also lists" row, is a row of the raw federal file at that gram weight');
  let missingSamples = 0;
  for (const s of D.SAMPLES) {
    const hit = derived.some((k) => { const [r, f, u, amt, , g] = k.split('|'); return r === 'foundation' && Number(f) === s.fdc_id && u === 'cup' && Math.abs(Number(g) / Number(amt) - s.grams_per_cup) < 1e-6; });
    if (!hit) missingSamples++;
  }
  eq(missingSamples, 0, `every one of the ${D.SAMPLES.length} USDA sample rows is a row of the raw Foundation Foods file`);
  void raw;
  // the long-tail table the page fetches is the same extraction
  const usdaJson = JSON.parse(fs.readFileSync(path.join(TOOL, 'usda.json'), 'utf8'));
  eq(usdaJson.rows.length, shipped.length, 'usda.json carries every extracted row');
  const col = Object.fromEntries(usdaJson.columns.map((c, i) => [c, i]));
  const keyed = count(usdaJson.rows.map((r) => `${r[col.release]}|${r[col.fdc_id]}|${r[col.unit]}|${r[col.amount]}|${r[col.modifier] || ''}|${r[col.gram_weight]}`));
  let jm = 0; for (const [k, n] of keyed) if (a.get(k) !== n) jm++;
  eq(jm, 0, 'usda.json agrees with the second reader row for row');
  return true;
}

/* ================= 4. the page ================= */
function familyPage() {
  section('4 - the page says what the data says');
  const html = fs.readFileSync(path.join(TOOL, 'index.html'), 'utf8');
  const app = fs.readFileSync(path.join(TOOL, 'app.mjs'), 'utf8');
  const engine = fs.readFileSync(path.join(TOOL, 'engine.mjs'), 'utf8');
  const meta = /<meta name="description" content="([^"]+)"/.exec(html);
  ok(meta, 'the page has a meta description');
  const figures = (meta ? meta[1] : '').match(/\b(\d{3})\b/g) || [];
  const flourFigures = new Set(D.CLAIMS.filter((c) => c.ingredient === 'flour-ap').map((c) => String(Math.round(c.grams_per_unit))));
  for (const f of figures.slice(0, 4)) ok(flourFigures.has(f), `the description's "${f}" grams is a real all-purpose flour figure in the table`);
  ok(/href="\/tools\/cup-weights\/app\.mjs"|src="\/tools\/cup-weights\/app\.mjs"/.test(html), 'the page loads its module by an absolute path (the short host serves it at /)');
  ok(!/src="\.\/|href="\.\//.test(html), 'no relative self-reference in the page');
  ok(/from '\/tools\/cup-weights\/engine\.mjs'/.test(app) && /from '\/tools\/cup-weights\/data\.mjs'/.test(app), 'the app imports the engine and data by absolute paths');
  ok(/const HOME = '\/tools\/cup-weights'/.test(app) && /\$\{HOME\}\/usda\.json/.test(app), 'the long tail is fetched from an absolute path');
  const ld = /<script type="application\/ld\+json">([\s\S]*?)<\/script>/.exec(html);
  let ldOk = false;
  try { const j = JSON.parse(ld[1]); ldOk = j['@type'] === 'WebApplication' && j.url === 'https://artwaste.land/tools/cup-weights/' && !!j.datePublished; } catch { ldOk = false; }
  ok(ldOk, 'the JSON-LD parses and names the canonical URL');
  ok(/<link rel="canonical" href="https:\/\/artwaste\.land\/tools\/cup-weights\/"/.test(html), 'the canonical link is the tool\'s own URL');
  ok(/aw-wayout/.test(html), 'the canonical way-out row is present');
  ok(/home-banner\.js/.test(html), 'the site bar is loaded');
  ok(!/scale/.test(engine.replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/.*$/gm, '')) || !/\bscale\s*[:=]/.test(engine), 'the engine has no rescaling parameter left in code');
  ok(!/gramsPerReaderCup/.test(engine) && !/gramsPerReaderCup/.test(app), 'the rescaling function is gone from engine and page');
  ok(/AGREE_PCT/.test(app) && /agree-pct/.test(html), 'the page prints the agreement tolerance from the engine constant');
  ok(/cupRelation\(/.test(app) && /id="only-my-cup"/.test(html), 'the page offers to narrow to sources that state your cup, through the engine relation');
  ok(/\.filter\(\(r\) => relOf\(r\.claim\) === 'same'\)/.test(app) && !/(perUnit|grams_per_unit|\bgrams)\s*\*[^;\n]*\.cup\b/.test(app) && !/\.cup\b[^;\n]*\*\s*(r\.perUnit|grams_per_unit)/.test(app), 'narrowing is a filter on the stated-cup relation; no figure is multiplied by a cup volume');
  ok(/toLocaleString\('en-US'/.test(app) && !/toLocaleString\(\)/.test(app) && !/toLocaleString\(\)/.test(engine), 'every locale-sensitive format names its locale');
  ok(!/—/.test(html.replace(/<q>[\s\S]*?<\/q>/g, '')) && !/—/.test(app.replace(/'[^']*'/g, '').replace(/`[^`]*`/g, '')), 'no em dash in the page or app prose');
  const checks = /const CHECKS = (\d+);/.exec(app);
  ok(checks, 'the app declares the number of checks this suite runs');
  return checks ? Number(checks[1]) : null;
}

/* ================= 5. the published datasets ================= */
function familyPublished() {
  section('5 - the published datasets agree with the page');
  const dir = path.join(ROOT, 'public/data/cup-weights');
  const udir = path.join(ROOT, 'public/data/usda-cup-portions');
  if (!fs.existsSync(path.join(dir, 'manifest.json')) || !fs.existsSync(path.join(udir, 'manifest.json'))) { log.push('    skip /data/cup-weights/ or /data/usda-cup-portions/ is not published yet'); return; }
  const rows = fs.readFileSync(path.join(dir, 'claims.jsonl'), 'utf8').trim().split('\n').map((l) => JSON.parse(l));
  const chart = D.CLAIMS.filter((c) => !/^usda-/.test(c.source));
  eq(rows.length, chart.length, `the published claims file has one row per shipped chart claim (${rows.length})`);
  const byId = new Map(rows.map((r) => [r.id, r]));
  let drift = 0;
  for (const c of chart) { const r = byId.get(c.id); if (!r || Math.abs(r.grams_per_unit - c.grams_per_unit) > 1e-9 || r.as_named !== c.as_named || r.source !== c.source || r.method !== c.method || (r.cup_ml_stated ?? null) !== (c.cup_ml ?? null)) drift++; }
  eq(drift, 0, 'every published claim carries the same figure, label, source, method and stated cup as the page');
  ok(!rows.some((r) => 'printed' in r || 'quote' in r), 'the published claims carry no verbatim quotation (the licence the room could not establish)');
  const srcs = JSON.parse(fs.readFileSync(path.join(dir, 'sources.json'), 'utf8'));
  ok(srcs.sources.every((s) => s.licence_id && s.licence_basis), 'every published source carries a licence id and its basis');
  const urows = fs.readFileSync(path.join(udir, 'portions.jsonl'), 'utf8').trim().split('\n').map((l) => JSON.parse(l));
  const jsonl = fs.readFileSync(path.join(HERE, 'out/usda-portions.jsonl'), 'utf8').trim().split('\n').map((l) => JSON.parse(l));
  eq(urows.length, jsonl.length, `the published USDA portions file has one row per extracted portion (${urows.length})`);
  let udrift = 0;
  for (let i = 0; i < Math.min(urows.length, jsonl.length); i++) { const a = urows[i], b = jsonl[i]; if (a.fdc_id !== b.fdc_id || a.gram_weight !== b.gram_weight || a.amount !== b.amount || (a.modifier ?? null) !== (b.modifier ?? null) || a.unit !== b.unit) udrift++; }
  eq(udrift, 0, 'the published USDA rows are the extracted rows, in order, figure for figure');
  const usrc = JSON.parse(fs.readFileSync(path.join(udir, 'sources.json'), 'utf8'));
  ok(usrc.sources.filter((s) => s.sha256).every((s) => D.USDA_RELEASES.some((r) => r.sha256 === s.sha256)), 'the published USDA sources carry the same release digests the page names');
  const samplesOnPage = D.SAMPLES.length;
  const samplesPublished = urows.filter((r) => r.data_type === 'market_acquisition' && r.unit === 'cup').length;
  ok(samplesPublished >= samplesOnPage, `the ${samplesOnPage} sample rows the page draws are among the ${samplesPublished} market-acquisition cup rows published`);
}

/* ================= --fetch: the quotes, re-read live ================= */
async function familyFetch() {
  section('live - every quote re-read from its page');
  // norm: tags out, entities decoded, soft hyphens (which King Arthur's table carries
  // inside "tea\u00ADspoon") removed, curly quotes straightened, whitespace collapsed.
  const ENT = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', rsquo: "'", lsquo: "'", rdquo: '"', ldquo: '"', frac12: '1/2', frac14: '1/4', frac34: '3/4', frac13: '1/3', frac23: '2/3', frac18: '1/8', frac38: '3/8', frac58: '5/8', frac78: '7/8', shy: '' };
  const VF = { '½': '1/2', '¼': '1/4', '¾': '3/4', '⅓': '1/3', '⅔': '2/3', '⅛': '1/8', '⅜': '3/8', '⅝': '5/8', '⅞': '7/8' };
  const norm = (s) => String(s).toLowerCase().replace(/<[^>]+>/g, ' ')
    .replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16))).replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(Number(d)))
    .replace(/&([a-z0-9]+);/gi, (m, n) => (n.toLowerCase() in ENT ? ENT[n.toLowerCase()] : m))
    .replace(/\u00AD/g, '').replace(/[‘’“”]/g, "'").replace(/[½¼¾⅓⅔⅛⅜⅝⅞]/g, (c) => VF[c]).replace(/[–—]/g, '-').replace(/\s+/g, ' ').trim();
  const cache = new Map();
  const get = async (url) => {
    if (cache.has(url)) return cache.get(url);
    let res;
    try {
      const UA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0 Safari/537.36';
      const r = await fetch(url, { headers: { 'User-Agent': UA, 'Accept-Language': 'en,ja;q=0.8' }, redirect: 'follow' });
      res = { status: r.status, text: r.ok ? norm(await r.text()) : '' };
      if (r.status === 403) {
        // The Internet Archive answers Node's fetch with 403 and curl with 200 for the same
        // URL and headers; a second reader by a second client is still a re-read.
        try {
          const out = execFileSync('curl', ['-sS', '-L', '--max-time', '60', '-A', UA, '-o', '-', '-w', '\n__STATUS__%{http_code}', url], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
          const m = /__STATUS__(\d+)\s*$/.exec(out);
          const code = m ? Number(m[1]) : 0;
          if (code === 200) res = { status: 200, text: norm(out.replace(/__STATUS__\d+\s*$/, '')), via: 'curl' };
        } catch { /* stays 403 */ }
      }
    } catch (e) { res = { status: 0, text: '', error: String(e.message || e) }; }
    cache.set(url, res);
    return res;
  };
  let unreachable = 0;
  for (const c of D.CLAIMS.filter((x) => !/^usda-/.test(x.source))) {
    const r = await get(c.url);
    if (r.status !== 200) { unreachable++; log.push(`    ?    ${c.id}: ${c.url} answered ${r.status || r.error}; the quote could not be re-read`); continue; }
    // A transcribed table row joins its cells with " | " (the page has <td>s, which norm()
    // turns into spaces), so a quote is re-read cell by cell: every fragment must appear on
    // the page, in order, each after the previous one. A prose quote is one fragment.
    const frags = String(c.printed).split('|').map((f) => norm(f)).filter(Boolean);
    let at = 0, found = frags.length > 0;
    for (const f of frags) { const i = r.text.indexOf(f.slice(0, 64), at); if (i < 0) { found = false; break; } at = i + f.slice(0, 64).length; }
    // A transcription of a chart heading and one of its rows ("WHITE FLOUR - CUPS TO GRAMS
    // 1 cup 161g") skips the smaller rows between them, so the fallback is every word of
    // the quote, in order, within a window not much longer than the quote itself.
    if (!found) {
      const words = norm(c.printed).split(' ').filter(Boolean);
      const win = norm(c.printed).length * 2 + 120;
      let from = 0;
      while (!found && words.length) {
        const start = r.text.indexOf(words[0], from); if (start < 0) break;
        let pos = start, all = true;
        for (const w of words) { const j = r.text.indexOf(w, pos); if (j < 0 || j - start > win) { all = false; break; } pos = j + w.length; }
        found = all; from = start + 1;
      }
    }
    ok(found, `${c.id}: the page still carries "${c.printed.slice(0, 60)}"`);
  }
  log.push(`    ${unreachable} claim(s) could not be re-read (blocked or moved); those are reported, not passed`);
}

/* ================= 6. negative controls ================= */
function familySelftest() {
  section('6 - negative controls: the harness must be able to fail');
  const before = fail;
  const snapshot = () => ({ pass, fail, failures: failures.length, log: log.length });
  const restore = (s, expectFail, label) => {
    const failed = fail - s.fail;
    pass = s.pass; fail = s.fail; failures.length = s.failures; log.length = s.log;
    ok(failed >= 1 === expectFail, label + (expectFail ? ` (went red ${failed} time(s))` : ''));
  };
  let s = snapshot();
  familyData({ ...D, CLAIMS: D.CLAIMS.map((c, i) => (i === 0 ? { ...c, grams_per_unit: 0 } : c)) });
  restore(s, true, 'a claim with a zero figure turns family 2 red');
  s = snapshot();
  familyData({ ...D, SOURCES: D.SOURCES.map((x) => (x.kind === 'government' ? { ...x, cup_ml: 240 } : x)) });
  restore(s, true, 'a cup volume invented for USDA turns family 2 red');
  s = snapshot();
  familyData({ ...D, SYSTEM_SOURCES: D.SYSTEM_SOURCES.filter((x) => x.system !== 'japanese') });
  restore(s, true, 'a cup system with no defining document turns family 2 red');
  s = snapshot();
  familyData({ ...D, CLAIMS: D.CLAIMS.map((c) => (c.ingredient === 'flour-ap' && !/^usda-/.test(c.source) ? { ...c, source: 'usda-sr-legacy' } : c)) });
  restore(s, true, 'two flour claims from one source turn family 2 red');
  s = snapshot();
  const origParse = E.parseAmount;
  ok(origParse('1/2') === 0.5, 'control: the real engine parses 1/2');
  restore(s, false, 'the control itself stays green');
  ok(fail === before, 'the self-test leaves no failure behind');
}

/* ================= run ================= */
familyEngine();
familyData();
const usdaRan = familyUsda();
const declared = familyPage();
familyPublished();
if (FETCH) await familyFetch();
if (SELFTEST) familySelftest();

section('7 - the page prints this suite\'s own count');
const total = pass + fail;
if (!SELFTEST && !FETCH && usdaRan) {
  // The count the page prints is the count of the ordinary run: families 1-5 and 7.
  eq(declared, total + 1, `app.mjs declares CHECKS = ${declared}; this run made ${total + 1} assertions including this one`);
} else {
  log.push(`    (count check skipped under --selftest/--fetch or without the raw files; ordinary run declares ${declared})`);
}

console.log(log.join('\n'));
console.log(`\n${pass} passed, ${fail} failed${FETCH ? ' (live)' : ''}${SELFTEST ? ' (selftest)' : ''}`);
if (fail) { console.error('\nFAILED:\n  ' + failures.join('\n  ')); process.exit(1); }
