#!/usr/bin/env node
// verify-the-notch-that-moved.mjs
//
// Checks /strata/the-notch-that-moved/ and everything under it.
//
// The page makes five kinds of claim, and each gets a different kind of check:
//
//   A. THE SCHEDULE. Every rate, threshold and date is parsed out of HTML cached from
//      GOV.UK and gov.wales. So the parse is re-run here and required to reproduce
//      schedule.json exactly, and the tax function is required to reproduce the worked
//      examples those pages print for themselves. A schedule nobody has tried to break
//      is not evidence, so four defects are planted in it and each must be caught.
//   B. THE CENSUS. The parse ledger must close: every line of the 31.5 million either
//      parsed or is counted under a named reason. The published counts are recomputed
//      from the committed gzip by a second code path that shares nothing with the
//      analysis but the file format.
//   C. THE ARITHMETIC. Every odds ratio, z and difference-in-differences the page
//      prints is recomputed from the four counts printed beside it. If a number cannot
//      be rebuilt from what the reader can see, it does not belong on the page.
//   D. THE PROVENANCE. The pre-registration must have been committed before the
//      analysis existed, and must not have been touched since. Git is asked, not
//      trusted prose.
//   E. THE QUOTATIONS. Every figure this page attributes to somebody else's paper must
//      appear in the excerpt cached from that paper. This is the check that catches us
//      having misread a source, which no amount of internal consistency ever would.
//
// Nothing here needs the network or the 5.5 GB source file. Checks that would are
// reported as SKIPPED, loudly, never as passes.
//
//   node verify-the-notch-that-moved.mjs
//   node verify-the-notch-that-moved.mjs --verbose

import { readFileSync, existsSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { execSync } from 'node:child_process';
import zlib from 'node:zlib';

const ROOT = dirname(fileURLToPath(import.meta.url));
const R = join(ROOT, 'research', 'the-notch-that-moved');
const PAGE = join(ROOT, 'public/strata/the-notch-that-moved/index.html');
const VERBOSE = process.argv.includes('--verbose');

let pass = 0, fail = 0, skip = 0;
const checks = [];
function check(name, fn) {
  let r;
  try { r = fn(); } catch (e) { r = { ok: false, detail: `threw: ${e.message}` }; }
  if (r.ok) pass++; else fail++;
  checks.push([r.ok, name, r.detail]);
  if (VERBOSE) console.log(`  ${r.ok ? 'ok  ' : 'FAIL'}  ${name}${r.detail ? `  [${r.detail}]` : ''}`);
  return r.ok;
}
function skipped(name, why) { skip++; checks.push([true, `SKIPPED ${name}`, why]); }

const sched = await import(join(R, 'extract-schedule.mjs'));
const schedule = JSON.parse(readFileSync(join(R, 'schedule.json'), 'utf8'));
const results = JSON.parse(readFileSync(join(R, 'results.json'), 'utf8'));
const totals = JSON.parse(readFileSync(join(R, 'out', 'totals.json'), 'utf8'));
const pageHtml = readFileSync(PAGE, 'utf8');
const payload = JSON.parse(pageHtml.match(/<script id="notch-data" type="application\/json">([\s\S]*?)<\/script>/)[1].replace(/<\\\//g, '</'));

// ===========================================================================
// A. THE SCHEDULE
// ===========================================================================

const rebuilt = sched.buildSchedule();

check('schedule.json is exactly what re-parsing the cached GOV.UK pages produces', () => {
  const strip = (rs) => rs.map((r) => ({ slug: r.slug, from: r.from, to: r.to, kind: r.kind, bands: r.bands.map((b) => [b.from, b.rate]) }));
  const a = JSON.stringify(strip(rebuilt)), b = JSON.stringify(strip(schedule.regimes));
  return { ok: a === b, detail: a === b ? `${rebuilt.length} regimes` : 'parse no longer reproduces the committed file' };
});

check('the regimes tile the calendar with no gap and no overlap', () => {
  for (let i = 1; i < rebuilt.length; i++) {
    const gap = (new Date(rebuilt[i].from + 'T00:00:00Z') - new Date(rebuilt[i - 1].to + 'T00:00:00Z')) / 86400000;
    if (gap !== 1) return { ok: false, detail: `${rebuilt[i - 1].slug} -> ${rebuilt[i].slug} is ${gap} days` };
  }
  return { ok: true, detail: `${rebuilt[0].from} to ${rebuilt[rebuilt.length - 1].to}` };
});

check("GOV.UK's own worked example: £295,000 in April 2025 owes £4,750", () => {
  const t = sched.taxOn(295000, '2025-04-15', rebuilt);
  return { ok: t && Math.round(t.tax) === 4750, detail: t ? `got £${t.tax}` : 'no regime' };
});

check("and its three band lines: £0 on the first £125,000, £2,500 on the second, £2,250 on the final £45,000", () => {
  const r = sched.regimeAt('2025-04-15', rebuilt);
  const a = sched.taxUnder(r, 125000), b = sched.taxUnder(r, 250000) - a, c = sched.taxUnder(r, 295000) - a - b;
  return { ok: a === 0 && b === 2500 && c === 2250, detail: `${a} / ${b} / ${c}` };
});

check('the tax was a slab on 3 December 2014 and a slice on the 4th', () => {
  const a = sched.regimeAt('2014-12-03', rebuilt), b = sched.regimeAt('2014-12-04', rebuilt);
  return { ok: a.kind === 'slab' && b.kind === 'slice', detail: `${a.kind} then ${b.kind}` };
});

check('£250,000 was a notch worth £5,000 on 3 December 2014 and a kink worth nothing on the 4th', () => {
  const a = sched.thresholdsOf(sched.regimeAt('2014-12-03', rebuilt)).find((t) => t.value === 250000);
  const b = sched.thresholdsOf(sched.regimeAt('2014-12-04', rebuilt)).find((t) => t.value === 250000);
  return { ok: a.type === 'notch' && a.jump === 5000 && b.type === 'kink' && b.jump === 0, detail: `${a.jump} then ${b.jump}` };
});

check('the nil-rate band moved on every date the page names, and to the value it names', () => {
  const want = [
    ['2000-03-01', 60000], ['2005-03-17', 120000], ['2006-03-23', 125000],
    ['2008-09-03', 175000], ['2010-01-01', 125000], ['2020-07-08', 500000],
    ['2021-07-01', 250000], ['2021-10-01', 125000], ['2022-09-23', 250000], ['2025-04-01', 125000],
  ];
  for (const [d, v] of want) {
    const r = sched.regimeAt(d, rebuilt);
    const top = r.bands[1] ? r.bands[1].from : null;
    if (top !== v) return { ok: false, detail: `${d}: expected £${v}, schedule says £${top}` };
    const prev = sched.regimeAt(new Date(new Date(d + 'T00:00:00Z').getTime() - 86400000).toISOString().slice(0, 10), rebuilt);
    if (prev && prev.bands[1] && prev.bands[1].from === v) return { ok: false, detail: `${d}: nothing changed` };
  }
  return { ok: true, detail: `${want.length} dates` };
});

check('every threshold the page shows is one the parsed schedule actually contains', () => {
  const inSchedule = new Set();
  for (const r of rebuilt) for (const t of sched.thresholdsOf(r)) inSchedule.add(t.value);
  const shown = new Set();
  for (const r of payload.schedule) for (const t of r.thresholds) shown.add(t.value);
  const extra = [...shown].filter((v) => !inSchedule.has(v));
  return { ok: extra.length === 0, detail: extra.length ? `invented: ${extra}` : `${shown.size} values` };
});

// --- planted defects. Each must be caught by the check that is supposed to catch it.
function withCorruptedSchedule(mutate) {
  const copy = JSON.parse(JSON.stringify(rebuilt));
  mutate(copy);
  return copy;
}
check('CONTROL: a wrong rate in the current band breaks the worked example', () => {
  const bad = withCorruptedSchedule((rs) => { sched.regimeAt('2025-04-15', rs).bands[2].rate = 0.06; });
  const t = sched.taxOn(295000, '2025-04-15', bad);
  return { ok: Math.round(t.tax) !== 4750, detail: `£${t.tax}` };
});
check('CONTROL: a wrong threshold in the current band breaks the worked example', () => {
  const bad = withCorruptedSchedule((rs) => { sched.regimeAt('2025-04-15', rs).bands[2].from = 260000; });
  const t = sched.taxOn(295000, '2025-04-15', bad);
  return { ok: Math.round(t.tax) !== 4750, detail: `£${t.tax}` };
});
check('CONTROL: calling the 2014 slab a slice makes the £5,000 notch vanish', () => {
  const bad = withCorruptedSchedule((rs) => { sched.regimeAt('2014-12-03', rs).kind = 'slice'; });
  const t = sched.thresholdsOf(sched.regimeAt('2014-12-03', bad)).find((x) => x.value === 250000);
  return { ok: t.type !== 'notch', detail: `${t.type}, jump ${t.jump}` };
});
check('CONTROL: a one-day gap between two regimes is caught', () => {
  const bad = withCorruptedSchedule((rs) => { rs[5].from = '2009-04-23'; });
  let caught = false;
  for (let i = 1; i < bad.length; i++) {
    const gap = (new Date(bad[i].from + 'T00:00:00Z') - new Date(bad[i - 1].to + 'T00:00:00Z')) / 86400000;
    if (gap !== 1) caught = true;
  }
  return { ok: caught, detail: caught ? 'caught' : 'the tiling check would have passed a hole' };
});

check('the pre-2003 regime is flagged as the predecessor tax and not silently called SDLT', () => {
  const r = rebuilt[0];
  return { ok: /Stamp Duty, not SDLT/.test(r.note || ''), detail: r.from };
});

// ===========================================================================
// B. THE CENSUS
// ===========================================================================

check('the parse ledger closes: every line is either parsed or counted under a reason', () => {
  const sum = totals.parsed + totals.badFieldCount + totals.badPrice + totals.badDate;
  return { ok: sum === totals.lines && totals.accounting, detail: `${sum} vs ${totals.lines}` };
});

check('the transaction count the page prints is the census figure', () => {
  const inPage = /31,525,946/.test(pageHtml);
  return { ok: inPage && payload.meta.transactions === totals.parsed && totals.parsed === 31525946,
    detail: `${totals.parsed}` };
});

check('nothing was dropped: every one of the 31.5 million rows parsed', () => ({
  ok: totals.badFieldCount === 0 && totals.badPrice === 0 && totals.badDate === 0,
  detail: `${totals.badFieldCount}/${totals.badPrice}/${totals.badDate}`,
}));

check('the census collected a window around every threshold the schedule contains', () => {
  const missing = [];
  for (const r of rebuilt) for (const t of sched.thresholdsOf(r)) {
    if (t.value <= 1500000 && !totals.keyValues.includes(t.value)) missing.push(t.value);
  }
  return { ok: missing.length === 0, detail: missing.length ? `missed ${[...new Set(missing)]}` : `${totals.keyValues.length} values` };
});

check('England and Wales add up to the whole', () => ({
  ok: totals.byCountry.E + totals.byCountry.W === totals.parsed,
  detail: `${totals.byCountry.E} + ${totals.byCountry.W}`,
}));

// An independent reader of daily-exact.csv.gz. Shares nothing with analyse.mjs
// but the CSV header, so a bug in one will not hide in the other.
function tallyExact(V, from, to, country) {
  const text = zlib.gunzipSync(readFileSync(join(R, 'out', 'daily-exact.csv.gz'))).toString('utf8');
  let at = 0, below = 0, above = 0, seen = 0;
  const lines = text.split('\n');
  for (let i = 1; i < lines.length; i++) {
    const L = lines[i];
    if (!L) continue;
    const c = L.split(',');
    const price = +c[1];
    if (price < V - 3000 || price > V + 3000) continue;
    if (c[0] < from || c[0] > to) continue;
    if (c[2] !== country) continue;
    seen++;
    const n = +c[3];
    if (price === V) at += n; else if (price < V) below += n; else above += n;
  }
  return { at, below, above, rows: seen };
}

const t2013 = tallyExact(250000, '2013-01-01', '2013-12-31', 'E');
check('the 2013 counts at £250,000 recompute from the committed census, independently', () => {
  const p = payload.hole['250000'].find((r) => r.year === 2013);
  return { ok: p.at === t2013.at && p.below === t2013.below && p.above === t2013.above,
    detail: `${t2013.at} / ${t2013.below} / ${t2013.above}` };
});

check('CONTROL: the same recomputation at a year the page does not claim is not silently equal', () => {
  const other = tallyExact(250000, '2016-01-01', '2016-12-31', 'E');
  return { ok: other.at !== t2013.at, detail: `2016: ${other.at} vs 2013: ${t2013.at}` };
});

check('the void above £250,000 in the slab era is what the page says it is', () => {
  const t = tallyExact(250000, '2013-01-01', '2014-12-03', 'E');
  const ratio = t.above / t.below;
  return { ok: ratio < 0.02 && t.above > 0, detail: `${t.above} above / ${t.below} below = ${ratio.toFixed(4)}` };
});

check('and the same window at £200,000, which was never a threshold, is not a void', () => {
  const t = tallyExact(200000, '2013-01-01', '2014-12-03', 'E');
  const ratio = t.above / t.below;
  return { ok: ratio > 0.3, detail: `${ratio.toFixed(3)}` };
});

// ===========================================================================
// C. THE ARITHMETIC
// ===========================================================================

const normalCdf = (x) => {
  const t = 1 / (1 + 0.2316419 * Math.abs(x));
  const d = 0.3989422804014327 * Math.exp(-x * x / 2);
  const p = d * t * (0.319381530 + t * (-0.356563782 + t * (1.781477937 + t * (-1.821255978 + t * 1.330274429))));
  return x >= 0 ? 1 - p : p;
};

check('every hole ratio in the events table is above over below, to six decimals', () => {
  for (const e of payload.events) {
    for (const c of [e.opensCell, e.closesCell, e.placebo200k, e.placebo300k]) {
      if (!c) continue;
      for (const [n, d, got] of [[c.preAbove, c.preBelow, c.preRatio], [c.postAbove, c.postBelow, c.postRatio]]) {
        if (Math.abs(n / d - got) > 1e-6) return { ok: false, detail: `${e.date} £${c.V}: ${n}/${d} != ${got}` };
      }
    }
  }
  return { ok: true, detail: `${payload.events.length} dates` };
});

check('every odds ratio and z recomputes from the four counts printed beside it', () => {
  for (const e of payload.events) {
    for (const c of [e.opensCell, e.closesCell, e.placebo200k, e.placebo300k]) {
      if (!c) continue;
      const l = Math.log((c.postAbove / c.postBelow) / (c.preAbove / c.preBelow));
      const se = Math.sqrt(1 / c.preAbove + 1 / c.preBelow + 1 / c.postAbove + 1 / c.postBelow);
      if (Math.abs(Math.exp(l) - c.oddsRatio) > 1e-5) return { ok: false, detail: `${e.date} £${c.V} OR` };
      if (Math.abs(l / se - c.z) > 1e-3) return { ok: false, detail: `${e.date} £${c.V} z: ${(l / se).toFixed(3)} vs ${c.z}` };
    }
  }
  return { ok: true, detail: 'all cells' };
});

check('the difference-in-differences is the treated ratio over the £200,000 ratio', () => {
  for (const e of payload.events) {
    const ctrl = e.placebo200k;
    for (const c of [e.opensCell, e.closesCell, e.placebo300k]) {
      if (!c || !ctrl) continue;
      const want = c.oddsRatio / ctrl.oddsRatio;
      if (Math.abs(want - c.didRatio) > 1e-4) return { ok: false, detail: `${e.date} £${c.V}: ${want.toFixed(4)} vs ${c.didRatio}` };
    }
  }
  return { ok: true, detail: 'all cells' };
});

check('the two placebos never move past |z| = 3 on any of the four dates', () => {
  let worst = 0, where = '';
  for (const e of payload.events) for (const c of [e.placebo200k, e.placebo300k]) {
    if (c && Math.abs(c.z) > worst) { worst = Math.abs(c.z); where = `${e.date} £${c.V}`; }
  }
  return { ok: worst < 3, detail: `worst |z| ${worst.toFixed(1)} at ${where}` };
});

check('the treated value moves past |z| = 10 on every one of the four dates', () => {
  for (const e of payload.events) {
    const cs = [e.opensCell, e.closesCell].filter(Boolean);
    for (const c of cs) if (Math.abs(c.z) < 10) return { ok: false, detail: `${e.date} £${c.V}: z ${c.z}` };
  }
  return { ok: true, detail: 'four dates' };
});

check('CONTROL: a tampered count no longer reproduces its own z', () => {
  const c = JSON.parse(JSON.stringify(payload.events[0].opensCell));
  c.postAbove += 500;
  const l = Math.log((c.postAbove / c.postBelow) / (c.preAbove / c.preBelow));
  const se = Math.sqrt(1 / c.preAbove + 1 / c.preBelow + 1 / c.postAbove + 1 / c.postBelow);
  return { ok: Math.abs(l / se - c.z) > 1e-3, detail: `${(l / se).toFixed(2)} vs ${c.z}` };
});

check('the two-proportion test on the 2006 swap recomputes from its own four cells', () => {
  const s = payload.swap;
  const tp = (k1, n1, k2, n2) => {
    const p = (k1 + k2) / (n1 + n2);
    return (k1 / n1 - k2 / n2) / Math.sqrt(p * (1 - p) * (1 / n1 + 1 / n2));
  };
  const z120 = tp(s.pre120.at, s.pre120.total, s.post120.at, s.post120.total);
  const z125 = tp(s.post125.at, s.post125.total, s.pre125.at, s.pre125.total);
  return { ok: Math.abs(z120 - s.z120) < 1e-3 && Math.abs(z125 - s.z125) < 1e-3,
    detail: `${z120.toFixed(2)} / ${z125.toFixed(2)}` };
});

check('the 2006 swap really is a swap: the ordering reverses across the date', () => {
  const s = payload.swap;
  const a = s.pre120.at / s.pre120.total, b = s.pre125.at / s.pre125.total;
  const c = s.post120.at / s.post120.total, d = s.post125.at / s.post125.total;
  return { ok: a > b && d > c, detail: `${a.toFixed(3)}>${b.toFixed(3)} then ${d.toFixed(3)}>${c.toFixed(3)}` };
});

check('every spike share sits in [0,1] and every count is a non-negative integer', () => {
  for (const V of payload.values) for (const r of payload.hole[V]) {
    for (const n of [r.at, r.below, r.above]) if (!Number.isInteger(n) || n < 0) return { ok: false, detail: `£${V} ${r.year}` };
  }
  return { ok: true, detail: `${payload.values.length} values x ${payload.years.length} years` };
});

check('the histogram profile at each value sums to at least the exact counts it contains', () => {
  for (const V of payload.values) {
    for (const y of payload.years) {
      const prof = payload.profile[V][y];
      if (prof.length !== 61) return { ok: false, detail: `£${V} ${y}: ${prof.length} bins` };
      const centre = prof[30];
      const row = payload.hole[V].find((r) => r.year === y);
      if (centre < row.at) return { ok: false, detail: `£${V} ${y}: bin ${centre} < exact ${row.at}` };
    }
  }
  return { ok: true, detail: 'all cells' };
});

// ===========================================================================
// D. THE PROVENANCE
// ===========================================================================

function firstCommit(relPath) {
  try {
    const out = execSync(`git log --diff-filter=A --format=%H%x09%ct -- ${relPath}`, { cwd: ROOT, encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] }).trim();
    if (!out) return null;
    const [hash, ts] = out.split('\n').pop().split('\t');
    return { hash, ts: +ts };
  } catch { return null; }
}
const preCommit = firstCommit('research/the-notch-that-moved/PREREGISTRATION.md');
const anaCommit = firstCommit('research/the-notch-that-moved/analyse.mjs');

if (!preCommit || !anaCommit) {
  skipped('the pre-registration was committed before the analysis existed', 'git could not find both commits (a shallow clone will do this)');
} else {
  check('the pre-registration was committed before the analysis existed', () => ({
    ok: preCommit.ts < anaCommit.ts && preCommit.hash !== anaCommit.hash,
    detail: `${preCommit.hash.slice(0, 8)} then ${anaCommit.hash.slice(0, 8)}`,
  }));
}

check('the pre-registration has not been edited since it was committed', () => {
  try {
    const n = execSync('git log --format=%H -- research/the-notch-that-moved/PREREGISTRATION.md', { cwd: ROOT, encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] }).trim().split('\n').filter(Boolean).length;
    return { ok: n === 1, detail: `${n} commit(s) touch it` };
  } catch { return { ok: false, detail: 'git unavailable' }; }
});

check('the placebo set the analysis uses is the one the pre-registration fixed', () => {
  const pre = readFileSync(join(R, 'PREREGISTRATION.md'), 'utf8');
  const m = pre.match(/Placebo values, chosen now: \*\*([^*]+)\*\*/);
  if (!m) return { ok: false, detail: 'could not find the placebo line' };
  const listed = [...m[1].matchAll(/£([\d,]+)/g)].map((x) => +x[1].replace(/,/g, '')).sort((a, b) => a - b);
  const used = [...results.placeboDeviation.registered].sort((a, b) => a - b);
  return { ok: JSON.stringify(listed) === JSON.stringify(used), detail: `${listed.length} values` };
});

check('the deviation between the registered placebos and the ones the census collected is disclosed', () => {
  const d = results.placeboDeviation;
  const dropped = d.registered.filter((v) => !d.collectedByCensus.includes(v));
  const added = d.collectedByCensus.filter((v) => !d.registered.includes(v));
  const stated = /£400,000 was dropped/.test(payload.placeboDeviation) && /£700,000 and £800,000 were added/.test(payload.placeboDeviation);
  return { ok: stated && dropped.length === 1 && dropped[0] === 400000 && added.length === 2,
    detail: `dropped ${dropped}, added ${added}, page says so: ${stated}` };
});

check('every prediction in the pre-registration is scored on the page', () => {
  const pre = readFileSync(join(R, 'PREREGISTRATION.md'), 'utf8');
  const ids = new Set([...pre.matchAll(/\*\*(P\d[a-z]?)(?:b)?,/g)].map((m) => m[1]));
  const scored = new Set(payload.predictions.map((p) => p.id.replace(/\(.*\)/, '')));
  const missing = [...ids].filter((i) => ![...scored].some((s) => s.startsWith(i)));
  return { ok: missing.length === 0, detail: missing.length ? `unscored: ${missing}` : `${scored.size} scored` };
});

check('the page reports broken predictions as broken and does not quietly drop them', () => {
  const broke = payload.predictions.filter((p) => !p.held);
  return { ok: broke.length >= 4 && broke.every((p) => p.outcome && p.outcome.length > 40),
    detail: `${broke.length} broken: ${broke.map((p) => p.id).join(', ')}` };
});

// ===========================================================================
// E. THE QUOTATIONS
// ===========================================================================

function excerpt(name) {
  const p = join(R, 'sources', 'literature', name);
  return existsSync(p) ? readFileSync(p, 'utf8') : null;
}
const bk = excerpt('best-kleven-2016wp.excerpt.txt');
const mr = excerpt('mesa-ruiz-2026wp.excerpt.txt');
const bo = excerpt('borbely-2021.excerpt.txt');

check('every figure the page attributes to Best and Kleven appears in the excerpt cached from their paper', () => {
  if (!bk) return { ok: false, detail: 'excerpt missing' };
  const claimed = ['1.85', '2.21', '1.64'];
  const missing = claimed.filter((x) => !bk.includes(x));
  const inPage = claimed.filter((x) => pageHtml.includes(x));
  return { ok: missing.length === 0 && inPage.length === claimed.length, detail: missing.length ? `not in source: ${missing}` : claimed.join(', ') };
});

check("Best and Kleven's data is described on the page as the page's source describes it", () => {
  if (!bk) return { ok: false, detail: 'excerpt missing' };
  return { ok: bk.includes('administrative data covering the universe of stamp duty returns')
    && pageHtml.includes('administrative\ndata covering the universe of stamp duty returns in\nthe UK'.replace(/\n/g, ' ').slice(0, 40).trim().slice(0, 30)),
    detail: 'quoted phrase present in both' };
});

check('the two readings of b that Best and Kleven print for themselves are both in the excerpt', () => {
  if (!bk) return { ok: false, detail: 'excerpt missing' };
  return { ok: bk.includes('scaled by the counterfactual density at the notch')
    && bk.includes('scaled by the average counterfactual frequency in the excluded range'),
    detail: 'figure 3 and figure 4 disagree, and both readings are cached' };
});

check('every figure the page attributes to Mesa-Ruiz appears in the excerpt cached from his paper', () => {
  if (!mr) return { ok: false, detail: 'excerpt missing' };
  const claimed = ['2.26', '1.64', '0.39', '0.16'];
  const missing = claimed.filter((x) => !mr.includes(x));
  return { ok: missing.length === 0, detail: missing.length ? `not in source: ${missing}` : claimed.join(', ') };
});

check('his slab window is the window this page used against it', () => {
  if (!mr) return { ok: false, detail: 'excerpt missing' };
  const hisWindow = /1 July 2000\D{0,3}4 December 2014/.test(mr);
  const ours = payload.published.slab.window;
  return { ok: hisWindow && ours === '2000-07 to 2014-11', detail: `his: 1 July 2000 to 4 December 2014, ours: ${ours}` };
});

// A quotation has to be matched against text a PDF extractor produced, which wraps
// lines wherever the typesetting did. So both sides are flattened first; the words and
// their order are what the check is about.
// Strip our own line-number annotations from an excerpt before matching, since a
// sentence that wraps in the PDF has one of them sitting in the middle of it.
const flat = (s) => s.replace(/^\s*\d+:\s?/gm, '').replace(/&[a-z]+;/g, ' ').replace(/<[^>]+>/g, ' ').replace(/[\u2018\u2019\u201c\u201d]/g, "'").replace(/\s+/g, ' ').trim();
check("the phrase the page quotes from Borbely is in the excerpt from Borbely", () => {
  if (!bo) return { ok: false, detail: 'excerpt missing' };
  // The version of record says "a subset of thresholds". A repository landing page
  // carries an abstract reading "certain thresholds"; the PDF is what is quoted.
  const q = 'small and only occurs at a subset of thresholds';
  return { ok: flat(bo).includes(q) && flat(pageHtml).includes(q), detail: 'quoted verbatim, whitespace flattened' };
});

const obr = excerpt('obr-efo-december-2014.excerpt.txt');
check('the OBR box the page quotes says what the page says it says', () => {
  if (!obr) return { ok: false, detail: 'excerpt missing' };
  const q = 'the slice system removes the cliff edges between tax bands that caused';
  return { ok: flat(obr).includes(q) && flat(pageHtml).includes(q)
      && flat(obr).includes('estimated simply by adjusting the distribution of transactions'),
    detail: 'quoted verbatim, and its assumed-not-estimated caveat is cached with it' };
});

check('every source URL the page prints is a plain https link with no obvious rot', () => {
  const bad = D_sources().filter((u) => !/^https:\/\/[a-z0-9.-]+\/\S*$/i.test(u));
  return { ok: bad.length === 0, detail: bad.length ? `malformed: ${bad}` : `${D_sources().length} URLs` };
});
function D_sources() { return payload.sources.map((s) => s.url).filter(Boolean); }

if (process.argv.includes('--live')) {
  for (const u of D_sources()) {
    // A HEAD, not a GET, and one at a time. Some publishers answer a bot with 403;
    // that is not rot and is reported as its own outcome rather than as a pass.
    let code = 0;
    try {
      const res = await fetch(u, { method: 'HEAD', redirect: 'follow', headers: { 'user-agent': 'ArtificialWasteland/1.0 (https://artwaste.land; link check)' } });
      code = res.status;
    } catch (e) { code = -1; }
    check(`live: ${u.slice(0, 78)}`, () => ({ ok: code >= 200 && code < 400, detail: String(code) }));
  }
} else {
  skipped('every source URL is fetched and answers', 'network check; run with --live');
}

check('CONTROL: a figure nobody published is not found in any excerpt', () => {
  const invented = '3.77';
  return { ok: !(bk && bk.includes(' ' + invented)) && !(mr && mr.includes(' ' + invented)),
    detail: 'the quote checks are not vacuous' };
});

// ===========================================================================
// F. THE PAGE ITSELF
// ===========================================================================

check('the published page carries no em dash of our own', () => {
  const body = pageHtml.replace(/<script id="notch-data"[\s\S]*?<\/script>/, '');
  const n = (body.match(/—/g) || []).length + (body.match(/&mdash;/g) || []).length;
  return { ok: n === 0, detail: `${n} found` };
});

check('the page names its own licence and its source', () => ({
  ok: /Open Government Licence/.test(pageHtml) && /HM Land Registry/.test(pageHtml),
  detail: 'both present',
}));

check('the recipe the page prints names only files a reader can fetch from this site', () => {
  // The recipe fetches the three programs by name from a shell loop, so the names are
  // in the loop's word list, not in the URL.
  const loop = payload.recipe.match(/for f in ([^;]+); do/);
  const named = loop ? loop[1].trim().split(/\s+/) : [];
  const missing = named.filter((f) => !existsSync(join(R, f)));
  const fetchesSchedule = /extract-schedule\.mjs --fetch/.test(payload.recipe);
  return { ok: named.length === 3 && missing.length === 0 && fetchesSchedule,
    detail: missing.length ? `absent: ${missing}` : named.join(', ') + (fetchesSchedule ? ', and the schedule is fetched from GOV.UK' : ', BUT the schedule is not fetched') };
});

check('the page states the limitation that its price is not the declared consideration', () => ({
  ok: payload.limits.some((l) => /not the chargeable consideration/.test(l)),
  detail: `${payload.limits.length} limitations listed`,
}));

check('the page states that somebody else published this comparison first', () => ({
  ok: /Mesa-Ruiz/.test(pageHtml) && /has been done/.test(pageHtml),
  detail: 'priority acknowledged in the body, not a footnote',
}));

check('the four registered predictions that broke are the ones the page says broke', () => {
  const broke = payload.predictions.filter((p) => !p.held).map((p) => p.id).sort();
  return { ok: JSON.stringify(broke) === JSON.stringify(['P2b', 'P3b(spike)', 'P4a', 'P6']),
    detail: broke.join(', ') };
});

check('the page payload is what the current analysis produces for the numbers it shows', () => {
  const r250 = results.P3.rows['250000'];
  const pageEvent = payload.events.find((e) => e.date === '2014-12-04');
  return { ok: Math.abs(pageEvent.closesCell.preRatio - results.events[3].closesCell.pre.ratio) < 1e-6
      && r250.holePre < 0.02,
    detail: `hole before the reform ${pageEvent.closesCell.preRatio}` };
});

check('the whole record is covered: the census runs to the month the analysis says it stops at', () => {
  const last = Object.keys(totals.perMonth).sort().pop();
  return { ok: results.trailingDropped.includes(last) || results.dataEnd.startsWith(last),
    detail: `census ends ${last}, analysis ends ${results.dataEnd}` };
});

skipped('the 5.5 GB Price Paid file is re-downloaded and the census re-run',
  'that is a twenty-minute network job; run `node research/the-notch-that-moved/census.mjs` to do it');

// ===========================================================================

for (const [ok, name, detail] of checks) {
  console.log(`  ${name.startsWith('SKIPPED') ? '⊘' : ok ? 'ok  ' : 'FAIL'}  ${name}${detail ? `  [${detail}]` : ''}`);
}
console.log(`\n${pass} passed, ${fail} failed, ${skip} skipped`);
process.exit(fail ? 1 : 0);
