// verify.mjs — "Drawn by Nothing" (a combine portal, program P3).
//
// The portal's load-bearing claim: four famous "effects" are what a NULL MODEL
// already produces — set every real cause to zero and the headline pattern is
// still there — and the same null model, turned around, is how you catch a thing
// (Mendel's data) that is *too clean* for chance. So before a pattern is evidence,
// subtract what pure noise draws for free.
//
// This file recomputes, from scratch, every number the page displays, with the
// SAME deterministic algorithms (and, where stochastic, the SAME fixed seed) the
// page uses — so "shown" == "checked". It also confirms each parent stratum's
// research/verifier still exists on disk (the join rests on real layers).
//
// Run:  node research/drawn-by-nothing/verify.mjs
// No dependencies. Node 18+.

import { existsSync, readFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, resolve } from 'node:path';

const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = resolve(HERE, '..', '..');

let pass = 0, fail = 0;
const eq  = (got, want, msg) => {
  const ok = got === want;
  console.log(`${ok ? 'PASS' : 'FAIL'}  ${msg}  ${ok ? '' : `→ got ${got}, want ${want}`}`);
  ok ? pass++ : fail++;
};
const near = (got, want, tol, msg) => {
  const ok = Math.abs(got - want) <= tol;
  console.log(`${ok ? 'PASS' : 'FAIL'}  ${msg}  ${ok ? `(${got})` : `→ got ${got}, want ${want}±${tol}`}`);
  ok ? pass++ : fail++;
};
const truth = (cond, msg) => {
  console.log(`${cond ? 'PASS' : 'FAIL'}  ${msg}`);
  cond ? pass++ : fail++;
};

// ── shared engine: a tiny seeded RNG, identical to the page's (mulberry32) ─────
function mulberry32(seed) {
  let a = seed >>> 0;
  return function () {
    a |= 0; a = (a + 0x6D2B79F5) | 0;
    let t = Math.imul(a ^ (a >>> 15), 1 | a);
    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
  };
}
// standard normal via Box–Muller, drawing from a given uniform rng
function gauss(rng) {
  let u = 0, v = 0;
  while (u === 0) u = rng();
  while (v === 0) v = rng();
  return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v);
}
// rank → percentile in [0,100]: fraction of the sample strictly below, ×100
function toPercentile(values) {
  const idx = values.map((v, i) => [v, i]).sort((a, b) => a[0] - b[0]);
  const pct = new Array(values.length);
  for (let r = 0; r < idx.length; r++) pct[idx[r][1]] = (r / (values.length - 1)) * 100;
  return pct;
}
const mean = a => a.reduce((s, x) => s + x, 0) / a.length;

console.log('\n# 1 · Dunning–Kruger, drawn from random numbers (rho = 0)');
console.log('#    the scissors appear with ZERO link between skill and self-rating.');
// Model (matches research/dunning-kruger/artefact.json and the page):
//   ability a ~ N(0,1); test score x = a + N(0, sTest^2); actual percentile = rank(x).
//   perceived = clamp( rho*abilityPct + (1-rho)*50 + bias + N(0,8^2), 0, 100 ).
//   quartiles cut on the ACTUAL-score percentile. rho = 0 ⇒ self-rating ⟂ skill.
const DK = { N: 4000, sTest: 0.5, bias: 15, noiseSD: 8, seed: 0x44_4b_00, rho: 0 };
function runDK(rho) {
  const rng = mulberry32(DK.seed);
  const a = [], x = [];
  for (let i = 0; i < DK.N; i++) { const ai = gauss(rng); a.push(ai); x.push(ai + DK.sTest * gauss(rng)); }
  const abilityPct = toPercentile(a);
  const actualPct  = toPercentile(x);
  const perceived = x.map((_, i) =>
    Math.max(0, Math.min(100, rho * abilityPct[i] + (1 - rho) * 50 + DK.bias + DK.noiseSD * gauss(rng))));
  // sort indices by actual-score percentile, cut into quartiles
  const order = actualPct.map((p, i) => [p, i]).sort((u, v) => u[0] - v[0]).map(z => z[1]);
  const q = n => order.slice(n * DK.N / 4, (n + 1) * DK.N / 4);
  const qStat = idxs => ({
    perceived: mean(idxs.map(i => perceived[i])),
    actual:    mean(idxs.map(i => actualPct[i])),
  });
  const bottom = qStat(q(0)), top = qStat(q(3));
  // overconfidence = perceived - actualPct; OLS slope of overconfidence on actualPct.
  // At rho=0 perceived ⟂ actual, so d(perceived-actual)/d(actual) = 0 - 1 = -1 exactly.
  const mx = mean(actualPct), oc = perceived.map((p, i) => p - actualPct[i]), my = mean(oc);
  let sxy = 0, sxx = 0;
  for (let i = 0; i < DK.N; i++) { sxy += (actualPct[i] - mx) * (oc[i] - my); sxx += (actualPct[i] - mx) ** 2; }
  const ocSlope = sxy / sxx;
  return { bottom, top, ocSlope };
}
const dk = runDK(0);
console.log(`   bottom quartile: perceived ${dk.bottom.perceived.toFixed(1)}th, actual ${dk.bottom.actual.toFixed(1)}th`);
console.log(`   top quartile:    perceived ${dk.top.perceived.toFixed(1)}th, actual ${dk.top.actual.toFixed(1)}th`);
console.log(`   overconfidence-vs-score OLS slope = ${dk.ocSlope.toFixed(4)}`);
// the published Kruger & Dunning (1999) bottom quartile: perceived ~62nd, actual ~12th.
near(dk.bottom.actual, 12.5, 1.0, 'DK bottom-quartile mean ACTUAL percentile ≈ 12.5 (structural)');
near(dk.bottom.perceived, 63, 4, 'DK bottom-quartile mean PERCEIVED ≈ 63 (lands near the published 62nd) from NO real effect');
truth(dk.bottom.perceived - dk.bottom.actual > 40, 'DK bottom-quartile self-flattery gap > 40 pts — from rho = 0');
truth(dk.top.perceived - dk.top.actual < -15, 'DK top-quartile under-rating < -15 pts — the OTHER blade of the scissors, also from noise');
near(dk.ocSlope, -1, 0.02, 'DK overconfidence-on-score slope ≈ -1 (exactly -1 absent the 0–100 clamp — the built-in autocorrelation)');
// And the cure: turn the real link on, the gap must shrink monotonically.
const sweep = [0, 0.25, 0.5, 0.75, 1].map(r => ({ r, gap: runDK(r).bottom.perceived - runDK(r).bottom.actual }));
console.log('   gap vs real link rho:', sweep.map(s => `${s.r}:${s.gap.toFixed(1)}`).join('  '));
truth(sweep.every((s, i) => i === 0 || s.gap <= sweep[i - 1].gap + 0.6), 'DK gap shrinks as the real skill→confidence link rho rises');

console.log('\n# 2 · The hot hand, from a fair coin (Miller–Sanjurjo selection bias)');
console.log('#    look only at flips AFTER a head: on average fewer than half are heads.');
// Exact finite-sample expectation, by full enumeration of all 2^n sequences.
// Estimator: among positions i (1..n-1) with flip i = H, the fraction with flip i+1 = H.
// Averaged over sequences where at least one such position exists (denominator > 0).
function hotHandExpectation(n) {
  let sum = 0, cnt = 0;
  for (let m = 0; m < (1 << n); m++) {
    let denom = 0, numer = 0;
    for (let i = 0; i < n - 1; i++) {
      const cur = (m >> i) & 1, nxt = (m >> (i + 1)) & 1;
      if (cur === 1) { denom++; if (nxt === 1) numer++; }
    }
    if (denom > 0) { sum += numer / denom; cnt++; }
  }
  return sum / cnt; // every sequence equally likely (fair coin), conditioned on denom>0
}
for (const n of [3, 4, 10, 20]) {
  const e = hotHandExpectation(n);
  console.log(`   ${n} flips: E[fraction heads after a head] = ${e.toFixed(4)}  (bias ${(e - 0.5).toFixed(4)})`);
  truth(e < 0.5, `hot-hand: ${n} fair flips ⇒ after-a-head fraction ${e.toFixed(3)} < 0.5 (no streakiness, pure bias)`);
}
const h4 = hotHandExpectation(4), h3 = hotHandExpectation(3);
near(h3, 5 / 12, 1e-12, 'hot-hand n=3 is EXACTLY 5/12 = 0.4167 (not 1/2) — enumerated, no streakiness anywhere');
truth(h3 < 0.5 && h4 < 0.5 && hotHandExpectation(20) > h4, 'hot-hand: the bias is largest for SHORT sequences and fades as n grows');

console.log('\n# 3 · Your friends are more popular than you (Feld 1991), forced by sampling');
// Zachary's karate club: 34 nodes, 78 undirected edges (the standard NetworkX edge list).
const KARATE_EDGES = [
  [1,2],[1,3],[1,4],[1,5],[1,6],[1,7],[1,8],[1,9],[1,11],[1,12],[1,13],[1,14],[1,18],[1,20],[1,22],[1,32],
  [2,3],[2,4],[2,8],[2,14],[2,18],[2,20],[2,22],[2,31],
  [3,4],[3,8],[3,9],[3,10],[3,14],[3,28],[3,29],[3,33],
  [4,8],[4,13],[4,14],[5,7],[5,11],[6,7],[6,11],[6,17],[7,17],
  [9,31],[9,33],[9,34],[10,34],[14,34],[15,33],[15,34],[16,33],[16,34],
  [19,33],[19,34],[20,34],[21,33],[21,34],[23,33],[23,34],
  [24,26],[24,28],[24,30],[24,33],[24,34],[25,26],[25,28],[25,32],[26,32],
  [27,30],[27,34],[28,34],[29,32],[29,34],[30,33],[30,34],[31,33],[31,34],[32,33],[32,34],[33,34],
];
function friendshipGap(n, edges) {
  const deg = new Array(n + 1).fill(0);
  for (const [a, b] of edges) { deg[a]++; deg[b]++; }
  const d = deg.slice(1);
  const mu = mean(d);
  const variance = mean(d.map(x => (x - mu) ** 2));
  const friendMean = mu + variance / mu;            // E[d^2]/E[d], the mean degree of a random friend
  // count people strictly below the mean degree of THEIR OWN friends
  const adj = Array.from({ length: n + 1 }, () => []);
  for (const [a, b] of edges) { adj[a].push(b); adj[b].push(a); }
  let below = 0;
  for (let i = 1; i <= n; i++) {
    if (adj[i].length === 0) continue;
    const fAvg = mean(adj[i].map(j => deg[j]));
    if (deg[i] < fAvg) below++;
  }
  return { mu, friendMean, gap: variance / mu, below, edges: edges.length };
}
const F = friendshipGap(34, KARATE_EDGES);
console.log(`   karate club: ${F.edges} edges, mean degree ${F.mu.toFixed(2)}, a random friend's mean degree ${F.friendMean.toFixed(2)}`);
console.log(`   gap = variance/mean = ${F.gap.toFixed(2)};  ${F.below}/34 sit below their friends' average`);
eq(F.edges, 78, 'friendship: karate edge list has 78 edges (transcription tripwire)');
near(F.mu, 156 / 34, 1e-9, 'friendship: mean degree = 156/34 = 4.588…');
near(F.friendMean, 7.77, 0.02, "friendship: a random friend's mean degree = mu + var/mu ≈ 7.77");
near(F.gap, 3.18, 0.02, 'friendship: forced gap = variance/mean ≈ 3.18 ≥ 0 always');
eq(F.below, 29, 'friendship: 29 of 34 below their friends average (the honest "most, not all")');
// the NULL point: even a structureless random graph shows it, because degrees vary.
const rng = mulberry32(0x5EED_01);
const erEdges = [];
for (let i = 1; i <= 34; i++) for (let j = i + 1; j <= 34; j++) if (rng() < 0.15) erEdges.push([i, j]);
const ER = friendshipGap(34, erEdges);
console.log(`   random graph (no social structure, p=0.15): gap = ${ER.gap.toFixed(2)} > 0`);
truth(ER.gap > 0, 'friendship: a STRUCTURELESS random graph still has gap > 0 — the paradox is sampling, not sociology');

console.log('\n# 4 · A monkey at a typewriter draws Zipf\'s law (Miller 1957)');
console.log('#    M equal letters + a space ⇒ rank-frequency is a power law, slope from a formula.');
// A word = a maximal run of letters. Block L (length-L words): M^L words, each of
// per-word frequency ∝ ((1-p)/M)^L. Rank-frequency is an exact staircase; the line
// through the block corners has slope -s with  s = 1 - ln(1-p)/ln(M).  (p = P(space))
function monkeyZipf(M, p) {
  const sClosed = 1 - Math.log(1 - p) / Math.log(M);
  // The closed form is the ASYMPTOTIC slope (large rank). Take it from the difference
  // between two deep blocks, where ln(r_{L+1}/r_L) → ln M exactly:
  //   r_L = M(M^L − 1)/(M − 1);  f_L ∝ ((1−p)/M)^L.
  const L = Math.max(3, Math.floor(13 / Math.log10(M)));     // deepest block kept in exact-double range
  const lnR = k => Math.log(M) + Math.log(Math.pow(M, k) - 1) - Math.log(M - 1);
  const lnF = k => k * Math.log((1 - p) / M);
  const asymSlope = -(lnF(L + 1) - lnF(L)) / (lnR(L + 1) - lnR(L));
  return { sClosed, sFitted: asymSlope };
}
for (const [M, p] of [[26, 0.18], [26, 0.5], [2, 0.3]]) {
  const z = monkeyZipf(M, p);
  console.log(`   M=${M} letters, P(space)=${p}: closed-form s = ${z.sClosed.toFixed(4)}, fitted ${z.sFitted.toFixed(4)}`);
  near(z.sFitted, z.sClosed, 0.03, `Zipf monkey M=${M},p=${p}: staircase slope matches s = 1 - ln(1-p)/ln(M)`);
}
const real = monkeyZipf(26, 0.18).sClosed;
near(real, 1.06, 0.01, 'Zipf monkey (26 letters, 18% spaces) gives s ≈ 1.06 — right where REAL English sits (~1.07)');
truth(real > 1.0, 'Zipf monkey: a Zipf line is not, by itself, evidence of linguistic structure');

console.log('\n# 5 · THE TURN — Mendel\'s data is too clean for chance (Fisher 1936)');
console.log('#    the same null model, as a yardstick, catches a real anomaly.');
// regularized lower incomplete gamma P(s,x) via series + continued fraction
function gammaln(x) {
  const c = [76.18009172947146, -86.50532032941677, 24.01409824083091,
    -1.231739572450155, 0.1208650973866179e-2, -0.5395239384953e-5];
  let y = x, t = x + 5.5; t -= (x + 0.5) * Math.log(t);
  let ser = 1.000000000190015;
  for (let j = 0; j < 6; j++) ser += c[j] / ++y;
  return -t + Math.log(2.5066282746310005 * ser / x);
}
function gammaP(s, x) { // lower regularized
  if (x <= 0) return 0;
  if (x < s + 1) { // series
    let ap = s, sum = 1 / s, del = sum;
    for (let n = 0; n < 500; n++) { ap++; del *= x / ap; sum += del; if (Math.abs(del) < Math.abs(sum) * 1e-15) break; }
    return sum * Math.exp(-x + s * Math.log(x) - gammaln(s));
  } else { // continued fraction for Q, then 1-Q
    let b = x + 1 - s, c = 1e30, d = 1 / b, h = d;
    for (let i = 1; i < 500; i++) {
      const an = -i * (i - s); b += 2;
      d = an * d + b; if (Math.abs(d) < 1e-30) d = 1e-30;
      c = b + an / c; if (Math.abs(c) < 1e-30) c = 1e-30;
      d = 1 / d; const del = d * c; h *= del; if (Math.abs(del - 1) < 1e-15) break;
    }
    return 1 - Math.exp(-x + s * Math.log(x) - gammaln(s)) * h;
  }
}
const chi2cdf = (x, df) => gammaP(df / 2, x / 2);
// Fisher's combined figure across Mendel's experiments: chi^2 = 41.6056 on 84 df.
const FISHER = { chi2: 41.6056, df: 84, publishedTail: 0.99993 };
const tooClean = chi2cdf(FISHER.chi2, FISHER.df);          // P(chi^2 <= observed): tiny ⇒ deficient variance
const upperTail = 1 - tooClean;                            // P(chi^2 > observed): the "too good" tail
console.log(`   Fisher chi^2 = ${FISHER.chi2} on ${FISHER.df} df.`);
console.log(`   Fisher (1936) PUBLISHED: P(chi^2 > obs) ≈ ${FISHER.publishedTail}  ⇒ ≈ 7 in 100,000 honest runs land this tidy.`);
console.log(`   our exact recomputation: lower tail P(chi^2 <= obs) = ${tooClean.toExponential(3)}  (≈ ${Math.round(tooClean * 100000)} in 100,000 land this close), i.e. P(chi^2 > obs) = ${upperTail.toFixed(6)}.`);
console.log(`   same order of magnitude; the gap is Fisher's pre-computer approximation. Both: < 1 in 10,000.`);
// chi2cdf verified against closed forms first, so this is our authoritative number:
near(chi2cdf(2, 2), 1 - Math.exp(-1), 1e-6, 'chi^2 CDF self-test: P(X<=2 | 2 df) = 1 - e^-1 (validates the gamma routine)');
near(chi2cdf(1, 1), 0.682689, 1e-5, 'chi^2 CDF self-test: P(X<=1 | 1 df) = 0.6827');
near(upperTail, 0.999971, 5e-5, 'Mendel: exact P(chi^2 > 41.6056, 84 df) = 0.99997 — consistent with Fisher\'s published 0.99993');
truth(tooClean < 1e-4, 'Mendel: fewer than 1 in 10,000 honest runs would be this clean — the null model flags it as REAL (the turn)');
// independent monohybrid sanity: round vs wrinkled seeds, 5474:1850 vs 3:1
const obs = [5474, 1850], N = obs[0] + obs[1], exp = [3 * N / 4, N / 4];
const chiMono = (obs[0] - exp[0]) ** 2 / exp[0] + (obs[1] - exp[1]) ** 2 / exp[1];
console.log(`   monohybrid 5474:1850 vs 3:1 ⇒ chi^2 = ${chiMono.toFixed(3)} on 1 df (Mendel's own count, far too small to reject)`);
near(chiMono, 0.263, 0.01, 'Mendel monohybrid chi^2 (5474:1850) ≈ 0.263 — recomputed from his published counts');

console.log('\n# 6 · the join rests on real layers (parent strata exist on disk)');
const PARENTS = [
  ['research/dunning-kruger/verify.mjs',      'Dunning–Kruger'],
  ['public/strata/the-cold-hand/index.html',  'the hot hand'],
  ['research/friendship-paradox/verify.mjs',  'the friendship paradox'],
  ['research/zipfs-law/verify.mjs',           "Zipf / the monkey"],
  ['research/closer-than-chance/verify.mjs',  'Mendel (the inverse)'],
  // the two sibling statistics portals this one is DELIBERATELY distinct from:
  ['src/content/strata/every-number-honest.md', 'sibling portal: grouping lies'],
  ['src/content/strata/no-number-wrong.md',     'sibling portal: summary projection'],
];
for (const [rel, label] of PARENTS) truth(existsSync(resolve(ROOT, rel)), `parent present: ${label}  (${rel})`);

console.log(`\n${'─'.repeat(60)}\n${fail === 0 ? 'ALL PASS' : 'FAILURES'} — ${pass} pass, ${fail} fail\n`);
process.exit(fail === 0 ? 0 : 1);
