// compare.mjs: the published model arm against its own re-run, scored by the published
// page's own sealed analysis module.
//
// THE ONE RULE THIS FILE OBEYS: it computes no statistic of its own. For each slug it
// imports public/strata/<slug>/analysis.mjs, the exact module whose bytes the page seals
// and displays a hash of, and calls the same summary function the page calls. Both the
// 2026-08 draws and the 2026-09 draws go through that one function. A re-run scored by a
// second parser would be comparing two things at once, and the difference could never be
// attributed.
//
// Everything below the import is bookkeeping: which function each page exposes, what shape
// it wants its argument in, and which field of its output is the number the page prints.
// Where a per-model figure is needed and the page's function reports only a pooled one, the
// runs are FILTERED to one model and the same function is called again, because filtering is not
// re-analysis.
//
// Usage:  node research/model-arm-reproduction/compare.mjs            (all slugs)
//         node research/model-arm-reproduction/compare.mjs --json     (machine-readable)

import fs from 'node:fs';
import path from 'node:path';

const HERE = path.resolve(import.meta.dirname);
const ROOT = path.resolve(HERE, '..', '..');
const PUBLISHED = path.join(ROOT, 'research', 'living-experiments', 'model-arm', 'results');
const RERUN = path.join(HERE, 'data', 'rerun');
const OUT = path.join(HERE, 'data', 'comparison.json');

const strataDir = (slug) => path.join(ROOT, 'public', 'strata', slug, 'analysis.mjs');
const pageData = (slug) => JSON.parse(fs.readFileSync(path.join(ROOT, 'public', 'strata', slug, 'page-data.json'), 'utf8'));

const withRuns = (arm, runs) => ({ ...arm, runs });

// ── per-slug adapters ────────────────────────────────────────────────────────
// Each returns rows of { model, protocol, metric, n, value } read straight out of the
// page's own summary object. `metric` names which printed quantity this is.
const ADAPTERS = {
  'the-coin-you-cant-fake': async (arm) => {
    const m = await import(strataDir('the-coin-you-cant-fake'));
    const s = m.modelSummary(arm);
    const rows = [];
    for (const b of s.byModel) for (const p of b.protocols) {
      rows.push({ model: b.model, protocol: p.id, metric: 'mean switch proportion', n: p.value, value: p.mean, extra: { raw: p.raw, unparseable: p.unparseable, uniqueSequences: p.uniqueSequences } });
    }
    return rows;
  },

  'guess-two-thirds': async (arm) => {
    const m = await import(strataDir('guess-two-thirds'));
    const s = m.modelSummary(arm);
    const rows = [];
    for (const b of s.byModel) for (const p of b.protocols) {
      rows.push({ model: b.model, protocol: p.id, metric: 'mean guess', n: p.value, value: p.mean, extra: { raw: p.raw, median: p.median, unparseable: p.unparseable } });
    }
    return rows;
  },

  'wason-246': async (arm) => {
    const m = await import(strataDir('wason-246'));
    const models = [...new Set(arm.runs.map((r) => r.model))].sort();
    const rows = [];
    for (const model of models) {
      const s = m.summarizeModels(arm.runs.filter((r) => r.model === model));
      for (const [id, p] of Object.entries(s.byProtocol)) {
        if (!p.raw) continue;
        // The page's headline for a protocol is the share of classified first actions that
        // are positive tests ("match"): match / value.
        rows.push({ model, protocol: id, metric: 'positive-test share of classified first actions', n: p.value, value: p.value ? p.match / p.value : null, extra: { raw: p.raw, match: p.match, not_match: p.not_match, no_current_guess: p.no_current_guess, unparseable: p.unparseable } });
      }
    }
    return rows;
  },

  'anchoring-rct': async (arm) => {
    const m = await import(strataDir('anchoring-rct'));
    const models = [...new Set(arm.runs.map((r) => r.model))].sort();
    const rows = [];
    for (const model of models) {
      const s = m.summarizeModelRuns(arm.runs.filter((r) => r.model === model));
      for (const [id, p] of Object.entries(s.byProtocol)) {
        if (!p.raw) continue;
        const vals = (p.values || []).slice().sort((a, b) => a - b);
        const med = vals.length ? (vals.length % 2 ? vals[(vals.length - 1) / 2] : (vals[vals.length / 2 - 1] + vals[vals.length / 2]) / 2) : null;
        rows.push({ model, protocol: id, metric: 'median estimate', n: p.value, value: med, extra: { raw: p.raw, unparseable: p.unparseable } });
      }
    }
    return rows;
  },

  'calibration-intervals': async (arm) => {
    const m = await import(strataDir('calibration-intervals'));
    const truth = pageData('calibration-intervals').truth_key.items;
    const models = [...new Set(arm.runs.map((r) => r.model))].sort();
    const rows = [];
    for (const model of models) {
      const s = m.summarizeModelRuns(withRuns(arm, arm.runs.filter((r) => r.model === model)), truth);
      rows.push({ model, protocol: 'direct_range', metric: 'mean eight-item coverage', n: s.direct.n, value: s.direct.mean, extra: {} });
      rows.push({ model, protocol: 'split (joined pairs)', metric: 'mean eight-item coverage', n: s.split.n, value: s.split.mean, extra: { joinedPairs: s.split.joinedPairs, invalidPairs: s.split.invalidPairs, unmatchedCalls: s.split.unmatchedCalls } });
    }
    return rows;
  },

  'serial-position': async (arm) => {
    const m = await import(strataDir('serial-position'));
    const models = [...new Set(arm.runs.map((r) => r.model))].sort();
    const rows = [];
    for (const model of models) {
      const s = m.summarizeModel(withRuns(arm, arm.runs.filter((r) => r.model === model)));
      const placed = s.positions.filter((p) => p.n > 0);
      const succ = placed.reduce((a, p) => a + p.success, 0);
      const n = placed.reduce((a, p) => a + p.n, 0);
      rows.push({ model, protocol: 'all 60 lists', metric: 'overall exact-match reporting rate', n, value: n ? succ / n : null, extra: { completions: s.completions, unparseable: s.counts.unparseable, error: s.counts.error } });
    }
    return rows;
  },
};

// ── the pooled row, and why it is restricted ─────────────────────────────────
// Per-model, per-protocol counts in this wave are small. The published coin arm classified
// as few as one usable completion for some model/protocol cells, because its parser demands
// exactly 100 characters of H and T and most models do not deliver that. So a pooled figure
// is the only one with enough n to look at.
//
// But pooling across the whole roster would confound the two things this study is trying to
// keep apart: the published run pooled six models and the re-run can only reach four, so a
// naive pooled difference would be mostly a roster difference wearing a drift costume. The
// pooled row here is therefore computed over the INTERSECTION of the two rosters, on both
// sides, using the same page function. It answers: holding the panel fixed to the models
// that still exist, did the number move?
const POOLED = {
  'the-coin-you-cant-fake': async (arm) => {
    const m = await import(strataDir('the-coin-you-cant-fake'));
    return m.modelSummary(arm).protocols.map((p) => ({ model: '(pooled, shared models)', protocol: p.id, metric: 'mean switch proportion', n: p.value, value: p.mean, extra: { raw: p.raw, unparseable: p.unparseable, uniqueSequences: p.uniqueSequences } }));
  },
  'guess-two-thirds': async (arm) => {
    const m = await import(strataDir('guess-two-thirds'));
    return m.modelSummary(arm).protocols.map((p) => ({ model: '(pooled, shared models)', protocol: p.id, metric: 'mean guess', n: p.value, value: p.mean, extra: { raw: p.raw, median: p.median, unparseable: p.unparseable } }));
  },
  'wason-246': async (arm) => {
    const m = await import(strataDir('wason-246'));
    const s = m.summarizeModels(arm.runs);
    return Object.entries(s.byProtocol).filter(([, p]) => p.raw).map(([id, p]) => ({ model: '(pooled, shared models)', protocol: id, metric: 'positive-test share of classified first actions', n: p.value, value: p.value ? p.match / p.value : null, extra: { raw: p.raw, match: p.match, unparseable: p.unparseable } }));
  },
  'anchoring-rct': async (arm) => {
    const m = await import(strataDir('anchoring-rct'));
    const s = m.summarizeModelRuns(arm.runs);
    return Object.entries(s.byProtocol).filter(([, p]) => p.raw).map(([id, p]) => {
      const vals = (p.values || []).slice().sort((a, b) => a - b);
      const med = vals.length ? (vals.length % 2 ? vals[(vals.length - 1) / 2] : (vals[vals.length / 2 - 1] + vals[vals.length / 2]) / 2) : null;
      return { model: '(pooled, shared models)', protocol: id, metric: 'median estimate', n: p.value, value: med, extra: { raw: p.raw, unparseable: p.unparseable } };
    });
  },
  'calibration-intervals': async (arm) => {
    const m = await import(strataDir('calibration-intervals'));
    const s = m.summarizeModelRuns(arm, pageData('calibration-intervals').truth_key.items);
    return [
      { model: '(pooled, shared models)', protocol: 'direct_range', metric: 'mean eight-item coverage', n: s.direct.n, value: s.direct.mean, extra: {} },
      { model: '(pooled, shared models)', protocol: 'split (joined pairs)', metric: 'mean eight-item coverage', n: s.split.n, value: s.split.mean, extra: { joinedPairs: s.split.joinedPairs } },
    ];
  },
  'serial-position': async (arm) => {
    const m = await import(strataDir('serial-position'));
    const s = m.summarizeModel(arm);
    const placed = s.positions.filter((p) => p.n > 0);
    const succ = placed.reduce((a, p) => a + p.success, 0);
    const n = placed.reduce((a, p) => a + p.n, 0);
    return [{ model: '(pooled, shared models)', protocol: 'all 60 lists', metric: 'overall exact-match reporting rate', n, value: n ? succ / n : null, extra: { completions: s.completions } }];
  },
};

// ── controls ─────────────────────────────────────────────────────────────────
// --selftest is the positive control this comparison needs: run every adapter over the
// published arm TWICE and require every delta to be exactly zero. It catches an adapter
// that reads a field the page computes non-deterministically (a Set iteration order, a
// timestamp, an unsorted values array), which would otherwise show up as a spurious drift
// and be read as a finding. It also catches an adapter that silently pairs nothing: a
// comparison with zero pairs is a failure, not a pass.
if (process.argv.includes('--selftest')) {
  let bad = 0, pairs = 0;
  for (const slug of Object.keys(ADAPTERS)) {
    const arm = JSON.parse(fs.readFileSync(path.join(PUBLISHED, `${slug}.json`), 'utf8'));
    const a = await ADAPTERS[slug](arm);
    const b = await ADAPTERS[slug](JSON.parse(JSON.stringify(arm)));
    const key = (r) => `${r.model}::${r.protocol}`;
    const byA = new Map(a.map((r) => [key(r), r]));
    let n = 0;
    for (const r of b) {
      const p = byA.get(key(r));
      if (!p) { console.log(`  ${slug}: unpaired row ${key(r)}`); bad++; continue; }
      n++;
      const same = (p.value === r.value || (p.value === null && r.value === null) || (Number.isNaN(p.value) && Number.isNaN(r.value))) && p.n === r.n;
      if (!same) { console.log(`  ${slug}: NOT STABLE ${key(r)} ${p.value}/${p.n} vs ${r.value}/${r.n}`); bad++; }
    }
    if (!n) { console.log(`  ${slug}: adapter produced ZERO pairs`); bad++; }
    pairs += n;
    console.log(`  ${slug}: ${n} rows stable`);
  }
  console.log(bad ? `\nSELFTEST FAILED (${bad} problems)` : `\nSELFTEST PASSED. ${pairs} rows, every adapter deterministic and non-empty`);
  process.exit(bad ? 1 : 0);
}

// ── comparison ───────────────────────────────────────────────────────────────
const SLUGS = Object.keys(ADAPTERS);

const out = { generated: new Date().toISOString(), slugs: {} };
for (const slug of SLUGS) {
  const pubFile = path.join(PUBLISHED, `${slug}.json`);
  const reFile = path.join(RERUN, `${slug}.json`);
  if (!fs.existsSync(reFile)) { out.slugs[slug] = { status: 'not re-run' }; continue; }
  const pub = JSON.parse(fs.readFileSync(pubFile, 'utf8'));
  const re = JSON.parse(fs.readFileSync(reFile, 'utf8'));
  const adapt = ADAPTERS[slug];
  const pubRows = await adapt(pub);
  const reRows = await adapt(re);

  // The shared roster, and the pooled rows computed on both sides restricted to it.
  const pubModels = new Set(pub.runs.map((r) => r.model));
  const reModels = new Set(re.runs.map((r) => r.model));
  const shared = [...pubModels].filter((m) => reModels.has(m)).sort();
  const only = (arm) => ({ ...arm, runs: arm.runs.filter((r) => shared.includes(r.model)) });
  if (shared.length) {
    pubRows.push(...await POOLED[slug](only(pub)));
    reRows.push(...await POOLED[slug](only(re)));
  }

  const key = (r) => `${r.model}::${r.protocol}`;
  const pubBy = new Map(pubRows.map((r) => [key(r), r]));
  const paired = [];
  for (const r of reRows) {
    const p = pubBy.get(key(r));
    if (!p) continue;
    paired.push({
      model: r.model, protocol: r.protocol, metric: r.metric,
      published: { n: p.n, value: p.value, extra: p.extra },
      rerun: { n: r.n, value: r.value, extra: r.extra },
      delta: (typeof p.value === 'number' && typeof r.value === 'number') ? r.value - p.value : null,
    });
  }
  out.slugs[slug] = {
    status: 'compared',
    published_generated: pub.generated,
    rerun_generated: re.generated,
    published_draws: pub.runs.length,
    rerun_draws: re.runs.length,
    published_answered: pub.runs.filter((r) => !r.error).length,
    rerun_answered: re.runs.filter((r) => !r.error).length,
    metric: paired.length ? paired[0].metric : null,
    published_roster: [...pubModels].sort(),
    rerun_roster: [...reModels].sort(),
    shared_roster: shared,
    pairs: paired,
    unpaired_rerun_rows: reRows.length - paired.length,
  };
}

fs.writeFileSync(OUT, JSON.stringify(out, null, 1));

if (process.argv.includes('--json')) { console.log(JSON.stringify(out, null, 1)); process.exit(0); }

const fmt = (v) => v === null || v === undefined || !Number.isFinite(v) ? '   —  ' : (Math.abs(v) >= 1000 ? v.toFixed(0) : v.toFixed(4));
for (const [slug, s] of Object.entries(out.slugs)) {
  console.log(`\n=== ${slug}`);
  if (s.status !== 'compared') { console.log('   ' + s.status); continue; }
  console.log(`   ${s.published_generated}: ${s.published_answered}/${s.published_draws} answered   ·   ${s.rerun_generated}: ${s.rerun_answered}/${s.rerun_draws} answered`);
  console.log(`   metric: ${s.metric}`);
  console.log('   ' + 'model'.padEnd(40) + 'protocol'.padEnd(34) + 'then'.padStart(9) + '  n' .padStart(5) + 'now'.padStart(11) + '  n'.padStart(5) + '   delta');
  for (const p of s.pairs) {
    console.log('   ' + p.model.padEnd(40) + p.protocol.padEnd(34)
      + fmt(p.published.value).padStart(9) + String(p.published.n).padStart(5)
      + fmt(p.rerun.value).padStart(11) + String(p.rerun.n).padStart(5)
      + '   ' + fmt(p.delta));
  }
}
console.log(`\nwritten: ${path.relative(process.cwd(), OUT)}`);
