#!/usr/bin/env python3
"""verify_headline_numbers.py — re-derive The Map's public headline numbers from the
committed corpus, so the figures on /strata/the-map/ (and every commercial surface
built on them, P17) are *checkable*, not asserted.

TWO CORPUS VERSIONS, kept apart on purpose (repaired 2026-07-26). This file was
written for v1 (2026-06-21) and then went RED without anyone noticing: it still
held v1 expectations for concepts / frequency / basins while reading the SHIPPED
cloud, which the v2 rebuild replaced under it. Three of its nine rows had stopped
reproducing, so a verifier whose entire job is to stop us publishing an unchecked
number was itself failing silently.

The repair is not to bump the numbers. v1 is a real, published edition (Edition 0
of The State of the Models Mind cites it and is honest that the map has since been
rebuilt), and its cloud is still committed at data/map/cloud_data.json. So the
file now checks BOTH, each against its own artifact, and names what neither can
settle. The ledger (P16) records v2 as canonical for what the site publishes today.

Pure stdlib (json/glob/collections) — runs with no numpy and no network. Re-run any
time before republishing a Map number:

    python3 research/the-map/verify_headline_numbers.py     # exits non-zero on mismatch

The universality RATIO is deliberately NOT asserted here. It is a *living* figure that
re-derives upward as the family set grows (0.80 @15 families -> 0.89 @39; the human,
2026-06-21: this is expected, the dataset expands), it needs the analysis stack
(numpy/scipy/spacy) plus the gitignored `data/concepts/concepts.jsonl`, and recomputing
+ date/family-stamping it is the Convergence Index's job (P17 rung 2). See
`results/headline-verification.md`.
"""
import json, glob, collections, sys
from pathlib import Path

HERE = Path(__file__).resolve().parent          # research/the-map
ROOT = HERE.parents[1]                            # repo root
RAW = sorted((HERE / "data" / "raw").glob("*.jsonl"))
CLOUD_V1 = HERE / "data" / "map" / "cloud_data.json"                       # the lab cloud, Edition 0
CLOUD_V2 = ROOT / "public" / "strata" / "the-map" / "cloud_data.json"      # what the site ships today

# v1 — the lab corpus and the cloud built from it. Every one of these still
# re-derives exactly; they are the numbers Edition 0 of the standing report cites.
EXPECT_V1 = {
    "v1 generations":       43655,
    "v1 model_families":    69,
    "v1 probes":            75,
    "v1 conversations":     34235,
    "v1 concepts_mapped":   60000,
    "v1 frequency_floor":   1775154,   # Σ of the 60k mapped concepts' frequencies ("1.78M")
    "v1 well_basin_count":  3038,
    "v1 daniel_oldfriends": 104,        # "Daniel" in the anchor-social-oldfriends probe
    "v1 daniel_oldfriends_fams": 24,   # families naming Daniel in old-friends stories
}

# v2 — the association rebuild, and the figures the live pages actually print.
# These come from the shipped cloud, which is the artifact a reader downloads.
EXPECT_V2 = {
    "v2 concepts_mapped":   53300,
    "v2 model_families":    86,
    "v2 well_basin_count":  2077,
    "v2 frequency_floor":   3141276,   # Σ of the shipped cloud's frequencies ("3.1M")
    "v2 regions":           16,
}

# Named rather than asserted: the v2 raw corpus was never committed to this
# repository (it lives in the separate private map repo), so nothing here can
# settle these. Printing them as unchecked is the point; quietly asserting them
# would be the failure this file exists to prevent.
NOT_DERIVABLE_HERE = {
    "v2 generations":         "121,558 — the v2 raw corpus is not committed here",
    "v2 concepts_extracted":  "unknown; the v2 extraction pass is not committed here",
}

# --- corpus counts (the raw lab notebook) ---
n = 0
fams, probes, convs = set(), set(), set()
daniel_probe = collections.Counter()
daniel_oldfriends_fams = set()
for f in RAW:
    with open(f, encoding="utf-8") as fh:
        for line in fh:
            line = line.strip()
            if not line:
                continue
            r = json.loads(line)
            n += 1
            fams.add(r.get("family"))
            probes.add(r.get("probe_id"))
            cid = r.get("conversation_id")
            if cid:
                convs.add(cid)
            if "Daniel" in (r.get("output_text") or ""):
                daniel_probe[r.get("probe_id")] += 1
                if r.get("probe_id") == "anchor-social-oldfriends":
                    daniel_oldfriends_fams.add(r.get("family"))

# --- the two clouds ---
c1 = json.loads(CLOUD_V1.read_text(encoding="utf-8"))
c2 = json.loads(CLOUD_V2.read_text(encoding="utf-8"))

got = {
    "v1 generations":       n,
    "v1 model_families":    len(fams),
    "v1 probes":            len(probes),
    "v1 conversations":     len(convs),
    "v1 concepts_mapped":   len(c1["f"]),
    "v1 frequency_floor":   sum(c1["f"]),
    "v1 well_basin_count":  c1["meta"]["basin_count"],
    "v1 daniel_oldfriends": daniel_probe["anchor-social-oldfriends"],
    "v1 daniel_oldfriends_fams": len(daniel_oldfriends_fams),
    "v2 concepts_mapped":   len(c2["f"]),
    "v2 model_families":    c2["meta"]["families"],
    "v2 well_basin_count":  c2["meta"]["basin_count"],
    "v2 frequency_floor":   sum(c2["f"]),
    "v2 regions":           len(c2["meta"]["regions"]),
}

EXPECT = {**EXPECT_V1, **EXPECT_V2}
print(f"{'check':24} {'expected':>10} {'got':>10}  status")
ok = fail = 0
for k in EXPECT:
    match = got[k] == EXPECT[k]
    ok += match
    fail += (not match)
    print(f"{k:24} {EXPECT[k]:>10} {got[k]:>10}  {'OK' if match else 'MISMATCH'}")

print(f"\n{ok}/{len(EXPECT)} headline numbers verified from the committed corpus.")
print("not re-derivable in this checkout, and therefore not asserted here:")
for k, why in NOT_DERIVABLE_HERE.items():
    print(f"  {k:24} {why}")
if fail:
    print(f"  {fail} MISMATCH — investigate before publishing.", file=sys.stderr)
    sys.exit(1)
print("  (universality ratio is a living figure — re-derive + date-stamp at Index time; "
      "not asserted here.)")