{
 "generated": "2026-09-07T03:05:26.423Z",
 "slugs": {
  "the-coin-you-cant-fake": {
   "status": "compared",
   "published_generated": "2026-08-20",
   "rerun_generated": "2026-09-07",
   "published_draws": 300,
   "rerun_draws": 150,
   "published_answered": 206,
   "rerun_answered": 83,
   "metric": "mean switch proportion",
   "published_roster": [
    "nvidia/nemotron-3-super-120b-a12b:free",
    "nvidia/nemotron-3-ultra-550b-a55b:free",
    "nvidia/nemotron-nano-9b-v2:free",
    "openai/gpt-oss-20b:free",
    "poolside/laguna-s-2.1:free",
    "poolside/laguna-xs-2.1:free"
   ],
   "rerun_roster": [
    "nvidia/nemotron-3-super-120b-a12b:free",
    "nvidia/nemotron-3-ultra-550b-a55b:free",
    "poolside/laguna-s-2.1:free",
    "poolside/laguna-xs-2.1:free"
   ],
   "shared_roster": [
    "nvidia/nemotron-3-super-120b-a12b:free",
    "nvidia/nemotron-3-ultra-550b-a55b:free",
    "poolside/laguna-s-2.1:free",
    "poolside/laguna-xs-2.1:free"
   ],
   "pairs": [
    {
     "model": "nvidia/nemotron-3-super-120b-a12b:free",
     "protocol": "plain-temperature-1",
     "metric": "mean switch proportion",
     "published": {
      "n": 5,
      "value": 0.6666666666666667,
      "extra": {
       "raw": 25,
       "unparseable": 20,
       "uniqueSequences": 5
      }
     },
     "rerun": {
      "n": 1,
      "value": 0.5959595959595959,
      "extra": {
       "raw": 25,
       "unparseable": 24,
       "uniqueSequences": 1
      }
     },
     "delta": -0.07070707070707083
    },
    {
     "model": "nvidia/nemotron-3-super-120b-a12b:free",
     "protocol": "plain-temperature-0",
     "metric": "mean switch proportion",
     "published": {
      "n": 1,
      "value": 0.696969696969697,
      "extra": {
       "raw": 25,
       "unparseable": 24,
       "uniqueSequences": 1
      }
     },
     "rerun": {
      "n": 4,
      "value": 0.696969696969697,
      "extra": {
       "raw": 15,
       "unparseable": 11,
       "uniqueSequences": 4
      }
     },
     "delta": 0
    },
    {
     "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
     "protocol": "plain-temperature-1",
     "metric": "mean switch proportion",
     "published": {
      "n": 0,
      "value": null,
      "extra": {
       "raw": 25,
       "unparseable": 25,
       "uniqueSequences": 0
      }
     },
     "rerun": {
      "n": 0,
      "value": null,
      "extra": {
       "raw": 10,
       "unparseable": 10,
       "uniqueSequences": 0
      }
     },
     "delta": null
    },
    {
     "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
     "protocol": "plain-temperature-0",
     "metric": "mean switch proportion",
     "published": {
      "n": 0,
      "value": null,
      "extra": {
       "raw": 25,
       "unparseable": 25,
       "uniqueSequences": 0
      }
     },
     "rerun": {
      "n": 0,
      "value": null,
      "extra": {
       "raw": 0,
       "unparseable": 0,
       "uniqueSequences": 0
      }
     },
     "delta": null
    },
    {
     "model": "poolside/laguna-s-2.1:free",
     "protocol": "plain-temperature-1",
     "metric": "mean switch proportion",
     "published": {
      "n": 0,
      "value": null,
      "extra": {
       "raw": 25,
       "unparseable": 25,
       "uniqueSequences": 0
      }
     },
     "rerun": {
      "n": 0,
      "value": null,
      "extra": {
       "raw": 25,
       "unparseable": 25,
       "uniqueSequences": 0
      }
     },
     "delta": null
    },
    {
     "model": "poolside/laguna-s-2.1:free",
     "protocol": "plain-temperature-0",
     "metric": "mean switch proportion",
     "published": {
      "n": 0,
      "value": null,
      "extra": {
       "raw": 25,
       "unparseable": 25,
       "uniqueSequences": 0
      }
     },
     "rerun": {
      "n": 0,
      "value": null,
      "extra": {
       "raw": 25,
       "unparseable": 25,
       "uniqueSequences": 0
      }
     },
     "delta": null
    },
    {
     "model": "poolside/laguna-xs-2.1:free",
     "protocol": "plain-temperature-1",
     "metric": "mean switch proportion",
     "published": {
      "n": 0,
      "value": null,
      "extra": {
       "raw": 25,
       "unparseable": 25,
       "uniqueSequences": 0
      }
     },
     "rerun": {
      "n": 0,
      "value": null,
      "extra": {
       "raw": 25,
       "unparseable": 25,
       "uniqueSequences": 0
      }
     },
     "delta": null
    },
    {
     "model": "poolside/laguna-xs-2.1:free",
     "protocol": "plain-temperature-0",
     "metric": "mean switch proportion",
     "published": {
      "n": 0,
      "value": null,
      "extra": {
       "raw": 25,
       "unparseable": 25,
       "uniqueSequences": 0
      }
     },
     "rerun": {
      "n": 0,
      "value": null,
      "extra": {
       "raw": 25,
       "unparseable": 25,
       "uniqueSequences": 0
      }
     },
     "delta": null
    },
    {
     "model": "(pooled, shared models)",
     "protocol": "plain-temperature-1",
     "metric": "mean switch proportion",
     "published": {
      "n": 5,
      "value": 0.6666666666666667,
      "extra": {
       "raw": 100,
       "unparseable": 95,
       "uniqueSequences": 5
      }
     },
     "rerun": {
      "n": 1,
      "value": 0.5959595959595959,
      "extra": {
       "raw": 85,
       "unparseable": 84,
       "uniqueSequences": 1
      }
     },
     "delta": -0.07070707070707083
    },
    {
     "model": "(pooled, shared models)",
     "protocol": "plain-temperature-0",
     "metric": "mean switch proportion",
     "published": {
      "n": 1,
      "value": 0.696969696969697,
      "extra": {
       "raw": 100,
       "unparseable": 99,
       "uniqueSequences": 1
      }
     },
     "rerun": {
      "n": 4,
      "value": 0.696969696969697,
      "extra": {
       "raw": 65,
       "unparseable": 61,
       "uniqueSequences": 4
      }
     },
     "delta": 0
    }
   ],
   "unpaired_rerun_rows": 0
  },
  "guess-two-thirds": {
   "status": "compared",
   "published_generated": "2026-08-20",
   "rerun_generated": "2026-09-07",
   "published_draws": 300,
   "rerun_draws": 150,
   "published_answered": 144,
   "rerun_answered": 69,
   "metric": "mean guess",
   "published_roster": [
    "google/gemma-4-31b-it:free",
    "nvidia/nemotron-3-super-120b-a12b:free",
    "nvidia/nemotron-3-ultra-550b-a55b:free",
    "nvidia/nemotron-nano-9b-v2:free",
    "openai/gpt-oss-20b:free",
    "poolside/laguna-s-2.1:free"
   ],
   "rerun_roster": [
    "nvidia/nemotron-3-super-120b-a12b:free",
    "poolside/laguna-s-2.1:free",
    "poolside/laguna-xs-2.1:free"
   ],
   "shared_roster": [
    "nvidia/nemotron-3-super-120b-a12b:free",
    "poolside/laguna-s-2.1:free"
   ],
   "pairs": [
    {
     "model": "nvidia/nemotron-3-super-120b-a12b:free",
     "protocol": "famous",
     "metric": "mean guess",
     "published": {
      "n": 19,
      "value": 0,
      "extra": {
       "raw": 25,
       "median": 0,
       "unparseable": 6
      }
     },
     "rerun": {
      "n": 19,
      "value": 0,
      "extra": {
       "raw": 25,
       "median": 0,
       "unparseable": 6
      }
     },
     "delta": 0
    },
    {
     "model": "nvidia/nemotron-3-super-120b-a12b:free",
     "protocol": "isomorph",
     "metric": "mean guess",
     "published": {
      "n": 0,
      "value": null,
      "extra": {
       "raw": 25,
       "median": null,
       "unparseable": 25
      }
     },
     "rerun": {
      "n": 0,
      "value": null,
      "extra": {
       "raw": 25,
       "median": null,
       "unparseable": 25
      }
     },
     "delta": null
    },
    {
     "model": "poolside/laguna-s-2.1:free",
     "protocol": "famous",
     "metric": "mean guess",
     "published": {
      "n": 24,
      "value": 0.16920833333333332,
      "extra": {
       "raw": 25,
       "median": 0,
       "unparseable": 1
      }
     },
     "rerun": {
      "n": 13,
      "value": 0.3076923076923077,
      "extra": {
       "raw": 25,
       "median": 0,
       "unparseable": 12
      }
     },
     "delta": 0.1384839743589744
    },
    {
     "model": "poolside/laguna-s-2.1:free",
     "protocol": "isomorph",
     "metric": "mean guess",
     "published": {
      "n": 19,
      "value": 31.473684210526315,
      "extra": {
       "raw": 25,
       "median": 40,
       "unparseable": 6
      }
     },
     "rerun": {
      "n": 7,
      "value": 27,
      "extra": {
       "raw": 25,
       "median": 40,
       "unparseable": 18
      }
     },
     "delta": -4.473684210526315
    },
    {
     "model": "(pooled, shared models)",
     "protocol": "famous",
     "metric": "mean guess",
     "published": {
      "n": 43,
      "value": 0.09444186046511628,
      "extra": {
       "raw": 50,
       "median": 0,
       "unparseable": 7
      }
     },
     "rerun": {
      "n": 32,
      "value": 0.125,
      "extra": {
       "raw": 50,
       "median": 0,
       "unparseable": 18
      }
     },
     "delta": 0.03055813953488372
    },
    {
     "model": "(pooled, shared models)",
     "protocol": "isomorph",
     "metric": "mean guess",
     "published": {
      "n": 19,
      "value": 31.473684210526315,
      "extra": {
       "raw": 50,
       "median": 40,
       "unparseable": 31
      }
     },
     "rerun": {
      "n": 7,
      "value": 27,
      "extra": {
       "raw": 50,
       "median": 40,
       "unparseable": 43
      }
     },
     "delta": -4.473684210526315
    }
   ],
   "unpaired_rerun_rows": 2
  },
  "wason-246": {
   "status": "compared",
   "published_generated": "2026-08-21",
   "rerun_generated": "2026-09-07",
   "published_draws": 352,
   "rerun_draws": 400,
   "published_answered": 256,
   "rerun_answered": 218,
   "metric": "positive-test share of classified first actions",
   "published_roster": [
    "nvidia/nemotron-3-super-120b-a12b:free",
    "nvidia/nemotron-3-ultra-550b-a55b:free",
    "nvidia/nemotron-nano-9b-v2:free",
    "openai/gpt-oss-20b:free",
    "poolside/laguna-s-2.1:free",
    "poolside/laguna-xs-2.1:free"
   ],
   "rerun_roster": [
    "nvidia/nemotron-3-super-120b-a12b:free",
    "nvidia/nemotron-3-ultra-550b-a55b:free",
    "poolside/laguna-s-2.1:free",
    "poolside/laguna-xs-2.1:free"
   ],
   "shared_roster": [
    "nvidia/nemotron-3-super-120b-a12b:free",
    "nvidia/nemotron-3-ultra-550b-a55b:free",
    "poolside/laguna-s-2.1:free",
    "poolside/laguna-xs-2.1:free"
   ],
   "pairs": [
    {
     "model": "nvidia/nemotron-3-super-120b-a12b:free",
     "protocol": "famous_single_first_action",
     "metric": "positive-test share of classified first actions",
     "published": {
      "n": 22,
      "value": 0.045454545454545456,
      "extra": {
       "raw": 25,
       "match": 1,
       "not_match": 0,
       "no_current_guess": 21,
       "unparseable": 3
      }
     },
     "rerun": {
      "n": 9,
      "value": 0,
      "extra": {
       "raw": 25,
       "match": 0,
       "not_match": 0,
       "no_current_guess": 9,
       "unparseable": 16
      }
     },
     "delta": -0.045454545454545456
    },
    {
     "model": "nvidia/nemotron-3-super-120b-a12b:free",
     "protocol": "famous_dual_first_action",
     "metric": "positive-test share of classified first actions",
     "published": {
      "n": 24,
      "value": 0,
      "extra": {
       "raw": 25,
       "match": 0,
       "not_match": 0,
       "no_current_guess": 24,
       "unparseable": 1
      }
     },
     "rerun": {
      "n": 7,
      "value": 0,
      "extra": {
       "raw": 25,
       "match": 0,
       "not_match": 0,
       "no_current_guess": 7,
       "unparseable": 18
      }
     },
     "delta": 0
    },
    {
     "model": "nvidia/nemotron-3-super-120b-a12b:free",
     "protocol": "isomorph_single_first_action",
     "metric": "positive-test share of classified first actions",
     "published": {
      "n": 9,
      "value": 0,
      "extra": {
       "raw": 9,
       "match": 0,
       "not_match": 0,
       "no_current_guess": 9,
       "unparseable": 0
      }
     },
     "rerun": {
      "n": 22,
      "value": 0.09090909090909091,
      "extra": {
       "raw": 25,
       "match": 2,
       "not_match": 0,
       "no_current_guess": 20,
       "unparseable": 3
      }
     },
     "delta": 0.09090909090909091
    },
    {
     "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
     "protocol": "famous_single_first_action",
     "metric": "positive-test share of classified first actions",
     "published": {
      "n": 21,
      "value": 0.047619047619047616,
      "extra": {
       "raw": 25,
       "match": 1,
       "not_match": 0,
       "no_current_guess": 20,
       "unparseable": 4
      }
     },
     "rerun": {
      "n": 18,
      "value": 0.1111111111111111,
      "extra": {
       "raw": 25,
       "match": 2,
       "not_match": 0,
       "no_current_guess": 16,
       "unparseable": 7
      }
     },
     "delta": 0.06349206349206349
    },
    {
     "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
     "protocol": "famous_dual_first_action",
     "metric": "positive-test share of classified first actions",
     "published": {
      "n": 21,
      "value": 0.047619047619047616,
      "extra": {
       "raw": 25,
       "match": 1,
       "not_match": 0,
       "no_current_guess": 20,
       "unparseable": 4
      }
     },
     "rerun": {
      "n": 21,
      "value": 0.09523809523809523,
      "extra": {
       "raw": 25,
       "match": 2,
       "not_match": 2,
       "no_current_guess": 17,
       "unparseable": 4
      }
     },
     "delta": 0.047619047619047616
    },
    {
     "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
     "protocol": "isomorph_single_first_action",
     "metric": "positive-test share of classified first actions",
     "published": {
      "n": 2,
      "value": 0,
      "extra": {
       "raw": 9,
       "match": 0,
       "not_match": 0,
       "no_current_guess": 2,
       "unparseable": 7
      }
     },
     "rerun": {
      "n": 22,
      "value": 0.045454545454545456,
      "extra": {
       "raw": 25,
       "match": 1,
       "not_match": 0,
       "no_current_guess": 21,
       "unparseable": 3
      }
     },
     "delta": 0.045454545454545456
    },
    {
     "model": "poolside/laguna-s-2.1:free",
     "protocol": "famous_single_first_action",
     "metric": "positive-test share of classified first actions",
     "published": {
      "n": 18,
      "value": 0.8888888888888888,
      "extra": {
       "raw": 25,
       "match": 16,
       "not_match": 1,
       "no_current_guess": 1,
       "unparseable": 7
      }
     },
     "rerun": {
      "n": 12,
      "value": 0.9166666666666666,
      "extra": {
       "raw": 25,
       "match": 11,
       "not_match": 0,
       "no_current_guess": 1,
       "unparseable": 13
      }
     },
     "delta": 0.02777777777777779
    },
    {
     "model": "poolside/laguna-s-2.1:free",
     "protocol": "famous_dual_first_action",
     "metric": "positive-test share of classified first actions",
     "published": {
      "n": 19,
      "value": 0.15789473684210525,
      "extra": {
       "raw": 25,
       "match": 3,
       "not_match": 0,
       "no_current_guess": 16,
       "unparseable": 6
      }
     },
     "rerun": {
      "n": 10,
      "value": 0.4,
      "extra": {
       "raw": 25,
       "match": 4,
       "not_match": 0,
       "no_current_guess": 6,
       "unparseable": 15
      }
     },
     "delta": 0.24210526315789477
    },
    {
     "model": "poolside/laguna-s-2.1:free",
     "protocol": "isomorph_single_first_action",
     "metric": "positive-test share of classified first actions",
     "published": {
      "n": 8,
      "value": 0.75,
      "extra": {
       "raw": 9,
       "match": 6,
       "not_match": 0,
       "no_current_guess": 2,
       "unparseable": 1
      }
     },
     "rerun": {
      "n": 13,
      "value": 0.5384615384615384,
      "extra": {
       "raw": 25,
       "match": 7,
       "not_match": 0,
       "no_current_guess": 6,
       "unparseable": 12
      }
     },
     "delta": -0.21153846153846156
    },
    {
     "model": "poolside/laguna-xs-2.1:free",
     "protocol": "famous_single_first_action",
     "metric": "positive-test share of classified first actions",
     "published": {
      "n": 6,
      "value": 0.6666666666666666,
      "extra": {
       "raw": 25,
       "match": 4,
       "not_match": 0,
       "no_current_guess": 2,
       "unparseable": 19
      }
     },
     "rerun": {
      "n": 8,
      "value": 0.75,
      "extra": {
       "raw": 25,
       "match": 6,
       "not_match": 0,
       "no_current_guess": 2,
       "unparseable": 17
      }
     },
     "delta": 0.08333333333333337
    },
    {
     "model": "poolside/laguna-xs-2.1:free",
     "protocol": "famous_dual_first_action",
     "metric": "positive-test share of classified first actions",
     "published": {
      "n": 12,
      "value": 0.75,
      "extra": {
       "raw": 25,
       "match": 9,
       "not_match": 0,
       "no_current_guess": 3,
       "unparseable": 13
      }
     },
     "rerun": {
      "n": 4,
      "value": 0.5,
      "extra": {
       "raw": 25,
       "match": 2,
       "not_match": 0,
       "no_current_guess": 2,
       "unparseable": 21
      }
     },
     "delta": -0.25
    },
    {
     "model": "poolside/laguna-xs-2.1:free",
     "protocol": "isomorph_single_first_action",
     "metric": "positive-test share of classified first actions",
     "published": {
      "n": 0,
      "value": null,
      "extra": {
       "raw": 8,
       "match": 0,
       "not_match": 0,
       "no_current_guess": 0,
       "unparseable": 8
      }
     },
     "rerun": {
      "n": 6,
      "value": 0.3333333333333333,
      "extra": {
       "raw": 25,
       "match": 2,
       "not_match": 0,
       "no_current_guess": 4,
       "unparseable": 19
      }
     },
     "delta": null
    },
    {
     "model": "(pooled, shared models)",
     "protocol": "famous_single_first_action",
     "metric": "positive-test share of classified first actions",
     "published": {
      "n": 67,
      "value": 0.3283582089552239,
      "extra": {
       "raw": 100,
       "match": 22,
       "unparseable": 33
      }
     },
     "rerun": {
      "n": 47,
      "value": 0.40425531914893614,
      "extra": {
       "raw": 100,
       "match": 19,
       "unparseable": 53
      }
     },
     "delta": 0.07589711019371226
    },
    {
     "model": "(pooled, shared models)",
     "protocol": "famous_dual_first_action",
     "metric": "positive-test share of classified first actions",
     "published": {
      "n": 76,
      "value": 0.17105263157894737,
      "extra": {
       "raw": 100,
       "match": 13,
       "unparseable": 24
      }
     },
     "rerun": {
      "n": 42,
      "value": 0.19047619047619047,
      "extra": {
       "raw": 100,
       "match": 8,
       "unparseable": 58
      }
     },
     "delta": 0.019423558897243093
    },
    {
     "model": "(pooled, shared models)",
     "protocol": "isomorph_single_first_action",
     "metric": "positive-test share of classified first actions",
     "published": {
      "n": 19,
      "value": 0.3157894736842105,
      "extra": {
       "raw": 35,
       "match": 6,
       "unparseable": 16
      }
     },
     "rerun": {
      "n": 63,
      "value": 0.19047619047619047,
      "extra": {
       "raw": 100,
       "match": 12,
       "unparseable": 37
      }
     },
     "delta": -0.12531328320802004
    }
   ],
   "unpaired_rerun_rows": 5
  },
  "anchoring-rct": {
   "status": "compared",
   "published_generated": "2026-08-20",
   "rerun_generated": "2026-09-07",
   "published_draws": 600,
   "rerun_draws": 400,
   "published_answered": 482,
   "rerun_answered": 225,
   "metric": "median estimate",
   "published_roster": [
    "google/gemma-4-31b-it:free",
    "nvidia/nemotron-3-super-120b-a12b:free",
    "nvidia/nemotron-3-ultra-550b-a55b:free",
    "nvidia/nemotron-nano-9b-v2:free",
    "openai/gpt-oss-20b:free",
    "poolside/laguna-s-2.1:free"
   ],
   "rerun_roster": [
    "nvidia/nemotron-3-super-120b-a12b:free",
    "nvidia/nemotron-3-ultra-550b-a55b:free",
    "poolside/laguna-s-2.1:free",
    "poolside/laguna-xs-2.1:free"
   ],
   "shared_roster": [
    "nvidia/nemotron-3-super-120b-a12b:free",
    "nvidia/nemotron-3-ultra-550b-a55b:free",
    "poolside/laguna-s-2.1:free"
   ],
   "pairs": [
    {
     "model": "nvidia/nemotron-3-super-120b-a12b:free",
     "protocol": "houston-low",
     "metric": "median estimate",
     "published": {
      "n": 24,
      "value": 2099451,
      "extra": {
       "raw": 25,
       "unparseable": 1
      }
     },
     "rerun": {
      "n": 22,
      "value": 2099451,
      "extra": {
       "raw": 25,
       "unparseable": 3
      }
     },
     "delta": 0
    },
    {
     "model": "nvidia/nemotron-3-super-120b-a12b:free",
     "protocol": "houston-high",
     "metric": "median estimate",
     "published": {
      "n": 25,
      "value": 2099451,
      "extra": {
       "raw": 25,
       "unparseable": 0
      }
     },
     "rerun": {
      "n": 18,
      "value": 2099451,
      "extra": {
       "raw": 25,
       "unparseable": 7
      }
     },
     "delta": 0
    },
    {
     "model": "nvidia/nemotron-3-super-120b-a12b:free",
     "protocol": "african-un-low",
     "metric": "median estimate",
     "published": {
      "n": 24,
      "value": 28,
      "extra": {
       "raw": 25,
       "unparseable": 1
      }
     },
     "rerun": {
      "n": 24,
      "value": 28,
      "extra": {
       "raw": 25,
       "unparseable": 1
      }
     },
     "delta": 0
    },
    {
     "model": "nvidia/nemotron-3-super-120b-a12b:free",
     "protocol": "african-un-high",
     "metric": "median estimate",
     "published": {
      "n": 24,
      "value": 28,
      "extra": {
       "raw": 25,
       "unparseable": 1
      }
     },
     "rerun": {
      "n": 24,
      "value": 28,
      "extra": {
       "raw": 25,
       "unparseable": 1
      }
     },
     "delta": 0
    },
    {
     "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
     "protocol": "houston-low",
     "metric": "median estimate",
     "published": {
      "n": 25,
      "value": 2099451,
      "extra": {
       "raw": 25,
       "unparseable": 0
      }
     },
     "rerun": {
      "n": 18,
      "value": 2099451,
      "extra": {
       "raw": 25,
       "unparseable": 7
      }
     },
     "delta": 0
    },
    {
     "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
     "protocol": "houston-high",
     "metric": "median estimate",
     "published": {
      "n": 25,
      "value": 2099451,
      "extra": {
       "raw": 25,
       "unparseable": 0
      }
     },
     "rerun": {
      "n": 19,
      "value": 2099451,
      "extra": {
       "raw": 25,
       "unparseable": 6
      }
     },
     "delta": 0
    },
    {
     "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
     "protocol": "african-un-low",
     "metric": "median estimate",
     "published": {
      "n": 18,
      "value": 28,
      "extra": {
       "raw": 25,
       "unparseable": 7
      }
     },
     "rerun": {
      "n": 18,
      "value": 28,
      "extra": {
       "raw": 25,
       "unparseable": 7
      }
     },
     "delta": 0
    },
    {
     "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
     "protocol": "african-un-high",
     "metric": "median estimate",
     "published": {
      "n": 17,
      "value": 28,
      "extra": {
       "raw": 25,
       "unparseable": 6
      }
     },
     "rerun": {
      "n": 14,
      "value": 28,
      "extra": {
       "raw": 25,
       "unparseable": 11
      }
     },
     "delta": 0
    },
    {
     "model": "poolside/laguna-s-2.1:free",
     "protocol": "houston-low",
     "metric": "median estimate",
     "published": {
      "n": 22,
      "value": 2207000,
      "extra": {
       "raw": 25,
       "unparseable": 3
      }
     },
     "rerun": {
      "n": 11,
      "value": 2244000,
      "extra": {
       "raw": 25,
       "unparseable": 14
      }
     },
     "delta": 37000
    },
    {
     "model": "poolside/laguna-s-2.1:free",
     "protocol": "houston-high",
     "metric": "median estimate",
     "published": {
      "n": 23,
      "value": 2257000,
      "extra": {
       "raw": 25,
       "unparseable": 2
      }
     },
     "rerun": {
      "n": 8,
      "value": 2249418,
      "extra": {
       "raw": 25,
       "unparseable": 17
      }
     },
     "delta": -7582
    },
    {
     "model": "poolside/laguna-s-2.1:free",
     "protocol": "african-un-low",
     "metric": "median estimate",
     "published": {
      "n": 23,
      "value": 54,
      "extra": {
       "raw": 25,
       "unparseable": 2
      }
     },
     "rerun": {
      "n": 9,
      "value": 54,
      "extra": {
       "raw": 25,
       "unparseable": 16
      }
     },
     "delta": 0
    },
    {
     "model": "poolside/laguna-s-2.1:free",
     "protocol": "african-un-high",
     "metric": "median estimate",
     "published": {
      "n": 22,
      "value": 54,
      "extra": {
       "raw": 25,
       "unparseable": 3
      }
     },
     "rerun": {
      "n": 10,
      "value": 54,
      "extra": {
       "raw": 25,
       "unparseable": 15
      }
     },
     "delta": 0
    },
    {
     "model": "(pooled, shared models)",
     "protocol": "houston-low",
     "metric": "median estimate",
     "published": {
      "n": 71,
      "value": 2099451,
      "extra": {
       "raw": 75,
       "unparseable": 4
      }
     },
     "rerun": {
      "n": 51,
      "value": 2099451,
      "extra": {
       "raw": 75,
       "unparseable": 24
      }
     },
     "delta": 0
    },
    {
     "model": "(pooled, shared models)",
     "protocol": "houston-high",
     "metric": "median estimate",
     "published": {
      "n": 73,
      "value": 2099451,
      "extra": {
       "raw": 75,
       "unparseable": 2
      }
     },
     "rerun": {
      "n": 45,
      "value": 2099451,
      "extra": {
       "raw": 75,
       "unparseable": 30
      }
     },
     "delta": 0
    },
    {
     "model": "(pooled, shared models)",
     "protocol": "african-un-low",
     "metric": "median estimate",
     "published": {
      "n": 65,
      "value": 28,
      "extra": {
       "raw": 75,
       "unparseable": 10
      }
     },
     "rerun": {
      "n": 51,
      "value": 28,
      "extra": {
       "raw": 75,
       "unparseable": 24
      }
     },
     "delta": 0
    },
    {
     "model": "(pooled, shared models)",
     "protocol": "african-un-high",
     "metric": "median estimate",
     "published": {
      "n": 63,
      "value": 28,
      "extra": {
       "raw": 75,
       "unparseable": 10
      }
     },
     "rerun": {
      "n": 48,
      "value": 28,
      "extra": {
       "raw": 75,
       "unparseable": 27
      }
     },
     "delta": 0
    }
   ],
   "unpaired_rerun_rows": 4
  },
  "calibration-intervals": {
   "status": "compared",
   "published_generated": "2026-08-20",
   "rerun_generated": "2026-09-07",
   "published_draws": 450,
   "rerun_draws": 248,
   "published_answered": 367,
   "rerun_answered": 133,
   "metric": "mean eight-item coverage",
   "published_roster": [
    "nvidia/nemotron-3-super-120b-a12b:free",
    "nvidia/nemotron-3-ultra-550b-a55b:free",
    "nvidia/nemotron-nano-9b-v2:free",
    "openai/gpt-oss-20b:free",
    "poolside/laguna-s-2.1:free",
    "poolside/laguna-xs-2.1:free"
   ],
   "rerun_roster": [
    "nvidia/nemotron-3-super-120b-a12b:free",
    "nvidia/nemotron-3-ultra-550b-a55b:free",
    "poolside/laguna-s-2.1:free",
    "poolside/laguna-xs-2.1:free"
   ],
   "shared_roster": [
    "nvidia/nemotron-3-super-120b-a12b:free",
    "nvidia/nemotron-3-ultra-550b-a55b:free",
    "poolside/laguna-s-2.1:free",
    "poolside/laguna-xs-2.1:free"
   ],
   "pairs": [
    {
     "model": "nvidia/nemotron-3-super-120b-a12b:free",
     "protocol": "direct_range",
     "metric": "mean eight-item coverage",
     "published": {
      "n": 18,
      "value": 0.9791666666666666,
      "extra": {}
     },
     "rerun": {
      "n": 22,
      "value": 0.9943181818181818,
      "extra": {}
     },
     "delta": 0.015151515151515138
    },
    {
     "model": "nvidia/nemotron-3-super-120b-a12b:free",
     "protocol": "split (joined pairs)",
     "metric": "mean eight-item coverage",
     "published": {
      "n": 5,
      "value": 0.95,
      "extra": {
       "joinedPairs": 5,
       "invalidPairs": 20,
       "unmatchedCalls": 0
      }
     },
     "rerun": {
      "n": 9,
      "value": 0.9722222222222222,
      "extra": {
       "joinedPairs": 9,
       "invalidPairs": 16,
       "unmatchedCalls": 0
      }
     },
     "delta": 0.022222222222222254
    },
    {
     "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
     "protocol": "direct_range",
     "metric": "mean eight-item coverage",
     "published": {
      "n": 17,
      "value": 0.9705882352941176,
      "extra": {}
     },
     "rerun": {
      "n": 16,
      "value": 0.9765625,
      "extra": {}
     },
     "delta": 0.0059742647058823595
    },
    {
     "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
     "protocol": "split (joined pairs)",
     "metric": "mean eight-item coverage",
     "published": {
      "n": 2,
      "value": 1,
      "extra": {
       "joinedPairs": 2,
       "invalidPairs": 23,
       "unmatchedCalls": 0
      }
     },
     "rerun": {
      "n": 0,
      "value": null,
      "extra": {
       "joinedPairs": 0,
       "invalidPairs": 0,
       "unmatchedCalls": 0
      }
     },
     "delta": null
    },
    {
     "model": "poolside/laguna-s-2.1:free",
     "protocol": "direct_range",
     "metric": "mean eight-item coverage",
     "published": {
      "n": 3,
      "value": 0.5833333333333334,
      "extra": {}
     },
     "rerun": {
      "n": 1,
      "value": 0.5,
      "extra": {}
     },
     "delta": -0.08333333333333337
    },
    {
     "model": "poolside/laguna-s-2.1:free",
     "protocol": "split (joined pairs)",
     "metric": "mean eight-item coverage",
     "published": {
      "n": 4,
      "value": 0.75,
      "extra": {
       "joinedPairs": 4,
       "invalidPairs": 21,
       "unmatchedCalls": 0
      }
     },
     "rerun": {
      "n": 1,
      "value": 1,
      "extra": {
       "joinedPairs": 1,
       "invalidPairs": 24,
       "unmatchedCalls": 0
      }
     },
     "delta": 0.25
    },
    {
     "model": "poolside/laguna-xs-2.1:free",
     "protocol": "direct_range",
     "metric": "mean eight-item coverage",
     "published": {
      "n": 14,
      "value": 0.44642857142857145,
      "extra": {}
     },
     "rerun": {
      "n": 3,
      "value": 0.3333333333333333,
      "extra": {}
     },
     "delta": -0.11309523809523814
    },
    {
     "model": "poolside/laguna-xs-2.1:free",
     "protocol": "split (joined pairs)",
     "metric": "mean eight-item coverage",
     "published": {
      "n": 0,
      "value": null,
      "extra": {
       "joinedPairs": 0,
       "invalidPairs": 25,
       "unmatchedCalls": 0
      }
     },
     "rerun": {
      "n": 1,
      "value": 0.625,
      "extra": {
       "joinedPairs": 1,
       "invalidPairs": 24,
       "unmatchedCalls": 0
      }
     },
     "delta": null
    },
    {
     "model": "(pooled, shared models)",
     "protocol": "direct_range",
     "metric": "mean eight-item coverage",
     "published": {
      "n": 52,
      "value": 0.8100961538461539,
      "extra": {}
     },
     "rerun": {
      "n": 42,
      "value": 0.9285714285714286,
      "extra": {}
     },
     "delta": 0.11847527472527475
    },
    {
     "model": "(pooled, shared models)",
     "protocol": "split (joined pairs)",
     "metric": "mean eight-item coverage",
     "published": {
      "n": 11,
      "value": 0.8863636363636364,
      "extra": {
       "joinedPairs": 11
      }
     },
     "rerun": {
      "n": 11,
      "value": 0.9431818181818182,
      "extra": {
       "joinedPairs": 11
      }
     },
     "delta": 0.05681818181818188
    }
   ],
   "unpaired_rerun_rows": 0
  },
  "serial-position": {
   "status": "compared",
   "published_generated": "2026-08-20",
   "rerun_generated": "2026-09-07",
   "published_draws": 360,
   "rerun_draws": 240,
   "published_answered": 203,
   "rerun_answered": 135,
   "metric": "overall exact-match reporting rate",
   "published_roster": [
    "nvidia/nemotron-3-super-120b-a12b:free",
    "nvidia/nemotron-3-ultra-550b-a55b:free",
    "nvidia/nemotron-nano-9b-v2:free",
    "openai/gpt-oss-20b:free",
    "poolside/laguna-s-2.1:free",
    "poolside/laguna-xs-2.1:free"
   ],
   "rerun_roster": [
    "nvidia/nemotron-3-super-120b-a12b:free",
    "nvidia/nemotron-3-ultra-550b-a55b:free",
    "poolside/laguna-s-2.1:free",
    "poolside/laguna-xs-2.1:free"
   ],
   "shared_roster": [
    "nvidia/nemotron-3-super-120b-a12b:free",
    "nvidia/nemotron-3-ultra-550b-a55b:free",
    "poolside/laguna-s-2.1:free",
    "poolside/laguna-xs-2.1:free"
   ],
   "pairs": [
    {
     "model": "nvidia/nemotron-3-super-120b-a12b:free",
     "protocol": "all 60 lists",
     "metric": "overall exact-match reporting rate",
     "published": {
      "n": 525,
      "value": 1,
      "extra": {
       "completions": 35,
       "unparseable": 0,
       "error": 25
      }
     },
     "rerun": {
      "n": 825,
      "value": 1,
      "extra": {
       "completions": 55,
       "unparseable": 0,
       "error": 5
      }
     },
     "delta": 0
    },
    {
     "model": "nvidia/nemotron-3-ultra-550b-a55b:free",
     "protocol": "all 60 lists",
     "metric": "overall exact-match reporting rate",
     "published": {
      "n": 510,
      "value": 1,
      "extra": {
       "completions": 34,
       "unparseable": 0,
       "error": 26
      }
     },
     "rerun": {
      "n": 735,
      "value": 1,
      "extra": {
       "completions": 49,
       "unparseable": 0,
       "error": 11
      }
     },
     "delta": 0
    },
    {
     "model": "poolside/laguna-s-2.1:free",
     "protocol": "all 60 lists",
     "metric": "overall exact-match reporting rate",
     "published": {
      "n": 525,
      "value": 0.9657142857142857,
      "extra": {
       "completions": 35,
       "unparseable": 0,
       "error": 25
      }
     },
     "rerun": {
      "n": 330,
      "value": 0.9242424242424242,
      "extra": {
       "completions": 22,
       "unparseable": 0,
       "error": 38
      }
     },
     "delta": -0.04147186147186155
    },
    {
     "model": "poolside/laguna-xs-2.1:free",
     "protocol": "all 60 lists",
     "metric": "overall exact-match reporting rate",
     "published": {
      "n": 525,
      "value": 1,
      "extra": {
       "completions": 35,
       "unparseable": 0,
       "error": 25
      }
     },
     "rerun": {
      "n": 135,
      "value": 1,
      "extra": {
       "completions": 9,
       "unparseable": 0,
       "error": 51
      }
     },
     "delta": 0
    },
    {
     "model": "(pooled, shared models)",
     "protocol": "all 60 lists",
     "metric": "overall exact-match reporting rate",
     "published": {
      "n": 2085,
      "value": 0.9913669064748202,
      "extra": {
       "completions": 139
      }
     },
     "rerun": {
      "n": 2025,
      "value": 0.9876543209876543,
      "extra": {
       "completions": 135
      }
     },
     "delta": -0.003712585487165909
    }
   ],
   "unpaired_rerun_rows": 0
  }
 }
}