{
 "study": "what-a-slot-buys, part 1 (efficiency)",
 "written": "2026-10-29",
 "author": "ccr-fc751e34-gmn18l",
 "seed": 20261029,
 "vocabularies": {
  "o200k-gptoss": {
   "census_id": "openai-gpt-oss-hf",
   "file": "tokenizer.json",
   "sha256": "0614fe83cadab421296e664e1f48f4261fa8fef6e03e63bb75c20f38e37d07d3",
   "model": "openai/gpt-oss-20b"
  },
  "llama3": {
   "census_id": "meta-llama3",
   "file": "tokenizer.json",
   "sha256": "e134af98b985517b4f068e3755ae90d4e9cd2d45d328325dc503f1c6b2d06cc7",
   "model": "meta-llama/Llama-3.2-1B"
  },
  "qwen3": {
   "census_id": "qwen3",
   "file": "tokenizer.json",
   "sha256": "aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4",
   "model": "Qwen/Qwen3-0.6B"
  }
 },
 "library": {
  "tokenizers": "0.23.3"
 },
 "budgets_K": [
  100,
  1000,
  5000
 ],
 "arms": [
  "FREE+SWAP",
  "APPEND",
  "baseline-purason-leaf-frequency (K=5000)",
  "baseline-didenko-descending-id (K=5000)",
  "sensitivity-latin-script-additions"
 ],
 "data": {
  "fineweb2": {
   "repo": "HuggingFaceFW/fineweb-2",
   "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9",
   "licence": "odc-by",
   "map": "data/fw2-map.json"
  },
  "fineweb_english": {
   "repo": "HuggingFaceFW/fineweb",
   "revision": "9bb295ddab0e05d785b879661af7260fed5140fc",
   "config": "sample/10BT",
   "licence": "odc-by",
   "value_MB": 100,
   "webtest_MB": 20,
   "rule": "draw 130 MB by seeded random row groups of sample/10BT; decontaminate as for every language; h mod 6 == 0 goes to WEB-TEST (capped 20 MB), the rest to VALUE (capped 100 MB), each in ascending h"
  },
  "per_language_draw_MB": 8,
  "merge_MB_cap": 3,
  "value_MB_cap": 3,
  "split_hash": "first 8 hex digits of sha256(utf-8 text); WEB-TEST = h mod 10 == 0 only where FineWeb-2's own test split is under 100 KB; MERGE = remaining h even, VALUE = remaining h odd; each capped in ascending h",
  "decontamination": "drop any document sharing a whitespace 10-gram with any FLORES-200 dev or devtest sentence of any language",
  "strata": "Bible and Wikipedia tagged by document url, using FineWeb-2's own domain lists",
  "no_fineweb2_config": [
   "ace_Arab",
   "acq_Arab",
   "knc_Arab",
   "min_Arab"
  ],
  "flores_devtest_archive_sha256": "b8b0b76783024b85797e5cc75064eb83fc5288b41e9654dabc7be6ae944011f6",
  "code_value_trees": [
   {
    "repo": "python/cpython",
    "tag": "v3.13.0",
    "commit": "60403a5409ff2c3f3b07dd2ca91a7a3e096839c7",
    "paths": [
     "Lib/"
    ]
   },
   {
    "repo": "golang/go",
    "tag": "go1.23.2",
    "commit": "ed07b321aef7632f956ce991dd10fdd7e1abd827",
    "paths": [
     "src/"
    ]
   },
   {
    "repo": "rust-lang/rust",
    "tag": "1.82.0",
    "commit": "f6e511eec7342f59a25f7c0534f1dbea00d01b14",
    "paths": [
     "library/",
     "compiler/"
    ]
   },
   {
    "repo": "nodejs/node",
    "tag": "v22.9.0",
    "commit": "4631be0311fbde7b77723757e15d025727399e63",
    "paths": [
     "lib/"
    ]
   }
  ],
  "code_value_rule": "text files in the listed paths, in sorted path order per tree, trees interleaved round-robin, until 100 MB of UTF-8",
  "code_test_trees": [
   {
    "repo": "microsoft/TypeScript",
    "tag": "v5.6.3",
    "commit": "d48a5cf89a62a62d6c6ed53ffa18f070d9458b85",
    "paths": [
     "src/"
    ]
   },
   {
    "repo": "apache/commons-lang",
    "tag": "rel/commons-lang-3.17.0",
    "commit": "29ccc7665f3bc5d84155a3092ab2209a053324e6",
    "paths": [
     "src/main/"
    ]
   },
   {
    "repo": "torvalds/linux",
    "tag": "v6.11",
    "commit": "98f7e32f20d28ec452afb208f9cffc08448a2652",
    "paths": [
     "kernel/",
     "mm/"
    ]
   }
  ],
  "code_test_rule": "the same, until 20 MB"
 },
 "weighting": {
  "primary": "equal content: w_L = bytes(FLORES dev, L) / bytes(VALUE, L)",
  "sensitivities": [
   "equal bytes: w_L = 1 / bytes(VALUE, L)",
   "Common Crawl CC-MAIN-2026-43 share from the census, floor 1e-4"
  ]
 },
 "protected_set": "ASCII, and the Unicode scripts Latin, Common and Inherited",
 "candidates": {
  "letters": "code points in MERGE with t_V(c) >= 2, as closures",
  "merges": "continued BPE per language on MERGE under V's normaliser and pre-tokeniser, up to 2000 merges per language, each in at least 5 distinct documents, as closures"
 },
 "removal_rank": "equal-content-weighted VALUE usage ascending; ties: higher id first",
 "removal_skip": "cumulative VALUE rise > 0.1% for any language, or > 0.01% for English or code",
 "addition_rule": "lazy greedy on equal-content-weighted VALUE savings per slot, re-evaluated after each batch of 50 closures; skip any addition that raises any language's VALUE count",
 "placement": "new merges appended after all existing merges in selection order; ids from free ids ascending, then freed ids ascending, in selection order",
 "guardrails": {
  "G1": "English WEB-TEST and CODE-TEST each rise <= 0.05%",
  "G2": "no FLORES devtest language and no WEB-TEST slice rises > 0.5%",
  "G3": "decode(encode(s)) == normalise(s) on all TEST text",
  "G4": "every added entry strictly reachable; no retained entry newly unreachable"
 },
 "predictions": {
  "H1": {
   "K": 1000,
   "min_reduction_in_mean_premium": {
    "o200k-gptoss": 0.05,
    "llama3": 0.05,
    "qwen3": 0.02
   },
   "requires": [
    "G1",
    "G2",
    "G3",
    "G4"
   ]
  },
  "H2": {
   "swap_over_append_at_least": {
    "1000": 0.95,
    "5000": 0.8
   }
  },
  "H3": {
   "K": 5000,
   "first_slots": 500,
   "share_of_reduction_at_least": 0.5
  },
  "H4": {
   "K": 1000,
   "languages_at_2x_fall_by_at_least": 0.25,
   "max_premium_falls_by_at_least": 0.4
  },
  "H5": {
   "K": 5000,
   "value_ranked_reduction_at_least_each_baseline": true,
   "fewer_languages_rise_than_each_baseline": true
  }
 },
 "bootstrap": {
  "iterations": 10000,
  "units": "FLORES sentences; web documents"
 }
}