#!/usr/bin/env python3 """ build_arcs.py — compute the emotional arc of famous public-domain books the same way Reagan, Mitchell, Kiley, Danforth & Dodds did in "The emotional arcs of stories are dominated by six basic shapes" (EPJ Data Science, 2016): a sliding window over the text, each word scored by the labMT happiness lexicon, with a "lens" that drops neutral words to cut noise. Then normalise every arc to the same length so books of different sizes sit on one axis. Sources (both fetched live, cached under data/): - labMT-en-v2 happiness lexicon, from the authors' own hedonometer.org API. - Project Gutenberg plain-text UTF-8 books (public domain). Output: public/strata/shapes-of-stories/arcs.json (shipped, read by the page); a copy run on its own writes arcs.json beside itself research/shapes-of-stories/data/* (caches + provenance) Run: python3 build_arcs.py Verify (recompute + compare to shipped): python3 verify.py """ import json, os, re, sys, urllib.request, hashlib from pathlib import Path HERE = Path(__file__).resolve().parent DATA = HERE / "data" CACHE = DATA / "texts" # In the repository the arcs are written where the page reads them. A copy downloaded on its own # (from /checks/) has no such tree around it, so it writes arcs.json beside itself instead of # creating directories above wherever it was saved. _REPO_OUT = HERE.parent.parent / "public" / "strata" / "shapes-of-stories" OUT = (_REPO_OUT if _REPO_OUT.is_dir() else HERE) / "arcs.json" DATA.mkdir(parents=True, exist_ok=True) CACHE.mkdir(parents=True, exist_ok=True) # --- method parameters (match the paper's approach, with one documented change) -- # Reagan et al. used a FIXED 10,000-word window over books of 20k–100k words. To let # beloved short works (Alice, the plays, The Metamorphosis) sit on the same axis, we # use a window of WIN_FRAC of each book, CAPPED at the paper's 10,000 words and FLOORED # at WIN_MIN. For books >=100k words this is exactly the paper's 10k cap; for shorter # works the window shrinks so the arc still spans ~90% of the book instead of a stub. # The trade-off is honest and flagged on the page: below ~10k words labMT is noisier. WIN_FRAC = 0.10 # sliding window = 10% of the book... WIN_CAP = 10000 # ...capped at the paper's 10,000-word window... WIN_MIN = 2500 # ...and floored here (below this the signal is too thin). STEP_FRAC = 0.25 # slide by 25% of a window each step (dense overlap) LENS_LO, LENS_HI = 4.0, 6.0 # drop words in [4,6]: the neutral band ("lens", Δh=1) RESAMPLE = 100 # every arc resampled to 100 points (0..100% through the book) WEAK_FIT = 0.20 # |best correlation| below this => "no dominant shape" (honest) # --- the curated corpus: famous, recognisable, public-domain, varied shapes --- # (id, poetic short title, author). IDs verified against Gutenberg title lines. BOOKS = [ (46, "A Christmas Carol", "Dickens"), (1342, "Pride and Prejudice", "Austen"), (84, "Frankenstein", "Shelley"), (11, "Alice in Wonderland", "Carroll"), (345, "Dracula", "Stoker"), (2701, "Moby-Dick", "Melville"), (5200, "The Metamorphosis", "Kafka"), (1400, "Great Expectations", "Dickens"), (174, "The Picture of Dorian Gray", "Wilde"), (514, "Little Women", "Alcott"), (1260, "Jane Eyre", "Bronte"), (768, "Wuthering Heights", "Bronte"), (76, "Huckleberry Finn", "Twain"), (74, "The Adventures of Tom Sawyer", "Twain"), (1513, "Romeo and Juliet", "Shakespeare"), (1524, "Hamlet", "Shakespeare"), (98, "A Tale of Two Cities", "Dickens"), (2554, "Crime and Punishment", "Dostoevsky"), (1184, "The Count of Monte Cristo", "Dumas"), (120, "Treasure Island", "Stevenson"), (1399, "Anna Karenina", "Tolstoy"), (1661, "The Adventures of Sherlock Holmes", "Doyle"), (16, "Peter Pan", "Barrie"), (55, "The Wizard of Oz", "Baum"), (219, "Heart of Darkness", "Conrad"), (158, "Emma", "Austen"), (730, "Oliver Twist", "Dickens"), (2591, "Grimms' Fairy Tales", "Grimm"), (2600, "War and Peace", "Tolstoy"), ] UA = "ArtificialWasteland/1.0 (shapes-of-stories research; contact via artwaste.land)" def fetch(url, dest, binary=False): if dest.exists() and dest.stat().st_size > 0: return dest.read_bytes() if binary else dest.read_text(encoding="utf-8", errors="replace") # Gutenberg is strict about rapid automated access; be polite (delay + backoff retry). import time last = None for attempt in range(4): try: req = urllib.request.Request(url, headers={"User-Agent": UA}) with urllib.request.urlopen(req, timeout=90) as r: raw = r.read() dest.write_bytes(raw) time.sleep(2) # ~human cadence between requests return raw if binary else raw.decode("utf-8", errors="replace") except Exception as e: last = e time.sleep(2 * (2 ** attempt)) # 2s, 4s, 8s, 16s raise last def load_labmt(): p = DATA / "labMT-en-v2.json" if not p.exists(): fetch("https://hedonometer.org/api/v1/words/?format=json&wordlist__title=labMT-en-v2&limit=30000", p, binary=True) objs = json.load(open(p))["objects"] # full map (for reporting) and the lensed scoring map (neutral band removed) full = {o["word"].lower(): float(o["happs"]) for o in objs} lensed = {w: h for w, h in full.items() if not (LENS_LO <= h <= LENS_HI)} return full, lensed GUT_START = re.compile(r"\*\*\*\s*START OF (?:THE|THIS) PROJECT GUTENBERG.*?\*\*\*", re.I) GUT_END = re.compile(r"\*\*\*\s*END OF (?:THE|THIS) PROJECT GUTENBERG.*?\*\*\*", re.I) def get_text(book_id): # canonical plain-text UTF-8 endpoint url = f"https://www.gutenberg.org/cache/epub/{book_id}/pg{book_id}.txt" raw = fetch(url, CACHE / f"{book_id}.txt") # strip Gutenberg boilerplate header/footer m1 = GUT_START.search(raw) body = raw[m1.end():] if m1 else raw m2 = GUT_END.search(body) if m2: body = body[:m2.start()] return body WORD = re.compile(r"[a-z]+(?:'[a-z]+)?") def tokens(text): return WORD.findall(text.lower()) def window_for(n): """sliding-window size for a book of n total words: 10% of the book, capped at the paper's 10,000 words, floored at WIN_MIN. Returns min(n, ...) so tiny texts still work.""" return min(n, max(WIN_MIN, min(WIN_CAP, round(WIN_FRAC * n)))) def arc(words, lensed): """windowed mean happiness over lensed words, resampled to RESAMPLE points in [0,1].""" n = len(words) scores = [(i, lensed[w]) for i, w in enumerate(words) if w in lensed] win = window_for(n) import bisect pos = [i for i, _ in scores] val = [h for _, h in scores] step = max(1, int(win * STEP_FRAC)) pts = [] last_start = max(0, n - win) for start in list(range(0, last_start + 1, step)) + [last_start]: end = start + win lo = bisect.bisect_left(pos, start) hi = bisect.bisect_left(pos, end) if hi > lo: centre = (start + min(end, n)) / 2 / n m = sum(val[lo:hi]) / (hi - lo) pts.append((centre, m)) pts = sorted(set((round(c, 6), v) for c, v in pts)) if not pts: return [5.0] * RESAMPLE, n, len(scores), win # resample to RESAMPLE evenly spaced points in [0,1] by linear interpolation xs = [p[0] for p in pts] ys = [p[1] for p in pts] out = [] for k in range(RESAMPLE): x = k / (RESAMPLE - 1) if x <= xs[0]: out.append(ys[0]); continue if x >= xs[-1]: out.append(ys[-1]); continue import bisect as _b r = _b.bisect_left(xs, x) x0, x1 = xs[r-1], xs[r] y0, y1 = ys[r-1], ys[r] t = (x - x0) / (x1 - x0) if x1 > x0 else 0 out.append(y0 + t * (y1 - y0)) return out, n, len(scores), win def zscore(a): m = sum(a) / len(a) var = sum((x - m) ** 2 for x in a) / len(a) sd = var ** 0.5 or 1.0 return [(x - m) / sd for x in a] # --- the six idealised shapes (Reagan et al. names), as templates over [0,1] --- import math def _tpl(fn): return [fn(k / (RESAMPLE - 1)) for k in range(RESAMPLE)] SHAPES = { "Rags to riches": _tpl(lambda x: x), # steady rise "Tragedy": _tpl(lambda x: -x), # steady fall "Man in a hole": _tpl(lambda x: -math.sin(math.pi * x)), # fall then rise "Icarus": _tpl(lambda x: math.sin(math.pi * x)), # rise then fall "Cinderella": _tpl(lambda x: math.sin(2 * math.pi * x)), # rise-fall-rise "Oedipus": _tpl(lambda x: -math.sin(2 * math.pi * x)), # fall-rise-fall } SHAPES_Z = {k: zscore(v) for k, v in SHAPES.items()} def corr(a, b): n = len(a) ma = sum(a)/n; mb = sum(b)/n num = sum((a[i]-ma)*(b[i]-mb) for i in range(n)) da = (sum((a[i]-ma)**2 for i in range(n)))**0.5 db = (sum((b[i]-mb)**2 for i in range(n)))**0.5 return num/(da*db) if da and db else 0.0 def classify(arc_vals): z = zscore(arc_vals) ranked = [] for name, tpl in SHAPES_Z.items(): ranked.append((name, round(corr(z, tpl), 3))) ranked.sort(key=lambda t: -t[1]) best, bestc = ranked[0] weak = bestc < WEAK_FIT # no template resembles this arc -> honest "no dominant shape" return best, bestc, ranked, weak def main(): full, lensed = load_labmt() print(f"labMT: {len(full)} words, {len(lensed)} after lens [{LENS_LO},{LENS_HI}] removed") results = [] for bid, title, author in BOOKS: try: text = get_text(bid) except Exception as e: print(f" SKIP {title} ({bid}): {e}") continue toks = tokens(text) a, nwords, nscored, win = arc(toks, lensed) best, bestc, ranked, weak = classify(a) # amplitude of the raw arc in happiness units (peak-to-trough), for the reader amp = round(max(a) - min(a), 3) results.append({ "id": bid, "title": title, "author": author, "words": nwords, "scoredWords": nscored, "window": win, "arc": [round(v, 4) for v in a], "shape": ("No dominant shape" if weak else best), "shapeCorr": round(bestc, 3), "weak": weak, "amp": amp, "ranked": ranked, }) tag = "weak-fit" if weak else best print(f" {title:32s} {nwords:>7} w win={win:>5} {tag:18s} r={bestc:+.2f} amp={amp:.2f}") payload = { "meta": { "method": "labMT windowed happiness (Reagan et al. 2016 method)", "winFrac": WIN_FRAC, "winCap": WIN_CAP, "winMin": WIN_MIN, "stepFrac": STEP_FRAC, "lens": [LENS_LO, LENS_HI], "resample": RESAMPLE, "weakFit": WEAK_FIT, "lexicon": "labMT-en-v2 (hedonometer.org)", "lexiconWords": len(full), "lexiconLensed": len(lensed), "generatedBy": "research/shapes-of-stories/build_arcs.py", }, "shapes": {k: [round(v, 4) for v in SHAPES[k]] for k in SHAPES}, "books": results, } OUT.write_text(json.dumps(payload, separators=(",", ":"))) print(f"\nwrote {OUT} ({OUT.stat().st_size} bytes, {len(results)} books)") if __name__ == "__main__": main()