#!/usr/bin/env python3
"""Measure the radif (the refrain after the rhyme) in every ghazal Ganjoor holds.

Input: the per-poet SQLite files Ganjoor publishes for its desktop and Android
apps (listed at https://i.ganjoor.net/android/androidgdbs.xml), unzipped into
GDB_DIR. Output: one TSV row per ghazal, and nothing interpretive. This is an
independent second implementation of engine.mjs, used to check it.

The rule being measured: in a ghazal both hemistichs of the first couplet (the
matla) and the second hemistich of every later couplet end the same way. The
radif is whatever run of whole words is repeated identically at the end of all
of those lines; the rhyme (qafia) sits just before it. So the detector is the
longest common word-suffix over those "rhyme lines". Nothing else.

Two tokenizations are measured, because Persian orthography does not settle
what a word is:
  S (space)  : words are separated by whitespace only; a zero-width non-joiner
               (ZWNJ, U+200C) joins, so "دل\u200Cها" is one word.
  Z (zwnj)   : ZWNJ also separates, so "دل\u200Cها" is "دل" + "ها".
"""
import sqlite3, glob, json, os, re, sys, unicodedata

GDB_DIR = sys.argv[1]
OUT = sys.argv[2]

# Arabic-script marks that are vocalisation, not letters: harakat, tanwin,
# shadda, sukun, superscript alef, the small hamza above (U+0654) that the
# edition puts on ezafe "هٔ", and Quranic annotation marks. Plus tatweel.
MARKS = re.compile('[\u064B-\u065F\u0670\u06D6-\u06ED\u0640]')
PUNCT = re.compile(r'[\u060C\u061B\u061F\u066A-\u066D\u06D4!"#$%&\'()*+,\-./:;<=>?@\[\\\]^_`{|}~\u00AB\u00BB\u2018-\u201F\u2026\u200E\u200F\u202A-\u202E]')
FOLD = str.maketrans({'ي': 'ی', 'ى': 'ی', 'ك': 'ک', 'ۀ': 'ه', 'ە': 'ه', 'أ': 'ا', 'إ': 'ا', 'ٱ': 'ا', 'ؤ': 'و', '\u00A0': ' '})

def norm(t):
    t = unicodedata.normalize('NFD', t or '')
    t = unicodedata.normalize('NFC', MARKS.sub('', t)).translate(FOLD)
    t = PUNCT.sub(' ', t)
    t = re.sub('\u200C+', '\u200C', t)
    t = re.sub(r'\s*\u200C\s*', '\u200C', t).strip('\u200C ')
    return t

def words(t, mode):
    t = norm(t)
    if mode == 'Z':
        t = t.replace('\u200C', ' ')
    return [w.strip('\u200C') for w in t.split() if w.strip('\u200C')]

def common_suffix(lines):
    n = 0
    shortest = min(len(l) for l in lines)
    while n < shortest - 1:  # never let the refrain swallow a whole line
        w = lines[0][-1 - n]
        if all(l[-1 - n] == w for l in lines):
            n += 1
        else:
            break
    return n

def tolerant_suffix(lines):
    """Longest suffix of the matla's second line shared by all but at most one rhyme line."""
    ref = lines[1]
    best = 0
    for n in range(1, len(ref)):
        suf = ref[-n:]
        miss = sum(1 for l in lines if len(l) <= n or l[-n:] != suf)
        if miss <= 1:
            best = n
        else:
            break
    return best


# Bound morphemes: suffixes that classical prosody counts inside the rhyme
# (the "added letters" after the rawi), never as a radif, however the edition
# spaces them. A refrain may not START with one of these.
BOUND = set('ها های هایی ام ات اش مان تان شان ایم اید اند یم ید ند ی ای یی تر ترین گان ان ست'.split())

def letters_and_bounds(t):
    """The line as bare letters, plus the set of offsets-from-the-end where a word starts."""
    toks = words(t, 'Z')
    s = ''.join(toks)
    starts = set()
    pos = 0
    for w in toks:
        starts.add(len(s) - pos)
        pos += len(w)
    return s, starts, toks, t

def char_radif(raw_lines, tol):
    """Separator-blind refrain. Compare letters only; the refrain is the longest
    common letter-suffix such that, in every agreeing line, it begins a word:
    either the edition wrote a boundary there (space or ZWNJ), or the letters
    before it in the joined word are themselves a word of the corpus (LEX), as
    with a preverb (بر + گرفت) or a slip of the typesetter (مینا + شکست). It may
    not begin with a bound suffix. tol = lines allowed to disagree (0 or 1)."""
    L = [letters_and_bounds(x) for x in raw_lines]
    n = len(L)
    best = (0, '')
    for ref_i in ([1] if tol == 0 else [0, 1, 2]):
        if ref_i >= n: continue
        ref = L[ref_i][0]
        k = 0
        for j in range(1, len(ref)):
            suf = ref[-j:]
            if sum(1 for s, _, _, _ in L if not (len(s) > j and s.endswith(suf))) <= tol:
                k = j
            else:
                break
        # j < k: at least one shared letter (the rhyme) must stand before the refrain
        for j in range(k - 1, 0, -1):
            if j <= best[0]: break
            suf = ref[-j:]
            agree = [x for x in L if len(x[0]) > j and x[0].endswith(suf)]
            if len(agree) < n - tol: continue
            def aligned(x):
                s, st, toks, _ = x
                if j in st: return True
                # the joined word that contains the refrain's first letter
                acc = 0
                for w in reversed(toks):
                    acc += len(w)
                    if acc > j:
                        pre = w[:acc - j]
                        return pre in PREVERBS or (len(pre) >= 3 and LEX.get(pre, 0) >= LEX_MIN)
                return False
            if not all(aligned(x) for x in agree): continue
            # read the refrain's words off a line where the edition wrote the boundary,
            # else split the reference line's letters
            # at least one line must write it as a separate word
            src = next((x for x in agree if j in x[1]), None)
            if not src: continue
            acc = 0; ws = []
            for w in reversed(src[2]):
                if acc >= j: break
                ws.insert(0, w); acc += len(w)
            first_shown = re.split(r'[\s\u200C]+', tail_letters(src[3], j))[0]
            if first_shown in BOUND or LEX.get(ws[0], 0) < LEX_MIN: continue
            if len(ws[0]) < 2 and ws[0] != 'و': continue
            best = (j, ' '.join(ws))
            break
    return best

LEX_MIN = 50

ENDINGS = ['است', 'ست', 'یم', 'ام', 'م', 'ات', 'ت', 'اش', 'ش', 'ی']
def shared_ending(raw_lines):
    S = [''.join(words(x, 'Z')) for x in raw_lines]
    for e in ENDINGS:
        if all(len(s) > len(e) and s.endswith(e) for s in S):
            return e
    return ''

PREVERBS = {'بر', 'در', 'سر', 'فرو', 'فرا', 'وا', 'باز', 'ور'}

def display(t):
    t = unicodedata.normalize('NFC', t or '')
    t = MARKS.sub('', t).translate(str.maketrans({'ي': 'ی', 'ى': 'ی', 'ك': 'ک', '\u00A0': ' '}))
    t = PUNCT.sub(' ', t)
    t = re.sub('\u200C+', '\u200C', t)
    return re.sub(r'\s*\u200C\s*', '\u200C', t).strip('\u200C ')

def tail_letters(t, j):
    """The last j letters of a line, in the edition's spelling (spaces, ZWNJ kept)."""
    t = display(t)
    out = []; cnt = 0
    for ch in reversed(t):
        if cnt == j: break
        out.append(ch)
        if ch not in ' \u200C': cnt += 1
    return ''.join(reversed(out)).strip(' \u200C')
LEX = {}
for f in sorted(glob.glob(os.path.join(GDB_DIR, '*.gdb'))):
    for (t,) in sqlite3.connect(f).execute('select text from verse'):
        for w in words(t, 'Z'):
            LEX[w] = LEX.get(w, 0) + 1

# Which Ganjoor categories are collections of whole ghazals. Excluded, by title:
# unfinished ghazals, selected couplets, mixed-genre sections, Turkish ghazals,
# poems marked as later additions, and tashbib/taghazzul (qasida openings).
EXCLUDE_WORDS = ('ناتمام', 'ابیات برگزیده', 'قصاید', 'قطعات', 'مقطعات', 'ترکی', 'الحاقی', 'تغزل')
def is_ghazal_cat(t):
    return bool(t) and 'غزل' in t and not any(w in t for w in EXCLUDE_WORDS)

rows = []
poets = {}
for f in sorted(glob.glob(os.path.join(GDB_DIR, '*.gdb'))):
    c = sqlite3.connect(f)
    for pid, pname, pcat in c.execute('select id, name, cat_id from poet'):
        poets[pid] = {'id': pid, 'name': pname, 'gdb': os.path.basename(f)}
    allv = {}
    for pid, vo, pos, tx in c.execute('select poem_id, vorder, position, text from verse order by poem_id, vorder'):
        allv.setdefault(pid, []).append((vo, pos, tx))
    cats = {cid: (text, url, poet) for cid, poet, text, url in c.execute('select id, poet_id, text, url from cat')}
    for cid, (text, url, poet) in cats.items():
        if not is_ghazal_cat(text):
            continue
        for pid, title, purl in c.execute('select id, title, url from poem where cat_id=? order by id', (cid,)):
            vs = allv.get(pid, [])
            positions = [v[1] for v in vs]
            row = {'poet': poet, 'cat': cid, 'catTitle': text, 'catUrl': url, 'poem': pid, 'title': title}
            # a ghazal here is a poem of plain couplets: right, left, right, left ...
            if len(vs) < 6 or len(vs) % 2 or positions != [0, 1] * (len(vs) // 2):
                row['skip'] = 'not-plain-couplets'
                rows.append(row)
                continue
            left = [vs[i][2] for i in range(1, len(vs), 2)]
            rhyme_raw = [vs[0][2]] + left
            row['couplets'] = len(vs) // 2
            for mode in ('S', 'Z'):
                L = [words(x, mode) for x in rhyme_raw]
                if any(len(l) < 2 for l in L):
                    row['skip'] = 'empty-line'
                    break
                n = common_suffix(L)
                row['r' + mode] = ' '.join(L[1][-n:]) if n else ''
                row['n' + mode] = n
                row['t' + mode] = tolerant_suffix(L)
                # the rhyme words: the word before the refrain, per line
                if mode == 'S':
                    row['q'] = [l[-1 - n] for l in L]
            if 'skip' not in row:
                for tol, key in ((0, 'C'), (1, 'D')):
                    j, rad = char_radif(rhyme_raw, tol)
                    row['r' + key] = rad
                    row['n' + key] = len(rad.split()) if rad else 0
                    if key == 'C':
                        row['j'] = j
                        row['shown'] = tail_letters(rhyme_raw[1], j) if j else ''
                        row['end'] = '' if j else shared_ending(rhyme_raw)
            rows.append(row)

with open(OUT, 'w') as fh:
    fh.write('poem\tS\tZ\tC\tD\tj\tend\tshown\n')
    for r in rows:
        if 'skip' in r: continue
        fh.write('\t'.join(str(r[k]) for k in ('poem', 'nS', 'nZ', 'nC', 'nD', 'j', 'end', 'shown')) + '\n')
ok = [r for r in rows if 'skip' not in r]
print(len(rows), 'poems in ghazal categories;', len(ok), 'measured')