#!/usr/bin/env python3
"""Mechanical convention lint for the Kuku Yalanji synthetic corpus.
CHECKING tool only (the no-scripts rule bans script-AUTHORED sentences, not verification).
Enforces the cheatsheet §0 orthography contract + §1 phonotactic guardrails over every row.
Scales to any corpus size; run after every batch insert. Rules grow with the error catalog.
"""
import sqlite3, re, sys, json, collections

DB = "/mnt/donto-data/donto-resources/research/translation-training/kuku-yalanji-runpod-2026-06-30/synthetic/claude-synthetic-v1-2026-07-02/synthetic.db"

# --- whitelists -----------------------------------------------------------
LOAN_TOKENS = {"store","school","tea","toy","motorcar","motorbike","toyota","dayim","hospital","naybu","wallaby"}  # bare/embedded loans
LOAN_PREFIX = tuple(sorted(LOAN_TOKENS))
# lexicalized solid partial reduplications (§2.5.3) + reduplicated loans
SOLID_REDUP = {"janjanay","bundanday","kankaday","wunanay","banbadiy","wanarriy","nukalkal",
               "bundandan","bundanda","kankadan","bikibiki","jukijuki","wanarrin","wanjawanja","yambayamba","nganga",
               "kalakalbay","badabada","wangkarwangkar","kaykay","bulbul","bajabaja","kunkun","junjun",
               "kurrinkurrin",  # dictionary headword 'mackerel sky' (batch-269); lexicalized solid reduplication, not productive redup
               "birrbirr","ngalngal","balbal","dakaldakal","kamukamu","bajaybajay"}  # bajaybajay='painful' solid dict headword (batch-143); kamukamu=alcohol/grog, lexicalized solid form in Patz ex.51/378/425/469, not productive redup.  # dakaldakal=trocus shell (dict headword, grep-verified batch-083)  # birrbirr=lorikeet, ngalngal=sand crab — LEXICALIZED solid animal names (dict headwords, grep-verified 2026-07-06 batch-063), not productive redup  # junjun = wild fruit (dict headword, grep-verified 2026-07-06)  # bulbul=pheasant coucal, bajabaja=blue-tongue, kunkun=periwinkle (lexemes, not redup)  # kalkal REMOVED 2026-07-06: kalka-l 'rest' is a PHANTOM (see DENY_ALWAYS) — grammar kalka='spear'; rest = mara-kunbay  # kalakalbay lexicalized 'far' (grammar.md:2387);
               # badabada 'underneath' / wangkarwangkar 'above' = lexicalized locationals (Patz §3.3)
# legitimate word-final -ny: PRECAUT/SUCC endings handled by suffix check; these are lexical
NY_FINAL_OK = {"muduwajuny","wunkuny","yununy","nyunguny",
               "ngaliny","yubaliny","ngalininy","ngalinyiny","bulaniny","ngananiny","yurraniny","jananiny","nganjininy",
               "karrkany","bunkuny"}  # lexicalized temporals + ACC pronoun cells [A 2581/2582] + karrkany 'digging stick', bunkuny 'yam' (dict -ny nouns, batch-067/068)
# ---- Phantom-lexeme denylist (established at the batch-048 close, 2026-07-06) ----
# Tokens verified ABSENT from dictionary.yaml + grammar.md that had crept into the corpus as
# invented senses. DENY_ALWAYS = zero legitimate uses remain anywhere; any occurrence is an error.
DENY_ALWAYS = {"jubuny","wural","bigan","jiwuny","bugal","bugal-bugal","warngkul","wanaba",
               "muyburr","jarrkunji","munjal","dama-kana","murku","buni","kunya","milkan","kulur-jambul","jinbi","janyji",
               "jani-mumbu","yamba-jubi","jani-jubi","jani-yindu",
               "bulmba-jarray","nganka-warral","balbay-nja",
               "kalkal","kalkanya",  # kalka-l 'rest/wait' PHANTOM (swept 2026-07-06 -> mara-kunbay/bundan); grammar kalka='spear' only. NB noun kalka (spear) + kalka-ji/kalka-bu/kalka-nka are NOT matched by these (no 'kalkal' substring).
               "janyul",  # 'sand' PHANTOM (swept 2026-07-06 batch-062 -> jukar); janyul absent from dict+grammar. (ngara-as-'listen' can't be denied: ngara-l='build', ngara-nyaku 'should have built' is legit — semantic watch only.)
               "jubay",  # 'strange/outsider-finds-strange' PHANTOM (swept 2026-07-06 batch-065 watchlist -> 'kari binal'); absent dict+grammar. NB attested avoidance terms: makuku-bawal 'avoid', yirmbal/jabul 'taboo', murru-kaday 'ashamed', biwul 'mother-in-law'.
               "buburr",  # 'headland' PHANTOM (swept 2026-07-06 batch-080 -> jalun-ngandal 'sea+mouth'); absent dict+grammar (only 'bubu' exists). NB 'kubu-burr' (smoke-PERL) is unrelated — substring 'buburr' only appears there via the hyphen, so match the solid token.
               "jarrmba","jinji","daga",  # promoted from DENY_NEW 2026-07-06: now 0 in ALL accepted rows. jarrmba 'yellow'->makirr; jinji 'mix'->murni-l; daga (garbage multi-sense) -> ngandal/dirra/jungku/nanda/maja per sense. Absent dict+grammar.
               "wanka",  # promoted from DENY_NEW 2026-07-06: 0 accepted rows. Was a MIS-SEGMENTATION of RECIP-PURP -wa-nka (daji-wanka->daji-wa-nka, nyaji-wanka->nyaji-wa-nka; cf. grammar l.2003 balka-wa-nka) + one 'dry' phantom (#2155 wanka-ma->marral-ma, dict L18496). Boundary-safe: bawanka(bawa-nka leave), ngawanka(ngawa-nka baby), -wa-nka all have 'wanka' preceded by 'a' or split by a hyphen, so (?<![a-z])wanka(?![a-z]) does not fire on them.
               "yumbul","kuku-yumbul"}  # promoted from DENY_NEW 2026-07-06: 0 accepted rows. Phantom (absent dict+grammar), drifting sense across 7 dense b046-047 rows -> mapped by sense to attested kuda 'question'(L14742) / ngangkal 'ignorant'(L22299) / kuku-kari 'wordless'. No legit word contains 'yumbul' (yulmbarril is unrelated).
# DENY_NEW = phantoms still present in ACCEPTED batches (<=048); corpus-wide reconciliation
# sweeps are pending (see the dictionary-errata watchlist). Flagged only for batches >= 49 so
# they cannot re-enter new work while the old rows await their sweep.
DENY_NEW_FROM_BATCH = 49
DENY_NEW = {"jubi",  # wanka, yumbul, kuku-yumbul all promoted to DENY_ALWAYS 2026-07-06 (0 accepted rows)
            "yilbay","jarrakay","janka-yukil",
            "jukarr","kunkul","bubu-jandu","bubu-warral","jani-warra"}
# bibar REMOVED from the denylist 2026-07-06: it IS attested ('shin', dict 2131) — only the
# batch-048 'tide' sense was phantom (tide = yira). Grep-verified before delisting.
# NOTE: bare 'jani' is NOT denied — it is attested ('well cooked, done', dict 8023; jani-kari,
# jani-ma-l are legal). Its phantom SENSES (habit/self/way/point/family) are covered by the
# denied compounds above; any other non-cooking use of jani is a review-lens matter.
# tokens legitimately containing 'baja'
BAJA_OK = {"wawubaja","bajaku","baja","baja-burray","bajabaja","wawubajanga","wawubajamuny",
           "wawubajangamuny","bajaburray",
           "wanyubaja"}  # 'what else' — lexicalized interrogative (wanyu + -baja), Patz §3.6, batch-068
# lexical words legitimately ending single-r + b + vowel (NOT a locative error)
RB_OK = {"jarba"}  # jarba 'snake' (lexeme); single-r LOC would be -a/-u, but this is a root
# lexical words legitimately ending -Cnga / -unga (NOT locative allomorphy errors)
LOC_OK = {"kalnga","nganga","bunga","wanga","manga","yinga","binga","nyunga","junga","kunga","wunga",
          "jurrnga","yunga","murrnga","dunga","yalnga"}  # jurrnga-l push, yunga-l send, murrngal break-off, dungal go — verb roots (IMP), not LOC. yalnga='sea turtle' (dict headword l.22927, grep-verified batch-083) — lexical -lnga root, not a LOC form.
ILLEGAL_LETTERS = set("eofchpqstvxz")  # outside loans
LEGAL_ONSET = tuple("bdjkmnwy") + ("ng","ny")  # + loans with l-
GEMINATES = ["bb","dd","jj","kk","mm","ll","ww","yy","nn"]  # nn only when not n+ng/ny boundary; checked crudely
RHOTIC_PALATAL = ["rrj","rrny","rny","rj"]  # rj: retroflex+palatal (rare false-positive risk: none in legal clusters)

def tokens(s):
    s = re.sub(r"[.!?,—]", " ", s)
    return [t for t in s.split() if t]

def is_loanish(tok):
    t = tok.lower()
    return any(t.startswith(p) for p in LOAN_PREFIX) or t in LOAN_TOKENS

def lint_row(seq_label, kuku, analysis, english):
    issues = []
    for tok in tokens(kuku):
        if tok.startswith(("'", '"')):   # quoted meta-linguistic citation ('rr', 'nyji') — not a word
            continue
        t = tok.lower().strip("'\"")
        core = t.replace("-", "")
        if not core or not core.isalpha():
            continue
        if is_loanish(t):
            continue
        # -warral directional / standalone 'warral' 'edge' is UNATTESTED (swept 2026-07-06; dict has only
        # 'warral-warral'). Allative = general LOC (grammar 1546); '-warra' = 'inhabitants of' (grammar 2045).
        if ("warral" in core) and ("warralwarral" not in core):
            issues.append(f"phantom '-warral'/'warral' in '{tok}' — directional 'toward'=LOC(gram 1546)/DAT; 'edge' has no lexeme (use bubu-kunbayn / muru); swept 2026-07-06")
        # letters
        bad = ILLEGAL_LETTERS.intersection(core)
        if bad:
            issues.append(f"illegal letter(s) {sorted(bad)} in '{tok}'")
        # onset
        if not (core.startswith(LEGAL_ONSET)):
            if core.startswith("l") or core.startswith("r"):
                if core != "rrunyuji":  # [A 895] the sole attested rhotic-initial word
                    issues.append(f"illegal onset in '{tok}' (l/r/rr cannot begin native words)")
            elif core[0] not in "bdjkmnwy":
                issues.append(f"suspicious onset in '{tok}'")
        # coda
        if core[-1] in "bdjkmw" or core.endswith("ng"):
            issues.append(f"illegal coda in '{tok}' (stops/m/ng/w cannot end words)")
        # geminates (crude; 'rr' digraph excluded; check per hyphen-segment to avoid cross-boundary FPs)
        for seg in t.split("-"):
          for g in GEMINATES:
            if g in seg:
                # allow n+ng, n+ny boundaries: 'nng'/'nny' contain 'nn'
                if g == "nn" and ("nng" in seg or "nny" in seg):
                    continue
                issues.append(f"geminate '{g}' in '{tok}'")
        # rhotic+palatal — check PER hyphen-segment (a rhotic ending one morpheme + a palatal
        # opening the next is a legal COMPOUND boundary, e.g. dingkar-jalbu). The COMITATIVE -ji
        # after a rhotic-final stem is also attested/legal (junkurr-ji, diburr-ji [A554]) and
        # surfaces unhyphenated, so exempt a segment whose rj/rrj is that -ji comitative.
        for seg in t.split("-"):
          for rp in RHOTIC_PALATAL:
            if rp in seg:
                if rp in ("rrj","rj") and re.search(r"r-?ji", seg) and ("COMIT" in analysis or "-ji" in analysis):
                    continue
                if "jarrjal" in core:  # attested headword 'message stick'
                    continue
                if "junkurrji" in core:  # attested headword 'strong' (junkurr+ji, dict) — surfaces unhyphenated
                    continue
                issues.append(f"rhotic+palatal '{rp}' in '{tok}'")
                break
        # verb-final -ny leakage (surface past must be -n)
        if core.endswith("ny") and t not in NY_FINAL_OK and not core.endswith(("nyji","nyjiku","muny","rriny")):
            # -rriny = plural demonstrative (yalurriny/yinyarriny 'these/those people', grammar §3.7 Table 3.13;
            #          cf. interrogative root wanyurri(y)-) — a legitimate derivational -ny, not a verbal past.
            # lexical ny-final nouns exist (jalbany...) — only flag if analysis maps this token to X-ny PAST
            if re.search(re.escape(core[:-2]) + r"-ny\b", analysis) and "PAST" in analysis:
                issues.append(f"surface past -ny leakage in '{tok}' (should be -n)")
            else:
                issues.append(f"word-final -ny in '{tok}' — verify lexical (ok) vs verbal past (must be -n)")
        # word-final -iy (surface must drop y after i, §2.5.2 / E-017)
        if core.endswith("iy"):
            issues.append(f"word-final -iy in '{tok}' (surface drops y after i: nandaji, wanarri)")
        # LOC allomorphy: after a CONSONANT the locative is -ba/-a (bayan-ba, wungar-a), after -u it is
        # -ngu (bubu-ngu). A word ending consonant+nga, or u+nga, is a LOC allomorphy error (should be
        # -ba/-ngu). Whitelist genuine lexical -nga roots (kalnga 'uncle', nganga 'alone', bunga 'make').
        if (t not in LOC_OK and not t.endswith("-bunga") and t != "bunga"  # -bunga = CAUSATIVE, not LOC
                and (re.search(r"(l|n|rr|r)nga$", core) or re.search(r"[^g]unga$", core))):
            issues.append(f"LOC allomorphy in '{tok}' (after consonant use -ba/-a; after -u use -ngu, not -nga)")
        # y-final LOC (Patz Table 3.1 -mbV/y-): after a y-final stem the locative is -mbV (buray-mba,
        # wakuy-mbu), and human y-final nouns take potent -anda (karrkay-anda); a y-final stem + -nga is
        # a LOC allomorphy error. (-bunga = CAUSATIVE, ends -unga not -ynga, so excluded.)
        if t not in LOC_OK and core.endswith("ynga") and not core.endswith("bunga"):
            issues.append(f"y-final LOC in '{tok}' (after y use -mba/-mbu, or potent -anda for humans; not -nga)")
        # Rhotic LOC split (Patz §3.2.3.1): SINGLE retroflex /r/ LOC is long-vowel -a/-u (wangkar-a,
        # jukar-a); TRILL /rr/ LOC is -ba/-bu (wabarr-ba, wujurr-bu). So a SINGLE r + -ba is a LOC error
        # (should be -a). NOTE -bu is NOT flagged: after a single r it is the INSTRUMENTAL (dawar-bu 'by
        # the star', muyar-bu 'by the wind'), a legal case. Whitelist true lexemes (jarba 'snake').
        if t not in RB_OK and t.split("-")[-1] not in RB_OK and re.search(r"[^r]rba$", core):  # -bu excluded: it is the INSTRUMENTAL (dawar-bu 'by the star'); last-segment check clears reduplicated roots (jarba-jarba 'snakes')
            issues.append(f"rhotic LOC in '{tok}' (single /r/ LOC is -a not -ba; -bu here would be instrumental)")
        # baja: DEG2 adjectival degree suffix -baja ('more/-er', Patz §3.2.3.8) is a LEGAL
        # hyphenated enclitic (ngulkurr-baja 'better', wubul-baja 'more', kalbay-baja 'longer').
        # Only the emphatic/coordinating clitic 'baja' fused with NO hyphen is an error.
        if t.endswith("baja") and not t.endswith("-baja") and t not in BAJA_OK:
            if not core.endswith("manubaja"):
                issues.append(f"fused clitic baja in '{tok}' (write as separate word)")
        # unhyphenated full reduplication
        if "-" not in t and len(core) >= 6 and len(core) % 2 == 0:
            h = len(core)//2
            if core[:h] == core[h:] and t not in SOLID_REDUP:
                issues.append(f"unhyphenated reduplication '{tok}' (productive redup is hyphenated)")
    # analysis field sanity
    if " | " not in analysis:
        issues.append("analysis missing ' | ' gloss separator")
    # english side
    e = english.strip()
    e_first = e.lstrip("'\"‘“")  # a leading quotation mark is a legal sentence opener (direct speech)
    e_last = e.rstrip("'\"’”")   # a trailing quotation mark is legal too (direct-speech question: ...?")
    if e and (not e_first[:1].isupper() or (e_last and e_last[-1] not in ".!?")):
        issues.append("english missing capital or final punctuation")
    return issues

def main():
    db = sqlite3.connect(DB)
    rows = db.execute("""SELECT b.label, b.id, s.seq, s.kuku, s.analysis, s.english
                         FROM sentences s JOIN batches b ON b.id=s.batch_id
                         WHERE s.status != 'rejected' ORDER BY b.id, s.seq""").fetchall()
    # exact-duplicate kuku detection
    seen = collections.defaultdict(list)
    for label, bid, seq, ku, an, en in rows:
        seen[ku.strip().lower()].append(f"{label}.{seq:02d}")
    n_issues = 0
    for label, bid, seq, ku, an, en in rows:
        issues = lint_row(f"{label}.{seq:02d}", ku, an, en)
        # phantom-lexeme denylist (word-boundary; hyphen counts as a boundary joiner)
        low = ku.lower()
        deny = DENY_ALWAYS | (DENY_NEW if bid >= DENY_NEW_FROM_BATCH else set())
        for pt in deny:
            if re.search(rf"(?<![a-z]){re.escape(pt)}(?![a-z])", low):
                issues.append(f"phantom lexeme '{pt}' (unattested in dictionary.yaml/grammar.md; see errata watchlist)")
        # kija sense guard (2026-07-06): kija = 'moon, month' ONLY (grammar 2509). Any kija token
        # whose row gloss lacks moon/month is the old 'cold' sense error (cold = kiway/kiway-kiway,
        # cold season = bulur-iji). Batches 39-48 reconciled 2026-07-06; batches 2-30 are legit moon.
        if re.search(r"(?<![a-z])kija", low) and not re.search(r"moon|month", an.lower()):
            issues.append("kija without moon/month in gloss — kija is 'moon' ONLY (cold = kiway/kiway-kiway; season = bulur-iji)")
        dups = seen[ku.strip().lower()]
        if len(dups) > 1:
            issues.append(f"exact duplicate kuku across rows: {dups}")
        if issues:
            n_issues += len(issues)
            print(f"{label}.{seq:02d} | {ku}")
            for i in issues:
                print(f"   ⚠ {i}")
    total = len(rows)
    words = db.execute("SELECT SUM(LENGTH(kuku)-LENGTH(REPLACE(kuku,' ',''))+1) FROM sentences WHERE status IN ('accepted','revised')").fetchone()[0]
    sents = db.execute("SELECT COUNT(*) FROM sentences WHERE status IN ('accepted','revised')").fetchone()[0]
    print(f"\nlint: {total} rows scanned, {n_issues} issues flagged")
    print(f"GOAL METRIC: {sents} / 20000 verified sentences  ({words or 0} Kuku words)")
    sys.exit(1 if n_issues else 0)

if __name__ == "__main__":
    main()
