#!/usr/bin/env python3
"""Builds bible.db for the Vellum app.

Markup inside verse text (KJV and WEB):
  {word|H430}   word group tagged with a Strong's number
  [word]        italic (words added by translators, KJV only)
  <...>         words of Jesus
"""
import json, os, re, sqlite3, glob, sys

RAW = '/home/claude/data/raw'
OUT = sys.argv[1] if len(sys.argv) > 1 else '/home/claude/vellum/app/src/main/assets/bible.db'

CODES = ['GEN','EXO','LEV','NUM','DEU','JOS','JDG','RUT','1SA','2SA','1KI','2KI','1CH','2CH','EZR','NEH','EST','JOB','PSA','PRO','ECC','SNG','ISA','JER','LAM','EZK','DAN','HOS','JOL','AMO','OBA','JON','MIC','NAM','HAB','ZEP','HAG','ZEC','MAL',
         'MAT','MRK','LUK','JHN','ACT','ROM','1CO','2CO','GAL','EPH','PHP','COL','1TH','2TH','1TI','2TI','TIT','PHM','HEB','JAS','1PE','2PE','1JN','2JN','3JN','JUD','REV']
NAMES = ['Genesis','Exodus','Leviticus','Numbers','Deuteronomy','Joshua','Judges','Ruth','1 Samuel','2 Samuel','1 Kings','2 Kings','1 Chronicles','2 Chronicles','Ezra','Nehemiah','Esther','Job','Psalms','Proverbs','Ecclesiastes','Song of Solomon','Isaiah','Jeremiah','Lamentations','Ezekiel','Daniel','Hosea','Joel','Amos','Obadiah','Jonah','Micah','Nahum','Habakkuk','Zephaniah','Haggai','Zechariah','Malachi',
         'Matthew','Mark','Luke','John','Acts','Romans','1 Corinthians','2 Corinthians','Galatians','Ephesians','Philippians','Colossians','1 Thessalonians','2 Thessalonians','1 Timothy','2 Timothy','Titus','Philemon','Hebrews','James','1 Peter','2 Peter','1 John','2 John','3 John','Jude','Revelation']
ABBR = ['Gen','Exod','Lev','Num','Deut','Josh','Judg','Ruth','1 Sam','2 Sam','1 Kgs','2 Kgs','1 Chr','2 Chr','Ezra','Neh','Esth','Job','Ps','Prov','Eccl','Song','Isa','Jer','Lam','Ezek','Dan','Hos','Joel','Amos','Obad','Jonah','Mic','Nah','Hab','Zeph','Hag','Zech','Mal',
        'Matt','Mark','Luke','John','Acts','Rom','1 Cor','2 Cor','Gal','Eph','Phil','Col','1 Thess','2 Thess','1 Tim','2 Tim','Titus','Phlm','Heb','Jas','1 Pet','2 Pet','1 John','2 John','3 John','Jude','Rev']
# openbible.info OSIS book names
OSIS = ['Gen','Exod','Lev','Num','Deut','Josh','Judg','Ruth','1Sam','2Sam','1Kgs','2Kgs','1Chr','2Chr','Ezra','Neh','Esth','Job','Ps','Prov','Eccl','Song','Isa','Jer','Lam','Ezek','Dan','Hos','Joel','Amos','Obad','Jonah','Mic','Nah','Hab','Zeph','Hag','Zech','Mal',
        'Matt','Mark','Luke','John','Acts','Rom','1Cor','2Cor','Gal','Eph','Phil','Col','1Thess','2Thess','1Tim','2Tim','Titus','Phlm','Heb','Jas','1Pet','2Pet','1John','2John','3John','Jude','Rev']

TRANSLATIONS = [
    # id, code, name, has_markup
    (0, 'KJV', 'King James Version (1769)'),
    (1, 'WEB', 'World English Bible'),
    (2, 'BSB', 'Berean Standard Bible'),
    (3, 'ASV', 'American Standard Version (1901)'),
    (4, 'YLT', "Young's Literal Translation (1898)"),
    (5, 'DRC', 'Douay-Rheims, Challoner Revision'),
]

def strip_strong(s):
    m = re.match(r'([HG])0*(\d+)([a-z]?)', s)
    return m.group(1) + m.group(2) if m else s

def convert_usfm_text(t, keep_italic=True):
    # footnotes and cross refs
    t = re.sub(r'\\f .*?\\f\*', '', t, flags=re.S)
    t = re.sub(r'\\x .*?\\x\*', '', t, flags=re.S)
    t = t.replace('[', '').replace(']', '')
    t = t.replace('¶', '')
    # strong's words, including nested \+w
    def wrep(m):
        word, attrs = m.group(2), m.group(3)
        sm = re.search(r'strong="([^"]+)"', attrs)
        word = word.strip()
        if not sm:
            return word
        num = strip_strong(sm.group(1).split(',')[0].split(' ')[0])
        return '{%s|%s}' % (word, num)
    t = re.sub(r'\\(\+?)w (.*?)\|([^\\]*?)\\\+?w\*', wrep, t)
    t = re.sub(r'\\\+?w (.*?)\\\+?w\*', r'\1', t)  # untagged \w
    t = re.sub(r'\\\+?wj\s*', '<', t)
    t = t.replace('<*', '>')
    if keep_italic:
        t = re.sub(r'\\\+?add\s*', '[', t)
        t = t.replace('[*', ']')
    else:
        t = re.sub(r'\\\+?add\*?\s*', '', t)
    t = re.sub(r'\\\+?(nd|tl|qs|k|bk|wh|sc|it|em|bd|pn|qt|sig|ord)\*', '', t)
    t = re.sub(r'\\\+?(nd|tl|qs|k|bk|wh|sc|it|em|bd|pn|qt|sig|ord) ', '', t)
    t = re.sub(r'\\[a-z]+\d*\*?', ' ', t)
    t = re.sub(r'\s+', ' ', t).strip()
    # tidy spaces around markup
    t = re.sub(r'<\s+', '<', t)
    t = re.sub(r'\s+>', '>', t)
    t = re.sub(r'\[\s+', '[', t)
    t = re.sub(r'\s+\]', ']', t)
    t = re.sub(r'\s+([,.;:?!’”)])', r'\1', t)
    t = t.replace('<>', '').replace('[]', '')
    return t

def plain(t):
    t = re.sub(r'\{([^|}]*)\|[^}]*\}', r'\1', t)
    return t.replace('[', '').replace(']', '').replace('<', '').replace('>', '')

def parse_usfm(folder, suffix, keep_italic):
    verses = {}
    titles = {}
    for bi, code in enumerate(CODES):
        f = glob.glob(os.path.join(folder, '*%s%s.usfm' % (code, suffix)))
        assert f, code
        src = open(f[0], encoding='utf-8').read()
        ch = 0
        # split into lines by markers at line start
        cur = None
        buf = []
        def flush():
            if cur is not None:
                txt = convert_usfm_text(' '.join(buf), keep_italic)
                if txt:
                    verses[(bi + 1, cur[0], cur[1])] = txt
        for line in src.split('\n'):
            line = line.rstrip()
            m = re.match(r'\\c (\d+)', line)
            if m:
                flush(); cur = None; buf = []
                ch = int(m.group(1)); continue
            if line.startswith('\\d '):
                flush(); cur = None; buf = []
                titles[(bi + 1, ch)] = plain(convert_usfm_text(line[3:], False))
                continue
            if re.match(r'\\(s\d?|ms\d?|mt\d?|toc\d|h|id|ide|mr|r|sp|rem|cl|is\d?|ip|imt\d?) ', line) or re.match(r'\\(s\d?|ms\d?)$', line):
                continue
            # line may contain multiple verses
            parts = re.split(r'(\\v \d+[a-z]?\s)', line)
            for p in parts:
                vm = re.match(r'\\v (\d+)', p)
                if vm:
                    flush(); buf = []
                    cur = (ch, int(vm.group(1)))
                elif cur is not None:
                    buf.append(p)
        flush()
    return verses, titles

def parse_json(code):
    d = json.load(open(os.path.join(RAW, code + '.json')))
    out = {}
    def norm(n):
        n = n.replace('Revelation of John', 'Revelation')
        n = re.sub(r'^III ', '3 ', n); n = re.sub(r'^II ', '2 ', n); n = re.sub(r'^I ', '1 ', n)
        return n
    byname = {norm(b['name']): b for b in d['books']}
    for bi in range(66):
        b = byname[NAMES[bi]]
        for c in b['chapters']:
            for v in c['verses']:
                t = re.sub(r'\s+', ' ', v['text']).strip()
                t = t.replace('¶', '').strip()
                if t:
                    out[(bi + 1, c['chapter'], v['verse'])] = t
    return out

def main():
    if os.path.exists(OUT):
        os.remove(OUT)
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    db = sqlite3.connect(OUT)
    db.executescript('''
    PRAGMA page_size=4096;
    CREATE TABLE android_metadata (locale TEXT);
    INSERT INTO android_metadata VALUES ('en_US');
    CREATE TABLE translations(id INTEGER PRIMARY KEY, code TEXT, name TEXT);
    CREATE TABLE books(id INTEGER PRIMARY KEY, name TEXT, abbr TEXT, chapters INTEGER);
    CREATE TABLE chapters(book INTEGER, chapter INTEGER, verses INTEGER, PRIMARY KEY(book,chapter)) WITHOUT ROWID;
    CREATE TABLE verses(tr INTEGER, book INTEGER, chapter INTEGER, verse INTEGER, text TEXT, plain TEXT, PRIMARY KEY(tr,book,chapter,verse)) WITHOUT ROWID;
    CREATE TABLE titles(book INTEGER, chapter INTEGER, text TEXT, PRIMARY KEY(book,chapter)) WITHOUT ROWID;
    CREATE TABLE xref(book INTEGER, chapter INTEGER, verse INTEGER, tb INTEGER, tc INTEGER, tv INTEGER, te INTEGER, votes INTEGER);
    CREATE TABLE strongs(id TEXT PRIMARY KEY, lemma TEXT, xlit TEXT, pron TEXT, deriv TEXT, def TEXT, kjv TEXT) WITHOUT ROWID;
    ''')
    for t in TRANSLATIONS:
        db.execute('INSERT INTO translations VALUES (?,?,?)', t)

    kjv, titles = parse_usfm(os.path.join(RAW, 'kjvusfm'), 'eng-kjv2006', True)
    web, _ = parse_usfm(os.path.join(RAW, 'webusfm'), 'eng-web', False)
    web = {k: re.sub(r'\{([^|}]*)\|[^}]*\}', r'\1', v) for k, v in web.items()}
    sets = {0: kjv, 1: web, 2: parse_json('BSB'), 3: parse_json('ASV'), 4: parse_json('YLT'), 5: parse_json('DRC')}
    print({k: len(v) for k, v in sets.items()})

    # canonical structure from KJV
    chap = {}
    for (b, c, v) in kjv:
        chap[(b, c)] = max(chap.get((b, c), 0), v)
    for bi in range(66):
        n = max(c for (b, c) in chap if b == bi + 1)
        db.execute('INSERT INTO books VALUES (?,?,?,?)', (bi + 1, NAMES[bi], ABBR[bi], n))
    for (b, c), n in sorted(chap.items()):
        db.execute('INSERT INTO chapters VALUES (?,?,?)', (b, c, n))
    for tr, vs in sets.items():
        rows = []
        for (b, c, v), t in vs.items():
            p = plain(t) if tr in (0, 1) else t
            rows.append((tr, b, c, v, t if tr in (0, 1) else None, p))
        db.executemany('INSERT OR REPLACE INTO verses VALUES (?,?,?,?,?,?)', rows)
    for (b, c), t in titles.items():
        db.execute('INSERT INTO titles VALUES (?,?,?)', (b, c, t))

    # cross references
    idx = {o: i + 1 for i, o in enumerate(OSIS)}
    def ref(s):
        p = s.split('.')
        return idx[p[0]], int(p[1]), int(p[2])
    per = {}
    for line in open(os.path.join(RAW, 'cross_references.txt'), encoding='utf-8'):
        if line.startswith('From') or not line.strip():
            continue
        a, bb, votes = line.rstrip('\n').split('\t')
        votes = int(votes)
        if votes < 1:
            continue
        try:
            f = ref(a)
            if '-' in bb:
                s, e = bb.split('-')
                t = ref(s); te = ref(e)
                tend = te[2] if te[:2] == t[:2] else 0
            else:
                t = ref(bb); tend = 0
        except (KeyError, ValueError, IndexError):
            continue
        per.setdefault(f, []).append((votes, t, tend))
    rows = []
    for f, lst in per.items():
        lst.sort(key=lambda x: -x[0])
        for votes, t, tend in lst[:8]:
            rows.append((f[0], f[1], f[2], t[0], t[1], t[2], tend, votes))
    db.executemany('INSERT INTO xref VALUES (?,?,?,?,?,?,?,?)', rows)
    db.execute('CREATE INDEX xi ON xref(book,chapter,verse)')
    print('xrefs', len(rows))

    # strongs
    for fn, var in (('heb.js', 'strongsHebrewDictionary'), ('grk.js', 'strongsGreekDictionary')):
        s = open(os.path.join(RAW, fn), encoding='utf-8').read()
        j = s[s.index('{'):s.rindex('}') + 1]
        d = json.loads(j)
        for k, v in d.items():
            db.execute('INSERT OR REPLACE INTO strongs VALUES (?,?,?,?,?,?,?)', (
                k, v.get('lemma', ''), v.get('xlit', v.get('translit', '')), v.get('pron', ''),
                v.get('derivation', ''), v.get('strongs_def', ''), v.get('kjv_def', '')))
    db.commit()
    db.execute('VACUUM')
    db.close()
    print('size', os.path.getsize(OUT))

if __name__ == '__main__':
    main()