230 lines
11 KiB
Python
230 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Builds bible.db for the Vellum app.
|
|
|
|
Markup inside verse text (KJV and WEB):
|
|
{word|H430} word group tagged with a Strong's number
|
|
[word] italic (words added by translators, KJV only)
|
|
<...> words of Jesus
|
|
"""
|
|
import json, os, re, sqlite3, glob, sys
|
|
|
|
RAW = '/home/claude/data/raw'
|
|
OUT = sys.argv[1] if len(sys.argv) > 1 else '/home/claude/vellum/app/src/main/assets/bible.db'
|
|
|
|
CODES = ['GEN','EXO','LEV','NUM','DEU','JOS','JDG','RUT','1SA','2SA','1KI','2KI','1CH','2CH','EZR','NEH','EST','JOB','PSA','PRO','ECC','SNG','ISA','JER','LAM','EZK','DAN','HOS','JOL','AMO','OBA','JON','MIC','NAM','HAB','ZEP','HAG','ZEC','MAL',
|
|
'MAT','MRK','LUK','JHN','ACT','ROM','1CO','2CO','GAL','EPH','PHP','COL','1TH','2TH','1TI','2TI','TIT','PHM','HEB','JAS','1PE','2PE','1JN','2JN','3JN','JUD','REV']
|
|
NAMES = ['Genesis','Exodus','Leviticus','Numbers','Deuteronomy','Joshua','Judges','Ruth','1 Samuel','2 Samuel','1 Kings','2 Kings','1 Chronicles','2 Chronicles','Ezra','Nehemiah','Esther','Job','Psalms','Proverbs','Ecclesiastes','Song of Solomon','Isaiah','Jeremiah','Lamentations','Ezekiel','Daniel','Hosea','Joel','Amos','Obadiah','Jonah','Micah','Nahum','Habakkuk','Zephaniah','Haggai','Zechariah','Malachi',
|
|
'Matthew','Mark','Luke','John','Acts','Romans','1 Corinthians','2 Corinthians','Galatians','Ephesians','Philippians','Colossians','1 Thessalonians','2 Thessalonians','1 Timothy','2 Timothy','Titus','Philemon','Hebrews','James','1 Peter','2 Peter','1 John','2 John','3 John','Jude','Revelation']
|
|
ABBR = ['Gen','Exod','Lev','Num','Deut','Josh','Judg','Ruth','1 Sam','2 Sam','1 Kgs','2 Kgs','1 Chr','2 Chr','Ezra','Neh','Esth','Job','Ps','Prov','Eccl','Song','Isa','Jer','Lam','Ezek','Dan','Hos','Joel','Amos','Obad','Jonah','Mic','Nah','Hab','Zeph','Hag','Zech','Mal',
|
|
'Matt','Mark','Luke','John','Acts','Rom','1 Cor','2 Cor','Gal','Eph','Phil','Col','1 Thess','2 Thess','1 Tim','2 Tim','Titus','Phlm','Heb','Jas','1 Pet','2 Pet','1 John','2 John','3 John','Jude','Rev']
|
|
# openbible.info OSIS book names
|
|
OSIS = ['Gen','Exod','Lev','Num','Deut','Josh','Judg','Ruth','1Sam','2Sam','1Kgs','2Kgs','1Chr','2Chr','Ezra','Neh','Esth','Job','Ps','Prov','Eccl','Song','Isa','Jer','Lam','Ezek','Dan','Hos','Joel','Amos','Obad','Jonah','Mic','Nah','Hab','Zeph','Hag','Zech','Mal',
|
|
'Matt','Mark','Luke','John','Acts','Rom','1Cor','2Cor','Gal','Eph','Phil','Col','1Thess','2Thess','1Tim','2Tim','Titus','Phlm','Heb','Jas','1Pet','2Pet','1John','2John','3John','Jude','Rev']
|
|
|
|
TRANSLATIONS = [
|
|
# id, code, name, has_markup
|
|
(0, 'KJV', 'King James Version (1769)'),
|
|
(1, 'WEB', 'World English Bible'),
|
|
(2, 'BSB', 'Berean Standard Bible'),
|
|
(3, 'ASV', 'American Standard Version (1901)'),
|
|
(4, 'YLT', "Young's Literal Translation (1898)"),
|
|
(5, 'DRC', 'Douay-Rheims, Challoner Revision'),
|
|
]
|
|
|
|
def strip_strong(s):
|
|
m = re.match(r'([HG])0*(\d+)([a-z]?)', s)
|
|
return m.group(1) + m.group(2) if m else s
|
|
|
|
def convert_usfm_text(t, keep_italic=True):
|
|
# footnotes and cross refs
|
|
t = re.sub(r'\\f .*?\\f\*', '', t, flags=re.S)
|
|
t = re.sub(r'\\x .*?\\x\*', '', t, flags=re.S)
|
|
t = t.replace('[', '').replace(']', '')
|
|
t = t.replace('¶', '')
|
|
# strong's words, including nested \+w
|
|
def wrep(m):
|
|
word, attrs = m.group(2), m.group(3)
|
|
sm = re.search(r'strong="([^"]+)"', attrs)
|
|
word = word.strip()
|
|
if not sm:
|
|
return word
|
|
num = strip_strong(sm.group(1).split(',')[0].split(' ')[0])
|
|
return '{%s|%s}' % (word, num)
|
|
t = re.sub(r'\\(\+?)w (.*?)\|([^\\]*?)\\\+?w\*', wrep, t)
|
|
t = re.sub(r'\\\+?w (.*?)\\\+?w\*', r'\1', t) # untagged \w
|
|
t = re.sub(r'\\\+?wj\s*', '<', t)
|
|
t = t.replace('<*', '>')
|
|
if keep_italic:
|
|
t = re.sub(r'\\\+?add\s*', '[', t)
|
|
t = t.replace('[*', ']')
|
|
else:
|
|
t = re.sub(r'\\\+?add\*?\s*', '', t)
|
|
t = re.sub(r'\\\+?(nd|tl|qs|k|bk|wh|sc|it|em|bd|pn|qt|sig|ord)\*', '', t)
|
|
t = re.sub(r'\\\+?(nd|tl|qs|k|bk|wh|sc|it|em|bd|pn|qt|sig|ord) ', '', t)
|
|
t = re.sub(r'\\[a-z]+\d*\*?', ' ', t)
|
|
t = re.sub(r'\s+', ' ', t).strip()
|
|
# tidy spaces around markup
|
|
t = re.sub(r'<\s+', '<', t)
|
|
t = re.sub(r'\s+>', '>', t)
|
|
t = re.sub(r'\[\s+', '[', t)
|
|
t = re.sub(r'\s+\]', ']', t)
|
|
t = re.sub(r'\s+([,.;:?!’”)])', r'\1', t)
|
|
t = t.replace('<>', '').replace('[]', '')
|
|
return t
|
|
|
|
def plain(t):
|
|
t = re.sub(r'\{([^|}]*)\|[^}]*\}', r'\1', t)
|
|
return t.replace('[', '').replace(']', '').replace('<', '').replace('>', '')
|
|
|
|
def parse_usfm(folder, suffix, keep_italic):
|
|
verses = {}
|
|
titles = {}
|
|
for bi, code in enumerate(CODES):
|
|
f = glob.glob(os.path.join(folder, '*%s%s.usfm' % (code, suffix)))
|
|
assert f, code
|
|
src = open(f[0], encoding='utf-8').read()
|
|
ch = 0
|
|
# split into lines by markers at line start
|
|
cur = None
|
|
buf = []
|
|
def flush():
|
|
if cur is not None:
|
|
txt = convert_usfm_text(' '.join(buf), keep_italic)
|
|
if txt:
|
|
verses[(bi + 1, cur[0], cur[1])] = txt
|
|
for line in src.split('\n'):
|
|
line = line.rstrip()
|
|
m = re.match(r'\\c (\d+)', line)
|
|
if m:
|
|
flush(); cur = None; buf = []
|
|
ch = int(m.group(1)); continue
|
|
if line.startswith('\\d '):
|
|
flush(); cur = None; buf = []
|
|
titles[(bi + 1, ch)] = plain(convert_usfm_text(line[3:], False))
|
|
continue
|
|
if re.match(r'\\(s\d?|ms\d?|mt\d?|toc\d|h|id|ide|mr|r|sp|rem|cl|is\d?|ip|imt\d?) ', line) or re.match(r'\\(s\d?|ms\d?)$', line):
|
|
continue
|
|
# line may contain multiple verses
|
|
parts = re.split(r'(\\v \d+[a-z]?\s)', line)
|
|
for p in parts:
|
|
vm = re.match(r'\\v (\d+)', p)
|
|
if vm:
|
|
flush(); buf = []
|
|
cur = (ch, int(vm.group(1)))
|
|
elif cur is not None:
|
|
buf.append(p)
|
|
flush()
|
|
return verses, titles
|
|
|
|
def parse_json(code):
|
|
d = json.load(open(os.path.join(RAW, code + '.json')))
|
|
out = {}
|
|
def norm(n):
|
|
n = n.replace('Revelation of John', 'Revelation')
|
|
n = re.sub(r'^III ', '3 ', n); n = re.sub(r'^II ', '2 ', n); n = re.sub(r'^I ', '1 ', n)
|
|
return n
|
|
byname = {norm(b['name']): b for b in d['books']}
|
|
for bi in range(66):
|
|
b = byname[NAMES[bi]]
|
|
for c in b['chapters']:
|
|
for v in c['verses']:
|
|
t = re.sub(r'\s+', ' ', v['text']).strip()
|
|
t = t.replace('¶', '').strip()
|
|
if t:
|
|
out[(bi + 1, c['chapter'], v['verse'])] = t
|
|
return out
|
|
|
|
def main():
|
|
if os.path.exists(OUT):
|
|
os.remove(OUT)
|
|
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
|
db = sqlite3.connect(OUT)
|
|
db.executescript('''
|
|
PRAGMA page_size=4096;
|
|
CREATE TABLE android_metadata (locale TEXT);
|
|
INSERT INTO android_metadata VALUES ('en_US');
|
|
CREATE TABLE translations(id INTEGER PRIMARY KEY, code TEXT, name TEXT);
|
|
CREATE TABLE books(id INTEGER PRIMARY KEY, name TEXT, abbr TEXT, chapters INTEGER);
|
|
CREATE TABLE chapters(book INTEGER, chapter INTEGER, verses INTEGER, PRIMARY KEY(book,chapter)) WITHOUT ROWID;
|
|
CREATE TABLE verses(tr INTEGER, book INTEGER, chapter INTEGER, verse INTEGER, text TEXT, plain TEXT, PRIMARY KEY(tr,book,chapter,verse)) WITHOUT ROWID;
|
|
CREATE TABLE titles(book INTEGER, chapter INTEGER, text TEXT, PRIMARY KEY(book,chapter)) WITHOUT ROWID;
|
|
CREATE TABLE xref(book INTEGER, chapter INTEGER, verse INTEGER, tb INTEGER, tc INTEGER, tv INTEGER, te INTEGER, votes INTEGER);
|
|
CREATE TABLE strongs(id TEXT PRIMARY KEY, lemma TEXT, xlit TEXT, pron TEXT, deriv TEXT, def TEXT, kjv TEXT) WITHOUT ROWID;
|
|
''')
|
|
for t in TRANSLATIONS:
|
|
db.execute('INSERT INTO translations VALUES (?,?,?)', t)
|
|
|
|
kjv, titles = parse_usfm(os.path.join(RAW, 'kjvusfm'), 'eng-kjv2006', True)
|
|
web, _ = parse_usfm(os.path.join(RAW, 'webusfm'), 'eng-web', False)
|
|
web = {k: re.sub(r'\{([^|}]*)\|[^}]*\}', r'\1', v) for k, v in web.items()}
|
|
sets = {0: kjv, 1: web, 2: parse_json('BSB'), 3: parse_json('ASV'), 4: parse_json('YLT'), 5: parse_json('DRC')}
|
|
print({k: len(v) for k, v in sets.items()})
|
|
|
|
# canonical structure from KJV
|
|
chap = {}
|
|
for (b, c, v) in kjv:
|
|
chap[(b, c)] = max(chap.get((b, c), 0), v)
|
|
for bi in range(66):
|
|
n = max(c for (b, c) in chap if b == bi + 1)
|
|
db.execute('INSERT INTO books VALUES (?,?,?,?)', (bi + 1, NAMES[bi], ABBR[bi], n))
|
|
for (b, c), n in sorted(chap.items()):
|
|
db.execute('INSERT INTO chapters VALUES (?,?,?)', (b, c, n))
|
|
for tr, vs in sets.items():
|
|
rows = []
|
|
for (b, c, v), t in vs.items():
|
|
p = plain(t) if tr in (0, 1) else t
|
|
rows.append((tr, b, c, v, t if tr in (0, 1) else None, p))
|
|
db.executemany('INSERT OR REPLACE INTO verses VALUES (?,?,?,?,?,?)', rows)
|
|
for (b, c), t in titles.items():
|
|
db.execute('INSERT INTO titles VALUES (?,?,?)', (b, c, t))
|
|
|
|
# cross references
|
|
idx = {o: i + 1 for i, o in enumerate(OSIS)}
|
|
def ref(s):
|
|
p = s.split('.')
|
|
return idx[p[0]], int(p[1]), int(p[2])
|
|
per = {}
|
|
for line in open(os.path.join(RAW, 'cross_references.txt'), encoding='utf-8'):
|
|
if line.startswith('From') or not line.strip():
|
|
continue
|
|
a, bb, votes = line.rstrip('\n').split('\t')
|
|
votes = int(votes)
|
|
if votes < 1:
|
|
continue
|
|
try:
|
|
f = ref(a)
|
|
if '-' in bb:
|
|
s, e = bb.split('-')
|
|
t = ref(s); te = ref(e)
|
|
tend = te[2] if te[:2] == t[:2] else 0
|
|
else:
|
|
t = ref(bb); tend = 0
|
|
except (KeyError, ValueError, IndexError):
|
|
continue
|
|
per.setdefault(f, []).append((votes, t, tend))
|
|
rows = []
|
|
for f, lst in per.items():
|
|
lst.sort(key=lambda x: -x[0])
|
|
for votes, t, tend in lst[:8]:
|
|
rows.append((f[0], f[1], f[2], t[0], t[1], t[2], tend, votes))
|
|
db.executemany('INSERT INTO xref VALUES (?,?,?,?,?,?,?,?)', rows)
|
|
db.execute('CREATE INDEX xi ON xref(book,chapter,verse)')
|
|
print('xrefs', len(rows))
|
|
|
|
# strongs
|
|
for fn, var in (('heb.js', 'strongsHebrewDictionary'), ('grk.js', 'strongsGreekDictionary')):
|
|
s = open(os.path.join(RAW, fn), encoding='utf-8').read()
|
|
j = s[s.index('{'):s.rindex('}') + 1]
|
|
d = json.loads(j)
|
|
for k, v in d.items():
|
|
db.execute('INSERT OR REPLACE INTO strongs VALUES (?,?,?,?,?,?,?)', (
|
|
k, v.get('lemma', ''), v.get('xlit', v.get('translit', '')), v.get('pron', ''),
|
|
v.get('derivation', ''), v.get('strongs_def', ''), v.get('kjv_def', '')))
|
|
db.commit()
|
|
db.execute('VACUUM')
|
|
db.close()
|
|
print('size', os.path.getsize(OUT))
|
|
|
|
if __name__ == '__main__':
|
|
main()
|