Add tools
This commit is contained in:
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Builds bible.db for the Vellum app.
|
||||
|
||||
Markup inside verse text (KJV and WEB):
|
||||
{word|H430} word group tagged with a Strong's number
|
||||
[word] italic (words added by translators, KJV only)
|
||||
<...> words of Jesus
|
||||
"""
|
||||
import json, os, re, sqlite3, glob, sys
|
||||
|
||||
RAW = '/home/claude/data/raw'
|
||||
OUT = sys.argv[1] if len(sys.argv) > 1 else '/home/claude/vellum/app/src/main/assets/bible.db'
|
||||
|
||||
CODES = ['GEN','EXO','LEV','NUM','DEU','JOS','JDG','RUT','1SA','2SA','1KI','2KI','1CH','2CH','EZR','NEH','EST','JOB','PSA','PRO','ECC','SNG','ISA','JER','LAM','EZK','DAN','HOS','JOL','AMO','OBA','JON','MIC','NAM','HAB','ZEP','HAG','ZEC','MAL',
|
||||
'MAT','MRK','LUK','JHN','ACT','ROM','1CO','2CO','GAL','EPH','PHP','COL','1TH','2TH','1TI','2TI','TIT','PHM','HEB','JAS','1PE','2PE','1JN','2JN','3JN','JUD','REV']
|
||||
NAMES = ['Genesis','Exodus','Leviticus','Numbers','Deuteronomy','Joshua','Judges','Ruth','1 Samuel','2 Samuel','1 Kings','2 Kings','1 Chronicles','2 Chronicles','Ezra','Nehemiah','Esther','Job','Psalms','Proverbs','Ecclesiastes','Song of Solomon','Isaiah','Jeremiah','Lamentations','Ezekiel','Daniel','Hosea','Joel','Amos','Obadiah','Jonah','Micah','Nahum','Habakkuk','Zephaniah','Haggai','Zechariah','Malachi',
|
||||
'Matthew','Mark','Luke','John','Acts','Romans','1 Corinthians','2 Corinthians','Galatians','Ephesians','Philippians','Colossians','1 Thessalonians','2 Thessalonians','1 Timothy','2 Timothy','Titus','Philemon','Hebrews','James','1 Peter','2 Peter','1 John','2 John','3 John','Jude','Revelation']
|
||||
ABBR = ['Gen','Exod','Lev','Num','Deut','Josh','Judg','Ruth','1 Sam','2 Sam','1 Kgs','2 Kgs','1 Chr','2 Chr','Ezra','Neh','Esth','Job','Ps','Prov','Eccl','Song','Isa','Jer','Lam','Ezek','Dan','Hos','Joel','Amos','Obad','Jonah','Mic','Nah','Hab','Zeph','Hag','Zech','Mal',
|
||||
'Matt','Mark','Luke','John','Acts','Rom','1 Cor','2 Cor','Gal','Eph','Phil','Col','1 Thess','2 Thess','1 Tim','2 Tim','Titus','Phlm','Heb','Jas','1 Pet','2 Pet','1 John','2 John','3 John','Jude','Rev']
|
||||
# openbible.info OSIS book names
|
||||
OSIS = ['Gen','Exod','Lev','Num','Deut','Josh','Judg','Ruth','1Sam','2Sam','1Kgs','2Kgs','1Chr','2Chr','Ezra','Neh','Esth','Job','Ps','Prov','Eccl','Song','Isa','Jer','Lam','Ezek','Dan','Hos','Joel','Amos','Obad','Jonah','Mic','Nah','Hab','Zeph','Hag','Zech','Mal',
|
||||
'Matt','Mark','Luke','John','Acts','Rom','1Cor','2Cor','Gal','Eph','Phil','Col','1Thess','2Thess','1Tim','2Tim','Titus','Phlm','Heb','Jas','1Pet','2Pet','1John','2John','3John','Jude','Rev']
|
||||
|
||||
TRANSLATIONS = [
|
||||
# id, code, name, has_markup
|
||||
(0, 'KJV', 'King James Version (1769)'),
|
||||
(1, 'WEB', 'World English Bible'),
|
||||
(2, 'BSB', 'Berean Standard Bible'),
|
||||
(3, 'ASV', 'American Standard Version (1901)'),
|
||||
(4, 'YLT', "Young's Literal Translation (1898)"),
|
||||
(5, 'DRC', 'Douay-Rheims, Challoner Revision'),
|
||||
]
|
||||
|
||||
def strip_strong(s):
|
||||
m = re.match(r'([HG])0*(\d+)([a-z]?)', s)
|
||||
return m.group(1) + m.group(2) if m else s
|
||||
|
||||
def convert_usfm_text(t, keep_italic=True):
|
||||
# footnotes and cross refs
|
||||
t = re.sub(r'\\f .*?\\f\*', '', t, flags=re.S)
|
||||
t = re.sub(r'\\x .*?\\x\*', '', t, flags=re.S)
|
||||
t = t.replace('[', '').replace(']', '')
|
||||
t = t.replace('¶', '')
|
||||
# strong's words, including nested \+w
|
||||
def wrep(m):
|
||||
word, attrs = m.group(2), m.group(3)
|
||||
sm = re.search(r'strong="([^"]+)"', attrs)
|
||||
word = word.strip()
|
||||
if not sm:
|
||||
return word
|
||||
num = strip_strong(sm.group(1).split(',')[0].split(' ')[0])
|
||||
return '{%s|%s}' % (word, num)
|
||||
t = re.sub(r'\\(\+?)w (.*?)\|([^\\]*?)\\\+?w\*', wrep, t)
|
||||
t = re.sub(r'\\\+?w (.*?)\\\+?w\*', r'\1', t) # untagged \w
|
||||
t = re.sub(r'\\\+?wj\s*', '<', t)
|
||||
t = t.replace('<*', '>')
|
||||
if keep_italic:
|
||||
t = re.sub(r'\\\+?add\s*', '[', t)
|
||||
t = t.replace('[*', ']')
|
||||
else:
|
||||
t = re.sub(r'\\\+?add\*?\s*', '', t)
|
||||
t = re.sub(r'\\\+?(nd|tl|qs|k|bk|wh|sc|it|em|bd|pn|qt|sig|ord)\*', '', t)
|
||||
t = re.sub(r'\\\+?(nd|tl|qs|k|bk|wh|sc|it|em|bd|pn|qt|sig|ord) ', '', t)
|
||||
t = re.sub(r'\\[a-z]+\d*\*?', ' ', t)
|
||||
t = re.sub(r'\s+', ' ', t).strip()
|
||||
# tidy spaces around markup
|
||||
t = re.sub(r'<\s+', '<', t)
|
||||
t = re.sub(r'\s+>', '>', t)
|
||||
t = re.sub(r'\[\s+', '[', t)
|
||||
t = re.sub(r'\s+\]', ']', t)
|
||||
t = re.sub(r'\s+([,.;:?!’”)])', r'\1', t)
|
||||
t = t.replace('<>', '').replace('[]', '')
|
||||
return t
|
||||
|
||||
def plain(t):
|
||||
t = re.sub(r'\{([^|}]*)\|[^}]*\}', r'\1', t)
|
||||
return t.replace('[', '').replace(']', '').replace('<', '').replace('>', '')
|
||||
|
||||
def parse_usfm(folder, suffix, keep_italic):
|
||||
verses = {}
|
||||
titles = {}
|
||||
for bi, code in enumerate(CODES):
|
||||
f = glob.glob(os.path.join(folder, '*%s%s.usfm' % (code, suffix)))
|
||||
assert f, code
|
||||
src = open(f[0], encoding='utf-8').read()
|
||||
ch = 0
|
||||
# split into lines by markers at line start
|
||||
cur = None
|
||||
buf = []
|
||||
def flush():
|
||||
if cur is not None:
|
||||
txt = convert_usfm_text(' '.join(buf), keep_italic)
|
||||
if txt:
|
||||
verses[(bi + 1, cur[0], cur[1])] = txt
|
||||
for line in src.split('\n'):
|
||||
line = line.rstrip()
|
||||
m = re.match(r'\\c (\d+)', line)
|
||||
if m:
|
||||
flush(); cur = None; buf = []
|
||||
ch = int(m.group(1)); continue
|
||||
if line.startswith('\\d '):
|
||||
flush(); cur = None; buf = []
|
||||
titles[(bi + 1, ch)] = plain(convert_usfm_text(line[3:], False))
|
||||
continue
|
||||
if re.match(r'\\(s\d?|ms\d?|mt\d?|toc\d|h|id|ide|mr|r|sp|rem|cl|is\d?|ip|imt\d?) ', line) or re.match(r'\\(s\d?|ms\d?)$', line):
|
||||
continue
|
||||
# line may contain multiple verses
|
||||
parts = re.split(r'(\\v \d+[a-z]?\s)', line)
|
||||
for p in parts:
|
||||
vm = re.match(r'\\v (\d+)', p)
|
||||
if vm:
|
||||
flush(); buf = []
|
||||
cur = (ch, int(vm.group(1)))
|
||||
elif cur is not None:
|
||||
buf.append(p)
|
||||
flush()
|
||||
return verses, titles
|
||||
|
||||
def parse_json(code):
|
||||
d = json.load(open(os.path.join(RAW, code + '.json')))
|
||||
out = {}
|
||||
def norm(n):
|
||||
n = n.replace('Revelation of John', 'Revelation')
|
||||
n = re.sub(r'^III ', '3 ', n); n = re.sub(r'^II ', '2 ', n); n = re.sub(r'^I ', '1 ', n)
|
||||
return n
|
||||
byname = {norm(b['name']): b for b in d['books']}
|
||||
for bi in range(66):
|
||||
b = byname[NAMES[bi]]
|
||||
for c in b['chapters']:
|
||||
for v in c['verses']:
|
||||
t = re.sub(r'\s+', ' ', v['text']).strip()
|
||||
t = t.replace('¶', '').strip()
|
||||
if t:
|
||||
out[(bi + 1, c['chapter'], v['verse'])] = t
|
||||
return out
|
||||
|
||||
def main():
|
||||
if os.path.exists(OUT):
|
||||
os.remove(OUT)
|
||||
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
||||
db = sqlite3.connect(OUT)
|
||||
db.executescript('''
|
||||
PRAGMA page_size=4096;
|
||||
CREATE TABLE android_metadata (locale TEXT);
|
||||
INSERT INTO android_metadata VALUES ('en_US');
|
||||
CREATE TABLE translations(id INTEGER PRIMARY KEY, code TEXT, name TEXT);
|
||||
CREATE TABLE books(id INTEGER PRIMARY KEY, name TEXT, abbr TEXT, chapters INTEGER);
|
||||
CREATE TABLE chapters(book INTEGER, chapter INTEGER, verses INTEGER, PRIMARY KEY(book,chapter)) WITHOUT ROWID;
|
||||
CREATE TABLE verses(tr INTEGER, book INTEGER, chapter INTEGER, verse INTEGER, text TEXT, plain TEXT, PRIMARY KEY(tr,book,chapter,verse)) WITHOUT ROWID;
|
||||
CREATE TABLE titles(book INTEGER, chapter INTEGER, text TEXT, PRIMARY KEY(book,chapter)) WITHOUT ROWID;
|
||||
CREATE TABLE xref(book INTEGER, chapter INTEGER, verse INTEGER, tb INTEGER, tc INTEGER, tv INTEGER, te INTEGER, votes INTEGER);
|
||||
CREATE TABLE strongs(id TEXT PRIMARY KEY, lemma TEXT, xlit TEXT, pron TEXT, deriv TEXT, def TEXT, kjv TEXT) WITHOUT ROWID;
|
||||
''')
|
||||
for t in TRANSLATIONS:
|
||||
db.execute('INSERT INTO translations VALUES (?,?,?)', t)
|
||||
|
||||
kjv, titles = parse_usfm(os.path.join(RAW, 'kjvusfm'), 'eng-kjv2006', True)
|
||||
web, _ = parse_usfm(os.path.join(RAW, 'webusfm'), 'eng-web', False)
|
||||
web = {k: re.sub(r'\{([^|}]*)\|[^}]*\}', r'\1', v) for k, v in web.items()}
|
||||
sets = {0: kjv, 1: web, 2: parse_json('BSB'), 3: parse_json('ASV'), 4: parse_json('YLT'), 5: parse_json('DRC')}
|
||||
print({k: len(v) for k, v in sets.items()})
|
||||
|
||||
# canonical structure from KJV
|
||||
chap = {}
|
||||
for (b, c, v) in kjv:
|
||||
chap[(b, c)] = max(chap.get((b, c), 0), v)
|
||||
for bi in range(66):
|
||||
n = max(c for (b, c) in chap if b == bi + 1)
|
||||
db.execute('INSERT INTO books VALUES (?,?,?,?)', (bi + 1, NAMES[bi], ABBR[bi], n))
|
||||
for (b, c), n in sorted(chap.items()):
|
||||
db.execute('INSERT INTO chapters VALUES (?,?,?)', (b, c, n))
|
||||
for tr, vs in sets.items():
|
||||
rows = []
|
||||
for (b, c, v), t in vs.items():
|
||||
p = plain(t) if tr in (0, 1) else t
|
||||
rows.append((tr, b, c, v, t if tr in (0, 1) else None, p))
|
||||
db.executemany('INSERT OR REPLACE INTO verses VALUES (?,?,?,?,?,?)', rows)
|
||||
for (b, c), t in titles.items():
|
||||
db.execute('INSERT INTO titles VALUES (?,?,?)', (b, c, t))
|
||||
|
||||
# cross references
|
||||
idx = {o: i + 1 for i, o in enumerate(OSIS)}
|
||||
def ref(s):
|
||||
p = s.split('.')
|
||||
return idx[p[0]], int(p[1]), int(p[2])
|
||||
per = {}
|
||||
for line in open(os.path.join(RAW, 'cross_references.txt'), encoding='utf-8'):
|
||||
if line.startswith('From') or not line.strip():
|
||||
continue
|
||||
a, bb, votes = line.rstrip('\n').split('\t')
|
||||
votes = int(votes)
|
||||
if votes < 1:
|
||||
continue
|
||||
try:
|
||||
f = ref(a)
|
||||
if '-' in bb:
|
||||
s, e = bb.split('-')
|
||||
t = ref(s); te = ref(e)
|
||||
tend = te[2] if te[:2] == t[:2] else 0
|
||||
else:
|
||||
t = ref(bb); tend = 0
|
||||
except (KeyError, ValueError, IndexError):
|
||||
continue
|
||||
per.setdefault(f, []).append((votes, t, tend))
|
||||
rows = []
|
||||
for f, lst in per.items():
|
||||
lst.sort(key=lambda x: -x[0])
|
||||
for votes, t, tend in lst[:8]:
|
||||
rows.append((f[0], f[1], f[2], t[0], t[1], t[2], tend, votes))
|
||||
db.executemany('INSERT INTO xref VALUES (?,?,?,?,?,?,?,?)', rows)
|
||||
db.execute('CREATE INDEX xi ON xref(book,chapter,verse)')
|
||||
print('xrefs', len(rows))
|
||||
|
||||
# strongs
|
||||
for fn, var in (('heb.js', 'strongsHebrewDictionary'), ('grk.js', 'strongsGreekDictionary')):
|
||||
s = open(os.path.join(RAW, fn), encoding='utf-8').read()
|
||||
j = s[s.index('{'):s.rindex('}') + 1]
|
||||
d = json.loads(j)
|
||||
for k, v in d.items():
|
||||
db.execute('INSERT OR REPLACE INTO strongs VALUES (?,?,?,?,?,?,?)', (
|
||||
k, v.get('lemma', ''), v.get('xlit', v.get('translit', '')), v.get('pron', ''),
|
||||
v.get('derivation', ''), v.get('strongs_def', ''), v.get('kjv_def', '')))
|
||||
db.commit()
|
||||
db.execute('VACUUM')
|
||||
db.close()
|
||||
print('size', os.path.getsize(OUT))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user