diff --git a/tools/build_db.py b/tools/build_db.py new file mode 100644 index 0000000..ed7f19c --- /dev/null +++ b/tools/build_db.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +"""Builds bible.db for the Vellum app. + +Markup inside verse text (KJV and WEB): + {word|H430} word group tagged with a Strong's number + [word] italic (words added by translators, KJV only) + <...> words of Jesus +""" +import json, os, re, sqlite3, glob, sys + +RAW = '/home/claude/data/raw' +OUT = sys.argv[1] if len(sys.argv) > 1 else '/home/claude/vellum/app/src/main/assets/bible.db' + +CODES = ['GEN','EXO','LEV','NUM','DEU','JOS','JDG','RUT','1SA','2SA','1KI','2KI','1CH','2CH','EZR','NEH','EST','JOB','PSA','PRO','ECC','SNG','ISA','JER','LAM','EZK','DAN','HOS','JOL','AMO','OBA','JON','MIC','NAM','HAB','ZEP','HAG','ZEC','MAL', + 'MAT','MRK','LUK','JHN','ACT','ROM','1CO','2CO','GAL','EPH','PHP','COL','1TH','2TH','1TI','2TI','TIT','PHM','HEB','JAS','1PE','2PE','1JN','2JN','3JN','JUD','REV'] +NAMES = ['Genesis','Exodus','Leviticus','Numbers','Deuteronomy','Joshua','Judges','Ruth','1 Samuel','2 Samuel','1 Kings','2 Kings','1 Chronicles','2 Chronicles','Ezra','Nehemiah','Esther','Job','Psalms','Proverbs','Ecclesiastes','Song of Solomon','Isaiah','Jeremiah','Lamentations','Ezekiel','Daniel','Hosea','Joel','Amos','Obadiah','Jonah','Micah','Nahum','Habakkuk','Zephaniah','Haggai','Zechariah','Malachi', + 'Matthew','Mark','Luke','John','Acts','Romans','1 Corinthians','2 Corinthians','Galatians','Ephesians','Philippians','Colossians','1 Thessalonians','2 Thessalonians','1 Timothy','2 Timothy','Titus','Philemon','Hebrews','James','1 Peter','2 Peter','1 John','2 John','3 John','Jude','Revelation'] +ABBR = ['Gen','Exod','Lev','Num','Deut','Josh','Judg','Ruth','1 Sam','2 Sam','1 Kgs','2 Kgs','1 Chr','2 Chr','Ezra','Neh','Esth','Job','Ps','Prov','Eccl','Song','Isa','Jer','Lam','Ezek','Dan','Hos','Joel','Amos','Obad','Jonah','Mic','Nah','Hab','Zeph','Hag','Zech','Mal', + 'Matt','Mark','Luke','John','Acts','Rom','1 Cor','2 Cor','Gal','Eph','Phil','Col','1 Thess','2 Thess','1 Tim','2 Tim','Titus','Phlm','Heb','Jas','1 Pet','2 Pet','1 John','2 John','3 John','Jude','Rev'] +# openbible.info OSIS book names +OSIS = ['Gen','Exod','Lev','Num','Deut','Josh','Judg','Ruth','1Sam','2Sam','1Kgs','2Kgs','1Chr','2Chr','Ezra','Neh','Esth','Job','Ps','Prov','Eccl','Song','Isa','Jer','Lam','Ezek','Dan','Hos','Joel','Amos','Obad','Jonah','Mic','Nah','Hab','Zeph','Hag','Zech','Mal', + 'Matt','Mark','Luke','John','Acts','Rom','1Cor','2Cor','Gal','Eph','Phil','Col','1Thess','2Thess','1Tim','2Tim','Titus','Phlm','Heb','Jas','1Pet','2Pet','1John','2John','3John','Jude','Rev'] + +TRANSLATIONS = [ + # id, code, name, has_markup + (0, 'KJV', 'King James Version (1769)'), + (1, 'WEB', 'World English Bible'), + (2, 'BSB', 'Berean Standard Bible'), + (3, 'ASV', 'American Standard Version (1901)'), + (4, 'YLT', "Young's Literal Translation (1898)"), + (5, 'DRC', 'Douay-Rheims, Challoner Revision'), +] + +def strip_strong(s): + m = re.match(r'([HG])0*(\d+)([a-z]?)', s) + return m.group(1) + m.group(2) if m else s + +def convert_usfm_text(t, keep_italic=True): + # footnotes and cross refs + t = re.sub(r'\\f .*?\\f\*', '', t, flags=re.S) + t = re.sub(r'\\x .*?\\x\*', '', t, flags=re.S) + t = t.replace('[', '').replace(']', '') + t = t.replace('¶', '') + # strong's words, including nested \+w + def wrep(m): + word, attrs = m.group(2), m.group(3) + sm = re.search(r'strong="([^"]+)"', attrs) + word = word.strip() + if not sm: + return word + num = strip_strong(sm.group(1).split(',')[0].split(' ')[0]) + return '{%s|%s}' % (word, num) + t = re.sub(r'\\(\+?)w (.*?)\|([^\\]*?)\\\+?w\*', wrep, t) + t = re.sub(r'\\\+?w (.*?)\\\+?w\*', r'\1', t) # untagged \w + t = re.sub(r'\\\+?wj\s*', '<', t) + t = t.replace('<*', '>') + if keep_italic: + t = re.sub(r'\\\+?add\s*', '[', t) + t = t.replace('[*', ']') + else: + t = re.sub(r'\\\+?add\*?\s*', '', t) + t = re.sub(r'\\\+?(nd|tl|qs|k|bk|wh|sc|it|em|bd|pn|qt|sig|ord)\*', '', t) + t = re.sub(r'\\\+?(nd|tl|qs|k|bk|wh|sc|it|em|bd|pn|qt|sig|ord) ', '', t) + t = re.sub(r'\\[a-z]+\d*\*?', ' ', t) + t = re.sub(r'\s+', ' ', t).strip() + # tidy spaces around markup + t = re.sub(r'<\s+', '<', t) + t = re.sub(r'\s+>', '>', t) + t = re.sub(r'\[\s+', '[', t) + t = re.sub(r'\s+\]', ']', t) + t = re.sub(r'\s+([,.;:?!’”)])', r'\1', t) + t = t.replace('<>', '').replace('[]', '') + return t + +def plain(t): + t = re.sub(r'\{([^|}]*)\|[^}]*\}', r'\1', t) + return t.replace('[', '').replace(']', '').replace('<', '').replace('>', '') + +def parse_usfm(folder, suffix, keep_italic): + verses = {} + titles = {} + for bi, code in enumerate(CODES): + f = glob.glob(os.path.join(folder, '*%s%s.usfm' % (code, suffix))) + assert f, code + src = open(f[0], encoding='utf-8').read() + ch = 0 + # split into lines by markers at line start + cur = None + buf = [] + def flush(): + if cur is not None: + txt = convert_usfm_text(' '.join(buf), keep_italic) + if txt: + verses[(bi + 1, cur[0], cur[1])] = txt + for line in src.split('\n'): + line = line.rstrip() + m = re.match(r'\\c (\d+)', line) + if m: + flush(); cur = None; buf = [] + ch = int(m.group(1)); continue + if line.startswith('\\d '): + flush(); cur = None; buf = [] + titles[(bi + 1, ch)] = plain(convert_usfm_text(line[3:], False)) + continue + if re.match(r'\\(s\d?|ms\d?|mt\d?|toc\d|h|id|ide|mr|r|sp|rem|cl|is\d?|ip|imt\d?) ', line) or re.match(r'\\(s\d?|ms\d?)$', line): + continue + # line may contain multiple verses + parts = re.split(r'(\\v \d+[a-z]?\s)', line) + for p in parts: + vm = re.match(r'\\v (\d+)', p) + if vm: + flush(); buf = [] + cur = (ch, int(vm.group(1))) + elif cur is not None: + buf.append(p) + flush() + return verses, titles + +def parse_json(code): + d = json.load(open(os.path.join(RAW, code + '.json'))) + out = {} + def norm(n): + n = n.replace('Revelation of John', 'Revelation') + n = re.sub(r'^III ', '3 ', n); n = re.sub(r'^II ', '2 ', n); n = re.sub(r'^I ', '1 ', n) + return n + byname = {norm(b['name']): b for b in d['books']} + for bi in range(66): + b = byname[NAMES[bi]] + for c in b['chapters']: + for v in c['verses']: + t = re.sub(r'\s+', ' ', v['text']).strip() + t = t.replace('¶', '').strip() + if t: + out[(bi + 1, c['chapter'], v['verse'])] = t + return out + +def main(): + if os.path.exists(OUT): + os.remove(OUT) + os.makedirs(os.path.dirname(OUT), exist_ok=True) + db = sqlite3.connect(OUT) + db.executescript(''' + PRAGMA page_size=4096; + CREATE TABLE android_metadata (locale TEXT); + INSERT INTO android_metadata VALUES ('en_US'); + CREATE TABLE translations(id INTEGER PRIMARY KEY, code TEXT, name TEXT); + CREATE TABLE books(id INTEGER PRIMARY KEY, name TEXT, abbr TEXT, chapters INTEGER); + CREATE TABLE chapters(book INTEGER, chapter INTEGER, verses INTEGER, PRIMARY KEY(book,chapter)) WITHOUT ROWID; + CREATE TABLE verses(tr INTEGER, book INTEGER, chapter INTEGER, verse INTEGER, text TEXT, plain TEXT, PRIMARY KEY(tr,book,chapter,verse)) WITHOUT ROWID; + CREATE TABLE titles(book INTEGER, chapter INTEGER, text TEXT, PRIMARY KEY(book,chapter)) WITHOUT ROWID; + CREATE TABLE xref(book INTEGER, chapter INTEGER, verse INTEGER, tb INTEGER, tc INTEGER, tv INTEGER, te INTEGER, votes INTEGER); + CREATE TABLE strongs(id TEXT PRIMARY KEY, lemma TEXT, xlit TEXT, pron TEXT, deriv TEXT, def TEXT, kjv TEXT) WITHOUT ROWID; + ''') + for t in TRANSLATIONS: + db.execute('INSERT INTO translations VALUES (?,?,?)', t) + + kjv, titles = parse_usfm(os.path.join(RAW, 'kjvusfm'), 'eng-kjv2006', True) + web, _ = parse_usfm(os.path.join(RAW, 'webusfm'), 'eng-web', False) + web = {k: re.sub(r'\{([^|}]*)\|[^}]*\}', r'\1', v) for k, v in web.items()} + sets = {0: kjv, 1: web, 2: parse_json('BSB'), 3: parse_json('ASV'), 4: parse_json('YLT'), 5: parse_json('DRC')} + print({k: len(v) for k, v in sets.items()}) + + # canonical structure from KJV + chap = {} + for (b, c, v) in kjv: + chap[(b, c)] = max(chap.get((b, c), 0), v) + for bi in range(66): + n = max(c for (b, c) in chap if b == bi + 1) + db.execute('INSERT INTO books VALUES (?,?,?,?)', (bi + 1, NAMES[bi], ABBR[bi], n)) + for (b, c), n in sorted(chap.items()): + db.execute('INSERT INTO chapters VALUES (?,?,?)', (b, c, n)) + for tr, vs in sets.items(): + rows = [] + for (b, c, v), t in vs.items(): + p = plain(t) if tr in (0, 1) else t + rows.append((tr, b, c, v, t if tr in (0, 1) else None, p)) + db.executemany('INSERT OR REPLACE INTO verses VALUES (?,?,?,?,?,?)', rows) + for (b, c), t in titles.items(): + db.execute('INSERT INTO titles VALUES (?,?,?)', (b, c, t)) + + # cross references + idx = {o: i + 1 for i, o in enumerate(OSIS)} + def ref(s): + p = s.split('.') + return idx[p[0]], int(p[1]), int(p[2]) + per = {} + for line in open(os.path.join(RAW, 'cross_references.txt'), encoding='utf-8'): + if line.startswith('From') or not line.strip(): + continue + a, bb, votes = line.rstrip('\n').split('\t') + votes = int(votes) + if votes < 1: + continue + try: + f = ref(a) + if '-' in bb: + s, e = bb.split('-') + t = ref(s); te = ref(e) + tend = te[2] if te[:2] == t[:2] else 0 + else: + t = ref(bb); tend = 0 + except (KeyError, ValueError, IndexError): + continue + per.setdefault(f, []).append((votes, t, tend)) + rows = [] + for f, lst in per.items(): + lst.sort(key=lambda x: -x[0]) + for votes, t, tend in lst[:8]: + rows.append((f[0], f[1], f[2], t[0], t[1], t[2], tend, votes)) + db.executemany('INSERT INTO xref VALUES (?,?,?,?,?,?,?,?)', rows) + db.execute('CREATE INDEX xi ON xref(book,chapter,verse)') + print('xrefs', len(rows)) + + # strongs + for fn, var in (('heb.js', 'strongsHebrewDictionary'), ('grk.js', 'strongsGreekDictionary')): + s = open(os.path.join(RAW, fn), encoding='utf-8').read() + j = s[s.index('{'):s.rindex('}') + 1] + d = json.loads(j) + for k, v in d.items(): + db.execute('INSERT OR REPLACE INTO strongs VALUES (?,?,?,?,?,?,?)', ( + k, v.get('lemma', ''), v.get('xlit', v.get('translit', '')), v.get('pron', ''), + v.get('derivation', ''), v.get('strongs_def', ''), v.get('kjv_def', ''))) + db.commit() + db.execute('VACUUM') + db.close() + print('size', os.path.getsize(OUT)) + +if __name__ == '__main__': + main() diff --git a/tools/textures.py b/tools/textures.py new file mode 100644 index 0000000..0c5ae21 --- /dev/null +++ b/tools/textures.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Procedural aged backgrounds: vellum, papyrus, parchment, candlelight (night).""" +import numpy as np +from PIL import Image +from scipy.ndimage import gaussian_filter, zoom +import os + +OUT = '/home/claude/vellum/app/src/main/res/drawable-nodpi' +os.makedirs(OUT, exist_ok=True) +W, H = 1440, 2560 +rng = np.random.default_rng(7) + +def fbm(w, h, octaves=6, base=4, persistence=0.55, seed=0): + r = np.random.default_rng(seed) + out = np.zeros((h, w)) + amp = 1.0; tot = 0 + for o in range(octaves): + n = base * 2 ** o + g = r.random((max(2, int(n * h / w)) + 1, n + 1)) + z = zoom(g, (h / (g.shape[0] - 1), w / (g.shape[1] - 1)), order=3)[:h, :w] + out += amp * z; tot += amp; amp *= persistence + out /= tot + out = (out - out.min()) / (out.max() - out.min()) + return out + +def vignette(w, h, strength=0.5, power=2.2): + y, x = np.mgrid[0:h, 0:w] + dx = (x - w / 2) / (w / 2); dy = (y - h / 2) / (h / 2) + d = np.sqrt(dx ** 2 * 1.0 + dy ** 2 * 0.85) + return 1 - strength * np.clip(d, 0, 1.6) ** power / (1.6 ** power) * 1.6 + +def colorize(v, c0, c1): + c0 = np.array(c0, float); c1 = np.array(c1, float) + return c0[None, None, :] * (1 - v[..., None]) + c1[None, None, :] * v[..., None] + +def save(img, name): + a = np.clip(img, 0, 255).astype(np.uint8) + Image.fromarray(a).save(os.path.join(OUT, name + '.webp'), quality=82, method=6) + Image.fromarray(a).resize((360, 640)).save('/home/claude/data/prev_' + name + '.png') + +def grain(w, h, amt, seed): + r = np.random.default_rng(seed) + return (r.random((h, w)) - 0.5) * amt + +# ---- Vellum: warm cream, soft mottled clouds, faint hair follicle speckle, darker edges +def vellum(): + m = fbm(W, H, 7, 3, 0.58, 1) + m2 = fbm(W, H, 5, 8, 0.5, 2) + base = colorize(m * 0.7 + m2 * 0.3, (222, 200, 160), (246, 234, 206)) + # stains + st = fbm(W, H, 4, 2, 0.6, 3) + stain = np.clip((st - 0.62) * 3.5, 0, 1) + base = base * (1 - 0.12 * stain[..., None]) + np.array([150, 110, 60]) * 0.12 * stain[..., None] + # follicle speckle + sp = rng.random((H, W)) > 0.9985 + sp = gaussian_filter(sp.astype(float), 1.2) * 6 + base -= sp[..., None] * np.array([40, 45, 55]) + base += grain(W, H, 10, 4)[..., None] + base *= vignette(W, H, 0.38, 2.6)[..., None] + save(base, 'bg_vellum') + +# ---- Papyrus: crossed plant strips, horizontal layer over vertical layer +def papyrus(): + r = np.random.default_rng(11) + # vertical strips + def strips(n, length, axis, seed): + rr = np.random.default_rng(seed) + widths = rr.integers(60, 140, n) + prof = [] + for wdt in widths: + t = np.linspace(0, 1, wdt) + edge = np.clip(np.minimum(t, 1 - t) * 9, 0, 1) + fibers = gaussian_filter(rr.random(wdt), 0.8) + prof.append(0.75 + 0.25 * edge + (fibers - 0.5) * 0.25 + rr.normal(0, 0.03)) + p = np.concatenate(prof) + return p + pv = strips(30, H, 0, 12)[:W] + pv = np.pad(pv, (0, max(0, W - len(pv))), mode='edge') + ph = strips(40, W, 1, 13)[:H] + ph = np.pad(ph, (0, max(0, H - len(ph))), mode='edge') + fv = gaussian_filter(r.random((H, W)), (40, 0.7)) # long vertical fibers + fh = gaussian_filter(r.random((H, W)), (0.7, 40)) # long horizontal fibers + fv = (fv - fv.mean()) / fv.std(); fh = (fh - fh.mean()) / fh.std() + v = 0.55 * ph[:, None] * np.ones((1, W)) + 0.25 * pv[None, :] * np.ones((H, 1)) + v += 0.05 * fh + 0.025 * fv + blot = fbm(W, H, 6, 3, 0.55, 14) + v = v * 0.8 + blot * 0.25 + v = (v - v.min()) / (v.max() - v.min()) + img = colorize(v, (204, 174, 120), (244, 228, 188)) + dark = np.clip((fbm(W, H, 4, 2, 0.6, 15) - 0.66) * 3, 0, 1) + img *= (1 - 0.10 * dark)[..., None] + img += grain(W, H, 12, 16)[..., None] + img *= vignette(W, H, 0.42, 2.4)[..., None] + save(img, 'bg_papyrus') + +# ---- Parchment: older, darker, more blotchy with burnt edges +def parchment(): + m = fbm(W, H, 7, 3, 0.62, 21) + img = colorize(m, (196, 160, 104), (238, 214, 168)) + bl = fbm(W, H, 5, 5, 0.55, 22) + b = np.clip((bl - 0.58) * 4, 0, 1) + img = img * (1 - 0.22 * b[..., None]) + np.array([120, 80, 35]) * 0.22 * b[..., None] + y, x = np.mgrid[0:H, 0:W] + edge = np.minimum.reduce([x, W - 1 - x, y, H - 1 - y]).astype(float) + wob = fbm(W, H, 5, 6, 0.5, 23) * 70 + e = np.clip(1 - (edge - wob) / 120, 0, 1) ** 1.6 + img = img * (1 - 0.65 * e[..., None]) + np.array([70, 40, 15]) * 0.65 * e[..., None] + img += grain(W, H, 12, 24)[..., None] + img *= vignette(W, H, 0.3, 2.2)[..., None] + save(img, 'bg_parchment') + +# ---- Candlelight night: deep umber parchment, warm glow from lower centre +def candle(): + m = fbm(W, H, 7, 3, 0.6, 31) + img = colorize(m, (22, 14, 8), (52, 34, 18)) + y, x = np.mgrid[0:H, 0:W] + d = np.sqrt(((x - W * 0.5) / W) ** 2 + ((y - H * 0.78) / H) ** 2 * 1.4) + glow = np.exp(-(d / 0.42) ** 2) + img += glow[..., None] * np.array([70, 40, 12]) + img += grain(W, H, 6, 32)[..., None] + img *= vignette(W, H, 0.55, 2.0)[..., None] + save(img, 'bg_candle') + +if __name__ == '__main__': + vellum(); papyrus(); parchment(); candle() + for f in sorted(os.listdir(OUT)): + print(f, os.path.getsize(os.path.join(OUT, f)))