Add tools

This commit is contained in:
2026-10-19 01:19:52 +00:00
parent 196b37ad52
commit 7027a8c9de
2 changed files with 356 additions and 0 deletions
+229
View File
@@ -0,0 +1,229 @@
#!/usr/bin/env python3
"""Builds bible.db for the Vellum app.
Markup inside verse text (KJV and WEB):
{word|H430} word group tagged with a Strong's number
[word] italic (words added by translators, KJV only)
<...> words of Jesus
"""
import json, os, re, sqlite3, glob, sys
RAW = '/home/claude/data/raw'
OUT = sys.argv[1] if len(sys.argv) > 1 else '/home/claude/vellum/app/src/main/assets/bible.db'
CODES = ['GEN','EXO','LEV','NUM','DEU','JOS','JDG','RUT','1SA','2SA','1KI','2KI','1CH','2CH','EZR','NEH','EST','JOB','PSA','PRO','ECC','SNG','ISA','JER','LAM','EZK','DAN','HOS','JOL','AMO','OBA','JON','MIC','NAM','HAB','ZEP','HAG','ZEC','MAL',
'MAT','MRK','LUK','JHN','ACT','ROM','1CO','2CO','GAL','EPH','PHP','COL','1TH','2TH','1TI','2TI','TIT','PHM','HEB','JAS','1PE','2PE','1JN','2JN','3JN','JUD','REV']
NAMES = ['Genesis','Exodus','Leviticus','Numbers','Deuteronomy','Joshua','Judges','Ruth','1 Samuel','2 Samuel','1 Kings','2 Kings','1 Chronicles','2 Chronicles','Ezra','Nehemiah','Esther','Job','Psalms','Proverbs','Ecclesiastes','Song of Solomon','Isaiah','Jeremiah','Lamentations','Ezekiel','Daniel','Hosea','Joel','Amos','Obadiah','Jonah','Micah','Nahum','Habakkuk','Zephaniah','Haggai','Zechariah','Malachi',
'Matthew','Mark','Luke','John','Acts','Romans','1 Corinthians','2 Corinthians','Galatians','Ephesians','Philippians','Colossians','1 Thessalonians','2 Thessalonians','1 Timothy','2 Timothy','Titus','Philemon','Hebrews','James','1 Peter','2 Peter','1 John','2 John','3 John','Jude','Revelation']
ABBR = ['Gen','Exod','Lev','Num','Deut','Josh','Judg','Ruth','1 Sam','2 Sam','1 Kgs','2 Kgs','1 Chr','2 Chr','Ezra','Neh','Esth','Job','Ps','Prov','Eccl','Song','Isa','Jer','Lam','Ezek','Dan','Hos','Joel','Amos','Obad','Jonah','Mic','Nah','Hab','Zeph','Hag','Zech','Mal',
'Matt','Mark','Luke','John','Acts','Rom','1 Cor','2 Cor','Gal','Eph','Phil','Col','1 Thess','2 Thess','1 Tim','2 Tim','Titus','Phlm','Heb','Jas','1 Pet','2 Pet','1 John','2 John','3 John','Jude','Rev']
# openbible.info OSIS book names
OSIS = ['Gen','Exod','Lev','Num','Deut','Josh','Judg','Ruth','1Sam','2Sam','1Kgs','2Kgs','1Chr','2Chr','Ezra','Neh','Esth','Job','Ps','Prov','Eccl','Song','Isa','Jer','Lam','Ezek','Dan','Hos','Joel','Amos','Obad','Jonah','Mic','Nah','Hab','Zeph','Hag','Zech','Mal',
'Matt','Mark','Luke','John','Acts','Rom','1Cor','2Cor','Gal','Eph','Phil','Col','1Thess','2Thess','1Tim','2Tim','Titus','Phlm','Heb','Jas','1Pet','2Pet','1John','2John','3John','Jude','Rev']
TRANSLATIONS = [
# id, code, name, has_markup
(0, 'KJV', 'King James Version (1769)'),
(1, 'WEB', 'World English Bible'),
(2, 'BSB', 'Berean Standard Bible'),
(3, 'ASV', 'American Standard Version (1901)'),
(4, 'YLT', "Young's Literal Translation (1898)"),
(5, 'DRC', 'Douay-Rheims, Challoner Revision'),
]
def strip_strong(s):
m = re.match(r'([HG])0*(\d+)([a-z]?)', s)
return m.group(1) + m.group(2) if m else s
def convert_usfm_text(t, keep_italic=True):
# footnotes and cross refs
t = re.sub(r'\\f .*?\\f\*', '', t, flags=re.S)
t = re.sub(r'\\x .*?\\x\*', '', t, flags=re.S)
t = t.replace('[', '').replace(']', '')
t = t.replace('', '')
# strong's words, including nested \+w
def wrep(m):
word, attrs = m.group(2), m.group(3)
sm = re.search(r'strong="([^"]+)"', attrs)
word = word.strip()
if not sm:
return word
num = strip_strong(sm.group(1).split(',')[0].split(' ')[0])
return '{%s|%s}' % (word, num)
t = re.sub(r'\\(\+?)w (.*?)\|([^\\]*?)\\\+?w\*', wrep, t)
t = re.sub(r'\\\+?w (.*?)\\\+?w\*', r'\1', t) # untagged \w
t = re.sub(r'\\\+?wj\s*', '<', t)
t = t.replace('<*', '>')
if keep_italic:
t = re.sub(r'\\\+?add\s*', '[', t)
t = t.replace('[*', ']')
else:
t = re.sub(r'\\\+?add\*?\s*', '', t)
t = re.sub(r'\\\+?(nd|tl|qs|k|bk|wh|sc|it|em|bd|pn|qt|sig|ord)\*', '', t)
t = re.sub(r'\\\+?(nd|tl|qs|k|bk|wh|sc|it|em|bd|pn|qt|sig|ord) ', '', t)
t = re.sub(r'\\[a-z]+\d*\*?', ' ', t)
t = re.sub(r'\s+', ' ', t).strip()
# tidy spaces around markup
t = re.sub(r'<\s+', '<', t)
t = re.sub(r'\s+>', '>', t)
t = re.sub(r'\[\s+', '[', t)
t = re.sub(r'\s+\]', ']', t)
t = re.sub(r'\s+([,.;:?!’”)])', r'\1', t)
t = t.replace('<>', '').replace('[]', '')
return t
def plain(t):
t = re.sub(r'\{([^|}]*)\|[^}]*\}', r'\1', t)
return t.replace('[', '').replace(']', '').replace('<', '').replace('>', '')
def parse_usfm(folder, suffix, keep_italic):
verses = {}
titles = {}
for bi, code in enumerate(CODES):
f = glob.glob(os.path.join(folder, '*%s%s.usfm' % (code, suffix)))
assert f, code
src = open(f[0], encoding='utf-8').read()
ch = 0
# split into lines by markers at line start
cur = None
buf = []
def flush():
if cur is not None:
txt = convert_usfm_text(' '.join(buf), keep_italic)
if txt:
verses[(bi + 1, cur[0], cur[1])] = txt
for line in src.split('\n'):
line = line.rstrip()
m = re.match(r'\\c (\d+)', line)
if m:
flush(); cur = None; buf = []
ch = int(m.group(1)); continue
if line.startswith('\\d '):
flush(); cur = None; buf = []
titles[(bi + 1, ch)] = plain(convert_usfm_text(line[3:], False))
continue
if re.match(r'\\(s\d?|ms\d?|mt\d?|toc\d|h|id|ide|mr|r|sp|rem|cl|is\d?|ip|imt\d?) ', line) or re.match(r'\\(s\d?|ms\d?)$', line):
continue
# line may contain multiple verses
parts = re.split(r'(\\v \d+[a-z]?\s)', line)
for p in parts:
vm = re.match(r'\\v (\d+)', p)
if vm:
flush(); buf = []
cur = (ch, int(vm.group(1)))
elif cur is not None:
buf.append(p)
flush()
return verses, titles
def parse_json(code):
d = json.load(open(os.path.join(RAW, code + '.json')))
out = {}
def norm(n):
n = n.replace('Revelation of John', 'Revelation')
n = re.sub(r'^III ', '3 ', n); n = re.sub(r'^II ', '2 ', n); n = re.sub(r'^I ', '1 ', n)
return n
byname = {norm(b['name']): b for b in d['books']}
for bi in range(66):
b = byname[NAMES[bi]]
for c in b['chapters']:
for v in c['verses']:
t = re.sub(r'\s+', ' ', v['text']).strip()
t = t.replace('', '').strip()
if t:
out[(bi + 1, c['chapter'], v['verse'])] = t
return out
def main():
if os.path.exists(OUT):
os.remove(OUT)
os.makedirs(os.path.dirname(OUT), exist_ok=True)
db = sqlite3.connect(OUT)
db.executescript('''
PRAGMA page_size=4096;
CREATE TABLE android_metadata (locale TEXT);
INSERT INTO android_metadata VALUES ('en_US');
CREATE TABLE translations(id INTEGER PRIMARY KEY, code TEXT, name TEXT);
CREATE TABLE books(id INTEGER PRIMARY KEY, name TEXT, abbr TEXT, chapters INTEGER);
CREATE TABLE chapters(book INTEGER, chapter INTEGER, verses INTEGER, PRIMARY KEY(book,chapter)) WITHOUT ROWID;
CREATE TABLE verses(tr INTEGER, book INTEGER, chapter INTEGER, verse INTEGER, text TEXT, plain TEXT, PRIMARY KEY(tr,book,chapter,verse)) WITHOUT ROWID;
CREATE TABLE titles(book INTEGER, chapter INTEGER, text TEXT, PRIMARY KEY(book,chapter)) WITHOUT ROWID;
CREATE TABLE xref(book INTEGER, chapter INTEGER, verse INTEGER, tb INTEGER, tc INTEGER, tv INTEGER, te INTEGER, votes INTEGER);
CREATE TABLE strongs(id TEXT PRIMARY KEY, lemma TEXT, xlit TEXT, pron TEXT, deriv TEXT, def TEXT, kjv TEXT) WITHOUT ROWID;
''')
for t in TRANSLATIONS:
db.execute('INSERT INTO translations VALUES (?,?,?)', t)
kjv, titles = parse_usfm(os.path.join(RAW, 'kjvusfm'), 'eng-kjv2006', True)
web, _ = parse_usfm(os.path.join(RAW, 'webusfm'), 'eng-web', False)
web = {k: re.sub(r'\{([^|}]*)\|[^}]*\}', r'\1', v) for k, v in web.items()}
sets = {0: kjv, 1: web, 2: parse_json('BSB'), 3: parse_json('ASV'), 4: parse_json('YLT'), 5: parse_json('DRC')}
print({k: len(v) for k, v in sets.items()})
# canonical structure from KJV
chap = {}
for (b, c, v) in kjv:
chap[(b, c)] = max(chap.get((b, c), 0), v)
for bi in range(66):
n = max(c for (b, c) in chap if b == bi + 1)
db.execute('INSERT INTO books VALUES (?,?,?,?)', (bi + 1, NAMES[bi], ABBR[bi], n))
for (b, c), n in sorted(chap.items()):
db.execute('INSERT INTO chapters VALUES (?,?,?)', (b, c, n))
for tr, vs in sets.items():
rows = []
for (b, c, v), t in vs.items():
p = plain(t) if tr in (0, 1) else t
rows.append((tr, b, c, v, t if tr in (0, 1) else None, p))
db.executemany('INSERT OR REPLACE INTO verses VALUES (?,?,?,?,?,?)', rows)
for (b, c), t in titles.items():
db.execute('INSERT INTO titles VALUES (?,?,?)', (b, c, t))
# cross references
idx = {o: i + 1 for i, o in enumerate(OSIS)}
def ref(s):
p = s.split('.')
return idx[p[0]], int(p[1]), int(p[2])
per = {}
for line in open(os.path.join(RAW, 'cross_references.txt'), encoding='utf-8'):
if line.startswith('From') or not line.strip():
continue
a, bb, votes = line.rstrip('\n').split('\t')
votes = int(votes)
if votes < 1:
continue
try:
f = ref(a)
if '-' in bb:
s, e = bb.split('-')
t = ref(s); te = ref(e)
tend = te[2] if te[:2] == t[:2] else 0
else:
t = ref(bb); tend = 0
except (KeyError, ValueError, IndexError):
continue
per.setdefault(f, []).append((votes, t, tend))
rows = []
for f, lst in per.items():
lst.sort(key=lambda x: -x[0])
for votes, t, tend in lst[:8]:
rows.append((f[0], f[1], f[2], t[0], t[1], t[2], tend, votes))
db.executemany('INSERT INTO xref VALUES (?,?,?,?,?,?,?,?)', rows)
db.execute('CREATE INDEX xi ON xref(book,chapter,verse)')
print('xrefs', len(rows))
# strongs
for fn, var in (('heb.js', 'strongsHebrewDictionary'), ('grk.js', 'strongsGreekDictionary')):
s = open(os.path.join(RAW, fn), encoding='utf-8').read()
j = s[s.index('{'):s.rindex('}') + 1]
d = json.loads(j)
for k, v in d.items():
db.execute('INSERT OR REPLACE INTO strongs VALUES (?,?,?,?,?,?,?)', (
k, v.get('lemma', ''), v.get('xlit', v.get('translit', '')), v.get('pron', ''),
v.get('derivation', ''), v.get('strongs_def', ''), v.get('kjv_def', '')))
db.commit()
db.execute('VACUUM')
db.close()
print('size', os.path.getsize(OUT))
if __name__ == '__main__':
main()
+127
View File
@@ -0,0 +1,127 @@
#!/usr/bin/env python3
"""Procedural aged backgrounds: vellum, papyrus, parchment, candlelight (night)."""
import numpy as np
from PIL import Image
from scipy.ndimage import gaussian_filter, zoom
import os
OUT = '/home/claude/vellum/app/src/main/res/drawable-nodpi'
os.makedirs(OUT, exist_ok=True)
W, H = 1440, 2560
rng = np.random.default_rng(7)
def fbm(w, h, octaves=6, base=4, persistence=0.55, seed=0):
r = np.random.default_rng(seed)
out = np.zeros((h, w))
amp = 1.0; tot = 0
for o in range(octaves):
n = base * 2 ** o
g = r.random((max(2, int(n * h / w)) + 1, n + 1))
z = zoom(g, (h / (g.shape[0] - 1), w / (g.shape[1] - 1)), order=3)[:h, :w]
out += amp * z; tot += amp; amp *= persistence
out /= tot
out = (out - out.min()) / (out.max() - out.min())
return out
def vignette(w, h, strength=0.5, power=2.2):
y, x = np.mgrid[0:h, 0:w]
dx = (x - w / 2) / (w / 2); dy = (y - h / 2) / (h / 2)
d = np.sqrt(dx ** 2 * 1.0 + dy ** 2 * 0.85)
return 1 - strength * np.clip(d, 0, 1.6) ** power / (1.6 ** power) * 1.6
def colorize(v, c0, c1):
c0 = np.array(c0, float); c1 = np.array(c1, float)
return c0[None, None, :] * (1 - v[..., None]) + c1[None, None, :] * v[..., None]
def save(img, name):
a = np.clip(img, 0, 255).astype(np.uint8)
Image.fromarray(a).save(os.path.join(OUT, name + '.webp'), quality=82, method=6)
Image.fromarray(a).resize((360, 640)).save('/home/claude/data/prev_' + name + '.png')
def grain(w, h, amt, seed):
r = np.random.default_rng(seed)
return (r.random((h, w)) - 0.5) * amt
# ---- Vellum: warm cream, soft mottled clouds, faint hair follicle speckle, darker edges
def vellum():
m = fbm(W, H, 7, 3, 0.58, 1)
m2 = fbm(W, H, 5, 8, 0.5, 2)
base = colorize(m * 0.7 + m2 * 0.3, (222, 200, 160), (246, 234, 206))
# stains
st = fbm(W, H, 4, 2, 0.6, 3)
stain = np.clip((st - 0.62) * 3.5, 0, 1)
base = base * (1 - 0.12 * stain[..., None]) + np.array([150, 110, 60]) * 0.12 * stain[..., None]
# follicle speckle
sp = rng.random((H, W)) > 0.9985
sp = gaussian_filter(sp.astype(float), 1.2) * 6
base -= sp[..., None] * np.array([40, 45, 55])
base += grain(W, H, 10, 4)[..., None]
base *= vignette(W, H, 0.38, 2.6)[..., None]
save(base, 'bg_vellum')
# ---- Papyrus: crossed plant strips, horizontal layer over vertical layer
def papyrus():
r = np.random.default_rng(11)
# vertical strips
def strips(n, length, axis, seed):
rr = np.random.default_rng(seed)
widths = rr.integers(60, 140, n)
prof = []
for wdt in widths:
t = np.linspace(0, 1, wdt)
edge = np.clip(np.minimum(t, 1 - t) * 9, 0, 1)
fibers = gaussian_filter(rr.random(wdt), 0.8)
prof.append(0.75 + 0.25 * edge + (fibers - 0.5) * 0.25 + rr.normal(0, 0.03))
p = np.concatenate(prof)
return p
pv = strips(30, H, 0, 12)[:W]
pv = np.pad(pv, (0, max(0, W - len(pv))), mode='edge')
ph = strips(40, W, 1, 13)[:H]
ph = np.pad(ph, (0, max(0, H - len(ph))), mode='edge')
fv = gaussian_filter(r.random((H, W)), (40, 0.7)) # long vertical fibers
fh = gaussian_filter(r.random((H, W)), (0.7, 40)) # long horizontal fibers
fv = (fv - fv.mean()) / fv.std(); fh = (fh - fh.mean()) / fh.std()
v = 0.55 * ph[:, None] * np.ones((1, W)) + 0.25 * pv[None, :] * np.ones((H, 1))
v += 0.05 * fh + 0.025 * fv
blot = fbm(W, H, 6, 3, 0.55, 14)
v = v * 0.8 + blot * 0.25
v = (v - v.min()) / (v.max() - v.min())
img = colorize(v, (204, 174, 120), (244, 228, 188))
dark = np.clip((fbm(W, H, 4, 2, 0.6, 15) - 0.66) * 3, 0, 1)
img *= (1 - 0.10 * dark)[..., None]
img += grain(W, H, 12, 16)[..., None]
img *= vignette(W, H, 0.42, 2.4)[..., None]
save(img, 'bg_papyrus')
# ---- Parchment: older, darker, more blotchy with burnt edges
def parchment():
m = fbm(W, H, 7, 3, 0.62, 21)
img = colorize(m, (196, 160, 104), (238, 214, 168))
bl = fbm(W, H, 5, 5, 0.55, 22)
b = np.clip((bl - 0.58) * 4, 0, 1)
img = img * (1 - 0.22 * b[..., None]) + np.array([120, 80, 35]) * 0.22 * b[..., None]
y, x = np.mgrid[0:H, 0:W]
edge = np.minimum.reduce([x, W - 1 - x, y, H - 1 - y]).astype(float)
wob = fbm(W, H, 5, 6, 0.5, 23) * 70
e = np.clip(1 - (edge - wob) / 120, 0, 1) ** 1.6
img = img * (1 - 0.65 * e[..., None]) + np.array([70, 40, 15]) * 0.65 * e[..., None]
img += grain(W, H, 12, 24)[..., None]
img *= vignette(W, H, 0.3, 2.2)[..., None]
save(img, 'bg_parchment')
# ---- Candlelight night: deep umber parchment, warm glow from lower centre
def candle():
m = fbm(W, H, 7, 3, 0.6, 31)
img = colorize(m, (22, 14, 8), (52, 34, 18))
y, x = np.mgrid[0:H, 0:W]
d = np.sqrt(((x - W * 0.5) / W) ** 2 + ((y - H * 0.78) / H) ** 2 * 1.4)
glow = np.exp(-(d / 0.42) ** 2)
img += glow[..., None] * np.array([70, 40, 12])
img += grain(W, H, 6, 32)[..., None]
img *= vignette(W, H, 0.55, 2.0)[..., None]
save(img, 'bg_candle')
if __name__ == '__main__':
vellum(); papyrus(); parchment(); candle()
for f in sorted(os.listdir(OUT)):
print(f, os.path.getsize(os.path.join(OUT, f)))