Add tools
This commit is contained in:
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Builds bible.db for the Vellum app.
|
||||
|
||||
Markup inside verse text (KJV and WEB):
|
||||
{word|H430} word group tagged with a Strong's number
|
||||
[word] italic (words added by translators, KJV only)
|
||||
<...> words of Jesus
|
||||
"""
|
||||
import json, os, re, sqlite3, glob, sys
|
||||
|
||||
RAW = '/home/claude/data/raw'
|
||||
OUT = sys.argv[1] if len(sys.argv) > 1 else '/home/claude/vellum/app/src/main/assets/bible.db'
|
||||
|
||||
CODES = ['GEN','EXO','LEV','NUM','DEU','JOS','JDG','RUT','1SA','2SA','1KI','2KI','1CH','2CH','EZR','NEH','EST','JOB','PSA','PRO','ECC','SNG','ISA','JER','LAM','EZK','DAN','HOS','JOL','AMO','OBA','JON','MIC','NAM','HAB','ZEP','HAG','ZEC','MAL',
|
||||
'MAT','MRK','LUK','JHN','ACT','ROM','1CO','2CO','GAL','EPH','PHP','COL','1TH','2TH','1TI','2TI','TIT','PHM','HEB','JAS','1PE','2PE','1JN','2JN','3JN','JUD','REV']
|
||||
NAMES = ['Genesis','Exodus','Leviticus','Numbers','Deuteronomy','Joshua','Judges','Ruth','1 Samuel','2 Samuel','1 Kings','2 Kings','1 Chronicles','2 Chronicles','Ezra','Nehemiah','Esther','Job','Psalms','Proverbs','Ecclesiastes','Song of Solomon','Isaiah','Jeremiah','Lamentations','Ezekiel','Daniel','Hosea','Joel','Amos','Obadiah','Jonah','Micah','Nahum','Habakkuk','Zephaniah','Haggai','Zechariah','Malachi',
|
||||
'Matthew','Mark','Luke','John','Acts','Romans','1 Corinthians','2 Corinthians','Galatians','Ephesians','Philippians','Colossians','1 Thessalonians','2 Thessalonians','1 Timothy','2 Timothy','Titus','Philemon','Hebrews','James','1 Peter','2 Peter','1 John','2 John','3 John','Jude','Revelation']
|
||||
ABBR = ['Gen','Exod','Lev','Num','Deut','Josh','Judg','Ruth','1 Sam','2 Sam','1 Kgs','2 Kgs','1 Chr','2 Chr','Ezra','Neh','Esth','Job','Ps','Prov','Eccl','Song','Isa','Jer','Lam','Ezek','Dan','Hos','Joel','Amos','Obad','Jonah','Mic','Nah','Hab','Zeph','Hag','Zech','Mal',
|
||||
'Matt','Mark','Luke','John','Acts','Rom','1 Cor','2 Cor','Gal','Eph','Phil','Col','1 Thess','2 Thess','1 Tim','2 Tim','Titus','Phlm','Heb','Jas','1 Pet','2 Pet','1 John','2 John','3 John','Jude','Rev']
|
||||
# openbible.info OSIS book names
|
||||
OSIS = ['Gen','Exod','Lev','Num','Deut','Josh','Judg','Ruth','1Sam','2Sam','1Kgs','2Kgs','1Chr','2Chr','Ezra','Neh','Esth','Job','Ps','Prov','Eccl','Song','Isa','Jer','Lam','Ezek','Dan','Hos','Joel','Amos','Obad','Jonah','Mic','Nah','Hab','Zeph','Hag','Zech','Mal',
|
||||
'Matt','Mark','Luke','John','Acts','Rom','1Cor','2Cor','Gal','Eph','Phil','Col','1Thess','2Thess','1Tim','2Tim','Titus','Phlm','Heb','Jas','1Pet','2Pet','1John','2John','3John','Jude','Rev']
|
||||
|
||||
TRANSLATIONS = [
|
||||
# id, code, name, has_markup
|
||||
(0, 'KJV', 'King James Version (1769)'),
|
||||
(1, 'WEB', 'World English Bible'),
|
||||
(2, 'BSB', 'Berean Standard Bible'),
|
||||
(3, 'ASV', 'American Standard Version (1901)'),
|
||||
(4, 'YLT', "Young's Literal Translation (1898)"),
|
||||
(5, 'DRC', 'Douay-Rheims, Challoner Revision'),
|
||||
]
|
||||
|
||||
def strip_strong(s):
|
||||
m = re.match(r'([HG])0*(\d+)([a-z]?)', s)
|
||||
return m.group(1) + m.group(2) if m else s
|
||||
|
||||
def convert_usfm_text(t, keep_italic=True):
|
||||
# footnotes and cross refs
|
||||
t = re.sub(r'\\f .*?\\f\*', '', t, flags=re.S)
|
||||
t = re.sub(r'\\x .*?\\x\*', '', t, flags=re.S)
|
||||
t = t.replace('[', '').replace(']', '')
|
||||
t = t.replace('¶', '')
|
||||
# strong's words, including nested \+w
|
||||
def wrep(m):
|
||||
word, attrs = m.group(2), m.group(3)
|
||||
sm = re.search(r'strong="([^"]+)"', attrs)
|
||||
word = word.strip()
|
||||
if not sm:
|
||||
return word
|
||||
num = strip_strong(sm.group(1).split(',')[0].split(' ')[0])
|
||||
return '{%s|%s}' % (word, num)
|
||||
t = re.sub(r'\\(\+?)w (.*?)\|([^\\]*?)\\\+?w\*', wrep, t)
|
||||
t = re.sub(r'\\\+?w (.*?)\\\+?w\*', r'\1', t) # untagged \w
|
||||
t = re.sub(r'\\\+?wj\s*', '<', t)
|
||||
t = t.replace('<*', '>')
|
||||
if keep_italic:
|
||||
t = re.sub(r'\\\+?add\s*', '[', t)
|
||||
t = t.replace('[*', ']')
|
||||
else:
|
||||
t = re.sub(r'\\\+?add\*?\s*', '', t)
|
||||
t = re.sub(r'\\\+?(nd|tl|qs|k|bk|wh|sc|it|em|bd|pn|qt|sig|ord)\*', '', t)
|
||||
t = re.sub(r'\\\+?(nd|tl|qs|k|bk|wh|sc|it|em|bd|pn|qt|sig|ord) ', '', t)
|
||||
t = re.sub(r'\\[a-z]+\d*\*?', ' ', t)
|
||||
t = re.sub(r'\s+', ' ', t).strip()
|
||||
# tidy spaces around markup
|
||||
t = re.sub(r'<\s+', '<', t)
|
||||
t = re.sub(r'\s+>', '>', t)
|
||||
t = re.sub(r'\[\s+', '[', t)
|
||||
t = re.sub(r'\s+\]', ']', t)
|
||||
t = re.sub(r'\s+([,.;:?!’”)])', r'\1', t)
|
||||
t = t.replace('<>', '').replace('[]', '')
|
||||
return t
|
||||
|
||||
def plain(t):
|
||||
t = re.sub(r'\{([^|}]*)\|[^}]*\}', r'\1', t)
|
||||
return t.replace('[', '').replace(']', '').replace('<', '').replace('>', '')
|
||||
|
||||
def parse_usfm(folder, suffix, keep_italic):
|
||||
verses = {}
|
||||
titles = {}
|
||||
for bi, code in enumerate(CODES):
|
||||
f = glob.glob(os.path.join(folder, '*%s%s.usfm' % (code, suffix)))
|
||||
assert f, code
|
||||
src = open(f[0], encoding='utf-8').read()
|
||||
ch = 0
|
||||
# split into lines by markers at line start
|
||||
cur = None
|
||||
buf = []
|
||||
def flush():
|
||||
if cur is not None:
|
||||
txt = convert_usfm_text(' '.join(buf), keep_italic)
|
||||
if txt:
|
||||
verses[(bi + 1, cur[0], cur[1])] = txt
|
||||
for line in src.split('\n'):
|
||||
line = line.rstrip()
|
||||
m = re.match(r'\\c (\d+)', line)
|
||||
if m:
|
||||
flush(); cur = None; buf = []
|
||||
ch = int(m.group(1)); continue
|
||||
if line.startswith('\\d '):
|
||||
flush(); cur = None; buf = []
|
||||
titles[(bi + 1, ch)] = plain(convert_usfm_text(line[3:], False))
|
||||
continue
|
||||
if re.match(r'\\(s\d?|ms\d?|mt\d?|toc\d|h|id|ide|mr|r|sp|rem|cl|is\d?|ip|imt\d?) ', line) or re.match(r'\\(s\d?|ms\d?)$', line):
|
||||
continue
|
||||
# line may contain multiple verses
|
||||
parts = re.split(r'(\\v \d+[a-z]?\s)', line)
|
||||
for p in parts:
|
||||
vm = re.match(r'\\v (\d+)', p)
|
||||
if vm:
|
||||
flush(); buf = []
|
||||
cur = (ch, int(vm.group(1)))
|
||||
elif cur is not None:
|
||||
buf.append(p)
|
||||
flush()
|
||||
return verses, titles
|
||||
|
||||
def parse_json(code):
|
||||
d = json.load(open(os.path.join(RAW, code + '.json')))
|
||||
out = {}
|
||||
def norm(n):
|
||||
n = n.replace('Revelation of John', 'Revelation')
|
||||
n = re.sub(r'^III ', '3 ', n); n = re.sub(r'^II ', '2 ', n); n = re.sub(r'^I ', '1 ', n)
|
||||
return n
|
||||
byname = {norm(b['name']): b for b in d['books']}
|
||||
for bi in range(66):
|
||||
b = byname[NAMES[bi]]
|
||||
for c in b['chapters']:
|
||||
for v in c['verses']:
|
||||
t = re.sub(r'\s+', ' ', v['text']).strip()
|
||||
t = t.replace('¶', '').strip()
|
||||
if t:
|
||||
out[(bi + 1, c['chapter'], v['verse'])] = t
|
||||
return out
|
||||
|
||||
def main():
|
||||
if os.path.exists(OUT):
|
||||
os.remove(OUT)
|
||||
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
||||
db = sqlite3.connect(OUT)
|
||||
db.executescript('''
|
||||
PRAGMA page_size=4096;
|
||||
CREATE TABLE android_metadata (locale TEXT);
|
||||
INSERT INTO android_metadata VALUES ('en_US');
|
||||
CREATE TABLE translations(id INTEGER PRIMARY KEY, code TEXT, name TEXT);
|
||||
CREATE TABLE books(id INTEGER PRIMARY KEY, name TEXT, abbr TEXT, chapters INTEGER);
|
||||
CREATE TABLE chapters(book INTEGER, chapter INTEGER, verses INTEGER, PRIMARY KEY(book,chapter)) WITHOUT ROWID;
|
||||
CREATE TABLE verses(tr INTEGER, book INTEGER, chapter INTEGER, verse INTEGER, text TEXT, plain TEXT, PRIMARY KEY(tr,book,chapter,verse)) WITHOUT ROWID;
|
||||
CREATE TABLE titles(book INTEGER, chapter INTEGER, text TEXT, PRIMARY KEY(book,chapter)) WITHOUT ROWID;
|
||||
CREATE TABLE xref(book INTEGER, chapter INTEGER, verse INTEGER, tb INTEGER, tc INTEGER, tv INTEGER, te INTEGER, votes INTEGER);
|
||||
CREATE TABLE strongs(id TEXT PRIMARY KEY, lemma TEXT, xlit TEXT, pron TEXT, deriv TEXT, def TEXT, kjv TEXT) WITHOUT ROWID;
|
||||
''')
|
||||
for t in TRANSLATIONS:
|
||||
db.execute('INSERT INTO translations VALUES (?,?,?)', t)
|
||||
|
||||
kjv, titles = parse_usfm(os.path.join(RAW, 'kjvusfm'), 'eng-kjv2006', True)
|
||||
web, _ = parse_usfm(os.path.join(RAW, 'webusfm'), 'eng-web', False)
|
||||
web = {k: re.sub(r'\{([^|}]*)\|[^}]*\}', r'\1', v) for k, v in web.items()}
|
||||
sets = {0: kjv, 1: web, 2: parse_json('BSB'), 3: parse_json('ASV'), 4: parse_json('YLT'), 5: parse_json('DRC')}
|
||||
print({k: len(v) for k, v in sets.items()})
|
||||
|
||||
# canonical structure from KJV
|
||||
chap = {}
|
||||
for (b, c, v) in kjv:
|
||||
chap[(b, c)] = max(chap.get((b, c), 0), v)
|
||||
for bi in range(66):
|
||||
n = max(c for (b, c) in chap if b == bi + 1)
|
||||
db.execute('INSERT INTO books VALUES (?,?,?,?)', (bi + 1, NAMES[bi], ABBR[bi], n))
|
||||
for (b, c), n in sorted(chap.items()):
|
||||
db.execute('INSERT INTO chapters VALUES (?,?,?)', (b, c, n))
|
||||
for tr, vs in sets.items():
|
||||
rows = []
|
||||
for (b, c, v), t in vs.items():
|
||||
p = plain(t) if tr in (0, 1) else t
|
||||
rows.append((tr, b, c, v, t if tr in (0, 1) else None, p))
|
||||
db.executemany('INSERT OR REPLACE INTO verses VALUES (?,?,?,?,?,?)', rows)
|
||||
for (b, c), t in titles.items():
|
||||
db.execute('INSERT INTO titles VALUES (?,?,?)', (b, c, t))
|
||||
|
||||
# cross references
|
||||
idx = {o: i + 1 for i, o in enumerate(OSIS)}
|
||||
def ref(s):
|
||||
p = s.split('.')
|
||||
return idx[p[0]], int(p[1]), int(p[2])
|
||||
per = {}
|
||||
for line in open(os.path.join(RAW, 'cross_references.txt'), encoding='utf-8'):
|
||||
if line.startswith('From') or not line.strip():
|
||||
continue
|
||||
a, bb, votes = line.rstrip('\n').split('\t')
|
||||
votes = int(votes)
|
||||
if votes < 1:
|
||||
continue
|
||||
try:
|
||||
f = ref(a)
|
||||
if '-' in bb:
|
||||
s, e = bb.split('-')
|
||||
t = ref(s); te = ref(e)
|
||||
tend = te[2] if te[:2] == t[:2] else 0
|
||||
else:
|
||||
t = ref(bb); tend = 0
|
||||
except (KeyError, ValueError, IndexError):
|
||||
continue
|
||||
per.setdefault(f, []).append((votes, t, tend))
|
||||
rows = []
|
||||
for f, lst in per.items():
|
||||
lst.sort(key=lambda x: -x[0])
|
||||
for votes, t, tend in lst[:8]:
|
||||
rows.append((f[0], f[1], f[2], t[0], t[1], t[2], tend, votes))
|
||||
db.executemany('INSERT INTO xref VALUES (?,?,?,?,?,?,?,?)', rows)
|
||||
db.execute('CREATE INDEX xi ON xref(book,chapter,verse)')
|
||||
print('xrefs', len(rows))
|
||||
|
||||
# strongs
|
||||
for fn, var in (('heb.js', 'strongsHebrewDictionary'), ('grk.js', 'strongsGreekDictionary')):
|
||||
s = open(os.path.join(RAW, fn), encoding='utf-8').read()
|
||||
j = s[s.index('{'):s.rindex('}') + 1]
|
||||
d = json.loads(j)
|
||||
for k, v in d.items():
|
||||
db.execute('INSERT OR REPLACE INTO strongs VALUES (?,?,?,?,?,?,?)', (
|
||||
k, v.get('lemma', ''), v.get('xlit', v.get('translit', '')), v.get('pron', ''),
|
||||
v.get('derivation', ''), v.get('strongs_def', ''), v.get('kjv_def', '')))
|
||||
db.commit()
|
||||
db.execute('VACUUM')
|
||||
db.close()
|
||||
print('size', os.path.getsize(OUT))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Procedural aged backgrounds: vellum, papyrus, parchment, candlelight (night)."""
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from scipy.ndimage import gaussian_filter, zoom
|
||||
import os
|
||||
|
||||
OUT = '/home/claude/vellum/app/src/main/res/drawable-nodpi'
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
W, H = 1440, 2560
|
||||
rng = np.random.default_rng(7)
|
||||
|
||||
def fbm(w, h, octaves=6, base=4, persistence=0.55, seed=0):
|
||||
r = np.random.default_rng(seed)
|
||||
out = np.zeros((h, w))
|
||||
amp = 1.0; tot = 0
|
||||
for o in range(octaves):
|
||||
n = base * 2 ** o
|
||||
g = r.random((max(2, int(n * h / w)) + 1, n + 1))
|
||||
z = zoom(g, (h / (g.shape[0] - 1), w / (g.shape[1] - 1)), order=3)[:h, :w]
|
||||
out += amp * z; tot += amp; amp *= persistence
|
||||
out /= tot
|
||||
out = (out - out.min()) / (out.max() - out.min())
|
||||
return out
|
||||
|
||||
def vignette(w, h, strength=0.5, power=2.2):
|
||||
y, x = np.mgrid[0:h, 0:w]
|
||||
dx = (x - w / 2) / (w / 2); dy = (y - h / 2) / (h / 2)
|
||||
d = np.sqrt(dx ** 2 * 1.0 + dy ** 2 * 0.85)
|
||||
return 1 - strength * np.clip(d, 0, 1.6) ** power / (1.6 ** power) * 1.6
|
||||
|
||||
def colorize(v, c0, c1):
|
||||
c0 = np.array(c0, float); c1 = np.array(c1, float)
|
||||
return c0[None, None, :] * (1 - v[..., None]) + c1[None, None, :] * v[..., None]
|
||||
|
||||
def save(img, name):
|
||||
a = np.clip(img, 0, 255).astype(np.uint8)
|
||||
Image.fromarray(a).save(os.path.join(OUT, name + '.webp'), quality=82, method=6)
|
||||
Image.fromarray(a).resize((360, 640)).save('/home/claude/data/prev_' + name + '.png')
|
||||
|
||||
def grain(w, h, amt, seed):
|
||||
r = np.random.default_rng(seed)
|
||||
return (r.random((h, w)) - 0.5) * amt
|
||||
|
||||
# ---- Vellum: warm cream, soft mottled clouds, faint hair follicle speckle, darker edges
|
||||
def vellum():
|
||||
m = fbm(W, H, 7, 3, 0.58, 1)
|
||||
m2 = fbm(W, H, 5, 8, 0.5, 2)
|
||||
base = colorize(m * 0.7 + m2 * 0.3, (222, 200, 160), (246, 234, 206))
|
||||
# stains
|
||||
st = fbm(W, H, 4, 2, 0.6, 3)
|
||||
stain = np.clip((st - 0.62) * 3.5, 0, 1)
|
||||
base = base * (1 - 0.12 * stain[..., None]) + np.array([150, 110, 60]) * 0.12 * stain[..., None]
|
||||
# follicle speckle
|
||||
sp = rng.random((H, W)) > 0.9985
|
||||
sp = gaussian_filter(sp.astype(float), 1.2) * 6
|
||||
base -= sp[..., None] * np.array([40, 45, 55])
|
||||
base += grain(W, H, 10, 4)[..., None]
|
||||
base *= vignette(W, H, 0.38, 2.6)[..., None]
|
||||
save(base, 'bg_vellum')
|
||||
|
||||
# ---- Papyrus: crossed plant strips, horizontal layer over vertical layer
|
||||
def papyrus():
|
||||
r = np.random.default_rng(11)
|
||||
# vertical strips
|
||||
def strips(n, length, axis, seed):
|
||||
rr = np.random.default_rng(seed)
|
||||
widths = rr.integers(60, 140, n)
|
||||
prof = []
|
||||
for wdt in widths:
|
||||
t = np.linspace(0, 1, wdt)
|
||||
edge = np.clip(np.minimum(t, 1 - t) * 9, 0, 1)
|
||||
fibers = gaussian_filter(rr.random(wdt), 0.8)
|
||||
prof.append(0.75 + 0.25 * edge + (fibers - 0.5) * 0.25 + rr.normal(0, 0.03))
|
||||
p = np.concatenate(prof)
|
||||
return p
|
||||
pv = strips(30, H, 0, 12)[:W]
|
||||
pv = np.pad(pv, (0, max(0, W - len(pv))), mode='edge')
|
||||
ph = strips(40, W, 1, 13)[:H]
|
||||
ph = np.pad(ph, (0, max(0, H - len(ph))), mode='edge')
|
||||
fv = gaussian_filter(r.random((H, W)), (40, 0.7)) # long vertical fibers
|
||||
fh = gaussian_filter(r.random((H, W)), (0.7, 40)) # long horizontal fibers
|
||||
fv = (fv - fv.mean()) / fv.std(); fh = (fh - fh.mean()) / fh.std()
|
||||
v = 0.55 * ph[:, None] * np.ones((1, W)) + 0.25 * pv[None, :] * np.ones((H, 1))
|
||||
v += 0.05 * fh + 0.025 * fv
|
||||
blot = fbm(W, H, 6, 3, 0.55, 14)
|
||||
v = v * 0.8 + blot * 0.25
|
||||
v = (v - v.min()) / (v.max() - v.min())
|
||||
img = colorize(v, (204, 174, 120), (244, 228, 188))
|
||||
dark = np.clip((fbm(W, H, 4, 2, 0.6, 15) - 0.66) * 3, 0, 1)
|
||||
img *= (1 - 0.10 * dark)[..., None]
|
||||
img += grain(W, H, 12, 16)[..., None]
|
||||
img *= vignette(W, H, 0.42, 2.4)[..., None]
|
||||
save(img, 'bg_papyrus')
|
||||
|
||||
# ---- Parchment: older, darker, more blotchy with burnt edges
|
||||
def parchment():
|
||||
m = fbm(W, H, 7, 3, 0.62, 21)
|
||||
img = colorize(m, (196, 160, 104), (238, 214, 168))
|
||||
bl = fbm(W, H, 5, 5, 0.55, 22)
|
||||
b = np.clip((bl - 0.58) * 4, 0, 1)
|
||||
img = img * (1 - 0.22 * b[..., None]) + np.array([120, 80, 35]) * 0.22 * b[..., None]
|
||||
y, x = np.mgrid[0:H, 0:W]
|
||||
edge = np.minimum.reduce([x, W - 1 - x, y, H - 1 - y]).astype(float)
|
||||
wob = fbm(W, H, 5, 6, 0.5, 23) * 70
|
||||
e = np.clip(1 - (edge - wob) / 120, 0, 1) ** 1.6
|
||||
img = img * (1 - 0.65 * e[..., None]) + np.array([70, 40, 15]) * 0.65 * e[..., None]
|
||||
img += grain(W, H, 12, 24)[..., None]
|
||||
img *= vignette(W, H, 0.3, 2.2)[..., None]
|
||||
save(img, 'bg_parchment')
|
||||
|
||||
# ---- Candlelight night: deep umber parchment, warm glow from lower centre
|
||||
def candle():
|
||||
m = fbm(W, H, 7, 3, 0.6, 31)
|
||||
img = colorize(m, (22, 14, 8), (52, 34, 18))
|
||||
y, x = np.mgrid[0:H, 0:W]
|
||||
d = np.sqrt(((x - W * 0.5) / W) ** 2 + ((y - H * 0.78) / H) ** 2 * 1.4)
|
||||
glow = np.exp(-(d / 0.42) ** 2)
|
||||
img += glow[..., None] * np.array([70, 40, 12])
|
||||
img += grain(W, H, 6, 32)[..., None]
|
||||
img *= vignette(W, H, 0.55, 2.0)[..., None]
|
||||
save(img, 'bg_candle')
|
||||
|
||||
if __name__ == '__main__':
|
||||
vellum(); papyrus(); parchment(); candle()
|
||||
for f in sorted(os.listdir(OUT)):
|
||||
print(f, os.path.getsize(os.path.join(OUT, f)))
|
||||
Reference in New Issue
Block a user