"""Rebuild a GPE application the way Psion built it. Psion's OpenEmbedded recipes pin an exact upstream version and then apply a list of their own patches to it. The upstream release tarballs are gone from gpe.linuxtogo.org but the whole tree is in the Wayback Machine, and every version Psion pinned is there, so each application can be reconstituted exactly: fetch the pinned tarball, apply Psion's patches in recipe order at the pnum each one declares, and build. This reads the recipe rather than hard-coding anything, so what gets applied is what Psion actually specified. python3 scripts/psionise.py [] """ import os import re import subprocess import sys import json import urllib.parse OE = os.path.expanduser('~/Downloads/PsionLinux-netBookPro/auditeon/oe-netbook') CDX = 'https://web.archive.org/cdx/search/cdx' # GPE_MIRROR in Psion's oe.conf is gpe.handhelds.org, which was never # crawled; the same tree at gpe.linuxtogo.org was, so the tarballs are # recoverable from there. MIRRORS = [ 'http://gpe.linuxtogo.org/download/source/', 'http://gpe.linuxtogo.org/download/releases/2.8/', 'http://gpe.handhelds.org/pub/projects/gpe/source/', 'http://ftp.handhelds.org/pub/projects/gpe/source/', ] def newest_recipe(pkg): cand = [f for f in os.listdir(os.path.join(OE, pkg)) if f.endswith('.oe')] if not cand: raise SystemExit(f'no recipe for {pkg}') def key(f): m = re.search(r'_([0-9][^_]*)\.oe$', f) return [int(x) if x.isdigit() else x for x in re.split(r'[._-]', m.group(1))] if m else [] return sorted(cand, key=key)[-1] def _sources(rp, text): """The recipe, plus anything it require/includes, in that order.""" out = [] for m in re.finditer(r'^\s*(?:require|include)\s+(\S+)', text, re.M): inc = m.group(1).replace('${PN}', os.path.basename(os.path.dirname(rp))) for base in (os.path.dirname(rp), OE, os.path.dirname(OE)): cand = os.path.join(base, inc) if os.path.isfile(cand): out.append(open(cand, errors='replace').read()) break out.append(text) return out def parse(pkg): """Version, and the patches in recipe order with their pnum.""" rp = os.path.join(OE, pkg, newest_recipe(pkg)) text = open(rp, errors='replace').read() m = re.search(r'_([0-9][^_]*)\.oe$', os.path.basename(rp)) pv = m.group(1) if m else None m = re.search(r'^PV\s*=\s*"([^"]+)"', text, re.M) if m: pv = m.group(1) # SRC_URI is not always a single assignment: recipes also use # "SRC_URI += " to add Psion's patches on top of what the gpe class # already set, and some pull in a shared .inc. Collect every form, # in file order, or patches are silently dropped. body = '' for chunk in _sources(rp, text): for m in re.finditer(r'SRC_URI\w*\s*[+?:]{0,2}=\s*"(.*?)"', chunk, re.S): body += m.group(1) + ' ' patches = [] seen = set() for entry in re.findall(r'file://(\S+)', body): name = entry.split(';')[0] if not name.endswith(('.patch', '.diff')) or name in seen: continue seen.add(name) pnum = 1 pm = re.search(r'pnum=(\d+)', entry) if pm: pnum = int(pm.group(1)) patches.append((name, pnum)) return pv, patches, rp, _urls(body, pkg, pv) def _urls(body, pkg, pv): """The upstream source URLs a recipe names, with OE's vars expanded.""" out = [] for u in re.findall(r'(?:https?|ftp)://\S+', body): u = u.split(';')[0].rstrip('\\').strip() u = (u.replace('${GPE_MIRROR}', MIRRORS[0].rstrip('/')) .replace('${PN}', pkg).replace('${P}', f'{pkg}-{pv}') .replace('${PV}', pv).replace('${BP}', f'{pkg}-{pv}')) if '${' in u: continue out.append(u) # the gpe class supplies the tarball name when a recipe gives no URL for ext in ('tar.gz', 'tar.bz2'): for m in MIRRORS: out.append(f'{m}{pkg}-{pv}.{ext}') seen, uniq = set(), [] for u in out: if u not in seen: seen.add(u) uniq.append(u) return uniq WB_DIR = os.path.expanduser( '~/Downloads/PsionLinux-netBookPro/workspace/build/.wayback') def _wb_index(): """basename -> (original url, timestamp), from a cached CDX listing. The archive rate-limits hard, so the whole gpe.linuxtogo.org source tree is indexed once into build/.wayback/index.json and every lookup after that is local. Deleting that file makes the next run rebuild it. """ f = os.path.join(WB_DIR, 'index.json') if os.path.exists(f): return {k: tuple(v) for k, v in json.load(open(f)).items()} return {} def wayback(url, dest, index=None): """Fetch url from the Internet Archive, if it was ever captured.""" index = _wb_index() if index is None else index hit = index.get(os.path.basename(url)) if hit: orig, ts = hit snaps = [f'https://web.archive.org/web/{ts}id_/{orig}'] else: q = (f'{CDX}?url={urllib.parse.quote(url, safe="")}' '&output=json&filter=statuscode:200&limit=2') r = subprocess.run(['curl', '-s', '--max-time', '30', q], capture_output=True, text=True) try: rows = json.loads(r.stdout or '[]') except ValueError: return False snaps = [f'https://web.archive.org/web/{row[1]}id_/{url}' for row in rows[1:]] for snap in snaps: d = subprocess.run(['curl', '-sL', '--max-time', '180', snap, '-o', dest], capture_output=True) if d.returncode == 0 and os.path.exists(dest) \ and os.path.getsize(dest) > 2000: with open(dest, 'rb') as fh: magic = fh.read(2) if magic in (b'\x1f\x8b', b'BZ'): return True return False def find_patch(pkg, name, pv): """Psion keep patches beside the recipe or in a version subdirectory.""" for d in (os.path.join(OE, pkg, f'{pkg}-{pv}'), os.path.join(OE, pkg, pkg), os.path.join(OE, pkg, 'files'), os.path.join(OE, pkg)): p = os.path.join(d, name) if os.path.isfile(p): return p for root, dirs, files in os.walk(os.path.join(OE, pkg)): if 'CVS' in root: continue if name in files: return os.path.join(root, name) return None GPE_SRC = os.path.expanduser( '~/Downloads/PsionLinux-netBookPro/workspace/build/gpe-src') def git_path(pkg): for sub in ('base', 'extra', 'games', 'legacy', 'eyecandy'): d = os.path.join(GPE_SRC, sub, pkg) if os.path.isdir(d): return f'{sub}/{pkg}' return None def declared_version(commit, path): """The version this package declares at a given commit. GPE is not consistent about this: newer packages use AC_INIT in configure.ac, older ones AM_INIT_AUTOMAKE in configure.in, and the plainest ones just set VERSION in a hand-written Makefile. Read all of them, or the pinned release is reported as missing when it is present. """ for f in ('configure.ac', 'configure.in', 'Makefile', 'config.makefile'): r = subprocess.run(['git', 'show', f'{commit}:{path}/{f}'], cwd=GPE_SRC, capture_output=True, text=True, errors='replace') if r.returncode: continue t = r.stdout for pat in (r'AC_INIT\s*\(\s*\[?[^,\]]*\]?\s*,\s*\[?\s*([0-9][^,\)\]\s]*)', r'AM_INIT_AUTOMAKE\s*\(\s*\[?[^,\]]*\]?\s*,\s*\[?\s*([0-9][^,\)\]\s]*)', r'^\s*VERSION\s*=\s*([0-9]\S*)'): m = re.search(pat, t, re.M) if m: return m.group(1).strip() return None def fetch_git(pkg, pv, into): """Take the pinned version out of the local GPE history. The upstream tarballs are only in the Wayback Machine, which rate-limits heavily; the git mirror is local and gives the same tree. There are no release tags, so the commit is found by walking the package's history and reading the version it declares. Psion patched the released tarball, so the commit wanted is the OLDEST one declaring pv -- the release point -- not a later one still declaring pv while carrying fixes. base/ has real history. extra/ and legacy/ were imported as a squashed snapshot, so for those the pinned release is simply not in the mirror; fall back to the oldest tree there and say so, since Psion's patches still carry most of their intent onto it. """ path = git_path(pkg) if not path: return None, None log = subprocess.run(['git', 'log', '--format=%H', '--all', '--', path], cwd=GPE_SRC, capture_output=True, text=True) commits = log.stdout.split() matching = [c for c in reversed(commits) if declared_version(c, path) == pv] note = 'exact' if not matching and commits: oldest = commits[-1] matching = [oldest] note = 'nearest: ' + (declared_version(oldest, path) or '?') for commit in matching: dest = os.path.join(into, f'{pkg}-{pv}.tar') with open(dest, 'wb') as fh: r = subprocess.run(['git', 'archive', '--format=tar', f'--prefix={pkg}-{pv}/', commit, path], cwd=GPE_SRC, stdout=fh) if r.returncode == 0 and os.path.getsize(dest) > 1000: return dest, note return None, None def fetch(pkg, pv, into, urls): """The pinned release tarball, from the Internet Archive if possible. Psion patched released tarballs. The archive has gpe.linuxtogo.org's whole source tree, so in almost every case the exact tarball can be had; the git mirror is the fallback for packages that were never mirrored there, and for those the tree is only an approximation. """ for u in urls: name = os.path.basename(u) dest = os.path.join(into, name) if os.path.isfile(dest) and os.path.getsize(dest) > 2000: return dest, 'cached ' + name if wayback(u, dest): return dest, 'archive ' + name if os.path.exists(dest): os.unlink(dest) return fetch_git(pkg, pv, into) def main(): pkg = sys.argv[1] stage = sys.argv[2] if len(sys.argv) > 2 else os.path.expanduser( '~/Downloads/PsionLinux-netBookPro/workspace/build/buildroot/build') cache = os.path.join(stage, '.tarballs') os.makedirs(cache, exist_ok=True) pv, patches, rp, urls = parse(pkg) print(f' {pkg}: version {pv}, {len(patches)} patches, from {os.path.basename(rp)}') tb, note = fetch(pkg, pv, cache, urls) if not tb: print(f' {pkg}: could not fetch the {pv} tarball'); return 1 print(f' fetched {os.path.basename(tb)} ({os.path.getsize(tb)} B) [{note}]') work = os.path.join(stage, pkg) subprocess.run(['rm', '-rf', work], check=False) os.makedirs(work, exist_ok=True) flag = 'xjf' if tb.endswith('bz2') else ('xzf' if tb.endswith('gz') else 'xf') strip = '3' if tb.endswith('.tar') else '1' # git archive keeps base// subprocess.run(['tar', flag, tb, '-C', work, f'--strip-components={strip}'], check=True) ok = bad = up = 0 for name, pnum in patches: p = find_patch(pkg, name, pv) if not p: print(f' {name:44s} NOT FOUND'); bad += 1; continue r = subprocess.run(['patch', f'-p{pnum}', '--batch', '--forward', '-l', '-F', '3', '-i', p], cwd=work, capture_output=True, text=True) out = r.stdout + r.stderr if r.returncode == 0: print(f' {name:44s} applied (-p{pnum})'); ok += 1 elif 'previously applied' in out or 'Reversed' in out: # the tree is newer than the release Psion pinned and this # change reached upstream; nothing to do, and not a failure print(f' {name:44s} already upstream'); up += 1 else: tail = out.strip().splitlines() print(f' {name:44s} FAILED (-p{pnum}): {tail[-1][:60] if tail else "?"}') bad += 1 print(f' {pkg}: {ok} applied, {up} already upstream, {bad} failed') return 0 if bad == 0 else 2 if __name__ == '__main__': sys.exit(main())