Skip to content

Commit 12e40d5

Browse files
authored
xpatch: run-time switchable instruction patches for the lifter (#9)
The lifter can lift a site twice, the original and a patched version behind a run-time switch (g_xpatch), so a feature can change game code that is translated ahead of time. Sites are listed in tools/xpatch_eu.txt; with an empty list the generated code is unchanged.
1 parent 44cf882 commit 12e40d5

8 files changed

Lines changed: 291 additions & 9 deletions

File tree

‎link.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ case "${BUILD:-eu}" in
4242
*) echo "BUILD must be eu or na"; exit 1 ;;
4343
esac
4444

45-
CSRC="mem dispatch genexec win32 win32_ext com debug res fileio overrides d3dx_hle vsemu enhance perflog wsfov gheap sndlead"
45+
CSRC="mem dispatch genexec win32 win32_ext com debug res fileio overrides d3dx_hle vsemu enhance perflog wsfov gheap sndlead xpatch"
4646
MSRC="metal audio bundle gamepad"
4747
FW="-lm -framework Cocoa -framework Metal -framework QuartzCore -framework ImageIO -framework CoreGraphics -framework CoreServices -framework AudioToolbox -framework GameController"
4848
SHARDS="$SOBJ/shard_*.o $SOBJ/fntab.o $SOBJ/stubs.o"

‎runtime/com.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4728,6 +4728,7 @@ void com_report(void){
47284728
fprintf(stderr,"\n=== COM / D3D9 KATMANI ===\n");
47294729
{ void ws_report(void); ws_report(); } /* wsfov.c */
47304730
{ void gamepad_report(void); gamepad_report(); } /* gamepad.m */
4731+
{ void xpatch_report(void); xpatch_report(); }
47314732
if (g_aspfix != 1.f)
47324733
{ fprintf(stderr," widescreen : aspfix %.4f, %lu perspective / %lu orthographic draws\n",
47334734
g_aspfix, g_asp_persp, g_asp_ortho);

‎runtime/xpatch.c‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/* Run-time switches for the "xpatch" sites (tools/xpatch.py).
2+
*
3+
* An xpatch site is lifted twice, the original code and a patched version
4+
* behind `if (g_xpatch[XP_<name>])`. This file owns the switch arrays; a
5+
* feature turns its patch on with xpatch_set("name", 1) or by writing
6+
* g_xpatch[XP_<NAME>] directly, and may change it while the game runs.
7+
* g_xpval[] holds the values of the `wv` sites. With no sites in
8+
* tools/xpatch_eu.txt both arrays have one unused entry and the generated
9+
* code is unchanged. */
10+
#include <stdio.h>
11+
#include <string.h>
12+
#include "xpatch.h"
13+
14+
unsigned char g_xpatch[XP_COUNT > 0 ? XP_COUNT : 1];
15+
unsigned int g_xpval[XPV_COUNT > 0 ? XPV_COUNT : 1];
16+
17+
/* 1 if the switch exists */
18+
int xpatch_set(const char *name, int on){
19+
for (int i = 0; i < XP_COUNT; i++)
20+
if (!strcmp(xp_names[i], name)){ g_xpatch[i] = on ? 1 : 0; return 1; }
21+
return 0;
22+
}
23+
24+
void xpatch_report(void){
25+
int n = 0;
26+
for (int i = 0; i < XP_COUNT; i++) n += g_xpatch[i] != 0;
27+
fprintf(stderr, " xpatch : %d switches, %d on", XP_COUNT, n);
28+
for (int i = 0; i < XP_COUNT; i++) if (g_xpatch[i]) fprintf(stderr, " %s", xp_names[i]);
29+
fprintf(stderr, "\n");
30+
}

‎tools/gen_v2.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ SHA="$(shasum -a 256 "$EXE" | cut -c1-64)"
2626
case "$SHA" in
2727
8f6a9ae2c4cfa01181d2fd755d99c6f4b7103f60273908ea1de1e719dfe2a297)
2828
BUILD=eu; DEF=gen/all; WORK=out
29-
export HOOKS="${HOOKS:-0x5c7ffe,0x4ec35c,0x726336}" ;; # runtime/wsfov.c, sndlead.c
29+
export HOOKS="${HOOKS:-0x5c7ffe,0x4ec35c,0x726336}" # runtime/wsfov.c, sndlead.c
30+
export XPATCH="${XPATCH:-tools/xpatch_eu.txt}" ;; # tools/xpatch.py
3031
f9dd86c054878ce6276beb07c1fd61874f7a1e4bf1f241b084c65b73e24168a7)
3132
BUILD=na; DEF=gen/na; WORK=out/na; OVR=$OVR_NA; export LIFT_MISSING=1
3233
export HOOKS="${HOOKS:-0x5c7fee,0x4ec35c,0x726336}" ;; # runtime/wsfov.c, sndlead.c (same sites, NA addresses)

‎tools/genall.py‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@
55
import pefile
66
from capstone import *
77
from capstone.x86 import *
8-
from lift import Lifter, Unsupported
8+
from lift import Lifter, Unsupported, XPatchError
9+
import xpatch
910

1011
EXE = sys.argv[1]
1112
NSHARD = int(sys.argv[2]) if len(sys.argv)>2 else 128
@@ -99,6 +100,7 @@ def _init(inl=None):
99100
cov = os.environ.get('COVERAGE','') == '1'
100101
_L = Lifter(EXE, comments=os.environ.get('COMMENTS','') == '1', trap=True, hooks=hooks, watch=watch)
101102
_L.cov = cov
103+
_L.xp = xpatch.load(os.environ.get('XPATCH')) # LOCAL PATCH: run-time switchable patches
102104

103105
def _inlprobe(a):
104106
"""leaf + small + not overridden -> (address, code) or (address, None)"""
@@ -121,6 +123,8 @@ def _shard(args):
121123
L.traps=0; L.truncated=0
122124
try:
123125
code,tg = L.lift(a)
126+
except XPatchError:
127+
raise
124128
except Exception as e:
125129
fb = f"void sub_{a:08x}(CPU *c){{ unsupported_insn(c,0x{a:x}u,\"lift-error\"); POP32(c->eip); }}"
126130
parts.append(_memarg(fb) if MEMARG else fb)
@@ -148,7 +152,7 @@ def _shard(args):
148152
f"static inline __attribute__((always_inline)) void inl_{x:08x}(CPU *restrict c) {{") + "\n")
149153
f.write("\n".join(parts))
150154
f.write("\n")
151-
return idx, defined, sorted(refs), traps, trunc, dict(kinds), os.path.getsize(path)
155+
return idx, defined, sorted(refs), traps, trunc, dict(kinds), os.path.getsize(path), sorted(L.xp_hits)
152156

153157
if __name__=='__main__':
154158
t0=time.time()
@@ -171,8 +175,9 @@ def _shard(args):
171175
print(f" [inline: {len(inl)} leaf functions <= {INLINE_MAX} instructions] ({time.time()-t1:.0f}s)")
172176
defined=set(); refs=set(); traps=0; trunc=0; kinds=collections.Counter(); bytes_=0
173177
with mp.Pool(mp.cpu_count(), initializer=_init, initargs=(inl,)) as pool:
174-
for n,(idx,d,r,tp,tr,kk,sz) in enumerate(pool.imap_unordered(_shard, chunks)):
175-
defined |= set(d); refs |= set(r); traps+=tp; trunc+=tr
178+
xhits=set()
179+
for n,(idx,d,r,tp,tr,kk,sz,xh) in enumerate(pool.imap_unordered(_shard, chunks)):
180+
defined |= set(d); refs |= set(r); traps+=tp; trunc+=tr; xhits |= set(xh)
176181
kinds.update(kk); bytes_+=sz
177182
if (n+1)%16==0: print(f" {n+1}/{NSHARD} shard ({time.time()-t1:.0f}s)")
178183
print(f" generation done ({time.time()-t1:.0f}s)")
@@ -186,10 +191,21 @@ def _shard(args):
186191
while True:
187192
more = sorted(a for a in refs - defined if _L.in_text(a))
188193
if not more: break
189-
idx, d, r, tp, tr, kk, sz = _shard((extra, more))
194+
idx, d, r, tp, tr, kk, sz, xh = _shard((extra, more)); xhits |= set(xh)
190195
print(f" [lift-missing] shard_{extra:04d}: {', '.join(hex(a) for a in more)}")
191196
defined |= set(d); refs |= set(r); traps += tp; trunc += tr; kinds.update(kk); bytes_ += sz
192197
extra += 1
198+
# LOCAL PATCH: xpatch header + a check that every site was found in some function
199+
spec = xpatch.load(os.environ.get('XPATCH'))
200+
with open(f"{OUT}/xpatch.h",'w') as f:
201+
f.write(spec.header() if spec else "#pragma once\n#define XP_COUNT 0\n#define XPV_COUNT 0\n"
202+
"static const char *const xp_names[] = { 0 };\nstatic const char *const xpv_names[] = { 0 };\n")
203+
if spec:
204+
hit_names = {n for n, a in xhits}
205+
lost = [n for n in spec.names if n not in hit_names]
206+
print(f" [xpatch: {len(spec.names)} switches, {len(xhits)} sites lifted]")
207+
for n, a in sorted(xhits, key=lambda t: t[1]): print(f" {a:#x} {n}")
208+
if lost: raise SystemExit(f"xpatch: no site found for {lost}")
193209
missing = sorted(refs - defined)
194210
print(f"\n defined: {len(defined)} referenced: {len(refs)} MISSING: {len(missing)}")
195211
if missing or (MEMARG and OVERRIDES):
@@ -207,6 +223,7 @@ def _shard(args):
207223
f.write("#ifndef DECLS_H\n#define DECLS_H\n#include \"cpu.h\"\n")
208224
f.write("void dispatch_indirect(CPU*,uint32_t);\nvoid unsupported_insn(CPU*,uint32_t,const char*);\n")
209225
f.write("void cpuid_shim(CPU*);\nuint64_t rdtsc_shim(void);\n")
226+
f.write("extern unsigned char g_xpatch[];\nextern unsigned int g_xpval[];\n") # LOCAL PATCH: xpatch
210227
for a in allf: f.write(f"void sub_{a:08x}(CPU*);\n")
211228
for a in sorted(OVERRIDES): f.write(f"void sub_{a:08x}_orig(CPU*);\n")
212229
if MEMARG:

‎tools/lift.py‎

Lines changed: 79 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,10 @@ class Unsupported(Exception): pass
291291
# call, ret, an instruction touching c->st/c->top, end of block) only the modified
292292
# slots still on the stack are written back and c->top is updated once.
293293
import re as _re
294+
import re
295+
PLACEHOLDERS_XP = (0x5d, 0x6b, 0x71, 0x4f, 0x63, 0x3b)
296+
class XPatchError(Exception):
297+
"""a bad xpatch site: generation must stop (genall re-raises it)"""
294298
_ST_RE = _re.compile(r'ST\((\d+)\)')
295299
_STA_RE = _re.compile(r'ST\((\d+)\)\s*=(?!=)')
296300
_BOUNDARY_RE = _re.compile(r'\b\w+\(c[,)]|\breturn\b|c->st|c->top|\bST\(|FPUSH|FPOP|fsw_get')
@@ -354,7 +358,7 @@ class RawLifter:
354358
def __init__(self, data, base=0x00401000, comments=True):
355359
self.comments=comments; self.trap=False; self.traps=0
356360
self.trap_kinds=collections.Counter(); self.hooks=set(); self.watch=False
357-
self.cov=False
361+
self.cov=False; self.xp=None
358362
self.secs=[('.text', base, len(data), bytes(data))]
359363
self.TVA=base; self.TSZ=len(data); self.TDATA=bytes(data)
360364
self.md=Cs(CS_ARCH_X86,CS_MODE_32); self.md.detail=True
@@ -364,7 +368,7 @@ class Lifter:
364368
def __init__(self, pe_path, comments=True, trap=False, hooks=None, watch=False):
365369
self.comments=comments; self.trap=trap; self.traps=0; self.trap_kinds=collections.Counter()
366370
self.hooks=set(hooks or ()); self.watch=watch
367-
self.cov=False
371+
self.cov=False; self.xp=None; self.xp_hits=set()
368372
pe = pefile.PE(pe_path); self.IB = pe.OPTIONAL_HEADER.ImageBase
369373
self.secs=[]
370374
for s in pe.sections:
@@ -1599,6 +1603,65 @@ def qwords(k, half):
15991603

16001604
raise Unsupported(m)
16011605

1606+
# ---------- LOCAL PATCH: run-time switchable patches (tools/xpatch.py) ----------
1607+
def _xp_lines(self, s, data, lbl, targets):
1608+
"""patched bytes -> C lines; returns (lines, ends_with_transfer)"""
1609+
L = []; pos = s.a; ends = False
1610+
saved = getattr(self, 'cur_fmask', F_ALL)
1611+
for i in self.md.disasm(data, s.a):
1612+
m = i.mnemonic; pos = i.address + i.size
1613+
if m == 'jmp' and i.operands[0].type == X86_OP_IMM:
1614+
t = i.operands[0].imm
1615+
if t not in lbl: raise XPatchError(f"xpatch {s.name}: jmp to {t:#x} is not a label")
1616+
L.append(f"goto L_{t:08x};"); ends = True; break
1617+
if m.startswith('j') and m[1:] in CC and i.operands[0].type == X86_OP_IMM:
1618+
t = i.operands[0].imm
1619+
if t not in lbl: raise XPatchError(f"xpatch {s.name}: {m} to {t:#x} is not a label")
1620+
L.append(f"if({CC[m[1:]]}) goto L_{t:08x};")
1621+
else:
1622+
self.cur_fmask = F_ALL
1623+
if m[:1] == 'f' or X86_GRP_FPU in i.groups:
1624+
raise XPatchError(f"xpatch {s.name}: x87 in patched code")
1625+
try: L += self.insn(i, targets)
1626+
except Unsupported as e: raise XPatchError(f"xpatch {s.name}: {m} not supported ({e})")
1627+
if m in ('ret', 'retf'): ends = True; break
1628+
if s.kind == 'w' and pos >= s.e: break
1629+
self.cur_fmask = saved
1630+
if s.kind == 'w' and not ends and pos != s.e:
1631+
raise XPatchError(f"xpatch {s.name}: patched code at {s.a:#x} ends at {pos:#x}, region ends at {s.e:#x}")
1632+
if s.kind == 'cave' and not ends and pos != s.a + len(data):
1633+
raise XPatchError(f"xpatch {s.name}: cave did not decode to its end")
1634+
return L, ends
1635+
1636+
def _xp_emit(self, s, lbl, targets, resume):
1637+
if s.kind == 'c':
1638+
L = [s.data]; ends = s.data.rstrip().endswith('return;')
1639+
elif s.kind == 'cave':
1640+
L, ends = self._xp_lines(s, s.data, lbl, targets)
1641+
else:
1642+
L = None
1643+
for ph in PLACEHOLDERS_XP:
1644+
buf = bytearray(s.data)
1645+
for off, var in s.var: buf[off] = ph
1646+
L0, ends = self._xp_lines(s, bytes(buf), lbl, targets)
1647+
if not s.var: L = L0; break
1648+
tok = re.compile(r'\b0x%xu\b' % ph)
1649+
text = "\n".join(L0)
1650+
if len(s.var) == 1 and len(tok.findall(text)) == 1:
1651+
var = s.var[0][1]
1652+
L = [tok.sub(f"((uint32_t)g_xpval[{self.xp.vars.index(var)}] /* {var} */)", text)]
1653+
break
1654+
if L is None:
1655+
raise XPatchError(f"xpatch {s.name}: no unique placeholder at {s.a:#x}")
1656+
self.xp_hits.add((s.name, s.a))
1657+
out = [f"if (g_xpatch[{s.idx}]) {{ /* xpatch {s.name} {s.a:#x}..{s.e:#x} */"]
1658+
out += [" " + l for l in L]
1659+
if not ends:
1660+
out.append(f" goto L_{s.e:08x};" if s.e in lbl else f" goto XR_{s.e:08x};")
1661+
if s.e not in lbl: resume.add(s.e)
1662+
out.append("}")
1663+
return out
1664+
16021665
# ---------- function -> C ----------
16031666
def lift(self, entry):
16041667
blocks, calls = self.discover(entry)
@@ -1622,12 +1685,26 @@ def lift(self, entry):
16221685
body=[]
16231686
x87opt = _os.environ.get('X87OPT','') == '1'
16241687
xb = X87Region()
1688+
# LOCAL PATCH: xpatch sites in this function; the resume labels needed
1689+
# are known after the sites are emitted, so they are collected first.
1690+
xsites = self.xp.sites_for(self, blocks) if self.xp else {}
1691+
xresume = set(); xr_done = set(); xs_done = set()
1692+
if xsites:
1693+
for s in xsites.values():
1694+
if s.e not in lbl: xresume.add(s.e)
16251695
for ba in addrs:
16261696
body.append(f"L_{ba:08x}:; /* --- block --- */")
16271697
ins=blocks[ba]
16281698
xb.reset()
16291699
for k,i in enumerate(ins):
16301700
m=i.mnemonic
1701+
if i.address in xresume and i.address not in xr_done:
1702+
if x87opt and xb.active: body += [" "+l for l in xb.close()]
1703+
body.append(f"XR_{i.address:08x}:;")
1704+
xr_done.add(i.address)
1705+
if i.address in xsites:
1706+
if x87opt and xb.active: body += [" "+l for l in xb.close()]
1707+
body += [" "+l for l in self._xp_emit(xsites[i.address], lbl, targets, set())]
16311708
if x87opt and xb.active and (m=='jmp' or (m.startswith('j') and m[1:] in CC)
16321709
or m in ('jecxz','jcxz','loop','loope','loopz','loopne','loopnz')):
16331710
body += [" "+l for l in xb.close()]

0 commit comments

Comments
 (0)