TP3/shell/tests/audit_helpers.py

#!/usr/bin/env python3
"""audit_helpers.py -- check that Runtime.mod's one-liner emitter procedures
emit the instruction their name claims.

Why this exists
---------------
Runtime.mod hand-assembles 8086 by writing raw bytes.  Most of it is wrapped
up as one-line helpers:

    PROCEDURE MovSiBx ; BEGIN B (89H) ; B (0DCH) END MovSiBx ;

and those names are the only documentation of what the bytes mean.  That is
exactly the setup for a silent, expensive mistake: write the right opcode and
the wrong ModRM, and nothing complains.  It happened twice here, in the same
direction both times:

    MovSiBx  emitted 89 DC   -- which is MOV SP,BX, not MOV SI,BX
    CmpSiBx  emitted 39 DC   -- ditto, CMP SP,BX

The comment next to CmpSiBx even spelled the ModRM out as "11 011 100" and
still got it wrong, because 11 011 100 in mod=11 means rm=100=SP; SI is
rm=100 only in mod=00, where it means [SI].  Both bugs shipped together into
EmitWrInt, which then stored decimal digits through an SI register it had
never initialised.  The structural checks in check_runtime.py could not see
any of this: the bytes decoded cleanly, the sweep stayed in sync, the branch
targets were all on boundaries and none of the entry prologues moved.  Only a
*name* versus a *decode* comparison finds it, because only that knows what
the author was trying to say.

So: for every helper whose body is a literal byte list, disassemble those
bytes with FCML and require the decode to match the name.  The name grammar
is deliberately narrow and mechanical:

    <Op><Dest><Src>        e.g. MovSiBx, StDiAx, CmpSpBx, MovDlSi
    <Op><Reg>              e.g. XorAxAx, NegAx, NotDx, IncSi, DecSi

with Op in {Mov, Lea, St, Ld, Cmp, Add, Sub, Xor, And, Or, Not, Inc, Dec,
Push, Pop, Int, Jmp, Jne, Jge, Jnl, Jle, Jl, Je, Jae, Jbe, Jb, Ja, JaE...}
and the register/operand words below.  A helper whose name does not parse is
reported as UNPARSED rather than silently skipped -- a name the grammar does
not understand is a name we are not checking, and that must be visible.

This is a source-level check, so it needs no re-baselining: it is a function
of the current source, and it gets stricter as more names are added to the
grammar.  It also runs without building anything.

Usage: audit_helpers.py [MODULE.mod]      # default ../Runtime.mod
       audit_helpers.py -v                 # print every helper, passing or not
"""

import os
import re
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
SHELL = os.path.dirname(HERE)
sys.path.insert(0, HERE)
import disasm16  # noqa: E402  (path set above)

RE_HELPER = re.compile(
    r"^PROCEDURE\s+(\w+)\s*;\s*BEGIN\s+(.*?)\s+END\s+\1\s*;", re.MULTILINE)
# The H suffix is optional: the source mixes B (8AH) and B (0) for the same
# kind of literal, and a byte written without H used to be silently dropped
# from the audit, which made three helpers look like truncated prefixes.
RE_BYTE = re.compile(r"B\s*\(\s*([0-9A-Fa-f]+)H?\s*\)")

# --- name grammar -----------------------------------------------------
#
# An ordered table of (name regex, mnemonic, operand specs, allowed opcodes).
#
# Why the opcode column exists
# ----------------------------
# "MovDlSi" and "MovBpSp" are ambiguous from the name alone: SI and BP are
# both in the register list and in the memory-base list.  What settles it is
# the opcode, and the rule is the asymmetry that makes 16-bit hand-assembly
# so error-prone:
#
#     88 /r  MOV r/m8, r8    reg is the SOURCE     (store)
#     89 /r  MOV r/m16, r16  reg is the SOURCE     (store)
#     8A /r  MOV r8, r/m8    reg is the DESTINATION (load)
#     8B /r  MOV r16, r/m16  reg is the DESTINATION (load)
#
# A byte move (8A/88) to or from a bare base register can only be a memory
# access, because "mov dl, si" is not an instruction.  A word move (8B/89)
# could be either, so for word moves the register reading is tried first.
# Encoding that rule in the table, rather than in pattern order alone, is what
# stops a helper being "verified" against the wrong reading of its own name.
#
# Operand spec tokens:
#
#     "r:Name"     a bare register spelled Name (Ax, Al, Dx, Si, Ds, ...)
#     "i:N"        the immediate N; the name spells it in DECIMAL
#     "ih:N"       like i: but the name spells it in HEX (only Int, whose
#                  vector 21h FCML prints as "21h" and which reads as decimal
#                  33 if you do not notice)
#     "m:Base"     memory at [Base], with no displacement
#     "m:Base+D"   memory at [Base + D]
#
# "Arg" is the emitter's alias for BP: on entry BP points at the return
# address, so a word argument starts at [BP+2].  Naming it "Arg" rather than
# "Bp" is what stops these being misread as [BP] accesses, and it is also
# how the grammar knows to expect the +2.
#
# A name matching NO pattern is reported UNPARSED, which counts as a failure
# on purpose: a helper the audit cannot read is a helper nobody is checking.

REGS = ["Ax", "Bx", "Cx", "Dx", "Si", "Di", "Bp", "Sp",
        "Al", "Bl", "Cl", "Dl", "Ah", "Bh", "Ch", "Dh"]
# Segment registers: the 8086 can only PUSH/POP them, never MOV to or from
# one, so they need names of their own.
SEGS = ["Ds", "Es", "Cs", "Ss"]
JCCS = ["Je", "Jne", "Jz", "Jnz", "Jge", "Jnl", "Jle", "Jl", "Ja", "Jae",
        "Jb", "Jbe", "Jg", "Jns", "Js", "Jo", "Jno", "Jp", "Jnp", "Jcxz",
        "Jecxz", "Jrcxz", "Loop", "Loope", "Loopne"]

_ALT = "|".join(REGS)
_BASE = "Si|Di|Bx|Bp|Sp|Arg|Data"
_STORE = (0x88, 0x89)      # reg is the source
_LOAD = (0x8A, 0x8B)       # reg is the destination

PATTERNS = [
    # --- no-operand and fixed-operand forms -------------------------
    (re.compile(r"^RetR?$"),            "ret",    [], None),
    (re.compile(r"^LeaveR?$"),          "leave",  [], None),
    (re.compile(r"^Int([0-9A-Fa-f]+)$"), "int",   ["ih:%(1)s"], {0xCD}),
    (re.compile(r"^(Push|Pop)(%s)$" % "|".join(SEGS)),
     None, ["r:%(2)s"], None),
    (re.compile(r"^(Push|Pop)(%s)$" % _ALT), None, ["r:%(2)s"], None),
    # --- jumps: the target is a fixup, so only the mnemonic is checked
    (re.compile(r"^(%s)(8|16)?$" % "|".join(JCCS)), None, [], None),
    (re.compile(r"^Jmp(%s)$" % _ALT),   "jmp",    ["r:%(1)s"], {0xFF}),
    # --- MUL names one operand; 16-bit DIV names two because it always
    #     divides DX:AX, so the decode has an AX the name has no room for
    (re.compile(r"^Mul(%s)$" % _ALT),   "mul",    ["r:%(1)s"], {0xF7}),
    (re.compile(r"^Div(%s)$" % _ALT),   "div",    ["r:Ax", "r:%(1)s"], {0xF7}),
    # --- store: name is St<base><reg>, decode puts the register last --
    (re.compile(r"^St(%s)(%s)$" % (_BASE, _ALT)),
     "mov", ["m:%(1)s", "r:%(2)s"], _STORE),
    # --- load a register from memory, with a displacement ------------
    # The displacement makes the name unambiguous, so any load opcode works.
    (re.compile(r"^Mov(%s)(%s)(\d+)$" % (_ALT, _BASE)),
     "mov", ["r:%(1)s", "m:%(2)s+%(3)s"], _LOAD),
    # --- two registers: MOV BP,SP / CMP SI,BX / XOR AX,AX / ADD DI,AX -
    # Tried before the bare-base reading below, because for a word move
    # "MovBpSp" means BP := SP and only "MovDlSi" means DL := [SI].  The two
    # readings cannot both match, because one demands a register operand
    # where the other demands a memory operand.
    (re.compile(r"^(Mov|Cmp|Add|Sub|Xor|And|Or|Xchg)(%s)(%s)$" % (_ALT, _ALT)),
     None, ["r:%(2)s", "r:%(3)s"], None),
    # --- byte move from a bare base register: necessarily memory ------
    # Restricted to 8A/88 on purpose: if this ever matched an 8B/89 it would
    # be a register move already claimed by the pattern above.
    (re.compile(r"^Mov(%s)(%s)$" % (_ALT, _BASE)),
     "mov", ["r:%(1)s", "m:%(2)s"], (0x8A, 0x88)),
    # --- immediate against a register --------------------------------
    (re.compile(r"^(Add|Sub|Cmp|Xor|And|Or)(%s)(\d+)$" % _ALT),
     None, ["r:%(2)s", "i:%(3)s"], {0x83}),
    # --- one register: INC CX / DEC SI / NOT DX / NEG AX ------------
    (re.compile(r"^(Inc|Dec|Not|Neg|Shl|Shr|Sar)(%s)$" % _ALT),
     None, ["r:%(2)s"], None),
]


def register_word(tok):
    """Map an FCML operand token to a REGS/SEGS word, or None."""
    t = tok.strip().lower()
    if not t or t.startswith("word ptr ") or t.startswith("byte ptr "):
        return None
    t = t.split()[-1]
    for r in REGS + SEGS:
        if t == r.lower():
            return r
    return None


def match_operand(spec, tok):
    """Does one FCML operand token satisfy one token spec?  Returns None on
    success, or a human-readable reason on failure."""
    kind, arg = spec.split(":", 1)
    t = tok.strip()
    if kind in ("i", "ih"):
        # FCML prints an immediate as hex with an h suffix ("0h", "2h", "21h")
        raw = t.lower().rstrip("h")
        try:
            v = int(raw, 16)
        except ValueError:
            return "%r is not an immediate" % t
        want = int(arg, 16) if kind == "ih" else int(arg, 10)
        if v != want:
            return "immediate is %d, name says %d" % (v, want)
        return None
    if kind == "r":
        got = register_word(t)
        if got != arg:
            return "%r is not the register %s" % (t, arg)
        return None
    if kind == "m":
        base, _, disp = arg.partition("+")
        real = {"Arg": "bp", "Data": "si"}.get(base, base.lower())
        if base == "Arg" and not disp:
            disp = "2"       # the first word argument lives at [BP+2]
        u = t.lower()
        u = re.sub(r"^(word|byte) ptr ", "", u)
        if not (u.startswith("[") and u.endswith("]")):
            return "%r is not a memory reference" % t
        body = u[1:-1]
        if not body.startswith(real):
            return "memory base is %r, name says [%s]" % (body, real)
        if not disp:
            if body != real:
                return ("memory is %r, name says [%s] with no displacement"
                        % (body, real))
            return None
        got = body[len(real):].strip()
        m = re.fullmatch(r"\+\s*([0-9a-f]+)h?", got)
        if not m:
            return "displacement %r is not a number" % got
        if int(m.group(1), 16) != int(disp):
            return "displacement is %d, name says %s" \
                   % (int(m.group(1), 16), disp)
        return None
    return "bad spec %r" % spec


def operands_of(decode):
    """Split an FCML decode like "mov si,bx" or "cmp word ptr [bp+4h],0h"
    into operand tokens.  Returns (mnemonic, [tokens])."""
    i = decode.find(" ")
    if i < 0:
        return decode.strip(), []
    return decode[:i].strip(), decode[i + 1:].split(",")


def main(argv):
    verbose = "-v" in argv
    files = [a for a in argv[1:] if not a.startswith("-")]
    path = files[0] if files else os.path.join(SHELL, "Runtime.mod")
    with open(path) as f:
        src = f.read()

    n_ok = n_unparsed = n_bad = 0
    problems = []
    if verbose:
        print("name                 bytes            FCML decode")

    for m in RE_HELPER.finditer(src):
        name, body = m.group(1), m.group(2)
        by = [int(b, 16) for b in RE_BYTE.findall(body)]
        if not by:
            continue
        raw = bytes(by)
        # Disassemble; helpers are 1-2 bytes so one instruction is the norm,
        # but StCx / Shl / Div style helpers can be longer, so decode all.
        decodes = []
        off = 0
        while off < len(raw):
            text, length = disasm16.decode(raw[off:], off)
            if length == 0:
                decodes.append(("<DECODE ERROR>", ""))
                break
            decodes.append(operands_of(text or "?"))
            off += length

        hexs = " ".join("%02X" % b for b in raw)
        dec_text = "; ".join(d[0] + " " + ", ".join(d[1]) for d in decodes)
        if verbose:
            print("%-20s %-16s %s" % (name, hexs, dec_text))

        # A name can have more than one reading (MovBpSp: BP := SP, or
        # BP := [SP]).  Take the first reading whose operand specs the decode
        # actually satisfies; if two readings both fit, the name is genuinely
        # ambiguous and that is itself reported, because it would mean the
        # audit could be satisfied by the wrong one.
        reads = []
        nopattern = True
        for rx, pmnem, specs, ops in PATTERNS:
            mm = rx.match(name)
            if not mm:
                continue
            if ops is not None and by[0] not in ops:
                continue
            nopattern = False
            gmap = {str(i): g for i, g in enumerate(mm.groups(), 1)}
            mn = pmnem
            if mn is None:
                mn = mm.group(1).lower()
            elif "%(" in mn:
                mn = mn % gmap
            if len(decodes) != 1:
                reads.append((mm, mn, specs,
                              "emits %d instructions, name describes one"
                              % len(decodes)))
                continue
            gmnem, got = decodes[0]
            if gmnem != mn:
                reads.append((mm, mn, specs,
                              "FCML decodes %r, name says %s" % (gmnem, mn)))
                continue
            if len(got) != len(specs):
                reads.append((mm, mn, specs,
                              "name asserts %d operand(s), FCML decoded %d"
                              % (len(specs), len(got))))
                continue
            why = [w for w in
                   (match_operand(sp % gmap if "%(" in sp else sp, tok)
                    for sp, tok in zip(specs, got)) if w]
            if why:
                reads.append((mm, mn, specs, "; ".join(why)))
                continue
            reads.append((mm, mn, specs, None))

        if nopattern:
            n_unparsed += 1
            problems.append("%s: no name pattern accepts it (emits %s %r) - "
                            "extend PATTERNS or fix the name"
                            % (name, hexs, dec_text))
            continue
        fitting = [r for r in reads if r[3] is None]
        if len(fitting) > 1:
            n_bad += 1
            problems.append("%s: %d name readings fit the same bytes - the "
                            "name is ambiguous" % (name, len(fitting)))
            continue
        if not fitting:
            n_bad += 1
            problems.append("%s: %s" % (name, reads[0][3]))
            continue
        n_ok += 1

    total = n_ok + n_unparsed + n_bad
    print("\n%d one-line emitter helpers audited: %d agree with their name, "
          "%d disagree, %d outside the grammar"
          % (total, n_ok, n_bad, n_unparsed))
    if problems:
        print("FAIL: %d problem(s)" % len(problems))
        for p in problems:
            print("  - %s" % p)
        return 1
    print("PASS: every audited helper emits what its name says")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))