TP3/shell/tests/audit_helpers.py

#!/usr/bin/env python3
"""audit_helpers.py -- check that Runtime.mod's one-liner emitter procedures
emit the instruction their name claims.

Why this exists
---------------
Runtime.mod hand-assembles 8086 by writing raw bytes.  Most of it is wrapped
up as one-line helpers:

    PROCEDURE MovSiBx ; BEGIN B (89H) ; B (0DCH) END MovSiBx ;

and those names are the only documentation of what the bytes mean.  That is
exactly the setup for a silent, expensive mistake: write the right opcode and
the wrong ModRM, and nothing complains.  It happened twice here, in the same
direction both times:

    MovSiBx  emitted 89 DC   -- which is MOV SP,BX, not MOV SI,BX
    CmpSiBx  emitted 39 DC   -- ditto, CMP SP,BX

The comment next to CmpSiBx even spelled the ModRM out as "11 011 100" and
still got it wrong, because 11 011 100 in mod=11 means rm=100=SP; SI is
rm=100 only in mod=00, where it means [SI].  Both bugs shipped together into
EmitWrInt, which then stored decimal digits through an SI register it had
never initialised.  The structural checks in check_runtime.py could not see
any of this: the bytes decoded cleanly, the sweep stayed in sync, the branch
targets were all on boundaries and none of the entry prologues moved.  Only a
*name* versus a *decode* comparison finds it, because only that knows what
the author was trying to say.

So: for every helper whose body is a literal byte list, disassemble those
bytes with FCML and require the decode to match the name.  The name grammar
is deliberately narrow and mechanical:

    <Op><Dest><Src>        e.g. MovSiBx, StDiAx, CmpSpBx, MovDlSi
    <Op><Reg>              e.g. XorAxAx, NegAx, NotDx, IncSi, DecSi

with Op in {Mov, Lea, St, Ld, Cmp, Add, Sub, Xor, And, Or, Not, Inc, Dec,
Push, Pop, Int, Jmp, Jne, Jge, Jnl, Jle, Jl, Je, Jae, Jbe, Jb, Ja, JaE...}
and the register/operand words below.  A helper whose name does not parse is
reported as UNPARSED rather than silently skipped -- a name the grammar does
not understand is a name we are not checking, and that must be visible.

This is a source-level check, so it needs no re-baselining: it is a function
of the current source, and it gets stricter as more names are added to the
grammar.  It also runs without building anything.

Usage: audit_helpers.py [MODULE.mod]      # default ../Runtime.mod
       audit_helpers.py -v                 # print every helper, passing or not
"""

import os
import re
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
SHELL = os.path.dirname(HERE)
sys.path.insert(0, HERE)
import disasm16  # noqa: E402  (path set above)

# An emitter may be declared with or without a parameter list, so the
# parentheses are optional.  This is not cosmetic: the version that required
# `PROCEDURE Name ;` matched ZERO of Compiler.mod's 22 emitters, because they
# are all declared `PROCEDURE EmName () ;`.  So the audit reported "everything
# agrees" for a module where it had examined nothing - a check that cannot fail
# is not a check.
RE_PROC_HEAD = re.compile(
    r"^PROCEDURE\s+(\w+)\s*(?:\(\s*\))?\s*;", re.MULTILINE)


RE_ANY_PROC = re.compile(r"^PROCEDURE\s+(\w+)", re.MULTILINE)

# --- what the one-line grammar deliberately does NOT reach ----------------
#
# The name audit checks procedures whose body is a list of CONSTANT bytes, so
# that the bytes can be disassembled and compared against the name.  Three
# kinds of emitter cannot be checked that way, and all three are listed here
# with their reason.  A name in this table is a documented exclusion; a name
# that is merely absent is a finding, and the coverage check below is what
# tells the two apart.
#
# 1. PARAMETERISED operands -- a byte that is a parameter, not a literal.
#    `SubAl (v)` emits 2C v, which disassembles to "sub al, 0" whatever v is,
#    so the name can be checked only by also trusting the source order.  The
#    grammar could learn this (the pattern Int([0-9A-Fa-f]+) already handles
#    the analogous case for INT); it does not yet, and until it does these are
#    unchecked BY THE AUDIT, not verified.
# 2. COMPUTED or DISPATCHED bytes -- a ModRM chosen by a comparison, a
#    rel16 with a patch slot, a form that depends on which kind of variable it
#    is.  There is no single byte sequence to disassemble.
# 3. DATA, not code -- the B(...) calls here emit the D_ block, so running
#    them through a disassembler yields "add [bx+si],al" and a decode error.
#    tests/check_runtime.py pins these bytes against runtime.golden, so they
#    are pinned; only the name-to-opcode correspondence does not apply.
#
# Named per module because the same name can be a one-liner in one and not the
# other.  The Vx family in Runtime.mod is the reason a name is worth listing
# separately: those four ARE audited, by audit_vx_helpers, because their shape
# is fixed even though their immediate is a data offset.
PARAM = "parameterised operand - the byte is an argument, not a literal"
COMP = "computed or dispatched bytes - no single sequence to disassemble"
DATA = "emits DATA (the D_ block), not instructions"

NON_ONE_LINE = {
    "Runtime.mod": {
        "MovBxImm": PARAM, "MovCxImm": PARAM, "MovDxImm": PARAM,
        "MovDl": PARAM, "MovAlD": PARAM, "MovAh": PARAM,
        "SubAl": PARAM, "CmpAl": PARAM, "AddDl": PARAM, "J8": COMP,
        "Jcc": COMP, "CmpCxV": PARAM,
        "B": DATA, "C8": DATA,
    },
    "Compiler.mod": {
        "AddSp": PARAM, "SubSp": PARAM, "Setcc": PARAM,
        "Call": COMP, "Jcc": COMP, "JmpNear": COMP,
        "BpDisp": COMP, "LoadVar": COMP, "StoreVar": COMP,
        "PushVarAddr": COMP, "MovAxi": COMP, "CmpAxi": COMP,
        "MovCxi": COMP, "LeaDi": COMP,
        # PushStrAddr is PushVarAddr's sibling one level up: it pushes the
        # ADDRESS of a whole-string operand, and a string LITERAL has no storage
        # to take the address of, so it emits the literal INLINE - a jump over
        # it, then <len><chars> - and pushes the address of those bytes.  The
        # emitted sequence therefore depends on the operand: a variable is a
        # single EmPushVarAddr, a literal is two bytes of jump plus the string
        # itself.  That is COMP, not PARAM (whose immediate varies but whose
        # opcode does not).  Listed here rather than left out: an emitter the
        # name grammar cannot describe has to be written down, or the report is
        # green and the helper unchecked.
        "PushStrAddr": COMP,
        # SetBlob is PushStrAddr's sibling for sets: a set cannot fold to a
        # word, so its bytes travel inline - a JMP over them, then the bytes -
        # and the address of those bytes is the value.  Like PushStrAddr the
        # emitted sequence depends on the operand (its length), so there is no
        # single sequence to disassemble; the LAYOUT it produces is pinned by
        # t61_setconst's .out instead.
        "SetBlob": COMP,
        # StrBlob is the string-literal blob on its own (factored out of
        # PushStrAddr so a string value parameter can copy from it); PushStrSlot
        # reserves a slot and copies a string value into it.  Both emit data
        # behind a jump / a data-dependent byte loop, so there is no single
        # sequence to disassemble.  The layouts they produce are pinned by the
        # string and string-parameter fixtures instead.
        "StrBlob": COMP,
        "PushStrSlot": COMP,
        # The CRT sequences (INT 10h) are one parameterised helper: several
        # instructions each, with no single name grammar.  Listed, not left
        # implicit.
        "CrtSeq": COMP,
        # A VAR parameter's slot holds an ADDRESS.  These three read and write
        # one level down: LoadVarInd fetches the pointer into BX and then the
        # byte/word at [BX]; StoreVarIndPrep parks the pointer in DI across the
        # right-hand side and StoreInd stores through it.  The sequence depends
        # on the width (one byte or a word), so there is no single opcode to
        # name; t73_varparam's execution pins both widths, the local/global
        # address forms, passing one on, and READ through one.
        "LoadVarInd": COMP,
        "StoreVarIndPrep": COMP,
        "StoreInd": COMP,
        # The WITH/2-D field helpers.  AddDiImm adds a record field's fixed
        # offset to an address already in DI; LdDiGlob and StDiGlob reload and
        # park that address in a data slot across a WITH body.  Each is a fixed
        # 3-4 byte sequence whose operand is a data/field offset, and t77_with
        # pins the behaviour (2-D element, constant element, field offsets).
        "AddDiImm": COMP,
        "LdDiGlob": COMP,
        "StDiGlob": COMP,
        # Set-field helpers: store one constructor byte through DI
        # (StDiImm), and test one element's bit in a DI-addressed set
        # (LdAlDiOff + AndAlImm).  t78_setfield pins both.
        "StDiImm": COMP,
        "LdAlDiOff": COMP,
        "AndAlImm": COMP,
        # A real copy through DI: one byte of the six.
        "StDiAlOff": COMP,
    },
}
# The Vx family is NOT here: those four are audited by audit_vx_helpers,
# because their shape is fixed even though their immediate is a data offset.
# They are named explicitly rather than filtered out of VX_KINDS so that this
# table stands on its own and a name added to VX_KINDS later cannot silently
# become unchecked - it would instead be reported, which is the point.
_NONLINE_VX = ("LdBxVx", "MovBxVx", "MovDxVx", "StVxBx")



def scan_emitters(src):
    """Yield (name, text) for every PROCEDURE, however it is written.

    The INVENTORY side of the coverage check, and deliberately as dumb as it
    can be: a name at the start of a line, and everything up to the next one.
    It does not check the parameter list, does not look for BEGIN, does not
    care where comments are, and cannot be defeated by the same mistake twice.

    The reason it exists at all is that the other list - the one the audit
    actually reads - is produced by find_helpers, which is a real parser and
    therefore fails on real inputs (a parameter list it did not expect, a
    comment in the wrong place).  Comparing a parser against itself finds
    nothing; comparing it against a scan that cannot parse anything finds
    exactly the cases where the parser is the thing that is wrong.
    """
    heads = list(RE_ANY_PROC.finditer(src))
    for k, h in enumerate(heads):
        stop = heads[k + 1].start() if k + 1 < len(heads) else len(src)
        yield h.group(1), src[h.end():stop]


def find_helpers(src):
    """Yield (name, body) for every PROCEDURE, in source order.

    A documented emitter must be auditable, so text between the header and
    BEGIN - which is where the explanatory comment belongs, and the only place
    it can be read next to the code it describes - is allowed through.  A
    regex that permitted it had nested quantifiers and took exponential time
    on these files, so this splits on procedure HEADERS instead and takes the
    text up to the next header.  That is linear, and it also cannot read one
    procedure's bytes as another's.

    The header regex is still strict about the parameter list (empty only),
    because that is what distinguishes an emitter from a real routine.
    """
    heads = list(RE_PROC_HEAD.finditer(src))
    for k, h in enumerate(heads):
        stop = heads[k + 1].start() if k + 1 < len(heads) else len(src)
        chunk = src[h.end():stop]
        m = re.search(r"\bBEGIN\b(.*)\bEND\s+%s\s*;" % re.escape(h.group(1)),
                      chunk, re.DOTALL)
        if not m:
            continue
        yield h.group(1), m.group(1)
# The H suffix is optional: the source mixes B (8AH) and B (0) for the same
# kind of literal, and a byte written without H used to be silently dropped
# from the audit, which made three helpers look like truncated prefixes.
#
# BOTH B (...) and Ebyte (...) are accepted.  Runtime.mod spells a byte B(v)
# and Compiler.mod spells the same thing Ebyte(v); the Em prefix is on the
# *procedure* names there, not on the byte call, so the byte regex has to cover
# both spellings or Compiler.mod's 22 emitters are silently skipped - which is
# what happened, and how EmXchgAxCx stayed wrong for its whole life with
# correct byte counts and a green compile matrix.
RE_BYTE = re.compile(r"(?:\bB|\bEbyte)\s*\(\s*([0-9A-Fa-f]+)H?\s*\)")

# Both modules emit bytes, so both must be swept.  Compiler.mod is where the
# procedure-skip jump, the FOR test ordering and EmXchgAxCx (93h = XCHG BX,AX
# where the name says XCHG AX,CX) all live: bugs that break every two-variable
# arithmetic and comparison in every program, invisible because the audit
# looked at Runtime.mod and found nothing wrong there.
MODULES = ["Runtime.mod", "Compiler.mod"]

# --- name grammar -----------------------------------------------------
#
# An ordered table of (name regex, mnemonic, operand specs, allowed opcodes).
#
# Why the opcode column exists
# ----------------------------
# "MovDlSi" and "MovBpSp" are ambiguous from the name alone: SI and BP are
# both in the register list and in the memory-base list.  What settles it is
# the opcode, and the rule is the asymmetry that makes 16-bit hand-assembly
# so error-prone:
#
#     88 /r  MOV r/m8, r8    reg is the SOURCE     (store)
#     89 /r  MOV r/m16, r16  reg is the SOURCE     (store)
#     8A /r  MOV r8, r/m8    reg is the DESTINATION (load)
#     8B /r  MOV r16, r/m16  reg is the DESTINATION (load)
#
# A byte move (8A/88) to or from a bare base register can only be a memory
# access, because "mov dl, si" is not an instruction.  A word move (8B/89)
# could be either, so for word moves the register reading is tried first.
# Encoding that rule in the table, rather than in pattern order alone, is what
# stops a helper being "verified" against the wrong reading of its own name.
#
# Operand spec tokens:
#
#     "r:Name"     a bare register spelled Name (Ax, Al, Dx, Si, Ds, ...)
#     "i:N"        the immediate N; the name spells it in DECIMAL
#     "ih:N"       like i: but the name spells it in HEX (only Int, whose
#                  vector 21h FCML prints as "21h" and which reads as decimal
#                  33 if you do not notice)
#     "m:Base"     memory at [Base], with no displacement
#     "m:Base+D"   memory at [Base + D]
#
# "Arg" is the emitter's alias for BP: on entry BP points at the return
# address, so a word argument starts at [BP+2].  Naming it "Arg" rather than
# "Bp" is what stops these being misread as [BP] accesses, and it is also
# how the grammar knows to expect the +2.
#
# "Arg" is the frame in which the entry did NOT push BP, and it is the only
# one where the argument is two bytes up: an entry that pushes BP first has
# moved it to +4, which is a different name and a different byte
# (MovAlArg2 / MovAlArg4, CmpArg2W0 / CmpArg4W0).  A name that says only
# "Arg" always means +2, with no exceptions -- the alternatives to spelling
# the displacement out were a Modula-2 parameter, which this audit cannot
# see, and a comment, which nothing checks.
#
# A name matching NO pattern is reported UNPARSED, which counts as a failure
# on purpose: a helper the audit cannot read is a helper nobody is checking.

REGS = ["Ax", "Bx", "Cx", "Dx", "Si", "Di", "Bp", "Sp",
        "Al", "Bl", "Cl", "Dl", "Ah", "Bh", "Ch", "Dh"]
# Segment registers: the 8086 can only PUSH/POP them, never MOV to or from
# one, so they need names of their own.
SEGS = ["Ds", "Es", "Cs", "Ss"]
JCCS = ["Je", "Jne", "Jz", "Jnz", "Jge", "Jnl", "Jle", "Jl", "Ja", "Jae",
        "Jb", "Jbe", "Jg", "Jns", "Js", "Jo", "Jno", "Jp", "Jnp", "Jcxz",
        "Jecxz", "Jrcxz", "Loop", "Loope", "Loopne"]

_ALT = "|".join(REGS)
_BASE = "Si|Di|Bx|Bp|Sp|Arg|Data"
_STORE = (0x88, 0x89)      # reg is the source
_LOAD = (0x8A, 0x8B)       # reg is the destination

PATTERNS = [
    # --- no-operand and fixed-operand forms -------------------------
    (re.compile(r"^RetR?$"),            "ret",    [], None),
    (re.compile(r"^LeaveR?$"),          "leave",  [], None),
    (re.compile(r"^Int([0-9A-Fa-f]+)$"), "int",   ["ih:%(1)s"], {0xCD}),
    (re.compile(r"^(Push|Pop)(%s)$" % "|".join(SEGS)),
     None, ["r:%(2)s"], None),
    (re.compile(r"^(Push|Pop)(%s)$" % _ALT), None, ["r:%(2)s"], None),
    # --- jumps: the target is a fixup, so only the mnemonic is checked
    (re.compile(r"^(%s)(8|16)?$" % "|".join(JCCS)), None, [], None),
    (re.compile(r"^Jmp(%s)$" % _ALT),   "jmp",    ["r:%(1)s"], {0xFF}),
    # --- MUL names one operand; 16-bit DIV names two because it always
    #     divides DX:AX, so the decode has an AX the name has no room for
    (re.compile(r"^Mul(%s)$" % _ALT),   "mul",    ["r:%(1)s"], {0xF7}),
    (re.compile(r"^Div(%s)$" % _ALT),   "div",    ["r:Ax", "r:%(1)s"], {0xF7}),
    # --- store: name is St<base><reg>, decode puts the register last --
    (re.compile(r"^St(%s)(%s)$" % (_BASE, _ALT)),
     "mov", ["m:%(1)s", "r:%(2)s"], _STORE),
    # --- load through a bare base register: Ld<reg><base> ------------
    # The mirror of the St pattern above, and spelled Ld rather than Mov on
    # purpose.  `8A 07` and `8A C3` are one byte apart and do opposite
    # things: LdAlBx reads the byte AT the pointer in BX, MovAlBl takes the
    # low byte OF BX.  Naming the first "MovAlBx" put the two one letter
    # apart, and getch's pushback path used the wrong one - it read memory
    # 010Ah, the address of the character, instead of the character.  So the
    # grammar is part of the safety: Ld* may only be spelled for a memory
    # source, which is what distinguishes it from Mov*<reg><lowreg>.
    (re.compile(r"^Ld(%s)(%s)$" % (_ALT, _BASE)),
     "mov", ["r:%(1)s", "m:%(2)s"], (0x8A, 0x8B)),
    # --- XCHG is symmetric, so the name's operand order carries no
    #     information and must not be checked positionally.  87 /r is
    #     XCHG r/m16, r16, so FCML always prints the r/m operand first:
    #     87 C7 - reg=AX, rm=DI - decodes as "xchg di,ax" whichever way the
    #     author thought about it.  What the audit is really for here is the
    #     PAIR: XchgAxDi must not come out as XCHG AX,CX.  So the two
    #     registers are compared as a set (see the "rx:" handling below).
    (re.compile(r"^Xchg(%s)(%s)$" % (_ALT, _ALT)),
     "xchg", ["rx:%(1)s|%(2)s"], {0x87, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96,
                                  0x97}),
    # --- the accumulator-implicit XCHGs, 91h..97h, join the /r form -----
    # One byte, no ModRM, both registers fixed by the opcode.  They are in
    # the same opcode set because the same `rx:` set comparison decides them,
    # and that comparison is exactly what catches 93h (XCHG BX,AX) under the
    # name XchgAxCx - the fault that broke every two-variable arithmetic
    # operation in every program.  90h is deliberately absent: it is NOP, and
    # FCML decodes it as "nop", not as an XCHG AX,AX.
    #
    # --- MOV r8, imm8 : B0+reg, and the AH-specific B4 form -----------
    # MOV AH,imm8 is B4 imm8, which is not B0+reg.  The name says which
    # register, so the opcode is checked against it: B4 must be AH and B0+4
    # must not be.  (The runtime's wrchar used to load the character with the
    # wrong register, so the store landed in AL-adjacent memory.)
    # B4 imm8 is MOV AH,imm8 and is NOT B0+reg, so AH and AL are listed
    # separately with their own opcodes rather than sharing one pattern that
    # would accept B4 under the name MovAl0.
    (re.compile(r"^MovAh(0|1)$"),
     "mov", ["r:Ah", "i:%(1)s"], {0xB4}),
    (re.compile(r"^MovAl(0|1)$"),
     "mov", ["r:Al", "i:%(1)s"], {0xB0}),
    # --- MUL/IMUL r/m16, accumulator implicit, /5 and /4 -----------------
    # The destination is AX:DX and the operand is the named register, so
    # "MulAxCx" means CX := CX, i.e. AX:DX := AX * CX.  FCML prints only the
    # one explicit operand ("imul cx"), which is why the spec has one token.
    (re.compile(r"^Mul(%s)(%s)$" % (_ALT, _ALT)),
     "imul", ["r:%(2)s"], {0xF7}),
    # --- CmpArg<N>W0 : CMP WORD [BP+N],0 between MOV BP,SP / MOV SP,BP -----
    # An odd but exact shape: [SP] is not encodable, so the frame pointer is
    # borrowed for the one comparison and handed back untouched.  The pair of
    # saves at the ends is what makes it correct, so the whole sequence is
    # checked - a missing MOV SP,BP would leave BP clobbered for the caller.
    #
    # The displacement is a CAPTURED GROUP, not a constant, and the name has to
    # spell it out.  That is the rule that keeps CmpArg2W0 and CmpArg4W0
    # honest: they differ by one byte, and one byte is the difference between
    # reading a BOOLEAN and reading a character, so the name -- which is the
    # only thing this audit can read -- has to carry it.  A single helper
    # taking the displacement as a Modula-2 parameter would decode correctly
    # under either name and be unchecked under both.
    (re.compile(r"^CmpArg(2|4)W0$"),
     None, [["mov", ["r:Bp", "r:Sp"]],
            ["cmp", ["m:Bp+%(1)s", "i:0"]],
            ["mov", ["r:Sp", "r:Bp"]]], {0x8B}),
    # --- a "Mov<reg>Sp" that is a POP/PUSH pair, not a memory access ----
    # MOV AX,[SP] does not exist on the 8086 at all, so the stack top is read
    # with POP and given back with PUSH: an observational no-op that leaves SP
    # where it found it.  Named MovAxSp because that is the OPERATION, and the
    # two-step shape is recorded here because it is the only correct encoding.
    (re.compile(r"^Mov(%s)Sp$" % _ALT),
     None, [["pop", ["r:%(1)s"]], ["push", ["r:%(1)s"]]], None),
    # --- IDIV: CWD then IDIV r/m16, sign-extending into DX:AX ---------
    # A two-instruction shape, so the specs are a list of one list per
    # instruction.  The CWD is not incidental: F7 /7 divides the 32-bit value
    # in DX:AX, and a signed dividend is only in DX:AX if CWD ran.  Drop the
    # 99h and the divisor goes to the same place, so the pair is checked
    # together - which is the reason the sequence form exists.
    # FCML prints BOTH ends of a /r divide ("idiv ax, cx"), because the
    # accumulator is not implicit in the mnemonic the way it is for MUL, so
    # the spec needs two operands and the AX comes first.
    (re.compile(r"^I?Div(%s)(%s)$" % (_ALT, _ALT)),
     None, [["cwd", []], ["idiv", ["r:%(1)s", "r:%(2)s"]]], {0xF7}),
    # --- load a register from memory, with a displacement ------------
    # The displacement makes the name unambiguous, so any load opcode works.
    (re.compile(r"^Mov(%s)(%s)(\d+)$" % (_ALT, _BASE)),
     "mov", ["r:%(1)s", "m:%(2)s+%(3)s"], _LOAD),
    # --- two registers: MOV BP,SP / CMP SI,BX / XOR AX,AX / ADD DI,AX -
    # Tried before the bare-base reading below, because for a word move
    # "MovBpSp" means BP := SP and only "MovDlSi" means DL := [SI].  The two
    # readings cannot both match, because one demands a register operand
    # where the other demands a memory operand.
    # Xchg is NOT in this list.  It has its own pattern above, which compares
    # the two registers as a SET because 87 /r is symmetric and FCML always
    # prints the r/m operand first - so the name's order carries no
    # information and must not be checked positionally.  Listing Xchg here as
    # well gave it a second, ordered reading, and the audit then reported
    # "2 name readings fit the same bytes" for XchgAxCx: not a real ambiguity
    # in the code, but two overlapping rows in the grammar saying the same
    # thing.  A grammar that can be satisfied two ways for one name is a
    # grammar that can be satisfied the wrong way.
    (re.compile(r"^(Mov|Cmp|Add|Sub|Xor|And|Or)(%s)(%s)$" % (_ALT, _ALT)),
     None, ["r:%(2)s", "r:%(3)s"], None),
    # --- byte move from a bare base register: necessarily memory ------
    # Restricted to 8A/88 on purpose: if this ever matched an 8B/89 it would
    # be a register move already claimed by the pattern above.
    (re.compile(r"^Mov(%s)(%s)$" % (_ALT, _BASE)),
     "mov", ["r:%(1)s", "m:%(2)s"], (0x8A, 0x88)),
    # --- immediate against a register --------------------------------
    (re.compile(r"^(Add|Sub|Cmp|Xor|And|Or)(%s)(\d+)$" % _ALT),
     None, ["r:%(2)s", "i:%(3)s"], {0x83}),
    # --- XOR AL,#imm8 is `34 ib': no ModRM byte at all, and only the
    #     accumulator.  The row above is pinned to 83 /r, which is the word
    #     `op r, imm8' encoding, so `XorAl01' matched it and was rejected on
    #     the opcode - leaving the helper with NO reading rather than a wrong
    #     one, which is how the audit reported it.  Narrow on purpose: 34 is
    #     XOR AL, nothing else, so a name claiming `xor al, 1h' against any
    #     other byte fails here.  This is TPSRC9 neglevel's boolean NOT.
    (re.compile(r"^XorAl(\d+)$"),
     "xor", ["r:Al", "i:%(1)s"], {0x34}),
    # --- one register: INC CX / DEC SI / NOT DX / NEG AX ------------
    (re.compile(r"^(Inc|Dec|Not|Neg|Shl|Shr|Sar)(%s)$" % _ALT),
     None, ["r:%(2)s"], None),
]


def register_word(tok):
    """Map an FCML operand token to a REGS/SEGS word, or None."""
    t = tok.strip().lower()
    if not t or t.startswith("word ptr ") or t.startswith("byte ptr "):
        return None
    t = t.split()[-1]
    for r in REGS + SEGS:
        if t == r.lower():
            return r
    return None


def match_operand(spec, tok):
    """Does one FCML operand token satisfy one token spec?  Returns None on
    success, or a human-readable reason on failure."""
    kind, arg = spec.split(":", 1)
    t = tok.strip()
    if kind in ("i", "ih"):
        # FCML prints an immediate as hex with an h suffix ("0h", "2h", "21h")
        raw = t.lower().rstrip("h")
        try:
            v = int(raw, 16)
        except ValueError:
            return "%r is not an immediate" % t
        want = int(arg, 16) if kind == "ih" else int(arg, 10)
        if v != want:
            return "immediate is %d, name says %d" % (v, want)
        return None
    if kind == "r":
        got = register_word(t)
        if got != arg:
            return "%r is not the register %s" % (t, arg)
        return None
    if kind == "m":
        base, _, disp = arg.partition("+")
        real = {"Arg": "bp", "Data": "si"}.get(base, base.lower())
        if base == "Arg" and not disp:
            disp = "2"       # the first word argument lives at [BP+2]
        u = t.lower()
        u = re.sub(r"^(word|byte) ptr ", "", u)
        if not (u.startswith("[") and u.endswith("]")):
            return "%r is not a memory reference" % t
        body = u[1:-1]
        if not body.startswith(real):
            return "memory base is %r, name says [%s]" % (body, real)
        if not disp:
            if body != real:
                return ("memory is %r, name says [%s] with no displacement"
                        % (body, real))
            return None
        got = body[len(real):].strip()
        m = re.fullmatch(r"\+\s*([0-9a-f]+)h?", got)
        if not m:
            return "displacement %r is not a number" % got
        if int(m.group(1), 16) != int(disp):
            return "displacement is %d, name says %s" \
                   % (int(m.group(1), 16), disp)
        return None
    return "bad spec %r" % spec


def operands_of(decode):
    """Split an FCML decode like "mov si,bx" or "cmp word ptr [bp+4h],0h"
    into operand tokens.  Returns (mnemonic, [tokens])."""
    i = decode.find(" ")
    if i < 0:
        return decode.strip(), []
    return decode[:i].strip(), decode[i + 1:].split(",")


# --- the Vx helpers, which the one-line grammar cannot see ---------------
#
# find_helpers takes a body spanning any number of lines, so a helper written
# as
#
#     PROCEDURE LdBxVx (delta : CARDINAL ) ;
#     BEGIN
#        B (8BH) ; B (01EH) ; Dd (delta)
#     END LdBxVx ;
#
# does not reach the one-line pass.  That is a gap rather than a documented
# exclusion, because these are exactly the helpers that are hard
# to get right and easy to get subtly wrong:
#
#   - the operand is a D_ data-block offset, so the immediate is a FIXUP.
#     Emitting W (delta) instead of Dd (delta) produces a byte-identical
#     instruction with the raw offset in it: `MOV BX,001Ch` for
#     `MOV BX,<D_PUSH>`, pointing into the entry JMP.  That happened, and
#     the result was a hang rather than a crash.
#   - in 16-bit mode there is no [BX] memory form, so a data address has to
#     go through mod=00 / rm=110, which is ModRM 1E.  rm=111 is [BX+SI], and
#     it decodes cleanly, so a wrong ModRM here is invisible to every other
#     check in the tree.
#   - LdBxVx and StVxBx are a load/store pair on the same word and MUST use
#     the same ModRM and the same fixup kind, or one of them silently
#     addresses something else.
#   - LdBxVx and MovBxVx differ only in the ModRM's mod field, i.e. "the word
#     at the address" versus "the address".  Both mistakes above have been
#     made here, in this pair.
#
# So they get their own check, keyed on "Vx" - the operand word Runtime.mod
# reserves for "a 16-bit absolute address into the data block" - and every one
# of them is declared explicitly, so ADDING a Vx helper is a deliberate act:
# an undeclared one is reported rather than ignored.
#
# The header, the body and the closing name are separated by `(.*?)` rather
# than by `\s*`, because two of these helpers carry an explanatory comment
# between the parameter list and BEGIN.  A `\s*` there skips them silently -
# which is the failure this whole check exists to prevent - so the pattern is
# deliberately loose about whitespace and strict about the `END <name>`.
RE_VX = re.compile(
    r"^PROCEDURE\s+(\w*Vx\w*)\s*\([^)]*\)\s*;.*?\bBEGIN\b(.*?)\bEND\s+\1\s*;",
    re.MULTILINE | re.DOTALL)

# (opcode, shape).  "address" = MOV reg,imm16: the register is in the opcode
# and there is no ModRM.  "load"/"store" = opcode + ModRM 1E + the
# Dd-supplied immediate.
VX_KINDS = {
    "MovBxVx": (0xBB, "address"),   # MOV BX, Vx
    "MovDxVx": (0xBA, "address"),   # MOV DX, Vx
    "LdBxVx":  (0x8B, "load"),      # MOV BX, WORD PTR [Vx]
    "StVxBx":  (0x89, "store"),     # MOV WORD PTR [Vx], BX
}


def audit_vx_helpers(src, mod, verbose):
    """Check the multi-line *Vx helpers.  Returns (n_checked, problems)."""
    found = {}
    for m in RE_VX.finditer(src):
        found[m.group(1)] = m.group(2)
    problems = []
    # The count reported is the number of helpers ACTUALLY examined, not the
    # size of VX_KINDS.  Those were the same number when only Runtime.mod was
    # swept, and printing the constant for Compiler.mod - which defines no Vx
    # emitter at all - made the report claim four subjects in a module that
    # has none.  A count that does not change when the module does is a
    # cosmetic lie, and this report exists to be believed.
    checked = 0
    for name in sorted(set(found) | set(VX_KINDS)):
        # A *Vx helper is a Runtime.mod concept (the V* data word at a fixed
        # offset).  Compiler.mod has none, so VX_KINDS is only a floor for the
        # module that is supposed to define them; asking Compiler.mod for four
        # Vx emitters it should not have produces four phantom failures.
        if mod != "Runtime.mod" and name in VX_KINDS and name not in found:
            continue
        if name not in VX_KINDS:
            problems.append("%s: a Vx helper with no declared kind - add it "
                            "to VX_KINDS with its opcode and shape" % name)
            continue
        if name not in found:
            # The module is named rather than hardcoded: these four live in
            # Runtime.mod, and reporting them as missing from Compiler.mod
            # would be a false alarm about a module that has no business
            # defining them.
            problems.append("%s: declared in VX_KINDS but not defined in %s"
                            % (name, mod))
            continue
        checked += 1
        body = found[name]
        want_op, shape = VX_KINDS[name]
        by = [int(b, 16) for b in RE_BYTE.findall(body)]
        hexs = " ".join("%02X" % b for b in by[:2])
        if not by or by[0] != want_op:
            problems.append("%s: starts %s, expected opcode %02X (%s)"
                            % (name, hexs or "<nothing>", want_op, shape))
            continue
        # The two shapes need different numbers of literal bytes, and asking
        # for a ModRM on an address move is a false alarm: MOV reg,imm16
        # carries the register in the opcode and has no ModRM at all.
        if shape == "address":
            if len(by) != 1:
                problems.append("%s: emits %s, expected just the opcode - an "
                                "address move is `MOV reg,imm16` and has no "
                                "ModRM" % (name, hexs))
                continue
        elif len(by) < 2:
            problems.append("%s: emits %s, expected opcode and ModRM"
                            % (name, hexs))
            continue
        elif by[1] != 0x1E:
            problems.append(
                "%s: ModRM is %02X, expected 1E.  16-bit mode has no [BX] "
                "form, so a data address must use mod=00 / rm=110; rm=111 "
                "is [BX+SI] and decodes cleanly, so nothing else would "
                "notice" % (name, by[1]))
            continue
        if "Dd" not in body:
            problems.append(
                "%s: does not call Dd, so the D_ offset is emitted raw "
                "instead of as a fixup - the instruction will be "
                "byte-correct and point at the wrong place" % name)
            continue
        if re.search(r"\bW\s*\(", body):
            problems.append("%s: calls W as well as Dd - the offset must be "
                            "placed by Dd alone" % name)
            continue
        if verbose:
            print("%-20s %-16s %s (%s, Dd fixup)"
                  % (name, hexs, shape,
                     "no ModRM" if shape == "address" else "mod=00 rm=110"))
    return checked, problems


def sweep(src, mod, verbose):
    """Audit every one-line emitter helper in one module.  Returns
    (n_ok, n_unparsed, n_bad, problems)."""
    n_ok = n_unparsed = n_bad = 0
    problems = []
    seen = set()
    # EmitData is a documented EXCLUSION, not a gap: its B(...) calls are DATA
    # (the D_ block), not instructions, so running the code buffer through a
    # disassembler produces nonsense like "add [bx+si],al" and a DECODE ERROR.
    # It is listed here rather than left implicit, because an exclusion that is
    # not written down is indistinguishable from a helper nobody audited.
    #
    # Its D_ offsets are asserted against this block by check_runtime.py's
    # golden, so "excluded from the name audit" is not "unchecked": the bytes
    # are pinned, only the name-to-opcode correspondence does not apply.
    excluded = {"EmitData"}
    if verbose:
        print("-- %s" % mod)
        print("name                 bytes            FCML decode")

    for name, body in find_helpers(src):
        # Compiler.mod prefixes its emitters with Em.  The name grammar is
        # shared with Runtime.mod and has no Em entries, so the prefix is
        # stripped here and the module name only ever appears in output.  It
        # is stripped for PATTERNS matching and for the Vx pass alike, which
        # is why a single RE_BYTE and a single PATTERNS table can serve both
        # modules instead of two drifting copies.
        if mod != "Runtime.mod" and name.startswith("Em"):
            name = name[2:]
        seen.add(name)
        if name in excluded:
            continue
        by = [int(b, 16) for b in RE_BYTE.findall(body)]
        if not by:
            continue
        raw = bytes(by)
        # Disassemble; helpers are 1-2 bytes so one instruction is the norm,
        # but StCx / Shl / Div style helpers can be longer, so decode all.
        decodes = []
        off = 0
        while off < len(raw):
            text, length = disasm16.decode(raw[off:], off)
            if length == 0:
                decodes.append(("<DECODE ERROR>", ""))
                break
            decodes.append(operands_of(text or "?"))
            off += length

        hexs = " ".join("%02X" % b for b in raw)
        dec_text = "; ".join(d[0] + " " + ", ".join(d[1]) for d in decodes)
        if verbose:
            print("%-20s %-16s %s" % (name, hexs, dec_text))

        # A name can have more than one reading (MovBpSp: BP := SP, or
        # BP := [SP]).  Take the first reading whose operand specs the decode
        # actually satisfies; if two readings both fit, the name is genuinely
        # ambiguous and that is itself reported, because it would mean the
        # audit could be satisfied by the wrong one.
        reads = []
        nopattern = True
        for rx, pmnem, specs, ops in PATTERNS:
            mm = rx.match(name)
            if not mm:
                continue
            # The opcode gate applies to the FIRST byte.  That is right for a
            # one-instruction helper, and wrong for a sequence, where the first
            # byte belongs to the setup step: IDivAxCx starts with 99h (CWD)
            # and its F7h /7 is the second instruction, so a first-byte gate
            # rejected the one helper whose opcode matters most.  Skipping the
            # gate for a sequence loses nothing, because the sequence check
            # below compares every decoded mnemonic against its own spec.
            is_seq = (isinstance(specs, list) and specs
                      and isinstance(specs[0], list))
            if ops is not None and not is_seq and by[0] not in ops:
                continue
            nopattern = False
            gmap = {str(i): g for i, g in enumerate(mm.groups(), 1)}
            mn = pmnem
            if mn is None:
                # A pattern with no capture group has no mnemonic to infer
                # from the name, and a pattern with none either names the
                # mnemonic outright (a sequence) or is checked by operands
                # alone.  A group-less pattern with pmnem None is a grammar
                # bug, not a helper bug, so say so instead of crashing.
                if not mm.re.groups:
                    reads.append((mm, None, specs,
                                  "pattern %r has no capture group and names "
                                  "no mnemonic" % rx.pattern))
                    continue
                mn = mm.group(1).lower()
            elif "%(" in mn:
                mn = mn % gmap
            if len(decodes) != 1:
                # A pattern may describe a fixed SEQUENCE of instructions by
                # passing a list of specs, one per instruction - IDIV is the
                # only user (CWD, then IDIV r/m16), and without this it was
                # reported as "emits 2 instructions, name describes one",
                # which is a true observation and a useless one: the two
                # instructions are one operation and the pair is the thing
                # that has to be checked, because a lone F7 /7 divides by an
                # un-sign-extended dividend.
                if not is_seq or len(specs) != len(decodes):
                    reads.append((mm, mn, specs,
                                  "emits %d instructions, name describes %s"
                                  % (len(decodes),
                                     len(specs) if is_seq else "one")))
                    continue
                why = []
                for k, sub in enumerate(specs):
                    smn, sspecs = sub
                    gmnem, got = decodes[k]
                    if gmnem != smn:
                        why.append("step %d is %r, name says %s"
                                   % (k + 1, gmnem, smn))
                        break
                    if len(got) != len(sspecs):
                        why.append("step %d: name asserts %d operand(s), "
                                   "FCML decoded %d"
                                   % (k + 1, len(sspecs), len(got)))
                        break
                    w = [x for x in
                         (match_operand(s % gmap if "%(" in s else s, t)
                          for s, t in zip(sspecs, got)) if x]
                    if w:
                        why.append("step %d: %s" % (k + 1, "; ".join(w)))
                        break
                reads.append((mm, mn, specs, "; ".join(why) if why else None))
                continue
            gmnem, got = decodes[0]
            if gmnem != mn:
                reads.append((mm, mn, specs,
                              "FCML decodes %r, name says %s" % (gmnem, mn)))
                continue
            if len(specs) == 1 and specs[0].startswith("rx:"):
                # XCHG: compare the two registers as a set, not in order.
                # This has to come before the operand-COUNT check, because
                # the "rx:" spec packs both registers into one token while
                # the decode has two operands - which is the whole point of
                # it.
                want = set((specs[0] % gmap)[3:].split("|"))
                got_regs = [register_word(t) for t in got]
                if len(got_regs) != 2 or None in got_regs:
                    reads.append((mm, mn, specs,
                                  "XCHG needs two bare registers, decoded %s"
                                  % ", ".join(got)))
                    continue
                if set(got_regs) != want:
                    reads.append((mm, mn, specs,
                                  "exchanges %s, name says %s"
                                  % (" and ".join(sorted(got_regs)),
                                     " and ".join(sorted(want)))))
                    continue
                reads.append((mm, mn, specs, None))
                continue
            if len(got) != len(specs):
                reads.append((mm, mn, specs,
                              "name asserts %d operand(s), FCML decoded %d"
                              % (len(specs), len(got))))
                continue
            why = [w for w in
                   (match_operand(sp % gmap if "%(" in sp else sp, tok)
                    for sp, tok in zip(specs, got)) if w]
            if why:
                reads.append((mm, mn, specs, "; ".join(why)))
                continue
            reads.append((mm, mn, specs, None))

        if nopattern:
            n_unparsed += 1
            problems.append("%s: no name pattern accepts it (emits %s %r) - "
                            "extend PATTERNS or fix the name"
                            % (name, hexs, dec_text))
            continue
        fitting = [r for r in reads if r[3] is None]
        if len(fitting) > 1:
            n_bad += 1
            problems.append("%s: %d name readings fit the same bytes - the "
                            "name is ambiguous" % (name, len(fitting)))
            continue
        if not fitting:
            n_bad += 1
            problems.append("%s: %s" % (name, reads[0][3]))
            continue
        n_ok += 1

    n_vx, vx_problems = audit_vx_helpers(src, mod, verbose)
    problems.extend(vx_problems)
    n_vx_ok = n_vx - len(vx_problems)

    # Every PROCEDURE that EMITS BYTES must have been looked at.  A helper
    # whose body the parser skipped is the failure mode that has bitten this
    # file three times now: the Em prefix, the missing parameter list, and a
    # comment in the wrong place - each time the suite stayed green while the
    # helper went unexamined and the reported subject count quietly dropped.
    #
    # The inventory is scanned INDEPENDENTLY of find_helpers, and that is the
    # whole design of this check.  Deriving both lists from the same parser
    # makes it vacuous: any input that stops find_helpers from matching also
    # removes the helper from the inventory, so the two lists agree and the
    # check reports nothing - which is exactly the failure it exists to catch.
    # So the inventory is a separate, deliberately dumb scan: every
    # `PROCEDURE <name>` at the start of a line, with everything up to the
    # next one as its text.  It makes no attempt to parse Modula-2, so it
    # cannot fail in the same way twice.  A helper the audit cannot reach is
    # therefore in `declared` and not in `seen`, which is a finding.
    #
    # The filter is the byte-call regex, so a parser, a lexer or a symbol
    # lookup that emits nothing is not expected to be audited - only helpers
    # that actually put bytes in the code buffer.  A helper that emitted bytes
    # under some spelling RE_BYTE does not know would be missed here too, which
    # is why RE_BYTE accepts both B(...) and Ebyte(...) and why a new spelling
    # has to be added there rather than relied on to be found by accident.
    strip = (lambda s: s[2:]) if mod != "Runtime.mod" else (lambda s: s)
    declared = {strip(nm) for nm, txt in scan_emitters(src)
                if RE_BYTE.search(txt)}
    # EmitData is a documented EXCLUSION, not a gap: its B(...) calls are DATA
    # (the D_ block), not instructions, so running the code buffer through a
    # disassembler produces nonsense like "add [bx+si],al" and a DECODE ERROR.
    # It is listed here rather than left implicit, because an exclusion that is
    # not written down is indistinguishable from a helper nobody audited.
    #
    # Its D_ offsets are asserted against this block by check_runtime.py's
    # golden, so "excluded from the name audit" is not "unchecked": the bytes
    # are pinned, only the name-to-opcode correspondence does not apply.
    excluded = {"EmitData"}
    strip = (lambda s: s[2:]) if mod != "Runtime.mod" else (lambda s: s)
    # The table is written in the names the audit uses (Runtime.mod's own
    # names, Compiler.mod's with the Em prefix already removed), so it is NOT
    # stripped again here.  Stripping it twice turns "AddSp" into "dSp", which
    # matches nothing - and a wrong match here is the dangerous direction: the
    # entries would stop suppressing their helpers and the report would look
    # like a genuine finding rather than like a broken table.
    documented = set(NON_ONE_LINE.get(mod, {}))
    audited_elsewhere = ({strip(n) for n in _NONLINE_VX}
                         if mod == "Runtime.mod" else set())
    unexamined = (declared - seen - {strip(e) for e in excluded}
                  - documented - audited_elsewhere)
    for n in sorted(unexamined):
        problems.append("%s: emits bytes but the audit never examined it - "
                        "the report would be green and the helper unchecked"
                        % n)
    print("\n  %s: %d one-line helpers audited, %d agree with their name, "
          "%d disagree, %d outside the grammar"
          % (mod, n_ok + n_unparsed + n_bad, n_ok, n_bad, n_unparsed))
    print("  %s: %d multi-line *Vx helpers audited, %d agree, %d problem(s)"
          % (mod, n_vx, n_vx_ok, len(vx_problems)))
    return n_ok + n_vx_ok, n_unparsed, n_bad, problems


def main(argv):
    verbose = "-v" in argv
    files = [a for a in argv[1:] if not a.startswith("-")]
    # No argument means "audit every module that emits bytes", not
    # "audit Runtime.mod".  The single-module default is why the audit was
    # green while EmXchgAxCx encoded XCHG BX,AX under a name that says
    # XCHG AX,CX - a fault that broke every two-variable arithmetic operation
    # in every program the compiler could produce.
    mods = [os.path.basename(f) for f in files] if files else list(MODULES)

    tot_ok = tot_unparsed = tot_bad = 0
    problems = []
    for mod in mods:
        path = os.path.join(SHELL, mod)
        if not os.path.exists(path):
            print("FAIL: no such module: %s" % path)
            return 1
        with open(path) as f:
            src = f.read()
        n_ok, n_unp, n_bad, pr = sweep(src, mod, verbose)
        tot_ok += n_ok
        tot_unparsed += n_unp
        tot_bad += n_bad
        problems.extend("%s: %s" % (mod, p) for p in pr)

    total = tot_ok + tot_unparsed + tot_bad
    print("\nAUDITED %d emitter helpers across %d module(s): %d agree with "
          "their name, %d disagree, %d outside the grammar"
          % (total, len(mods), tot_ok, tot_bad, tot_unparsed))
    if problems:
        print("FAIL: %d problem(s)" % len(problems))
        for p in problems:
            print("  - %s" % p)
        return 1
    print("PASS: every audited helper emits what its name says")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))