TP3/shell/tests/nonvacuity.sh

#!/bin/sh
# nonvacuity.sh -- prove the runtime checks can actually fail.
#
# A test that has never been seen red is not a test.  This script breaks the
# runtime on purpose, once per check, and asserts that the check goes red and
# says something useful about the breakage.  Then it restores the source and
# asserts everything is green again.
#
# Each mutation below is a real bug that was in this file at some point, not an
# invented one.  That is the point: these are the mistakes we actually make
# with 16-bit ModRM, so these are the ones the checks have to catch.
#
#   audit_helpers.py   name-versus-decode: catches a wrong ModRM that still
#                      decodes cleanly
#   audit_helpers.py   coverage:           catches a helper that has silently
#                      dropped OUT of the audit, which is a green report about
#                      a subject nobody looked at
#   run_com_tests.sh   the .COM layout:    catches a header that cannot be
#                      located, a runtime size that disagrees with the image,
#                      and an entry jump that starts in the wrong place
#   check_runtime.py   golden:            catches the same thing in the built
#                                           image
#   check_runtime.py   decode sweep:      catches a wrong instruction LENGTH
#   check_runtime.py   branch targets:    catches a wrong fixup
#   check_runtime.py   entry goldens:     catches a broken prologue
#   probe/modrm11.py   the mod=11 table:   catches the ModRM column itself
#                      going wrong, which no amount of decoding will show
#   check_framedisp.py the BP disp rule:   catches a displacement that reads
#                      a different address than the symbol table named
#   rt_exec.py        the BP contract:      catches an entry that borrows BP to
#                      reach its argument and does not hand it back.  The bytes
#                      are well formed, the golden is satisfied, the audit says
#                      every helper emits what its name says, and the machine
#                      triple-faults on the second call - so nothing short of
#                      running it, or of stating the register contract
#                      explicitly, can see it.
#   run_com_exec.py   behaviour:            catches a*b that emits an ADD, `>'
#                      and `>=' swapped, REPEAT..UNTIL that stops after one
#                      pass, and two procedures whose parameters collide.
#                      All four sat in fixtures that COMPILED and were never
#                      RUN, with every byte-level check green.
#
# The mod=11 cases do not need a rebuild -- they read the probe sources
# directly -- so they are cheap, and they are the ones that matter most: the
# table they guard is the one thing in this project that was wrong in the
# documentation while the code was right, and a table that is wrong in the
# code produces bytes that decode perfectly.
#
# Usage: tests/nonvacuity.sh        (from shell/; leaves Runtime.mod restored)

set -u
cd "$(dirname "$0")/.." || exit 1

GM2=/home/eric/bin/Modula2/Gm2/bin/gm2
SAVED=/tmp/opencode/nonvacuity.Runtime.mod
PROBE=/tmp/opencode/nonvacuity.rtprobe
DUMP=/tmp/opencode/nonvacuity.dump

cp Runtime.mod "$SAVED" || exit 1
trap 'cp "$SAVED" Runtime.mod; "$GM2" -fiso -c Runtime.mod >/dev/null 2>&1' EXIT

pass=0
fail=0

# mutate <file> <sed-expr> -- apply a deliberate breakage and INSIST it landed.
#
# Four cases in this file were already dead when first run, all the same way:
# the helper they name had been renamed or reformatted since the case was
# written, the sed matched nothing, the source was unchanged, and the check
# correctly passed - so the harness reported "NOT NON-VACUOUS" and, worse, a
# reader skimming the output could take "the check still passed" for a passing
# test.  A case that cannot fire is worse than no case: it is a claim of
# coverage that was never tested.
#
# So the mutation is verified, not assumed.  If the file is byte-identical
# afterwards, that is reported as a FAILURE of the harness, naming the sed, and
# the case is not run - because running it would only produce a meaningless
# green.  The message says what to do (fix the sed) rather than what it found.
mutate () {
    mf=$1
    msed=$2
    cp "$mf" /tmp/opencode/nonvacuity.mut.bak
    sed -i "$msed" "$mf"
    if cmp -s "$mf" /tmp/opencode/nonvacuity.mut.bak; then
        echo "  BROKEN CASE: the mutation did not change $mf"
        echo "       sed: $msed"
        echo "       the named code has probably been renamed or reformatted -"
        echo "       fix this case, it is asserting nothing"
        fail=$((fail + 1))
        return 1
    fi
    return 0
}

# rebuild <label> -- re-emit the runtime and dump it
rebuild () {
    "$GM2" -fiso -c Runtime.mod >/dev/null 2>&1 || return 1
    "$GM2" -fiso -o "$PROBE" tests/RtProbe.mod Runtime.o Posix.o \
        >/dev/null 2>&1 || return 1
    "$PROBE" > "$DUMP" || return 1
    return 0
}

# expect_red <label> <pattern> <checker-cmd...>
#   <pattern> is a grep the failure output must match, so a check cannot
#   "pass" by failing for some unrelated reason.
expect_red () {
    label=$1
    want=$2
    shift 2
    if out=$("$@" 2>&1); then
        echo "NOT NON-VACUOUS: $label -- the check still passed"
        fail=$((fail + 1))
    elif ! printf '%s\n' "$out" | grep -qi "$want"; then
        echo "WRONG FAILURE: $label -- went red, but not for the stated reason"
        printf '%s\n' "$out" | sed 's/^/       /'
        fail=$((fail + 1))
    else
        echo "  ok: $label"
        printf '%s\n' "$out" | grep -im1 "$want" | sed 's/^/       /'
        pass=$((pass + 1))
    fi
}

echo "== each mutation must turn the named check red"
echo

# --- 1. name-versus-decode -------------------------------------------
# MovSiBx was `89 DC`, which is MOV SP,BX.  Two bytes either way, decodes
# cleanly, and no structural check can see it.
cp "$SAVED" Runtime.mod
mutate Runtime.mod 's|B (0DEH) END MovSiBx|B (0DCH) END MovSiBx|'
expect_red "audit_helpers catches MovSiBx emitting MOV SP,BX" \
    "MovSiBx" python3 tests/audit_helpers.py

# CmpSiBx had the identical mistake, which is how you know a single fix is
# not enough -- the same misreading was written twice.
cp "$SAVED" Runtime.mod
mutate Runtime.mod 's|B (39H) ; B (0DEH) END CmpSiBx|B (39H) ; B (0DCH) END CmpSiBx|'
expect_red "audit_helpers catches CmpSiBx emitting CMP SP,BX" \
    "CmpSiBx" python3 tests/audit_helpers.py

# --- 2. golden, and entry goldens ------------------------------------
# MovDlAl was `88 C0` = MOV AL,AL instead of MOV DL,AL.  This is the case that
# motivated runtime.golden: the sweep stayed in sync, every branch target
# stayed on a boundary, no entry's first bytes moved, and the size did not
# change.  The target helper was MovAlDh when this case was written, which is
# the fourth way a case here can rot - see the note on `mutate` below.
cp "$SAVED" Runtime.mod
mutate Runtime.mod 's|PROCEDURE MovDlAl  ; BEGIN B (88H) ; B (0C2H)|PROCEDURE MovDlAl  ; BEGIN B (88H) ; B (0C0H)|'
rebuild
expect_red "runtime.golden catches MOV AL,AL" \
    "mov al,al" python3 tests/check_runtime.py "$DUMP"

# initmem opened with the mis-emitted MovSiAx, so its entry golden was the
# thing that noticed the prologue was a no-op.
cp "$SAVED" Runtime.mod
mutate Runtime.mod 's|B (0F0H) END MovSiAx|B (0C0H) END MovSiAx|'
rebuild
expect_red "check_runtime catches a broken initmem prologue" \
    "mov ax,ax" python3 tests/check_runtime.py "$DUMP"

# --- 3. decode sweep / length ----------------------------------------
# StDiDl was `88 97` = [BX+disp16],DL: mod=10, so the instruction needs a
# disp16 it was not given, and the sweep loses sync two bytes later.
cp "$SAVED" Runtime.mod
mutate Runtime.mod 's|PROCEDURE StDiDl   ; BEGIN B (88H) ; B (15H)|PROCEDURE StDiDl   ; BEGIN B (88H) ; B (97H)|'
rebuild
expect_red "decode sweep catches a mod=10 byte move with no displacement" \
    "mov byte ptr \[bx+5b5fh\],dl" python3 tests/check_runtime.py "$DUMP"

# --- 4. branch targets ------------------------------------------------
# FixUp measures a rel8 from the end of the instruction, one byte past the
# displacement field.  Drop the +1 and every short branch lands one byte into
# its target, which for a 3-byte instruction means the middle of it.  The
# bytes themselves are all perfectly well formed -- only the fixups are
# wrong -- so this is the one failure mode the golden cannot be expected to
# catch on its own.
cp "$SAVED" Runtime.mod
mutate Runtime.mod 's|rel := (t + 100H - (fix \[i\].place + 1)) MOD 100H|rel := (t + 100H - fix [i].place) MOD 100H|'
rebuild
expect_red "branch check catches rel8 fixups measured from the wrong byte" \
    "not an instruction boundary" \
    python3 tests/check_runtime.py "$DUMP"

echo
echo "== everything restored and green again"
cp "$SAVED" Runtime.mod
if rebuild; then
    if python3 tests/audit_helpers.py >/dev/null 2>&1 &&
       python3 tests/check_runtime.py "$DUMP" >/dev/null 2>&1; then
        echo "  ok: both checks pass on the restored source"
        pass=$((pass + 1))
    else
        echo "NOT RESTORED: a check is red after restoring Runtime.mod"
        fail=$((fail + 1))
    fi
else
    echo "NOT RESTORED: the runtime would not rebuild"
    fail=$((fail + 1))
fi

echo
echo "== the mod=11 table (probe/modrm11.py)"
# These mutate the probe's own sources, not the runtime, so there is no
# rebuild in the loop.  SAVED_PY / SAVED_S are restored after each case.
SAVED_PY=/tmp/opencode/nonvacuity.modrm11.py
SAVED_S=/tmp/opencode/nonvacuity.modrm11.s
cp tests/probe/modrm11.py "$SAVED_PY" || exit 1
cp tests/probe/modrm11.s "$SAVED_S" || exit 1

M11="python3 tests/probe/modrm11.py"
restore_probe () {
    cp "$SAVED_PY" tests/probe/modrm11.py
    cp "$SAVED_S" tests/probe/modrm11.s
}

# 1. one cell of the table moved
mutate tests/probe/modrm11.py 's|"Si", "Di"\]$|"Bp", "Di"]|'
expect_red "anchor pins a moved table cell" \
    "anchor ADD SI, 2" $M11
restore_probe

# 2. the table this project actually shipped: AX dropped off the front and a
#    duplicate BX invented at the end, which shifts every code down by one
mutate tests/probe/modrm11.py 's|^REG = .*$|REG = ["Cx", "Dx", "Bx", "Sp", "Bp", "Si", "Di", "Bx"]|'
expect_red "the table shifted by one (AX dropped, BX duplicated)" \
    "anchor MOV SP, BP" $M11
restore_probe

# 3. the .s edited to contradict the table.  This is the case that shows why
#    the hard-coded EXPECT bytes exist: the assembler encodes the new claim
#    correctly, so comparing the .s against `as` alone can never fail here.
mutate tests/probe/modrm11.s 's|movw    %sp, %di         # reg 100|movw    %bp, %di         # reg 100|'
expect_red "probe source edited away from the recorded bytes" \
    "expected 89 E7" $M11
restore_probe

# 4. the 8-bit list edited, which is a different table from the word one
mutate tests/probe/modrm11.s 's|movb    %al, %dl         # 88 C2  ->  DL := AL|movb    %al, %bl         # was DL|'
expect_red "the 8-bit register list edited" \
    "expected 88 C2" $M11
restore_probe

# 5. an anchor's recorded byte corrupted, so the anchor can no longer
#    corroborate itself
mutate tests/probe/modrm11.py 's|"8B EC", "8B E5"|"8B ED", "8B E5"|'
expect_red "anchor byte no longer matches the emitted code" \
    "expected 8B ED" $M11
restore_probe

if $M11 >/dev/null 2>&1; then
    echo "  ok: modrm11.py passes on the restored probe sources"
    pass=$((pass + 1))
else
    echo "NOT RESTORED: modrm11.py is red after restoring its sources"
    $M11 2>&1 | sed 's/^/       /'
    fail=$((fail + 1))
fi

echo
echo "== the BP displacement rule (check_framedisp.py)"
# This one is about Compiler.mod rather than the runtime, and it needs the
# whole toolchain rebuilt (comtest, not rtprobe), so it gets its own rebuild.
SAVED_C=/tmp/opencode/nonvacuity.Compiler.mod
cp Compiler.mod "$SAVED_C" || exit 1

rebuild_compiler () {
    $GM2 -fiso -c Compiler.mod >/dev/null 2>&1 || return 1
    $GM2 -fiso -fgen-module-list=tests/ct.lst -o /dev/null \
        tests/ComTest.mod TextBuf.o Posix.o Compiler.o Runtime.o Linker.o \
        >/dev/null 2>&1
    $GM2 -fiso -fuse-list=tests/ct.lst -o comtest \
        tests/ComTest.mod TextBuf.o Posix.o Compiler.o Runtime.o Linker.o \
        >/dev/null 2>&1 || return 1
    return 0
}

# 1. the original bug: `off MOD 100H`, always disp8.  Restores exactly the code
#    that was there before EmBpDisp existed.  t28's [BP+128] read becomes
#    [BP-128], which is the failure this whole check is named after.
python3 - "$SAVED_C" <<'PYEOF'
import sys
p = 'Compiler.mod'
s = open(p).read()
old = """BEGIN
   IF off <= 127 THEN
      Ebyte (46H) ; Ebyte (VAL (BYTE, off))
   ELSE
      Ebyte (86H) ; Eword (off)
   END
END EmBpDisp ;"""
new = """VAR disp : CARDINAL ;
BEGIN
   disp := off MOD 100H ;
   Ebyte (46H) ; Ebyte (VAL (BYTE, disp))
END EmBpDisp ;"""
assert old in s, "EmBpDisp body not found -- update this mutation"
open(p, 'w').write(s.replace(old, new))
PYEOF
if rebuild_compiler; then
    expect_red "displacement truncation reads a different address" \
        "no 8B access at \[BP+128\]" python3 tests/check_framedisp.py
else
    echo "  FAIL: the compiler would not rebuild with the truncation"
    fail=$((fail + 1))
fi
cp "$SAVED_C" Compiler.mod

# 2. the other half of the rule: always use the 4-byte form, ignoring the
#    <= 127 case.  This is over-cautious rather than wrong, so the checker must
#    still be happy -- which is worth asserting, because a check that only
#    ever fails on a smaller encoding is a check that pins one answer instead
#    of the rule.
python3 - <<'PYEOF'
p = 'Compiler.mod'
s = open(p).read()
old = """   IF off <= 127 THEN
      Ebyte (46H) ; Ebyte (VAL (BYTE, off))
   ELSE
      Ebyte (86H) ; Eword (off)
   END"""
new = """   Ebyte (86H) ; Eword (off)"""
assert old in s, "EmBpDisp branch not found -- update this mutation"
open(p, 'w').write(s.replace(old, new))
PYEOF
if rebuild_compiler; then
    if python3 tests/check_framedisp.py >/dev/null 2>&1; then
        echo "  ok: always-disp16 is accepted, so the check pins the rule"
        echo "       and not one particular encoding"
        pass=$((pass + 1))
    else
        echo "  FAIL: check_framedisp rejects a safe, over-long encoding"
        python3 tests/check_framedisp.py 2>&1 | sed 's/^/       /'
        fail=$((fail + 1))
    fi
else
    echo "  FAIL: the compiler would not rebuild with always-disp16"
    fail=$((fail + 1))
fi
cp "$SAVED_C" Compiler.mod

if rebuild_compiler; then
    if python3 tests/check_framedisp.py >/dev/null 2>&1; then
        echo "  ok: check_framedisp passes on the restored source"
        pass=$((pass + 1))
    else
        echo "NOT RESTORED: check_framedisp is red after restoring Compiler.mod"
        python3 tests/check_framedisp.py 2>&1 | sed 's/^/       /'
        fail=$((fail + 1))
    fi
else
    echo "NOT RESTORED: the compiler would not rebuild"
    fail=$((fail + 1))
fi

# --- 3. the operator bugs the nine dead fixtures were hiding ------------
echo
echo "== the four bugs the nine never-executed fixtures were hiding"
echo
# A different KIND of case from everything above.  The others break the code
# and assert a byte-level check notices; these break the code and assert a
# BEHAVIOURAL check notices, which is the only kind that could have found them.
# All four shipped with a green compile matrix, a passing .COM layout check, a
# passing golden and a passing emitter audit:
#
#   OpMul = 1        `a * b` emitted ADD AX,CX.  `*' and `+' both numbered
#                    their operator 1, and BinOpEmit cannot see which
#                    precedence level called it, so every multiplication
#                    dispatched to the addition.  The constant-folding arm was
#                    correct, which is why `n * n' with n a CONST was right and
#                    `a * a' with a a variable was not -- and t08_const is the
#                    only fixture that ever multiplied.
#   9Dh / 9FH        `>' got SETGE and `>=' got SETG: swapped, one letter
#                    apart in the mnemonic.  Only a==b could see it.
#   JNZ -> body      REPEAT..UNTIL looped back while the condition was TRUE,
#                    which is WHILE, so the body ran once and stopped.
#
# They are mutated back to the original defect and run_com_exec.py must go red
# on the exact fixture that pins the behaviour.  The fourth (HideLocals) is a
# scoping bug rather than an operator bug; it was hiding in the same place.
mutate_compiler () {   # reuse mutate's verified-change discipline on Compiler.mod
    mf=Compiler.mod
    msed=$1
    cp "$SAVED_C" /tmp/opencode/nonvacuity.mut2.bak
    sed -i "$msed" "$mf"
    if cmp -s "$mf" /tmp/opencode/nonvacuity.mut2.bak; then
        echo "  BROKEN CASE: the mutation did not change Compiler.mod"
        echo "       sed: $msed"
        echo "       the named code has probably been renamed or reformatted -"
        echo "       fix this case, it is asserting nothing"
        fail=$((fail + 1))
        return 1
    fi
    return 0
}

# The SETcc swap and the HideLocals removal are done in python rather than
# with sed: both need to match source text containing `*` and `(` in a way that
# is tedious and fragile as a regex, and a case whose only failure mode is a
# malformed sed is a case that silently asserts nothing.
#
# And a python helper fails in a way sed does not: a syntax error in the helper
# is a non-zero exit, `if mutate_foo; then` is simply false, and the case is
# SKIPPED -- with no failure counted and nothing on stdout but whatever python
# printed.  That is how the SETcc case spent its first run: an apostrophe in an
# assert message ("the `>' arm") closed the string early, python died, the
# compiler was never broken, and the suite still reported 0 failed.  A skipped
# case and a passing case look the same in the total.  So each helper below
# fails LOUDLY: a non-zero exit from python is reported as a BROKEN CASE and
# counted, never swallowed.
#
# Each one also counts its targets before replacing.  `assert s != before' only
# says the file changed; with two edits it would pass if just one of them
# landed, and with two identical HideLocals call sites it would happily delete
# the wrong one -- still a changed file, still a working compiler, still green
# for the wrong reason.
mutate_cc_swap () {
    if python3 - <<'PYX'
p = 'Compiler.mod'
s = open(p).read()
GT = 'EmSetcc (9FH) ;          (* >  SETG  *)'    # the greater-than arm
GE = 'EmSetcc (9DH) ;          (* >= SETGE *)'    # the greater-equal arm
assert s.count(GT) == 1, 'expected 1 greater-than arm, found %d' % s.count(GT)
assert s.count(GE) == 1, 'expected 1 greater-equal arm, found %d' % s.count(GE)
s = s.replace(GT, GT.replace('9FH', '9DH'))
s = s.replace(GE, GE.replace('9DH', '9FH'))
open(p, 'w').write(s)
PYX
    then
        return 0
    fi
    echo "  BROKEN CASE: the SETcc swap did not apply"
    echo "       the two EmSetcc arms are probably renamed or reformatted -"
    echo "       fix this case, it is asserting nothing"
    fail=$((fail + 1))
    return 1
}

mutate_no_hidelocals () {
    if python3 - <<'PYX'
p = 'Compiler.mod'
s = open(p).read()
# Two HideLocals calls exist.  Only the body-exit one may go: deleting the
# FORWARD one instead would still change the file, still rebuild, and still
# leave t13_proc compiling, so the case would go green for the wrong reason.
HL = '   HideLocals (nestMark) ;         (* parameters and locals stop here *)\n'
assert s.count(HL) == 1, 'expected 1 body-exit HideLocals, found %d' % s.count(HL)
open(p, 'w').write(s.replace(HL, ''))
PYX
    then
        return 0
    fi
    echo "  BROKEN CASE: HideLocals was not removed"
    echo "       the call or its comment has probably been reformatted -"
    echo "       fix this case, it is asserting nothing"
    fail=$((fail + 1))
    return 1
}

cp "$SAVED_C" Compiler.mod
if mutate_compiler 's|^         op := OpMul ; DropCh|         op := OpAdd ; DropCh|'; then
    if rebuild_compiler; then
        expect_red "execution catches '*' emitting an ADD (t08_const, n*n)" \
            "t08_const" python3 tests/run_com_exec.py t08_const
        # t08 only ever multiplied two CONSTANTS, which is the one path that was
        # never wrong, because BinOpEmit folds it.  So the case above is close
        # to vacuous: it proves the mutation changed the binary, not that the
        # emitted multiply is covered.  The check that matters needs a
        # VARIABLE operand, and no shipped fixture has one -- which is why the
        # bug survived at all.  So this writes a throwaway fixture that
        # multiplies a variable, runs it, and asserts the multiply is right.
        # The fixture is deleted afterwards; it is here to close the coverage
        # hole, not to become a permanent test (that is what a real fixture
        # with a `*' in it would be for).
        cat > tests/fixtures/zzmul.pas <<'ZZEOF'
program zzmul;
var a : integer ;
begin
  a := 7 ;
  writeln (a * 6)
end.
ZZEOF
        printf '42\r\n' > tests/fixtures/zzmul.out
        expect_red "execution catches '*' on a VARIABLE (the unfolded path)" \
            "zzmul" python3 tests/run_com_exec.py zzmul
        rm -f tests/fixtures/zzmul.pas tests/fixtures/zzmul.out
    else
        echo "  FAIL: the compiler would not rebuild with OpAdd for '*'"
        fail=$((fail + 1))
    fi
fi
cp "$SAVED_C" Compiler.mod

cp "$SAVED_C" Compiler.mod
if mutate_cc_swap; then
    if rebuild_compiler; then
        expect_red "execution catches '>' and '>=' swapped (t09_if)" \
            "t09_if" python3 tests/run_com_exec.py t09_if
    else
        echo "  FAIL: the compiler would not rebuild with the SETcc swap"
        fail=$((fail + 1))
    fi
fi
cp "$SAVED_C" Compiler.mod

cp "$SAVED_C" Compiler.mod
if mutate_compiler 's|      DropC (EmJcc (84H, L1)) ;        (\* JZ -> body again \*)|      zj := EmJcc (85H, L1) ;|'; then
    if rebuild_compiler; then
        expect_red "execution catches REPEAT..UNTIL exiting after one pass (t12)" \
            "t12_repeat" python3 tests/run_com_exec.py t12_repeat
    else
        echo "  FAIL: the compiler would not rebuild with the JNZ repeat"
        fail=$((fail + 1))
    fi
fi
cp "$SAVED_C" Compiler.mod

# The fourth: sibling procedures shared one parameter namespace, because a
# finished procedure's symbols were left at a level Search still accepts.
# Removing HideLocals puts two procedures' `a : integer' back in collision.
cp "$SAVED_C" Compiler.mod
if mutate_no_hidelocals; then
    if rebuild_compiler; then
        expect_red "a duplicate parameter in two procedures is a compile error again" \
            "ERROR 41" python3 tests/run_com_exec.py t13_proc
    else
        echo "  FAIL: the compiler would not rebuild without HideLocals"
        fail=$((fail + 1))
    fi
else
    echo "  FAIL: could not remove HideLocals to test the scoping fix"
    fail=$((fail + 1))
fi
cp "$SAVED_C" Compiler.mod

if rebuild_compiler; then
    if python3 tests/run_com_exec.py >/dev/null 2>&1; then
        echo "  ok: all 30 executed fixtures pass on the restored compiler"
        pass=$((pass + 1))
    else
        echo "NOT RESTORED: run_com_exec.py is red after restoring Compiler.mod"
        python3 tests/run_com_exec.py 2>&1 | grep -i fail | head -3 | sed 's/^/       /'
        fail=$((fail + 1))
    fi
else
    echo "NOT RESTORED: the compiler would not rebuild"
    fail=$((fail + 1))
fi

echo
echo "== the emitter-name audit of Compiler.mod (audit_helpers.py)"
# These need no rebuild: the audit reads the SOURCE, not the built object, so
# they are the cheapest cases here and they cover the module the audit used
# not to look at at all.  That is the point of the section: the audit reported
# "every helper agrees with its name" for a module it had never examined, and
# EmXchgAxCx was `93` (XCHG BX,AX) under a name that says XCHG AX,CX for the
# whole life of the project.  Two of these five are for faults that were real.
SAVED_C2=/tmp/opencode/nonvacuity.Compiler.mod.2
SAVED_R2=/tmp/opencode/nonvacuity.Runtime.mod.2
cp Compiler.mod "$SAVED_C2" || exit 1
cp Runtime.mod "$SAVED_R2" || exit 1
restore_audit_sources () {
    cp "$SAVED_C2" Compiler.mod
    cp "$SAVED_R2" Runtime.mod
}

AUD="python3 tests/audit_helpers.py"

# 1. THE fault.  91h is XCHG AX,CX; 93h is XCHG BX,AX.  Both are one byte, so
#    the compile matrix never moved and the byte counts never moved.
cp "$SAVED_C2" Compiler.mod
mutate Compiler.mod 's|^   Ebyte (91H)$|   Ebyte (93H)|'
expect_red "audit catches XchgAxCx emitting XCHG BX,AX" \
    "exchanges Ax and Bx" $AUD
restore_audit_sources

# 2. the coverage check itself.  A parameter list that find_helpers does not
#    accept is exactly how the real emitter was missed, and the inventory is
#    scanned separately on purpose so this can be caught.  Without the
#    independent scan this case is silent, because both lists would come from
#    the same parser and agree that the helper does not exist.
cp "$SAVED_C2" Compiler.mod
mutate Compiler.mod 's|^PROCEDURE EmXchgAxCx () ;$|PROCEDURE EmXchgAxCx (why : CARDINAL) ;|'
expect_red "audit reports an emitter it cannot reach, rather than skipping it" \
    "never examined it" $AUD
restore_audit_sources

# 3. EmXchgAxDx was named EmMoveAxDx, which said MOV where the bytes say XCHG.
#    93h here is XCHG AX,BX - one letter away, the exact class of mistake the
#    name is supposed to make impossible.
cp "$SAVED_C2" Compiler.mod
mutate Compiler.mod 's|^   Ebyte (92H)$|   Ebyte (93H)|'
expect_red "audit catches XchgAxDx emitting XCHG BX,AX" \
    "XchgAxDx" $AUD
restore_audit_sources

# 4. CmpArgW0's [BP+2] written as the 386 SIB form, which decodes on a 8086 as
#    [SI+24h].  A real bug: the runtime was clearing the wrong memory.
cp "$SAVED_R2" Runtime.mod
mutate Runtime.mod 's|   B (83H) ; B (7EH) ; B (2) ; B (0) ;|   B (83H) ; B (7CH) ; B (24) ; B (0) ; B (0) ;|'
expect_red "audit catches the [SI+24h] encoding of [BP+2]" \
    "memory base is 'si" $AUD
restore_audit_sources

# 5. MovAxSp is POP then PUSH, because MOV AX,[SP] does not exist on an 8086.
#    Dropping the POP leaves the stack one word short - a fault in the shape,
#    not in a byte value.
cp "$SAVED_C2" Compiler.mod
python3 - <<'PYEOF'
p='Compiler.mod'; s=open(p).read()
a="   Ebyte (58H) ;                     (* POP AX  *)\n"
assert s.count(a)==1, "EmMovAxSp POP line not found -- update this mutation"
open(p,'w').write(s.replace(a, ""))
PYEOF
expect_red "audit catches MovAxSp with its POP missing" \
    "MovAxSp" $AUD
restore_audit_sources

# 6. The two-instruction shape: IDIV is CWD then IDIV, and dropping the CWD
#    leaves an un-sign-extended dividend in DX:AX.  Both are still present as
#    a two-step spec, so a missing step has to be visible.
cp "$SAVED_C2" Compiler.mod
mutate Compiler.mod 's|   Ebyte (99H) ; Ebyte (0F7H) ; Ebyte (0F9H)|   Ebyte (0F7H) ; Ebyte (0F9H)|'
expect_red "audit catches IDiv without the CWD that extends the dividend" \
    "IDivAxCx" $AUD
restore_audit_sources

if $AUD >/dev/null 2>&1; then
    echo "  ok: the audit passes on both restored sources"
    pass=$((pass + 1))
else
    echo "NOT RESTORED: the audit is red after restoring the sources"
    $AUD 2>&1 | sed 's/^/       /'
    fail=$((fail + 1))
fi

echo
echo "== the BP contract rt_exec.py checks before it starts a machine"
# wrchar and wrbool both borrowed BP to reach their argument -- [SP] is not
# encodable in 16-bit mode -- and neither saved it.  The driver's cursor into
# the case record lives in BP precisely because BP is the one register an entry
# may keep, so "wrchar borrowed it and did not give it back" sent the second
# call to a garbage address, the machine triple-faulted, and the run printed the
# record header twice and hung.  Both the golden and the audit call that shape
# CORRECT: the bytes are well formed, every branch is on a boundary, the size
# is unchanged, and the decode says exactly what it says.  So the rule is now
# checked directly, and these two cases are what make that check more than a
# claim.
#
# rt_exec.py needs the whole runtime rebuilt and then boots 36 machines, so this
# section is the slow one.  The baseline comes first and is asserted: a case
# that mutates a red tree proves nothing.
SAVED_R3=/tmp/opencode/nonvacuity.Runtime.mod.3
cp Runtime.mod "$SAVED_R3" || exit 1

if rebuild; then
    if python3 tests/rt_exec.py >/dev/null 2>&1; then
        echo "  ok: baseline - rt_exec.py passes on the unmutated runtime"
        pass=$((pass + 1))
    else
        echo "  FAIL: the baseline is already red, so the cases below prove"
        echo "        nothing - fix the baseline before reading them"
        python3 tests/rt_exec.py 2>&1 | tail -3 | sed 's/^/       /'
        fail=$((fail + 1))
    fi
else
    echo "  FAIL: could not rebuild the runtime for the baseline"
    fail=$((fail + 1))
fi

# 1. THE fault: drop the PUSH, exactly as EmitWrChar was written.  This is the
#    shape the behavioural case found, so the byte-level check must find it too
#    -- a check that only the expensive test can trip is a check that has not
#    been made to earn its place.
cp "$SAVED_R3" Runtime.mod
python3 - <<'PYEOF'
p = 'Runtime.mod'
s = open(p).read()
a = '   M ("wrchar") ;\n   PushBp ; MovBpSp ;\n'
assert s.count(a) == 1, "EmitWrChar prologue not found exactly once"
open(p, 'w').write(s.replace(a, '   M ("wrchar") ;\n   MovBpSp ;\n'))
PYEOF
if rebuild; then
    expect_red "the BP contract catches wrchar borrowing BP unsaved" \
        "borrows BP but does not open with" python3 tests/rt_exec.py
else
    echo "  FAIL: the runtime would not rebuild with the PUSH removed"
    fail=$((fail + 1))
fi
cp "$SAVED_R3" Runtime.mod

# 2. The mirror image: push it and never pop it.  A different one-instruction
#    omission with the same consequence for a caller, and the reason the rule
#    asks for the 5D and not just the 55.
cp "$SAVED_R3" Runtime.mod
mutate Runtime.mod 's|^   MovSpBp ; PopBp ;$|   MovSpBp ;|'
if rebuild; then
    expect_red "the BP contract catches a BP that is pushed and never popped" \
        "pushes BP but never pops it" python3 tests/rt_exec.py
else
    echo "  FAIL: the runtime would not rebuild with the POP removed"
    fail=$((fail + 1))
fi
cp "$SAVED_R3" Runtime.mod

# 3. The scan's other silent failure: a rule that matches nothing looks exactly
#    like a rule that passes.  The pattern the check looks for is changed to one
#    the blob does not contain -- MOV BP,DI, which the runtime has no reason to
#    emit -- so the check has examined thirteen entries and matched nothing and
#    MUST say so rather than report a clean sweep.  This is the general shape of
#    the fault this project keeps making: a check whose SUBJECT has drifted
#    reports a confident answer about the wrong thing.
cp "$SAVED_R3" Runtime.mod
SAVED_X=/tmp/opencode/nonvacuity.rt_exec.py
cp tests/rt_exec.py "$SAVED_X" || exit 1
mutate tests/rt_exec.py 's|^MOV_BP_SP = b"\\x8b\\xec" .*$|MOV_BP_SP = b"\\x8b\\xed"               # MOV BP,DI: never emitted|' \
expect_red "a check that matched nothing is a failure, not a pass" \
    "matched nothing" python3 tests/rt_exec.py
cp "$SAVED_X" tests/rt_exec.py

if rebuild; then
    if python3 tests/rt_exec.py >/dev/null 2>&1; then
        echo "  ok: rt_exec.py passes on the restored source"
        pass=$((pass + 1))
    else
        echo "NOT RESTORED: rt_exec.py is red after restoring Runtime.mod"
        python3 tests/rt_exec.py 2>&1 | tail -3 | sed 's/^/       /'
        fail=$((fail + 1))
    fi
else
    echo "NOT RESTORED: the runtime would not rebuild"
    fail=$((fail + 1))
fi

# 4. The GOLDEN and the entry goldens, which are the other half of the same
#    rule.  rt_exec.py states the contract; check_runtime.py pins the bytes.
#    They are separate mechanisms and the case below is what shows the golden
#    one works on its own -- a re-baseline of runtime.golden would otherwise
#    have absorbed the new prologues silently, and the next person to bless it
#    would have no way to know the PUSH and POP were ever missing.
cp "$SAVED_R3" Runtime.mod
python3 - <<'PYEOF'
p = 'Runtime.mod'
s = open(p).read()
a = '   M ("wrchar") ;\n   PushBp ; MovBpSp ;\n   MovAlArg4 ;\n   MovSpBp ; PopBp ;\n'
assert s.count(a) == 1, "EmitWrChar frame not found exactly once"
open(p, 'w').write(s.replace(a, '   M ("wrchar") ;\n   MovBpSp ;\n   MovAlArg2 ;\n   MovSpBp ;\n'))
PYEOF
if rebuild; then
    expect_red "the entry golden catches wrchar without its PUSH BP" \
        "entry wrchar starts 8B EC" python3 tests/check_runtime.py "$DUMP"
else
    echo "  FAIL: the runtime would not rebuild with wrchar's frame removed"
    fail=$((fail + 1))
fi
cp "$SAVED_R3" Runtime.mod

echo
echo "== the .COM layout check, and the runtime size it now measures"
# The checker used to RESTATE the runtime's size as a literal.  It was wrong
# by 41 bytes for an unknown time, and every one of the 30 .COM files "failed"
# on a header read out of the code stream.  A duplicated constant that has
# drifted does not fail loudly; it re-reports the same falsehood, in which the
# real failures hide.  The size is now MEASURED from the image.
#
# These cases corrupt a real emitted .COM and require the checker to notice.
# They need the images, so they are built once and copied; the checker has a
# --check-only mode for exactly this, because its scratch directory is normally
# deleted on exit and a check that has only ever seen the truth is not a check.
KEEPDIR=/tmp/opencode/nonvacuity.com
rm -rf "$KEEPDIR"
TP_COM_KEEP=1 tests/run_com_tests.sh >/tmp/opencode/nonvacuity.com.log 2>&1
KEEP=$(sed -n 's/^TP_COM_KEEP=1: images left in //p' \
       /tmp/opencode/nonvacuity.com.log | tail -1)
if [ -z "$KEEP" ] || [ ! -d "$KEEP" ]; then
    echo "  FAIL: could not obtain emitted .COM images for the layout cases"
    fail=$((fail + 1))
else
    COMCHK="tests/run_com_tests.sh --check-only"

    # 0. The baseline.  Every case below is a claim that a specific assertion
    #    turns red, and none of them means anything if the copies of untouched
    #    images already fail.  (The stale RT_SZ produced exactly that: 30
    #    failures that were not findings.)  So this is asserted first, and a
    #    failure here is reported as a broken baseline rather than a red test.
    #
    #    The baseline is also WHERE THE HEADER OFFSET COMES FROM.  Cases 1 and 2
    #    used to carry it as the literal 435 and 439, which were ENT_SZ + the
    #    runtime size at the time -- 432.  Two 4-byte changes to the runtime
    #    later, both cases were editing the wrong bytes: 435 had become four
    #    bytes inside the runtime itself, so case 1 had stopped testing "the
    #    header cannot be found" and had started testing "the checker also
    #    notices you scribbled on the code", and case 2 had become a no-op
    #    that wrote the header where it already was.  Both still went red, for
    #    reasons the messages did not name, which is the whole problem: a
    #    hard-coded offset is a claim about the world that expires silently.
    #
    #    So the offset is read back out of the checker's own report on the
    #    untouched images, and the expected numbers below are ARITHMETIC ON IT.
    #    If the runtime grows again, these cases still test what they say.
    rm -rf "$KEEPDIR"; mkdir -p "$KEEPDIR"
    cp "$KEEP"/*.COM "$KEEP"/raw.txt "$KEEPDIR"/
    BASE_OUT=$($COMCHK "$KEEPDIR" 2>&1)
    if [ $? -eq 0 ]; then
        echo "  ok: baseline - untouched copies of the real images all pass"
        pass=$((pass + 1))
    else
        echo "  FAIL: the baseline is already red, so the cases below prove"
        echo "        nothing - fix the baseline before reading them"
        printf '%s\n' "$BASE_OUT" | grep FAIL | head -3 | sed 's/^/       /'
        fail=$((fail + 1))
    fi
    BASE_RT=$(printf '%s\n' "$BASE_OUT" \
        | sed -n 's/.*measured runtime size: \([0-9][0-9]*\) bytes.*/\1/p' \
        | head -1)
    if [ -z "$BASE_RT" ]; then
        echo "  FAIL: the checker did not report the runtime size it measured,"
        echo "        so the cases below cannot find the header to edit"
        fail=$((fail + 1))
        BASE_RT=0
    fi
    # The layout constants are ENT_SZ 3 and HDR_SZ 16 (run_com_tests.sh), so the
    # header sits at hdrOff = 3 + rtSz and the entry jump's displacement must be
    # (hdrOff + 16) - 3.  The wanted number is therefore computed from the header
    # OFFSET, not from rtSz: the two differ by 3, and getting that backwards
    # produces a plausible-looking expectation three bytes out, which is how
    # this case came to expect "want 452" and then be reported as not proving
    # what it said.
    BASE_OFF=$((3 + BASE_RT))
    # Case 2 claims a runtime four bytes LONGER, so the header - and with it the
    # demanded jump target - moves four bytes further on.
    SHIFTED_WANT=$((BASE_OFF + 4 + 16 - 3))
    echo "       (measured runtime size $BASE_RT, header at image offset $BASE_OFF)"

    # 1. Break hdrDS so it no longer ties the header to its own offset.  The
    #    header must become UNFINDABLE and be reported as such - a checker that
    #    fell back to a remembered offset would report a confident number here,
    #    which is the failure mode the measurement was introduced to remove.
    rm -rf "$KEEPDIR"; mkdir -p "$KEEPDIR"
    cp "$KEEP"/*.COM "$KEEP"/raw.txt "$KEEPDIR"/
    python3 - "$KEEPDIR/t01_minimal.COM" "$BASE_OFF" <<'PYEOF'
import sys
p, off = sys.argv[1], int(sys.argv[2])
d = bytearray(open(p, 'rb').read())
d[off + 4:off + 6] = (0x1234).to_bytes(2, 'little')     # hdrDS, no longer self-consistent
open(p, 'wb').write(bytes(d))
PYEOF
    expect_red "a header that cannot be located is reported, not assumed" \
        "no program header found" $COMCHK "$KEEPDIR"

    # 2. A complete, self-consistent header four bytes later, so the measured
    #    runtime size becomes $((BASE_RT + 4)) instead of $BASE_RT.  This is the
    #    positive half of the same check: the derivation must FOLLOW the file,
    #    and the entry jump assertion - expressed in terms of the measurement -
    #    must follow it too, demanding $SHIFTED_WANT rather than $((BASE_OFF + 16 - 3)).
    #
    #    hdrCS is copied from the header already in the file rather than written
    #    as a literal.  It used to be the literal 464+100h, which was the image
    #    length when the image was 720 bytes; four bytes of runtime later it was
    #    464+100h against a 724-byte image, so this case was ALSO failing on
    #    hdrCS, for a reason three lines below the one it was written to test.
    rm -rf "$KEEPDIR"; mkdir -p "$KEEPDIR"
    cp "$KEEP"/*.COM "$KEEP"/raw.txt "$KEEPDIR"/
    python3 - "$KEEPDIR/t01_minimal.COM" "$BASE_OFF" "$SHIFTED_WANT" <<'PYEOF'
import sys
p, off, want = sys.argv[1], int(sys.argv[2]), int(sys.argv[3])
d = bytearray(open(p, 'rb').read())
hdrCS = int.from_bytes(d[off + 2:off + 4], 'little')   # unchanged: still the image end
off += 4
ds = off + 0x1000 + 0x100
w = [1, hdrCS, ds, ds + 4, 0, 0, 0, 0]
for i, x in enumerate(w):
    d[off + 2 * i:off + 2 * i + 2] = x.to_bytes(2, 'little')
open(p, 'wb').write(bytes(d))
PYEOF
    expect_red "the measured runtime size follows the image ($BASE_RT -> $((BASE_RT + 4)))" \
        "want $SHIFTED_WANT" $COMCHK "$KEEPDIR"

    # 3. The entry jump's opcode.  One byte, and the only assertion in the
    #    project that can see where execution STARTS.
    rm -rf "$KEEPDIR"; mkdir -p "$KEEPDIR"
    cp "$KEEP"/*.COM "$KEEP"/raw.txt "$KEEPDIR"/
    python3 - "$KEEPDIR/t01_minimal.COM" <<'PYEOF'
import sys
p = sys.argv[1]
d = bytearray(open(p, 'rb').read())
d[0] = 0xEA
open(p, 'wb').write(bytes(d))
PYEOF
    expect_red "the entry jump must be E9, not a near JMP" \
        "not the E9 of the entry jump" $COMCHK "$KEEPDIR"

    # 4. The entry jump's target, moved one instruction earlier.  A .COM that
    #    lands in the middle of the prologue runs, prints something and exits
    #    cleanly, so no size or structure check can see this.
    rm -rf "$KEEPDIR"; mkdir -p "$KEEPDIR"
    cp "$KEEP"/*.COM "$KEEP"/raw.txt "$KEEPDIR"/
    python3 - "$KEEPDIR/t01_minimal.COM" <<'PYEOF'
import sys
p = sys.argv[1]
d = bytearray(open(p, 'rb').read())
d[1:3] = (100).to_bytes(2, 'little')
open(p, 'wb').write(bytes(d))
PYEOF
    expect_red "the entry jump must land on the first instruction" \
        "entry jump rel16=100" $COMCHK "$KEEPDIR"

    rm -rf "$KEEPDIR"
fi

echo
echo "non-vacuity: $pass ok, $fail failed"
[ "$fail" -eq 0 ]