TP3/shell/tests/nonvacuity.sh

#!/bin/sh
# nonvacuity.sh -- prove the runtime checks can actually fail.
#
# A test that has never been seen red is not a test.  This script breaks the
# runtime on purpose, once per check, and asserts that the check goes red and
# says something useful about the breakage.  Then it restores the source and
# asserts everything is green again.
#
# Each mutation below is a real bug that was in this file at some point, not an
# invented one.  That is the point: these are the mistakes we actually make
# with 16-bit ModRM, so these are the ones the checks have to catch.
#
#   audit_helpers.py   name-versus-decode: catches a wrong ModRM that still
#                      decodes cleanly, and both halves of a name that admits
#                      an operand at all - `34 02' where the name promises
#                      `34 01', and `35 01' where the row's opcode gate admits
#                      only 34h.  Both rows are new, and a row nobody has seen
#                      reject anything accepts whatever it is shown.
#   audit_helpers.py   coverage:           catches a helper that has silently
#                      dropped OUT of the audit, which is a green report about
#                      a subject nobody looked at
#   run_com_tests.sh   the .COM layout:    catches a header that cannot be
#                      located, a runtime size that disagrees with the image,
#                      and an entry jump that starts in the wrong place
#   check_runtime.py   golden:            catches the same thing in the built
#                                           image
#   check_runtime.py   decode sweep:      catches a wrong instruction LENGTH
#   check_runtime.py   branch targets:    catches a wrong fixup
#   check_runtime.py   entry goldens:     catches a broken prologue
#   probe/modrm11.py   the mod=11 table:   catches the ModRM column itself
#                      going wrong, which no amount of decoding will show
#   check_framedisp.py the BP disp rule:   catches a displacement that reads
#                      a different address than the symbol table named
#   tests/comimage.py the image layout:     catches a sweep region built from a
#                      literal instead of from two independent readings of the
#                      file - the old RT_SZ went on reporting PASS while its
#                      region began inside the runtime - and the layout itself
#                      being written down a second time, now that one file
#                      defines it and all three readers import it
#   rt_exec.py        the BP contract:      catches an entry that borrows BP to
#                      reach its argument and does not hand it back.  The bytes
#                      are well formed, the golden is satisfied, the audit says
#                      every helper emits what its name says, and the machine
#                      triple-faults on the second call - so nothing short of
#                      running it, or of stating the register contract
#                      explicitly, can see it.
#   check_8086.py    the 8086 opcodes:    catches a conditional branch or a
#                      SETcc emitted as `0F 8x'/`0F 9x', which are 386-only.
#                      Nothing else here can: qemu-system-i386's lowest CPU
#                      model is 486, where both are ordinary instructions, and
#                      FCML's -m16 mode is a 386 too.  Clause H additionally
#                      catches the branch polarity being inverted, which is
#                      still a legal opcode and therefore invisible to every
#                      shape-based check.
#   run_com_exec.py   behaviour:            catches a*b that emits an ADD, `>'
#                      and `>=' swapped, REPEAT..UNTIL that stops after one
#                      pass, two procedures whose parameters collide, a left
#                      operand overwritten by the right one's code (SaveLeft),
#                      and a boolean `not' lowered as the integer one.  Every
#                      one of them sat in a fixture that COMPILED and was
#                      never RUN, with every byte-level check green.
#   run_exec86.py     behaviour (Exec86):   catches the interpreter's OWN
#                      reading of the machine: JE inverted in Exec86's Cond,
#                      which swaps the two arms of every `=' in every program
#                      while the emitted bytes, the sizes and every
#                      byte-level check stay green.  The image is fine; only
#                      the thing reading it is wrong, so nothing that looks at
#                      the image can fail this one.
#   runtest.py        the R key:            catches CmdRun poking nothing into
#                      the interpreter at all, which faults on the first step
#                      and prints "interpreter fault" instead of the guest's
#                      answer.  It is the only case here that needs a rebuilt
#                      SHELL rather than a rebuilt compiler or runtime.
#
# The mod=11 cases do not need a rebuild -- they read the probe sources
# directly -- so they are cheap, and they are the ones that matter most: the
# table they guard is the one thing in this project that was wrong in the
# documentation while the code was right, and a table that is wrong in the
# code produces bytes that decode perfectly.
#
# Usage: tests/nonvacuity.sh        (from shell/; leaves Runtime.mod restored)

set -u
cd "$(dirname "$0")/.." || exit 1

GM2=/home/eric/bin/Modula2/Gm2/bin/gm2
SAVED=../tmp/nonvacuity.Runtime.mod
PROBE=../tmp/nonvacuity.rtprobe
DUMP=../tmp/nonvacuity.dump
# Exec86.mod is UNTRACKED, so unlike Runtime.mod git cannot put a botched
# mutation back: the copy taken here is the only correct one in existence.
# Shell.mod is tracked but is mutated by the R-key case below, and a trap that
# restored only the sources would leave tpshell BUILT FROM the mutation - so
# the trap rebuilds too.  All three copies live in the project's own tmp/ and
# are taken here, before the first mutation, rather than beside each case.
#
# comimage.py and comtest.py are the image layout and one of its three readers;
# both are mutated by cases further down, and neither is recoverable from git
# once the mutation is staged over - so they join the trap here for the same
# reason, and an interrupted run cannot leave the layout half-written.
mkdir -p ../tmp
SAVED_E=../tmp/nonvacuity.Exec86.mod
SAVED_SH=../tmp/nonvacuity.Shell.mod
SAVED_CI=../tmp/nonvacuity.comimage.py
SAVED_T=../tmp/nonvacuity.comtest.py

cp Runtime.mod "$SAVED" || exit 1
cp Exec86.mod "$SAVED_E" || exit 1
cp Shell.mod "$SAVED_SH" || exit 1
cp tests/comimage.py "$SAVED_CI" || exit 1
cp tests/comtest.py "$SAVED_T" || exit 1

restore_all () {
    cp "$SAVED" Runtime.mod
    cp "$SAVED_E" Exec86.mod
    cp "$SAVED_SH" Shell.mod
    cp "$SAVED_CI" tests/comimage.py
    cp "$SAVED_T" tests/comtest.py
    "$GM2" -fiso -c Runtime.mod >/dev/null 2>&1
    "$GM2" -fiso -c Exec86.mod >/dev/null 2>&1
    # The shell is rebuilt as well: a test that runs against a binary built
    # from a half-restored tree is reporting on the mutation, not on the code.
    make >/dev/null 2>&1
}
trap restore_all EXIT

pass=0
fail=0

# mutate <file> <sed-expr> -- apply a deliberate breakage and INSIST it landed.
#
# Four cases in this file were already dead when first run, all the same way:
# the helper they name had been renamed or reformatted since the case was
# written, the sed matched nothing, the source was unchanged, and the check
# correctly passed - so the harness reported "NOT NON-VACUOUS" and, worse, a
# reader skimming the output could take "the check still passed" for a passing
# test.  A case that cannot fire is worse than no case: it is a claim of
# coverage that was never tested.
#
# So the mutation is verified, not assumed.  If the file is byte-identical
# afterwards, that is reported as a FAILURE of the harness, naming the sed, and
# the case is not run - because running it would only produce a meaningless
# green.  The message says what to do (fix the sed) rather than what it found.
mutate () {
    mf=$1
    msed=$2
    cp "$mf" ../tmp/nonvacuity.mut.bak
    sed -i "$msed" "$mf"
    if cmp -s "$mf" ../tmp/nonvacuity.mut.bak; then
        echo "  BROKEN CASE: the mutation did not change $mf"
        echo "       sed: $msed"
        echo "       the named code has probably been renamed or reformatted -"
        echo "       fix this case, it is asserting nothing"
        fail=$((fail + 1))
        return 1
    fi
    return 0
}

# rebuild <label> -- re-emit the runtime and dump it
rebuild () {
    "$GM2" -fiso -c Runtime.mod >/dev/null 2>&1 || return 1
    "$GM2" -fiso -o "$PROBE" tests/RtProbe.mod Runtime.o Posix.o \
        >/dev/null 2>&1 || return 1
    "$PROBE" > "$DUMP" || return 1
    return 0
}

# expect_red <label> <pattern> <checker-cmd...>
#   <pattern> is a grep the failure output must match, so a check cannot
#   "pass" by failing for some unrelated reason.
expect_red () {
    label=$1
    want=$2
    shift 2
    if out=$("$@" 2>&1); then
        echo "NOT NON-VACUOUS: $label -- the check still passed"
        fail=$((fail + 1))
    elif ! printf '%s\n' "$out" | grep -qi "$want"; then
        echo "WRONG FAILURE: $label -- went red, but not for the stated reason"
        printf '%s\n' "$out" | sed 's/^/       /'
        fail=$((fail + 1))
    else
        echo "  ok: $label"
        printf '%s\n' "$out" | grep -im1 "$want" | sed 's/^/       /'
        pass=$((pass + 1))
    fi
}

echo "== each mutation must turn the named check red"
echo

# --- 1. name-versus-decode -------------------------------------------
# MovSiBx was `89 DC`, which is MOV SP,BX.  Two bytes either way, decodes
# cleanly, and no structural check can see it.
cp "$SAVED" Runtime.mod
mutate Runtime.mod 's|B (0DEH) END MovSiBx|B (0DCH) END MovSiBx|'
expect_red "audit_helpers catches MovSiBx emitting MOV SP,BX" \
    "MovSiBx" python3 tests/audit_helpers.py

# CmpSiBx had the identical mistake, which is how you know a single fix is
# not enough -- the same misreading was written twice.
cp "$SAVED" Runtime.mod
mutate Runtime.mod 's|B (39H) ; B (0DEH) END CmpSiBx|B (39H) ; B (0DCH) END CmpSiBx|'
expect_red "audit_helpers catches CmpSiBx emitting CMP SP,BX" \
    "CmpSiBx" python3 tests/audit_helpers.py

# --- 2. golden, and entry goldens ------------------------------------
# MovDlAl was `88 C0` = MOV AL,AL instead of MOV DL,AL.  This is the case that
# motivated runtime.golden: the sweep stayed in sync, every branch target
# stayed on a boundary, no entry's first bytes moved, and the size did not
# change.  The target helper was MovAlDh when this case was written, which is
# the fourth way a case here can rot - see the note on `mutate` below.
cp "$SAVED" Runtime.mod
mutate Runtime.mod 's|PROCEDURE MovDlAl  ; BEGIN B (88H) ; B (0C2H)|PROCEDURE MovDlAl  ; BEGIN B (88H) ; B (0C0H)|'
rebuild
expect_red "runtime.golden catches MOV AL,AL" \
    "mov al,al" python3 tests/check_runtime.py "$DUMP"

# initmem opened with the mis-emitted MovSiAx, so its entry golden was the
# thing that noticed the prologue was a no-op.
cp "$SAVED" Runtime.mod
mutate Runtime.mod 's|B (0F0H) END MovSiAx|B (0C0H) END MovSiAx|'
rebuild
expect_red "check_runtime catches a broken initmem prologue" \
    "mov ax,ax" python3 tests/check_runtime.py "$DUMP"

# --- 3. decode sweep / length ----------------------------------------
# StDiDl was `88 97` = [BX+disp16],DL: mod=10, so the instruction needs a
# disp16 it was not given, and the sweep loses sync two bytes later.
cp "$SAVED" Runtime.mod
mutate Runtime.mod 's|PROCEDURE StDiDl   ; BEGIN B (88H) ; B (15H)|PROCEDURE StDiDl   ; BEGIN B (88H) ; B (97H)|'
rebuild
expect_red "decode sweep catches a mod=10 byte move with no displacement" \
    "mov byte ptr \[bx+5b5fh\],dl" python3 tests/check_runtime.py "$DUMP"

# --- 4. branch targets ------------------------------------------------
# FixUp measures a rel8 from the end of the instruction, one byte past the
# displacement field.  Drop the +1 and every short branch lands one byte into
# its target, which for a 3-byte instruction means the middle of it.  The
# bytes themselves are all perfectly well formed -- only the fixups are
# wrong -- so this is the one failure mode the golden cannot be expected to
# catch on its own.
cp "$SAVED" Runtime.mod
mutate Runtime.mod 's|rel := (t + 100H - (fix \[i\].place + 1)) MOD 100H|rel := (t + 100H - fix [i].place) MOD 100H|'
rebuild
expect_red "branch check catches rel8 fixups measured from the wrong byte" \
    "not an instruction boundary" \
    python3 tests/check_runtime.py "$DUMP"

echo
echo "== everything restored and green again"
cp "$SAVED" Runtime.mod
if rebuild; then
    if python3 tests/audit_helpers.py >/dev/null 2>&1 &&
       python3 tests/check_runtime.py "$DUMP" >/dev/null 2>&1; then
        echo "  ok: both checks pass on the restored source"
        pass=$((pass + 1))
    else
        echo "NOT RESTORED: a check is red after restoring Runtime.mod"
        fail=$((fail + 1))
    fi
else
    echo "NOT RESTORED: the runtime would not rebuild"
    fail=$((fail + 1))
fi

echo
echo "== the mod=11 table (probe/modrm11.py)"
# These mutate the probe's own sources, not the runtime, so there is no
# rebuild in the loop.  SAVED_PY / SAVED_S are restored after each case.
SAVED_PY=../tmp/nonvacuity.modrm11.py
SAVED_S=../tmp/nonvacuity.modrm11.s
cp tests/probe/modrm11.py "$SAVED_PY" || exit 1
cp tests/probe/modrm11.s "$SAVED_S" || exit 1

M11="python3 tests/probe/modrm11.py"
restore_probe () {
    cp "$SAVED_PY" tests/probe/modrm11.py
    cp "$SAVED_S" tests/probe/modrm11.s
}

# 1. one cell of the table moved
mutate tests/probe/modrm11.py 's|"Si", "Di"\]$|"Bp", "Di"]|'
expect_red "anchor pins a moved table cell" \
    "anchor ADD SI, 2" $M11
restore_probe

# 2. the table this project actually shipped: AX dropped off the front and a
#    duplicate BX invented at the end, which shifts every code down by one
mutate tests/probe/modrm11.py 's|^REG = .*$|REG = ["Cx", "Dx", "Bx", "Sp", "Bp", "Si", "Di", "Bx"]|'
expect_red "the table shifted by one (AX dropped, BX duplicated)" \
    "anchor MOV SP, BP" $M11
restore_probe

# 3. the .s edited to contradict the table.  This is the case that shows why
#    the hard-coded EXPECT bytes exist: the assembler encodes the new claim
#    correctly, so comparing the .s against `as` alone can never fail here.
mutate tests/probe/modrm11.s 's|movw    %sp, %di         # reg 100|movw    %bp, %di         # reg 100|'
expect_red "probe source edited away from the recorded bytes" \
    "expected 89 E7" $M11
restore_probe

# 4. the 8-bit list edited, which is a different table from the word one
mutate tests/probe/modrm11.s 's|movb    %al, %dl         # 88 C2  ->  DL := AL|movb    %al, %bl         # was DL|'
expect_red "the 8-bit register list edited" \
    "expected 88 C2" $M11
restore_probe

# 5. an anchor's recorded byte corrupted, so the anchor can no longer
#    corroborate itself
mutate tests/probe/modrm11.py 's|"8B EC", "8B E5"|"8B ED", "8B E5"|'
expect_red "anchor byte no longer matches the emitted code" \
    "expected 8B ED" $M11
restore_probe

if $M11 >/dev/null 2>&1; then
    echo "  ok: modrm11.py passes on the restored probe sources"
    pass=$((pass + 1))
else
    echo "NOT RESTORED: modrm11.py is red after restoring its sources"
    $M11 2>&1 | sed 's/^/       /'
    fail=$((fail + 1))
fi

echo
echo "== the BP displacement rule (check_framedisp.py)"
# This one is about Compiler.mod rather than the runtime, and it needs the
# whole toolchain rebuilt (comtest, not rtprobe), so it gets its own rebuild.
SAVED_C=../tmp/nonvacuity.Compiler.mod
cp Compiler.mod "$SAVED_C" || exit 1

rebuild_compiler () {
    $GM2 -fiso -c Compiler.mod >/dev/null 2>&1 || return 1
    $GM2 -fiso -fgen-module-list=tests/ct.lst -o /dev/null \
        tests/ComTest.mod TextBuf.o Posix.o Compiler.o Runtime.o Linker.o \
        >/dev/null 2>&1
    $GM2 -fiso -fuse-list=tests/ct.lst -o comtest \
        tests/ComTest.mod TextBuf.o Posix.o Compiler.o Runtime.o Linker.o \
        >/dev/null 2>&1 || return 1
    return 0
}

# 1. the original bug: `off MOD 100H`, always disp8.  Restores exactly the code
#    that was there before EmBpDisp existed.  t28's [BP+128] read becomes
#    [BP-128], which is the failure this whole check is named after.
python3 - "$SAVED_C" <<'PYEOF'
import sys
p = 'Compiler.mod'
s = open(p).read()
old = """BEGIN
   IF off <= 127 THEN
      Ebyte (46H) ; Ebyte (VAL (BYTE, off))
   ELSE
      Ebyte (86H) ; Eword (off)
   END
END EmBpDisp ;"""
new = """VAR disp : CARDINAL ;
BEGIN
   disp := off MOD 100H ;
   Ebyte (46H) ; Ebyte (VAL (BYTE, disp))
END EmBpDisp ;"""
assert old in s, "EmBpDisp body not found -- update this mutation"
open(p, 'w').write(s.replace(old, new))
PYEOF
if rebuild_compiler; then
    expect_red "displacement truncation reads a different address" \
        "no 8B access at \[BP+128\]" python3 tests/check_framedisp.py
else
    echo "  FAIL: the compiler would not rebuild with the truncation"
    fail=$((fail + 1))
fi
cp "$SAVED_C" Compiler.mod

# 2. the other half of the rule: always use the 4-byte form, ignoring the
#    <= 127 case.  This is over-cautious rather than wrong, so the checker must
#    still be happy -- which is worth asserting, because a check that only
#    ever fails on a smaller encoding is a check that pins one answer instead
#    of the rule.
python3 - <<'PYEOF'
p = 'Compiler.mod'
s = open(p).read()
old = """   IF off <= 127 THEN
      Ebyte (46H) ; Ebyte (VAL (BYTE, off))
   ELSE
      Ebyte (86H) ; Eword (off)
   END"""
new = """   Ebyte (86H) ; Eword (off)"""
assert old in s, "EmBpDisp branch not found -- update this mutation"
open(p, 'w').write(s.replace(old, new))
PYEOF
if rebuild_compiler; then
    if python3 tests/check_framedisp.py >/dev/null 2>&1; then
        echo "  ok: always-disp16 is accepted, so the check pins the rule"
        echo "       and not one particular encoding"
        pass=$((pass + 1))
    else
        echo "  FAIL: check_framedisp rejects a safe, over-long encoding"
        python3 tests/check_framedisp.py 2>&1 | sed 's/^/       /'
        fail=$((fail + 1))
    fi
else
    echo "  FAIL: the compiler would not rebuild with always-disp16"
    fail=$((fail + 1))
fi
cp "$SAVED_C" Compiler.mod

if rebuild_compiler; then
    if python3 tests/check_framedisp.py >/dev/null 2>&1; then
        echo "  ok: check_framedisp passes on the restored source"
        pass=$((pass + 1))
    else
        echo "NOT RESTORED: check_framedisp is red after restoring Compiler.mod"
        python3 tests/check_framedisp.py 2>&1 | sed 's/^/       /'
        fail=$((fail + 1))
    fi
else
    echo "NOT RESTORED: the compiler would not rebuild"
    fail=$((fail + 1))
fi

# --- the region check_framedisp sweeps, and the one file it comes from -----
echo
echo "== the image layout, measured once (tests/comimage.py)"
# check_framedisp.py decodes instructions over a REGION of the image, and a
# wrong region does not stop it: the byte patterns it searches for are in the
# image wherever they happen to be, and a decode that starts mid-instruction
# reached the same offsets anyway.  That is how RT_SZ = 391 - a literal that had
# drifted from the runtime it described - went on reporting PASS while its sweep
# began inside the runtime, part way through an instruction.  Nothing in the
# check could see where its own region began.
#
# So the region is two independent readings of the file - where the entry jump
# says execution starts, and where the program header says the runtime ends -
# and the two must AGREE or the region is undetermined.  Both readings, and the
# layout itself, live in tests/comimage.py, which is also where comtest.py and
# the independent checker in run_com_tests.sh now get the copy each of them
# used to keep: a duplicated constant that has silently drifted is not an
# independent check, it is a second source of truth that lies, and it lies in
# the direction of looking like the code under test is broken.
#
# None of this needs a rebuild - comimage.py is Python the checkers import -
# so these cases are cheap, and they turn the READERS red one at a time.

# 1. entry_target answers with the literal the region used to be built from.
#    The two readings now disagree, and there is no measured region to sweep.
python3 - <<'PYEOF'
p = 'tests/comimage.py'
s = open(p).read()
old = '    return ENT_SZ + int.from_bytes(d[1:ENT_SZ], "little", signed=True)'
new = '    return ENT_SZ + 391                        # the literal RT_SZ was'
assert s.count(old) == 1, "entry_target's return not found -- update this mutation"
open(p, 'w').write(s.replace(old, new))
PYEOF
expect_red "a sweep region that is not where execution starts is rejected" \
    "neither region is swept" python3 tests/check_framedisp.py
cp "$SAVED_CI" tests/comimage.py

# 2 and 3. The header's own equation, one byte out, so the header cannot be
#    located at all - the case where a checker that fell back on a remembered
#    offset would report a confident number instead.  Two readers, two
#    different sentences: each must say in its own words that it does not know
#    where the program's code begins.
python3 - <<'PYEOF'
p = 'tests/comimage.py'
s = open(p).read()
old = '        if w(HDR_DS_WORD) != off + 0x1000 + LOAD_BIAS:    # hdrDS'
new = '        if w(HDR_DS_WORD) != off + 0x1000 + LOAD_BIAS + 1:  # hdrDS'
assert s.count(old) == 1, "the hdrDS equation not found -- update this mutation"
open(p, 'w').write(s.replace(old, new))
PYEOF
expect_red "check_framedisp reports a header it cannot locate" \
    "no program header found" python3 tests/check_framedisp.py
expect_red "check_8086 reports the same unlocatable header" \
    "could not locate the code region" python3 tests/check_8086.py
cp "$SAVED_CI" tests/comimage.py

# 4-7. The single copy itself, now that there is one.  A second find_header,
#      a second HDR_SZ, a caller that reaches the helper through another
#      consumer instead of through comimage, and - the case that exists
#      because of rt_exec.py's "matched nothing" lesson - the definition the
#      audit counts.  A rule that matches nothing looks exactly like a rule
#      that passes, so a deleted definition must be a red report rather than a
#      silent green one.
python3 - <<'PYEOF'
p = 'tests/comtest.py'
s = open(p).read()
old = 'def check_com(path, expected_src_len):'
new = ('def find_header(d):\n'
       '    return None                    # a second copy of the helper\n'
       '\n\n'
       'def check_com(path, expected_src_len):')
assert s.count(old) == 1, "check_com not found -- update this mutation"
open(p, 'w').write(s.replace(old, new))
PYEOF
expect_red "the audit reports a find_header copied out of comimage" \
    "comtest.py defines its own find_header" python3 tests/check_comimage.py
cp "$SAVED_T" tests/comtest.py

python3 - <<'PYEOF'
p = 'tests/comtest.py'
s = open(p).read()
old = 'from comimage import (ENT_SZ, HDR_SZ, LOAD_BIAS, HEAD, HDR_DS_WORD,'
new = ('HDR_SZ = 16                   # a second copy of the layout\n'
       'from comimage import (ENT_SZ, HDR_SZ, LOAD_BIAS, HEAD, HDR_DS_WORD,')
assert s.count(old) == 1, "the comimage import not found -- update this mutation"
open(p, 'w').write(s.replace(old, new))
PYEOF
expect_red "the audit reports a layout constant defined elsewhere" \
    "comtest.py defines its own HDR_SZ" python3 tests/check_comimage.py
cp "$SAVED_T" tests/comtest.py

python3 - <<'PYEOF'
p = 'tests/comimage.py'
s = open(p).read()
old = 'def find_header(d):'
new = 'def _find_header(d):                # the definition is gone'
assert s.count(old) == 1, "def find_header not found -- update this mutation"
open(p, 'w').write(s.replace(old, new))
PYEOF
expect_red "the audit counts its own subject rather than matching nothing" \
    "defines find_header 0 times" python3 tests/check_comimage.py
cp "$SAVED_CI" tests/comimage.py

# 7. The import dropped while the CALLS stay - the shape a refactor makes when
#    one reader starts taking the helper from another reader instead.  It would
#    NameError the moment comtest ran, but the audit says so first, and in the
#    vocabulary of the layout rather than in the vocabulary of Python: the
#    failure this is guarding is "two sources of truth", not "undefined name".
python3 - <<'PYEOF'
p = 'tests/comtest.py'
s = open(p).read()
old = ('from comimage import (ENT_SZ, HDR_SZ, LOAD_BIAS, HEAD, HDR_DS_WORD,\n'
       '                      HDR_HEAP_WORD, find_header)')
new = '# MUTATION: the shared import is gone, but the calls remain\n'
assert s.count(old) == 1, "the comimage import not found -- update this mutation"
open(p, 'w').write(s.replace(old, new))
PYEOF
expect_red "the audit reports a caller that does not take it from comimage" \
    "comtest.py calls find_header without importing it from comimage" \
    python3 tests/check_comimage.py
cp "$SAVED_T" tests/comtest.py

# Green again, on all three readers at once: the case above mutated the one
# file all of them import, so restoring it has to satisfy each of them, not
# just the one that was last run.
for c in tests/check_framedisp.py tests/check_comimage.py tests/check_8086.py; do
    if python3 "$c" >/dev/null 2>&1; then
        echo "  ok: $c passes on the restored sources"
        pass=$((pass + 1))
    else
        echo "NOT RESTORED: $c is red after restoring comimage.py and comtest.py"
        python3 "$c" 2>&1 | tail -3 | sed 's/^/       /'
        fail=$((fail + 1))
    fi
done

# --- 3. the operator bugs the nine dead fixtures were hiding ------------
echo
echo "== the bugs the never-executed fixtures were hiding"
echo
# A different KIND of case from everything above.  The others break the code
# and assert a byte-level check notices; these break the code and assert a
# BEHAVIOURAL check notices, which is the only kind that could have found them.
# Each of the original four shipped with a green compile matrix, a passing .COM
# layout check, a passing golden and a passing emitter audit:
#
#   OpMul = 1        `a * b` emitted ADD AX,CX.  `*' and `+' both numbered
#                    their operator 1, and BinOpEmit cannot see which
#                    precedence level called it, so every multiplication
#                    dispatched to the addition.  The constant-folding arm was
#                    correct, which is why `n * n' with n a CONST was right and
#                    `a * a' with a a variable was not -- and t08_const is the
#                    only fixture that ever multiplied.
#   9Dh / 9FH        `>' got SETGE and `>=' got SETG: swapped, one letter
#                    apart in the mnemonic.  Only a==b could see it.
#   JNZ -> body      REPEAT..UNTIL looped back while the condition was TRUE,
#                    which is WHILE, so the body ran once and stopped.
#
# They are mutated back to the original defect and run_com_exec.py must go red
# on the exact fixture that pins the behaviour.  The fourth (HideLocals) is a
# scoping bug rather than an operator bug; it was hiding in the same place.
#
# Two joined them when the operand-lifetime and `not' fixes landed, each run
# red by hand before it was written down here, and each with the same property
# as the four above -- green everywhere except execution:
#
#   SaveLeft's park `(p > q) or (q > p)' evaluated `(q > p) or (q > p)'.  A
#                    kind-2 value exists only in AX, and the right operand's
#                    code is emitted before the two are ever brought together,
#                    so without the push the left one is simply gone.
#   neglevel's split `not' on a boolean emitted the INTEGER `not', which left
#                    0FFFEh where a boolean belongs, and wrbool reads anything
#                    non-zero as TRUE -- so `not (a = a)' answered TRUE and so
#                    did every other `not'.
mutate_compiler () {   # reuse mutate's verified-change discipline on Compiler.mod
    mf=Compiler.mod
    msed=$1
    cp "$SAVED_C" ../tmp/nonvacuity.mut2.bak
    sed -i "$msed" "$mf"
    if cmp -s "$mf" ../tmp/nonvacuity.mut2.bak; then
        echo "  BROKEN CASE: the mutation did not change Compiler.mod"
        echo "       sed: $msed"
        echo "       the named code has probably been renamed or reformatted -"
        echo "       fix this case, it is asserting nothing"
        fail=$((fail + 1))
        return 1
    fi
    return 0
}

# The SETcc swap and the HideLocals removal are done in python rather than
# with sed: both need to match source text containing `*` and `(` in a way that
# is tedious and fragile as a regex, and a case whose only failure mode is a
# malformed sed is a case that silently asserts nothing.
#
# And a python helper fails in a way sed does not: a syntax error in the helper
# is a non-zero exit, `if mutate_foo; then` is simply false, and the case is
# SKIPPED -- with no failure counted and nothing on stdout but whatever python
# printed.  That is how the SETcc case spent its first run: an apostrophe in an
# assert message ("the `>' arm") closed the string early, python died, the
# compiler was never broken, and the suite still reported 0 failed.  A skipped
# case and a passing case look the same in the total.  So each helper below
# fails LOUDLY: a non-zero exit from python is reported as a BROKEN CASE and
# counted, never swallowed.
#
# Each one also counts its targets before replacing.  `assert s != before' only
# says the file changed; with two edits it would pass if just one of them
# landed, and with two identical HideLocals call sites it would happily delete
# the wrong one -- still a changed file, still a working compiler, still green
# for the wrong reason.
mutate_cc_swap () {
    if python3 - <<'PYX'
p = 'Compiler.mod'
s = open(p).read()
GT = 'EmSetcc (9FH) ;          (* >  SETG  *)'    # the greater-than arm
GE = 'EmSetcc (9DH) ;          (* >= SETGE *)'    # the greater-equal arm
assert s.count(GT) == 1, 'expected 1 greater-than arm, found %d' % s.count(GT)
assert s.count(GE) == 1, 'expected 1 greater-equal arm, found %d' % s.count(GE)
s = s.replace(GT, GT.replace('9FH', '9DH'))
s = s.replace(GE, GE.replace('9DH', '9FH'))
open(p, 'w').write(s)
PYX
    then
        return 0
    fi
    echo "  BROKEN CASE: the SETcc swap did not apply"
    echo "       the two EmSetcc arms are probably renamed or reformatted -"
    echo "       fix this case, it is asserting nothing"
    fail=$((fail + 1))
    return 1
}

mutate_no_hidelocals () {
    if python3 - <<'PYX'
p = 'Compiler.mod'
s = open(p).read()
# Two HideLocals calls exist.  Only the body-exit one may go: deleting the
# FORWARD one instead would still change the file, still rebuild, and still
# leave t13_proc compiling, so the case would go green for the wrong reason.
HL = '   HideLocals (nestMark) ;         (* parameters and locals stop here *)\n'
assert s.count(HL) == 1, 'expected 1 body-exit HideLocals, found %d' % s.count(HL)
open(p, 'w').write(s.replace(HL, ''))
PYX
    then
        return 0
    fi
    echo "  BROKEN CASE: HideLocals was not removed"
    echo "       the call or its comment has probably been reformatted -"
    echo "       fix this case, it is asserting nothing"
    fail=$((fail + 1))
    return 1
}

cp "$SAVED_C" Compiler.mod
if mutate_compiler 's|^         op := OpMul ; DropCh|         op := OpAdd ; DropCh|'; then
    if rebuild_compiler; then
        expect_red "execution catches '*' emitting an ADD (t08_const, n*n)" \
            "t08_const" python3 tests/run_com_exec.py t08_const
        # t08 only ever multiplied two CONSTANTS, which is the one path that was
        # never wrong, because BinOpEmit folds it.  So the case above is close
        # to vacuous: it proves the mutation changed the binary, not that the
        # emitted multiply is covered.  The check that matters needs a
        # VARIABLE operand, and no shipped fixture has one -- which is why the
        # bug survived at all.  So this writes a throwaway fixture that
        # multiplies a variable, runs it, and asserts the multiply is right.
        # The fixture is deleted afterwards; it is here to close the coverage
        # hole, not to become a permanent test (that is what a real fixture
        # with a `*' in it would be for).
        cat > tests/fixtures/zzmul.pas <<'ZZEOF'
program zzmul;
var a : integer ;
begin
  a := 7 ;
  writeln (a * 6)
end.
ZZEOF
        printf '42\r\n' > tests/fixtures/zzmul.out
        expect_red "execution catches '*' on a VARIABLE (the unfolded path)" \
            "zzmul" python3 tests/run_com_exec.py zzmul
        rm -f tests/fixtures/zzmul.pas tests/fixtures/zzmul.out
    else
        echo "  FAIL: the compiler would not rebuild with OpAdd for '*'"
        fail=$((fail + 1))
    fi
fi
cp "$SAVED_C" Compiler.mod

cp "$SAVED_C" Compiler.mod
if mutate_cc_swap; then
    if rebuild_compiler; then
        expect_red "execution catches '>' and '>=' swapped (t09_if)" \
            "t09_if" python3 tests/run_com_exec.py t09_if
    else
        echo "  FAIL: the compiler would not rebuild with the SETcc swap"
        fail=$((fail + 1))
    fi
fi
cp "$SAVED_C" Compiler.mod

cp "$SAVED_C" Compiler.mod
if mutate_compiler 's|      DropC (EmJcc (84H, L1)) ;        (\* JZ -> body again \*)|      zj := EmJcc (85H, L1) ;|'; then
    if rebuild_compiler; then
        expect_red "execution catches REPEAT..UNTIL exiting after one pass (t12)" \
            "t12_repeat" python3 tests/run_com_exec.py t12_repeat
    else
        echo "  FAIL: the compiler would not rebuild with the JNZ repeat"
        fail=$((fail + 1))
    fi
fi
cp "$SAVED_C" Compiler.mod

# The fourth: sibling procedures shared one parameter namespace, because a
# finished procedure's symbols were left at a level Search still accepts.
# Removing HideLocals puts two procedures' `a : integer' back in collision.
cp "$SAVED_C" Compiler.mod
if mutate_no_hidelocals; then
    if rebuild_compiler; then
        expect_red "a duplicate parameter in two procedures is a compile error again" \
            "ERROR 41" python3 tests/run_com_exec.py t13_proc
    else
        echo "  FAIL: the compiler would not rebuild without HideLocals"
        fail=$((fail + 1))
    fi
else
    echo "  FAIL: could not remove HideLocals to test the scoping fix"
    fail=$((fail + 1))
fi
# M6: the LEFT operand, parked for the right one.  kind 2 means "this value
# exists in AX and nowhere else", and the right-hand operand's code is emitted
# between recognizing the operator and using both operands - so without
# SaveLeft's push the left value is overwritten before it is ever read, and
# LoadPair's kind-4 shape can never trigger.  `(p > q) or (q > p)' then
# evaluates `(q > p) or (q > p)'.  Every byte, every size and the whole compile
# matrix stay green: nothing is malformed, the value is simply the wrong one.
cp "$SAVED_C" Compiler.mod
if python3 - <<'PYX'
p = 'Compiler.mod'
s = open(p).read()
old = """BEGIN
   IF left.kind = 2 THEN
      EmPushAx () ;
      left.kind := 4                      (* 4 = on the top of the stack *)
   END
END SaveLeft ;"""
new = """BEGIN
   (* MUTATION: the park is unreachable, so nothing survives the parse *)
   IF left.kind = 99 THEN
      EmPushAx () ;
      left.kind := 4
   END
END SaveLeft ;"""
assert s.count(old) == 1, 'SaveLeft body found %d times -- update this mutation' % s.count(old)
open(p, 'w').write(s.replace(old, new))
PYX
then
    if rebuild_compiler; then
        expect_red "execution catches a left operand clobbered by the right one" \
            "t34_arith" python3 tests/run_com_exec.py t34_arith
    else
        echo "  FAIL: the compiler would not rebuild without SaveLeft's push"
        fail=$((fail + 1))
    fi
else
    echo "  BROKEN CASE: the SaveLeft mutation did not apply"
    fail=$((fail + 1))
fi
cp "$SAVED_C" Compiler.mod

# M7: the type split in TPSRC9's neglevel, wrong half.  A boolean NOT lowered
# as the INTEGER one leaves 0FFFEh/0FFFFh, and wrbool tests [BP+4] <> 0 - so
# `not (a = a)' answers TRUE, and so does every other `not'.  One instruction,
# same length, same sizes, and the compile matrix cannot see it because both
# halves emit well-formed code for a type the parser already accepted.
cp "$SAVED_C" Compiler.mod
if python3 - <<'PYX'
p = 'Compiler.mod'
s = open(p).read()
old = """      IF r.cls = TBool THEN
         LoadAtom (r) ;
         EmXorAl01 () ;"""
new = """      IF r.cls = TBool THEN
         LoadAtom (r) ;
         EmNotAx () ;                      (* MUTATION: integer NOT on a boolean *)"""
assert s.count(old) == 1, 'the TBool arm of ParseNeg found %d times -- update this mutation' % s.count(old)
open(p, 'w').write(s.replace(old, new))
PYX
then
    if rebuild_compiler; then
        expect_red "execution catches a boolean NOT lowered as an integer one" \
            "t35_not" python3 tests/run_com_exec.py t35_not
    else
        echo "  FAIL: the compiler would not rebuild with EmNotAx for a boolean"
        fail=$((fail + 1))
    fi
else
    echo "  BROKEN CASE: the neglevel type-split mutation did not apply"
    fail=$((fail + 1))
fi
cp "$SAVED_C" Compiler.mod

if rebuild_compiler; then
    if python3 tests/run_com_exec.py >/dev/null 2>&1; then
        # No count: the matrix grows every time a fixture is added, and a
        # number here would be right only until the next one.
        echo "  ok: every executed fixture passes on the restored compiler"
        pass=$((pass + 1))
    else
        echo "NOT RESTORED: run_com_exec.py is red after restoring Compiler.mod"
        python3 tests/run_com_exec.py 2>&1 | grep -i fail | head -3 | sed 's/^/       /'
        fail=$((fail + 1))
    fi
else
    echo "NOT RESTORED: the compiler would not rebuild"
    fail=$((fail + 1))
fi

echo
echo "== the second execution oracle (run_exec86.py)"
# Everything above runs the image under qemu-system-i386.  This one runs the
# SAME image inside shell/Exec86.mod, this project's own in-process 8086
# interpreter, and requires its output to agree with BOTH the hand-derived .out
# and qemu, byte for byte.  Two execution checks that could only ever agree
# with each other would be one check: the point of this one is that it was
# written against the 8086's own reference rather than against qemu, whose
# lowest CPU model is a 486 and whose `0F 84' is an ordinary JZ.
#
# So the only mutation worth writing here is one qemu cannot make at all - the
# interpreter's own reading of the machine.  Cond is that reading: nibble 4 is
# JE, and inverting it swaps the two arms of every `=' in every program.  The
# emitted bytes, the sizes and the compile matrix are untouched, because
# nothing is emitted here - the fault is in who interprets it.
#
# Exec86.mod is recompiled explicitly rather than left to the harness:
# ensure_exec86run() notices the source changed and RELINKS, but does not
# recompile it, so a mutated source with a stale object would link the good
# interpreter straight back in and stay green - which is the trap its own
# docstring records, and the reason this case compiles first.
#
# SAVED_E itself was taken at the top of this file, next to the EXIT trap that
# puts it back.

if python3 - <<'PYX'
p = 'Exec86.mod'
s = open(p).read()
old = "| 4H  : r := ZF"
new = "| 4H  : r := NOT ZF                    (* MUTATION: JE inverted *)"
assert s.count(old) == 1, 'the JE arm of Cond found %d times -- update this mutation' % s.count(old)
open(p, 'w').write(s.replace(old, new))
PYX
then
    if $GM2 -fiso -c Exec86.mod >/dev/null 2>&1; then
        expect_red "the interpreter's own oracle catches JE inverted" \
            "t33_cmpops" python3 tests/run_exec86.py t33_cmpops
    else
        echo "  FAIL: Exec86.mod would not compile with JE inverted"
        fail=$((fail + 1))
    fi
else
    echo "  BROKEN CASE: the Cond JE mutation did not apply"
    fail=$((fail + 1))
fi
cp "$SAVED_E" Exec86.mod

# And the check on the RESTORED interpreter, because a green above could also
# come from a mutation that never took and an object left mutated by a run
# that died in between.
if $GM2 -fiso -c Exec86.mod >/dev/null 2>&1; then
    if python3 tests/run_exec86.py >/dev/null 2>&1; then
        echo "  ok: run_exec86 green on the restored interpreter"
        pass=$((pass + 1))
    else
        echo "NOT RESTORED: run_exec86.py is red after restoring Exec86.mod"
        python3 tests/run_exec86.py 2>&1 | grep -i "fail" | head -3 | sed 's/^/       /'
        fail=$((fail + 1))
    fi
else
    echo "NOT RESTORED: Exec86.mod would not recompile"
    fail=$((fail + 1))
fi

echo
echo "== the R key: compile, poke, run, report (runtest.py)"
# The only check in the project that exercises CmdRun.  Everything above looks
# at bytes, or at what qemu says the image does; this one drives the shell
# through a pty the way a person would, presses R, and compares the GUEST's
# output against the fixture's hand-derived .out - so it fails on things no
# byte check can see and no file records either, because R writes no file at
# all (which is itself asserted).
#
# The mutation is the poke loop's count.  With n = 0 nothing is copied into
# the interpreter, loadHi stays where Clear86 left it - 0100h - and Run86
# faults on its very first step, "execution left the loaded image", before the
# guest has executed a single instruction.  That is fast and deterministic, which
# matters: the obvious alternative (poking the image somewhere else) leaves the
# machine executing zeros and buys a step-limit timeout instead of a finding.
cp "$SAVED_SH" Shell.mod
if python3 - <<'PYX'
p = 'Shell.mod'
s = open(p).read()
old = "   n := LinkSize () ;"
new = "   n := 0 ;                           (* MUTATION: nothing is poked *)"
assert s.count(old) == 1, 'the poke count in CmdRun found %d times -- update this mutation' % s.count(old)
open(p, 'w').write(s.replace(old, new))
PYX
then
    if make > ../tmp/nonvacuity.make.log 2>&1; then
        expect_red "the R check catches an image that was never poked" \
            "AH=4Ch exit" python3 tests/runtest.py
    else
        echo "  FAIL: the shell would not rebuild with the poke loop neutered"
        tail -5 ../tmp/nonvacuity.make.log | sed 's/^/       /'
        fail=$((fail + 1))
    fi
else
    echo "  BROKEN CASE: the CmdRun poke mutation did not apply"
    fail=$((fail + 1))
fi
cp "$SAVED_SH" Shell.mod

if make > ../tmp/nonvacuity.make.log 2>&1; then
    if python3 tests/runtest.py >/dev/null 2>&1; then
        echo "  ok: runtest green on the restored shell"
        pass=$((pass + 1))
    else
        echo "NOT RESTORED: runtest.py is red after restoring Shell.mod"
        python3 tests/runtest.py 2>&1 | grep -i "fail" | head -3 | sed 's/^/       /'
        fail=$((fail + 1))
    fi
else
    echo "NOT RESTORED: the shell would not rebuild"
    tail -5 ../tmp/nonvacuity.make.log | sed 's/^/       /'
    fail=$((fail + 1))
fi

echo
echo "== 8086 legality of the conditional lowering (check_8086.py)"
# This whole section exists because of a fault that every other check in the
# project was blind to, and being blunt about WHY is the point of listing it.
#
# EmJcc and EmSetcc emitted `0F 8x rel16' and `0F 9x rel8'.  `0F' is a
# 386-and-later opcode prefix; the 8086 has none.  So EVERY conditional branch
# and EVERY comparison in EVERY compiled program was an illegal instruction on
# the machine this compiler targets.  And all of the following were green while
# it was: the compile matrix, the .COM layout checker, the runtime golden, the
# emitter audit, and the whole execution suite answering correctly under qemu.
#
# Two reasons, and the second is the one worth keeping:
#   1. qemu-system-i386 has no 8086 model.  Its lowest is 486, where `0F 84' is
#      a perfectly ordinary JZ.  So the execution oracle CANNOT see this class
#      of fault, ever.
#   2. FCML is this project's INDEPENDENT disassembler, and FCML's -m16 mode is
#      a 386.  The one tool whose job is to say "this is not a real instruction"
#      was architecturally guaranteed to agree with the bug.
#
# So a check was needed that asks the question nothing else was asking.  The
# five mutations below are the mutations that check has to catch, and each was
# run by hand and confirmed red BEFORE this section existed.
#
# M4 and M5 are the interesting pair.  M4 is a fault this project actually
# introduced while fixing the above: EmJcc jumps TO its target, but the 8086
# shape steps OVER a 3-byte EJMP, so the naive port inverts every conditional
# in every program.  It was caught by execution, not by any byte check.  M5 is
# the partial version -- the inversion dropped for the IF and CASE sites only,
# kept for FOR -- and MEASURING that is what produced clause H, because the
# check was blind to M5 as first written: IF declares nibble 4, CASE declares
# 5, they negate into each other, and both are declared, so nothing complained
# while every conditional in every program took the wrong path.  FOR declares
# C and F, whose negations D and E are declared for nothing at all, so the
# whole-suite version was caught by luck.
#
# Note these need `rebuild_compiler' only to refresh comtest; check_8086.py
# relinks the fixtures itself.  Getting that wrong is how this section's first
# draft reported a mutation as VACUOUS: the build was skipped, a stale comtest
# ran the GOOD compiler, and the check passed.  A helper that exits non-zero
# makes `if mutate_x; then' false and the case is silently skipped, which looks
# exactly like a pass -- so rebuild_compiler is checked, not assumed.

# M1: the original defect, restored verbatim.
cp "$SAVED_C" Compiler.mod
if python3 - <<'PYX'
p = 'Compiler.mod'
s = open(p).read()
old = """   Ebyte (JccShortInv (cc)) ;        (* Jcc_s, taken when cc does NOT hold *)
   Ebyte (03H) ;                     (* rel8: step over the 3-byte EJMP *)
   RETURN EmJmpNear (target)         (* target = 0 => forward, see above *)"""
new = """   Ebyte (0FH) ; Ebyte (cc) ; Eword (0) ;
   IF target = 0 THEN
      RETURN nPatch
   END ;
   RETURN 0"""
assert s.count(old) == 1, 'EmJcc body found %d times -- update this mutation' % s.count(old)
open(p, 'w').write(s.replace(old, new))
PYX
then
    if rebuild_compiler; then
        expect_red "check_8086 catches a 0F 8x branch (the original bug)" \
            "0F 84" python3 tests/check_8086.py
    else
        echo "  FAIL: the compiler would not rebuild with the 0F branch"
        fail=$((fail + 1))
    fi
else
    echo "  BROKEN CASE: the 0F branch mutation did not apply"
    fail=$((fail + 1))
fi
cp "$SAVED_C" Compiler.mod

# M2: the other original defect, SETcc plus the MOV AH,0 it needed.
cp "$SAVED_C" Compiler.mod
if python3 - <<'PYX'
p = 'Compiler.mod'
s = open(p).read()
old = """   Ebyte (0B8H) ; Eword (1) ;                           (* MOV AX,#0001 *)
   Ebyte (JccShort (cc)) ;            (* taken when the comparison HOLDS *)
   Ebyte (01H) ;                     (* rel8: step over the DEC AX *)
   Ebyte (48H)                                          (* DEC AX *)"""
new = """   Ebyte (0FH) ; Ebyte (cc) ; Ebyte (0C0H) ;
   EmMovAh0 ()"""
assert s.count(old) == 1, 'EmSetcc body found %d times -- update this mutation' % s.count(old)
open(p, 'w').write(s.replace(old, new))
PYX
then
    if rebuild_compiler; then
        expect_red "check_8086 catches a 0F 9x SETcc (the original bug)" \
            "0F 9F C0" python3 tests/check_8086.py
    else
        echo "  FAIL: the compiler would not rebuild with the 0F SETcc"
        fail=$((fail + 1))
    fi
else
    echo "  BROKEN CASE: the 0F SETcc mutation did not apply"
    fail=$((fail + 1))
fi
cp "$SAVED_C" Compiler.mod

# M4: the polarity inversion, everywhere.  Note the expected text is clause H's,
# not a shape complaint: every byte here is a legal 8086 shape, which is the
# entire reason a separate clause had to be written.
cp "$SAVED_C" Compiler.mod
if python3 - <<'PYX'
p = 'Compiler.mod'
s = open(p).read()
old = '   Ebyte (JccShortInv (cc)) ;        (* Jcc_s, taken when cc does NOT hold *)'
new = '   Ebyte (JccShort (cc)) ;           (* MUTATION: inversion dropped *)'
assert s.count(old) == 1, 'EmJcc Ebyte found %d times -- update this mutation' % s.count(old)
open(p, 'w').write(s.replace(old, new))
PYX
then
    if rebuild_compiler; then
        expect_red "clause H catches the branch polarity inverted everywhere" \
            "but reading the source says" python3 tests/check_8086.py
    else
        echo "  FAIL: the compiler would not rebuild with the inversion dropped"
        fail=$((fail + 1))
    fi
else
    echo "  BROKEN CASE: the polarity mutation did not apply"
    fail=$((fail + 1))
fi
cp "$SAVED_C" Compiler.mod

# M5: the same inversion dropped for the IF and CASE sites ONLY.  This one is
# the reason clause H exists: it was run by hand and the check stayed GREEN,
# and FOR's C/F nibbles were the only reason the whole-suite version (M4)
# happened to be caught.
cp "$SAVED_C" Compiler.mod
if python3 - <<'PYX'
p = 'Compiler.mod'
s = open(p).read()
old = '   Ebyte (JccShortInv (cc)) ;        (* Jcc_s, taken when cc does NOT hold *)'
new = """   IF (cc = 84H) OR (cc = 85H) THEN
      Ebyte (JccShort (cc))            (* MUTATION: no inversion here *)
   ELSE
      Ebyte (JccShortInv (cc))
   END ;"""
assert s.count(old) == 1, 'EmJcc Ebyte found %d times -- update this mutation' % s.count(old)
open(p, 'w').write(s.replace(old, new))
PYX
then
    if rebuild_compiler; then
        expect_red "clause H catches the inversion dropped for IF and CASE only" \
            "t22_case" python3 tests/check_8086.py
    else
        echo "  FAIL: the compiler would not rebuild with the partial inversion"
        fail=$((fail + 1))
    fi
else
    echo "  BROKEN CASE: the partial-inversion mutation did not apply"
    fail=$((fail + 1))
fi
cp "$SAVED_C" Compiler.mod

# And the check on the RESTORED compiler, so a green above cannot come from a
# mutation that failed to take and left the real defect in place.
if rebuild_compiler; then
    if python3 tests/check_8086.py >/dev/null 2>&1; then
        echo "  ok: check_8086 green on the restored compiler"
        pass=$((pass + 1))
    else
        echo "NOT RESTORED: check_8086.py is red after restoring Compiler.mod"
        python3 tests/check_8086.py 2>&1 | grep -m3 -- '  - ' | sed 's/^/       /'
        fail=$((fail + 1))
    fi
else
    echo "NOT RESTORED: the compiler would not rebuild"
    fail=$((fail + 1))
fi

echo
echo "== the emitter-name audit of Compiler.mod (audit_helpers.py)"
# These need no rebuild: the audit reads the SOURCE, not the built object, so
# they are the cheapest cases here and they cover the module the audit used
# not to look at at all.  That is the point of the section: the audit reported
# "every helper agrees with its name" for a module it had never examined, and
# EmXchgAxCx was `93` (XCHG BX,AX) under a name that says XCHG AX,CX for the
# whole life of the project.  Two of these five are for faults that were real.
SAVED_C2=../tmp/nonvacuity.Compiler.mod.2
SAVED_R2=../tmp/nonvacuity.Runtime.mod.2
cp Compiler.mod "$SAVED_C2" || exit 1
cp Runtime.mod "$SAVED_R2" || exit 1
restore_audit_sources () {
    cp "$SAVED_C2" Compiler.mod
    cp "$SAVED_R2" Runtime.mod
}

AUD="python3 tests/audit_helpers.py"

# 1. THE fault.  91h is XCHG AX,CX; 93h is XCHG BX,AX.  Both are one byte, so
#    the compile matrix never moved and the byte counts never moved.
cp "$SAVED_C2" Compiler.mod
mutate Compiler.mod 's|^   Ebyte (91H)$|   Ebyte (93H)|'
expect_red "audit catches XchgAxCx emitting XCHG BX,AX" \
    "exchanges Ax and Bx" $AUD
restore_audit_sources

# 2. the coverage check itself.  A parameter list that find_helpers does not
#    accept is exactly how the real emitter was missed, and the inventory is
#    scanned separately on purpose so this can be caught.  Without the
#    independent scan this case is silent, because both lists would come from
#    the same parser and agree that the helper does not exist.
cp "$SAVED_C2" Compiler.mod
mutate Compiler.mod 's|^PROCEDURE EmXchgAxCx () ;$|PROCEDURE EmXchgAxCx (why : CARDINAL) ;|'
expect_red "audit reports an emitter it cannot reach, rather than skipping it" \
    "never examined it" $AUD
restore_audit_sources

# 3. EmXchgAxDx was named EmMoveAxDx, which said MOV where the bytes say XCHG.
#    93h here is XCHG AX,BX - one letter away, the exact class of mistake the
#    name is supposed to make impossible.
cp "$SAVED_C2" Compiler.mod
mutate Compiler.mod 's|^   Ebyte (92H)$|   Ebyte (93H)|'
expect_red "audit catches XchgAxDx emitting XCHG BX,AX" \
    "XchgAxDx" $AUD
restore_audit_sources

# 4. CmpArgW0's [BP+2] written as the 386 SIB form, which decodes on a 8086 as
#    [SI+24h].  A real bug: the runtime was clearing the wrong memory.
cp "$SAVED_R2" Runtime.mod
mutate Runtime.mod 's|   B (83H) ; B (7EH) ; B (2) ; B (0) ;|   B (83H) ; B (7CH) ; B (24) ; B (0) ; B (0) ;|'
expect_red "audit catches the [SI+24h] encoding of [BP+2]" \
    "memory base is 'si" $AUD
restore_audit_sources

# 5. MovAxSp is POP then PUSH, because MOV AX,[SP] does not exist on an 8086.
#    Dropping the POP leaves the stack one word short - a fault in the shape,
#    not in a byte value.
cp "$SAVED_C2" Compiler.mod
python3 - <<'PYEOF'
p='Compiler.mod'; s=open(p).read()
a="   Ebyte (58H) ;                     (* POP AX  *)\n"
assert s.count(a)==1, "EmMovAxSp POP line not found -- update this mutation"
open(p,'w').write(s.replace(a, ""))
PYEOF
expect_red "audit catches MovAxSp with its POP missing" \
    "MovAxSp" $AUD
restore_audit_sources

# 6. The two-instruction shape: IDIV is CWD then IDIV, and dropping the CWD
#    leaves an un-sign-extended dividend in DX:AX.  Both are still present as
#    a two-step spec, so a missing step has to be visible.
cp "$SAVED_C2" Compiler.mod
mutate Compiler.mod 's|   Ebyte (99H) ; Ebyte (0F7H) ; Ebyte (0F9H)|   Ebyte (0F7H) ; Ebyte (0F9H)|'
expect_red "audit catches IDiv without the CWD that extends the dividend" \
    "IDivAxCx" $AUD
restore_audit_sources

# 7. The byte under EmXorAl01's name, and the row in PATTERNS that was added
#    to admit it.  `34 01' is XOR AL,#01 - the boolean NOT, and the one emitter
#    in Compiler.mod with no ModRM byte at all.  A grammar row nobody has ever
#    seen reject anything cannot be trusted to accept only the truth, so here
#    is the rejection: `34 02' is the same length, decodes just as cleanly, and
#    `not x' would flip bit 1 instead of bit 0.
cp "$SAVED_C2" Compiler.mod
mutate Compiler.mod 's|^   Ebyte (34H) ; Ebyte (01H)|   Ebyte (34H) ; Ebyte (02H)|'
expect_red "audit catches EmXorAl01 emitting xor al,#2 under a name saying #1" \
    "XorAl01" $AUD
restore_audit_sources

# 7b. The other half of that row: the opcode gate.  34h is XOR AL,#imm and
#     35h is XOR AX,#imm - a word, not a byte, under a name that says AL.  The
#     row is pinned to {034h} on purpose, and a pin that has never been asked
#     to hold is a pin.  The report is the one thing the row above cannot
#     produce: with no reading at all, the helper falls out of the grammar.
cp "$SAVED_C2" Compiler.mod
mutate Compiler.mod 's|^   Ebyte (34H) ; Ebyte (01H)|   Ebyte (35H) ; Ebyte (01H)|'
expect_red "the opcode gate rejects XOR AX under an XOR AL name" \
    "no name pattern accepts it" $AUD
restore_audit_sources

if $AUD >/dev/null 2>&1; then
    echo "  ok: the audit passes on both restored sources"
    pass=$((pass + 1))
else
    echo "NOT RESTORED: the audit is red after restoring the sources"
    $AUD 2>&1 | sed 's/^/       /'
    fail=$((fail + 1))
fi

echo
echo "== the BP contract rt_exec.py checks before it starts a machine"
# wrchar and wrbool both borrowed BP to reach their argument -- [SP] is not
# encodable in 16-bit mode -- and neither saved it.  The driver's cursor into
# the case record lives in BP precisely because BP is the one register an entry
# may keep, so "wrchar borrowed it and did not give it back" sent the second
# call to a garbage address, the machine triple-faulted, and the run printed the
# record header twice and hung.  Both the golden and the audit call that shape
# CORRECT: the bytes are well formed, every branch is on a boundary, the size
# is unchanged, and the decode says exactly what it says.  So the rule is now
# checked directly, and these two cases are what make that check more than a
# claim.
#
# rt_exec.py needs the whole runtime rebuilt and then boots 36 machines, so this
# section is the slow one.  The baseline comes first and is asserted: a case
# that mutates a red tree proves nothing.
SAVED_R3=../tmp/nonvacuity.Runtime.mod.3
cp Runtime.mod "$SAVED_R3" || exit 1

if rebuild; then
    if python3 tests/rt_exec.py >/dev/null 2>&1; then
        echo "  ok: baseline - rt_exec.py passes on the unmutated runtime"
        pass=$((pass + 1))
    else
        echo "  FAIL: the baseline is already red, so the cases below prove"
        echo "        nothing - fix the baseline before reading them"
        python3 tests/rt_exec.py 2>&1 | tail -3 | sed 's/^/       /'
        fail=$((fail + 1))
    fi
else
    echo "  FAIL: could not rebuild the runtime for the baseline"
    fail=$((fail + 1))
fi

# 1. THE fault: drop the PUSH, exactly as EmitWrChar was written.  This is the
#    shape the behavioural case found, so the byte-level check must find it too
#    -- a check that only the expensive test can trip is a check that has not
#    been made to earn its place.
cp "$SAVED_R3" Runtime.mod
python3 - <<'PYEOF'
p = 'Runtime.mod'
s = open(p).read()
a = '   M ("wrchar") ;\n   PushBp ; MovBpSp ;\n'
assert s.count(a) == 1, "EmitWrChar prologue not found exactly once"
open(p, 'w').write(s.replace(a, '   M ("wrchar") ;\n   MovBpSp ;\n'))
PYEOF
if rebuild; then
    expect_red "the BP contract catches wrchar borrowing BP unsaved" \
        "borrows BP but does not open with" python3 tests/rt_exec.py
else
    echo "  FAIL: the runtime would not rebuild with the PUSH removed"
    fail=$((fail + 1))
fi
cp "$SAVED_R3" Runtime.mod

# 2. The mirror image: push it and never pop it.  A different one-instruction
#    omission with the same consequence for a caller, and the reason the rule
#    asks for the 5D and not just the 55.
cp "$SAVED_R3" Runtime.mod
mutate Runtime.mod 's|^   MovSpBp ; PopBp ;$|   MovSpBp ;|'
if rebuild; then
    expect_red "the BP contract catches a BP that is pushed and never popped" \
        "pushes BP but never pops it" python3 tests/rt_exec.py
else
    echo "  FAIL: the runtime would not rebuild with the POP removed"
    fail=$((fail + 1))
fi
cp "$SAVED_R3" Runtime.mod

# 3. The scan's other silent failure: a rule that matches nothing looks exactly
#    like a rule that passes.  The pattern the check looks for is changed to one
#    the blob does not contain -- MOV BP,DI, which the runtime has no reason to
#    emit -- so the check has examined thirteen entries and matched nothing and
#    MUST say so rather than report a clean sweep.  This is the general shape of
#    the fault this project keeps making: a check whose SUBJECT has drifted
#    reports a confident answer about the wrong thing.
cp "$SAVED_R3" Runtime.mod
SAVED_X=../tmp/nonvacuity.rt_exec.py
cp tests/rt_exec.py "$SAVED_X" || exit 1
mutate tests/rt_exec.py 's|^MOV_BP_SP = b"\\x8b\\xec" .*$|MOV_BP_SP = b"\\x8b\\xed"               # MOV BP,DI: never emitted|' \
expect_red "a check that matched nothing is a failure, not a pass" \
    "matched nothing" python3 tests/rt_exec.py
cp "$SAVED_X" tests/rt_exec.py

if rebuild; then
    if python3 tests/rt_exec.py >/dev/null 2>&1; then
        echo "  ok: rt_exec.py passes on the restored source"
        pass=$((pass + 1))
    else
        echo "NOT RESTORED: rt_exec.py is red after restoring Runtime.mod"
        python3 tests/rt_exec.py 2>&1 | tail -3 | sed 's/^/       /'
        fail=$((fail + 1))
    fi
else
    echo "NOT RESTORED: the runtime would not rebuild"
    fail=$((fail + 1))
fi

# 4. The GOLDEN and the entry goldens, which are the other half of the same
#    rule.  rt_exec.py states the contract; check_runtime.py pins the bytes.
#    They are separate mechanisms and the case below is what shows the golden
#    one works on its own -- a re-baseline of runtime.golden would otherwise
#    have absorbed the new prologues silently, and the next person to bless it
#    would have no way to know the PUSH and POP were ever missing.
cp "$SAVED_R3" Runtime.mod
python3 - <<'PYEOF'
p = 'Runtime.mod'
s = open(p).read()
a = '   M ("wrchar") ;\n   PushBp ; MovBpSp ;\n   MovAlArg4 ;\n   MovSpBp ; PopBp ;\n'
assert s.count(a) == 1, "EmitWrChar frame not found exactly once"
open(p, 'w').write(s.replace(a, '   M ("wrchar") ;\n   MovBpSp ;\n   MovAlArg2 ;\n   MovSpBp ;\n'))
PYEOF
if rebuild; then
    expect_red "the entry golden catches wrchar without its PUSH BP" \
        "entry wrchar starts 8B EC" python3 tests/check_runtime.py "$DUMP"
else
    echo "  FAIL: the runtime would not rebuild with wrchar's frame removed"
    fail=$((fail + 1))
fi
cp "$SAVED_R3" Runtime.mod

echo
echo "== the .COM layout check, and the runtime size it now measures"
# The checker used to RESTATE the runtime's size as a literal.  It had drifted
# from the runtime it described, so the header was read out of the code stream
# and EVERY linked fixture "failed" on it - a whole file of "failures" that were
# not findings at all.  A duplicated constant that has drifted does not fail
# loudly; it re-reports the same falsehood, in which the real failures hide.
# The size is now MEASURED from the image.
#
# These cases corrupt a real emitted .COM and require the checker to notice.
# They need the images, so they are built once and copied; the checker has a
# --check-only mode for exactly this, because its scratch directory is normally
# deleted on exit and a check that has only ever seen the truth is not a check.
KEEPDIR=../tmp/nonvacuity.com
rm -rf "$KEEPDIR"
TP_COM_KEEP=1 tests/run_com_tests.sh >../tmp/nonvacuity.com.log 2>&1
KEEP=$(sed -n 's/^TP_COM_KEEP=1: images left in //p' \
       ../tmp/nonvacuity.com.log | tail -1)
if [ -z "$KEEP" ] || [ ! -d "$KEEP" ]; then
    echo "  FAIL: could not obtain emitted .COM images for the layout cases"
    fail=$((fail + 1))
else
    COMCHK="tests/run_com_tests.sh --check-only"

    # 0. The baseline.  Every case below is a claim that a specific assertion
    #    turns red, and none of them means anything if the copies of untouched
    #    images already fail.  (The stale RT_SZ produced exactly that: a whole
    #    file of "failures" that were not findings.)  So this is asserted first,
    #    and a failure here is reported as a broken baseline rather than a red
    #    test.
    #
    #    The baseline is also WHERE THE HEADER OFFSET COMES FROM.  Cases 1 and 2
    #    used to carry it as the literal 435 and 439, which were ENT_SZ + the
    #    runtime size at the time -- 432.  Two 4-byte changes to the runtime
    #    later, both cases were editing the wrong bytes: 435 had become four
    #    bytes inside the runtime itself, so case 1 had stopped testing "the
    #    header cannot be found" and had started testing "the checker also
    #    notices you scribbled on the code", and case 2 had become a no-op
    #    that wrote the header where it already was.  Both still went red, for
    #    reasons the messages did not name, which is the whole problem: a
    #    hard-coded offset is a claim about the world that expires silently.
    #
    #    So the offset is read back out of the checker's own report on the
    #    untouched images, and the expected numbers below are ARITHMETIC ON IT.
    #    If the runtime grows again, these cases still test what they say.
    rm -rf "$KEEPDIR"; mkdir -p "$KEEPDIR"
    cp "$KEEP"/*.COM "$KEEP"/raw.txt "$KEEPDIR"/
    BASE_OUT=$($COMCHK "$KEEPDIR" 2>&1)
    if [ $? -eq 0 ]; then
        echo "  ok: baseline - untouched copies of the real images all pass"
        pass=$((pass + 1))
    else
        echo "  FAIL: the baseline is already red, so the cases below prove"
        echo "        nothing - fix the baseline before reading them"
        printf '%s\n' "$BASE_OUT" | grep FAIL | head -3 | sed 's/^/       /'
        fail=$((fail + 1))
    fi
    BASE_RT=$(printf '%s\n' "$BASE_OUT" \
        | sed -n 's/.*measured runtime size: \([0-9][0-9]*\) bytes.*/\1/p' \
        | head -1)
    if [ -z "$BASE_RT" ]; then
        echo "  FAIL: the checker did not report the runtime size it measured,"
        echo "        so the cases below cannot find the header to edit"
        fail=$((fail + 1))
        BASE_RT=0
    fi
    # The layout constants are ENT_SZ 3 and HDR_SZ 16 (tests/comimage.py), so the
    # header sits at hdrOff = 3 + rtSz and the entry jump's displacement must be
    # (hdrOff + 16) - 3.  The wanted number is therefore computed from the header
    # OFFSET, not from rtSz: the two differ by 3, and getting that backwards
    # produces a plausible-looking expectation three bytes out, which is how
    # this case came to expect "want 452" and then be reported as not proving
    # what it said.
    BASE_OFF=$((3 + BASE_RT))
    # Case 2 claims a runtime four bytes LONGER, so the header - and with it the
    # demanded jump target - moves four bytes further on.
    SHIFTED_WANT=$((BASE_OFF + 4 + 16 - 3))
    echo "       (measured runtime size $BASE_RT, header at image offset $BASE_OFF)"

    # 1. Break hdrDS so it no longer ties the header to its own offset.  The
    #    header must become UNFINDABLE and be reported as such - a checker that
    #    fell back to a remembered offset would report a confident number here,
    #    which is the failure mode the measurement was introduced to remove.
    rm -rf "$KEEPDIR"; mkdir -p "$KEEPDIR"
    cp "$KEEP"/*.COM "$KEEP"/raw.txt "$KEEPDIR"/
    python3 - "$KEEPDIR/t01_minimal.COM" "$BASE_OFF" <<'PYEOF'
import sys
p, off = sys.argv[1], int(sys.argv[2])
d = bytearray(open(p, 'rb').read())
d[off + 4:off + 6] = (0x1234).to_bytes(2, 'little')     # hdrDS, no longer self-consistent
open(p, 'wb').write(bytes(d))
PYEOF
    expect_red "a header that cannot be located is reported, not assumed" \
        "no program header found" $COMCHK "$KEEPDIR"

    # 2. A complete, self-consistent header four bytes later, so the measured
    #    runtime size becomes $((BASE_RT + 4)) instead of $BASE_RT.  This is the
    #    positive half of the same check: the derivation must FOLLOW the file,
    #    and the entry jump assertion - expressed in terms of the measurement -
    #    must follow it too, demanding $SHIFTED_WANT rather than $((BASE_OFF + 16 - 3)).
    #
    #    hdrCS is copied from the header already in the file rather than written
    #    as a literal.  It used to be the literal 464+100h, which was the image
    #    length when the image was 720 bytes; four bytes of runtime later it was
    #    464+100h against a 724-byte image, so this case was ALSO failing on
    #    hdrCS, for a reason three lines below the one it was written to test.
    rm -rf "$KEEPDIR"; mkdir -p "$KEEPDIR"
    cp "$KEEP"/*.COM "$KEEP"/raw.txt "$KEEPDIR"/
    python3 - "$KEEPDIR/t01_minimal.COM" "$BASE_OFF" "$SHIFTED_WANT" <<'PYEOF'
import sys
p, off, want = sys.argv[1], int(sys.argv[2]), int(sys.argv[3])
d = bytearray(open(p, 'rb').read())
hdrCS = int.from_bytes(d[off + 2:off + 4], 'little')   # unchanged: still the image end
off += 4
ds = off + 0x1000 + 0x100
w = [1, hdrCS, ds, ds + 4, 0, 0, 0, 0]
for i, x in enumerate(w):
    d[off + 2 * i:off + 2 * i + 2] = x.to_bytes(2, 'little')
open(p, 'wb').write(bytes(d))
PYEOF
    expect_red "the measured runtime size follows the image ($BASE_RT -> $((BASE_RT + 4)))" \
        "want $SHIFTED_WANT" $COMCHK "$KEEPDIR"

    # 3. The entry jump's opcode.  One byte, and the only assertion in the
    #    project that can see where execution STARTS.
    rm -rf "$KEEPDIR"; mkdir -p "$KEEPDIR"
    cp "$KEEP"/*.COM "$KEEP"/raw.txt "$KEEPDIR"/
    python3 - "$KEEPDIR/t01_minimal.COM" <<'PYEOF'
import sys
p = sys.argv[1]
d = bytearray(open(p, 'rb').read())
d[0] = 0xEA
open(p, 'wb').write(bytes(d))
PYEOF
    expect_red "the entry jump must be E9, not a near JMP" \
        "not the E9 of the entry jump" $COMCHK "$KEEPDIR"

    # 4. The entry jump's target, moved one instruction earlier.  A .COM that
    #    lands in the middle of the prologue runs, prints something and exits
    #    cleanly, so no size or structure check can see this.
    rm -rf "$KEEPDIR"; mkdir -p "$KEEPDIR"
    cp "$KEEP"/*.COM "$KEEP"/raw.txt "$KEEPDIR"/
    python3 - "$KEEPDIR/t01_minimal.COM" <<'PYEOF'
import sys
p = sys.argv[1]
d = bytearray(open(p, 'rb').read())
d[1:3] = (100).to_bytes(2, 'little')
open(p, 'wb').write(bytes(d))
PYEOF
    expect_red "the entry jump must land on the first instruction" \
        "entry jump rel16=100" $COMCHK "$KEEPDIR"

    rm -rf "$KEEPDIR"
fi

echo
echo "non-vacuity: $pass ok, $fail failed"
[ "$fail" -eq 0 ]