TP3/shell/tests/nonvacuity.sh

#!/bin/sh
# nonvacuity.sh -- prove the runtime checks can actually fail.
#
# A test that has never been seen red is not a test.  This script breaks the
# runtime on purpose, once per check, and asserts that the check goes red and
# says something useful about the breakage.  Then it restores the source and
# asserts everything is green again.
#
# Each mutation below is a real bug that was in this file at some point, not an
# invented one.  That is the point: these are the mistakes we actually make
# with 16-bit ModRM, so these are the ones the checks have to catch.
#
#   audit_helpers.py   name-versus-decode: catches a wrong ModRM that still
#                      decodes cleanly
#   check_runtime.py   golden:            catches the same thing in the built
#                                           image
#   check_runtime.py   decode sweep:      catches a wrong instruction LENGTH
#   check_runtime.py   branch targets:    catches a wrong fixup
#   check_runtime.py   entry goldens:     catches a broken prologue
#   probe/modrm11.py   the mod=11 table:   catches the ModRM column itself
#                      going wrong, which no amount of decoding will show
#
# The mod=11 cases do not need a rebuild -- they read the probe sources
# directly -- so they are cheap, and they are the ones that matter most: the
# table they guard is the one thing in this project that was wrong in the
# documentation while the code was right, and a table that is wrong in the
# code produces bytes that decode perfectly.
#
# Usage: tests/nonvacuity.sh        (from shell/; leaves Runtime.mod restored)

set -u
cd "$(dirname "$0")/.." || exit 1

GM2=/home/eric/bin/Modula2/Gm2/bin/gm2
SAVED=/tmp/opencode/nonvacuity.Runtime.mod
PROBE=/tmp/opencode/nonvacuity.rtprobe
DUMP=/tmp/opencode/nonvacuity.dump

cp Runtime.mod "$SAVED" || exit 1
trap 'cp "$SAVED" Runtime.mod; "$GM2" -fiso -c Runtime.mod >/dev/null 2>&1' EXIT

pass=0
fail=0

# rebuild <label> -- re-emit the runtime and dump it
rebuild () {
    "$GM2" -fiso -c Runtime.mod >/dev/null 2>&1 || return 1
    "$GM2" -fiso -o "$PROBE" tests/RtProbe.mod Runtime.o Posix.o \
        >/dev/null 2>&1 || return 1
    "$PROBE" > "$DUMP" || return 1
    return 0
}

# expect_red <label> <pattern> <checker-cmd...>
#   <pattern> is a grep the failure output must match, so a check cannot
#   "pass" by failing for some unrelated reason.
expect_red () {
    label=$1
    want=$2
    shift 2
    if out=$("$@" 2>&1); then
        echo "NOT NON-VACUOUS: $label -- the check still passed"
        fail=$((fail + 1))
    elif ! printf '%s\n' "$out" | grep -qi "$want"; then
        echo "WRONG FAILURE: $label -- went red, but not for the stated reason"
        printf '%s\n' "$out" | sed 's/^/       /'
        fail=$((fail + 1))
    else
        echo "  ok: $label"
        printf '%s\n' "$out" | grep -im1 "$want" | sed 's/^/       /'
        pass=$((pass + 1))
    fi
}

echo "== each mutation must turn the named check red"
echo

# --- 1. name-versus-decode -------------------------------------------
# MovSiBx was `89 DC`, which is MOV SP,BX.  Two bytes either way, decodes
# cleanly, and no structural check can see it.
cp "$SAVED" Runtime.mod
sed -i 's|B (0DEH) END MovSiBx|B (0DCH) END MovSiBx|' Runtime.mod
expect_red "audit_helpers catches MovSiBx emitting MOV SP,BX" \
    "MovSiBx" python3 tests/audit_helpers.py

# CmpSiBx had the identical mistake, which is how you know a single fix is
# not enough -- the same misreading was written twice.
cp "$SAVED" Runtime.mod
sed -i 's|B (39H) ; B (0DEH) END CmpSiBx|B (39H) ; B (0DCH) END CmpSiBx|' Runtime.mod
expect_red "audit_helpers catches CmpSiBx emitting CMP SP,BX" \
    "CmpSiBx" python3 tests/audit_helpers.py

# --- 2. golden, and entry goldens ------------------------------------
# MovAlDh was `8A C0` = MOV AL,AL instead of MOV AL,DH.  This is the case
# that motivated runtime.golden: the sweep stayed in sync, every branch
# target stayed on a boundary, and no entry's first bytes moved.
cp "$SAVED" Runtime.mod
sed -i 's|B (0C6H) END MovAlDh|B (0C0H) END MovAlDh|' Runtime.mod
rebuild
expect_red "runtime.golden catches MOV AL,AL" \
    "mov al,al" python3 tests/check_runtime.py "$DUMP"

# initmem opened with the mis-emitted MovSiAx, so its entry golden was the
# thing that noticed the prologue was a no-op.
cp "$SAVED" Runtime.mod
sed -i 's|B (0F0H) END MovSiAx|B (0C0H) END MovSiAx|' Runtime.mod
rebuild
expect_red "check_runtime catches a broken initmem prologue" \
    "mov ax,ax" python3 tests/check_runtime.py "$DUMP"

# --- 3. decode sweep / length ----------------------------------------
# StBxDl was `88 97` = [BX],DL with mod=10, so the instruction needs a
# disp16 and the sweep loses sync two bytes later.
cp "$SAVED" Runtime.mod
sed -i 's|B (88H) ; B (17H) END StBxDl|B (88H) ; B (97H) END StBxDl|' Runtime.mod
rebuild
expect_red "decode sweep catches a mod=10 byte move with no displacement" \
    "585Bh" python3 tests/check_runtime.py "$DUMP"

# --- 4. branch targets ------------------------------------------------
# FixUp measures a rel8 from the end of the instruction, one byte past the
# displacement field.  Drop the +1 and every short branch lands one byte into
# its target, which for a 3-byte instruction means the middle of it.  The
# bytes themselves are all perfectly well formed -- only the fixups are
# wrong -- so this is the one failure mode the golden cannot be expected to
# catch on its own.
cp "$SAVED" Runtime.mod
sed -i 's|rel := (t + 100H - (fix \[i\].place + 1)) MOD 100H|rel := (t + 100H - fix [i].place) MOD 100H|' \
    Runtime.mod
rebuild
expect_red "branch check catches rel8 fixups measured from the wrong byte" \
    "not an instruction boundary" \
    python3 tests/check_runtime.py "$DUMP"

echo
echo "== everything restored and green again"
cp "$SAVED" Runtime.mod
if rebuild; then
    if python3 tests/audit_helpers.py >/dev/null 2>&1 &&
       python3 tests/check_runtime.py "$DUMP" >/dev/null 2>&1; then
        echo "  ok: both checks pass on the restored source"
        pass=$((pass + 1))
    else
        echo "NOT RESTORED: a check is red after restoring Runtime.mod"
        fail=$((fail + 1))
    fi
else
    echo "NOT RESTORED: the runtime would not rebuild"
    fail=$((fail + 1))
fi

echo
echo "== the mod=11 table (probe/modrm11.py)"
# These mutate the probe's own sources, not the runtime, so there is no
# rebuild in the loop.  SAVED_PY / SAVED_S are restored after each case.
SAVED_PY=/tmp/opencode/nonvacuity.modrm11.py
SAVED_S=/tmp/opencode/nonvacuity.modrm11.s
cp tests/probe/modrm11.py "$SAVED_PY" || exit 1
cp tests/probe/modrm11.s "$SAVED_S" || exit 1

M11="python3 tests/probe/modrm11.py"
restore_probe () {
    cp "$SAVED_PY" tests/probe/modrm11.py
    cp "$SAVED_S" tests/probe/modrm11.s
}

# 1. one cell of the table moved
sed -i 's|"Si", "Di"\]$|"Bp", "Di"]|' tests/probe/modrm11.py
expect_red "anchor pins a moved table cell" \
    "anchor ADD SI, 2" $M11
restore_probe

# 2. the table this project actually shipped: AX dropped off the front and a
#    duplicate BX invented at the end, which shifts every code down by one
sed -i 's|^REG = .*$|REG = ["Cx", "Dx", "Bx", "Sp", "Bp", "Si", "Di", "Bx"]|' \
    tests/probe/modrm11.py
expect_red "the table shifted by one (AX dropped, BX duplicated)" \
    "anchor MOV SP, BP" $M11
restore_probe

# 3. the .s edited to contradict the table.  This is the case that shows why
#    the hard-coded EXPECT bytes exist: the assembler encodes the new claim
#    correctly, so comparing the .s against `as` alone can never fail here.
sed -i 's|movw    %sp, %di         # reg 100|movw    %bp, %di         # reg 100|' \
    tests/probe/modrm11.s
expect_red "probe source edited away from the recorded bytes" \
    "expected 89 E7" $M11
restore_probe

# 4. the 8-bit list edited, which is a different table from the word one
sed -i 's|movb    %al, %dl         # 88 C2  ->  DL := AL|movb    %al, %bl         # was DL|' \
    tests/probe/modrm11.s
expect_red "the 8-bit register list edited" \
    "expected 88 C2" $M11
restore_probe

# 5. an anchor's recorded byte corrupted, so the anchor can no longer
#    corroborate itself
sed -i 's|"8B EC", "8B E5"|"8B ED", "8B E5"|' tests/probe/modrm11.py
expect_red "anchor byte no longer matches the emitted code" \
    "expected 8B ED" $M11
restore_probe

if $M11 >/dev/null 2>&1; then
    echo "  ok: modrm11.py passes on the restored probe sources"
    pass=$((pass + 1))
else
    echo "NOT RESTORED: modrm11.py is red after restoring its sources"
    $M11 2>&1 | sed 's/^/       /'
    fail=$((fail + 1))
fi

echo
echo "non-vacuity: $pass ok, $fail failed"
[ "$fail" -eq 0 ]