TP3/shell/tests/run_all.sh

#!/bin/bash
# Run every check.  Non-zero exit if anything fails.
#
# Four suites, in increasing order of "how much could be lying to me":
#
#   1. compile matrix   33 fixtures, verdict + code size + data size ASSERTED
#                       from expected.tsv.  Fast, no pty.
#   2. .COM linker      links every fixture, then re-verifies the bytes with an
#                       independent checker that restates the layout constants
#                       instead of asking the compiler.
#   3. UI error path    pty: W, C on a bad file, editor opens on the error,
#                       Ctrl-K D, Q.  Covers the shell behaviour that only
#                       exists interactively.
#   4. UI success path  pty: Options -> Destination=COM, C, then the .COM is
#                       checked ON DISK.  This is the end-to-end path from
#                       keystroke to artifact.
#
#   5. frame displ      links the two fixtures that touch a stack frame and
#                       asserts the [BP+off] displacement encoding, which no
#                       other check can see: disp8 and disp16 are both
#                       well-formed and both decode cleanly, and only one of
#                       them reads the variable the symbol table named.
#
#   RtProbe             dumps the runtime size and its 14 entry offsets, so a
#                       runtime change that moves an entry is visible here.
#
# Three runtime suites run before all of those, because everything else
# trusts the runtime's bytes:
#
#   audit_helpers       disassembles every one-line emitter in BOTH Runtime.mod
#                       and Compiler.mod and compares the DECODE against the
#                       procedure's NAME.  This is the only check that can see
#                       a wrong ModRM that still decodes cleanly, which is the
#                       mistake this project actually makes: MovSiBx and
#                       CmpSiBx were both `DC` (= MOV SP,BX / CMP SP,BX) for a
#                       long time, and EmXchgAxCx was `93` (XCHG BX,AX) under a
#                       name that says XCHG AX,CX.
#
#                       It sweeps Compiler.mod because that is where the
#                       emitter was.  Auditing one module said "every helper
#                       agrees with its name" about a module it had not
#                       examined, which is why the XCHG fault survived a green
#                       run.  It also cross-checks its own coverage: a
#                       procedure that emits bytes but that the audit never
#                       matched is itself a failure, because a helper that
#                       drops out of the audit when you add a comment is worse
#                       than a helper that is checked and found wanting.
#   check_runtime       sweeps the built runtime's code region with FCML: no
#                       desync, every entry and every branch target on an
#                       instruction boundary, and the whole disassembly equal
#                       to tests/runtime.golden.
#   nonvacuity          breaks the runtime and the compiler in the ways each
#                       named check is meant to catch, and asserts it goes red
#                       for the STATED reason, not merely red.  Slower, so it
#                       is not in the default run: run it when a check's
#                       sensitivity is in question, or after editing a check.
#
# nonvacuity.sh is NOT run by default: it rebuilds the runtime and the compiler
# a dozen times over and it is a meta-check, so it belongs to "is the harness
# honest" reviews rather than to every change.  Run tests/nonvacuity.sh
# directly after touching any of the checks above.  (The count of rebuilds is
# not stated here on purpose: it drifts every time a case is added, and a
# number in a comment that is only right until the next edit is the kind of
# restated constant that hides a fault here.)
#
# Two more checks guard the ModR/M table itself, which is the input every hand
# written emitter in Runtime.mod depends on:
#
#   probe/modrm11.py    asks GNU `as` (.code16) to encode the register moves
#                       and FCML to decode them, and requires the result to
#                       agree with the hard-coded bytes, the four
#                       hand-checking anchors, and the table in Runtime.mod.
#                       This is the only check that can catch the TABLE being
#                       wrong rather than one emitter: the table shipped once
#                       with AX dropped off the front of the mod=11 column
#                       and a duplicate BX invented at the end, which shifts
#                       every code down by one and makes `89 DC` (MOV SP,BX)
#                       look like the SI move the name asks for.
#   probe/modrm19.s     the execution probe for mod=00/01/10, kept as the
#                       record of how the memory forms were measured.  It is
#                       not run by default: it needs qemu and a hand-built
#                       floppy image, and its result is already written into
#                       the table in Runtime.mod, which modrm11.py checks.
#                       See probe/README.md to re-run it.
#
# rt_exec.py DOES run.  It used to run on Unicorn and fail 33 of 33, because
# Unicorn 2.1.4 mis-decodes 16-bit ModRM memory operands - so every failure was
# the emulator's, and a wrong oracle is worse than none: it cannot tell "my
# codegen is broken" from "the machine is broken".  It now boots the same
# bootcom.s the .COM fixtures use, one fresh machine per case, so every case
# gets a machine that has provably never run anything.  See its own docstring
# and the EXPECTATION FIXES section in tests/rt_exec.py.
set -u
D=/home/eric/Projets/Projets-Modula2/MyWork/TP3-comp/shell
GM2=/home/eric/bin/Modula2/Gm2/bin/gm2
cd "$D" || exit 9

fail=0
run () {
   name=$1
   shift
   echo
   echo "=============================================================="
   echo "== $name"
   echo "=============================================================="
   if "$@" ; then
      echo "-- $name: PASS"
   else
      echo "-- $name: FAIL (rc=$?)"
      fail=1
   fi
}

echo "== build =="
make clean >/dev/null 2>&1
if ! make >/tmp/tp_all_mk 2>&1 ; then
   echo "BUILD FAIL"
   grep -m10 "error:" /tmp/tp_all_mk
   exit 1
fi
echo "make rc=0, tpshell $(stat -c%s tpshell) bytes"
# gm2 reports a pass-3 rollup on the compiler during phase 1 of the two-phase
# link; that is expected and is not a build failure (make rc=0 is the check).

# The probe step asks rt_exec.py for its dump rather than building RtProbe a
# second time by hand.  Two recipes for one artifact is how a stale binary gets
# believed: ensure_rtprobe() is the only one that knows Runtime.mod's mtime, and
# a probe older than the runtime it reports on is worse than no probe, because
# its failure -- a blob with the wrong addresses baked in -- is invisible at the
# byte level and catastrophic at the behaviour level.
run "runtime probe"  python3 tests/rt_exec.py --probe

run "helper audit"    python3 tests/audit_helpers.py
run "mod=11 table"     python3 tests/probe/modrm11.py
run "runtime image"   python3 tests/check_runtime.py
run "compile matrix"  tests/run_compile_tests.sh
run "COM linker"      tests/run_com_tests.sh
# The byte checks above can only ask "are these bytes well formed".  This one
# asks "does the program do the right thing": every fixture with a .out is
# compiled, booted in qemu, and its output compared byte for byte, together
# with the DOS exit code.  It is the step that found eleven runtime bugs, the
# FOR off-by-one, the missing procedure-skip jump and a parser bug - none of
# which produced a malformed byte.
run "EXECUTE under qemu" python3 tests/run_com_exec.py
# And this one asks whether the runtime ENTRIES do what they claim, one qemu
# boot per call, 35 cases plus a pre-flight.  It is the only check that can catch
# an entry that is well-formed, decodes cleanly, preserves every register the
# driver happens to need - and computes the wrong answer: wrchar and wrbool
# borrowed BP to reach their argument and never gave it back, which a byte check
# and a register-convention check both call correct.  One case (four calls back
# to back) noticed; the BP-contract pre-flight is what stops the next entry from
# depending on there being such a case.
run "runtime entries"  python3 tests/rt_exec.py
run "frame displ"     python3 tests/check_framedisp.py
# The machine this compiler targets is an 8086, and the byte checks above all
# ask "is this well formed" -- which `0F 84 lo hi' is.  qemu-system-i386's
# lowest CPU model is 486, so the execution suite could not object either.
# This one asks "does the target have this opcode at all", and asserts the
# lowering that replaced the two illegal ones is the original compiler's shape.
run "8086 opcodes"   python3 tests/check_8086.py
run "UI error path"   python3 tests/uitest.py
run "UI success path" python3 tests/comtest.py

echo
echo "=============================================================="
if [ "$fail" -eq 0 ]; then
   echo "OVERALL: ALL PASS"
else
   echo "OVERALL: FAILURES PRESENT"
fi
echo "=============================================================="
exit $fail