TP3/shell/tests/run_all.sh

#!/bin/bash
# Run every check.  Non-zero exit if anything fails.
#
# Four suites, in increasing order of "how much could be lying to me":
#
#   1. compile matrix   27 fixtures, verdict + code size + data size ASSERTED
#                       from expected.tsv.  Fast, no pty.
#   2. .COM linker      links every fixture, then re-verifies the bytes with an
#                       independent checker that restates the layout constants
#                       instead of asking the compiler.
#   3. UI error path    pty: W, C on a bad file, editor opens on the error,
#                       Ctrl-K D, Q.  Covers the shell behaviour that only
#                       exists interactively.
#   4. UI success path  pty: Options -> Destination=COM, C, then the .COM is
#                       checked ON DISK.  This is the end-to-end path from
#                       keystroke to artifact.
#
#   5. frame displ      links the two fixtures that touch a stack frame and
#                       asserts the [BP+off] displacement encoding, which no
#                       other check can see: disp8 and disp16 are both
#                       well-formed and both decode cleanly, and only one of
#                       them reads the variable the symbol table named.
#
#   RtProbe             dumps the runtime size and its 14 entry offsets, so a
#                       runtime change that moves an entry is visible here.
#
# Three runtime suites run before all of those, because everything else
# trusts the runtime's bytes:
#
#   audit_helpers       disassembles every one-line emitter in BOTH Runtime.mod
#                       and Compiler.mod and compares the DECODE against the
#                       procedure's NAME.  This is the only check that can see
#                       a wrong ModRM that still decodes cleanly, which is the
#                       mistake this project actually makes: MovSiBx and
#                       CmpSiBx were both `DC` (= MOV SP,BX / CMP SP,BX) for a
#                       long time, and EmXchgAxCx was `93` (XCHG BX,AX) under a
#                       name that says XCHG AX,CX.
#
#                       It sweeps Compiler.mod because that is where the
#                       emitter was.  Auditing one module said "every helper
#                       agrees with its name" about a module it had not
#                       examined, which is why the XCHG fault survived a green
#                       run.  It also cross-checks its own coverage: a
#                       procedure that emits bytes but that the audit never
#                       matched is itself a failure, because a helper that
#                       drops out of the audit when you add a comment is worse
#                       than a helper that is checked and found wanting.
#   check_runtime       sweeps the built runtime's code region with FCML: no
#                       desync, every entry and every branch target on an
#                       instruction boundary, and the whole disassembly equal
#                       to tests/runtime.golden.
#   nonvacuity          breaks the runtime six ways and asserts each named
#                       check goes red for the stated reason.  Slower, so it
#                       is not in the default run: run it when a check's
#                       sensitivity is in question, or after editing a check.
#
# nonvacuity.sh is NOT run by default: it rebuilds the runtime six times and
# it is a meta-check, so it belongs to "is the harness honest" reviews rather
# than to every change.  Run tests/nonvacuity.sh directly after touching any
# of the checks above.
#
# Two more checks guard the ModR/M table itself, which is the input every hand
# written emitter in Runtime.mod depends on:
#
#   probe/modrm11.py    asks GNU `as` (.code16) to encode the register moves
#                       and FCML to decode them, and requires the result to
#                       agree with the hard-coded bytes, the four
#                       hand-checking anchors, and the table in Runtime.mod.
#                       This is the only check that can catch the TABLE being
#                       wrong rather than one emitter: the table shipped once
#                       with AX dropped off the front of the mod=11 column
#                       and a duplicate BX invented at the end, which shifts
#                       every code down by one and makes `89 DC` (MOV SP,BX)
#                       look like the SI move the name asks for.
#   probe/modrm19.s     the execution probe for mod=00/01/10, kept as the
#                       record of how the memory forms were measured.  It is
#                       not run by default: it needs qemu and a hand-built
#                       floppy image, and its result is already written into
#                       the table in Runtime.mod, which modrm11.py checks.
#                       See probe/README.md to re-run it.
#
# rt_exec.py is NOT run: it needs Unicorn, whose 16-bit ModRM decoding is
# wrong on this machine (see SUMMARY.md), so its failures would be the
# emulator's, not the runtime's.  Running it by default would be noise.
set -u
D=/home/eric/Projets/Projets-Modula2/MyWork/TP3-comp/shell
GM2=/home/eric/bin/Modula2/Gm2/bin/gm2
cd "$D" || exit 9

fail=0
run () {
   name=$1
   shift
   echo
   echo "=============================================================="
   echo "== $name"
   echo "=============================================================="
   if "$@" ; then
      echo "-- $name: PASS"
   else
      echo "-- $name: FAIL (rc=$?)"
      fail=1
   fi
}

echo "== build =="
make clean >/dev/null 2>&1
if ! make >/tmp/tp_all_mk 2>&1 ; then
   echo "BUILD FAIL"
   grep -m10 "error:" /tmp/tp_all_mk
   exit 1
fi
echo "make rc=0, tpshell $(stat -c%s tpshell) bytes"
# gm2 reports a pass-3 rollup on the compiler during phase 1 of the two-phase
# link; that is expected and is not a build failure (make rc=0 is the check).

run "runtime probe"  bash -c '
   GM2=/home/eric/bin/Modula2/Gm2/bin/gm2
   $GM2 -fiso -o /tmp/tp_rtprobe tests/RtProbe.mod Runtime.o Posix.o \
       >/tmp/tp_rt 2>&1 || { grep -m5 error: /tmp/tp_rt; exit 1; }
   /tmp/tp_rtprobe'

run "helper audit"    python3 tests/audit_helpers.py
run "mod=11 table"     python3 tests/probe/modrm11.py
run "runtime image"   python3 tests/check_runtime.py
run "compile matrix"  tests/run_compile_tests.sh
run "COM linker"      tests/run_com_tests.sh
# The byte checks above can only ask "are these bytes well formed".  This one
# asks "does the program do the right thing": every fixture with a .out is
# compiled, booted in qemu, and its output compared byte for byte, together
# with the DOS exit code.  It is the step that found eleven runtime bugs, the
# FOR off-by-one, the missing procedure-skip jump and a parser bug - none of
# which produced a malformed byte.
run "EXECUTE under qemu" python3 tests/run_com_exec.py
run "frame displ"     python3 tests/check_framedisp.py
run "UI error path"   python3 tests/uitest.py
run "UI success path" python3 tests/comtest.py

echo
echo "=============================================================="
if [ "$fail" -eq 0 ]; then
   echo "OVERALL: ALL PASS"
else
   echo "OVERALL: FAILURES PRESENT"
fi
echo "=============================================================="
exit $fail