#!/bin/bash # Build and run the .COM linker harness (tests/ComTest.mod), then verify every # .COM it produced with an INDEPENDENT checker. # # The independent pass matters: ComTest computes the expectations from the same # Compiler state it is testing, so a bug in the compiler would be invisible to # it. The Python pass re-derives what the file must contain - the runtime's # first bytes, a zero gap, a size that covers the data area - from the # constants only, and cross-checks. set -u D=/home/eric/Projets/Projets-Modula2/MyWork/TP3-comp/shell GM2=/home/eric/bin/Modula2/Gm2/bin/gm2 cd "$D" || exit 9 FLAGS="-fiso" # build logs go beside the tree (TP3-comp/tmp), never in /tmp mkdir -p ../tmp # --check-only DIR -- skip the build and the link, and run only the # independent Python checker over the .COM files already in DIR (plus a # hand-written raw.txt in the same shape the linker emits). # # This exists for non-vacuity, and it is the only reason to make it. The # checker normally runs over a scratch directory that the EXIT trap deletes, # so there is no way to hand it a DELIBERATELY WRONG .COM and see whether it # notices. A check that has only ever been shown the truth is not a check: # it could be reporting the truth about every file because it says nothing at # all. tests/nonvacuity.sh uses this to feed it a corrupted image and # require the named assertion to go red. CHECK_ONLY="" if [ "${1:-}" = "--check-only" ]; then CHECK_ONLY=${2:-} shift 2 fi echo "== support modules ==" if [ -n "$CHECK_ONLY" ]; then echo "check-only mode: not rebuilding, not linking" OUT="$CHECK_ONLY" [ -d "$OUT" ] || { echo "RESULT: FAIL (no such directory: $OUT)"; exit 1; } else [ -f Posix.o ] || cc -c Posix.c || exit 1 for m in TextBuf Compiler Runtime Linker; do $GM2 $FLAGS -c $m.mod >../tmp/cm_c_$m 2>&1 \ || { echo "COMPILE_FAIL $m"; grep -m5 "error:" ../tmp/cm_c_$m; exit 1; } done $GM2 $FLAGS -c tests/ComTest.mod >../tmp/cm_c_ComTest 2>&1 \ || { echo "COMPILE_FAIL ComTest"; grep -m5 "error:" ../tmp/cm_c_ComTest; exit 1; } rm -f tests/ct.lst comtest $GM2 $FLAGS -fgen-module-list=tests/ct.lst -o /dev/null \ tests/ComTest.mod TextBuf.o Posix.o Compiler.o Runtime.o Linker.o \ >../tmp/cm_p1 2>&1 p1=$? $GM2 $FLAGS -fuse-list=tests/ct.lst -o comtest \ tests/ComTest.mod TextBuf.o Posix.o Compiler.o Runtime.o Linker.o \ >../tmp/cm_p2 2>&1 p2=$? if [ $p2 -ne 0 ]; then echo "LINK_FAIL p1_rc=$p1 p2_rc=$p2" grep -E "error:|undefined" ../tmp/cm_p2 | head -10 exit 1 fi echo "comtest built (p1_rc=$p1, phase 1 rc=1 is the expected rollup)" # .COM files are written beside the harness, so run it in a scratch dir OUT=$(mktemp -d) || exit 9 # TP_COM_KEEP=1 leaves the scratch dir behind, for tests/nonvacuity.sh to # corrupt a copy of a real image and hand it back through --check-only. The # alternative - rebuilding the whole toolchain inside the non-vacuity script # to produce one throwaway byte - is slow for no benefit, and the point of # the case is the CHECKER's sensitivity, not the compiler's. if [ "${TP_COM_KEEP:-0}" = "1" ]; then echo "TP_COM_KEEP=1: images left in $OUT" else trap 'rm -rf "$OUT"' EXIT fi cd "$OUT" || exit 9 ls "$D"/tests/fixtures/*.pas | "$D"/comtest > "$OUT/raw.txt" 2>&1 sed 's/^.*fixtures\///' "$OUT/raw.txt" echo "----------------------------------------------------------------" nok=$(grep -c " OK com=" "$OUT/raw.txt") nerr=$(grep -c " ERROR " "$OUT/raw.txt") nbad=$(grep -cE "WRITE_COM_FAILED|CANNOT" "$OUT/raw.txt") echo "linked: $nok .COM files, $nerr fixtures rejected at compile time, $nbad harness failures" if [ "$nbad" -ne 0 ]; then echo "RESULT: FAIL (harness could not link every compiling fixture)" exit 1 fi fi # ---- independent verification of the bytes on disk ------------------------ python3 - "$OUT" <<'PYEOF' import sys, os, re, glob out = sys.argv[1] # ENT_SZ and HDR_SZ are the layout constants, and they are RESTATED here on # purpose: the checker must not ask the code under test what the answer is. # # RT_SZ is different, and it is NOT restated. It used to be a literal, and it # was WRONG - 391, against a runtime of 432 bytes - so the header was read at # offset 394 instead of 435 and every one of the 30 .COM files "failed" on a # header full of code bytes. A duplicated constant that has silently drifted # is not an independent check; it is a second source of truth that lies, and # it lies in the direction of looking like the compiler is broken. # # So the runtime's size is MEASURED, from the .COM itself: the runtime is the # region between the entry jump and the program header, and the header is # found by its own signature rather than by an assumed offset (hdrFlag = 1, # with the code END and the data base where the layout says they are). If # the runtime ever changes size, this follows automatically; if the LAYOUT # changes, the header stops being found and the checker says so instead of # quietly measuring the wrong thing. # # The measurement is still independent of the compiler - it reads the emitted # file, not a Modula-2 variable - so it cannot be satisfied by the code under # test agreeing with itself. tests/check_runtime.py pins the size explicitly, # which is where a deliberate size change should be noticed. RT_SZ = None # measured per .COM by find_header, below # The image starts with a three-byte JMP at offset 0 -- see the layout comment # in Compiler.Inittur. It has to be there: a .COM is entered at file offset # 0, and until the jump existed this checker ASSERTED that the runtime was at # offset 0, which is precisely the bug. A checker that pins a wrong invariant # is worse than no checker, because it makes the wrong thing look tested. ENT_SZ = 3 # E9 lo hi HDR_SZ = 16 # 5 header words + 3 buffer words, see Compiler # RTSZ (image offset of the program header), PROLOG (RTSZ + HDR_SZ, where the # entry jump must land) and DATAB (RTSZ + 1000h, the compiler's data base) are # all DERIVED PER FILE by find_header below, not written down here. They used # to be module constants built on the restated RT_SZ, which is the bug this # whole block exists to remove: every one of them was wrong by 41 bytes, and # a checker that is consistently wrong in a simple direction does not fail - # it re-reports the same false 30 failures, in which the real ones hide. # The load bias: DOS puts a .COM's first byte at CS:0100, and CS = DS, so an # image offset K lives at DS:(K + 0100h). Every ABSOLUTE address in the # image must carry it; relative encodings (the entry jump, every CALL) must # not, since both operands shift together. Restated here so that the header # checks below compare against the addresses the program will actually use, # and so that the +0100h in them is a decision this checker made rather than # an accident of the compiler's. See Runtime.LoadBias. # # This is the THIRD bias of the same family in this file, and the subtlest: # the entry jump (a jump that landed on the end of the code), the RT_Entry # offsets (CALLs that landed inside a neighbouring runtime entry) and this # one (absolute addresses that landed 0100h low, inside the runtime) all # produce a program that STARTS, RUNS and PRINTS something. Only running it # finds this one; the byte checks are all satisfied by an address that is # consistently 0100h wrong. LOAD_BIAS = 0x100 # initmem's prologue, which is the runtime's only reader of the program # header. The displacements +4 and +6 below are the whole point of this # constant: the header word checks further down read hdrDS at +4 and hdrHeap # at +6, and initmem has to read the SAME two words or it clears the wrong # range. It used to read +8 (hdrMax, which the compiler patches to 0), so it # zeroed nothing at all, and nothing here noticed -- the emitted loop was # perfectly well formed, it just never ran. Asserting the bytes and the # header offsets together is what closes that gap. # # It is the FIRST ELEVEN BYTES OF THE RUNTIME, so it sits at image offset # ENT_SZ, not 0. HEAD = '8B F0 8B 54 04 8B 4C 06' # 11 bytes of initmem, see below HDR_DS_WORD = 4 # header word holding the data base HDR_HEAP_WORD = 6 # header word holding the data end # Byte 4 of HEAD is the displacement of initmem's MOV DX,[SI+?], and byte 7 # the displacement of its MOV CX,[SI+?]. The checks below read the header # words at HDR_DS_WORD and HDR_HEAP_WORD, so tying those two displacements to # the same two constants is what makes the runtime and the compiler agree by # construction rather than by coincidence. assert [int(HEAD.split()[4], 16), int(HEAD.split()[7], 16)] == \ [HDR_DS_WORD, HDR_HEAP_WORD], \ 'initmem no longer reads the two header words this checker verifies' raw = open(os.path.join(out, 'raw.txt')).read() rows = re.findall(r'(\S+\.pas)\s+OK\s+com=(\d+)\s+image=(\d+)\s+data=(\d+)\s+nonzeroInGap=(\d+)', raw) if not rows: print('RESULT: FAIL (no linked fixtures found in output)') sys.exit(1) def find_header(d): """Locate the program header by its own signature. Returns its image offset, or None. The header is eight words at image offset ENT_SZ + rtSz, and the layout says what they are (offsets here are BYTES into the header, which is why HDR_DS_WORD is 4 and not 2 - the words are 2 bytes each and 1-based by two, not by one): +0 1 hdrFlag, always 1 +2 code end + bias hdrCS +4 data base + bias hdrDS, where data base = hdrOff + 1000h +6 data end + bias hdrHeap, which is hdrDS + dataBytes hdrDS ties the header to its OWN offset, so the offset is recoverable from the file without assuming a runtime size: hdrOff = hdrDS - 1000h - bias. A candidate is accepted only if hdrFlag is 1, hdrDS satisfies that equation, hdrHeap is above hdrDS (a heap below its own base is not a layout, it is a coincidence), and the runtime's known first bytes are where they belong. initmem is the ONLY code in the image that reads the header, so its bytes cannot themselves move: they are at ENT_SZ always. Measuring beats restating the size, and it is not a loss of independence: it reads the EMITTED FILE, so the compiler cannot satisfy it by agreeing with itself. tests/check_runtime.py is where the runtime's size is pinned deliberately, and this checker reports the size it measured on every run, so a change there is visible rather than absorbed. """ head = bytes(int(x, 16) for x in HEAD.split()) if d[ENT_SZ:ENT_SZ + len(head)] != head: return None # no runtime: nothing to measure for off in range(ENT_SZ, len(d) - HDR_SZ + 1): w = (lambda b: int.from_bytes(d[off + b:off + b + 2], 'little')) if w(0) != 1: # hdrFlag continue if w(HDR_DS_WORD) != off + 0x1000 + LOAD_BIAS: # hdrDS continue if w(HDR_HEAP_WORD) <= w(HDR_DS_WORD): # hdrHeap continue if w(2) - LOAD_BIAS < off + HDR_SZ: # hdrCS continue return off return None bad = 0 last_rt = None for name, com, image, data, nzg in rows: com, image, data, nzg = int(com), int(image), int(data), int(nzg) # ComTest writes the .COM by BASENAME beside itself (it cannot graft a # directory onto a source path), so the checker must look for the bare # name, not the full source path the fixture was read from. path = os.path.join(out, os.path.basename(name)[:-4] + '.COM') errs = [] if not os.path.exists(path): errs.append('no .COM file') d = b'' else: d = open(path, 'rb').read() # Everything below is expressed in terms of where the header actually is, # measured from this file, rather than where a literal says it should be. hdrOff = find_header(d) if hdrOff is None: errs.append('no program header found: the layout this checker knows ' 'how to look for is not the one in the file') RTSZ, PROLOG, DATAB = ENT_SZ, ENT_SZ + HDR_SZ, ENT_SZ + 0x1000 else: RTSZ = hdrOff PROLOG = hdrOff + HDR_SZ DATAB = hdrOff + 0x1000 if RTSZ != last_rt: last_rt = RTSZ print(' measured runtime size: %d bytes (header at image offset ' '%d)' % (RTSZ - ENT_SZ, RTSZ)) # The entry jump. This is the one assertion in the whole project that can # see where execution STARTS, because it is the only one that cares. It # has caught THREE real bugs, all in the same three bytes, and all of them # invisible to every other check here: # # 1. no jump at all, so a .COM began by executing the runtime's initmem # with whatever the loader left in AX; # 2. a jump to `pc`, one byte past the last instruction, into the # zero-filled code/data gap, where the CPU slides through # `ADD [BX+SI],AL` until it faults; # 3. a jump to RTSZ, which is the program HEADER - sixteen bytes of DATA # that the CPU then decodes as instructions. This one is the reason # the target is pinned to PROLOG and not to RTSZ: whether it works # depends entirely on how those sixteen bytes happen to decode, so # writeln('hi') ran correctly by sliding through them while t07 hung # on a LOCK-prefixed ADD with a displacement crossing a page. The # correct target is the first instruction, and there is no reason for # a checker to accept a range. if len(d) >= ENT_SZ: if d[0] != 0xE9: errs.append('byte 0 is %02X, not the E9 of the entry jump' % d[0]) # The jump's displacement is measured from the end of the jump. want_rel = PROLOG - ENT_SZ got_rel = int.from_bytes(d[1:ENT_SZ], 'little') if got_rel != want_rel: errs.append('entry jump rel16=%d, want %d; it lands on image ' 'offset %d, want %d (the first instruction, %d bytes ' 'past the header - not the header at %d, and not the ' 'end of the code at %d)' % (got_rel, want_rel, ENT_SZ + got_rel, PROLOG, HDR_SZ, RTSZ, image)) # The runtime's own first bytes must follow the jump, and the jump must be # the only thing before them. if d[ENT_SZ:ENT_SZ + len(HEAD.split())].hex(' ').upper() != HEAD: errs.append('runtime not at offset %d (bytes there %s, want %s)' % (ENT_SZ, d[ENT_SZ:ENT_SZ + len(HEAD.split())].hex(' ').upper(), HEAD)) if len(d) != com: errs.append('file is %d bytes, harness said %d' % (len(d), com)) if DATAB + data != len(d): errs.append('size %d != dataBase+data %d' % (len(d), DATAB + data)) # the gap between the image and the data area must be entirely zero gap = d[image:DATAB] if any(gap): errs.append('%d non-zero bytes in the code/data gap' % sum(1 for b in gap if b)) if nzg != 0: errs.append('harness itself reported %d non-zero gap bytes' % nzg) # the program must start exactly where the runtime ends if image < RTSZ: errs.append('image %d shorter than the runtime %d' % (image, RTSZ)) # the program header sits at offset rtSz, and its words must describe the # image that is actually in the file if len(d) > RTSZ + 8: hdr = d[RTSZ:RTSZ+16] flag = int.from_bytes(hdr[0:2], 'little') cs = int.from_bytes(hdr[2:4], 'little') ds = int.from_bytes(hdr[HDR_DS_WORD:HDR_DS_WORD+2], 'little') heap = int.from_bytes(hdr[HDR_HEAP_WORD:HDR_HEAP_WORD+2], 'little') if flag != 1: errs.append('hdrFlag=%d' % flag) # hdrCS is the end of the code, as a SEGMENT offset like every other # offset in the header, so the load bias comes off before comparing it # with the harness's `image` (= rtSz + code, already an image offset). # Adding RTSZ here would count the runtime twice. if cs - LOAD_BIAS != image: errs.append('hdrCS=%d, want %d (end of image, as a segment ' 'offset)' % (cs, image + LOAD_BIAS)) if ds != DATAB + LOAD_BIAS: errs.append('hdrDS=%d, want %d (= data base %d + load bias %d)' % (ds, DATAB + LOAD_BIAS, DATAB, LOAD_BIAS)) if heap != DATAB + data + LOAD_BIAS: errs.append('hdrHeap=%d, want %d (= data base %d + %d + load bias %d)' % (heap, DATAB + data + LOAD_BIAS, DATAB, data, LOAD_BIAS)) # initmem must read hdrDS and hdrHeap, not some other pair of header # words. (It read +8, hdrMax, which the compiler patches to 0, so # the range it cleared was empty and every global kept whatever the # loader left in it.) if [d[ENT_SZ + 4], d[ENT_SZ + 7]] != [HDR_DS_WORD, HDR_HEAP_WORD]: errs.append('initmem reads header words +%d/+%d, but the data ' 'base and data end are at +%d/+%d' % (d[ENT_SZ + 4], d[ENT_SZ + 7], HDR_DS_WORD, HDR_HEAP_WORD)) if errs: bad += 1 print(' %-24s FAIL %s' % (os.path.basename(name)[:-4], '; '.join(errs))) else: print(' %-24s PASS %d bytes' % (os.path.basename(name)[:-4], com)) print('----------------------------------------------------------------') print('independent .COM check: %d checked, %d failed' % (len(rows), bad)) sys.exit(1 if bad else 0) PYEOF rc=$? [ "$rc" -eq 0 ] || exit 1 echo "RESULT: ALL PASS"