TurboModula2/tools/mcdiff.sh

#!/bin/bash
# mcdiff.sh — verification harness for the src/compiler re-read.
# Stage 1 (runs HERE, no VM needed): static consistency checks.
# Stage 2 (needs the Reloaded VM + original compiler): true MCD diff;
#   procedure documented below. Exit nonzero on any stage-1 failure.
#
# Stage 2 procedure (once, per module MOD in dependency order
#   Compiler Scanner SymTab Errors CodeGen Express Statemen Pass1):
#   1. Build the VM:  cd Turbo-Modula-2-Reloaded-main/vm_linux && make
#   2. Stage sources: copy src/compiler/X.{DEF,MOD} onto m2.dsk
#      (mtools: mcopy -i m2.dsk, or sudo mount -o loop).
#      File names must be 8+3 uppercase (COMPILER.DEF etc. already are).
#   3. Boot: ./m2_disk m2.dsk ; at the M2 prompt run the resident
#      compiler on each MOD (c <file>), producing X.MCD on the image.
#   4. Extract: mcopy -i m2.dsk ::COMPILER.DIR/X.MCD ./build/
#   5. Disassemble both: unassemble.c built from Reversing-Turbo-Modula2-main
#      (cc -o unassemble unassemble.c), run on original .MCD from
#      Turbo-Modula-2-Reloaded-main/disk_content/COMPILER.DIR/ and on build/.
#   NOTE 2026-10-03: the VM needs a real terminal (termios raw mode); plain
#   pipes hang. Drive it under a pty, e.g.
#     script -qec "./vm_linux/m2 m2.dsk" /dev/null
#   or expect(1), sending keys with delays (CR-terminated lines). mtools is
#   NOT installed in this environment — staging currently needs manual mount.
#   6. Diff against Reversing-Turbo-Modula2-main/MCode_disassembly/<mod>.txt.
#      Byte-identical MCD (hence identical disassembly modulo addresses) =
#      proof of a faithful rename/decompile. Record result per module in
#      docs/compiler/70-rename-plan.md style commit notes.
set -u
SRC="${1:-src/compiler}"
fail=0

say() { printf '%s\n' "$*"; }
err() { printf 'FAIL: %s\n' "$*" >&2; fail=1; }

say "== stage 1a: numbered identifiers (must be zero outside comments) =="
# strip (* *) comments, then look for procN/globalN/paramN/localN/PROC N
tmp=$(mktemp -d)
for f in "$SRC"/*.MOD "$SRC"/*.DEF; do
  [ -f "$f" ] || continue
  python3 - "$f" "$tmp" <<'PY'
import re, sys, pathlib
src = pathlib.Path(sys.argv[1]).read_text()
nos = re.sub(r'\(\*.*?\*\)', '', src, flags=re.S)  # drop (* *) comments
hits = re.findall(r'\b(?:global|proc|param|local)\d+\b|\bPROC\d+\b', nos)
if hits:
    from collections import Counter
    print(f'{sys.argv[1]}: {dict(Counter(hits))}')
PY
done | tee "$tmp/hits" | grep . && err "numbered identifiers in code" || say "clean"

say "== stage 1b: FORWARD/def consistency per MOD =="
python3 - "$SRC" <<'PY'
import re, sys, pathlib
bad = False
for f in sorted(pathlib.Path(sys.argv[1]).glob('*.MOD')):
    nos = re.sub(r'\(\*.*?\*\)', '', f.read_text(), flags=re.S)
    fwds = re.findall(r'PROCEDURE\s+(\w+)\s*(?:\([^;]*\))?(?:\s*:\s*\w+)?\s*;\s*FORWARD\s*;', nos)
    defs = re.findall(r'PROCEDURE\s+(\w+)\s*(?:\([^;]*\))?(?:\s*:\s*\w+)?\s*;', nos)
    from collections import Counter
    dc = Counter(defs)
    dupes = [k for k, v in dc.items() if v > 1 and k not in fwds]
    missing = [w for w in fwds if w not in defs]
    if dupes or missing:
        bad = True
        print(f'{f.name}: dupes={dupes} missing-defs={missing}')
print('FORWARD-OK' if not bad else 'FORWARD-BAD')
PY

say "== stage 1c: FIXME inventory =="
grep -c "FIXME" "$SRC"/*.MOD "$SRC"/*.DEF 2>/dev/null || true

say "== stage 1d: unknown cross-module callees (spot check) =="
# every Module.Proc call should resolve to a renamed export or a flagged FIXME nearby
python3 - "$SRC" <<'PY'
import re, sys, pathlib
known = set()
for f in pathlib.Path(sys.argv[1]).glob('*.DEF'):
    nos = re.sub(r'\(\*.*?\*\)', '', f.read_text(), flags=re.S)
    mod = f.stem.capitalize()
    for m in re.findall(r'PROCEDURE\s+(\w+)', nos):
        known.add(f'{mod}.{m}')
    for m in re.findall(r'^\s{4}(\w+)\s*[:,\[]', nos, re.M):
        known.add(f'{mod}.{m}')
# + known drafts without DEF
for mod, procs in {
  'Express': ['ParseExpression','ParseSimpleExpression','EmitRangeCheck','EmitCompare','ParseDesignatorTail','ParseTerm','ParseFactor','ParseSetCons','ParseFuncCall','ParseTypeCast','ExpandStdProc43','ExpandStdProc44','ExpandStdProc45','ExpandStdProc46','ParseSetOrCastTail','LoadOperand','StoreOperand','FoldRelation','CheckRelationTypes','GetConstVal','GetTypeDesc','ParseAssignment','IsOrdinal','IsSet','BaseTypeOf','FoldConstOp','ConstToCard','ConstToInt','MatchOpClass','CheckAssignable','ParseSelector','PushExprDesc','PopExprDesc','CentralError','EvalConstExpr'],
  'Statemen': ['ParseStatement','GuardedStatSeq','ParseBeginBlock','ParseCaseStmt','ParseCaseTail','ParseFor','ParseWhile','ParseRepeat','ParseIf','ParseLoop','ParseExit','ParseResult','ParseWith','ParseReturn','CalcMemOffset','SyncToStatStart','ParseStatSeqLoop','EmitJumpFixup','NextSymWrap','CloseBlockChain','EmitNameHash','EmitComBytes'],
  'Pass1': ['ParseType','ParseFormalParams','ParseValParam','ParseVarParam','ParseFieldList','EvalConstExpr','CheckConformant','ParseRecordType','CloseScopeFixup','ParseConstDecl','ParseVarDecl','GetIdent','ParseImportItem','OpenScopeEmit','TypeSizeRec','ParseProcedureDecl','ParseModuleHead','ParseExportList','ParseOneDecl','ParseDeclarations','ParseImportFrom','ParseModuleBlock','ParseProcBody','ParseMainModule','WriteMcdHeader','WriteOutputFiles','EmitConstVal','EmitTypeDesc','PushTempMark','PopTempMark','AllocSymEntry','LinkSymEntry','InsertSetConst','CondInsert','InsertEntry','CheckForwardResolved','Pass1Main','InitPass1','EmitModuleProlog','ParseDeclSync','ParseConstOrTypeArm','ParseModuleArm','TryParseDecl','ParseImportScope','CappedStrLen'],
  'SymTab': ['FindModule','EnterModule','LookupChain','MatchSym','SymbolAssert','CheckNode','NewString','GetName','PutAttr','GetAttr','AllocSym','EnterChained','EnterIdent','UnlinkScope','InitHeaps','InsertBuiltin','EnterStandards','LoadSymFile','FindExport','ListExports','ImportOne','ImportAll','SkipQualident','DefineDefinitionModule','BuildBitsetConst','DefineProgramModule','InitSymbolTables','NoteImported','DoImportList','CompileBody','CheckForwards','TryCompile','CompileDispatch','Checksum7w','SaveSymbolFile','RestoreAfterEdit'],
}.items():
    known.update(f'{mod}.{p}' for p in procs)
import collections
susp = collections.Counter()
for f in sorted(pathlib.Path(sys.argv[1]).glob('*.MOD')):
    for i, line in enumerate(f.read_text().splitlines(), 1):
        code = line.split('(*')[0]
        for m in re.findall(r'\b(Scanner|Errors|CodeGen|Compiler|SymTab|Express|Statemen|Pass1|Loader|Files|Texts|Terminal|ComLine|Doubles|Editor|Storage|System)\.(\w+)', code):
            q = f'{m[0]}.{m[1]}'
            if q not in known and m[1] not in ('WriteString','WriteLn','WriteChar','WriteCard','SetCol','ReadBytes','WriteBytes','SetPos','Open','Close','Create','Delete','Allocate','MARK','RELEASE','MOVE','ADR','BITSET','CARDINAL','ADDRESS','ORD','CHR','CAP','HIGH','NIL','TRUE','TRUE','HALT','RETURN','INC','DEC','INCL','EXCL','IN','ABS','ODD','SIZE','HALT','Close','Create','Delete','Open','ReadBytes','WriteBytes','SetPos','FileSize','NextPos','NoTrailer','ReadByte','WriteByte','ResetSys','UseError','EndError','DiskFull','Call','LibOpen','LibRead','InitLibs','LoadError','firstDrive','Edit','editPosition','editOkFlag','ReadChar','ReadAgain','numCols','commandLine','inName','outName','codepos','execute','RedirectOutput','PromptFor','qcp','qadd','qsub','qmul','qdiv','qneg','qabs','StrToDouble','DerefAliasType','StrLen','FindIdent','InsertSymbol','CopyStringToHeap','Allocate','GetStackMark','CheckStackMark','OpenScope','NewNode','NewSizedNode','DeclareIdentifier','MakeIdentifierNode','ExpectStringLiteral','ChangeFileExtension','GetSym','AcceptSymbol','ExpectSymbol','TestSymbolInSet','NeedIdentifier','ReportError'):
                susp[(f.name, q)] += 1
for (f, q), n in sorted(susp.items()):
    print(f'{f}:{q} x{n}')
PY

if [ "$fail" -ne 0 ]; then say "STAGE 1 FAILED"; exit 1; fi
say "STAGE 1 PASSED"
rm -rf "$tmp"