#!/usr/bin/env python3 """run_exec86.py -- run the same .COM under Exec86 and under qemu, and require the two machines to agree byte for byte. .pas --Compiler.mod--> image --Linker.mod--> .COM .COM --Exec86Run--> our interpreter ----------\ .COM --bootcom.s--> floppy --qemu-----------> compare This is the cross-validation the summary asked for, and it is a claim about a *pair*: either one alone would pass while being wrong in the same direction. * the expected output is still the fixture's hand-derived .out, so the interpreter is not being compared only against qemu. If both machines were wrong the same way, .out - which no machine has ever written - would still catch it; * and the two machines are compared against each other directly, so a disagreement is reported as the two byte strings rather than as one test going red for a reason three assertions away; * the exit code is compared too, so "produced the right bytes and then fell over" cannot pass. Nothing here is copied from run_com_exec.py: the boot sector, the floppy layout, the input descriptor's offsets, qemu's exit-code convention and the fixture list are all imported from it. Two copies of that layout would be two things that can drift apart, and the layout has already produced two silent bugs once (see to_file's docstring there). Usage: tests/run_exec86.py # every fixture that has a .out tests/run_exec86.py t02_writeln # one, by name tests/run_exec86.py --list tests/run_exec86.py --show t02 # print what came out, assert nothing tests/run_exec86.py --no-qemu # skip the cross-check, for iteration There is deliberately no --rebless. The .out files are derived by hand from what the Pascal means, and a script that can regenerate them from what a machine did is a way to turn a red test into a green one without ever looking at the program. run_com_exec.py still has it, under that project's rules. """ import os import subprocess import sys import tempfile HERE = os.path.dirname(os.path.abspath(__file__)) SHELL = os.path.dirname(HERE) GM2 = "/home/eric/bin/Modula2/Gm2/bin/gm2" # run_com_exec.py owns the boot sector, the floppy, the input descriptor and # the qemu exit-code convention. Importing is what keeps this file from # becoming a second, subtly different copy of all four. sys.path.insert(0, HERE) import run_com_exec as rce # noqa: E402 # ------------------------------------------------------------ the interpreter def ensure_exec86run(): """Build tests/Exec86Run.mod into ./exec86run if it is out of date. Freshness is measured against the .o files that get LINKED, not against the sources they came from, for the same reason rce.ensure_comtest does it that way: `make` relinks an .o from an edited .mod, and if only the .mod were watched, a freshly rebuilt Exec86.o could sit next to an exec86run that still contained the old interpreter - so the fix under test would never actually be in the binary being tested. """ exe = os.path.join(SHELL, "exec86run") objs = ["Posix.o", "Exec86.o"] newer = objs + ["Exec86.mod", "Posix.c", os.path.join("tests", "Exec86Run.mod")] if os.path.exists(exe): t = os.path.getmtime(exe) if all(os.path.exists(os.path.join(SHELL, f)) and os.path.getmtime(os.path.join(SHELL, f)) <= t for f in newer): return exe lst = os.path.join(SHELL, "tests", "e86.lst") link = [os.path.join(SHELL, "tests", "Exec86Run.mod")] objs_abs = [os.path.join(SHELL, o) for o in objs] subprocess.run([GM2, "-fiso", "-fgen-module-list=" + lst, "-o", "/dev/null"] + link + objs_abs, capture_output=True, cwd=SHELL) r = subprocess.run([GM2, "-fiso", "-fuse-list=" + lst, "-o", exe] + link + objs_abs, capture_output=True, cwd=SHELL) if r.returncode != 0: sys.exit("linking exec86run failed:\n" + r.stderr.decode(errors="replace")) return exe def parse_status(err): """The driver's one report line, or None if it is not there. Parsed strictly: a driver that printed no line, or a line this does not understand, is a failure to report rather than something to guess at. """ for line in err.decode("latin-1").splitlines(): if not line.startswith("exec86: status="): continue out = {} for field in line[len("exec86: "):].split(): k, _, v = field.partition("=") if not v.isdigit(): return None out[k] = int(v) if set(out) != {"status", "exit", "steps"}: return None return out return None def run_exec86(exe, com, guest_in): """Run one .COM under Exec86. Return (output, report or None, note).""" payload = com.encode() + b"\n" + guest_in try: r = subprocess.run([exe], input=payload, capture_output=True, timeout=rce.TIMEOUT) except subprocess.TimeoutExpired: return b"", None, "the interpreter never halted (timeout %ds)" % rce.TIMEOUT rep = parse_status(r.stderr) if rep is None: return r.stdout, None, ("no report line; stderr was:\n " + r.stderr.decode("latin-1").replace("\n", "\n ")) return r.stdout, rep, "" # ------------------------------------------------------------------- reporting def visible(raw): return raw.decode("latin-1").replace("\r", "\\r").replace("\n", "\\n") def main(argv): if "--list" in argv: for c in rce.cases(): print(c["name"]) return 0 show = "--show" in argv cross = "--no-qemu" not in argv names = [a for a in argv if not a.startswith("-")] all_cases = list(rce.cases()) if names: sel = [c for c in all_cases if c["name"] in names] missing = set(names) - set(c["name"] for c in sel) if missing: sys.exit("no such fixture: " + ", ".join(sorted(missing))) else: sel = all_cases exe = ensure_exec86run() boot = rce.build_boot_sector() if cross else None workdir = tempfile.mkdtemp(prefix="tpexec86-") passed = failed = 0 for c in sel: pas = os.path.join(HERE, "fixtures", c["name"] + ".pas") com, log = rce.emit_com(rce.ensure_comtest(), pas, workdir) if com is None: print(" %-22s FAIL comtest wrote no .COM" % c["name"]) if log: print(" %s" % log) failed += 1 continue got, rep, note = run_exec86(exe, com, c["stdin"]) want = open(c["out"], "rb").read() if show: print(" %-22s %s" % (c["name"], visible(got))) if rep: print(" %s" % rep) continue problems = [] if rep is None: problems.append(note) elif rep["status"] != 0: problems.append("interpreter status=%d (0 = halted through INT 21h " "AH=4Ch)" % rep["status"]) if got != want: problems.append("output differs from the hand-derived .out\n" " want %s\n" " got %s" % (visible(want), visible(got))) if cross: img = os.path.join(workdir, c["name"] + ".img") open(img, "wb").write(rce.build_floppy(boot, open(com, "rb").read(), c["stdin"])) qout, rc, qnote = rce.run_qemu(img) if rc is None: problems.append("qemu: " + qnote) else: qcode = rce.exit_code_of(rc) if qcode is None: problems.append("qemu rc=%d, which no .COM exit can produce" % rc) else: if qout != got: problems.append("disagrees with qemu\n" " qemu %s\n" " exec86 %s" % (visible(qout), visible(got))) if rep is not None and rep["status"] == 0 \ and rep["exit"] != qcode: problems.append("exit code differs: exec86=%d qemu=%d" % (rep["exit"], qcode)) if problems: failed += 1 print(" %-22s FAIL" % c["name"]) for p in problems: print(" %s" % p) else: passed += 1 tail = "" if rep is None else ", exit %d, %d steps" % (rep["exit"], rep["steps"]) print(" %-22s PASS %d bytes%s" % (c["name"], len(got), tail)) if show: return 0 print() print("exec86: %d passed, %d failed (of %d)%s" % (passed, failed, len(sel), ", cross-checked against qemu" if cross else ", qemu cross-check off")) return 1 if failed else 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))