TP3/shell/tests/run_exec86.py

#!/usr/bin/env python3
"""run_exec86.py -- run the same .COM under Exec86 and under qemu, and require
the two machines to agree byte for byte.

    .pas --Compiler.mod--> image --Linker.mod--> .COM
    .COM --Exec86Run--> our interpreter ----------\
    .COM --bootcom.s--> floppy --qemu-----------> compare

This is the cross-validation the summary asked for, and it is a claim about a
*pair*: either one alone would pass while being wrong in the same direction.

  * the expected output is still the fixture's hand-derived .out, so the
    interpreter is not being compared only against qemu.  If both machines
    were wrong the same way, .out - which no machine has ever written - would
    still catch it;
  * and the two machines are compared against each other directly, so a
    disagreement is reported as the two byte strings rather than as one test
    going red for a reason three assertions away;
  * the exit code is compared too, so "produced the right bytes and then fell
    over" cannot pass.

Nothing here is copied from run_com_exec.py: the boot sector, the floppy
layout, the input descriptor's offsets, qemu's exit-code convention and the
fixture list are all imported from it.  Two copies of that layout would be two
things that can drift apart, and the layout has already produced two silent
bugs once (see to_file's docstring there).

Usage:

    tests/run_exec86.py                 # every fixture that has a .out
    tests/run_exec86.py t02_writeln     # one, by name
    tests/run_exec86.py --list
    tests/run_exec86.py --show t02      # print what came out, assert nothing
    tests/run_exec86.py --no-qemu       # skip the cross-check, for iteration

There is deliberately no --rebless.  The .out files are derived by hand from
what the Pascal means, and a script that can regenerate them from what a
machine did is a way to turn a red test into a green one without ever looking
at the program.  run_com_exec.py still has it, under that project's rules.
"""

import os
import subprocess
import sys
import tempfile

HERE = os.path.dirname(os.path.abspath(__file__))
SHELL = os.path.dirname(HERE)
GM2 = "/home/eric/bin/Modula2/Gm2/bin/gm2"

# run_com_exec.py owns the boot sector, the floppy, the input descriptor and
# the qemu exit-code convention.  Importing is what keeps this file from
# becoming a second, subtly different copy of all four.
sys.path.insert(0, HERE)
import run_com_exec as rce  # noqa: E402


# ------------------------------------------------------------ the interpreter
def ensure_exec86run():
    """Build tests/Exec86Run.mod into ./exec86run if it is out of date.

    Freshness is measured against the .o files that get LINKED, not against
    the sources they came from, for the same reason rce.ensure_comtest does
    it that way: `make` relinks an .o from an edited .mod, and if only the
    .mod were watched, a freshly rebuilt Exec86.o could sit next to an
    exec86run that still contained the old interpreter - so the fix under
    test would never actually be in the binary being tested.
    """
    exe = os.path.join(SHELL, "exec86run")
    objs = ["Posix.o", "Exec86.o"]
    newer = objs + ["Exec86.mod", "Posix.c", os.path.join("tests", "Exec86Run.mod")]
    if os.path.exists(exe):
        t = os.path.getmtime(exe)
        if all(os.path.exists(os.path.join(SHELL, f))
               and os.path.getmtime(os.path.join(SHELL, f)) <= t
               for f in newer):
            return exe
    lst = os.path.join(SHELL, "tests", "e86.lst")
    link = [os.path.join(SHELL, "tests", "Exec86Run.mod")]
    objs_abs = [os.path.join(SHELL, o) for o in objs]
    subprocess.run([GM2, "-fiso", "-fgen-module-list=" + lst, "-o", "/dev/null"]
                   + link + objs_abs, capture_output=True, cwd=SHELL)
    r = subprocess.run([GM2, "-fiso", "-fuse-list=" + lst, "-o", exe]
                       + link + objs_abs, capture_output=True, cwd=SHELL)
    if r.returncode != 0:
        sys.exit("linking exec86run failed:\n"
                 + r.stderr.decode(errors="replace"))
    return exe


def parse_status(err):
    """The driver's one report line, or None if it is not there.

    Parsed strictly: a driver that printed no line, or a line this does not
    understand, is a failure to report rather than something to guess at.
    """
    for line in err.decode("latin-1").splitlines():
        if not line.startswith("exec86: status="):
            continue
        out = {}
        for field in line[len("exec86: "):].split():
            k, _, v = field.partition("=")
            if not v.isdigit():
                return None
            out[k] = int(v)
        if set(out) != {"status", "exit", "steps"}:
            return None
        return out
    return None


def run_exec86(exe, com, guest_in):
    """Run one .COM under Exec86.  Return (output, report or None, note)."""
    payload = com.encode() + b"\n" + guest_in
    try:
        r = subprocess.run([exe], input=payload, capture_output=True,
                           timeout=rce.TIMEOUT)
    except subprocess.TimeoutExpired:
        return b"", None, "the interpreter never halted (timeout %ds)" % rce.TIMEOUT
    rep = parse_status(r.stderr)
    if rep is None:
        return r.stdout, None, ("no report line; stderr was:\n  "
                                + r.stderr.decode("latin-1").replace("\n", "\n  "))
    return r.stdout, rep, ""


# ------------------------------------------------------------------- reporting
def visible(raw):
    return raw.decode("latin-1").replace("\r", "\\r").replace("\n", "\\n")


def main(argv):
    if "--list" in argv:
        for c in rce.cases():
            print(c["name"])
        return 0
    show = "--show" in argv
    cross = "--no-qemu" not in argv
    names = [a for a in argv if not a.startswith("-")]
    all_cases = list(rce.cases())
    if names:
        sel = [c for c in all_cases if c["name"] in names]
        missing = set(names) - set(c["name"] for c in sel)
        if missing:
            sys.exit("no such fixture: " + ", ".join(sorted(missing)))
    else:
        sel = all_cases

    exe = ensure_exec86run()
    boot = rce.build_boot_sector() if cross else None
    workdir = tempfile.mkdtemp(prefix="tpexec86-")
    passed = failed = 0

    for c in sel:
        pas = os.path.join(HERE, "fixtures", c["name"] + ".pas")
        com, log = rce.emit_com(rce.ensure_comtest(), pas, workdir)
        if com is None:
            print("  %-22s FAIL  comtest wrote no .COM" % c["name"])
            if log:
                print("       %s" % log)
            failed += 1
            continue

        got, rep, note = run_exec86(exe, com, c["stdin"])
        want = open(c["out"], "rb").read()

        if show:
            print("  %-22s %s" % (c["name"], visible(got)))
            if rep:
                print("       %s" % rep)
            continue

        problems = []
        if rep is None:
            problems.append(note)
        elif rep["status"] != 0:
            problems.append("interpreter status=%d (0 = halted through INT 21h "
                            "AH=4Ch)" % rep["status"])

        if got != want:
            problems.append("output differs from the hand-derived .out\n"
                            "         want  %s\n"
                            "         got   %s" % (visible(want), visible(got)))

        if cross:
            img = os.path.join(workdir, c["name"] + ".img")
            open(img, "wb").write(rce.build_floppy(boot, open(com, "rb").read(),
                                                   c["stdin"]))
            qout, rc, qnote = rce.run_qemu(img)
            if rc is None:
                problems.append("qemu: " + qnote)
            else:
                qcode = rce.exit_code_of(rc)
                if qcode is None:
                    problems.append("qemu rc=%d, which no .COM exit can produce"
                                    % rc)
                else:
                    if qout != got:
                        problems.append("disagrees with qemu\n"
                                        "         qemu    %s\n"
                                        "         exec86  %s"
                                        % (visible(qout), visible(got)))
                    if rep is not None and rep["status"] == 0 \
                            and rep["exit"] != qcode:
                        problems.append("exit code differs: exec86=%d qemu=%d"
                                        % (rep["exit"], qcode))

        if problems:
            failed += 1
            print("  %-22s FAIL" % c["name"])
            for p in problems:
                print("       %s" % p)
        else:
            passed += 1
            tail = "" if rep is None else ", exit %d, %d steps" % (rep["exit"],
                                                                  rep["steps"])
            print("  %-22s PASS  %d bytes%s" % (c["name"], len(got), tail))

    if show:
        return 0
    print()
    print("exec86: %d passed, %d failed (of %d)%s"
          % (passed, failed, len(sel),
             ", cross-checked against qemu" if cross else ", qemu cross-check off"))
    return 1 if failed else 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))