TP3/shell/tests/runtest.py

#!/usr/bin/env python3
"""End-to-end UI test: R compiles and RUNS the image, with no file written.

uitest.py covers the compile error path, comtest.py the artifact path.  This
covers TP3's `R', which produces no artifact at all: the linked image is poked
into the interpreter's own 64 KB at 0100h and executed inside this process, so
the only evidence it ever existed is what the program printed.

  1. W loads a work file (LoadWorkFile ends with a Pause: needs a filler key)
  2. R compiles, reports the code and data sizes, pokes LinkSize() bytes at
     0100h, runs them, and reports a status and a step count
  3. The GUEST'S OWN OUTPUT is compared against the fixture's .out

Step 3 is the point, and it is deliberately not "the shell said OK".  The .out
files in this project are hand-derived from Pascal's semantics and never
blessed from a machine's output, so comparing against one checks the
interpreter against a third thing that the code under test did not produce:
the compiler chose the bytes, the interpreter executed them, and neither had a
say in what the answer was supposed to be.

Step 4-5: the reported status must be a clean INT 21h AH=4Ch with code 0, the
step count must be non-zero (a run that executed nothing is not a run), ESC
must get back to the main menu, and the .COM on disk must be untouched - R is
Memory and .COM alike by design, and "it didn't write a file" is a claim about
behaviour that only this test can make.

Usage: runtest.py [fixture.pas]        (the fixture must have a .out)

Passing a fixture whose .out does not match is expected to be RED - that is
the test being non-vacuous, not a failure of the shell.
"""
import os
import re
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from ptyharness import SHELL_DIR, drain, reap, send, spawn, status_str, visible

FIXTURE = os.path.abspath(sys.argv[1]) if len(sys.argv) > 1 else os.path.join(
    os.path.dirname(os.path.abspath(__file__)), "fixtures", "t34_arith.pas")
OUT = os.path.splitext(FIXTURE)[0] + ".out"
COM = os.path.splitext(FIXTURE)[0] + ".COM"

if not os.path.exists(OUT):
    sys.exit("runtest.py: %s has no .out, so there would be nothing to "
             "compare the guest's output against" % os.path.basename(OUT))

# Hand-derived expectation, read as the raw CRLF bytes the runtime emits.
EXPECTED = open(OUT, "rb").read()


def steps_of(text):
    m = re.search(r"steps executed: (\d+)", text)
    return int(m.group(1)) if m else -1


def main():
    com_before = os.stat(COM).st_mtime_ns if os.path.exists(COM) else None
    pid, fd = spawn()
    checks = []
    run = ""
    after = ""
    try:
        drain(fd, quiet=0.6)
        send(fd, b"w")
        send(fd, FIXTURE.encode())
        send(fd, b"\r", quiet=0.8)
        send(fd, b" ", quiet=0.5)              # clears LoadWorkFile's Pause

        run = visible(send(fd, b"r", quiet=2.5))
        checks.append(("R reported a successful compile",
                       "Compiled OK - code" in run))
        checks.append(("R reported poking the image at 0100h",
                       "bytes at 0100h" in run))
        # The status line is what Run86 answered: 0 is the guest's own
        # INT 21h AH=4Ch, 1 an interpreter fault, 2 the step limit.  Only 0
        # means the program finished because it decided to.
        checks.append(("guest ran to a clean AH=4Ch exit with code 0",
                       "program terminated (INT 21h AH=4Ch, code 0)" in run))
        checks.append(("R reported a non-zero step count", steps_of(run) > 0))
        checks.append(("guest output matches the hand-derived .out",
                       EXPECTED.decode("latin-1") in run))

        after = visible(send(fd, b"\x1b", quiet=0.8))
        checks.append(("ESC after the run returned to the main menu",
                       "Main file" in after))
        send(fd, b"q", quiet=0.8)
    finally:
        status = reap(fd, pid)

    checks.append(("shell exited cleanly (status 0)", os.WIFEXITED(status)
                   and os.WEXITSTATUS(status) == 0))
    com_after = os.stat(COM).st_mtime_ns if os.path.exists(COM) else None
    checks.append(("R wrote no .COM (it runs the image where it already is)",
                   com_before == com_after))

    print("UI TEST (R): %s" % os.path.basename(FIXTURE))
    print("-" * 60)
    ok = True
    for name, passed in checks:
        print("%-58s %s" % (name[:58], "PASS" if passed else "FAIL"))
        ok = ok and passed
    print("-" * 60)
    if not ok:
        # Only when something failed, and only what is needed to say why: the
        # captured run and the bytes wanted, so a red line is readable without
        # re-running it by hand.
        print("wanted: %r" % EXPECTED)
        print("captured after R:")
        for line in run.splitlines():
            if line.strip():
                print("   | %s" % line)
        print("steps: %d" % steps_of(run))
    print("child: %s" % status_str(status))
    print("RESULT: %s" % ("ALL PASS" if ok else "FAILURES PRESENT"))
    return 0 if ok else 1


if __name__ == "__main__":
    sys.exit(main())