TP3/shell/tests/comtest.py

#!/usr/bin/env python3
"""End-to-end UI test: C writes a real .COM, verified by reading the disk.

uitest.py covers the error path (compile fails -> editor opens on the error).
This covers the success path, which is the one that produces an artifact:

  1. W loads a work file (LoadWorkFile ends with a Pause: needs a filler key)
  2. O opens the Options submenu and C selects Destination = .COM
  3. ESC returns to the main menu
  4. C compiles, and the shell reports the .COM it wrote
  5. The .COM is then INDEPENDENTLY checked on disk: the runtime's first
     bytes, a fully zero code/data gap, a size that covers the data area, and
     the program header words

Step 5 is the point.  "The shell said it wrote a file" is not evidence that
the file is right, and the shell reports success from the same state that
produced the bytes, so the check that matters happens outside the shell.

The .COM is removed afterwards so the test can be re-run.

Usage: comtest.py [fixture.pas]

Passing a fixture that does NOT compile is expected to be RED - that is the
test being non-vacuous, not a failure of the shell.  It was checked that way
(t14_types.pas, which raises ENoLib for 'array' at its point of use).
"""
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from ptyharness import SHELL_DIR, Screen, drain, reap, send, spawn, status_str, visible

FIXTURE = os.path.abspath(sys.argv[1]) if len(sys.argv) > 1 else os.path.join(
    os.path.dirname(os.path.abspath(__file__)), "fixtures", "t26_str_mixed_args.pas")
COM = os.path.splitext(FIXTURE)[0] + ".COM"

# Restated here on purpose - the checker must not ask the code under test.
# RTSZ was re-baselined 385 -> 391 together with the runtime bug fixes; see
# the rationale in tests/run_com_tests.sh and tests/runtime.golden.  HEAD is
# initmem's prologue, whose two SI displacements are the header words this
# checker goes on to verify, so the two are tied together by assertion rather
# than by coincidence (that is how initmem came to read hdrMax instead of
# hdrHeap and silently zero nothing).
RTSZ = 391                    # Runtime.RT_Size()
DATAB = RTSZ + 0x1000         # Compiler: data base = rtSz + 1000H
HEAD = "8B F0 8B 54 04 8B 4C 06"   # MOV SI,AX / MOV DX,[SI+4] / MOV CX,[SI+6]
HDR_DS_WORD = 4               # header word holding the data base
HDR_HEAP_WORD = 6             # header word holding the data end
assert [int(HEAD.split()[4], 16), int(HEAD.split()[7], 16)] == \
       [HDR_DS_WORD, HDR_HEAP_WORD], \
       "initmem no longer reads the two header words this checker verifies"


def check_com(path, expected_src_len):
    """Return a list of problems (empty = the .COM is correct)."""
    errs = []
    if not os.path.exists(path):
        return ["no .COM file was written"]
    d = open(path, "rb").read()
    if d[:len(HEAD.split())].hex(" ").upper() != HEAD:
        errs.append("runtime not at offset 0: first bytes %s, want %s"
                    % (d[:len(HEAD.split())].hex(" ").upper(), HEAD))
    if len(d) < DATAB:
        errs.append("file is %d bytes, shorter than the data base %d - the "
                    "globals would be outside the file" % (len(d), DATAB))
    # the runtime must be followed immediately by the program header,
    # whose first word is the "image is valid" flag
    if len(d) >= RTSZ + 2:
        flag = int.from_bytes(d[RTSZ:RTSZ + 2], "little")
        if flag != 1:
            errs.append("hdrFlag=%d at the program header, want 1" % flag)
    if len(d) >= RTSZ + 10:
        ds = int.from_bytes(d[RTSZ + HDR_DS_WORD:RTSZ + HDR_DS_WORD + 2], "little")
        heap = int.from_bytes(d[RTSZ + HDR_HEAP_WORD:RTSZ + HDR_HEAP_WORD + 2], "little")
        if ds != DATAB:
            errs.append("hdrDS=%d, want %d" % (ds, DATAB))
        if heap < ds:
            errs.append("hdrHeap=%d < hdrDS=%d" % (heap, ds))
        if [d[4], d[7]] != [HDR_DS_WORD, HDR_HEAP_WORD]:
            errs.append("initmem reads header words +%d/+%d, but the data base "
                        "and data end are at +%d/+%d"
                        % (d[4], d[7], HDR_DS_WORD, HDR_HEAP_WORD))
    # the gap between the end of the code and the data area must be all zero
    cs = int.from_bytes(d[RTSZ + 2:RTSZ + 4], "little") if len(d) >= RTSZ + 4 else 0
    if cs and cs < DATAB:
        nz = sum(1 for b in d[cs:DATAB] if b)
        if nz:
            errs.append("%d non-zero bytes in the code/data gap" % nz)
    return errs


def main():
    if os.path.exists(COM):
        os.unlink(COM)
    pid, fd = spawn()
    checks = []
    try:
        drain(fd, quiet=0.6)
        send(fd, b"w")
        send(fd, FIXTURE.encode())
        send(fd, b"\r", quiet=0.8)
        send(fd, b" ", quiet=0.5)              # clears LoadWorkFile's Pause

        # Options submenu -> Destination = .COM.
        # The submenu is left with Q (that is its Quit key, as in the
        # original's optmenu); ESC is swallowed by its REPEAT loop.
        send(fd, b"o", quiet=0.8)
        opt = visible(send(fd, b"c", quiet=0.5))
        checks.append(("Options submenu opened", "Compile" in opt))
        checks.append(("selecting C set Destination = COM", "COM" in opt))
        after = visible(send(fd, b"q", quiet=0.8))
        checks.append(("Q left the submenu for the main menu", "Main file" in after))

        comp = visible(send(fd, b"c", quiet=2.0))
        checks.append(("compile reported success", "Compiled OK" in comp))
        checks.append(("shell reported the Destination it used",
                       "Destination" in comp))
        checks.append(("shell named the .COM it wrote",
                       os.path.basename(COM) in comp))

        send(fd, b"\x1b", quiet=0.6)
        send(fd, b"q", quiet=0.8)
    finally:
        status = reap(fd, pid)

    checks.append(("shell exited cleanly (status 0)", os.WIFEXITED(status)
                   and os.WEXITSTATUS(status) == 0))

    # now verify the artifact independently
    errs = check_com(COM, len(open(FIXTURE, "rb").read()))
    checks.append((".COM exists on disk and passes the independent check"
                   if not errs else ".COM check: " + "; ".join(errs), not errs))
    checks.append((".COM was written next to the source, not in the shell's cwd",
                   os.path.dirname(os.path.abspath(COM)) == os.path.dirname(FIXTURE)))

    print("UI TEST: %s" % os.path.basename(FIXTURE))
    print("-" * 60)
    ok = True
    for name, passed in checks:
        print("%-58s %s" % (name[:58], "PASS" if passed else "FAIL"))
        ok = ok and passed
    print("-" * 60)
    print("child: %s" % status_str(status))
    print("RESULT: %s" % ("ALL PASS" if ok else "FAILURES PRESENT"))
    if os.path.exists(COM):
        print("(removed %s)" % COM)
        os.unlink(COM)
    return 0 if ok else 1


if __name__ == "__main__":
    sys.exit(main())