TP3/shell/tests/comtest.py

#!/usr/bin/env python3
"""End-to-end UI test: C writes a real .COM, verified by reading the disk.

uitest.py covers the error path (compile fails -> editor opens on the error).
This covers the success path, which is the one that produces an artifact:

  1. W loads a work file (LoadWorkFile ends with a Pause: needs a filler key)
  2. O opens the Options submenu and C selects Destination = .COM
  3. ESC returns to the main menu
  4. C compiles, and the shell reports the .COM it wrote
  5. The .COM is then INDEPENDENTLY checked on disk: the entry jump and the
     runtime's first bytes, a code-only file ending at hdrCS, the fixed data
     base outside the file, and the program header words

Step 5 is the point.  "The shell said it wrote a file" is not evidence that
the file is right, and the shell reports success from the same state that
produced the bytes, so the check that matters happens outside the shell.

The .COM is removed afterwards so the test can be re-run.

Usage: comtest.py [fixture.pas]

Passing a fixture that does NOT compile is expected to be RED - that is the
test being non-vacuous, not a failure of the shell.  It was checked that way
(t14_types.pas, which raises ENoLib for 'array' at its point of use).
"""
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from ptyharness import SHELL_DIR, Screen, drain, reap, send, spawn, status_str, visible

FIXTURE = os.path.abspath(sys.argv[1]) if len(sys.argv) > 1 else os.path.join(
    os.path.dirname(os.path.abspath(__file__)), "fixtures", "t26_str_mixed_args.pas")
COM = os.path.splitext(FIXTURE)[0] + ".COM"

# The image layout - the entry jump, the header's size, the load bias,
# initmem's first bytes - and the function that MEASURES where the header is
# live in tests/comimage.py and are imported, not copied.  This checker had one
# copy of both and the independent checker in tests/run_com_tests.sh had a
# second; the third reader (tests/check_framedisp.py) had neither and wrote
# down RT_SZ = 391 instead, which had long since drifted from the runtime it
# claimed to track, so its sweep began inside the runtime rather than at the
# program's first instruction and reported a clean PASS over bytes the program
# never executes.  Two copies of a constant that moves are two chances to be
# confidently wrong, so there is one now, and tests/check_comimage.py asserts
# that it stays one.
#
# RTSZ, PROLOG and DATAB below are derived per .COM from find_header(d) - never
# written down.  The image starts with a three-byte JMP (Compiler.Inittur) and
# a .COM is entered at file offset 0, so before that jump existed this checker
# ASSERTED that the runtime was at offset 0, which was precisely the bug: every
# .COM began by executing initmem with whatever the loader left in AX.  A
# checker that pins a wrong invariant is worse than none, because it makes the
# wrong thing look tested.
from comimage import (ENT_SZ, HDR_SZ, LOAD_BIAS, DATA_IMAGE, HEAD,
                      HDR_DS_WORD, HDR_HEAP_WORD, find_header)


def check_com(path, expected_src_len):
    """Return a list of problems (empty = the .COM is correct)."""
    errs = []
    if not os.path.exists(path):
        return ["no .COM file was written"]
    d = open(path, "rb").read()
    # Everything below is in terms of where the header actually is in THIS
    # file, measured, rather than where a literal once said it should be.
    hdr_off = find_header(d)
    if hdr_off is None:
        return ["no program header found: the layout this checker knows how to "
                "look for is not the one in the file, so every offset below "
                "would be a guess - reporting the guesses individually would "
                "be worse than saying so once"]
    RTSZ = hdr_off
    PROLOG = hdr_off + HDR_SZ
    DATAB = DATA_IMAGE
    cs = (int.from_bytes(d[RTSZ + 2:RTSZ + 4], "little") - LOAD_BIAS) \
        if len(d) >= RTSZ + 4 else 0
    # The entry jump.  This is the only assertion in the project that can see
    # where execution STARTS, because it is the only one that cares.  It has
    # caught three real bugs in these same three bytes: no jump at all; a jump
    # to `pc` (the END of the code, in the zero gap); and a jump to RTSZ,
    # which is the program HEADER - data, which the CPU then decodes as
    # instructions.  That last one is why the target is PROLOG and not RTSZ:
    # whether it works depends on how the header happens to decode, so
    # writeln('hi') slid through it unharmed while t07 hung on a LOCK-prefixed
    # ADD.  Every other check here reads bytes at an offset the compiler chose
    # for itself.
    if len(d) >= ENT_SZ:
        if d[0] != 0xE9:
            errs.append("byte 0 is %02X, not the E9 of the entry jump" % d[0])
        want_rel = PROLOG - ENT_SZ
        got_rel = int.from_bytes(d[1:ENT_SZ], "little")
        if got_rel != want_rel:
            errs.append("entry jump rel16=%d, want %d; lands on image offset "
                        "%d, want %d (the first instruction, %d bytes past the "
                        "header - not the header at %d)"
                        % (got_rel, want_rel, ENT_SZ + got_rel, PROLOG,
                           HDR_SZ, RTSZ))
    if d[ENT_SZ:ENT_SZ + len(HEAD.split())].hex(" ").upper() != HEAD:
        errs.append("runtime not at offset %d: bytes there %s, want %s"
                    % (ENT_SZ,
                       d[ENT_SZ:ENT_SZ + len(HEAD.split())].hex(" ").upper(),
                       HEAD))
    # The .COM is code-only.  hdrCS - bias is the code end, which must be the
    # file length; the globals at DATA_IMAGE are outside the file and initmem
    # zeroes them.
    if cs != len(d):
        errs.append("file is %d bytes, but hdrCS says the code ends at %d"
                    % (len(d), cs))
    # the runtime must be followed immediately by the program header,
    # whose first word is the "image is valid" flag
    if len(d) >= RTSZ + 2:
        flag = int.from_bytes(d[RTSZ:RTSZ + 2], "little")
        if flag != 1:
            errs.append("hdrFlag=%d at the program header, want 1" % flag)
    if len(d) >= RTSZ + 10:
        ds = int.from_bytes(d[RTSZ + HDR_DS_WORD:RTSZ + HDR_DS_WORD + 2], "little")
        heap = int.from_bytes(d[RTSZ + HDR_HEAP_WORD:RTSZ + HDR_HEAP_WORD + 2], "little")
        if ds != DATAB + LOAD_BIAS:
            errs.append("hdrDS=%d, want %d (= data base %d + load bias %d)"
                        % (ds, DATAB + LOAD_BIAS, DATAB, LOAD_BIAS))
        if heap < ds:
            errs.append("hdrHeap=%d < hdrDS=%d" % (heap, ds))
        if [d[ENT_SZ + 4], d[ENT_SZ + 7]] != [HDR_DS_WORD, HDR_HEAP_WORD]:
            errs.append("initmem reads header words +%d/+%d, but the data base "
                        "and data end are at +%d/+%d"
                        % (d[ENT_SZ + 4], d[ENT_SZ + 7],
                           HDR_DS_WORD, HDR_HEAP_WORD))
    return errs


def main():
    if os.path.exists(COM):
        os.unlink(COM)
    pid, fd = spawn()
    checks = []
    try:
        drain(fd, quiet=0.6)
        send(fd, b"w")
        send(fd, FIXTURE.encode())
        send(fd, b"\r", quiet=0.8)
        send(fd, b" ", quiet=0.5)              # clears LoadWorkFile's Pause

        # Options submenu -> Destination = .COM.
        # The submenu is left with Q (that is its Quit key, as in the
        # original's optmenu); ESC is swallowed by its REPEAT loop.
        send(fd, b"o", quiet=0.8)
        opt = visible(send(fd, b"c", quiet=0.5))
        checks.append(("Options submenu opened", "Compile" in opt))
        checks.append(("selecting C set Destination = COM", "COM" in opt))
        after = visible(send(fd, b"q", quiet=0.8))
        checks.append(("Q left the submenu for the main menu", "Main file" in after))

        comp = visible(send(fd, b"c", quiet=2.0))
        checks.append(("compile reported success", "Compiled OK" in comp))
        checks.append(("shell reported the Destination it used",
                       "Destination" in comp))
        checks.append(("shell named the .COM it wrote",
                       os.path.basename(COM) in comp))

        send(fd, b"\x1b", quiet=0.6)
        send(fd, b"q", quiet=0.8)
    finally:
        status = reap(fd, pid)

    checks.append(("shell exited cleanly (status 0)", os.WIFEXITED(status)
                   and os.WEXITSTATUS(status) == 0))

    # now verify the artifact independently
    errs = check_com(COM, len(open(FIXTURE, "rb").read()))
    checks.append((".COM exists on disk and passes the independent check"
                   if not errs else ".COM check: " + "; ".join(errs), not errs))
    checks.append((".COM was written next to the source, not in the shell's cwd",
                   os.path.dirname(os.path.abspath(COM)) == os.path.dirname(FIXTURE)))

    print("UI TEST: %s" % os.path.basename(FIXTURE))
    print("-" * 60)
    ok = True
    for name, passed in checks:
        print("%-58s %s" % (name[:58], "PASS" if passed else "FAIL"))
        ok = ok and passed
    print("-" * 60)
    print("child: %s" % status_str(status))
    print("RESULT: %s" % ("ALL PASS" if ok else "FAILURES PRESENT"))
    if os.path.exists(COM):
        print("(removed %s)" % COM)
        os.unlink(COM)
    return 0 if ok else 1


if __name__ == "__main__":
    sys.exit(main())