TP3/shell/tests/comtest.py

#!/usr/bin/env python3
"""End-to-end UI test: C writes a real .COM, verified by reading the disk.

uitest.py covers the error path (compile fails -> editor opens on the error).
This covers the success path, which is the one that produces an artifact:

  1. W loads a work file (LoadWorkFile ends with a Pause: needs a filler key)
  2. O opens the Options submenu and C selects Destination = .COM
  3. ESC returns to the main menu
  4. C compiles, and the shell reports the .COM it wrote
  5. The .COM is then INDEPENDENTLY checked on disk: the runtime's first
     bytes, a fully zero code/data gap, a size that covers the data area, and
     the program header words

Step 5 is the point.  "The shell said it wrote a file" is not evidence that
the file is right, and the shell reports success from the same state that
produced the bytes, so the check that matters happens outside the shell.

The .COM is removed afterwards so the test can be re-run.

Usage: comtest.py [fixture.pas]

Passing a fixture that does NOT compile is expected to be RED - that is the
test being non-vacuous, not a failure of the shell.  It was checked that way
(t14_types.pas, which raises ENoLib for 'array' at its point of use).
"""
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from ptyharness import SHELL_DIR, Screen, drain, reap, send, spawn, status_str, visible

FIXTURE = os.path.abspath(sys.argv[1]) if len(sys.argv) > 1 else os.path.join(
    os.path.dirname(os.path.abspath(__file__)), "fixtures", "t26_str_mixed_args.pas")
COM = os.path.splitext(FIXTURE)[0] + ".COM"

# Restated here on purpose - the checker must not ask the code under test.
# ENT_SZ and HDR_SZ are the layout, and the load bias below is a decision this
# checker made on its own account; those stay written down.
#
# The RUNTIME'S SIZE is not restated, and that is the correction.  It used to
# be a literal (385, then 391) beside a comment saying it tracks
# Runtime.RT_Size().  It did not: the runtime is 432 bytes, so this checker
# read the program header 41 bytes early, out of the middle of the code, and
# reported a .COM full of nonsense - hdrFlag=61579 - as a compiler fault.  A
# duplicated constant that has drifted is not an independent check, it is a
# second source of truth that lies, and it lies in a way that looks like the
# thing under test is broken.  The size is now MEASURED, by the same
# self-consistent-header argument documented in find_header below.
ENT_SZ = 3                    # E9 lo hi, the entry jump
HDR_SZ = 16                   # 5 header words + 3 buffer words
# RTSZ, PROLOG and DATAB are derived per .COM from find_header(d).
# The image starts with a three-byte JMP - see Compiler.Inittur.  A .COM is
# entered at file offset 0, so before that jump existed this checker ASSERTED
# that the runtime was at offset 0, which was precisely the bug: every .COM
# began by executing initmem with whatever the loader left in AX.  A checker
# that pins a wrong invariant is worse than none, because it makes the wrong
# thing look tested.
# The load bias.  A DOS .COM's first byte is at CS:0100 and CS = DS, so an
# image offset K is at DS:(K + 0100h); every absolute address the image
# contains has to carry it, or it points 0100h low and - since the code region
# and the runtime are all below the bias - almost always lands inside the
# runtime instead of inside the data.  Relative encodings must not carry it.
# Restated, not asked of the code under test.  See Runtime.LoadBias.
LOAD_BIAS = 0x100
HEAD = "8B F0 8B 54 04 8B 4C 06"   # MOV SI,AX / MOV DX,[SI+4] / MOV CX,[SI+6]
HDR_DS_WORD = 4               # header word holding the data base
HDR_HEAP_WORD = 6             # header word holding the data end
assert [int(HEAD.split()[4], 16), int(HEAD.split()[7], 16)] == \
       [HDR_DS_WORD, HDR_HEAP_WORD], \
       "initmem no longer reads the two header words this checker verifies"


def find_header(d):
    """Return the image offset of the program header, or None.

    The header is eight words, and hdrDS ties it to its OWN offset: the data
    base is header offset + 1000h, and hdrDS is the data base with the load
    bias added, so header offset = hdrDS - 1000h - bias.  That makes the
    offset recoverable from the file with no remembered runtime size, which is
    the whole point - see the note on RT_SZ above.

    A candidate is accepted only if hdrFlag is 1, hdrDS satisfies that
    equation, hdrHeap is above hdrDS, and hdrCS leaves room for the header
    itself.  initmem is the only code in the image that reads the header, so
    its bytes are pinned at ENT_SZ and a match that also has them is not a
    coincidence in the code stream.

    Measuring is not the same as asking the compiler: this reads the emitted
    file, so it cannot be satisfied by the code under test agreeing with
    itself.  tests/check_runtime.py is where the runtime's size is pinned on
    purpose, and tests/run_com_tests.sh measures and prints it on every run.
    """
    head = bytes(int(x, 16) for x in HEAD.split())
    if d[ENT_SZ:ENT_SZ + len(head)] != head:
        return None
    for off in range(ENT_SZ, len(d) - HDR_SZ + 1):
        w = (lambda b: int.from_bytes(d[off + b:off + b + 2], "little"))
        if w(0) != 1:
            continue
        if w(HDR_DS_WORD) != off + 0x1000 + LOAD_BIAS:
            continue
        if w(HDR_HEAP_WORD) <= w(HDR_DS_WORD):
            continue
        if w(2) - LOAD_BIAS < off + HDR_SZ:
            continue
        return off
    return None


def check_com(path, expected_src_len):
    """Return a list of problems (empty = the .COM is correct)."""
    errs = []
    if not os.path.exists(path):
        return ["no .COM file was written"]
    d = open(path, "rb").read()
    # Everything below is in terms of where the header actually is in THIS
    # file, measured, rather than where a literal once said it should be.
    hdr_off = find_header(d)
    if hdr_off is None:
        return ["no program header found: the layout this checker knows how to "
                "look for is not the one in the file, so every offset below "
                "would be a guess - reporting the guesses individually would "
                "be worse than saying so once"]
    RTSZ = hdr_off
    PROLOG = hdr_off + HDR_SZ
    DATAB = hdr_off + 0x1000
    # The entry jump.  This is the only assertion in the project that can see
    # where execution STARTS, because it is the only one that cares.  It has
    # caught three real bugs in these same three bytes: no jump at all; a jump
    # to `pc` (the END of the code, in the zero gap); and a jump to RTSZ,
    # which is the program HEADER - data, which the CPU then decodes as
    # instructions.  That last one is why the target is PROLOG and not RTSZ:
    # whether it works depends on how the header happens to decode, so
    # writeln('hi') slid through it unharmed while t07 hung on a LOCK-prefixed
    # ADD.  Every other check here reads bytes at an offset the compiler chose
    # for itself.
    if len(d) >= ENT_SZ:
        if d[0] != 0xE9:
            errs.append("byte 0 is %02X, not the E9 of the entry jump" % d[0])
        want_rel = PROLOG - ENT_SZ
        got_rel = int.from_bytes(d[1:ENT_SZ], "little")
        if got_rel != want_rel:
            errs.append("entry jump rel16=%d, want %d; lands on image offset "
                        "%d, want %d (the first instruction, %d bytes past the "
                        "header - not the header at %d)"
                        % (got_rel, want_rel, ENT_SZ + got_rel, PROLOG,
                           HDR_SZ, RTSZ))
    if d[ENT_SZ:ENT_SZ + len(HEAD.split())].hex(" ").upper() != HEAD:
        errs.append("runtime not at offset %d: bytes there %s, want %s"
                    % (ENT_SZ,
                       d[ENT_SZ:ENT_SZ + len(HEAD.split())].hex(" ").upper(),
                       HEAD))
    if len(d) < DATAB:
        errs.append("file is %d bytes, shorter than the data base %d - the "
                    "globals would be outside the file" % (len(d), DATAB))
    # the runtime must be followed immediately by the program header,
    # whose first word is the "image is valid" flag
    if len(d) >= RTSZ + 2:
        flag = int.from_bytes(d[RTSZ:RTSZ + 2], "little")
        if flag != 1:
            errs.append("hdrFlag=%d at the program header, want 1" % flag)
    if len(d) >= RTSZ + 10:
        ds = int.from_bytes(d[RTSZ + HDR_DS_WORD:RTSZ + HDR_DS_WORD + 2], "little")
        heap = int.from_bytes(d[RTSZ + HDR_HEAP_WORD:RTSZ + HDR_HEAP_WORD + 2], "little")
        if ds != DATAB + LOAD_BIAS:
            errs.append("hdrDS=%d, want %d (= data base %d + load bias %d)"
                        % (ds, DATAB + LOAD_BIAS, DATAB, LOAD_BIAS))
        if heap < ds:
            errs.append("hdrHeap=%d < hdrDS=%d" % (heap, ds))
        if [d[ENT_SZ + 4], d[ENT_SZ + 7]] != [HDR_DS_WORD, HDR_HEAP_WORD]:
            errs.append("initmem reads header words +%d/+%d, but the data base "
                        "and data end are at +%d/+%d"
                        % (d[ENT_SZ + 4], d[ENT_SZ + 7],
                           HDR_DS_WORD, HDR_HEAP_WORD))
    # The gap between the end of the code and the data area must be all zero.
    # hdrCS is a SEGMENT offset, like every other offset in the header, so
    # LOAD_BIAS comes back off before it is used to index the file.
    cs = (int.from_bytes(d[RTSZ + 2:RTSZ + 4], "little") - LOAD_BIAS) \
        if len(d) >= RTSZ + 4 else 0
    if cs and cs < DATAB:
        nz = sum(1 for b in d[cs:DATAB] if b)
        if nz:
            errs.append("%d non-zero bytes in the code/data gap" % nz)
    return errs


def main():
    if os.path.exists(COM):
        os.unlink(COM)
    pid, fd = spawn()
    checks = []
    try:
        drain(fd, quiet=0.6)
        send(fd, b"w")
        send(fd, FIXTURE.encode())
        send(fd, b"\r", quiet=0.8)
        send(fd, b" ", quiet=0.5)              # clears LoadWorkFile's Pause

        # Options submenu -> Destination = .COM.
        # The submenu is left with Q (that is its Quit key, as in the
        # original's optmenu); ESC is swallowed by its REPEAT loop.
        send(fd, b"o", quiet=0.8)
        opt = visible(send(fd, b"c", quiet=0.5))
        checks.append(("Options submenu opened", "Compile" in opt))
        checks.append(("selecting C set Destination = COM", "COM" in opt))
        after = visible(send(fd, b"q", quiet=0.8))
        checks.append(("Q left the submenu for the main menu", "Main file" in after))

        comp = visible(send(fd, b"c", quiet=2.0))
        checks.append(("compile reported success", "Compiled OK" in comp))
        checks.append(("shell reported the Destination it used",
                       "Destination" in comp))
        checks.append(("shell named the .COM it wrote",
                       os.path.basename(COM) in comp))

        send(fd, b"\x1b", quiet=0.6)
        send(fd, b"q", quiet=0.8)
    finally:
        status = reap(fd, pid)

    checks.append(("shell exited cleanly (status 0)", os.WIFEXITED(status)
                   and os.WEXITSTATUS(status) == 0))

    # now verify the artifact independently
    errs = check_com(COM, len(open(FIXTURE, "rb").read()))
    checks.append((".COM exists on disk and passes the independent check"
                   if not errs else ".COM check: " + "; ".join(errs), not errs))
    checks.append((".COM was written next to the source, not in the shell's cwd",
                   os.path.dirname(os.path.abspath(COM)) == os.path.dirname(FIXTURE)))

    print("UI TEST: %s" % os.path.basename(FIXTURE))
    print("-" * 60)
    ok = True
    for name, passed in checks:
        print("%-58s %s" % (name[:58], "PASS" if passed else "FAIL"))
        ok = ok and passed
    print("-" * 60)
    print("child: %s" % status_str(status))
    print("RESULT: %s" % ("ALL PASS" if ok else "FAILURES PRESENT"))
    if os.path.exists(COM):
        print("(removed %s)" % COM)
        os.unlink(COM)
    return 0 if ok else 1


if __name__ == "__main__":
    sys.exit(main())