TP3/shell/tests/corpus_check.py

#!/usr/bin/env python3
"""Compile the TP3 example corpus in Sources/Dezip/pascsrc.

The corpus is deliberately outside the build tree, so it is easy for a compiler
change to regress it without any of the fixture suites noticing.  This script
runs each source in its own compiletest process (one crashing file must not hide
the rest), prints a compact result, and requires the whole corpus to compile.

    python3 tests/corpus_check.py
    python3 tests/corpus_check.py --expected 71
    python3 tests/corpus_check.py --verbose

A missing Sources tree is a skip rather than a failure: the corpus is user
material and is not tracked by git.  Once the tree exists, both the file count
and every verdict are checked, so adding or losing a source is visible too.
"""

import argparse
import pathlib
import re
import subprocess
import sys

HERE = pathlib.Path(__file__).resolve().parent
SHELL = HERE.parent
SOURCES = SHELL.parent / "Sources"
CORPUS = SOURCES / "Dezip" / "pascsrc"
COMPILETEST = SHELL / "compiletest"
BUILD = HERE / "run_compile_tests.sh"


def build_compiletest() -> bool:
    """Build the harness through its single, timestamp-checked recipe."""
    r = subprocess.run([str(BUILD), "--build-only"], cwd=SHELL,
                       capture_output=True, text=True)
    if r.returncode != 0:
        print(r.stdout, end="")
        print(r.stderr, end="", file=sys.stderr)
        return False
    return True


def compile_one(path: pathlib.Path) -> tuple[bool, str]:
    try:
        r = subprocess.run([str(COMPILETEST)],
                           input=str(path) + "\n",
                           capture_output=True, text=True, timeout=5)
    except subprocess.TimeoutExpired:
        return False, "timeout"
    out = r.stdout + r.stderr
    if re.search(r"\bOK\s+code=\d+\s+data=\d+", out):
        return True, "OK"
    m = re.search(r"\bERROR\s+(\d+)\s+at pos\s+(\d+)", out)
    if m:
        return False, "ERROR %s at pos %s" % (m.group(1), m.group(2))
    if "ERROR" in out:
        return False, out.strip().splitlines()[-1]
    return False, "no compile result (exit %d)" % r.returncode


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--expected", type=int, default=71,
                    help="required number of .pas files (default: 71)")
    ap.add_argument("--verbose", action="store_true",
                    help="print every passing source as well as failures")
    args = ap.parse_args()

    if not CORPUS.is_dir():
        print("SKIP: %s does not exist" % CORPUS)
        return 0

    if not build_compiletest():
        print("FAIL: compiletest did not build")
        return 1

    files = sorted(CORPUS.rglob("*.[pP][aA][sS]"))
    if len(files) != args.expected:
        print("FAIL: expected %d .pas files, found %d in %s"
              % (args.expected, len(files), CORPUS))
        for p in files:
            print("  " + str(p.relative_to(SOURCES)))
        return 1

    failed = []
    for p in files:
        ok, detail = compile_one(p)
        rel = p.relative_to(SOURCES)
        if ok:
            if args.verbose:
                print("  PASS  %s" % rel)
        else:
            failed.append((rel, detail))
            print("  FAIL  %s  %s" % (rel, detail))

    print("\ncorpus: %d passed, %d failed (of %d)"
          % (len(files) - len(failed), len(failed), len(files)))
    return 1 if failed else 0


if __name__ == "__main__":
    sys.exit(main())