corpus: SYSTEM storage types, anchored subranges, unnamed proc-type params, import resolution

- SymTab: predefine SYSTEM storage types BYTE/SBYTE/WORD/SWORD/LONGWORD/
  SHORTADDR/LONGADDR/PROC/LOC (-> integer family / ADDRESS), so
  FROM SYSTEM IMPORT BYTE and ARRAY OF BYTE resolve.
- M2.atg Type: anchored subranges T[lo..hi] (INTEGER [-1 .. 24]).
- M2.atg ProcTypeSection: allow unnamed parameters (VAR ARRAY OF REAL)
  and open arrays in procedure types; SymTab.ParamTypesOk gives
  structural proc-type formal compatibility (open array vs 1-D array).
- tools/v3-corpus/corpus.sh: resolve imported modules via $V3_LIBS
  (pass each import's .def), removing LIB noise.
- Results (with $V3_LIBS): gm2 testsuite 272 OK/331 LANG (was 242/185),
  gm2 stdlib 115 OK/221 LANG (was 78/82); CRASH 0; testsuite LIB 185->9.
- Suite 152/152; fixpoint OK (2,364,879 bytes).

compiler/src/M2.atg

@@ -244,7 +244,7 @@ PRODUCTIONS
                                            ELSE SymTab.SetSymType(n,
                                              SymTab.NewAlias()) END; .) ) .
   Type<VAR t: SymTab.TypeIndex; allowOpen: BOOLEAN>
-    = TypeIdent<t>
+    = TypeIdent<t> [ Subrange<t> ]     (* anchored subrange: T[lo..hi] *)
     | Subrange<t>
     | Enum<t>
     | ArrayType<t, allowOpen>
@@ -268,14 +268,14 @@ PRODUCTIONS
                                              names: ARRAY [0 .. 15] OF SymTab.Name; .)
     =                                   (. isV := FALSE; cnt := 0; .)
       [ "VAR"                           (. isV := TRUE; .) ]
-      GetIdent<names[cnt]>              (. INC(cnt); .)
-      { "," GetIdent<names[cnt]>        (. INC(cnt); .) }
-      ( ":" Type<pt, FALSE>             (. k := 0;
+      ( GetIdent<names[cnt]>            (. INC(cnt); .)
+        { "," GetIdent<names[cnt]>      (. INC(cnt); .) }
+        ( ":" Type<pt, TRUE>           (. k := 0;
                                            WHILE k < cnt DO
                                              SymTab.ProcTypeAdd(t, isV, pt);
                                              INC(k)
                                            END; .)
-      |                                 (. (* type-only parameter list:
+        |                               (. (* type-only parameter list:
                                               each name is a type (GNU
                                               shorthand used by the
                                               Coco/R scanner frame) *)
@@ -290,7 +290,10 @@ PRODUCTIONS
                                              END;
                                              SymTab.ProcTypeAdd(t, isV, pt);
                                              INC(k)
-                                           END; .) ) .
+                                           END; .) )
+      | Type<pt, TRUE>                 (. (* unnamed parameter (PIM):
+                                              e.g. PROCEDURE (VAR ARRAY OF REAL) *)
+                                           SymTab.ProcTypeAdd(t, isV, pt); .) ) .
   (* Arrays: "OF" without bounds is an open formal (allowed only
      where allowOpen); "[lo..hi, ...]" nests bounded levels inside
      out. Bounds are folded literals (int/char); anything else 230.

compiler/src/SymTab.mod

@@ -2491,6 +2491,22 @@ PROCEDURE SetBasesOk (a, b: TypeIndex): BOOLEAN;
     RETURN BaseSpanOk(a) AND BaseSpanOk(b)
   END SetBasesOk;
 
+PROCEDURE ParamTypesOk (x, y: TypeIndex): BOOLEAN;
+(* Procedure-type formal compatibility: identical, or an open array
+   matching an open/fixed 1-D array of the same element type (open
+   arrays are distinct descriptors, so index equality is too strict). *)
+  BEGIN
+    IF SameType(x, y) THEN RETURN TRUE END;
+    IF (x = InvalidType) OR (y = InvalidType) THEN RETURN TRUE END;
+    IF IsOpenArray(x) AND (ClassOf(y) = ClArray) AND (ArrayDepth(y) = 1) THEN
+      RETURN SameType(ArrayElem(x), ArrayElem(y))
+    END;
+    IF IsOpenArray(y) AND (ClassOf(x) = ClArray) AND (ArrayDepth(x) = 1) THEN
+      RETURN SameType(ArrayElem(x), ArrayElem(y))
+    END;
+    RETURN FALSE
+  END ParamTypesOk;
+
 PROCEDURE ProcTypesOk (a, b: TypeIndex): BOOLEAN;
 (* Structural compatibility of two procedure types: same result and
    same parameter types / VAR flags. *)
@@ -2505,8 +2521,8 @@ PROCEDURE ProcTypesOk (a, b: TypeIndex): BOOLEAN;
       IF procvis[procstart[a] + i] # procvis[procstart[b] + i] THEN
         RETURN FALSE
       END;
-      IF NOT SameType(procp[procstart[a] + i],
-                      procp[procstart[b] + i]) THEN
+      IF NOT ParamTypesOk(procp[procstart[a] + i],
+                          procp[procstart[b] + i]) THEN
         RETURN FALSE
       END;
       INC(i)
@@ -2805,6 +2821,19 @@ PROCEDURE Init;
     Predef("ADDRESS", KindPredef, dAddr);
     Predef("UCHAR", KindPredef, dUChar);
     Predef("BITSET", KindPredef, dBitset);
+    (* SYSTEM storage types (PIM/gm2): mapped onto V3's integer family
+       (BYTE/WORD -> the 32-bit ordinal, LONGWORD -> LONGINT, the
+       address forms -> ADDRESS) so `FROM SYSTEM IMPORT BYTE;` and
+       `ARRAY OF BYTE` resolve. *)
+    Predef("BYTE", KindPredef, dCard);
+    Predef("SBYTE", KindPredef, dInt);
+    Predef("WORD", KindPredef, dCard);
+    Predef("SWORD", KindPredef, dInt);
+    Predef("LONGWORD", KindPredef, dLong);
+    Predef("SHORTADDR", KindPredef, dAddr);
+    Predef("LONGADDR", KindPredef, dAddr);
+    Predef("PROC", KindPredef, dAddr);
+    Predef("LOC", KindPredef, dAddr);
     Predef("NIL", KindConst, dNil)
   END Init;
 

docs/summary_v3-corpus.md

@@ -8,88 +8,88 @@ The TopSpeed corpus validated the *sidecar grammar*, not V3 the
 compiler (the sidecar is a separate `TSM2` binary built from
 `TopSpeed-V3-M2.atg`; it reuses V3's `SymTab`/`QbeGen`/`FileIO` sources
 but never runs `compiler/M2`).  So this work runs **V3 itself** over two
-real Modula-2 corpora.
+real Modula-2 corpora and fixes what it finds.
 
 Harness: `tools/v3-corpus/corpus.sh` — compiles each file with
-`compiler/M2` alone (pairing an IMPLEMENTATION with its sibling
-DEFINITION), in a scratch tree, and splits failures into CRASH (M2
-died) / LIB (imports a module outside V3's known stdlib) / LANG (real
+`compiler/M2` in a scratch tree, together with its sibling DEFINITION
+and the DEFINITION of every module it imports (`$V3_LIBS` search path),
+and splits failures into CRASH / LIB (unresolved import) / LANG (real
 gap).
 
 ## Corpora and results
 
-**1. gm2 testsuite** — `testsuite/gm2/pim/pass` + `iso/run/pass`
-(612 files):
+**1. gm2 testsuite** (`gm2/pim/pass` + `gm2/iso/run/pass`, 612 files):
 
-| | files |
-| --- | --- |
-| **compile-OK** | **242** |
-| CRASH | 0 |
-| LIB (missing gm2 library) | 185 |
-| LANG (real V3 gap) | 185 |
+| | no libs | with `$V3_LIBS` |
+| --- | --- | --- |
+| **compile-OK** | 242 | **272** |
+| CRASH | 0 | 0 |
+| LIB | 185 | **9** |
+| LANG | 185 | **331** |
 
-**2. gm2 standard libraries** — `$HOME/bin/Modula2/Gm2/lib/gcc/…/m2/`
-(`m2pim`/`m2iso`/`m2log`/`m2cor`/`m2min`, 336 files, `.def`+`.mod`):
+**2. gm2 standard libraries** (`m2pim`/`m2iso`/`m2log`/`m2cor`/`m2min`,
+336 files):
 
-| | files |
-| --- | --- |
-| **compile-OK** | **78** |
-| CRASH | 0 |
-| LIB | 176 |
-| LANG | 82 |
+| | no libs | with `$V3_LIBS` |
+| --- | --- | --- |
+| **compile-OK** | 78 | **115** |
+| LIB | 176 | **0** |
+| LANG | 82 | **221** |
 
-(LIB is high in both because each file is compiled alone — cross-module
-imports are not resolved; the real signal is LANG.)
+(With the import path set, LIB noise is gone and the true LANG counts
+are visible.)
 
 ## Fixes landed
 
-### A. Long-line buffer overflow (the 7 "set" crashes) — `v3-fix-longline-crash`
-
-`PrintListing`'s line buffer was `ARRAY [0 .. 255]` and `GetLine` wrote
-without bounding to `HIGH(line)`, so a source line > 255 chars (an
-860-char enum line in the gm2 set tests) overflowed the stack.  Fixed
-in `compiler/src/compiler.frm` (truncate in `GetLine`, buffer →
-4096, bounded caret loop); regression `t_longline.mod`; CRASH → 0.
-
-### B. `EXPORT QUALIFIED` in DEFINITION modules — this step
-
-gm2/PIM definition modules carry an export list
-(`EXPORT QUALIFIED ReadString, WriteString, …;`).  V3's `DefUnit` did
-not accept it → `'END' expected` on the first such file.  Added the
-clause (parsed; the names are already exported by the module scope).
-This raised **both** corpora: testsuite compile-OK **219 → 242**, gm2
-libs **42 → 78**.
+1. **Long-line buffer overflow** (`v3-fix-longline-crash`) — `GetLine`
+   wrote past a 256-byte listing buffer; a long line (an 860-char enum)
+   overflowed the stack.  Fixed (`GetLine` truncates, buffer 4096,
+   bounded caret loop); CRASH → 0.
+2. **`EXPORT QUALIFIED`** in DEFINITION modules (`v3-export-qualified`).
+3. **`SYSTEM` facilities** — predefine `BYTE`/`SBYTE`/`WORD`/`SWORD`/
+   `LONGWORD`/`SHORTADDR`/`LONGADDR`/`PROC`/`LOC` (mapped onto V3's
+   integer family / `ADDRESS`) so `FROM SYSTEM IMPORT BYTE;` and
+   `ARRAY OF BYTE` resolve.
+4. **Anchored subranges** `T[lo..hi]` (`INTEGER [-1 .. 24]`).
+5. **Unnamed procedure-type parameters** (`PROCEDURE (VAR ARRAY OF
+   REAL)`) plus **open arrays in procedure types** and structural
+   proc-type formal compatibility (`ParamTypesOk`), so assigning a
+   procedure with an open-array parameter to a procedure variable works.
+6. **Harness import resolution** — `corpus.sh` now finds and passes the
+   `.def` of each imported module (`$V3_LIBS`), removing the LIB noise.
 
 ## Remaining LANG gaps
 
-- Tests suite: **185** files, mostly gm2-specific libraries' call sites
-  and deliberate V3 `230`s (array constructors, …).
-- gm2 libs: **82** files, dominated by **gm2-specific syntax** —
-  `PROCEDURE __BUILTIN__ name(...)` (`Builtins`/`libc`/`libm`/…),
-  C-name pragmas, ISO module attributes — which are *not* V3's dialect.
-- Genuine V3 gaps still seen: **`SYSTEM` facilities** (`FROM SYSTEM
-  IMPORT BYTE`), **anchored subranges** `T[lo..hi]`, **unnamed
-  procedure-type parameters**.
+- **`ARRAY <ordinalType> OF T`** (index-type arrays, e.g.
+  `ARRAY btype OF INTEGER` in `array2`/`array3`) — V3 only has
+  `ARRAY [lo..hi] OF` / `ARRAY OF`.  The big one.
+- **`undeclared identifier`** — SYSTEM facilities beyond the set
+  above, and small library holes.
+- **gm2-specific syntax** — `PROCEDURE __BUILTIN__ name(…)`, C-name
+  pragmas, ISO module attributes (not V3's dialect; skip).
+- **`not supported yet`** — deliberate `230`s (array constructors, …).
 
 ## Reproduce
 
 ```sh
 cd tools/v3-corpus
-./corpus.sh                       # gm2 testsuite -> 242 OK / 185 LANG
+./corpus.sh                       # testsuite, no libs
 G=$HOME/bin/Modula2/Gm2/lib/gcc/x86_64-pc-linux-gnu/16.0.1/m2
-./corpus.sh $G/m2pim $G/m2iso $G/m2log $G/m2cor $G/m2min   # 78 OK
+V3_LIBS="$G/m2pim:$G/m2iso:$G/m2log:$G/m2cor:$G/m2min" ./corpus.sh
+V3_LIBS="$G/m2pim:$G/m2iso:$G/m2log:$G/m2cor:$G/m2min" \
+  ./corpus.sh "$G/m2pim" "$G/m2iso" "$G/m2log" "$G/m2cor" "$G/m2min"
 ```
 
 ## Suggested next steps
 
-1. **`SYSTEM` facilities**: expose `BYTE`/`WORD`/`SHORTADDR` etc.
-2. **Anchored subranges** `T[lo..hi]` (small grammar addition).
-3. **Unnamed proc-type parameters** (small grammar addition).
-4. Resolve cross-module imports in the harness (pass each file's
-   imported `.def`s) to remove the LIB noise and isolate LANG.
+1. **`ARRAY <type> OF T`** (index-type arrays) — a real PIM feature.
+2. More `SYSTEM` facilities / library holes.
+3. Array constructors `array{…}` (codegen).
 
 ## Files
 
 `tools/v3-corpus/{corpus.sh,README.md}`, `compiler/src/compiler.frm`
-(long-line fix), `compiler/src/M2.atg` (`EXPORT QUALIFIED`),
-`compiler/tests/t_longline.mod`, `compiler/run_tests.sh`, this doc.
+(long-line), `compiler/src/M2.atg` (`EXPORT QUALIFIED`, anchored
+subranges, unnamed proc-type params), `compiler/src/SymTab.def`/`.mod`
+(SYSTEM predefs, `ParamTypesOk`), `compiler/tests/t_longline.mod`,
+`compiler/run_tests.sh`, this doc.

tools/v3-corpus/README.md

@@ -24,8 +24,15 @@ testsuite files are DEFINITION/IMPLEMENTATION modules with no program;
 V3 reports those as `Incorrect source` but with 0 errors).
 
 An IMPLEMENTATION file is compiled together with its sibling
-DEFINITION (same basename, `.def`/`.DEF`) when present, so a `.mod`
-does not fail on its own interface.
+DEFINITION (same basename, `.def`/`.DEF`) when present, and with the
+DEFINITION of every module it imports, searched in the source directory
+and in the colon-separated `$V3_LIBS` directories — so cross-module
+imports resolve.  E.g.:
+
+```sh
+G=$HOME/bin/Modula2/Gm2/lib/gcc/x86_64-pc-linux-gnu/16.0.1/m2
+V3_LIBS="$G/m2pim:$G/m2iso:$G/m2log:$G/m2cor:$G/m2min" ./corpus.sh
+```
 
 Corpora used:
 - gm2 testsuite `pim/pass` + `iso/run/pass`;

tools/v3-corpus/corpus.sh

@@ -3,13 +3,15 @@
 # sources and reports how many compile cleanly.
 #
 # Each source is copied into a scratch tree (so its .LST listing never
-# touches the corpus) and compiled alone.  A file with 0 errors is a
-# pass.  Failures are split into:
-#   CRASH  -- M2 died (rc >= 128): a real compiler bug
-#   LIB    -- the file imports a module outside V3's known stdlib set
-#             (gm2's StrIO/NumberIO/STextIO/...), so "undeclared" is
-#             expected and not a language gap
-#   LANG   -- otherwise: a genuine parse/semantic gap in V3
+# touches the corpus) and compiled alone, together with:
+#   * its sibling DEFINITION (same basename, .def/.DEF) when present, and
+#   * the DEFINITION of every module it imports, searched in the source
+#     directory and in the -I-style library dirs listed in $V3_LIBS
+#     (colon-separated), so cross-module imports resolve.
+#
+# A file with 0 errors is a pass.  Failures are split into CRASH (M2
+# died), LIB (an import still unresolved: no .def found) and LANG (a
+# real V3 gap).
 #
 # Usage: ./corpus.sh [corpus-dir ...]
 cd "$(dirname "$0")"
@@ -24,23 +26,51 @@ KNOWN="SYSTEM SysIO TextIO Strings Math Conversions RealIO ProgramArgs IOChan Fi
 
 known() { for k in $KNOWN; do [ "$1" = "$k" ] && return 0; done; return 1; }
 
+# find a DEFINITION file for module $1 in the source dir or $V3_LIBS
+finddef() {
+  for d in "$scrdir" $(echo "$V3_LIBS" | tr ':' ' '); do
+    [ -d "$d" ] || continue
+    for cand in "$d/$1.def" "$d/$1.DEF" "$d/$(echo "$1" | tr 'A-Z' 'a-z').def"; do
+      [ -f "$cand" ] && { echo "$cand"; return; }
+    done
+  done
+}
+
+# extract imported module names from a source
+imports() {
+  { sed -nE 's/^[[:space:]]*FROM[[:space:]]+([A-Za-z][A-Za-z0-9_]*).*/\1/p' "$1"
+    sed -nE 's/^[[:space:]]*IMPORT[[:space:]]+(.*)/\1/p' "$1" | sed 's/;.*//' | tr ',' '\n'
+  } | sed -E 's/[[:space:]]//g' | grep -E '^[A-Za-z][A-Za-z0-9_]*$'
+}
+
 rm -rf "$WORK"; mkdir -p "$WORK"; : > "$WORK/results.txt"
 n=0
-for dir in "$@"; do
-  find "$dir" -maxdepth 1 \( -iname '*.mod' -o -iname '*.def' \) 2>/dev/null | sort |
+for scrdir in "$@"; do
+  find "$scrdir" -maxdepth 1 \( -iname '*.mod' -o -iname '*.def' \) 2>/dev/null | sort |
   while read -r f; do
     n=$((n+1))
     w="$WORK/f$n"; mkdir -p "$w"; base=$(basename "$f")
     cp "$f" "$w/$base"
-    # if this is an IMPLEMENTATION, compile its sibling DEFINITION first
     files="$w/$base"
+    # sibling DEFINITION first
     case "$base" in
       *.mod|*.MOD)
         sib="$(echo "$base" | sed -E 's/\.[^.]*$/.def/')"
-        [ -f "$dir/$sib" ] || sib="$(echo "$base" | sed -E 's/\.[^.]*$/.DEF/')"
-        if [ -f "$dir/$sib" ]; then cp "$dir/$sib" "$w/$sib"; files="$w/$sib $w/$base"; fi
+        [ -f "$scrdir/$sib" ] || sib="$(echo "$base" | sed -E 's/\.[^.]*$/.DEF/')"
+        if [ -f "$scrdir/$sib" ]; then cp "$scrdir/$sib" "$w/$sib"; files="$w/$sib $files"; fi
         ;;
     esac
+    # imported DEFINITIONs (for the LIB classification and for resolution)
+    lib=0
+    for m in $(imports "$w/$base"); do
+      known "$m" && continue
+      df=$(finddef "$m")
+      if [ -n "$df" ]; then
+        cp "$df" "$w/$(basename "$df")"; files="$w/$(basename "$df") $files"
+      else
+        lib=1
+      fi
+    done
     "$M2" $files >/dev/null 2>&1
     rc=$?
     lst="$w/$(echo "$base" | sed 's/\.[^.]*$//').LST"
@@ -49,12 +79,6 @@ for dir in "$@"; do
     e=$(grep -oE '[0-9]+ errors?' "$lst" | head -1 | grep -oE '[0-9]+')
     [ -z "$e" ] && e=-1
     if [ "$e" = 0 ]; then echo "OK||$f" >> "$WORK/results.txt"; continue; fi
-    # classify: does it import a module outside the known set?
-    names=$( { sed -nE 's/^[[:space:]]*FROM[[:space:]]+([A-Za-z][A-Za-z0-9_]*).*/\1/p' "$w/$base"
-               sed -nE 's/^[[:space:]]*IMPORT[[:space:]]+(.*)/\1/p' "$w/$base" | sed 's/;.*//' | tr ',' '\n'
-             } | sed -E 's/[[:space:]]//g' | grep -E '^[A-Za-z][A-Za-z0-9_]*$' )
-    lib=0
-    for m in $names; do known "$m" || lib=1; done
     m=$(grep -E '^\*\*\*\*\*' "$lst" | head -1 | sed -E 's/^\*\*\*\*\* *\^? *//; s/^\*\*\*\*\*//')
     if [ "$lib" = 1 ]; then echo "LIB|$m|$f" >> "$WORK/results.txt"
     else echo "LANG|$m|$f" >> "$WORK/results.txt"; fi
@@ -65,8 +89,8 @@ cnt() { grep -c "^$1" "$WORK/results.txt" 2>/dev/null || true; }
 echo "total      : $total"
 echo "compile-OK : $(cnt OK)"
 echo "CRASH      : $(cnt CRASH)"
-echo "LIB (needs a gm2 library): $(cnt LIB)"
-echo "LANG (V3 gap)            : $(cnt LANG)"
+echo "LIB (unresolved import): $(cnt LIB)"
+echo "LANG (V3 gap)          : $(cnt LANG)"
 echo "=== LANG error kinds (top) ==="
 grep '^LANG' "$WORK/results.txt" | cut -d'|' -f2 | sed -E 's/[0-9]+/N/g' | sort | uniq -c | sort -rn | head -15
 echo "=== LANG files (first 40) ==="