topspeed corpus: build the sidecar and validate it on TS-V3 (631/662)

- tools/topspeed-grammar/{build.sh,corpus.sh,README.md}: build TSM2 from
  TopSpeed-V3-M2.atg (frames + V3 SymTab/QbeGen/FileIO) and run it over
  a corpus, classifying parse-OK (deliberate 230s) vs syntax errors.
- TopSpeed-V3-M2.atg: API-drift patches (NewOpenArray, FixPending/
  SetProcRes results, EndModule(name)) and three real grammar fixes:
  tagless variant CASE : T OF, manifest constants without CONST,
  trailing '|' in variant lists.
- Result on $HOME/bin/Dos/M2/TS-V3 (662 files): 631 parse-OK, 31
  syntax-bad (17 need (*% conditional compilation).  Coverage 590->631.
- docs/summary_topspeed-corpus.md.  V3 compiler/fixpoint untouched.

.gitignore

@@ -1,3 +1,4 @@
 stdlib/*.LST
 git.txt
 runtime/syslib/*.LST
+tools/topspeed-grammar/build/

compiler/src/TopSpeed-V3-M2.atg

@@ -100,6 +100,7 @@ PRODUCTIONS
 ";"
        { Import }
        { ConstBlock | TypeBlock<TRUE> | VarBlock
+       | ConstDecl ";"                   (. (* manifest const, no CONST *) .)
        | ProcHeading<pn> ";"             (. SymTab.CloseProc; .)
        | ClassItem ";" }
       "END"
@@ -140,10 +141,12 @@ PRODUCTIONS
       "END"
       GetIdent<m2>                      (. IF ~SymTab.Equal(m1, m2) THEN
                                              SemError(202) END; .)
-                                        (. QbeGen.EndModule;
+                                        (. QbeGen.EndModule(m1);
                                            SymTab.PrintTable; .) .
   DeclSeq
-    = { ConstBlock | TypeBlock<FALSE> | VarBlock | [ "INLINE" ] ProcDecl ";"
+    = { ConstBlock | TypeBlock<FALSE> | VarBlock
+      | ConstDecl ";"                   (. (* manifest const, no CONST *) .)
+      | [ "INLINE" ] ProcDecl ";"
       | NestedModule ";" | ClassItem ";"
       | "LABEL" LabelList ";" } .
   LabelList                             (. VAR nm: SymTab.Name; .)
@@ -288,7 +291,7 @@ ArrayType<VAR t: SymTab.TypeIndex>    (. VAR e: SymTab.TypeIndex;
       ( "[" Bound { "," Bound } "]"      (. t := SymTab.InvalidType; .)
       | Bound                            (. t := SymTab.InvalidType; .)
       |                                   (. t := SymTab.InvalidType; .) )
-      "OF" Type<e>                      (. t := SymTab.NewArray(e);
+      "OF" Type<e>                      (. t := SymTab.NewOpenArray(e);
                                               SemError(230); .) .
   Bound                                 (. VAR t: SymTab.TypeIndex;
                                              q: QbeGen.QVal; .)
@@ -310,8 +313,10 @@ ArrayType<VAR t: SymTab.TypeIndex>    (. VAR e: SymTab.TypeIndex;
   FieldList<rec: SymTab.TypeIndex>      (. VAR tag: SymTab.Name;
                                              ti: SymTab.TypeIndex; .)
     = "CASE"                            (. SemError(230); .)
-      [ GetIdent<tag> ":"
-        ] Type<ti>
+      ( GetIdent<tag> ":"               (. (* named selector *) .)
+      | ":"                             (. (* anonymous selector *) .)
+      |                                 (. (* no selector *) .) )
+      Type<ti>
       "OF" VariantList<rec>
       "END"
     | FieldIdents<rec> .
@@ -323,7 +328,7 @@ ArrayType<VAR t: SymTab.TypeIndex>    (. VAR e: SymTab.TypeIndex;
                                             THEN SemError(200) END; .) }
       ":" Type<t>                       (. SymTab.FixPendingF(rec, t); .) .
   VariantList<rec: SymTab.TypeIndex>
-    = [ "|" ] Variant<rec> { "|" Variant<rec> }
+    = [ "|" ] Variant<rec> { "|" [ Variant<rec> ] }
       [ "ELSE" FieldListSequence<rec> ] .
   Variant<rec: SymTab.TypeIndex>
     = VariantLabels ":"                 (. SymTab.PushScope; .)
@@ -527,7 +532,7 @@ ClassRest                             (. VAR cn, m2, pn, q: SymTab.Name;
                                                QbeGen.DeclVar(nm, t);
                                                INC(i)
                                              END;
-                                             SymTab.FixPending(t); .) .
+                                             IF NOT SymTab.FixPending(t) THEN END; .) .
 VarIdents                             (. VAR n: SymTab.Name;
                                              t: SymTab.TypeIndex;
                                              q: QbeGen.QVal; .)
@@ -561,14 +566,14 @@ VarIdents                             (. VAR n: SymTab.Name;
                                               SemError(200) END
                                             END; .)
       [ FormalParams ]
-      [ ":" TypeIdent2<t>               (. SymTab.SetProcRes(t); .) ] .
+      [ ":" TypeIdent2<t>               (. IF NOT SymTab.SetProcRes(t) THEN END; .) ] .
   FormalParams
     = "(" [ ParamSection { ";" ParamSection } ] ")" .
   ParamSection                          (. VAR t: SymTab.TypeIndex;
                                              isV: BOOLEAN; .)
     =                                   (. isV := FALSE; .)
       [ "VAR"                           (. isV := TRUE; .) ]
-      ParIdents<isV> ":" Type<t>        (. SymTab.FixPending(t); .) .
+      ParIdents<isV> ":" Type<t>        (. IF NOT SymTab.FixPending(t) THEN END; .) .
   ProcDecl                              (. VAR pn, assem: SymTab.Name;
                                              at: SymTab.TypeIndex;
                                              ak: INTEGER;

docs/summary_topspeed-corpus.md

@@ -0,0 +1,85 @@
+# Step: TopSpeed V3 corpus validation
+
+Tag `v3-topspeed-corpus`. V3 suite **151/151**, fixpoint OK
+(unchanged — this touches only the sidecar).
+
+## Why
+
+After self-hosting, the next "normal" step was to exercise the
+front-end on a **real corpus**. The historical TopSpeed/JPI V3 corpus
+is the natural target: `$HOME/bin/Dos/M2/TS-V3` — **662 files**
+(352 `.DEF` + 310 `.MOD`).
+
+V3's own `M2.atg` is the classic/Redux subset and would not take this
+dialect (pragmas `(*#...)`, conditional compilation `(*%...)`, `H`/`C`/
+`B` literals, `::=`, address constructors `[seg:ofs]^`, `CLASS`/`IS`,
+manifest constants, ...).  The project already carries a
+**sidecar** grammar for it, `compiler/src/TopSpeed-V3-M2.atg`, whose
+deliverable is *parsing* the dialect (unlowered constructs end in the
+standard `230`, parse-now / lower-later).
+
+That sidecar had never been compiled and run over the full corpus.
+
+## What was done
+
+1. **Built the sidecar** (`tools/topspeed-grammar/build.sh`) — copies
+   the grammar + the V3 frames + `SymTab`/`QbeGen`/`FileIO`, runs
+   `CR -m -C`, `gm2 -fiso`, links `build/TSM2`.
+2. **Fixed API drift** vs the current `SymTab`/`QbeGen` (the sidecar
+   predated several changes):
+   - `SymTab.NewArray(e)` → `SymTab.NewOpenArray(e)`;
+   - `SymTab.FixPending(t)` / `SetProcRes(t)` now return `BOOLEAN`;
+   - `QbeGen.EndModule` now takes the module name.
+3. **Three real grammar gaps** the corpus exposed:
+   - **tagless variant** `CASE : T OF` (anonymous selector) — TopSpeed
+     accepted, the sidecar required a named tag;
+   - **manifest constants without `CONST`** (`NAME = value;` at
+     declaration level);
+   - **trailing `|`** in a variant list (`TRUE: …; | FALSE: …; |`).
+4. **Corpus harness** (`tools/topspeed-grammar/corpus.sh`) — runs
+   `TSM2` over every `.DEF`/`.MOD` in a scratch tree (so listings never
+   touch the corpus) and classifies each file as *parse-OK* (only the
+   deliberate `230`/semantic markers) or *syntax-bad* (a real Coco/R
+   error).
+
+## Result
+
+| | files |
+| --- | --- |
+| corpus | **662** (352 `.DEF` + 310 `.MOD`) |
+| **parse-OK** (230/semantic only) | **631** |
+| syntax-bad | **31** |
+
+Coverage rose **590 → 627 → 631** as the fixes landed.
+
+### Remaining 31 syntax-bad
+
+- **17 use `(*%…*)` conditional compilation** — a lexer feature the
+  sidecar does not implement; text inside a disabled block is parsed
+  as code.  (Pragmas `(*#…)` are already fine — they lex as ordinary
+  `(* … *)` comments.)
+- **14 assorted**, e.g. a typed set constructor in an argument list
+  (`GetMenu("…", CharSet{'N','P','T'}, ch)`), address constructors,
+  and a few files with malformed/commented-out regions
+  (`END (*QMLB` …).  Full list in the harness output.
+
+## Reproduce
+
+```sh
+cd tools/topspeed-grammar
+./build.sh          # -> build/TSM2
+./corpus.sh         # -> 631 parse-OK / 31 syntax-bad
+```
+
+## Notes
+
+- V3's own compiler (`M2.atg`) and the self-hosting path are untouched;
+  the sidecar is a separate grammar built on demand.
+- The sidecar now tracks the V3 `SymTab`/`QbeGen` API; if those drift
+  again, `build.sh` is where the small patch belongs.
+
+## Files
+
+`compiler/src/TopSpeed-V3-M2.atg` (three grammar fixes),
+`tools/topspeed-grammar/{build.sh,corpus.sh,README.md}`,
+`.gitignore` (build dir), this doc.

tools/topspeed-grammar/README.md

@@ -0,0 +1,40 @@
+# topspeed-grammar
+
+Corpus harness for the historical **TopSpeed / JPI V3** dialect, using
+the sidecar grammar `compiler/src/TopSpeed-V3-M2.atg`.
+
+The sidecar is a *grammar* deliverable: it parses the TS-V3 dialect
+(pragmas, `H`/`C`/`B` literals, `::=`, anchored subranges, address
+constructors, `CLASS`/`IS`/`VIRTUAL`, manifest constants, tagless and
+trailing-`|` variant records, ...). Constructs the current backend
+cannot lower yet still **parse** and end with the standard
+`SemError(230)` marker (parse-now / lower-later), exactly like
+`M2.atg`.
+
+## Build
+
+```sh
+./build.sh        # -> build/TSM2   (build/ is git-ignored)
+```
+
+It copies the grammar, the V3 frames (`scanner/parser/compiler.frm`)
+and the V3 `SymTab`/`QbeGen`/`FileIO` sources into `build/`, runs
+`CR -m -C`, compiles with `gm2 -fiso` and links `TSM2`. Needs the
+project `CR` (`$HOME/Projets/Projets-Modula2/MyWork/CocoGm2/CR`) and
+GNU Modula-2.
+
+## Corpus check
+
+```sh
+./corpus.sh [corpus-dir]     # default: $HOME/bin/Dos/M2/TS-V3
+```
+
+Runs `TSM2` over every `.DEF`/`.MOD`, copying each into a scratch tree
+so the `.LST` listings never touch the corpus, and reports:
+
+- **parse-OK** — only the deliberate `230`/semantic markers,
+- **syntax-bad** — a real Coco/R error (`'x' expected`, `invalid X`).
+
+Current result on the TS-V3 tree (662 files, 352 `.DEF` + 310 `.MOD`):
+**631 parse-OK / 31 syntax-bad** — see
+`docs/summary_topspeed-corpus.md`.

tools/topspeed-grammar/build.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+# Builds the TopSpeed-V3 sidecar front-end (TSM2) from
+# compiler/src/TopSpeed-V3-M2.atg.
+#
+# The sidecar is a *grammar* deliverable: it parses the historical
+# TopSpeed/JPI V3 dialect (pragmas, H/C/B literals, ::=, address
+# constructors, CLASS/IS/VIRTUAL, manifest constants, tagless variant
+# records, ...).  It reuses the V3 frontend frame (scanner/parser/
+# compiler .frm) and the V3 SymTab/QbeGen/FileIO modules, so it tracks
+# the V3 API -- a small drift patch is applied here if a call site has
+# moved on (see the sed lines below).
+#
+# Output: build/TSM2  (build/ is git-ignored)
+set -e
+cd "$(dirname "$0")"
+ROOT=../..
+SRC="$ROOT/compiler/src"
+CR="${CR:-$HOME/Projets/Projets-Modula2/MyWork/CocoGm2/CR}"
+B=build
+
+rm -rf "$B"; mkdir -p "$B"
+cp "$SRC/TopSpeed-V3-M2.atg" "$B/M2.atg"
+cp "$SRC/scanner.frm" "$SRC/parser.frm" "$SRC/compiler.frm" "$B/"
+cp "$SRC/SymTab.def" "$SRC/SymTab.mod" "$SRC/QbeGen.def" "$SRC/QbeGen.mod" \
+   "$SRC/FileIO.def" "$SRC/FileIO.mod" "$B/"
+
+cd "$B"
+echo "=== CR ==="; CRFRAMES="$(pwd)" "$CR" -m -C M2.atg
+echo "=== compile ==="
+for m in FileIO SymTab QbeGen M2S M2P M2; do
+  gm2 -fiso -I . -c "$m.mod"
+done
+echo "=== link ==="
+gm2 -fiso -I . -fgen-module-list=modules.lst -o /dev/null \
+    M2S.o M2P.o FileIO.o SymTab.o QbeGen.o M2.mod
+gm2 -fiso -I . -fuse-list=modules.lst -o TSM2 \
+    M2S.o M2P.o FileIO.o SymTab.o QbeGen.o M2.mod
+echo "=== build/TSM2 built ==="

tools/topspeed-grammar/corpus.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+# Runs the TopSpeed sidecar front-end (build/TSM2) over a corpus of
+# .DEF/.MOD files and reports parse coverage.
+#
+# A *syntax* error is a Coco/R error (`'x' expected`, `invalid X`); the
+# sidecar's own `not supported yet` (230) markers are the deliberate
+# parse-now/lower-later discipline and do NOT count as failures.
+#
+# Usage: ./corpus.sh [corpus-dir]   (default: the TS-V3 tree)
+set -e
+cd "$(dirname "$0")"
+TSM2="$(pwd)/build/TSM2"
+CORPUS="${1:-$HOME/bin/Dos/M2/TS-V3}"
+WORK="/tmp/opencode/tscorpus"
+[ -x "$TSM2" ] || { echo "build/TSM2 missing -- run ./build.sh"; exit 1; }
+[ -d "$CORPUS" ] || { echo "corpus not found: $CORPUS"; exit 1; }
+
+rm -rf "$WORK"; mkdir -p "$WORK"
+: > "$WORK/results.txt"
+n=0
+find "$CORPUS" \( -iname '*.DEF' -o -iname '*.MOD' \) 2>/dev/null | sort |
+while read -r f; do
+  n=$((n+1))
+  w="$WORK/f$n"; mkdir -p "$w"; base=$(basename "$f")
+  cp "$f" "$w/$base"
+  ( cd "$w" && "$TSM2" "$base" >/dev/null 2>&1 )
+  lst="$w/$(echo "$base" | sed 's/\.[^.]*$//').LST"
+  if [ ! -f "$lst" ]; then echo "NOLST|$f" >> "$WORK/results.txt"; continue; fi
+  if grep -qE "^\*\*\*\*\*.*(expected|invalid [A-Z])" "$lst"; then
+    m=$(grep -E "^\*\*\*\*\*.*(expected|invalid [A-Z])" "$lst" | head -1 |
+        sed -E 's/^\*\*\*\*\* *[^ ]* *//; s/^\*\*\*\*\* *//')
+    echo "SYN|$m|$f" >> "$WORK/results.txt"
+  else
+    echo "OK|0|$f" >> "$WORK/results.txt"
+  fi
+done
+
+total=$(wc -l < "$WORK/results.txt")
+ok=$(grep -c '^OK' "$WORK/results.txt" || true)
+syn=$(grep -c '^SYN' "$WORK/results.txt" || true)
+echo "corpus: $CORPUS"
+echo "total                      : $total"
+echo "parse-OK (semantic/230 only): $ok"
+echo "syntax-bad                 : $syn"
+echo "=== syntax-error kinds ==="
+grep '^SYN' "$WORK/results.txt" | cut -d'|' -f2 |
+  sed -E 's/[0-9]+/N/g' | sort | uniq -c | sort -rn
+echo "=== syntax-bad files ==="
+grep '^SYN' "$WORK/results.txt" | cut -d'|' -f3 | sed "s|$CORPUS/||"