two-phase parse milestone 2c (WIP): defer bodies and re-parse at unit end

Behind -twophase: module-level ProcDecl bodies are skipped in phase 1
(scanner left at the matching END; ProcDecl closes like FORWARD) and
re-parsed at unit end with SymTab/Lower/QbeGen re-armed (inlined loop at
ProgModule/ImplUnit; unit astDecl/astStmt saved/restored). Fixed: PopPos
restored the error count and wiped repass diagnostics.

Suite 199/199; fixpoint OK. 142 single-file tests: 0 signals, 138
byte-identical; the 4 forward-reference tests differ because deferral
makes a forward var compile like a normally-ordered one (the legacy
FwdVar path is the divergent one). Known gap: some multi-module
implementation sessions record a spurious 'ident expected' inside a
repassed body while emitting identical .ssa, so the pass stays opt-in.

compiler/run_tests.sh

@@ -481,10 +481,46 @@ twophase_same() {
   rm -f "gen_ssa/$mod.off.ssa"
 }
 
-twophase_same t_fwdvar.mod
 twophase_same t_nestedproc.mod
 twophase_same t_classmethod.mod
 
+# Forward references compile *differently* under -twophase than under the
+# legacy FwdVar path (the deferral makes them ordinary, so the codegen is
+# the normal one) — that is the point.  Gate them functionally: same exit
+# code, and the forward diagnostic still fires.
+twophase_run() {
+  # $1 = test file, $2 = expected exit code
+  name="$1"; want="$2"; mod=$(modof "$name")
+  rm -f "gen_ssa/$mod.ssa" "gen_ssa/$mod.s" "gen_ssa/$mod"
+  if ./M2 -twophase "tests/$name" 2>&1 | grep -q "Parsed correctly"; then :; else
+    fail=$((fail+1)); echo "FAIL(twophase-run): $name rejected"; return
+  fi
+  if ./M2 -twophase "tests/$name" 2>&1 | grep -qE "twophase .*(mismatch|failed)"; then
+    fail=$((fail+1)); echo "FAIL(twophase-run): $name signal"; return
+  fi
+  qbe -o "gen_ssa/$mod.s" "gen_ssa/$mod.ssa" 2>/dev/null || { fail=$((fail+1)); echo "FAIL(twophase-run): $name qbe"; return; }
+  cc "gen_ssa/$mod.s" "$SHIM" -o "gen_ssa/$mod" 2>/dev/null || { fail=$((fail+1)); echo "FAIL(twophase-run): $name cc"; return; }
+  timeout 10 "gen_ssa/$mod" >/dev/null 2>&1; got=$?
+  if [ "$got" = "$want" ]; then pass=$((pass+1)); echo "PASS(twophase-run): $name -> $got"
+  else fail=$((fail+1)); echo "FAIL(twophase-run): $name got $got want $want"; fi
+}
+
+twophase_fail() {
+  # $1 = test file, $2 = message expected in the .LST
+  name="$1"; want="$2"
+  if ./M2 -twophase "tests/$name" 2>&1 | grep -q "Incorrect source"; then :; else
+    fail=$((fail+1)); echo "FAIL(twophase-fail): $name accepted"; return
+  fi
+  if grep -q "$want" "tests/$(basename "$name" .mod).LST" 2>/dev/null; then
+    pass=$((pass+1)); echo "PASS(twophase-fail): $name [$want]"
+  else fail=$((fail+1)); echo "FAIL(twophase-fail): [$want] not found"; fi
+}
+
+twophase_run t_fwdvar.mod 42
+twophase_run t_fwdsel.mod 42
+twophase_run t_fwdprop.mod 42
+twophase_fail t_bad_fwd.mod "incompatible assignment"
+
 # --- step 7 gate: compiler sources free of gm2-only imports.
 #     FileIO is the last module to port (see docs/summary_step7.md);
 #     it is excluded here so the gate guards everything else. ---

compiler/src/Lower.def

@@ -37,6 +37,8 @@ PROCEDURE ScopeLeave;
    the top and pops. ReenterScopeProc resumes a noted entry for the
    re-parse pass without re-registering it. *)
 PROCEDURE ReenterScopeProc (ty: INTEGER);
+PROCEDURE CurScopeProc (): INTEGER;
+(* The proc entry NoteProc most recently made current. *)
 PROCEDURE NoteProc (name: ARRAY OF CHAR; uid: CARDINAL;
                     res: SymTab.TypeIndex; depth: CARDINAL; ext: BOOLEAN);
 PROCEDURE MarkProcExternal;

compiler/src/Lower.mod

@@ -212,6 +212,13 @@ PROCEDURE ReenterScopeProc (ty: INTEGER);
     curScopeProc := ty
   END ReenterScopeProc;
 
+PROCEDURE CurScopeProc (): INTEGER;
+(* The proc entry NoteProc most recently made current (ty stamped on the
+   NkProcDecl); captured by the parser's span table for the re-parse. *)
+  BEGIN
+    RETURN curScopeProc
+  END CurScopeProc;
+
 PROCEDURE NoteProc (name: ARRAY OF CHAR; uid: CARDINAL;
                     res: SymTab.TypeIndex; depth: CARDINAL; ext: BOOLEAN);
   BEGIN

compiler/src/M2.atg

@@ -76,6 +76,19 @@ VAR
   spanName: ARRAY [0 .. 1023] OF SymTab.Name;
   spanStart: ARRAY [0 .. 1023] OF M2S.INT32;
   spanIdx: ARRAY [0 .. 1023] OF CARDINAL;
+  spanTy: ARRAY [0 .. 1023] OF INTEGER;
+  spanNode: ARRAY [0 .. 1023] OF AST.Node;
+
+  (* Two-phase parse (milestone 2c): phase-1 skips a module-level proc
+     body (DeferSkip) and phase 2 re-parses it at unit end (RepassAll).
+     `deferredBody` reports the skip to ProcDecl's close choice;
+     `repassing` disables deferral during phase 2; `allowDefer` limits
+     deferral to plain ProcDecls (methods stay inline); `curProcNode` is
+     the NkProcDecl whose body is about to be parsed. *)
+  deferredBody: BOOLEAN;
+  repassing: BOOLEAN;
+  allowDefer: BOOLEAN;
+  curProcNode: AST.Node;
 
   (* Result slot for the expression-AST builder (slice 2).  Every
      Expr/SimExpr/Term/Fact leaves its node here; the combination
@@ -364,14 +377,11 @@ PROCEDURE NoteSpan (pn: SymTab.Name; start: M2S.INT32);
   END NoteSpan;
 
 PROCEDURE ProbeBody (pn: SymTab.Name);
-(* Shadow-skip the body starting at the current token: consume tokens
-   counting block openers until the matching END, check the procedure
-   name after it, record the body span for the re-parse pass, verify
-   SeekTo repositions exactly, then restore the exact scanner/parser
-   state.  The body then parses normally, so a correct skipper is
-   output-neutral; a wrong one reports to StdOut (never into the
-   listing).  The SeekTo check's side effects are discarded by the
-   trailing PopPos, so it is output-neutral too. *)
+(* Shadow-skip a body: consume tokens counting block openers until the
+   matching END, check the name, record the span, verify SeekTo
+   repositions exactly, then restore the exact scanner/parser state.
+   Superseded by DeferSkip for deferred bodies; kept for the plain
+   probe of non-deferred Bodies (methods). *)
   VAR savedSym, savedErr, depth, guard: CARDINAL;
     nm: SymTab.Name;
     ok, sok: BOOLEAN;
@@ -420,6 +430,71 @@ PROCEDURE ProbeBody (pn: SymTab.Name);
     END
   END ProbeBody;
 
+PROCEDURE Openers (): BOOLEAN;
+(* TRUE when the current token opens a block that must be closed by an
+   END (BEGIN is excluded: a block's BEGIN shares the declaration END;
+   REPEAT..UNTIL and EXIT/RETURN need no END). *)
+  BEGIN
+    RETURN TokIs("MODULE") OR TokIs("PROCEDURE") OR TokIs("CLASS")
+       OR TokIs("RECORD") OR TokIs("IF") OR TokIs("CASE")
+       OR TokIs("WHILE") OR TokIs("FOR") OR TokIs("LOOP")
+       OR TokIs("WITH")
+  END Openers;
+
+PROCEDURE DeferSkip (pn: SymTab.Name): BOOLEAN;
+(* Phase 1 of the deferral: skip the body starting at the current token
+   (the first token after the heading's ';') up to the matching END,
+   verify the END's name, and LEAVE the scanner positioned exactly at
+   that END (so Block's trailing "END" ident consumes it for real), with
+   the span recorded for the re-parse pass.  On any inconsistency,
+   restore the scanner and return FALSE so the caller parses normally. *)
+  VAR depth, guard: CARDINAL;
+    nm: SymTab.Name;
+    ok: BOOLEAN;
+    start, endPos: M2S.INT32;
+  BEGIN
+    deferredBody := FALSE;
+    start := M2S.nextPos;
+    M2S.PushPos;
+    depth := 0; guard := 0; ok := TRUE;
+    LOOP
+      IF sym = 0 THEN ok := FALSE; EXIT END;
+      IF TokIs("END") THEN
+        IF depth = 0 THEN EXIT END;
+        DEC(depth)
+      ELSIF Openers() THEN
+        INC(depth)
+      END;
+      Adv;
+      INC(guard);
+      IF guard > 1000000 THEN ok := FALSE; EXIT END
+    END;
+    IF ok THEN
+      (* sym is the matching END; check the name after it, then seek back
+         so Block sees END (its "END" ident then consumes it for real). *)
+      endPos := M2S.nextPos;
+      Adv;
+      M2S.GetName(M2S.nextPos, M2S.nextLen, nm);
+      IF (sym = 0) OR NOT SymTab.Equal(nm, pn) THEN ok := FALSE END;
+      IF ok THEN
+        M2S.SeekTo(endPos); Adv
+      END
+    END;
+    IF ok THEN
+      NoteSpan(pn, start);
+      spanTy[nSpans - 1] := Lower.CurScopeProc();
+      spanNode[nSpans - 1] := curProcNode;
+      deferredBody := TRUE;
+      RETURN TRUE
+    ELSE
+      M2S.PopPos;
+      FileIO.WriteString(FileIO.StdOut, "twophase defer mismatch in ");
+      FileIO.WriteString(FileIO.StdOut, pn);
+      FileIO.WriteLn(FileIO.StdOut);
+      RETURN FALSE
+    END
+  END DeferSkip;
+
 CHARACTERS
   eol      = CHR(13) .
   lf       = CHR(10) .
@@ -485,7 +560,9 @@ PRODUCTIONS
                                              refActual[refN] := at; INC(refN)
                                            END; .) .
   Unit
-    =                                   (. isGen := FALSE; .)
+    =                                   (. isGen := FALSE;
+                                           allowDefer := FALSE;
+                                           repassing := FALSE; .)
       [ "GENERIC"                       (. isGen := TRUE; .) ]
       ( DefUnit | ImplUnit | ProgModule ) .
   (* Step 4.3: one session compiles DEFINITION, its IMPLEMENTATION
@@ -566,6 +643,12 @@ PRODUCTIONS
   ImplUnit                              (. VAR m1, m2: SymTab.Name;
                                              g: SymTab.Name;
                                              k: CARDINAL;
+                                             rp: CARDINAL;
+                                             rpOk: BOOLEAN;
+                                             rErrs: INTEGER;
+                                             rDecl, rStmt: AST.Node;
+                                             rnm: SymTab.Name;
+                                             rSym, rErr: CARDINAL;
                                              fname: SymTab.Name;
                                              unres: BOOLEAN;
                                              astNode: AST.Node; .)
@@ -603,6 +686,44 @@ PRODUCTIONS
 
                                            nLab := 0; nGot := 0; .)
         [ StatSeq ] ]
+                                        (. IF twoPhaseParse THEN
+                                             rSym := sym; rErr := errDist;
+                                             rDecl := astDecl; rStmt := astStmt;
+                                             M2S.PushPos;
+                                             repassing := TRUE;
+                                             rp := 0;
+                                             WHILE rp < nSpans DO
+                                               k := 0;
+                                               WHILE (k <= HIGH(rnm))
+                                                 AND (spanName[rp][k] # CHR(0)) DO
+                                                 rnm[k] := spanName[rp][k]; INC(k)
+                                               END;
+                                               rnm[k] := CHR(0);
+                                               M2S.SeekTo(spanStart[rp]);
+                                               Adv;
+                                               rpOk := SymTab.ReenterDeferred(spanIdx[rp]);
+                                               IF NOT rpOk THEN
+                                                 FileIO.WriteString(FileIO.StdOut,
+                                                   "twophase reenter failed");
+                                                 FileIO.WriteLn(FileIO.StdOut)
+                                               END;
+                                               Lower.ReenterScopeProc(spanTy[rp]);
+                                               QbeGen.BeginFunc("");
+                                               Block(rnm);
+                                               AST.SetChild(spanNode[rp], 1, astStmt);
+                                               AST.SetChild(spanNode[rp], 2, astBlkDecls);
+                                               Lower.ScopeLeave;
+                                               SymTab.CloseProc;
+                                               QbeGen.EndFunc(SymTab.ProcRes(rnm));
+                                               INC(rp)
+                                             END;
+                                             rErrs := M2S.errors;
+                                             repassing := FALSE;
+                                             M2S.PopPos;
+                                             sym := rSym; errDist := rErr;
+                                             M2S.errors := rErrs;
+                                             astDecl := rDecl; astStmt := rStmt
+                                           END; .)
       "END"
       GetIdent<m2>                      (. IF instActive THEN InstNameGet(m2) END;
                                            FlushPend;
@@ -640,6 +761,12 @@ PRODUCTIONS
                                            SymTab.EndUnit; .) ) .
   ProgModule                            (. VAR m1, m2: SymTab.Name;
                                              k: CARDINAL;
+                                             rp: CARDINAL;
+                                             rpOk: BOOLEAN;
+                                             rErrs: INTEGER;
+                                             rDecl, rStmt: AST.Node;
+                                             rnm: SymTab.Name;
+                                             rSym, rErr: CARDINAL;
                                              fname: SymTab.Name;
                                              unres: BOOLEAN;
                                              astNode: AST.Node; .)
@@ -668,6 +795,44 @@ PRODUCTIONS
 
                                            nLab := 0; nGot := 0; .)
         [ StatSeq ] ]
+                                        (. IF twoPhaseParse THEN
+                                             rSym := sym; rErr := errDist;
+                                             rDecl := astDecl; rStmt := astStmt;
+                                             M2S.PushPos;
+                                             repassing := TRUE;
+                                             rp := 0;
+                                             WHILE rp < nSpans DO
+                                               k := 0;
+                                               WHILE (k <= HIGH(rnm))
+                                                 AND (spanName[rp][k] # CHR(0)) DO
+                                                 rnm[k] := spanName[rp][k]; INC(k)
+                                               END;
+                                               rnm[k] := CHR(0);
+                                               M2S.SeekTo(spanStart[rp]);
+                                               Adv;
+                                               rpOk := SymTab.ReenterDeferred(spanIdx[rp]);
+                                               IF NOT rpOk THEN
+                                                 FileIO.WriteString(FileIO.StdOut,
+                                                   "twophase reenter failed");
+                                                 FileIO.WriteLn(FileIO.StdOut)
+                                               END;
+                                               Lower.ReenterScopeProc(spanTy[rp]);
+                                               QbeGen.BeginFunc("");
+                                               Block(rnm);
+                                               AST.SetChild(spanNode[rp], 1, astStmt);
+                                               AST.SetChild(spanNode[rp], 2, astBlkDecls);
+                                               Lower.ScopeLeave;
+                                               SymTab.CloseProc;
+                                               QbeGen.EndFunc(SymTab.ProcRes(rnm));
+                                               INC(rp)
+                                             END;
+                                             rErrs := M2S.errors;
+                                             repassing := FALSE;
+                                             M2S.PopPos;
+                                             sym := rSym; errDist := rErr;
+                                             M2S.errors := rErrs;
+                                             astDecl := rDecl; astStmt := rStmt
+                                           END; .)
       "END"
       GetIdent<m2>                      (. IF instActive THEN InstNameGet(m2) END;
                                            FlushPend;
@@ -1540,17 +1705,32 @@ PRODUCTIONS
                                            SymTab.CloseProc;
                                            QbeGen.AbortFunc; .)
       |                                 
-        Block<pn>                       (. AST.SetChild(astNode, 1, astStmt);
-                                           AST.SetChild(astNode, 2, astBlkDecls);
-                                           astDecl := astNode;
-                                           Lower.ScopeLeave;
-                                           SymTab.CloseProc;
-                                           QbeGen.EndFunc(
-                                             SymTab.ProcRes(pn)); .) ) .
-  Block<pn: SymTab.Name>                (. VAR m2: SymTab.Name; .)
-    =                                   (. IF twoPhaseParse AND (procNest = 0) THEN
-                                            ProbeBody(pn) END;
-                                          INC(procNest); .)
+        (. allowDefer := TRUE; deferredBody := FALSE;
+                                           curProcNode := astNode; .)
+        Block<pn>                       (. IF deferredBody THEN
+                                             (* body skipped: close like
+                                                FORWARD (no body node) *)
+                                             astDecl := astNode;
+                                             Lower.ScopeLeave;
+                                             SymTab.CloseProc;
+                                             QbeGen.AbortFunc
+                                           ELSE
+                                             AST.SetChild(astNode, 1, astStmt);
+                                             AST.SetChild(astNode, 2, astBlkDecls);
+                                             astDecl := astNode;
+                                             Lower.ScopeLeave;
+                                             SymTab.CloseProc;
+                                             QbeGen.EndFunc(
+                                               SymTab.ProcRes(pn))
+                                           END; .) ) .
+  Block<pn: SymTab.Name>                (. VAR m2: SymTab.Name; dfr: BOOLEAN; .)
+    =                                   (. dfr := FALSE;
+                                           IF allowDefer AND twoPhaseParse
+                                              AND NOT repassing THEN
+                                             dfr := DeferSkip(pn)
+                                           END;
+                                           allowDefer := FALSE;
+                                           IF NOT dfr THEN INC(procNest) END; .)
       DeclSeq                           (. astBlkDecls := astDecl;
                                            astStmt := AST.NoNode;
                                            nLab := 0; nGot := 0; .)
@@ -1558,16 +1738,22 @@ PRODUCTIONS
                                              body: mark it so Lower
                                              does not read it as a
                                              definition heading *)
-                                           IF astStmt = AST.NoNode THEN
-                                             astStmt :=
-                                               AST.MakeNode(AST.NkBlock)
+                                           IF NOT dfr THEN
+                                             IF astStmt = AST.NoNode THEN
+                                               astStmt :=
+                                                 AST.MakeNode(AST.NkBlock)
+                                             END
                                            END; .)
         [ StatSeq ] ]
       "END"
       GetIdent<m2>                      (. IF NOT SymTab.Equal(pn, m2) THEN
                                              SemError(202) END;
-                                           DEC(procNest);
+                                           IF NOT dfr THEN DEC(procNest) END;
                                            CheckGotos; .) .
+  (* Two-phase, phase 2 (milestone 2c): re-parse every deferred module-
+     level body once the unit's declarations are complete.  Inlined at
+     both call sites because a stand-alone epsilon production is
+     "unreachable" to Coco/R (referenced only from actions). *)
   StatSeq                               (. VAR astSeq, astTail: AST.Node; .)
     =                                   (. astSeq := AST.NoNode;
                                            astTail := AST.NoNode; .)

docs/plan-two-phase-parse.md

@@ -196,3 +196,43 @@ alone.
     pairs with `EndFunc`); **registrations run once** (skip pass) with
     the re-parse on resume paths; nested spans append to the table
     during phase 2 (worklist); DefUnit/EXTERNAL/FORWARD paths unchanged.
+- **2026-10-16, milestone 2c (real deferral + repass, behind
+  `-twophase`).**
+  - Phase 1: `DeferSkip` (from `Block`, only for plain `ProcDecl`
+    bodies — methods stay inline) skips the body for real, leaves the
+    scanner at the matching `END` so `Block`'s trailing `END ident`
+    consumes it, records the span, and `ProcDecl` closes like `FORWARD`
+    (`AbortFunc`, no body node).  Phase 2: an inlined loop at both
+    module ends (`ProgModule`/`ImplUnit`) `SeekTo`s each span, re-arms
+    `SymTab`/`Lower`/`QbeGen`, parses `Block`, attaches body + block
+    decls to the proc node, and closes with `ScopeLeave`/`CloseProc`/
+    `EndFunc`.  Scanner/parser state and the unit's `astDecl`/`astStmt`
+    are saved and restored around the loop.
+  - Two bugs found and fixed: the loop clobbered the unit's
+    `astDecl`/`astStmt` (now saved/restored), and `PopPos` restored
+    `errors`, wiping every diagnostic recorded during the repass (now
+    the post-loop count is re-instated — this had silently dropped
+    210/232 etc. in deferred bodies).
+  - Verified: suite **199/199** (twophase gates: identity on
+    forward-free files, functional `-> 42` on `t_fwdvar`/`t_fwdsel`/
+    `t_fwdprop`, `incompatible assignment` on `t_bad_fwd`);
+    **fixpoint OK (3,620,461 bytes)**.
+  - Measured: all **142 single-file tests** run under `-twophase` with
+    **zero signals**; **138/142 byte-identical**.  The 4 that differ are
+    exactly the forward-reference tests (`t_fwdvar`/`t_fwdsel`/
+    `t_fwdprop`/`t_bad_fwd`): deferral makes a forward variable ordinary,
+    so it compiles like a normally-ordered one — the legacy `FwdVar`
+    path is the divergent one (it drops a `loadw`).  Deferred output for
+    those matches the *normal* codegen (verified against a reordered
+    fixture) and runs to 42.  Sessions `DBasicProg`/`TString`/
+    `Showcase12` byte-identical.
+  - **Known gap (why this is not flipped).**  On some multi-module
+    implementation sessions (e.g. `Conversions.mod` in the Library1
+    session), `-twophase` records a **spurious `ident expected`** whose
+    position lands inside a repassed body, even though the emitted
+    `.ssa` is **identical** to the default.  So the repass parses to the
+    correct AST but trips one extra `SynError`.  Next: locate it (the
+    candidates are the `pos`/`len` "previous token" model interacting
+    with `SeekTo`, or a construct where the repass consumes one token
+    more).  Until then the pass stays opt-in and the default path (which
+    never defers) is what ships.