Theia-Modula-2/attic/oberon-language/src/oberonSymbols.ts

/** Local Oberon symbol model for Outline, completion and hover.
 *
 *  Conservative by design: only declaration shapes recognised with full
 *  confidence are emitted (misses are acceptable, wrong entries are not).
 *  Bodies are skipped with depth counting, so statement-level noise can
 *  never leak into the symbol table. Scopes: module globals plus one
 *  routine level (receiver, params and locals); deeper nesting folds
 *  into its owner. Covers Oberon-07, Oberon-2 and Ulm Oberon shapes;
 *  Oberon is case-sensitive throughout.
 */

export interface ObSymbol {
  name: string;
  kind: 'procedure' | 'function' | 'constant' | 'type' | 'variable' | 'parameter' | 'field' | 'module';
  detail: string;
  line: number; ch: number; endLine: number; endCh: number;
  children: ObSymbol[];
  /** Last line covered by this scope (routines/modules), if known. */
  scopeEndLine?: number;
  scopeEndCh?: number;
}

export interface Tok {
  text: string;
  upper: string;
  line: number; ch: number; endLine: number; endCh: number;
}

/** Strip comments and string literals (length-preserving) so the
 *  declaration scan never sees keywords inside them. Exported for the
 *  resolver's occurrence scan, which needs identical masking. */
export function mask(text: string): string {
  const out = text.split('');
  const n = text.length;
  let i = 0;
  const blank = (a: number, b: number) => { for (let k = a; k < b; k++) if (out[k] !== '\n') out[k] = ' '; };
  while (i < n) {
    const c = text[i];
    const nx = i + 1 < n ? text[i + 1] : '';
    if (c === '(' && nx === '*') {
      let depth = 1;
      let j = i + 2;
      while (j < n && depth > 0) {
        if (text[j] === '(' && text[j + 1] === '*') { depth++; j += 2; }
        else if (text[j] === '*' && text[j + 1] === ')') { depth--; j += 2; }
        else j++;
      }
      blank(i, j);
      i = j;
    } else if (c === '"' || c === "'") {
      let j = i + 1;
      while (j < n && text[j] !== c && text[j] !== '\n') j++;
      blank(i, Math.min(j + 1, n));
      i = Math.min(j + 1, n);
    } else {
      i++;
    }
  }
  return out.join('');
}

export function lex(masked: string): Tok[] {
  const toks: Tok[] = [];
  const re = /[A-Za-z_][A-Za-z0-9_]*|\d+(?:\.\d+)?(?:[EDed][+-]?\d+)?|:=|<=|>=|\.\.|[^\sA-Za-z0-9_]/g;
  let m: RegExpExecArray | null;
  // Incremental line tracking (O(n)); never re-slices from the file start.
  let line = 0;
  let prevEnd = 0;
  while ((m = re.exec(masked)) !== null) {
    for (let k = prevEnd; k < m.index; k++) {
      if (masked[k] === '\n') line++;
    }
    const ch = m.index - (masked.lastIndexOf('\n', m.index - 1) + 1);
    const mid = m[0].split('\n');
    toks.push({
      text: m[0], upper: m[0].toUpperCase(),
      line, ch,
      endLine: line + mid.length - 1,
      endCh: mid.length > 1 ? mid[mid.length - 1].length : ch + m[0].length
    });
    line += mid.length - 1;
    prevEnd = m.index + m[0].length;
  }
  return toks;
}

export const isIdent = (t: Tok) => /^[A-Za-z_]/.test(t.text);

class Parser {
  pos = 0;
  constructor(readonly toks: Tok[], readonly marks?: Set<string>) {}

  peek(off = 0): Tok | null {
    return this.pos + off < this.toks.length ? this.toks[this.pos + off] : null;
  }

  at(word: string): boolean {
    const t = this.peek();
    return !!t && t.upper === word;
  }

  next(): Tok {
    return this.toks[this.pos++];
  }

  /** Skip a parenthesis/bracket group, returns true when balanced. */
  skipGroup(open: string, close: string): boolean {
    if (!this.at(open)) return false;
    let depth = 0;
    while (this.pos < this.toks.length) {
      const t = this.next();
      if (t.upper === open) depth++;
      else if (t.upper === close) {
        depth--;
        if (depth === 0) return true;
      }
    }
    return false;
  }

  /** Skip a routine body: declarations then an optional compound. */
  skipBody(): void {
    for (;;) {
      const t = this.peek();
      if (!t) return;
      if (t.upper === 'BEGIN') {
        this.skipCompound();
        return;
      }
      if (['CONST', 'TYPE', 'VAR', 'PROCEDURE'].includes(t.upper)) {
        this.skipSection();
        continue;
      }
      return;
    }
  }

  skipCompound(): void {
    if (!this.at('BEGIN')) return;
    this.next();
    let depth = 1;
    while (this.pos < this.toks.length && depth > 0) {
      const t = this.next();
      // BEGIN/CASE/RECORD/LOOP/WITH each consume their own END.
      if (t.upper === 'BEGIN' || t.upper === 'CASE' || t.upper === 'RECORD' ||
          t.upper === 'LOOP' || t.upper === 'WITH') depth++;
      else if (t.upper === 'END') depth--;
    }
  }

  /** Skip one declaration section (const/type/var). */
  skipSection(): void {
    const kw = this.peek()?.upper;
    if (kw !== 'CONST' && kw !== 'TYPE' && kw !== 'VAR') return;
    this.next();
    for (;;) {
      const t = this.peek();
      if (!t) return;
      if (['CONST', 'TYPE', 'VAR', 'PROCEDURE', 'BEGIN', 'END'].includes(t.upper)) return;
      this.skipDeclItem();
      if (this.at(';')) this.next();
      else return;
    }
  }

  /** Skip one declaration item up to `;` at depth 0. */
  skipDeclItem(): void {
    let depth = 0;
    while (this.pos < this.toks.length) {
      const t = this.peek()!;
      if (t.upper === '(' || t.upper === '[') depth++;
      else if (t.upper === ')' || t.upper === ']') depth--;
      else if (t.upper === 'RECORD' || t.upper === 'CASE') depth++;
      else if (t.upper === 'END' && depth > 0) depth--;
      else if (depth === 0 && (t.upper === ';' || ['CONST', 'TYPE', 'VAR', 'PROCEDURE', 'BEGIN', 'END'].includes(t.upper))) return;
      this.next();
    }
  }

  /** Parse an identifier list with optional `*`/`-` export marks,
   *  returning the name tokens. Mark positions are recorded for the
   *  formatter (export marks attach; operators keep spaces). */
  nameList(): Tok[] {
    const out: Tok[] = [];
    for (;;) {
      const t = this.peek();
      if (!t || !isIdent(t)) return out;
      out.push(this.next());
      const mark = this.peek();
      if (mark?.text === '*' || mark?.text === '-') {
        this.next();
        this.marks?.add(`${mark.line}:${mark.ch}`);
      }
      if (this.at(',')) { this.next(); continue; }
      return out;
    }
  }

  /** Consume an optional `*`/`-` export mark after a declared name. */
  exportMark(): void {
    const mark = this.peek();
    if (mark?.text === '*' || mark?.text === '-') {
      this.next();
      this.marks?.add(`${mark.line}:${mark.ch}`);
    }
  }
}

/** Parse a `RECORD ... END` field list; RECORD already consumed.
 *
 *  Only plain `name [, ...] : type;` items at depth 1 become fields
 *  (export marks stripped). Anything unrecognised is skipped without
 *  emitting. The closing END is consumed. */
export function parseRecordFields(p: Parser): ObSymbol[] {
  const fields: ObSymbol[] = [];
  let depth = 1;
  for (;;) {
    const t = p.peek();
    if (!t) return fields;
    if (t.upper === 'RECORD' || t.upper === 'CASE') { depth++; p.next(); continue; }
    if (t.upper === 'END') {
      depth--;
      p.next();
      if (depth <= 0) return fields;
      continue;
    }
    if (depth !== 1) { p.next(); continue; }
    if (!isIdent(t)) {
      p.skipDeclItem();
      if (p.at(';')) p.next();
      else if (p.peek() && p.peek()!.upper !== 'END') p.next();
      continue;
    }
    if (['CONST', 'TYPE', 'VAR', 'PROCEDURE', 'BEGIN'].includes(t.upper)) {
      while (p.pos < p.toks.length && depth > 0) {
        const k = p.next();
        if (k.upper === 'RECORD' || k.upper === 'CASE') depth++;
        else if (k.upper === 'END') depth--;
      }
      return fields;
    }
    const names = p.nameList();
    if (!p.at(':')) { p.skipDeclItem(); if (p.at(';')) p.next(); continue; }
    p.next();
    const typeToks: string[] = [];
    let dd = 0;
    while (p.pos < p.toks.length) {
      const k = p.peek()!;
      if (k.upper === '(' || k.upper === '[') dd++;
      else if (k.upper === ')' || k.upper === ']') dd--;
      else if (k.upper === 'RECORD' || k.upper === 'CASE') dd++;
      else if (k.upper === 'END' && dd > 0) dd--;
      else if (dd === 0 && (k.upper === ';' || k.upper === 'END')) break;
      else if (dd === 0 && ['CONST', 'TYPE', 'VAR', 'PROCEDURE', 'BEGIN'].includes(k.upper)) break;
      typeToks.push(k.text);
      p.next();
    }
    const typeText = typeToks.join(' ').replace(/\s+([,;:()[\]])/g, '$1').replace(/([(\[])\s+/g, '$1').replace(/\s*\.\s*/g, '.');
    for (const nm of names) {
      fields.push({
        name: nm.text, kind: 'field', detail: `${nm.text}: ${typeText}`,
        line: nm.line, ch: nm.ch, endLine: nm.endLine, endCh: nm.endCh, children: []
      });
    }
    if (p.at(';')) p.next();
  }
}

/** An `IMPORT` clause entry: local name, real module, positions. */
export interface ObImport {
  /** Name used in this file (alias when `alias := Module` was written). */
  name: string;
  /** Real module name (after `:=`, or the name itself). */
  module: string;
  line: number; ch: number; endCh: number;
}

/** All `IMPORT a, b, alias := C, ...;` entries at paren-depth 0.
 *
 *  An entry not shaped `ident [:=> ident]` (or a clause without a
 *  terminating `;`) yields nothing. */
export function parseImports(text: string): ObImport[] {
  const toks = lex(mask(text));
  const out: ObImport[] = [];
  let depth = 0;
  for (let k = 0; k < toks.length; k++) {
    const t = toks[k];
    if (t.upper === '(') { depth++; continue; }
    if (t.upper === ')') { depth--; continue; }
    if (depth !== 0 || t.upper !== 'IMPORT') continue;
    let j = k + 1;
    let ok = false;
    for (;;) {
      const u = toks[j];
      if (!u) break;
      if (u.upper === ';') { ok = true; break; }
      if (u.upper === 'BEGIN' || u.upper === 'END' || u.upper === 'CONST' ||
          u.upper === 'TYPE' || u.upper === 'VAR' || u.upper === 'PROCEDURE') break;
      if (!isIdent(u)) {
        if (u.upper !== ',') break;
        j++;
        continue;
      }
      // ident [ := ident ] followed by ',' or ';'
      let real = u.text;
      let endCh = u.endCh;
      let jj = j + 1;
      if (toks[jj]?.text === ':=') {
        const mod = toks[jj + 1];
        if (!mod || !isIdent(mod)) break;
        real = mod.text;
        endCh = mod.endCh;
        jj += 2;
      }
      const after = toks[jj];
      if (!after || (after.upper !== ',' && after.upper !== ';')) break;
      out.push({ name: u.text, module: real, line: u.line, ch: u.ch, endCh });
      j = jj;
      if (after.upper === ';') { ok = true; break; }
      j++;
    }
    if (ok) k = j;
  }
  return out;
}

/** Parse the declarations of one scope; stops before BEGIN/END/EOF. */
function parseScope(p: Parser, out: ObSymbol[], stopAtBegin: boolean): void {
  for (;;) {
    const t = p.peek();
    if (!t) return;
    const u = t.upper;
    if (u === 'BEGIN') {
      if (stopAtBegin) return;
      p.skipBody();
      return;
    }
    if (u === 'END') return;
    if (u === 'CONST') {
      p.next();
      for (;;) {
        const n = p.peek();
        if (!n || !isIdent(n)) break;
        if (['TYPE', 'VAR', 'CONST', 'PROCEDURE', 'BEGIN', 'END'].includes(n.upper)) break;
        const nm = p.next();
        p.exportMark();
        if (!p.at('=')) { p.skipDeclItem(); if (p.at(';')) p.next(); continue; }
        p.next();
        p.skipDeclItem();
        out.push({ name: nm.text, kind: 'constant', detail: `CONST ${nm.text}`, line: nm.line, ch: nm.ch, endLine: nm.endLine, endCh: nm.endCh, children: [] });
        if (p.at(';')) p.next();
        else break;
      }
      continue;
    }
    if (u === 'TYPE') {
      p.next();
      for (;;) {
        const n = p.peek();
        if (!n || !isIdent(n)) break;
        if (['TYPE', 'VAR', 'CONST', 'PROCEDURE', 'BEGIN', 'END'].includes(n.upper)) break;
        const nm = p.next();
        p.exportMark();
        if (!p.at('=')) { p.skipDeclItem(); if (p.at(';')) p.next(); continue; }
        p.next();
        const typeChildren: ObSymbol[] = [];
        if (p.at('RECORD')) {
          p.next();
          typeChildren.push(...parseRecordFields(p));
        } else {
          p.skipDeclItem();
        }
        out.push({ name: nm.text, kind: 'type', detail: `TYPE ${nm.text}`, line: nm.line, ch: nm.ch, endLine: nm.endLine, endCh: nm.endCh, children: typeChildren });
        if (p.at(';')) p.next();
        else break;
      }
      continue;
    }
    if (u === 'VAR') {
      p.next();
      for (;;) {
        const n = p.peek();
        if (!n || !isIdent(n)) break;
        if (['TYPE', 'VAR', 'CONST', 'PROCEDURE', 'BEGIN', 'END'].includes(n.upper)) break;
        const names = p.nameList();
        if (!p.at(':')) { p.skipDeclItem(); if (p.at(';')) p.next(); continue; }
        p.next();
        if (p.at('RECORD')) {
          p.next();
          const fields = parseRecordFields(p);
          for (const nm of names) {
            out.push({ name: nm.text, kind: 'variable', detail: `VAR ${nm.text}: RECORD`, line: nm.line, ch: nm.ch, endLine: nm.endLine, endCh: nm.endCh, children: fields });
          }
          if (p.at(';')) p.next();
          else break;
          continue;
        }
        // capture type text up to ';' at depth 0
        const typeToks: string[] = [];
        let depth = 0;
        while (p.pos < p.toks.length) {
          const k = p.peek()!;
          if (k.upper === '(' || k.upper === '[') depth++;
          else if (k.upper === ')' || k.upper === ']') depth--;
          else if (k.upper === 'RECORD' || k.upper === 'CASE') depth++;
          else if (k.upper === 'END' && depth > 0) depth--;
          else if (depth === 0 && k.upper === ';') break;
          else if (depth === 0 && ['CONST', 'TYPE', 'VAR', 'PROCEDURE', 'BEGIN', 'END'].includes(k.upper)) break;
          typeToks.push(k.text);
          p.next();
        }
        const typeText = typeToks.join(' ').replace(/\s+([,;:()[\]])/g, '$1').replace(/([(\[])\s+/g, '$1').replace(/\s*\.\s*/g, '.');
        for (const nm of names) {
          out.push({ name: nm.text, kind: 'variable', detail: `VAR ${nm.text}: ${typeText}`, line: nm.line, ch: nm.ch, endLine: nm.endLine, endCh: nm.endCh, children: [] });
        }
        if (p.at(';')) p.next();
        else break;
      }
      continue;
    }
    if (u === 'PROCEDURE') {
      const kw = p.next();
      // Optional receiver `( [VAR] self [, ...] : Type )`: a type-bound
      // procedure. Its names are in scope in the body, like parameters.
      const receiver: Tok[] = [];
      if (p.at('(')) {
        p.next();
        let rd = 1;
        let sawColon = false;
        while (p.pos < p.toks.length && rd > 0) {
          const q = p.next();
          if (q.upper === '(') { rd++; continue; }
          if (q.upper === ')') {
            rd--;
            if (rd === 0) break;
            continue;
          }
          if (rd !== 1) continue;
          if (q.upper === ':') { sawColon = true; continue; }
          if (q.upper === ';' || q.upper === ',') continue;
          if (!sawColon && isIdent(q) && q.upper !== 'VAR' && q.upper !== 'IN') receiver.push(q);
        }
        if (rd !== 0) { p.skipDeclItem(); if (p.at(';')) p.next(); continue; }
      }
      const nt = p.peek();
      if (!nt || !isIdent(nt)) {
        p.skipDeclItem();
        if (p.at(';')) p.next();
        continue;
      }
      const nameTok = p.next();
      p.exportMark();
      // formal params
      const params: ObSymbol[] = [];
      if (p.at('(')) {
        p.next();
        let pd = 1;
        const pnames: Tok[] = [];
        let ptype: string[] = [];
        let inType = false;
        const flush = () => {
          const tt = ptype.join(' ').replace(/\s+([,;:()[\]])/g, '$1').replace(/([(\[])\s+/g, '$1').replace(/\s*\.\s*/g, '.');
          for (const nm of pnames) {
            params.push({ name: nm.text, kind: 'parameter', detail: `${nm.text}: ${tt}`, line: nm.line, ch: nm.ch, endLine: nm.endLine, endCh: nm.endCh, children: [] });
          }
          pnames.length = 0; ptype = []; inType = false;
        };
        while (p.pos < p.toks.length && pd > 0) {
          const k = p.next();
          if (k.upper === '(') { pd++; if (inType) ptype.push(k.text); continue; }
          if (k.upper === ')') { pd--; if (pd === 0) { if (pnames.length || ptype.length) flush(); break; } if (inType) ptype.push(k.text); continue; }
          if (pd !== 1) { if (inType) ptype.push(k.text); continue; }
          if (k.upper === ';') { flush(); continue; }
          if (k.upper === ':' && !inType) { inType = true; continue; }
          if (k.upper === ',') continue;
          if ((k.upper === 'VAR' || k.upper === 'IN') && !inType && pnames.length === 0) continue;
          if (k.text === '*' || k.text === '-') continue;
          if (isIdent(k)) { if (inType) ptype.push(k.text); else pnames.push(k); }
          else if (inType) ptype.push(k.text);
        }
      }
      // result type
      let restype = '';
      if (p.at(':')) {
        p.next();
        const rt: string[] = [];
        while (p.pos < p.toks.length) {
          const k = p.peek()!;
          if (k.upper === ';') break;
          rt.push(k.text);
          p.next();
        }
        restype = rt.join(' ').replace(/\s+([,;:()[\]])/g, '$1').replace(/([(\[])\s+/g, '$1').replace(/\s*\.\s*/g, '.');
      }
      const sym: ObSymbol = {
        name: nameTok.text,
        kind: 'procedure',
        detail: `PROCEDURE ${nameTok.text}${restype ? ': ' + restype : ''}`,
        line: kw.line, ch: kw.ch, endLine: nameTok.endLine, endCh: nameTok.endCh,
        children: [
          ...receiver.map(nm => ({
            name: nm.text, kind: 'parameter' as const,
            detail: `${nm.text}: receiver`,
            line: nm.line, ch: nm.ch, endLine: nm.endLine, endCh: nm.endCh, children: [] as ObSymbol[]
          })),
          ...params
        ]
      };
      out.push(sym);
      if (p.at(';')) p.next();
      for (;;) {
        const k = p.peek();
        if (!k) break;
        if (k.upper === 'BEGIN') { p.skipBody(); break; }
        if (k.upper === 'END' || k.upper === 'PROCEDURE') break;
        if (['CONST', 'TYPE', 'VAR'].includes(k.upper)) {
          parseScope(p, sym.children, true);
          continue;
        }
        break;
      }
      const tail = p.toks[p.pos - 1];
      if (tail) { sym.scopeEndLine = tail.endLine; sym.scopeEndCh = tail.endCh; }
      continue;
    }
    // anything else at scope level: skip one item to stay in sync
    p.skipDeclItem();
    if (p.at(';')) p.next();
    else if (p.pos < p.toks.length) p.next();
  }
}

/** Entry: top-level symbols of an Oberon module.
 *  When `marks` is given, export-mark `*`/`-` positions (`line:ch`)
 *  are collected for the formatter. */
export function parseOberon(text: string, marks?: Set<string>): ObSymbol[] {
  const toks = lex(mask(text));
  const p = new Parser(toks, marks);
  // skip an optional DEFINITION prefix and the MODULE header to ';'
  if (p.at('DEFINITION')) p.next();
  if (p.at('MODULE')) {
    p.next();
    let depth = 0;
    while (p.pos < p.toks.length) {
      const t = p.next();
      if (t.upper === '(') depth++;
      else if (t.upper === ')') depth--;
      else if (t.upper === ';' && depth === 0) break;
    }
  }
  const out: ObSymbol[] = [];
  parseScope(p, out, false);
  const first = toks.length > 0 ? toks[0] : null;
  if (first && first.upper === 'MODULE' && toks.length > 1 && isIdent(toks[1])) {
    const nm = toks[1];
    const last = toks[toks.length - 1];
    const mod: ObSymbol = { name: nm.text, kind: 'module', detail: `MODULE ${nm.text}`, line: first.line, ch: first.ch, endLine: nm.endLine, endCh: nm.endCh, children: out };
    if (last) { mod.scopeEndLine = last.endLine; mod.scopeEndCh = last.endCh; }
    return [mod];
  }
  return out;
}

/** True when (line, ch) lies within the symbol's scope, if known. */
function inScope(s: ObSymbol, line: number, ch: number): boolean {
  if (s.scopeEndLine === undefined) return true;
  if (line < s.line) return false;
  if (line > s.scopeEndLine) return false;
  if (line === s.scopeEndLine && s.scopeEndCh !== undefined && ch > s.scopeEndCh) return false;
  return true;
}

/** Symbols visible at a position: globals plus the innermost enclosing
 *  routine's receiver, params and locals. Forgiving (completion-only). */
export function visibleAt(symbols: ObSymbol[], line: number, ch: number): ObSymbol[] {
  const top: ObSymbol[] = [];
  for (const s of symbols) {
    if (s.kind === 'module') top.push(...s.children);
    else top.push(s);
  }
  const result: ObSymbol[] = [];
  for (const s of top) {
    if (s.kind === 'constant' || s.kind === 'type' || s.kind === 'variable') result.push(s);
  }
  const routines = top.filter(s => s.kind === 'procedure' && inScope(s, line, ch));
  routines.sort((a, b) => (b.line - a.line) || ((b.ch ?? 0) - (a.ch ?? 0)));
  for (const r of routines) {
    result.push(...r.children);
    result.push(r);
  }
  for (const s of top) {
    if (s.kind === 'procedure' && !result.includes(s)) result.push(s);
  }
  return result;
}