Semantic tokens for Modula-2 editors

Server-driven highlighting for variables, parameters, fields, types,
procedures, modules and enum members via textDocument/semanticTokens,
reusing the hover/definition resolver. Unresolvable identifiers are
skipped so broken code falls back to TextMate colors.

Workbench contribution enables fetching (editor.semanticHighlighting
defaults to true unless configured) and patches the bundled monaco
themes with one additive token rule per type.

extensions/modula2-language/src/m2/semantic.ts

@@ -0,0 +1,127 @@
+/** Semantic tokens for Modula-2 editors (LSP `textDocument/semanticTokens`).
+ *
+ *  Declarations come from the symbol tree; identifier *uses* are resolved
+ *  through the same machinery as hover/definition, so variables, parameters,
+ *  fields, enum members and cross-file imports all highlight by meaning,
+ *  not just by spelling. Anything unresolvable is skipped, so broken code
+ *  under editing degrades to TextMate highlighting instead of errors.
+ */
+
+import { M2Symbol, M2Unit, parseUnitText } from './parser';
+import { flattenUnit, identifierOccurrences, resolveName } from './resolve';
+
+export const SEMANTIC_TOKEN_TYPES = [
+  'namespace', // modules
+  'type', // declared types
+  'function', // procedures
+  'variable', // variables and constants
+  'parameter', // parameters
+  'property', // record fields
+  'enumMember', // enumeration literals
+];
+
+export const SEMANTIC_TOKEN_MODIFIERS = ['readonly'];
+
+export function semanticTokensLegend(): { tokenTypes: string[]; tokenModifiers: string[] } {
+  return { tokenTypes: [...SEMANTIC_TOKEN_TYPES], tokenModifiers: [...SEMANTIC_TOKEN_MODIFIERS] };
+}
+
+interface RawToken {
+  line: number;
+  ch: number;
+  length: number;
+  type: number;
+  modifiers: number;
+}
+
+/** LSP delta-encoded token data for a document. Never throws on broken input. */
+export function computeSemanticTokens(text: string, filePath: string, docDir: string): number[] {
+  const unit = parseUnitText(text, filePath);
+  const out: RawToken[] = [];
+  const declared = new Set<string>();
+  for (const { sym, chain } of flattenUnit(unit)) {
+    if (!sym.name) continue;
+    // The file's own module symbol is the root scope, not a declaration.
+    if (sym.kind === 'module' && chain.length === 0) continue;
+    declared.add(`${sym.nameRange.startLine}:${sym.nameRange.startCh}`);
+    const mapped = symbolToken(sym, chain);
+    if (!mapped) continue;
+    out.push({
+      line: sym.nameRange.startLine,
+      ch: sym.nameRange.startCh,
+      length: Math.max(1, sym.nameRange.endCh - sym.nameRange.startCh),
+      type: mapped.type,
+      modifiers: mapped.modifiers,
+    });
+  }
+  for (const occ of identifierOccurrences(text)) {
+    if (declared.has(`${occ.line}:${occ.ch}`)) continue;
+    const resolved = resolveName(unit, docDir, occ.line, occ.ch, occ.qualifier, occ.name);
+    if (!resolved) continue;
+    const kind = useTokenKind(resolved.sym);
+    if (kind === null) continue;
+    out.push({
+      line: occ.line,
+      ch: occ.ch,
+      length: Math.max(1, occ.endCh - occ.ch),
+      type: kind,
+      modifiers: 0,
+    });
+  }
+  out.sort((a, b) => a.line - b.line || a.ch - b.ch);
+  const data: number[] = [];
+  let prevLine = 0;
+  let prevCh = 0;
+  for (const t of out) {
+    data.push(
+      t.line - prevLine,
+      t.line === prevLine ? t.ch - prevCh : t.ch,
+      t.length, t.type, t.modifiers,
+    );
+    prevLine = t.line;
+    prevCh = t.ch;
+  }
+  return data;
+}
+
+function typeIndex(name: string): number {
+  const i = SEMANTIC_TOKEN_TYPES.indexOf(name);
+  return i >= 0 ? i : 0;
+}
+
+function symbolToken(
+  sym: M2Symbol, chain: M2Symbol[],
+): { type: number; modifiers: number } | null {
+  switch (sym.kind) {
+    case 'module': return { type: typeIndex('namespace'), modifiers: 0 };
+    case 'procedure': return { type: typeIndex('function'), modifiers: 0 };
+    case 'variable': return { type: typeIndex('variable'), modifiers: 0 };
+    case 'parameter': return { type: typeIndex('parameter'), modifiers: 0 };
+    case 'field': return { type: typeIndex('property'), modifiers: 0 };
+    case 'constant': {
+      // Enumeration literals live under their type; other constants read-only.
+      const underType = chain.length > 0 && chain[chain.length - 1].kind === 'type';
+      return underType
+        ? { type: typeIndex('enumMember'), modifiers: 0 }
+        : { type: typeIndex('variable'), modifiers: 1 };
+    }
+    case 'type': return { type: typeIndex('type'), modifiers: 0 };
+    default: return null;
+  }
+}
+
+/** Token type for a resolved identifier use (null = no token). */
+function useTokenKind(sym: M2Symbol): number | null {
+  switch (sym.kind) {
+    case 'module': return typeIndex('namespace');
+    case 'procedure': return typeIndex('function');
+    case 'variable': return typeIndex('variable');
+    case 'parameter': return typeIndex('parameter');
+    case 'field': return typeIndex('property');
+    case 'constant':
+      // Enumeration literals are recorded as `Type.Literal`.
+      return /^\w+\.\w+$/.test(sym.detail) ? typeIndex('enumMember') : typeIndex('variable');
+    case 'type': return typeIndex('type');
+    default: return null;
+  }
+}

extensions/modula2-language/src/server.ts

@@ -13,6 +13,7 @@ import { pathToFileURL } from 'url';
 import { lex, Token } from './m2/lexer';
 import { M2Symbol, M2Unit, parseUnitText } from './m2/parser';
 import { analyseUnit } from './m2/analyse';
+import { computeSemanticTokens, semanticTokensLegend } from './m2/semantic';
 import { DEFAULT_FORMAT_OPTIONS, FormatOptions, formatDocument, normalizeFormatOptions } from './m2/format';
 import {
   moduleExports, recordFieldsOf, resolveName, toLocation, visibleSymbols,
@@ -43,6 +44,10 @@ connection.onInitialize((params: InitializeParams): InitializeResult => {
       referencesProvider: true,
       renameProvider: { prepareProvider: true },
       documentFormattingProvider: true,
+      semanticTokensProvider: {
+        legend: semanticTokensLegend(),
+        full: true,
+      },
     },
   };
 });
@@ -485,5 +490,19 @@ connection.onDocumentFormatting(params => {
   return [edit];
 });
 
+connection.languages.semanticTokens.on(params => {
+  const document = documents.get(params.textDocument.uri);
+  if (!document) return { data: [] };
+  const filePath = uriToFilePath(document.uri);
+  try {
+    return {
+      data: computeSemanticTokens(document.getText(), filePath, path.dirname(filePath)),
+    };
+  } catch (error) {
+    connection.console.error(`semantic tokens failed: ${error instanceof Error ? error.message : String(error)}`);
+    return { data: [] };
+  }
+});
+
 documents.listen(connection);
 connection.listen();

extensions/modula2-workbench/package.json

@@ -4,8 +4,8 @@
   "version": "0.1.0",
   "keywords": ["theia-extension"],
   "files": ["lib", "src"],
-  "dependencies": {"@theia/core": "1.75.0"},
+  "dependencies": {"@theia/core": "1.75.0", "@theia/monaco": "1.75.0"},
   "devDependencies": {"rimraf": "^6.0.1", "typescript": "^5.9.2"},
   "scripts": {"clean": "rimraf lib", "build": "tsc -p tsconfig.json", "watch": "tsc -p tsconfig.json --watch", "prepare": "yarn clean && yarn build"},
   "theiaExtensions": [{"frontend": "lib/browser/modula2-workbench-frontend-module"}]
-}
+}

extensions/modula2-workbench/src/browser/modula2-semantic-colors.ts

@@ -0,0 +1,62 @@
+/** Pure semantic-token color support (no Theia/monaco imports, testable headlessly). */
+
+/** Semantic token colors by theme brightness (VS Code conventions). */
+export function semanticTokenRules(dark: boolean): Record<string, string> {
+  if (dark) {
+    return {
+      variable: '#9CDCFE',
+      parameter: '#9CDCFE',
+      property: '#9CDCFE',
+      function: '#DCDCAA',
+      type: '#4EC9B0',
+      namespace: '#4EC9B0',
+      enumMember: '#4EC9B0',
+    };
+  }
+  return {
+    variable: '#001080',
+    parameter: '#001080',
+    property: '#001080',
+    function: '#795E26',
+    type: '#267F99',
+    namespace: '#267F99',
+    enumMember: '#267F99',
+  };
+}
+
+export interface TokenRuleLike {
+  token: string;
+  foreground?: string;
+  [key: string]: unknown;
+}
+
+export interface ThemeDataLike {
+  semanticHighlighting?: boolean;
+  semanticTokenColors?: Record<string, string>;
+  rules?: TokenRuleLike[];
+  [key: string]: unknown;
+}
+
+/** Add semantic highlighting to theme data without touching anything else.
+ *
+ *  The bundled Theia monaco build hardcodes `StandaloneTheme.semanticHighlighting`
+ *  to false and styles semantic tokens through the TextMate `rules` (see
+ *  `standaloneThemeService.js` / `semanticTokensProviderStyling.js`), so the
+ *  `semanticTokenColors` map alone is inert there. We therefore add one `rules`
+ *  entry per semantic type as well (keeping `semanticTokenColors` for builds
+ *  that do honor it). Existing entries always win. Idempotent.
+ */
+export function withSemanticColors<T extends ThemeDataLike>(data: T, dark: boolean): T {
+  const wanted = semanticTokenRules(dark);
+  const existing = Array.isArray(data.rules) ? data.rules : [];
+  const have = new Set(existing.map(r => r.token));
+  const added = Object.entries(wanted)
+    .filter(([token]) => !have.has(token))
+    .map(([token, foreground]) => ({ token, foreground }));
+  return {
+    ...data,
+    semanticHighlighting: true,
+    semanticTokenColors: { ...wanted, ...(data.semanticTokenColors ?? {}) },
+    rules: [...existing, ...added],
+  };
+}

extensions/modula2-workbench/src/browser/modula2-semantic-style.ts

@@ -0,0 +1,74 @@
+import { inject, injectable } from '@theia/core/shared/inversify';
+import { FrontendApplicationContribution } from '@theia/core/lib/browser/frontend-application-contribution';
+import { ThemeService } from '@theia/core/lib/browser/theming';
+import { PreferenceService } from '@theia/core/lib/common/preferences/preference-service';
+import { PreferenceScope } from '@theia/core/lib/common/preferences/preference-scope';
+import { MonacoThemeRegistry } from '@theia/monaco/lib/browser/textmate/monaco-theme-registry';
+import type { ThemeMix } from '@theia/monaco/lib/browser/textmate/monaco-theme-types';
+import { withSemanticColors, ThemeDataLike } from './modula2-semantic-colors';
+
+/** Bundled monaco theme names patched with semantic colors (best-effort). */
+const MONACO_THEMES = ['dark-theia', 'light-theia', 'hc-theia', 'hc-theia-light'];
+
+/** Preference gate monaco checks before even fetching semantic tokens. */
+const SEMANTIC_PREF = 'editor.semanticHighlighting.enabled';
+
+/** Enables semantic highlighting in the bundled editor themes.
+ *
+ *  Stock Theia themes ship no rules for our token types, so semantic tokens
+ *  would fall back to TextMate colors even when fetched — and worse, the
+ *  bundled monaco hardcodes `StandaloneTheme.semanticHighlighting` to false,
+ *  so with the default `configuredByTheme` setting tokens are never fetched
+ *  at all. This (a) patches the registered monaco themes in place with one
+ *  additive `rules` entry per token type, and (b) defaults the preference to
+ *  `true` unless the user configured any value at any scope. Monaco refreshes
+ *  open editors automatically.
+ */
+@injectable()
+export class Modula2SemanticStyling implements FrontendApplicationContribution {
+  @inject(MonacoThemeRegistry) protected readonly themes: MonacoThemeRegistry;
+  @inject(ThemeService) protected readonly themeService: ThemeService;
+  @inject(PreferenceService) protected readonly prefs: PreferenceService;
+
+  onStart(): void {
+    this.ensureEnabled();
+    this.patchAll();
+    this.themeService.onDidColorThemeChange(() => this.patchAll());
+  }
+
+  /** Default the fetch gate to on, without overriding an explicit user choice. */
+  protected ensureEnabled(): void {
+    try {
+      const inspection = this.prefs.inspect<boolean>(SEMANTIC_PREF);
+      if (!inspection) return;
+      const configured = inspection.globalValue ?? inspection.workspaceValue
+        ?? inspection.workspaceFolderValue ?? inspection.sessionValue;
+      if (typeof configured === 'undefined') {
+        this.prefs.set(SEMANTIC_PREF, true, PreferenceScope.User).catch(error => {
+          console.warn('Modula-2: could not enable semantic highlighting:', error);
+        });
+      }
+    } catch (error) {
+      console.warn('Modula-2: could not inspect semantic highlighting preference:', error);
+    }
+  }
+
+  protected patchAll(): void {
+    const done: string[] = [];
+    for (const name of MONACO_THEMES) {
+      try {
+        const theme = this.themes.getTheme(name);
+        const data = this.themes.getThemeData(name);
+        if (!theme || !data) continue;
+        const base = (theme as unknown as { base?: string }).base;
+        const dark = base === 'vs-dark' || base === 'hc-black';
+        this.themes.setTheme(name, withSemanticColors(data as unknown as ThemeDataLike, dark) as unknown as ThemeMix);
+        done.push(name + ':' + base);
+      } catch (error) {
+        console.warn(`Modula-2: could not patch semantic colors into '${name}':`, error);
+      }
+    }
+    // eslint-disable-next-line no-console
+    console.info(`Modula-2: semantic colors applied to ${done.length} editor theme(s): ${done.join(', ')}`);
+  }
+}

extensions/modula2-workbench/src/browser/modula2-workbench-frontend-module.ts

@@ -5,6 +5,7 @@ import { PreferenceContribution } from '@theia/core/lib/common/preferences/prefe
 import { modula2UIPreferenceSchema } from './modula2-ui-preferences';
 import { Modula2ThemeCommands, SwitchThemeCommand, ToggleThemeCommand } from './modula2-theme-commands';
 import { Modula2UIFontContribution } from './modula2-ui-font-contribution';
+import { Modula2SemanticStyling } from './modula2-semantic-style';
 
 const category = 'Modula-2';
 
@@ -35,5 +36,6 @@ export default new ContainerModule(bind => {
   bind(CommandContribution).to(CommandsContribution).inSingletonScope();
   bind(CommandContribution).to(Modula2ThemeCommands).inSingletonScope();
   bind(FrontendApplicationContribution).to(Modula2UIFontContribution).inSingletonScope();
+  bind(FrontendApplicationContribution).to(Modula2SemanticStyling).inSingletonScope();
   bind(MenuContribution).to(MenusContribution).inSingletonScope();
 });

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "modula2-ide",
-  "version": "0.19.0",
+  "version": "0.20.0",
   "description": "GNU Modula-2 IDE built on Eclipse Theia",
   "engines": {
     "node": ">=24",