TestATNDeserialization, TestATNInterpreter and TestAttributeChecks done

- Brought back the Tool specific error managers. Needed for tests, where we create more than one Tool instance.
- Further improvements of the action splitter. It's pretty difficult to simulate the exact behavior of the old ANTLR3 lexer (where element references were allowed). Had to change a couple of error messages in the tests, where the column differed by 1.
- Further improvements of the parse tree to AST converter.
- Change tool parameter parsing. It's no longer taken from the process parameters directly. This way we can dynamically use varying parameters (e.g. for the unit tests).
- TestATNDeserialization, TestATNInterpreter and TestAttributeChecks done.
- Bug fixes.

.vscode/launch.json

@@ -4,6 +4,15 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "type": "antlr-debug",
+            "request": "launch",
+            "name": "Debug Current Grammar",
+            "input": "/Volumes/Extern/Work/projects/ANTLRng/tests/temp/input.txt",
+            "visualParseTree": true,
+            "startRule": "a",
+            "grammar": "${file}",
+        },
         {
             "type": "node",
             "request": "launch",

cli/Interpreter.ts

@@ -16,6 +16,7 @@ import { Grammar } from "../src/tool/Grammar.js";
 import type { GrammarParserInterpreter } from "../src/tool/GrammarParserInterpreter.js";
 import { LexerGrammar } from "../src/tool/LexerGrammar.js";
 import { encodings, parseBoolean } from "./cli-options.js";
+import type { Tool } from "../src/Tool.js";
 
 /** CLI parameters for the interpreter tool. */
 export interface IInterpreterCliParameters {
@@ -62,6 +63,9 @@ export class Interpreter {
         }
     };
 
+    public constructor(private tool: Tool) {
+    }
+
     public static getValue(decisionInfo: DecisionInfo, ruleNamesByDecision: string[], decision: number,
         col: number): number | string {
         switch (col) { // laborious but more efficient than reflection
@@ -107,7 +111,7 @@ export class Interpreter {
 
         let g: Grammar;
         let lg = null;
-        const listener = new DefaultToolListener();
+        const listener = new DefaultToolListener(this.tool.errorManager);
         if (interpreterOptions.grammarFileName) {
             const grammarContent = await readFile(interpreterOptions.grammarFileName, "utf8");
             g = new Interpreter.IgnoreTokenVocabGrammar(interpreterOptions.grammarFileName, grammarContent, undefined,
@@ -206,5 +210,5 @@ export class Interpreter {
     }
 }
 
-const interpreter = new Interpreter();
-await interpreter.interp();
+// todo: const interpreter = new Interpreter();
+// todo: await interpreter.interp();

src/Tool.ts

@@ -20,7 +20,7 @@ import { LexerATNFactory } from "./automata/LexerATNFactory.js";
 import { ParserATNFactory } from "./automata/ParserATNFactory.js";
 import { CodeGenPipeline } from "./codegen/CodeGenPipeline.js";
 import { CodeGenerator } from "./codegen/CodeGenerator.js";
-import { grammarOptions } from "./grammar-options.js";
+import { grammarOptions, parseToolParameters } from "./grammar-options.js";
 import { Graph } from "./misc/Graph.js";
 import { ToolANTLRLexer } from "./parse/ToolANTLRLexer.js";
 import { ToolANTLRParser } from "./parse/ToolANTLRParser.js";
@@ -47,21 +47,24 @@ export class Tool implements ITool {
 
     public static readonly ALL_GRAMMAR_EXTENSIONS = [Tool.GRAMMAR_EXTENSION, Tool.LEGACY_GRAMMAR_EXTENSION];
 
-    public inputDirectory: string;
-
     public readonly args: string[];
 
     public logMgr = new LogManager();
 
+    public readonly errorManager;
+
     // helper vars for option management
     protected haveOutputDir = false;
 
     protected grammarFiles = new Array<string>();
 
     private readonly importedGrammars = new Map<string, Grammar>();
 
-    public constructor(args?: string[]) {
-        this.args = args ?? [];
+    public constructor(args: string[] = []) {
+        this.args = args;
+        parseToolParameters(this.args);
+        this.grammarFiles = grammarOptions.args;
+        this.errorManager = new ErrorManager();
     }
 
     public static main(args: string[]): void {
@@ -74,7 +77,7 @@ export class Tool implements ITool {
                     const logName = antlr.logMgr.save();
                     console.log("wrote " + logName);
                 } catch (ioe) {
-                    ErrorManager.get().toolError(ErrorType.INTERNAL_ERROR, ioe);
+                    antlr.errorManager.toolError(ErrorType.INTERNAL_ERROR, ioe);
                 }
             }
         }
@@ -139,7 +142,7 @@ export class Tool implements ITool {
                 const dep = new BuildDependencyGenerator(this, g);
                 console.log(dep.getDependencies().render());
             } else {
-                if (ErrorManager.get().errors === 0) {
+                if (this.errorManager.errors === 0) {
                     this.process(g, true);
                 }
             }
@@ -185,13 +188,13 @@ export class Tool implements ITool {
             return;
         }
 
-        const prevErrors = ErrorManager.get().errors;
+        const prevErrors = this.errorManager.errors;
 
         // MAKE SURE GRAMMAR IS SEMANTICALLY CORRECT (FILL IN GRAMMAR OBJECT)
         const sem = new SemanticPipeline(g);
         sem.process();
 
-        if (ErrorManager.get().errors > prevErrors) {
+        if (this.errorManager.errors > prevErrors) {
             return;
         }
 
@@ -216,7 +219,7 @@ export class Tool implements ITool {
                 const fileName = this.getOutputFile(g, g.name + ".interp");
                 writeFileSync(fileName, interpFile);
             } catch (ioe) {
-                ErrorManager.get().toolError(ErrorType.CANNOT_WRITE_FILE, ioe);
+                this.errorManager.toolError(ErrorType.CANNOT_WRITE_FILE, ioe);
             }
         }
 
@@ -259,15 +262,15 @@ export class Tool implements ITool {
             const prev = ruleToAST.get(ruleName);
             if (prev) {
                 const prevChild = prev.getChild(0) as GrammarAST;
-                ErrorManager.get().grammarError(ErrorType.RULE_REDEFINITION, g.fileName, id.token!, ruleName,
+                this.errorManager.grammarError(ErrorType.RULE_REDEFINITION, g.fileName, id.token!, ruleName,
                     prevChild.token!.line);
                 redefinition = true;
                 continue;
             }
             ruleToAST.set(ruleName, ruleAST);
         }
 
-        const chk = new UndefChecker(g.isLexer(), ruleToAST);
+        const chk = new UndefChecker(g.isLexer(), ruleToAST, this.errorManager);
         chk.visitGrammar(g.ast);
 
         return redefinition; //  || chk.errors;
@@ -351,13 +354,12 @@ export class Tool implements ITool {
 
     public parseGrammar(fileName: string): GrammarRootAST | undefined {
         try {
-            const fullPath = path.join(this.inputDirectory, fileName);
-            const content = readFileSync(fullPath, { encoding: grammarOptions.grammarEncoding as BufferEncoding });
+            const content = readFileSync(fileName, { encoding: grammarOptions.grammarEncoding as BufferEncoding });
             const input = CharStream.fromString(content.toString());
 
             return this.parse(fileName, input);
         } catch (ioe) {
-            ErrorManager.get().toolError(ErrorType.CANNOT_OPEN_FILE, ioe, fileName);
+            this.errorManager.toolError(ErrorType.CANNOT_OPEN_FILE, ioe, fileName);
             throw ioe;
         }
     }
@@ -398,7 +400,7 @@ export class Tool implements ITool {
             }
 
             if (!importedFile) {
-                ErrorManager.get().grammarError(ErrorType.CANNOT_FIND_IMPORTED_GRAMMAR, g.fileName, nameNode.token!,
+                this.errorManager.grammarError(ErrorType.CANNOT_FIND_IMPORTED_GRAMMAR, g.fileName, nameNode.token!,
                     name);
 
                 return null;
@@ -425,7 +427,7 @@ export class Tool implements ITool {
     }
 
     public parse(fileName: string, input: CharStream): GrammarRootAST | undefined {
-        const lexer = new ToolANTLRLexer(input);
+        const lexer = new ToolANTLRLexer(input, this);
         const tokens = new CommonTokenStream(lexer);
         const p = new ToolANTLRParser(tokens, this);
         const grammarSpec = p.grammarSpec();
@@ -450,7 +452,7 @@ export class Tool implements ITool {
                     const dot = dotGenerator.getDOTFromState(g.atn!.ruleToStartState[r.index]!, g.isLexer());
                     this.writeDOTFile(g, r, dot);
                 } catch (ioe) {
-                    ErrorManager.get().toolError(ErrorType.CANNOT_WRITE_FILE, ioe);
+                    this.errorManager.toolError(ErrorType.CANNOT_WRITE_FILE, ioe);
                     throw ioe;
                 }
             }
@@ -494,22 +496,21 @@ export class Tool implements ITool {
     }
 
     public getImportedGrammarFile(g: Grammar, fileName: string): string | undefined {
-        let importedFile = path.join(this.inputDirectory, fileName);
-        if (!existsSync(importedFile)) {
-            const parentDir = basename(importedFile); // Check the parent dir of input directory.
-            importedFile = path.join(parentDir, fileName);
-            if (!existsSync(importedFile)) { // try in lib dir
+        if (!existsSync(fileName)) {
+            const parentDir = basename(fileName); // Check the parent dir of input directory.
+            fileName = path.join(parentDir, fileName);
+            if (!existsSync(fileName)) { // try in lib dir
                 const libDirectory = grammarOptions.libDirectory;
                 if (libDirectory) {
-                    importedFile = path.join(libDirectory, fileName);
-                    if (!existsSync(importedFile)) {
+                    fileName = path.join(libDirectory, fileName);
+                    if (!existsSync(fileName)) {
                         return undefined;
                     }
                 }
             }
         }
 
-        return importedFile;
+        return fileName;
     }
 
     /**
@@ -533,7 +534,7 @@ export class Tool implements ITool {
     }
 
     public getNumErrors(): number {
-        return ErrorManager.get().errors;
+        return this.errorManager.errors;
     }
 
     public exit(e: number): void {

src/UndefChecker.ts

@@ -16,7 +16,8 @@ import { GrammarTreeVisitor } from "./tree-walkers/GrammarTreeVisitor.js";
 export class UndefChecker extends GrammarTreeVisitor {
     public badRef = false;
 
-    public constructor(private isLexer: boolean, private ruleToAST: Map<string, RuleAST>) {
+    public constructor(private isLexer: boolean, private ruleToAST: Map<string, RuleAST>,
+        private errorManager: ErrorManager) {
         super();
     }
 
@@ -38,11 +39,11 @@ export class UndefChecker extends GrammarTreeVisitor {
         if (Character.isUpperCase(this.currentRuleName!.codePointAt(0)!) &&
             Character.isLowerCase(ref.getText().codePointAt(0)!)) {
             this.badRef = true;
-            ErrorManager.get().grammarError(ErrorType.PARSER_RULE_REF_IN_LEXER_RULE,
+            this.errorManager.grammarError(ErrorType.PARSER_RULE_REF_IN_LEXER_RULE,
                 fileName, ref.token!, ref.getText(), this.currentRuleName);
         } else if (!ruleAST) {
             this.badRef = true;
-            ErrorManager.get().grammarError(ErrorType.UNDEFINED_RULE_REF, fileName, ref.token!, ref.getText());
+            this.errorManager.grammarError(ErrorType.UNDEFINED_RULE_REF, fileName, ref.token!, ref.getText());
         }
     }
 

src/analysis/AnalysisPipeline.ts

@@ -8,7 +8,6 @@
 
 import { IntervalSet, LL1Analyzer, Token } from "antlr4ng";
 import { Utils } from "../misc/Utils.js";
-import { ErrorManager } from "../tool/ErrorManager.js";
 import { ErrorType } from "../tool/ErrorType.js";
 import { Grammar } from "../tool/Grammar.js";
 import { GrammarAST } from "../tool/ast/GrammarAST.js";
@@ -67,7 +66,7 @@ export class AnalysisPipeline {
             const analyzer = new LL1Analyzer();
             const look = analyzer.look(this.g.atn!, this.g.atn!.ruleToStartState[rule.index]!, undefined);
             if (look.contains(Token.EPSILON)) {
-                ErrorManager.get().grammarError(ErrorType.EPSILON_TOKEN, this.g.fileName,
+                this.g.tool.errorManager.grammarError(ErrorType.EPSILON_TOKEN, this.g.fileName,
                     (rule.ast.getChild(0) as GrammarAST).token!, rule.name);
             }
         }

src/analysis/LeftRecursionDetector.ts

@@ -8,7 +8,6 @@
 
 import { ATN, ATNState, HashSet, RuleStartState, RuleStopState, RuleTransition } from "antlr4ng";
 
-import { ErrorManager } from "../tool/ErrorManager.js";
 import type { Grammar } from "../tool/Grammar.js";
 import { LeftRecursionCyclesMessage } from "../tool/LeftRecursionCyclesMessage.js";
 import type { Rule } from "../tool/Rule.js";
@@ -138,6 +137,6 @@ export class LeftRecursionDetector {
 
     private leftRecursionCycles(fileName: string, cycles: Rule[][]): void {
         const msg = new LeftRecursionCyclesMessage(fileName, cycles);
-        ErrorManager.get().error(msg);
+        this.g.tool.errorManager.error(msg);
     }
 }

src/analysis/LeftRecursiveRuleAnalyzer.ts

@@ -15,7 +15,6 @@ import { CommonTreeNodeStream } from "../antlr3/tree/CommonTreeNodeStream.js";
 import { CodeGenerator, type SupportedLanguage } from "../codegen/CodeGenerator.js";
 import { ANTLRv4Parser } from "../generated/ANTLRv4Parser.js";
 import { GrammarASTAdaptor } from "../parse/GrammarASTAdaptor.js";
-import { ErrorManager } from "../tool/ErrorManager.js";
 import { ErrorType } from "../tool/ErrorType.js";
 import { AltAST } from "../tool/ast/AltAST.js";
 import { GrammarAST } from "../tool/ast/GrammarAST.js";
@@ -120,14 +119,14 @@ export class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
                 if (a === Associativity.Left.toString()) {
                     assoc = Associativity.Left;
                 } else {
-                    ErrorManager.get().grammarError(ErrorType.ILLEGAL_OPTION_VALUE, t.g.fileName,
+                    this.tool.errorManager.grammarError(ErrorType.ILLEGAL_OPTION_VALUE, t.g.fileName,
                         t.getOptionAST("assoc")!.token!, "assoc", assoc);
                 }
             }
         }
 
         if (this.altAssociativity.get(alt) && this.altAssociativity.get(alt) !== assoc) {
-            ErrorManager.get().toolError(ErrorType.INTERNAL_ERROR, "all operators of alt " + alt +
+            this.tool.errorManager.toolError(ErrorType.INTERNAL_ERROR, "all operators of alt " + alt +
                 " of left-recursive rule must have same associativity");
         }
         this.altAssociativity.set(alt, assoc);

src/analysis/LeftRecursiveRuleTransformer.ts

@@ -24,7 +24,6 @@ import { RuleCollector } from "../semantics/RuleCollector.js";
 import { ParseTreeToASTConverter } from "../support/ParseTreeToASTConverter.js";
 import { isTokenName } from "../support/helpers.js";
 import { DictType } from "../tool/DictType.js";
-import { ErrorManager } from "../tool/ErrorManager.js";
 import { ErrorType } from "../tool/ErrorType.js";
 import { Grammar } from "../tool/Grammar.js";
 import { GrammarTransformPipeline } from "../tool/GrammarTransformPipeline.js";
@@ -72,7 +71,7 @@ export class LeftRecursiveRuleTransformer {
                     if (fitsPattern) {
                         leftRecursiveRuleNames.push(r.name);
                     } else { // better given an error that non-conforming left-recursion exists
-                        ErrorManager.get().grammarError(ErrorType.NONCONFORMING_LR_RULE, this.g.fileName,
+                        this.g.tool.errorManager.grammarError(ErrorType.NONCONFORMING_LR_RULE, this.g.fileName,
                             (r.ast.getChild(0) as GrammarAST).token!, r.name);
                     }
                 }
@@ -160,7 +159,7 @@ export class LeftRecursiveRuleTransformer {
         r.recPrimaryAlts = new Array<LeftRecursiveRuleAltInfo>();
         r.recPrimaryAlts.push(...leftRecursiveRuleWalker.prefixAndOtherAlts);
         if (r.recPrimaryAlts.length === 0) {
-            ErrorManager.get().grammarError(ErrorType.NO_NON_LR_ALTS, this.g.fileName,
+            this.g.tool.errorManager.grammarError(ErrorType.NO_NON_LR_ALTS, this.g.fileName,
                 (r.ast.getChild(0) as GrammarAST).token!, r.name);
         }
 
@@ -224,7 +223,7 @@ export class LeftRecursiveRuleTransformer {
 
             return ruleAST;
         } catch (e) {
-            ErrorManager.get().toolError(ErrorType.INTERNAL_ERROR, e, ruleStart,
+            this.g.tool.errorManager.toolError(ErrorType.INTERNAL_ERROR, e, ruleStart,
                 "error parsing rule created during left-recursion detection: " + ruleText);
         }
 

src/automata/ATNOptimizer.ts

@@ -10,10 +10,9 @@ import {
 } from "antlr4ng";
 
 import { CharSupport } from "../misc/CharSupport.js";
+import { Character } from "../support/Character.js";
 import { ErrorType } from "../tool/ErrorType.js";
 import { Grammar } from "../tool/Grammar.js";
-import { Character } from "../support/Character.js";
-import { ErrorManager } from "../tool/ErrorManager.js";
 
 export class ATNOptimizer {
     private constructor() {
@@ -91,9 +90,8 @@ export class ATNOptimizer {
                             for (let v = a; v <= b; v++) {
                                 if (matchSet.contains(v)) {
                                     // TODO: Token is missing (i.e. position in source is not displayed).
-                                    ErrorManager.get().grammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName,
-                                        null,
-                                        CharSupport.getANTLRCharLiteralForChar(v),
+                                    g.tool.errorManager.grammarError(ErrorType.CHARACTERS_COLLISION_IN_SET,
+                                        g.fileName, null, CharSupport.getANTLRCharLiteralForChar(v),
                                         CharSupport.getIntervalSetEscapedString(matchSet));
                                     break;
                                 }