Fixed grammar creation and started working on the pipelines

- Lots of fine tuning and fixing. - Many detail changes in the tree walkers (got a good part fixed). - Template loading. - Unified all token values (types) into one list to make tree walkers working properly. - Removed the need for DFA prediction in tree walkers (except in one spot).
mike-lischke · Nov 10, 2024 · 7a94e99 · 7a94e99
1 parent f0cf56f
commit 7a94e99
Show file tree

Hide file tree

Showing 59 changed files with 2,060 additions and 3,107 deletions.
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -9,7 +9,10 @@
                 "$tsc-watch"
             ],
             "group": "build",
-            "label": "tsc: watch"
+            "label": "tsc: watch",
+            "runOptions": {
+                "runOn": "folderOpen"
+            }
         },
         {
             "label": "ESLint",

diff --git a/cli/Interpreter.ts b/cli/Interpreter.ts
@@ -4,21 +4,17 @@
  * can be found in the LICENSE.txt file in the project root.
  */
 
-/* eslint-disable jsdoc/require-param, jsdoc/require-returns */
-
-
-import { readFile } from "fs/promises";
-import { createWriteStream } from "fs";
 import { Option, program } from "commander";
+import { createWriteStream } from "fs";
+import { readFile } from "fs/promises";
 
 import { CharStream, CommonToken, CommonTokenStream, DecisionInfo, ParseInfo } from "antlr4ng";
 
-import { Tool } from "../Tool.js";
-import { ANTLRToolListener } from "../tool/ANTLRToolListener.js";
-import { DefaultToolListener } from "../tool/DefaultToolListener.js";
-import { Grammar } from "../tool/Grammar.js";
-import { GrammarParserInterpreter } from "../tool/GrammarParserInterpreter.js";
-import { LexerGrammar } from "../tool/LexerGrammar.js";
+import type { ANTLRToolListener } from "../src/tool/ANTLRToolListener.js";
+import { DefaultToolListener } from "../src/tool/DefaultToolListener.js";
+import { Grammar } from "../src/tool/Grammar.js";
+import type { GrammarParserInterpreter } from "../src/tool/GrammarParserInterpreter.js";
+import { LexerGrammar } from "../src/tool/LexerGrammar.js";
 import { encodings, parseBoolean } from "./cli-options.js";
 
 /** CLI parameters for the interpreter tool. */
@@ -111,7 +107,7 @@ export class Interpreter {
 
         let g: Grammar;
         let lg = null;
-        const listener = new DefaultToolListener(new Tool());
+        const listener = new DefaultToolListener();
         if (interpreterOptions.grammarFileName) {
             const grammarContent = await readFile(interpreterOptions.grammarFileName, "utf8");
             g = new Interpreter.IgnoreTokenVocabGrammar(interpreterOptions.grammarFileName, grammarContent, undefined,

diff --git a/src/constants.ts → src/Constants1.ts b/src/constants.ts → src/Constants1.ts
@@ -19,18 +19,16 @@ export class Constants {
     public static readonly PRECEDENCE_OPTION_NAME = "p";
     public static readonly TOKENINDEX_OPTION_NAME = "tokenIndex";
 
-    public static readonly TEMPLATE_ROOT = "org/antlr/v4/tool/templates/codegen";
     public static readonly VOCAB_FILE_EXTENSION = ".tokens";
 
     public static readonly GRAMMAR_FROM_STRING_NAME = "<string>";
 
-    public static readonly EOR_TOKEN_TYPE: number = 1;
-    public static readonly DOWN = -2;
-    public static readonly UP = -3;
+    public static readonly EOR_TOKEN_TYPE = 1;
+    public static readonly DOWN = 2;
+    public static readonly UP = 3;
 
     public static readonly MEMO_RULE_FAILED = -2;
     public static readonly MEMO_RULE_UNKNOWN = -1;
-    public static readonly INITIAL_FOLLOW_STACK_SIZE = 100;
 
     // copies from Token object for convenience in actions
     public static readonly DEFAULT_TOKEN_CHANNEL = Token.DEFAULT_CHANNEL;

diff --git a/src/Tool.ts b/src/Tool.ts
@@ -159,12 +159,11 @@ export class Tool implements ITool {
         const transform = new GrammarTransformPipeline(g, this);
         transform.process();
 
-        let lexerGrammar: LexerGrammar;
         let lexerAST: GrammarRootAST | undefined;
         if (g.ast.grammarType === GrammarType.Combined) {
             lexerAST = transform.extractImplicitLexer(g); // alters g.ast
             if (lexerAST) {
-                lexerGrammar = ClassFactory.createLexerGrammar(this, lexerAST);
+                const lexerGrammar = ClassFactory.createLexerGrammar(this, lexerAST);
                 lexerGrammar.fileName = g.fileName;
                 lexerGrammar.originalGrammar = g;
                 g.implicitLexer = lexerGrammar;
@@ -173,7 +172,9 @@ export class Tool implements ITool {
             }
         }
 
-        g.importVocab(g.implicitLexer);
+        if (g.implicitLexer) {
+            g.importVocab(g.implicitLexer);
+        }
 
         this.processNonCombinedGrammar(g, genCode);
     }
@@ -194,7 +195,7 @@ export class Tool implements ITool {
             return;
         }
 
-        const codeGenerator = new CodeGenerator(this, g, g.getLanguage());
+        const codeGenerator = new CodeGenerator(this, g, g.getLanguage() ?? "Java");
 
         // BUILD ATN FROM AST
         let factory: IATNFactory;
@@ -243,9 +244,10 @@ export class Tool implements ITool {
      */
     public checkForRuleIssues(g: Grammar): boolean {
         // check for redefined rules
-        const rules: GrammarAST[] = [];
+        const rulesNode = g.ast.getFirstChildWithType(ANTLRv4Parser.RULES) as GrammarAST;
+        const rules: GrammarAST[] = [...rulesNode.getAllChildrenWithType(ANTLRv4Parser.RULE)];
         for (const mode of g.ast.getAllChildrenWithType(ANTLRv4Parser.MODE)) {
-            rules.push(...mode.getAllChildrenWithType(ANTLRv4Parser.RULE_REF));
+            rules.push(...mode.getAllChildrenWithType(ANTLRv4Parser.RULE));
         }
 
         let redefinition = false;
@@ -265,7 +267,7 @@ export class Tool implements ITool {
             ruleToAST.set(ruleName, ruleAST);
         }
 
-        const chk = new UndefChecker(g.isLexer());
+        const chk = new UndefChecker(g.isLexer(), ruleToAST);
         chk.visitGrammar(g.ast);
 
         return redefinition; //  || chk.errors;

diff --git a/src/UndefChecker.ts b/src/UndefChecker.ts
@@ -4,20 +4,19 @@
  * can be found in the LICENSE.txt file in the project root.
  */
 
-import { GrammarTreeVisitor } from "./tree-walkers/GrammarTreeVisitor.js";
-import type { ErrorManager } from "./tool/ErrorManager.js";
+import { Character } from "./support/Character.js";
+import { ErrorManager } from "./tool/ErrorManager.js";
 import { ErrorType } from "./tool/ErrorType.js";
 import type { ActionAST } from "./tool/ast/ActionAST.js";
 import type { GrammarAST } from "./tool/ast/GrammarAST.js";
 import type { RuleAST } from "./tool/ast/RuleAST.js";
 import type { TerminalAST } from "./tool/ast/TerminalAST.js";
+import { GrammarTreeVisitor } from "./tree-walkers/GrammarTreeVisitor.js";
 
 export class UndefChecker extends GrammarTreeVisitor {
     public badRef = false;
-    private errMgr: ErrorManager;
-    private ruleToAST: Map<string, RuleAST>;
 
-    public constructor(private isLexer: boolean) {
+    public constructor(private isLexer: boolean, private ruleToAST: Map<string, RuleAST>) {
         super();
     }
 
@@ -33,20 +32,18 @@ export class UndefChecker extends GrammarTreeVisitor {
     }
 
     public override ruleRef(ref: GrammarAST, arg: ActionAST | null): void {
-        const ruleAST: RuleAST | undefined = this.ruleToAST.get(ref.getText()!);
-        const fileName = ref.getToken()?.inputStream?.getSourceName() ?? "<unknown>";
-        if (this.currentRuleName?.charAt(0).toUpperCase() === this.currentRuleName?.charAt(0) &&
-            ref.getText()?.charAt(0).toLowerCase() === ref.getText()?.charAt(0)) {
+        const ruleAST: RuleAST | undefined = this.ruleToAST.get(ref.getText());
+        const fileName = ref.token?.inputStream?.getSourceName() ?? "<unknown>";
+
+        if (Character.isUpperCase(this.currentRuleName!.codePointAt(0)!) &&
+            Character.isLowerCase(ref.getText().codePointAt(0)!)) {
             this.badRef = true;
-            this.errMgr.grammarError(ErrorType.PARSER_RULE_REF_IN_LEXER_RULE,
-                fileName, ref.getToken(), ref.getText(), this.currentRuleName);
+            ErrorManager.get().grammarError(ErrorType.PARSER_RULE_REF_IN_LEXER_RULE,
+                fileName, ref.token!, ref.getText(), this.currentRuleName);
         } else if (!ruleAST) {
             this.badRef = true;
-            this.errMgr.grammarError(ErrorType.UNDEFINED_RULE_REF, fileName, ref.token, ref.getText());
+            ErrorManager.get().grammarError(ErrorType.UNDEFINED_RULE_REF, fileName, ref.token!, ref.getText());
         }
     }
 
-    public override getErrorManager(): ErrorManager {
-        return this.errMgr;
-    }
 }
diff --git a/src/analysis/AnalysisPipeline.ts b/src/analysis/AnalysisPipeline.ts
@@ -65,7 +65,7 @@ export class AnalysisPipeline {
             const look = analyzer.look(this.g.atn, this.g.atn.ruleToStartState[rule.index]!, undefined);
             if (look.contains(Token.EPSILON)) {
                 ErrorManager.get().grammarError(ErrorType.EPSILON_TOKEN, this.g.fileName,
-                    (rule.ast.getChild(0) as GrammarAST).getToken(), rule.name);
+                    (rule.ast.getChild(0) as GrammarAST).token!, rule.name);
             }
         }
     }