TestDollarParser done

- Implemented execParser in the test tool utils, which is need in TestDollarParser.
- TestDollarParser done.
- Added helper string templates for parser generation in tests.
- Updated the TypeScript string template in code gen (no underlines in template names).
- Added support for `define` ("-D")in command line parsing. That field is now also searched through when looking for an option.

src/grammar-options.ts

@@ -14,6 +14,8 @@ export interface IToolParameters {
     /** The grammar files. */
     args: string[];
 
+    define?: Record<string, string>,
+
     outputDirectory?: string,
     libDirectory?: string,
     generateATNDot?: boolean,
@@ -41,10 +43,13 @@ const parseBoolean = (value: string | null): boolean => {
     return lower === "true" || lower === "1" || lower === "on" || lower === "yes";
 };
 
-const parseKeyValuePair = (input: string): { key: string, value: string; } => {
+const defines: Record<string, string> = {};
+
+const parseKeyValuePair = (input: string): Record<string, string> => {
     const [key, value] = input.split("=");
+    defines[key] = value;
 
-    return { key, value };
+    return defines;
 };
 
 const prepared = new Command()
@@ -61,7 +66,7 @@ const prepared = new Command()
     .option<boolean>("-v, --visitor [boolean]", "Generate parse tree visitor.", parseBoolean, false)
     .option("-p, --package <name>", "Specify a package/namespace for the generated code.")
     .option<boolean>("-d, --dependencies [boolean]", "Generate file dependencies.", parseBoolean, false)
-    .option("-D, --define <key=value>", "Set/override a grammar-level option.", parseKeyValuePair)
+    .option("-D, --define <key=value...>", "Set/override a grammar-level option.", parseKeyValuePair)
     .option<boolean>("-w, --warnings-are-errors [boolean]", "Treat warnings as errors.", parseBoolean, false)
     .option<boolean>("-f, --force-atn [boolean]", "Use the ATN simulator for all predictions.", parseBoolean, false)
     .option<boolean>("--log [boolean]", "Dump lots of logging info to antlr-timestamp.log.", parseBoolean, false)

src/support/ParseTreeToASTConverter.ts

@@ -20,8 +20,7 @@ import {
     type LexerRuleSpecContext, type ModeSpecContext, type NotSetContext, type OptionsSpecContext,
     type ParserRuleSpecContext, type PredicateOptionsContext, type PrequelConstructContext, type RuleActionContext,
     type RuleBlockContext, type RulerefContext, type RulesContext, type RuleSpecContext, type SetElementContext,
-    type TerminalDefContext, type TokensSpecContext,
-    type WildcardContext
+    type TerminalDefContext, type TokensSpecContext, type WildcardContext
 } from "../generated/ANTLRv4Parser.js";
 
 import { Constants } from "../Constants1.js";

src/tool/Grammar.ts

@@ -669,7 +669,7 @@ export class Grammar implements IGrammar, AttributeResolver {
             qualifiedName += this.name;
         }
 
-        if (this.isCombined() || (this.isLexer())) {
+        if (this.isCombined() /*|| (this.isLexer())*/) {
             suffix = Grammar.getGrammarTypeToFileNameSuffix(this.type);
         }
 

src/tool/ast/GrammarRootAST.ts

@@ -93,10 +93,17 @@ export class GrammarRootAST extends GrammarASTWithOptions implements IGrammarRoo
     }
 
     public override getOptionString(key: string): string | undefined {
+        // Standard options.
         if (typeof grammarOptions[key] === "string") {
             return grammarOptions[key];
         }
 
+        // Defines.
+        const define = grammarOptions.define?.[key];
+        if (define !== undefined) {
+            return define;
+        }
+
         return super.getOptionString(key);
     }
 

templates/codegen/TypeScript/TypeScript.stg

@@ -642,25 +642,25 @@ SetAttr(s,rhsChunks) ::= "<ctx(s)>!.<s.escapedName> = <rhsChunks>;"
 TokenLabelType()  ::= "<file.TokenLabelType; null={Token}>"
 InputSymbolType() ::= "<file.InputSymbolType; null={Token}>"
 
-TokenPropertyRef_text(t)    ::= "(<ctx(t)>._<t.label>?.text ?? '')"
-TokenPropertyRef_type(t)    ::= "(<ctx(t)>._<t.label>?.type ?? 0)"
-TokenPropertyRef_line(t)    ::= "(<ctx(t)>._<t.label>?.line ?? 0)"
-TokenPropertyRef_pos(t)     ::= "(<ctx(t)>._<t.label>?.column ?? 0)"
-TokenPropertyRef_channel(t) ::= "(<ctx(t)>._<t.label>?.channel ?? 0)"
-TokenPropertyRef_index(t)   ::= "(<ctx(t)>._<t.label>?.tokenIndex ?? 0)"
-TokenPropertyRef_int(t)     ::= "Number(<ctx(t)>._<t.label>?.text ?? '0')"
-
-RulePropertyRef_start(r)  ::= "(<ctx(r)>._<r.label>?.start)"
-RulePropertyRef_stop(r)   ::= "(<ctx(r)>._<r.label>?.stop)"
-RulePropertyRef_text(r)   ::= "(<ctx(r)>._<r.label> != null ? this.tokenStream.getTextFromRange(<ctx(r)>._<r.label>.start, <ctx(r)>._<r.label>.stop) : '')"
-RulePropertyRef_ctx(r)    ::= "<ctx(r)>._<r.label>"
-RulePropertyRef_parser(r) ::= "this"
-
-ThisRulePropertyRef_start(r)  ::= "localContext.start"
-ThisRulePropertyRef_stop(r)   ::= "localContext.stop"
-ThisRulePropertyRef_text(r)   ::= "this.tokenStream.getTextFromRange(localContext.start, this.tokenStream.LT(-1))"
-ThisRulePropertyRef_ctx(r)    ::= "localContext"
-ThisRulePropertyRef_parser(r) ::= "this"
+TokenPropertyRefText(t)    ::= "(<ctx(t)>._<t.label>?.text ?? '')"
+TokenPropertyRefType(t)    ::= "(<ctx(t)>._<t.label>?.type ?? 0)"
+TokenPropertyRefLine(t)    ::= "(<ctx(t)>._<t.label>?.line ?? 0)"
+TokenPropertyRefPos(t)     ::= "(<ctx(t)>._<t.label>?.column ?? 0)"
+TokenPropertyRefChannel(t) ::= "(<ctx(t)>._<t.label>?.channel ?? 0)"
+TokenPropertyRefIndex(t)   ::= "(<ctx(t)>._<t.label>?.tokenIndex ?? 0)"
+TokenPropertyRefInt(t)     ::= "Number(<ctx(t)>._<t.label>?.text ?? '0')"
+
+RulePropertyRefStart(r)  ::= "(<ctx(r)>._<r.label>?.start)"
+RulePropertyRefStop(r)   ::= "(<ctx(r)>._<r.label>?.stop)"
+RulePropertyRefText(r)   ::= "(<ctx(r)>._<r.label> != null ? this.tokenStream.getTextFromRange(<ctx(r)>._<r.label>.start, <ctx(r)>._<r.label>.stop) : '')"
+RulePropertyRefCtx(r)    ::= "<ctx(r)>._<r.label>"
+RulePropertyRefParser(r) ::= "this"
+
+ThisRulePropertyRefStart(r)  ::= "localContext.start"
+ThisRulePropertyRefStop(r)   ::= "localContext.stop"
+ThisRulePropertyRefText(r)   ::= "this.tokenStream.getTextFromRange(localContext.start, this.tokenStream.LT(-1))"
+ThisRulePropertyRefCtx(r)    ::= "localContext"
+ThisRulePropertyRefParser(r) ::= "this"
 
 NonLocalAttrRef(s)            ::= "(this.getInvokingContext(<s.ruleIndex>) as <s.ruleName; format=\"cap\">Context).<s.escapedName>"
 SetNonLocalAttr(s, rhsChunks) ::=

tests/TestCommonTokenStream.spec.ts

@@ -179,38 +179,38 @@ describe("TestCommonTokenStream", () => {
 
         const tokens = new CommonTokenStream(lexer);
         tokens.fill();
-        expect(tokens.getHiddenTokensToLeft(0)).toBeUndefined();
-        expect(tokens.getHiddenTokensToRight(0));
+        expect(tokens.getHiddenTokensToLeft(0, -1)).toBeUndefined();
+        expect(tokens.getHiddenTokensToRight(0, -1));
 
-        expect(tokens.getHiddenTokensToLeft(1)!.join(", ")).toBe("[@0,0:0=' ',<1>,channel=1,0:-1]");
-        expect(tokens.getHiddenTokensToRight(1)!.join(", ")).toBe("[@2,0:0=' ',<1>,channel=1,0:-1]");
+        expect(tokens.getHiddenTokensToLeft(1, -1)!.join(", ")).toBe("[@0,0:0=' ',<1>,channel=1,0:-1]");
+        expect(tokens.getHiddenTokensToRight(1, -1)!.join(", ")).toBe("[@2,0:0=' ',<1>,channel=1,0:-1]");
 
-        expect(tokens.getHiddenTokensToLeft(2)).toBeUndefined();
-        expect(tokens.getHiddenTokensToRight(2)).toBeUndefined();
+        expect(tokens.getHiddenTokensToLeft(2, -1)).toBeUndefined();
+        expect(tokens.getHiddenTokensToRight(2, -1)).toBeUndefined();
 
-        expect(tokens.getHiddenTokensToLeft(3)!.join(", ")).toBe("[@2,0:0=' ',<1>,channel=1,0:-1]");
-        expect(tokens.getHiddenTokensToRight(3)).toBeUndefined();
+        expect(tokens.getHiddenTokensToLeft(3, -1)!.join(", ")).toBe("[@2,0:0=' ',<1>,channel=1,0:-1]");
+        expect(tokens.getHiddenTokensToRight(3, -1)).toBeUndefined();
 
-        expect(tokens.getHiddenTokensToLeft(4)).toBeUndefined();
-        expect(tokens.getHiddenTokensToRight(4)!.join(", ")).toBe(
+        expect(tokens.getHiddenTokensToLeft(4, -1)).toBeUndefined();
+        expect(tokens.getHiddenTokensToRight(4, -1)!.join(", ")).toBe(
             "[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]");
 
-        expect(tokens.getHiddenTokensToLeft(5)).toBeUndefined();
-        expect(tokens.getHiddenTokensToRight(5)!.join(", ")).toBe("[@6,0:0=' ',<1>,channel=1,0:-1]");
+        expect(tokens.getHiddenTokensToLeft(5, -1)).toBeUndefined();
+        expect(tokens.getHiddenTokensToRight(5, -1)!.join(", ")).toBe("[@6,0:0=' ',<1>,channel=1,0:-1]");
 
-        expect(tokens.getHiddenTokensToLeft(6)!.join(", ")).toBe("[@5,0:0=' ',<1>,channel=1,0:-1]");
-        expect(tokens.getHiddenTokensToRight(6)).toBeUndefined();
+        expect(tokens.getHiddenTokensToLeft(6, -1)!.join(", ")).toBe("[@5,0:0=' ',<1>,channel=1,0:-1]");
+        expect(tokens.getHiddenTokensToRight(6, -1)).toBeUndefined();
 
-        expect(tokens.getHiddenTokensToLeft(7)!.join(", ")).toBe(
+        expect(tokens.getHiddenTokensToLeft(7, -1)!.join(", ")).toBe(
             "[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]");
-        expect(tokens.getHiddenTokensToRight(7)!.join(", ")).toBe(
+        expect(tokens.getHiddenTokensToRight(7, -1)!.join(", ")).toBe(
             "[@8,0:0=' ',<1>,channel=1,0:-1], [@9,0:0='\\n',<1>,channel=1,0:-1]",);
 
-        expect(tokens.getHiddenTokensToLeft(8)).toBeUndefined();
-        expect(tokens.getHiddenTokensToRight(8)!.join(", ")).toBe("[@9,0:0='\\n',<1>,channel=1,0:-1]");
+        expect(tokens.getHiddenTokensToLeft(8, -1)).toBeUndefined();
+        expect(tokens.getHiddenTokensToRight(8, -1)!.join(", ")).toBe("[@9,0:0='\\n',<1>,channel=1,0:-1]");
 
-        expect(tokens.getHiddenTokensToLeft(9)!.join(", ")).toBe("[@8,0:0=' ',<1>,channel=1,0:-1]");
-        expect(tokens.getHiddenTokensToRight(9)).toBeUndefined();
+        expect(tokens.getHiddenTokensToLeft(9, -1)!.join(", ")).toBe("[@8,0:0=' ',<1>,channel=1,0:-1]");
+        expect(tokens.getHiddenTokensToRight(9, -1)).toBeUndefined();
     });
 
     it("testSingleEOF", (): void => {

tests/TestDollarParser.spec.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) Mike Lischke. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ */
+
+import { describe, expect, it } from "vitest";
+
+import { ToolTestUtils } from "./ToolTestUtils.js";
+
+describe("TestDollarParser", () => {
+    it("testSimpleCall", async () => {
+        const orgLog = console.log;
+        let output = "";
+        console.log = (str) => {
+            output += str;
+        };
+
+        try {
+            const grammar = "grammar T;\n" +
+                "a : ID  { console.log($parser.getSourceName()); }\n" +
+                "  ;\n" +
+                "ID : 'a'..'z'+ ;\n";
+
+            const errors = await ToolTestUtils.execParser("T.g4", grammar, "TParser", "TLexer", "a", "x", true);
+            expect(output.includes("input")).toBe(true);
+            expect(errors).toHaveLength(0);
+        } finally {
+            console.log = orgLog;
+        }
+    });
+});