TestActionTranslation done

Lots of fixes were needed for the first render test to succeed. And that's a pretty tough given, given that it's testing various action code.
mike-lischke · Nov 18, 2024 · ac69a79 · ac69a79
1 parent 3e110b0
commit ac69a79
Show file tree

Hide file tree

Showing 29 changed files with 240 additions and 167 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/src/analysis/LeftRecursiveRuleTransformer.ts b/src/analysis/LeftRecursiveRuleTransformer.ts
@@ -182,7 +182,7 @@ export class LeftRecursiveRuleTransformer {
         this.setAltASTPointers(r, t);
 
         // update Rule to just one alt and add prec alt
-        const arg = r.ast.getFirstChildWithType(ANTLRv4Parser.BEGIN_ARGUMENT) as ActionAST | null;
+        const arg = r.ast.getFirstChildWithType(ANTLRv4Parser.ARG_ACTION) as ActionAST | null;
         if (arg !== null) {
             r.args = ScopeParser.parseTypedArgList(arg, arg.getText(), this.g);
             r.args.type = DictType.Argument;

diff --git a/src/automata/LexerATNFactory.ts b/src/automata/LexerATNFactory.ts
@@ -58,6 +58,8 @@ export class LexerATNFactory extends ParserATNFactory {
         ["EOF", Lexer.EOF],
         //["MAX_CHAR_VALUE", Lexer.MAX_CHAR_VALUE], // TODO: are these constants needed?
         //["MIN_CHAR_VALUE", Lexer.MIN_CHAR_VALUE],
+        ["MAX_CHAR_VALUE", 0x1FFFF],
+        ["MIN_CHAR_VALUE", 0],
     ]);
 
     public static CharSetParseState = class CharSetParseState {
@@ -133,8 +135,8 @@ export class LexerATNFactory extends ParserATNFactory {
         this.codegenTemplates = codeGenerator.getTemplates();
     }
 
-    public static getCommonConstants(): MapIterator<string> {
-        return LexerATNFactory.COMMON_CONSTANTS.keys();
+    public static getCommonConstants(): Map<string, number> {
+        return LexerATNFactory.COMMON_CONSTANTS;
     }
 
     public override createATN(): ATN {

diff --git a/src/codegen/ActionTranslator.ts b/src/codegen/ActionTranslator.ts
@@ -132,8 +132,8 @@ export class ActionTranslator implements ActionSplitterListener {
         }
 
         const input = CharStream.fromString(action);
-        //input.setLine(tokenWithinAction.getLine());
-        //input.setCharPositionInLine(tokenWithinAction.getCharPositionInLine());
+        // TODO: input.setLine(tokenWithinAction.getLine());
+        // TODO: input.setCharPositionInLine(tokenWithinAction.getCharPositionInLine());
         const trigger = new ActionSplitter(input);
 
         // forces eval, triggers listener methods

diff --git a/src/codegen/OutputModelWalker.ts b/src/codegen/OutputModelWalker.ts
@@ -4,13 +4,14 @@
  * can be found in the LICENSE.txt file in the project root.
  */
 
+import { HashSet, OrderedHashSet } from "antlr4ng";
 import { ST, STGroup, type IST } from "stringtemplate4ts";
 
+import { isModelElement } from "../misc/ModelElement.js";
 import { Tool } from "../Tool.js";
 import { ErrorManager } from "../tool/ErrorManager.js";
 import { ErrorType } from "../tool/ErrorType.js";
 import { OutputModelObject } from "./model/OutputModelObject.js";
-import { isModelElement } from "../misc/ModelElement.js";
 
 /**
  * Convert an output model tree to template hierarchy by walking
@@ -92,7 +93,7 @@ export class OutputModelWalker {
                 const nestedST = this.walk(nestedOmo, header);
                 st.add(fieldName, nestedST);
             } else {
-                if (o instanceof Set || Array.isArray(o)) {
+                if (o instanceof Set || o instanceof HashSet || o instanceof OrderedHashSet || Array.isArray(o)) {
                     for (const nestedOmo of o) {
                         if (!nestedOmo) {
                             continue;

diff --git a/src/codegen/Target.ts b/src/codegen/Target.ts
@@ -31,7 +31,18 @@ import { RuleFunction } from "./model/RuleFunction.js";
 export type char = number;
 
 export abstract class Target {
-    protected static readonly defaultCharValueEscape: Map<char, string>;
+    protected static readonly defaultCharValueEscape = new Map<char, string>([
+        [0x08, "\\b"],
+        [0x09, "\\t"],
+        [0x0A, "\\n"],
+        [0x0C, "\\f"],
+        [0x0D, "\\r"],
+        [0x1B, "\\e"],
+        [0x22, "\\\""],
+        [0x27, "\\'"],
+        [0x5C, "\\\\"]
+    ]);
+
     private static readonly languageTemplates = new Map<string, STGroup>();
 
     public constructor(protected gen: CodeGenerator) {
@@ -598,7 +609,7 @@ export abstract class Target {
     }
 
     protected escapeChar(v: number): string {
-        return v.toString(16).padStart(4, "0");
+        return `\\u${v.toString(16).padStart(4, "0")}`;
     }
 
     /**

diff --git a/src/codegen/model/InvokeRule.ts b/src/codegen/model/InvokeRule.ts
@@ -58,7 +58,7 @@ export class InvokeRule extends RuleElement implements LabeledOp {
             rf.addContextDecl(ast.getAltLabel()!, decl);
         }
 
-        const arg = ast.getFirstChildWithType(ANTLRv4Parser.BEGIN_ARGUMENT) as ActionAST | null;
+        const arg = ast.getFirstChildWithType(ANTLRv4Parser.ARG_ACTION) as ActionAST | null;
         if (arg !== null) {
             this.argExprsChunks = ActionTranslator.translateAction(factory, rf, arg.token!, arg);
         }

diff --git a/src/codegen/model/SerializedATN.ts b/src/codegen/model/SerializedATN.ts
@@ -14,17 +14,17 @@ import { OutputModelObject } from "./OutputModelObject.js";
  *  except for java, which requires a 16-bit char encoding. See {@link SerializedJavaATN}.
  */
 export class SerializedATN extends OutputModelObject {
-    public serialized: number[];
+    #serialized: number[];
 
     public constructor(factory: OutputModelFactory, atn?: ATN) {
         super(factory);
 
         if (atn) {
-            this.serialized = ATNSerializer.getSerialized(atn);
+            this.#serialized = ATNSerializer.getSerialized(atn);
         }
     }
 
     public getSerialized(): object {
-        return this.serialized;
+        return this.#serialized;
     }
 }
diff --git a/src/codegen/model/SerializedJavaATN.ts b/src/codegen/model/SerializedJavaATN.ts
@@ -39,6 +39,10 @@ export class SerializedJavaATN extends SerializedATN {
         this.serializedAsString = this.segments[0]; // serializedAsString is valid if only one segment
     }
 
+    public get serialized(): string[] {
+        return this.serializedAsString;
+    };
+
     public override getSerialized(): object {
         return this.serializedAsString;
     }

diff --git a/src/codegen/model/chunk/SetAttr.ts b/src/codegen/model/chunk/SetAttr.ts
@@ -4,11 +4,13 @@
  * can be found in the LICENSE.txt file in the project root.
  */
 
+import { ModelElement } from "../../../misc/ModelElement.js";
 import { StructDecl } from "../decl/StructDecl.js";
 import { ActionChunk } from "./ActionChunk.js";
 import { SymbolRefChunk } from "./SymbolRefChunk.js";
 
 export class SetAttr extends SymbolRefChunk {
+    @ModelElement
     public rhsChunks: ActionChunk[];
 
     public constructor(ctx: StructDecl, name: string, escapedName: string, rhsChunks: ActionChunk[]) {

diff --git a/src/codegen/model/decl/StructDecl.ts b/src/codegen/model/decl/StructDecl.ts
@@ -4,6 +4,8 @@
  * can be found in the LICENSE.txt file in the project root.
  */
 
+import { HashSet, OrderedHashSet } from "antlr4ng";
+
 import { grammarOptions } from "../../../grammar-options.js";
 import { ModelElement } from "../../../misc/ModelElement.js";
 import type { IAttribute } from "../../../tool/IAttribute.js";
@@ -33,21 +35,21 @@ export class StructDecl extends Decl {
     // Track these separately; Go target needs to generate getters/setters
     // Do not make them templates; we only need the Decl object not the ST
     // built from it. Avoids adding args to StructDecl template
-    public tokenDecls = new Set<Decl>();
-    public tokenTypeDecls = new Set<Decl>();
-    public tokenListDecls = new Set<Decl>();
-    public ruleContextDecls = new Set<Decl>();
-    public ruleContextListDecls = new Set<Decl>();
-    public attributeDecls = new Set<Decl>();
+    public tokenDecls = new OrderedHashSet<Decl>();
+    public tokenTypeDecls = new OrderedHashSet<Decl>();
+    public tokenListDecls = new OrderedHashSet<Decl>();
+    public ruleContextDecls = new OrderedHashSet<Decl>();
+    public ruleContextListDecls = new OrderedHashSet<Decl>();
+    public attributeDecls = new HashSet<Decl>();
 
     @ModelElement
     public dispatchMethods: DispatchMethod[] = [];
 
     @ModelElement
-    public attrs = new Set<Decl>();
+    public attrs = new OrderedHashSet<Decl>();
 
     @ModelElement
-    public getters = new Set<Decl>();
+    public getters = new OrderedHashSet<Decl>();
 
     @ModelElement
     public ctorAttrs: AttributeDecl[] = [];
@@ -60,7 +62,7 @@ export class StructDecl extends Decl {
 
     // Used to generate method signatures in Go target interfaces
     @ModelElement
-    public signatures = new Set<Decl>();
+    public signatures = new HashSet<Decl>();
 
     public constructor(factory: OutputModelFactory, r: Rule, name?: string) {
         super(factory, name ?? factory.getGenerator()!.getTarget().getRuleFunctionContextStructName(r));

diff --git a/src/grammars/ANTLRv4Parser.g4 b/src/grammars/ANTLRv4Parser.g4
@@ -399,7 +399,7 @@ elementOptions
 
 elementOption
     : identifier
-    | identifier ASSIGN (identifier | STRING_LITERAL)
+    | identifier ASSIGN (identifier | STRING_LITERAL | INT)
     ;
 
 identifier

diff --git a/src/grammars/ActionSplitter.g4 b/src/grammars/ActionSplitter.g4
@@ -17,25 +17,27 @@ import type { ActionSplitterListener } from "../parse/ActionSplitterListener.js"
 }
 
 @members {
-private readonly attrValueExpr = /[^=][^;]*/;
-private readonly ws = /[ \t\n\r]+/;
-private readonly id = /[a-zA-Z_][a-zA-Z0-9_]*/;
-private readonly setNonLocalAttr = new RegExp(`\\$(?<x>${this.id.source})::(<y>${this.id.source})\s?=(?<expr>${this.attrValueExpr.source});`);
-private readonly nonLocalAttr = new RegExp(`\\$(?<x>${this.id.source})::(?<y>${this.id.source})`);
-private readonly qualifiedAttr = new RegExp(`\\$(?<x>${this.id.source}).(?<y>${this.id.source})`);
-private readonly setAttr = new RegExp(`\\$(?<x>${this.id.source})\s?=(?<expr>${this.attrValueExpr.source});`);
-private readonly attr = new RegExp(`\\$(?<x>${this.id.source})`);
+static readonly #attrValueExpr = /[^=][^;]*/;
+static readonly #ws = /[ \t\n\r]+/;
+static readonly #id = /[a-zA-Z_][a-zA-Z0-9_]*/;
+static readonly #setNonLocalAttr = new RegExp(`\\$(?<x>${this.#id.source})::(<y>${this.#id.source})\s?=(?<expr>${this.#attrValueExpr.source});`);
+static readonly #nonLocalAttr = new RegExp(`\\$(?<x>${this.#id.source})::(?<y>${this.#id.source})`);
+static readonly #qualifiedAttr = new RegExp(`\\$(?<x>${this.#id.source}).(?<y>${this.#id.source})`);
+static readonly #setAttr = new RegExp(`\\$(?<x>${this.#id.source})\s?=(?<expr>${this.#attrValueExpr.source});`);
+static readonly #attr = new RegExp(`\\$(?<x>${this.#id.source})`);
 
 /** Force filtering (and return tokens). Sends token values to the delegate. */
-    public getActionTokens(delegate: ActionSplitterListener): Token[] {
+public getActionTokens(delegate: ActionSplitterListener): Token[] {
     const tokens = this.getAllTokens();
     for (let i = 0; i < tokens.length; i++) {
         const t = tokens[i];
         switch (t.type) {
             case ActionSplitter.COMMENT:
             case ActionSplitter.LINE_COMMENT:
             case ActionSplitter.TEXT: {
-                delegate.text(t.text!);
+                // Replace $ with $.
+                const text = t.text!.replaceAll("\\$", "$");
+                delegate.text(text);
 
                 break;
             }
@@ -44,7 +46,7 @@ private readonly attr = new RegExp(`\\$(?<x>${this.id.source})`);
                 const text = t.text!;
 
                 // Parse the text and extract the named groups from the match result.
-                const match = text.match(this.setNonLocalAttr);
+                const match = text.match(ActionSplitter.#setNonLocalAttr);
                 if (match === null) {
                     throw new Error(`Mismatched input '${text}'`);
                 }
@@ -58,7 +60,7 @@ private readonly attr = new RegExp(`\\$(?<x>${this.id.source})`);
             case ActionSplitter.NONLOCAL_ATTR: {
                 const text = t.text!;
 
-                const match = text.match(this.nonLocalAttr);
+                const match = text.match(ActionSplitter.#nonLocalAttr);
                 if (match === null) {
                     throw new Error(`Mismatched input '${text}'`);
                 }
@@ -72,7 +74,7 @@ private readonly attr = new RegExp(`\\$(?<x>${this.id.source})`);
             case ActionSplitter.QUALIFIED_ATTR: {
                 let text = t.text!;
 
-                const match = text.match(this.qualifiedAttr);
+                const match = text.match(ActionSplitter.#qualifiedAttr);
                 if (match === null) {
                     throw new Error(`Mismatched input '${text}'`);
                 }
@@ -100,7 +102,7 @@ private readonly attr = new RegExp(`\\$(?<x>${this.id.source})`);
             case ActionSplitter.SET_ATTR: {
                 const text = t.text!;
 
-                const match = text.match(this.setAttr);
+                const match = text.match(ActionSplitter.#setAttr);
                 if (match === null) {
                     throw new Error(`Mismatched input '${text}'`);
                 }
@@ -114,7 +116,7 @@ private readonly attr = new RegExp(`\\$(?<x>${this.id.source})`);
             case ActionSplitter.ATTR: {
                 const text = t.text!;
 
-                const match = text.match(this.attr);
+                const match = text.match(ActionSplitter.#attr);
                 if (match === null) {
                     throw new Error(`Mismatched input '${text}'`);
                 }
@@ -154,7 +156,7 @@ ATTR: '$' ID;
 
 // Anything else is just random text
 
-TEXT: ( ~('\\' | '$') | '\\$' | '\\' ~('$') | {!this.isIDStartChar(this.inputStream.LA(2))}? '$')+;
+TEXT: (~'$' | '\\$' | '$' ~[a-zA-Z_])+;
 
 fragment ID: ('a' ..'z' | 'A' ..'Z' | '_') ('a' ..'z' | 'A' ..'Z' | '0' ..'9' | '_')*;