Skip to content

Commit

Permalink
The first 3 tests of the TestActionTranslation spec succeed
Browse files Browse the repository at this point in the history
  • Loading branch information
@mike-lischke
mike-lischke committed Nov 16, 2024
1 parent e6869b5 commit 3e110b0
Show file tree
Hide file tree
Showing 10 changed files with 446 additions and 457 deletions.
792 changes: 385 additions & 407 deletions package-lock.json
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
],
"license": "MIT",
"dependencies": {
"antlr4ng": "3.0.7",
"antlr4ng": "3.0.8",
"commander": "12.1.0",
"fast-printf": "1.6.9",
"stringtemplate4ts": "1.0.2",
"stringtemplate4ts": "1.0.3",
"unicode-properties": "1.4.1"
},
"devDependencies": {
Expand Down
10 changes: 4 additions & 6 deletions src/codegen/OutputModelWalker.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ export class OutputModelWalker {
continue;
}

console.log(`${omo.constructor.name}.${fieldName}`);
//console.log(`${omo.constructor.name}.${fieldName}`);
if (usedFieldNames.has(fieldName)) {
ErrorManager.get().toolError(ErrorType.INTERNAL_ERROR, "Model object " + omo.constructor.name +
" has multiple fields named '" + fieldName + "'");
Expand All @@ -86,17 +86,15 @@ export class OutputModelWalker {
continue;
}

//const o = fi.get(omo);
const o = omo[fieldName];
if (o instanceof OutputModelObject) { // SINGLE MODEL OBJECT?
const nestedOmo = o;
const nestedST = this.walk(nestedOmo, header);
st.add(fieldName, nestedST);
} else {
if (Array.isArray(o)) {
const nestedObjects = o as unknown[];
for (const nestedOmo of nestedObjects) {
if (nestedOmo === null) {
if (o instanceof Set || Array.isArray(o)) {
for (const nestedOmo of o) {
if (!nestedOmo) {
continue;
}

Expand Down
2 changes: 1 addition & 1 deletion src/codegen/model/OutputFile.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export abstract class OutputFile extends OutputModelObject {
const namedActions = new Map<string, Action>();
for (const name of g.namedActions.keys()) {
const ast = g.namedActions.get(name)!;
if (filter?.(ast)) {
if (!filter || filter(ast)) {
namedActions.set(name, new Action(this.factory!, ast));
}
}
Expand Down
20 changes: 15 additions & 5 deletions src/codegen/model/RuleFunction.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { ANTLRv4Lexer } from "../../generated/ANTLRv4Lexer.js";
import type { CommonTree } from "../../tree/CommonTree.js";

import { FrequencySet } from "../../misc/FrequencySet.js";
import { ModelElement } from "../../misc/ModelElement.js";
import { GrammarASTAdaptor } from "../../parse/GrammarASTAdaptor.js";
import { ErrorManager } from "../../tool/ErrorManager.js";
import { ErrorType } from "../../tool/ErrorType.js";
Expand Down Expand Up @@ -52,24 +53,33 @@ export class RuleFunction extends OutputModelObject {
public readonly altToContext: AltLabelStructDecl[];
public hasLookaheadBlock: boolean;

@ModelElement
public code: SrcOp[];

@ModelElement
public locals = new Set<Decl>(); // TODO: move into ctx?

@ModelElement
public args?: AttributeDecl[];

@ModelElement
public ruleCtx: StructDecl;

@ModelElement
public altLabelCtxs?: Map<string, AltLabelStructDecl>;

@ModelElement
public namedActions: Map<string, Action>;

@ModelElement
public finallyAction: Action;

@ModelElement
public exceptions: ExceptionClause[];

@ModelElement
public postamble: SrcOp[];

private locals = new Set<Decl>(); // TODO: move into ctx?

public constructor(factory: OutputModelFactory, r: Rule) {
super(factory);
this.name = r.name;
Expand Down Expand Up @@ -230,7 +240,7 @@ export class RuleFunction extends OutputModelObject {
public getDeclForAltElement(t: GrammarAST, refLabelName: string, needList: boolean, optional: boolean): Decl[] {
const decls = new Array<Decl>();
if (t.getType() === ANTLRv4Lexer.RULE_REF) {
const ruleRef = this.factory!.getGrammar()!.getRule(t.getText()!)!;
const ruleRef = this.factory!.getGrammar()!.getRule(t.getText())!;
const ctxName = this.factory!.getGenerator()!.getTarget().getRuleFunctionContextStructName(ruleRef);
if (needList) {
if (this.factory!.getGenerator()!.getTarget().supportsOverloadedMethods()) {
Expand Down Expand Up @@ -326,7 +336,7 @@ export class RuleFunction extends OutputModelObject {
}

private getName(token: GrammarAST): string | null {
const tokenText = token.getText()!;
const tokenText = token.getText();
const tokenName = token.getType() !== ANTLRv4Lexer.STRING_LITERAL
? tokenText
: token.g.getTokenName(Number.parseInt(tokenText));
Expand All @@ -338,7 +348,7 @@ export class RuleFunction extends OutputModelObject {
private nodesToStrings<T extends GrammarAST>(nodes: T[]): string[] {
const a = new Array<string>();
for (const t of nodes) {
a.push(t.getText()!);
a.push(t.getText());
}

return a;
Expand Down
5 changes: 3 additions & 2 deletions src/codegen/model/decl/StructDecl.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,6 @@ export class StructDecl extends Decl {
public derivedFromName: string; // rule name or label name
public provideCopyFrom: boolean;

public dispatchMethods: DispatchMethod[] = [];

// Track these separately; Go target needs to generate getters/setters
// Do not make them templates; we only need the Decl object not the ST
// built from it. Avoids adding args to StructDecl template
Expand All @@ -42,6 +40,9 @@ export class StructDecl extends Decl {
public ruleContextListDecls = new Set<Decl>();
public attributeDecls = new Set<Decl>();

@ModelElement
public dispatchMethods: DispatchMethod[] = [];

@ModelElement
public attrs = new Set<Decl>();

Expand Down
2 changes: 1 addition & 1 deletion src/support/ParseTreeToASTConverter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -809,7 +809,7 @@ export class ParseTreeToASTConverter {
}

private static createASTNode(astType: number, context: ParserRuleContext | TerminalNode): GrammarAST {
const token = this.createToken(astType, context);
const token = this.createToken(astType, context, context.getText());

// The token determines the payload of the AST node as well as the start token index.
const ast = new GrammarAST(token);
Expand Down
18 changes: 13 additions & 5 deletions src/tool/Grammar.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -850,8 +850,10 @@ export class Grammar implements IGrammar, AttributeResolver {
* @see #getTokenDisplayName
* @returns The display names of all tokens defined in the grammar.
*/
public getTokenDisplayNames(): string[] {
const tokenNames: string[] = [];
public getTokenDisplayNames(): Array<string | null> {
const numTokens = this.getMaxTokenType();
const tokenNames = new Array<string | null>(numTokens + 1);
tokenNames.fill(null);
for (let i = 0; i < tokenNames.length; i++) {
tokenNames[i] = this.getTokenDisplayName(i);
}
Expand All @@ -863,7 +865,10 @@ export class Grammar implements IGrammar, AttributeResolver {
* Gets the literal names assigned to tokens in the grammar.
*/
public getTokenLiteralNames(): Array<string | null> {
const literalNames: Array<string | null> = [];
const numTokens = this.getMaxTokenType();
const literalNames = new Array<string | null>(numTokens + 1);
literalNames.fill(null);

for (let i = 0; i < Math.min(literalNames.length, this.typeToStringLiteralList.length); i++) {
literalNames[i] = this.typeToStringLiteralList[i];
}
Expand All @@ -880,8 +885,11 @@ export class Grammar implements IGrammar, AttributeResolver {
/**
* Gets the symbolic names assigned to tokens in the grammar.
*/
public getTokenSymbolicNames(): string[] {
const symbolicNames: string[] = [];
public getTokenSymbolicNames(): Array<string | null> {
const numTokens = this.getMaxTokenType();
const symbolicNames = new Array<string | null>(numTokens + 1);
symbolicNames.fill(null);

for (let i = 0; i < Math.min(symbolicNames.length, this.typeToTokenList.length); i++) {
const name = this.typeToTokenList[i];
if (!name || name.startsWith(Grammar.AUTO_GENERATED_TOKEN_NAME_PREFIX)) {
Expand Down
2 changes: 1 addition & 1 deletion templates/codegen/Java/Java.stg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ public class <file.grammarName>BaseVisitor\<T> extends AbstractParseTreeVisitor\
>>

fileHeader(grammarFileName, ANTLRVersion) ::= <<
// Generated from <grammarFileName> by ANTLR <ANTLRVersion>
// Generated from <grammarFileName> by ANTLRng <ANTLRVersion>
>>

Parser(parser, funcs, atn, sempredFuncs, superClass) ::= <<
Expand Down
48 changes: 21 additions & 27 deletions tests/TestActionTranslation.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,11 @@ import "../src/Tool.js"; // To kick off the loading of the tool
import { AnalysisPipeline } from "../src/analysis/AnalysisPipeline.js";
import { LexerATNFactory } from "../src/automata/LexerATNFactory.js";
import { ParserATNFactory } from "../src/automata/ParserATNFactory.js";
import { CodeGenerator } from "../src/codegen/CodeGenerator.js";
import { SemanticPipeline } from "../src/semantics/SemanticPipeline.js";
import { Grammar } from "../src/tool/Grammar.js";
import type { LexerGrammar } from "../src/tool/LexerGrammar.js";
import { ErrorQueue } from "./support/ErrorQueue.js";
import { CodeGenerator } from "../src/codegen/CodeGenerator.js";
import { ErrorManager } from "../src/tool/ErrorManager.js";

describe("TestActionTranslation", () => {
const attributeTemplate =
Expand Down Expand Up @@ -67,31 +66,26 @@ describe("TestActionTranslation", () => {
const outputFileST = gen.generateParser(false);
const output = outputFileST.render();

const errorCount = ErrorManager.get().errors;
if (errorCount > 0) {
console.log(errorQueue.toString(true));
}

const b = "#" + actionName + "#";
const start = output.indexOf(b);
const e = "#end-" + actionName + "#";
const end = output.indexOf(e);
const snippet = output.substring(start + b.length, end);
//expect(snippet).toEqual(expected);
expect(snippet).toEqual(expected);
}
};

it.only("testEscapedLessThanInAction", (): void => {
it("testEscapedLessThanInAction", (): void => {
const action = "i<3; '<xmltag>'";
const expected = "i<3; '<xmltag>'";
testActions(attributeTemplate, "members", action, expected);
//testActions(attributeTemplate, "init", action, expected);
//testActions(attributeTemplate, "inline", action, expected);
//testActions(attributeTemplate, "finally", action, expected);
//testActions(attributeTemplate, "inline2", action, expected);
testActions(attributeTemplate, "init", action, expected);
testActions(attributeTemplate, "inline", action, expected);
testActions(attributeTemplate, "finally", action, expected);
testActions(attributeTemplate, "inline2", action, expected);
});

it("testEscapedInAction", (): void => {
it.skip("testEscapedInAction", (): void => {
const action = "int \\$n; \"\\$in string\\$\"";
const expected = "int $n; \"$in string$\"";
testActions(attributeTemplate, "members", action, expected);
Expand All @@ -105,7 +99,7 @@ describe("TestActionTranslation", () => {
* Regression test for "in antlr v4 lexer, $ translation issue in action".
* https://github.com/antlr/antlr4/issues/176
*/
it("testUnescapedInAction", (): void => {
it.skip("testUnescapedInAction", (): void => {
const action = "\\$string$";
const expected = "$string$";
testActions(attributeTemplate, "members", action, expected);
Expand Down Expand Up @@ -137,7 +131,7 @@ describe("TestActionTranslation", () => {
testActions(attributeTemplate, "inline2", action, expected);
});

it("testComplicatedArgParsingWithTranslation", (): void => {
it.skip("testComplicatedArgParsingWithTranslation", (): void => {
const action = "x, $ID.text+\"3242\", (*$ID).foo(21,33), 3.2+1, '\\n', " +
"\"a,oo\\nick\", {bl, \"fdkj\"eck}";
const expected =
Expand All @@ -146,51 +140,51 @@ describe("TestActionTranslation", () => {
testActions(attributeTemplate, "inline", action, expected);
});

it("testArguments", (): void => {
it.skip("testArguments", (): void => {
const action = "$x; $ctx.x";
const expected = "_localctx.x; _localctx.x";
testActions(attributeTemplate, "inline", action, expected);
});

it("testReturnValue", (): void => {
it.skip("testReturnValue", (): void => {
const action = "$y; $ctx.y";
const expected = "_localctx.y; _localctx.y";
testActions(attributeTemplate, "inline", action, expected);
});

it("testReturnValueWithNumber", (): void => {
it.skip("testReturnValueWithNumber", (): void => {
const action = "$ctx.x1";
const expected = "_localctx.x1";
testActions(attributeTemplate, "inline", action, expected);
});

it("testReturnValuesCurrentRule", (): void => {
it.skip("testReturnValuesCurrentRule", (): void => {
const action = "$y; $ctx.y;";
const expected = "_localctx.y; _localctx.y;";
testActions(attributeTemplate, "inline", action, expected);
});

it("testReturnValues", (): void => {
it.skip("testReturnValues", (): void => {
const action = "$lab.e; $b.e; $y.e = \"\";";
const expected = "((AContext)_localctx).lab.e; ((AContext)_localctx).b.e; _localctx.y.e = \"\";";
testActions(attributeTemplate, "inline", action, expected);
});

it("testReturnWithMultipleRuleRefs", (): void => {
it.skip("testReturnWithMultipleRuleRefs", (): void => {
const action = "$c.x; $c.y;";
const expected = "((AContext)_localctx).c.x; ((AContext)_localctx).c.y;";
testActions(attributeTemplate, "inline", action, expected);
});

it("testTokenRefs", (): void => {
it.skip("testTokenRefs", (): void => {
const action = "$id; $ID; $id.text; $id.getText(); $id.line;";
const expected = "((AContext)_localctx).id; ((AContext)_localctx).ID; (((AContext)_localctx).id!=" +
"null?((AContext)_localctx).id.getText():null); ((AContext)_localctx).id.getText(); " +
"(((AContext)_localctx).id!=null?((AContext)_localctx).id.getLine():0);";
testActions(attributeTemplate, "inline", action, expected);
});

it("testRuleRefs", (): void => {
it.skip("testRuleRefs", (): void => {
const action = "$lab.start; $c.text;";
const expected = "(((AContext)_localctx).lab!=null?(((AContext)_localctx).lab.start):null); " +
"(((AContext)_localctx).c!=null?_input.getText(((AContext)_localctx).c.start,((AContext)_localctx)." +
Expand All @@ -199,7 +193,7 @@ describe("TestActionTranslation", () => {
});

/** Added in response to https://github.com/antlr/antlr4/issues/1211 */
it("testUnknownAttr", (): void => {
it.skip("testUnknownAttr", (): void => {
const action = "$qqq.text";
const expected = ""; // was causing an exception
testActions(attributeTemplate, "inline", action, expected);
Expand All @@ -210,7 +204,7 @@ describe("TestActionTranslation", () => {
* $e.v yields incorrect value 0 in "e returns [int v] : '1' {$v = 1;} | '(' e ')' {$v = $e.v;} ;"
* https://github.com/antlr/antlr4/issues/1295
*/
it("testRuleRefsRecursive", (): void => {
it.skip("testRuleRefsRecursive", (): void => {
const recursiveTemplate =
"recursiveTemplate(inline) ::= <<\n" +
"parser grammar A;\n" +
Expand Down Expand Up @@ -245,7 +239,7 @@ describe("TestActionTranslation", () => {
testActions(leftRecursiveTemplate, "inline", action, expected);
});

it("testRefToTextAttributeForCurrentRule", (): void => {
it.skip("testRefToTextAttributeForCurrentRule", (): void => {
const action = "$ctx.text; $text";

// this is the expected translation for all cases
Expand Down

0 comments on commit 3e110b0

Please sign in to comment.