Skip to content

Commit

Permalink
TestBasicSemanticErrors done
Browse files Browse the repository at this point in the history
  • Loading branch information
mike-lischke committed Dec 1, 2024
1 parent c16b27e commit c9f40aa
Show file tree
Hide file tree
Showing 9 changed files with 167 additions and 146 deletions.
6 changes: 4 additions & 2 deletions src/Tool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -354,8 +354,10 @@ export class Tool implements ITool {

public parseGrammar(fileName: string): GrammarRootAST | undefined {
try {
const content = readFileSync(fileName, { encoding: grammarOptions.grammarEncoding as BufferEncoding });
const input = CharStream.fromString(content.toString());
const encoding = grammarOptions.grammarEncoding ?? "utf-8";
const content = readFileSync(fileName, { encoding: encoding as BufferEncoding });
const input = CharStream.fromString(content);
input.name = basename(fileName);

return this.parse(fileName, input);
} catch (ioe) {
Expand Down
2 changes: 1 addition & 1 deletion src/grammars/predefined.tokens
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ SRC = 57
STAR = 58
STRING_LITERAL = 59
THROWS = 60
TOKENS_SPEC = 61
UNUSED2 = 61
TOKEN_REF = 62
UNICODE_ESC = 63
UNICODE_EXTENDED_ESC = 64
Expand Down
3 changes: 2 additions & 1 deletion src/semantics/BasicSemanticChecks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -317,7 +317,8 @@ export class BasicSemanticChecks extends GrammarTreeVisitor {
}

// Don't warn about diff if this is implicit lexer.
if (fullyQualifiedName.startsWith(nameToken.text!) &&
const fileName = fullyQualifiedName.substring(0, fullyQualifiedName.lastIndexOf("."));
if (fileName !== nameToken.text! &&
fullyQualifiedName !== Constants.GRAMMAR_FROM_STRING_NAME) {
this.g.tool.errorManager.grammarError(ErrorType.FILE_AND_GRAMMAR_NAME_DIFFER, fullyQualifiedName, nameToken,
nameToken.text, fullyQualifiedName);
Expand Down
8 changes: 3 additions & 5 deletions src/support/ParseTreeToASTConverter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
// cspell: ignore RULEMODIFIERS, ruleref

import {
CommonToken, ParserRuleContext, type CharStream, type TerminalNode, type TokenSource, type TokenStream
CommonToken, ParserRuleContext, type TerminalNode, type TokenStream
} from "antlr4ng";

import {
Expand Down Expand Up @@ -276,7 +276,7 @@ export class ParseTreeToASTConverter {

public static convertTokensSpec(tokensSpec: TokensSpecContext, ast: GrammarAST): void {
if (tokensSpec.idList()) {
const tokens = this.createVirtualASTNode(GrammarAST, ANTLRv4Parser.TOKENS_SPEC, tokensSpec);
const tokens = this.createVirtualASTNode(GrammarAST, ANTLRv4Parser.TOKENS, tokensSpec);
ast.addChild(tokens);

tokensSpec.idList()!.identifier().forEach((id) => {
Expand Down Expand Up @@ -929,9 +929,7 @@ export class ParseTreeToASTConverter {
ref: ParserRuleContext | TerminalNode, text?: string): T {

const sourceToken = ref instanceof ParserRuleContext ? ref.start! : ref.symbol;
const source: [TokenSource | null, CharStream | null] = [sourceToken.tokenSource, sourceToken.inputStream];
const token = CommonToken.fromSource(source, astType, Constants.DEFAULT_TOKEN_CHANNEL, sourceToken.start,
sourceToken.stop);
const token = this.createToken(astType, ref, text);

if (text) {
token.text = text;
Expand Down
39 changes: 20 additions & 19 deletions src/tool/DOTGenerator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,16 @@ import { Grammar } from "./Grammar.js";
export class DOTGenerator {
public static readonly STRIP_NONREDUCED_STATES = false;

/** Library of output templates; use {@code <attrname>} format. */
private static readonly templateUrl = new URL("../../templates/dot/graphs.stg", import.meta.url);

private static readonly stLib = new STGroupFile(this.templateUrl.pathname);

protected arrowhead = "normal";
protected rankdir = "LR";

protected grammar: Grammar;

/** Library of output templates; use {@code <attrname>} format. */
static readonly #templateUrl = new URL("../../templates/dot/graphs.stg", import.meta.url);
static readonly #stLib = new STGroupFile(this.#templateUrl.pathname);

/** This aspect is associated with a grammar */
public constructor(grammar: Grammar) {
this.grammar = grammar;
Expand All @@ -45,7 +46,7 @@ export class DOTGenerator {

// The output DOT graph for visualization
const markedStates = new Set<ATNState>();
const dot = DOTGenerator.#stLib.getInstanceOf("atn");
const dot = DOTGenerator.stLib.getInstanceOf("atn");
if (!dot) {
throw new Error("no such template: atn");
}
Expand Down Expand Up @@ -79,7 +80,7 @@ export class DOTGenerator {
const rr = (edge);

// don't jump to other rules, but display edge to follow node
edgeST = DOTGenerator.#stLib.getInstanceOf("edge");
edgeST = DOTGenerator.stLib.getInstanceOf("edge");
if (!edgeST) {
throw new Error("no such template: edge");
}
Expand All @@ -100,21 +101,21 @@ export class DOTGenerator {
}

if (edge instanceof ActionTransition) {
edgeST = DOTGenerator.#stLib.getInstanceOf("action-edge");
edgeST = DOTGenerator.stLib.getInstanceOf("action-edge");
if (!edgeST) {
throw new Error("no such template: action-edge");
}

edgeST.add("label", this.getEdgeLabel(edge.toString()));
} else if (edge instanceof AbstractPredicateTransition) {
edgeST = DOTGenerator.#stLib.getInstanceOf("edge");
edgeST = DOTGenerator.stLib.getInstanceOf("edge");
if (!edgeST) {
throw new Error("no such template: edge");
}

edgeST.add("label", this.getEdgeLabel(String(edge)));
} else if (edge.isEpsilon) {
edgeST = DOTGenerator.#stLib.getInstanceOf("epsilon-edge");
edgeST = DOTGenerator.stLib.getInstanceOf("epsilon-edge");
if (!edgeST) {
throw new Error("no such template: epsilon-edge");
}
Expand All @@ -131,7 +132,7 @@ export class DOTGenerator {

edgeST.add("loopback", loopback);
} else if (edge instanceof AtomTransition) {
edgeST = DOTGenerator.#stLib.getInstanceOf("edge");
edgeST = DOTGenerator.stLib.getInstanceOf("edge");
if (!edgeST) {
throw new Error("no such template: edge");
}
Expand All @@ -145,7 +146,7 @@ export class DOTGenerator {

edgeST.add("label", this.getEdgeLabel(label));
} else if (edge instanceof SetTransition) {
edgeST = DOTGenerator.#stLib.getInstanceOf("edge");
edgeST = DOTGenerator.stLib.getInstanceOf("edge");
if (!edgeST) {
throw new Error("no such template: edge");
}
Expand All @@ -163,7 +164,7 @@ export class DOTGenerator {

edgeST.add("label", this.getEdgeLabel(label));
} else if (edge instanceof RangeTransition) {
edgeST = DOTGenerator.#stLib.getInstanceOf("edge");
edgeST = DOTGenerator.stLib.getInstanceOf("edge");
if (!edgeST) {
throw new Error("no such template: edge");
}
Expand All @@ -177,7 +178,7 @@ export class DOTGenerator {

edgeST.add("label", this.getEdgeLabel(label));
} else {
edgeST = DOTGenerator.#stLib.getInstanceOf("edge");
edgeST = DOTGenerator.stLib.getInstanceOf("edge");
if (!edgeST) {
throw new Error("no such template: edge");
}
Expand All @@ -204,7 +205,7 @@ export class DOTGenerator {
continue;
}

const st = DOTGenerator.#stLib.getInstanceOf("stopstate");
const st = DOTGenerator.stLib.getInstanceOf("stopstate");
if (!st) {
throw new Error("no such template: stopstate");
}
Expand All @@ -219,7 +220,7 @@ export class DOTGenerator {
continue;
}

const st = DOTGenerator.#stLib.getInstanceOf("state");
const st = DOTGenerator.stLib.getInstanceOf("state");
if (!st) {
throw new Error("no such template: state");
}
Expand All @@ -238,7 +239,7 @@ export class DOTGenerator {
return "";
}

const dot = DOTGenerator.#stLib.getInstanceOf("dfa");
const dot = DOTGenerator.stLib.getInstanceOf("dfa");
if (!dot) {
throw new Error("no such template: dfa");
}
Expand All @@ -253,7 +254,7 @@ export class DOTGenerator {
continue;
}

const st = DOTGenerator.#stLib.getInstanceOf("stopstate");
const st = DOTGenerator.stLib.getInstanceOf("stopstate");
if (!st) {
throw new Error("no such template: stopstate");
}
Expand All @@ -272,7 +273,7 @@ export class DOTGenerator {
continue;
}

const st = DOTGenerator.#stLib.getInstanceOf("state");
const st = DOTGenerator.stLib.getInstanceOf("state");
if (!st) {
throw new Error("no such template: state");
}
Expand All @@ -297,7 +298,7 @@ export class DOTGenerator {
label = this.grammar.getTokenDisplayName(ttype)!;
}

const st = DOTGenerator.#stLib.getInstanceOf("edge");
const st = DOTGenerator.stLib.getInstanceOf("edge");
if (!st) {
throw new Error("no such template: edge");
}
Expand Down
16 changes: 8 additions & 8 deletions src/tree-walkers/GrammarTreeVisitor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ export class GrammarTreeVisitor extends TreeParser {
"LOCALS", "LPAREN", "LT", "MODE", "NESTED_ACTION", "NLCHARS", "NOT", "NameChar",
"NameStartChar", "OPTIONS", "OR", "PARSER", "PLUS", "PLUS_ASSIGN", "POUND",
"QUESTION", "RANGE", "RARROW", "RBRACE", "RETURNS", "RPAREN", "RULE_REF",
"SEMI", "SEMPRED", "SRC", "STAR", "STRING_LITERAL", "THROWS", "TOKENS_SPEC",
"SEMI", "SEMPRED", "SRC", "STAR", "STRING_LITERAL", "THROWS", "TOKENS",
"TOKEN_REF", "UNICODE_ESC", "UNICODE_EXTENDED_ESC", "UnicodeBOM", "WS",
"WSCHARS", "WSNLCHARS", "ALT", "BLOCK", "CLOSURE", "COMBINED", "ELEMENT_OPTIONS",
"EPSILON", "LEXER_ACTION_CALL", "LEXER_ALT_ACTION", "OPTIONAL", "POSITIVE_CLOSURE",
Expand Down Expand Up @@ -705,7 +705,7 @@ export class GrammarTreeVisitor extends TreeParser {
// org/antlr/v4/parse/GrammarTreeVisitor.g:364:2: ( ( prequelConstruct )+ |)
let alt3 = 2;
const LA3_0 = this.input.LA(1);
if ((LA3_0 === ANTLRv4Parser.AT || LA3_0 === ANTLRv4Parser.CHANNELS || LA3_0 === ANTLRv4Parser.IMPORT || LA3_0 === ANTLRv4Parser.OPTIONS || LA3_0 === ANTLRv4Parser.TOKENS_SPEC)) {
if ((LA3_0 === ANTLRv4Parser.AT || LA3_0 === ANTLRv4Parser.CHANNELS || LA3_0 === ANTLRv4Parser.IMPORT || LA3_0 === ANTLRv4Parser.OPTIONS || LA3_0 === ANTLRv4Parser.TOKENS)) {
alt3 = 1;
} else {
if ((LA3_0 === ANTLRv4Parser.RULES)) {
Expand All @@ -728,7 +728,7 @@ export class GrammarTreeVisitor extends TreeParser {
while (true) {
let alt2 = 2;
const LA2_0 = this.input.LA(1);
if ((LA2_0 === ANTLRv4Parser.AT || LA2_0 === ANTLRv4Parser.CHANNELS || LA2_0 === ANTLRv4Parser.IMPORT || LA2_0 === ANTLRv4Parser.OPTIONS || LA2_0 === ANTLRv4Parser.TOKENS_SPEC)) {
if ((LA2_0 === ANTLRv4Parser.AT || LA2_0 === ANTLRv4Parser.CHANNELS || LA2_0 === ANTLRv4Parser.IMPORT || LA2_0 === ANTLRv4Parser.OPTIONS || LA2_0 === ANTLRv4Parser.TOKENS)) {
alt2 = 1;
}

Expand Down Expand Up @@ -809,7 +809,7 @@ export class GrammarTreeVisitor extends TreeParser {
break;
}

case ANTLRv4Parser.TOKENS_SPEC: {
case ANTLRv4Parser.TOKENS: {
{
alt4 = 3;
}
Expand Down Expand Up @@ -1189,18 +1189,18 @@ export class GrammarTreeVisitor extends TreeParser {
}

// $ANTLR start "tokensSpec"
// org/antlr/v4/parse/GrammarTreeVisitor.g:443:1: tokensSpec : ^( TOKENS_SPEC ( tokenSpec )+ ) ;
// org/antlr/v4/parse/GrammarTreeVisitor.g:443:1: tokensSpec : ^( TOKENS ( tokenSpec )+ ) ;
public tokensSpec(): GrammarTreeVisitor.tokensSpec_return {
const retval = new GrammarTreeVisitor.tokensSpec_return();
retval.start = this.input.LT(1);

this.enterTokensSpec((retval.start as GrammarAST));

try {
// org/antlr/v4/parse/GrammarTreeVisitor.g:450:2: ( ^( TOKENS_SPEC ( tokenSpec )+ ) )
// org/antlr/v4/parse/GrammarTreeVisitor.g:450:6: ^( TOKENS_SPEC ( tokenSpec )+ )
// org/antlr/v4/parse/GrammarTreeVisitor.g:450:2: ( ^( TOKENS ( tokenSpec )+ ) )
// org/antlr/v4/parse/GrammarTreeVisitor.g:450:6: ^( TOKENS ( tokenSpec )+ )
{
this.match(this.input, ANTLRv4Parser.TOKENS_SPEC, null);
this.match(this.input, ANTLRv4Parser.TOKENS, null);
this.match(this.input, Constants.DOWN, null);
// org/antlr/v4/parse/GrammarTreeVisitor.g:450:20: ( tokenSpec )+
let cnt8 = 0;
Expand Down
112 changes: 112 additions & 0 deletions tests/TestBasicSemanticErrors.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
/*
* Copyright (c) Mike Lischke. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*/

// cspell: ignore blech

import { describe, it } from "vitest";

import { ST } from "stringtemplate4ts";

import { ErrorType } from "../src/tool/ErrorType.js";
import { ToolTestUtils } from "./ToolTestUtils.js";

describe("TestBasicSemanticErrors", () => {
const grammarU = [
// INPUT
"parser grammar U;\n" +
"options { foo=bar; k=3;}\n" +
"tokens {\n" +
" ID,\n" +
" f,\n" +
" S\n" +
"}\n" +
"tokens { A }\n" +
"options { x=y; }\n" +
"\n" +
"a\n" +
"options { blech=bar; greedy=true; }\n" +
" : ID\n" +
" ;\n" +
"b : ( options { ick=bar; greedy=true; } : ID )+ ;\n" +
"c : ID<blue> ID<x=y> ;",

// YIELDS
"warning(" + ErrorType.ILLEGAL_OPTION.code + "): U.g4:2:10: unsupported option foo\n" +
"warning(" + ErrorType.ILLEGAL_OPTION.code + "): U.g4:2:19: unsupported option k\n" +
"error(" + ErrorType.TOKEN_NAMES_MUST_START_UPPER.code + "): U.g4:5:8: token names must start with an " +
"uppercase letter: f\n" +
"warning(" + ErrorType.ILLEGAL_OPTION.code + "): U.g4:9:10: unsupported option x\n" +
"error(" + ErrorType.REPEATED_PREQUEL.code + "): U.g4:9:0: repeated grammar prequel spec (options, tokens," +
" or import); please merge\n" +
"error(" + ErrorType.REPEATED_PREQUEL.code + "): U.g4:8:0: repeated grammar prequel spec (options, tokens, " +
"or import); please merge\n" +
"warning(" + ErrorType.ILLEGAL_OPTION.code + "): U.g4:12:10: unsupported option blech\n" +
"warning(" + ErrorType.ILLEGAL_OPTION.code + "): U.g4:12:21: unsupported option greedy\n" +
"warning(" + ErrorType.ILLEGAL_OPTION.code + "): U.g4:15:16: unsupported option ick\n" +
"warning(" + ErrorType.ILLEGAL_OPTION.code + "): U.g4:15:25: unsupported option greedy\n" +
"warning(" + ErrorType.ILLEGAL_OPTION.code + "): U.g4:16:16: unsupported option x\n",
];

it("testU", (): void => {
ToolTestUtils.testErrors(grammarU, true);
});

/**
* Regression test for #25 "Don't allow labels on not token set subrules".
* https://github.com/antlr/antlr4/issues/25
*/
it("testIllegalNonSetLabel", (): void => {
const grammar =
"grammar T;\n" +
"ss : op=('=' | '+=' | expr) EOF;\n" +
"expr : '=' '=';\n" +
"";

const expected =
"error(" + ErrorType.LABEL_BLOCK_NOT_A_SET.code + "): T.g4:2:5: label op assigned to a block which " +
"is not a set\n";

ToolTestUtils.testErrors([grammar, expected], false);
});

it("testArgumentRetvalLocalConflicts", (): void => {
const grammarTemplate =
"grammar T;\n" +
"ss<if(args)>[<args>]<endif> <if(retvals)>returns [<retvals>]<endif>\n" +
"<if(locals)>locals [<locals>]<endif>\n" +
" : <body> EOF;\n" +
"expr : '=';\n";

const expected =
"error(" + ErrorType.ARG_CONFLICTS_WITH_RULE.code + "): T.g4:2:7: parameter expr conflicts with rule " +
"with same name\n" +
"error(" + ErrorType.RETVAL_CONFLICTS_WITH_RULE.code + "): T.g4:2:26: return value expr conflicts with " +
"rule with same name\n" +
"error(" + ErrorType.LOCAL_CONFLICTS_WITH_RULE.code + "): T.g4:3:12: local expr conflicts with rule " +
"with same name\n" +
"error(" + ErrorType.RETVAL_CONFLICTS_WITH_ARG.code + "): T.g4:2:26: return value expr conflicts with " +
"parameter with same name\n" +
"error(" + ErrorType.LOCAL_CONFLICTS_WITH_ARG.code + "): T.g4:3:12: local expr conflicts with parameter" +
" with same name\n" +
"error(" + ErrorType.LOCAL_CONFLICTS_WITH_RETVAL.code + "): T.g4:3:12: local expr conflicts with return" +
" value with same name\n" +
"error(" + ErrorType.LABEL_CONFLICTS_WITH_RULE.code + "): T.g4:4:4: label expr conflicts with rule with" +
" same name\n" +
"error(" + ErrorType.LABEL_CONFLICTS_WITH_ARG.code + "): T.g4:4:4: label expr conflicts with parameter" +
" with same name\n" +
"error(" + ErrorType.LABEL_CONFLICTS_WITH_RETVAL.code + "): T.g4:4:4: label expr conflicts with return" +
" value with same name\n" +
"error(" + ErrorType.LABEL_CONFLICTS_WITH_LOCAL.code + "): T.g4:4:4: label expr conflicts with local" +
" with same name\n";

const grammarST = new ST(grammarTemplate);
grammarST.add("args", "int expr");
grammarST.add("retvals", "int expr");
grammarST.add("locals", "int expr");
grammarST.add("body", "expr=expr");
ToolTestUtils.testErrors([grammarST.render(), expected], false);
});

});
Loading

0 comments on commit c9f40aa

Please sign in to comment.