Skip to content

Commit

Permalink
The first unit test runs successfully
Browse files Browse the repository at this point in the history 
Some more clean up done too.
  • Loading branch information
@mike-lischke
mike-lischke committed Oct 20, 2024
1 parent e5f3ef3 commit caecc04
Show file tree
Hide file tree
Showing 11 changed files with 328 additions and 389 deletions.
14 changes: 14 additions & 0 deletions src/codegen/model/chunk/RulePropertyRefText.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
/*
* Copyright (c) The ANTLR Project. All rights reserved.
* Use of this file is governed by the BSD 3-clause license that
* can be found in the LICENSE.txt file in the project root.
*/

import { StructDecl } from "../decl/StructDecl.js";
import { RulePropertyRef } from "./RulePropertyRef.js";

export class RulePropertyRefText extends RulePropertyRef {
public constructor(ctx: StructDecl, label: string) {
super(ctx, label);
}
}
67 changes: 42 additions & 25 deletions src/grammars/ActionSplitter.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,27 @@ lexer grammar ActionSplitter;
// $antlr-format allowShortRulesOnASingleLine on, alignSemicolons none, minEmptyLines 0

@header {
/* eslint-disable */
import { Character } from "../support/Character.js";
import type { ActionSplitterListener } from "../parse/ActionSplitterListener.js";
}

@members {
/** Force filtering (and return tokens).Sends token values to the delegate. */
public getActionTokens(delegate: ActionSplitterListener): Token[] {
const chunks = new Array<Token>();
let t = this.nextToken();
while (t.type !== Token.EOF) {
private readonly attrValueExpr = /[^=][^;]*/;
private readonly ws = /[ \t\n\r]+/;
private readonly id = /[a-zA-Z_][a-zA-Z0-9_]*/;
private readonly setNonLocalAttr = new RegExp(`\\$(?<x>${this.id.source})::(<y>${this.id.source})\s?=(?<expr>${this.attrValueExpr.source});`);
private readonly nonLocalAttr = new RegExp(`\\$(?<x>${this.id.source})::(?<y>${this.id.source})`);
private readonly qualifiedAttr = new RegExp(`\\$(?<x>${this.id.source}).(?<y>${this.id.source})`);
private readonly setAttr = new RegExp(`\\$(?<x>${this.id.source})\s?=(?<expr>${this.attrValueExpr.source});`);
private readonly attr = new RegExp(`\\$(?<x>${this.id.source})`);

/** Force filtering (and return tokens). Sends token values to the delegate. */
public getActionTokens(delegate: ActionSplitterListener): Token[] {
const tokens = this.getAllTokens();
for (let i = 0; i < tokens.length; i++) {
const t = tokens[i];
switch (t.type) {
case ActionSplitter.COMMENT:
case ActionSplitter.LINE_COMMENT:
Expand All @@ -31,11 +41,10 @@ public getActionTokens(delegate: ActionSplitterListener): Token[] {
}

case ActionSplitter.SET_NONLOCAL_ATTR: {
const regex = /\$(?<x>[a-zA-Z_][a-zA-Z0-9_]*)::\s*(?<y>[a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*(?<expr>[^=][^;]*);/;
const text = t.text!;
// Parse the text and extract the named groups from the match result.
const match = text.match(regex);
const match = text.match(this.setNonLocalAttr);
if (match === null) {
throw new Error(`Mismatched input '${text}'`);
}
Expand All @@ -47,10 +56,9 @@ public getActionTokens(delegate: ActionSplitterListener): Token[] {
}

case ActionSplitter.NONLOCAL_ATTR: {
const regex = /\$(?<x>[a-zA-Z_][a-zA-Z0-9_]*)::\s*(?<y>[a-zA-Z_][a-zA-Z0-9_]*)/;
const text = t.text!;
const match = text.match(regex);
const match = text.match(this.nonLocalAttr);
if (match === null) {
throw new Error(`Mismatched input '${text}'`);
}
Expand All @@ -62,25 +70,37 @@ public getActionTokens(delegate: ActionSplitterListener): Token[] {
}

case ActionSplitter.QUALIFIED_ATTR: {
const regex = /\$(?<x>[a-zA-Z_][a-zA-Z0-9_]*)::\s*(?<y>[a-zA-Z_][a-zA-Z0-9_]*)/;
const text = t.text!;
let text = t.text!;
const match = text.match(regex);
const match = text.match(this.qualifiedAttr);
if (match === null) {
throw new Error(`Mismatched input '${text}'`);
}

const { x, y } = match.groups!;

// In the ANTLR3 version of the grammar, QUALIFIED_ATTR was not matched if followed by a '('.
// We have to simulate this behavior here.
if (i + 1 < tokens.length && tokens[i + 1].text?.startsWith("(")) {
// Pretend we matched ATTR instead of QUALIFIED_ATTR, with the first part of the qualified name.
delegate.attr("$" + x, x);
// Pretend we matched TEXT instead of QUALIFIED_ATTR, with the rest of the qualified name.
text = "." + y + tokens[++i].text!;
delegate.text(text);
break;
}

delegate.qualifiedAttr(text, x, y);

break;
}

case ActionSplitter.SET_ATTR: {
const regex = /\$(?<x>[a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*(?<expr>[^=][^;]*);/;
const text = t.text!;
const match = text.match(regex);
const match = text.match(this.setAttr);
if (match === null) {
throw new Error(`Mismatched input '${text}'`);
}
Expand All @@ -92,10 +112,9 @@ public getActionTokens(delegate: ActionSplitterListener): Token[] {
}

case ActionSplitter.ATTR: {
const regex = /\$(?<x>[a-zA-Z_][a-zA-Z0-9_]*)/;
const text = t.text!;
const match = text.match(regex);
const match = text.match(this.attr);
if (match === null) {
throw new Error(`Mismatched input '${text}'`);
}
Expand All @@ -108,19 +127,15 @@ public getActionTokens(delegate: ActionSplitterListener): Token[] {

default:
}

chunks.push(t);
t = this.nextToken();
}

return chunks;
// TODO: need to remove the EOF token from the list of tokens?
return tokens;
}

private isIDStartChar(c: number): boolean {
return c == 0x5F /* "_" */ || Character.isLetter(c);
}

}
return c == 0x5F /* "_" */ || Character.isLetter(c);
}}

// ignore comments right away

Expand All @@ -138,11 +153,13 @@ SET_ATTR: '$' ID WS? '=' ATTR_VALUE_EXPR ';';
ATTR: '$' ID;

// Anything else is just random text

TEXT: ( ~('\\' | '$') | '\\$' | '\\' ~('$') | {!this.isIDStartChar(this.inputStream.LA(2))}? '$')+;

fragment ID: ('a' ..'z' | 'A' ..'Z' | '_') ('a' ..'z' | 'A' ..'Z' | '0' ..'9' | '_')*;

/** Don't allow an = as first char to prevent $x == 3; kind of stuff. */

fragment ATTR_VALUE_EXPR: ~'=' (~';')*;

fragment WS: (' ' | '\t' | '\n' | '\r')+;
14 changes: 7 additions & 7 deletions src/tool/GrammarParserInterpreter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ export class GrammarParserInterpreter extends ParserInterpreter {
// Create a new parser interpreter to parse the ambiguous sub phrase
const parser = GrammarParserInterpreter.deriveTempParserInterpreter(g, originalParser, tokens);

const decisionState = originalParser.atn.decisionToState[decision];
const decisionState = originalParser.#atn.decisionToState[decision];

for (let alt = 1; alt <= decisionState.transitions.length; alt++) {
// re-parse entire input for all ambiguous alternatives
Expand Down Expand Up @@ -294,7 +294,7 @@ export class GrammarParserInterpreter extends ParserInterpreter {
if (originalParser instanceof ParserInterpreter) {
try {
const ctor = originalParser.constructor as Constructor<ParserInterpreter>;
parser = new ctor(g, originalParser.atn, originalParser.tokenStream);
parser = new ctor(g, originalParser.#atn, originalParser.tokenStream);
} catch (e) {
if (e instanceof Error) {
throw new Error("can't create parser to match incoming " + originalParser.constructor.name, e);
Expand All @@ -304,7 +304,7 @@ export class GrammarParserInterpreter extends ParserInterpreter {
}
} else { // must've been a generated parser
parser = new ParserInterpreter(originalParser.grammarFileName, originalParser.vocabulary,
originalParser.ruleNames, originalParser.atn, tokens);
originalParser.ruleNames, originalParser.#atn, tokens);
}

parser.inputStream = tokens;
Expand All @@ -327,10 +327,10 @@ export class GrammarParserInterpreter extends ParserInterpreter {
*/
public findOuterMostDecisionStates(): BitSet {
const track = new BitSet();
const numberOfDecisions = this.atn.getNumberOfDecisions();
const numberOfDecisions = this.#atn.getNumberOfDecisions();
for (let i = 0; i < numberOfDecisions; i++) {
const decisionState = this.atn.getDecisionState(i)!;
const startState = this.atn.ruleToStartState[decisionState.ruleIndex];
const decisionState = this.#atn.getDecisionState(i)!;
const startState = this.#atn.ruleToStartState[decisionState.ruleIndex];
// Look for StarLoopEntryState that is in any left recursive rule
if (decisionState instanceof StarLoopEntryState) {
const loopEntry = decisionState;
Expand Down Expand Up @@ -412,7 +412,7 @@ export class GrammarParserInterpreter extends ParserInterpreter {
if (this.decisionStatesThatSetOuterAltNumInContext.get(p.stateNumber)) {
ctx.outerAltNum = predictedAlt;
const r = this.g.getRule(p.ruleIndex)!;
if (this.atn.ruleToStartState[r.index]?.isLeftRecursiveRule) {
if (this.#atn.ruleToStartState[r.index]?.isLeftRecursiveRule) {
let alts = this.stateToAltsMap[p.stateNumber];
const lr = this.g.getRule(p.ruleIndex)! as LeftRecursiveRule;
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
Expand Down
7 changes: 3 additions & 4 deletions tests/InterpreterTreeTextProvider.ts
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
/*
* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
* Use of this file is governed by the BSD 3-clause license that
* can be found in the LICENSE.txt file in the project root.
* Copyright (c) Mike Lischke. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*/

import { ErrorNode, Trees, type ParseTree } from "antlr4ng";

export class InterpreterTreeTextProvider implements TreeTextProvider {
export class InterpreterTreeTextProvider {
public ruleNames: string[];
public constructor(ruleNames: string[]) {
this.ruleNames = ruleNames;
Expand Down
Loading

0 comments on commit caecc04

Please sign in to comment.