Skip to content

Commit

Permalink
TestBasicSemanticErrors, TestCompositeGrammars, TestDollarParser, Te…
Browse files Browse the repository at this point in the history 
…stUnicodeEscapes done

This was a tough one. Had to change quite a few places (including the antlr4ng dependency) to make the new tests work.
  • Loading branch information
@mike-lischke
mike-lischke committed Dec 9, 2024
1 parent 4051542 commit be57803
Show file tree
Hide file tree
Showing 36 changed files with 1,282 additions and 1,277 deletions.
1 change: 1 addition & 0 deletions cspell.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@
"prec",
"precpred",
"preds",
"println",
"rankdir",
"rarr",
"recog",
Expand Down
8 changes: 4 additions & 4 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
],
"license": "MIT",
"dependencies": {
"antlr4ng": "3.0.10",
"antlr4ng": "3.0.11",
"commander": "12.1.0",
"fast-printf": "1.6.9",
"stringtemplate4ts": "1.0.4",
Expand Down
27 changes: 15 additions & 12 deletions src/Tool.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

import { ATNSerializer, CharStream, CommonTokenStream } from "antlr4ng";
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
import path, { basename } from "path";
import path, { basename, dirname } from "path";

import { ANTLRv4Parser } from "./generated/ANTLRv4Parser.js";

Expand Down Expand Up @@ -273,7 +273,7 @@ export class Tool implements ITool {
const chk = new UndefChecker(g.isLexer(), ruleToAST, this.errorManager);
chk.visitGrammar(g.ast);

return redefinition; // || chk.errors;
return redefinition || chk.badRef;
}

public sortGrammarByTokenVocab(fileNames: string[]): GrammarRootAST[] {
Expand Down Expand Up @@ -376,7 +376,6 @@ export class Tool implements ITool {
const grammarAST = this.parseGrammar(fileName)!;
const g = this.createGrammar(grammarAST);
g.fileName = fileName;
this.process(g, false);

return g;
}
Expand Down Expand Up @@ -408,9 +407,10 @@ export class Tool implements ITool {
return null;
}

const grammarEncoding = grammarOptions.grammarEncoding as BufferEncoding;
const grammarEncoding = grammarOptions.encoding as BufferEncoding;
const content = readFileSync(importedFile, { encoding: grammarEncoding });
const input = CharStream.fromString(content.toString());
const input = CharStream.fromString(content);
input.name = basename(importedFile);
const result = this.parse(g.fileName, input);
if (!result) {
return null;
Expand Down Expand Up @@ -498,21 +498,24 @@ export class Tool implements ITool {
}

public getImportedGrammarFile(g: Grammar, fileName: string): string | undefined {
if (!existsSync(fileName)) {
const parentDir = basename(fileName); // Check the parent dir of input directory.
fileName = path.join(parentDir, fileName);
if (!existsSync(fileName)) { // try in lib dir
let candidate = fileName;
if (!existsSync(candidate)) {
const parentDir = dirname(g.fileName); // Check the parent dir of input directory.
candidate = path.join(parentDir, fileName);
if (!existsSync(candidate)) { // try in lib dir
const libDirectory = grammarOptions.libDirectory;
if (libDirectory) {
fileName = path.join(libDirectory, fileName);
if (!existsSync(fileName)) {
candidate = path.join(libDirectory, fileName);
if (!existsSync(candidate)) {
return undefined;
}

return candidate;
}
}
}

return fileName;
return candidate;
}

/**
Expand Down
6 changes: 3 additions & 3 deletions src/analysis/AnalysisPipeline.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ export class AnalysisPipeline {
continue;
}

const analyzer = new LL1Analyzer();
const look = analyzer.look(this.g.atn!, this.g.atn!.ruleToStartState[rule.index]!, undefined);
const analyzer = new LL1Analyzer(this.g.atn!);
const look = analyzer.look(this.g.atn!.ruleToStartState[rule.index]!, undefined);
if (look.contains(Token.EPSILON)) {
this.g.tool.errorManager.grammarError(ErrorType.EPSILON_TOKEN, this.g.fileName,
(rule.ast.getChild(0) as GrammarAST).token!, rule.name);
Expand All @@ -84,7 +84,7 @@ export class AnalysisPipeline {
if (s.nonGreedy) { // nongreedy decisions can't be LL(1)
look = new Array<IntervalSet>(s.transitions.length + 1);
} else {
const anal = new LL1Analyzer();
const anal = new LL1Analyzer(this.g.atn!);
look = anal.getDecisionLookahead(s) as IntervalSet[];
this.g.tool.logInfo({ component: "LL1", msg: "look=" + look });
}
Expand Down
10 changes: 5 additions & 5 deletions src/analysis/LeftRecursionDetector.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,14 @@ export class LeftRecursionDetector {
* filling the cycles in listOfRecursiveCycles and also, as a
* side-effect, set leftRecursiveRules.
*/
public check(enclosingRule: Rule, s: ATNState, visitedStates: HashSet<ATNState>): boolean;
public check(enclosingRule: Rule, s: ATNState, visitedStates: Set<ATNState>): boolean;
public check(...args: unknown[]): undefined | boolean {
if (args.length === 0) {
for (const start of this.atn.ruleToStartState) {
this.rulesVisitedPerRuleCheck.clear();
this.rulesVisitedPerRuleCheck.add(start!);

this.check(this.g.getRule(start!.ruleIndex)!, start!, new HashSet<ATNState>());
this.check(this.g.getRule(start!.ruleIndex)!, start!, new Set<ATNState>());
}

if (this.listOfRecursiveCycles.length > 0) {
Expand All @@ -60,13 +60,13 @@ export class LeftRecursionDetector {
return;
}

const [enclosingRule, s, visitedStates] = args as [Rule, ATNState, HashSet<ATNState>];
const [enclosingRule, s, visitedStates] = args as [Rule, ATNState, Set<ATNState>];

if (s instanceof RuleStopState) {
return true;
}

if (visitedStates.contains(s)) {
if (visitedStates.has(s)) {
return false;
}

Expand All @@ -86,7 +86,7 @@ export class LeftRecursionDetector {
this.rulesVisitedPerRuleCheck.add(t.target as RuleStartState);

// send new visitedStates set per rule invocation
const nullable = this.check(r, t.target, new HashSet<ATNState>());
const nullable = this.check(r, t.target, new Set<ATNState>());

// we're back from visiting that rule
this.rulesVisitedPerRuleCheck.remove(t.target as RuleStartState);
Expand Down
4 changes: 2 additions & 2 deletions src/automata/IATNFactory.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import type { Constructor } from "../misc/Utils.js";
* state submachine. Used to build ATNs.
*/
export interface IStatePair {
left: ATNState;
left: ATNState | null;
right: ATNState | null;
}

Expand Down Expand Up @@ -79,7 +79,7 @@ export interface IATNFactory {
*/
action(action: ActionAST | string): IStatePair;

alt(els: IStatePair[]): IStatePair;
alt(els: Array<IStatePair | null>): IStatePair;

/**
* From A|B|..|Z alternative block build
Expand Down
2 changes: 1 addition & 1 deletion src/automata/LexerATNFactory.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ export class LexerATNFactory extends ParserATNFactory {

public override lexerAltCommands(alt: IStatePair, commands: IStatePair): IStatePair {
const h = { left: alt.left, right: commands.right };
this.epsilon(alt.right, commands.left);
this.epsilon(alt.right, commands.left!);

return h;
}
Expand Down
45 changes: 26 additions & 19 deletions src/automata/ParserATNFactory.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ export class ParserATNFactory implements IParserATNFactory, IATNFactory {
continue;
}

const analyzer = new LL1Analyzer();
if (analyzer.look(this.atn, startState, atnState2).contains(Token.EPSILON)) {
const analyzer = new LL1Analyzer(this.atn);
if (analyzer.look(startState, atnState2).contains(Token.EPSILON)) {
this.g.tool.errorManager.grammarError(ErrorType.EPSILON_OPTIONAL, this.g.fileName,
(rule.ast.getChild(0) as GrammarAST).token!, rule.name);
continue optionalCheck;
Expand Down Expand Up @@ -137,7 +137,7 @@ export class ParserATNFactory implements IParserATNFactory, IATNFactory {
public rule(ruleAST: GrammarAST, name: string, blk: IStatePair): IStatePair {
const r = this.g.getRule(name)!;
const start = this.atn.ruleToStartState[r.index]!;
this.epsilon(start, blk.left);
this.epsilon(start, blk.left!);
const stop = this.atn.ruleToStopState[r.index]!;
this.epsilon(blk.right, stop);
const h = { left: start, right: stop };
Expand Down Expand Up @@ -349,7 +349,7 @@ export class ParserATNFactory implements IParserATNFactory, IATNFactory {
if (ebnfRoot === null) {
if (alts.length === 1) {
const h = alts[0];
blkAST.atnState = h.left;
blkAST.atnState = h.left!;

return h;
}
Expand Down Expand Up @@ -404,20 +404,20 @@ export class ParserATNFactory implements IParserATNFactory, IATNFactory {
return this.elemList(els);
}

public elemList(els: IStatePair[]): IStatePair {
public elemList(els: Array<IStatePair | null>): IStatePair {
const n = els.length;
for (let i = 0; i < n - 1; i++) { // hook up elements (visit all but last)
const el = els[i];
const el = els[i]!;

// if el is of form o-x->o for x in {rule, action, pred, token, ...}
// and not last in alt
let tr = null;
if (el.left.transitions.length === 1) {
tr = el.left.transitions[0];
if (el.left!.transitions.length === 1) {
tr = el.left!.transitions[0];
}

const isRuleTrans = tr instanceof RuleTransition;
if ((el.left.constructor as typeof ATNState).stateType === ATNState.BASIC
if ((el.left!.constructor as typeof ATNState).stateType === ATNState.BASIC
&& el.right && (el.right.constructor as typeof ATNState).stateType === ATNState.BASIC
&& tr !== null
&& (isRuleTrans && (tr as RuleTransition).followState === el.right || tr.target === el.right)) {
Expand All @@ -429,21 +429,28 @@ export class ParserATNFactory implements IParserATNFactory, IATNFactory {

if (handle !== null) {
if (isRuleTrans) {
(tr as RuleTransition).followState = handle.left;
(tr as RuleTransition).followState = handle.left!;
} else {
tr.target = handle.left;
tr.target = handle.left!;
}
}
this.atn.removeState(el.right); // we skipped over this state
} else { // need epsilon if previous block's right end node is complicated
this.epsilon(el.right, els[i + 1].left);
this.epsilon(el.right, els[i + 1]!.left!);
}
}

const first = els[0];
const last = els[n - 1];
const left = first.left;
const right = last.right;
let left: ATNState | null = null;
if (first !== null) {
left = first.left;
}

let right: ATNState | null = null;
if (last !== null) {
right = last.right;
}

return { left, right };
}
Expand All @@ -469,7 +476,7 @@ export class ParserATNFactory implements IParserATNFactory, IATNFactory {
blkStart.nonGreedy = !greedy;
this.epsilon(blkStart, blk.right!, !greedy);

optAST.atnState = blk.left;
optAST.atnState = blk.left!;

return blk;
}
Expand Down Expand Up @@ -668,10 +675,10 @@ export class ParserATNFactory implements IParserATNFactory, IATNFactory {

protected checkEpsilonClosure(): void {
for (const [rule, atnState1, atnState2] of this.preventEpsilonClosureBlocks) {
const analyzer = new LL1Analyzer();
const analyzer = new LL1Analyzer(this.atn);
const blkStart = atnState1;
const blkStop = atnState2;
const lookahead = analyzer.look(this.atn, blkStart, blkStop);
const lookahead = analyzer.look(blkStart, blkStop);
if (lookahead.contains(Token.EPSILON)) {
const errorType = rule instanceof LeftRecursiveRule
? ErrorType.EPSILON_LR_FOLLOW
Expand Down Expand Up @@ -719,12 +726,12 @@ export class ParserATNFactory implements IParserATNFactory, IATNFactory {
start.endState = end;
for (const alt of alts) {
// hook alts up to decision block
this.epsilon(start, alt.left);
this.epsilon(start, alt.left!);
this.epsilon(alt.right, end);
// no back link in ATN so must walk entire alt to see if we can
// strip out the epsilon to 'end' state
const opt = new TailEpsilonRemover(this.atn);
opt.visit(alt.left);
opt.visit(alt.left!);
}

blkAST.atnState = start;
Expand Down
28 changes: 15 additions & 13 deletions src/codegen/ParserFactory.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,8 @@ export class ParserFactory extends DefaultOutputModelFactory {
return new CodeBlockForAlt(this);
}

public override finishAlternative(blk: CodeBlockForAlt, ops: SrcOp[]): CodeBlockForAlt {
blk.ops = ops;
public override finishAlternative(blk: CodeBlockForAlt, ops: SrcOp[] | undefined): CodeBlockForAlt {
blk.ops = ops ?? [];

return blk;
}
Expand Down Expand Up @@ -147,24 +147,26 @@ export class ParserFactory extends DefaultOutputModelFactory {
return new TokenListDecl(this, this.gen.getTarget().getListLabel(label));
}

public override set(setAST: GrammarAST, labelAST: GrammarAST, invert: boolean): SrcOp[] {
public override set(setAST: GrammarAST, labelAST: GrammarAST | null, invert: boolean): SrcOp[] {
let matchOp: MatchSet;
if (invert) {
matchOp = new MatchNotSet(this, setAST);
} else {
matchOp = new MatchSet(this, setAST);
}

const label = labelAST.getText();
const rf = this.getCurrentRuleFunction()!;
if (labelAST.parent?.getType() === ANTLRv4Parser.PLUS_ASSIGN) {
this.defineImplicitLabel(setAST, matchOp);
const l = this.getTokenListLabelDecl(label);
rf.addContextDecl(setAST.getAltLabel()!, l);
} else {
const d = this.getTokenLabelDecl(label);
matchOp.labels.push(d);
rf.addContextDecl(setAST.getAltLabel()!, d);
if (labelAST !== null) {
const label = labelAST.getText();
const rf = this.getCurrentRuleFunction()!;
if (labelAST.parent?.getType() === ANTLRv4Parser.PLUS_ASSIGN) {
this.defineImplicitLabel(setAST, matchOp);
const l = this.getTokenListLabelDecl(label);
rf.addContextDecl(setAST.getAltLabel()!, l);
} else {
const d = this.getTokenLabelDecl(label);
matchOp.labels.push(d);
rf.addContextDecl(setAST.getAltLabel()!, d);
}
}

if (this.controller.needsImplicitLabel(setAST, matchOp)) {
Expand Down
Loading

0 comments on commit be57803

Please sign in to comment.