Skip to content

Commit

Permalink
Folders analysis + automata done
Browse files Browse the repository at this point in the history
  • Loading branch information
@mike-lischke
mike-lischke committed Oct 6, 2024
1 parent 35f9b2f commit 8415551
Show file tree
Hide file tree
Showing 18 changed files with 892 additions and 1,002 deletions.
3 changes: 2 additions & 1 deletion eslint.config.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,8 @@ export default tslint.config(
"@typescript-eslint/no-unnecessary-condition": ["error", { "allowConstantLoopConditions": true }],
"@typescript-eslint/no-extraneous-class": "off",
"@typescript-eslint/array-type": ["error", { "default": "array-simple" }],
"@typescript-eslint/prefer-return-this-type": "off"
"@typescript-eslint/prefer-return-this-type": "off",
"@typescript-eslint/no-invalid-void-type": "off",
},
},
);
1 change: 0 additions & 1 deletion src/antlr3/tree/TreeNodeStream.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ export interface TreeNodeStream extends IntStream {
* instead of a {@link Token}. Makes code generation identical for both
* parser and tree grammars.</p>
*/
// eslint-disable-next-line @typescript-eslint/naming-convention
LT(k: number): Tree | null;

/**
Expand Down
31 changes: 19 additions & 12 deletions src/tree-walkers/ATNBuilder.ts
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,22 @@
/* java2ts: keep */

/*
* Copyright (c) The ANTLR Project. All rights reserved.
* Use of this file is governed by the BSD 3-clause license that
* can be found in the LICENSE.txt file in the project root.
*/

import { RecognitionException, type ParserRuleContext } from "antlr4ng";
import { RecognitionException } from "antlr4ng";

import { ANTLRv4Lexer } from "../generated/ANTLRv4Lexer.js";
import { AlternativeContext, BlockContext, LabeledAltContext, RuleBlockContext, RuleSpecContext, type ElementContext, type LexerAltContext, type LexerCommandContext, type LexerCommandsContext } from "../generated/ANTLRv4Parser.js";
import {
AlternativeContext, BlockContext, RuleBlockContext, RuleSpecContext, type ElementContext, type LexerAltContext,
type LexerCommandContext, type LexerCommandsContext
} from "../generated/ANTLRv4Parser.js";

import { IATNFactory, type IStatePair } from "../../tool/src/org/antlr/v4/automata/ATNFactory.js";
import { IATNFactory, type IStatePair } from "../../tool/src/org/antlr/v4/automata/IATNFactory.js";
import type { BlockAST } from "../../tool/src/org/antlr/v4/tool/ast/BlockAST.js";
import type { GrammarAST } from "../../tool/src/org/antlr/v4/tool/ast/GrammarAST.js";
import type { TreeNodeStream } from "../antlr3/tree/TreeNodeStream.js";

export class ATNBuilder {
public static element_return = class element_return extends TreeRuleReturnScope {
Expand All @@ -36,21 +42,22 @@ export class ATNBuilder {
public p: IStatePair;
};

public constructor(protected factory: IATNFactory, private root: RuleSpecContext) {
public constructor(private input: TreeNodeStream, protected factory: IATNFactory) {
}

public ruleBlock(ebnfRoot: GrammarAST, ruleBlock: RuleBlockContext): IStatePair | undefined { // Four outer blocks.
public ruleBlock(ebnfRoot: GrammarAST | null/*, ruleBlock: RuleBlockContext*/): IStatePair | null { // For outer blocks.
let p: IStatePair | undefined;

const alts = new Array<IStatePair>();
let alt = 1;
const alt = 1;
this.factory.setCurrentOuterAlt(alt);

for (const labledAlt of ruleBlock.ruleAltList().labeledAlt()) {
alts.push(this.alternative(labledAlt.alternative()));
this.factory.setCurrentOuterAlt(++alt);
}

/*
for (const labledAlt of ruleBlock.ruleAltList().labeledAlt()) {
alts.push(this.alternative(labledAlt.alternative()));
this.factory.setCurrentOuterAlt(++alt);
}
*/
return this.factory.block(BLOCK1 as BlockAST, ebnfRoot, alts);
}

Expand Down
2 changes: 1 addition & 1 deletion tool/src/org/antlr/v4/Tool.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import { ANTLRv4Parser } from "../../../../../src/generated/ANTLRv4Parser.js";

import { UndefChecker } from "./UndefChecker.js";
import { AnalysisPipeline } from "./analysis/AnalysisPipeline.js";
import { IATNFactory } from "./automata/ATNFactory.js";
import { IATNFactory } from "./automata/IATNFactory.js";
import { LexerATNFactory } from "./automata/LexerATNFactory.js";
import { ParserATNFactory } from "./automata/ParserATNFactory.js";
import { CodeGenPipeline } from "./codegen/CodeGenPipeline.js";
Expand Down
6 changes: 3 additions & 3 deletions tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ export class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {
public prefixAndOtherAlts = new Array<LeftRecursiveRuleAltInfo>();

/** Pointer to ID node of ^(= ID element) */
public leftRecursiveRuleRefLabels = new Array<[GrammarAST, string | undefined]>();
public leftRecursiveRuleRefLabels = new Array<[GrammarAST, string]>();

/** Tokens from which rule AST comes from */
public readonly tokenStream: TokenStream;
Expand Down Expand Up @@ -138,7 +138,7 @@ export class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {

public override binaryAlt(originalAltTree: AltAST, alt: number): void {
let altTree = originalAltTree.dupTree() as AltAST;
const altLabel = altTree.altLabel?.getText() ?? undefined;
const altLabel = altTree.altLabel!.getText()!;

let label: string | undefined;
let isListLabel = false;
Expand Down Expand Up @@ -183,7 +183,7 @@ export class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker {

public override suffixAlt(originalAltTree: AltAST, alt: number): void {
const altTree = originalAltTree.dupTree() as AltAST;
const altLabel = altTree.altLabel?.getText() ?? undefined;
const altLabel = altTree.altLabel!.getText()!;

let label: string | undefined;
let isListLabel = false;
Expand Down
104 changes: 54 additions & 50 deletions tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,18 @@
* can be found in the LICENSE.txt file in the project root.
*/

import { OrderedHashMap, RecognitionException } from "antlr4ng";
import { CharStream, CommonTokenStream, RecognitionException } from "antlr4ng";

import { ANTLRv4Lexer } from "../../../../../../src/generated/ANTLRv4Lexer.js";
import { ANTLRv4Parser } from "../../../../../../src/generated/ANTLRv4Parser.js";
import { Tool } from "../Tool.js";
import type { SupportedLanguage } from "../codegen/CodeGenerator.js";
import { GrammarASTAdaptor } from "../parse/GrammarASTAdaptor.js";
import { ScopeParser } from "../parse/ScopeParser.js";
import { ToolANTLRParser } from "../parse/ToolANTLRParser.js";
import { BasicSemanticChecks } from "../semantics/BasicSemanticChecks.js";
import { RuleCollector } from "../semantics/RuleCollector.js";
import { AttributeDict } from "../tool/AttributeDict.js";
import { DictType } from "../tool/DictType.js";
import { ErrorType } from "../tool/ErrorType.js";
import { Grammar } from "../tool/Grammar.js";
import { GrammarTransformPipeline } from "../tool/GrammarTransformPipeline.js";
Expand All @@ -28,7 +31,6 @@ import { GrammarRootAST } from "../tool/ast/GrammarRootAST.js";
import { RuleAST } from "../tool/ast/RuleAST.js";
import { LeftRecursiveRuleAltInfo } from "./LeftRecursiveRuleAltInfo.js";
import { LeftRecursiveRuleAnalyzer } from "./LeftRecursiveRuleAnalyzer.js";
import type { SupportedLanguage } from "../codegen/CodeGenerator.js";

/**
* Remove left-recursive rule refs, add precedence args to recursive rule refs.
Expand Down Expand Up @@ -113,14 +115,17 @@ export class LeftRecursiveRuleTransformer {
}

// replace old rule's AST; first create text of altered rule
const RULES = ast.getFirstChildWithType(ANTLRv4Parser.RULES) as GrammarAST;
const RULES = ast.getFirstChildWithType(ANTLRv4Parser.RULE_REF) as GrammarAST;
const newRuleText = leftRecursiveRuleWalker.getArtificialOpPrecRule();

// now parse within the context of the grammar that originally created
// the AST we are transforming. This could be an imported grammar so
// we cannot just reference this.g because the role might come from
// the imported grammar and not the root grammar (this.g)
const t = this.parseArtificialRule(prevRuleAST.g, newRuleText);
if (t === undefined) {
return false;
}

// Reuse the name token from the original AST since it refers to the proper source location in the original
// grammar.
Expand All @@ -146,74 +151,77 @@ export class LeftRecursiveRuleTransformer {

// track recursive alt info for codegen
r.recPrimaryAlts = new Array<LeftRecursiveRuleAltInfo>();
r.recPrimaryAlts.addAll(leftRecursiveRuleWalker.prefixAndOtherAlts);
if (r.recPrimaryAlts.isEmpty()) {
this.tool.errMgr.grammarError(ErrorType.NO_NON_LR_ALTS, this.g.fileName, (r.ast.getChild(0) as GrammarAST).getToken(), r.name);
r.recPrimaryAlts.push(...leftRecursiveRuleWalker.prefixAndOtherAlts);
if (r.recPrimaryAlts.length === 0) {
this.tool.errMgr.grammarError(ErrorType.NO_NON_LR_ALTS, this.g.fileName,
(r.ast.getChild(0) as GrammarAST).getToken(), r.name);
}

r.recOpAlts = new OrderedHashMap<number, LeftRecursiveRuleAltInfo>();
r.recOpAlts.putAll(leftRecursiveRuleWalker.binaryAlts);
r.recOpAlts.putAll(leftRecursiveRuleWalker.ternaryAlts);
r.recOpAlts.putAll(leftRecursiveRuleWalker.suffixAlts);
r.recOpAlts = new Map<number, LeftRecursiveRuleAltInfo>();
leftRecursiveRuleWalker.binaryAlts.forEach((value, key) => {
r.recOpAlts.set(key, value);
});

leftRecursiveRuleWalker.ternaryAlts.forEach((value, key) => {
r.recOpAlts.set(key, value);
});

leftRecursiveRuleWalker.suffixAlts.forEach((value, key) => {
r.recOpAlts.set(key, value);
});

// walk alt info records and set their altAST to point to appropriate ALT subtree
// from freshly created AST
this.setAltASTPointers(r, t);

// update Rule to just one alt and add prec alt
const arg = r.ast.getFirstChildWithType(ANTLRv4Parser.ARG_ACTION) as ActionAST;
const arg = r.ast.getFirstChildWithType(ANTLRv4Parser.BEGIN_ARGUMENT) as ActionAST | null;
if (arg !== null) {
r.args = ScopeParser.parseTypedArgList(arg, arg.getText(), this.g);
r.args.type = AttributeDict.DictType.ARG;
r.args = ScopeParser.parseTypedArgList(arg, arg.getText()!, this.g);
r.args.type = DictType.ARG;
r.args.ast = arg;
arg.resolver = r.alt[1]; // todo: isn't this Rule or something?
}

// define labels on recursive rule refs we delete; they don't point to nodes of course
// these are so $label in action translation works
for (const pair of leftRecursiveRuleWalker.leftRecursiveRuleRefLabels) {
const labelNode = pair.a;
const labelOpNode = labelNode.getParent() as GrammarAST;
for (const [ast, _] of leftRecursiveRuleWalker.leftRecursiveRuleRefLabels) {
const labelOpNode = ast.getParent() as GrammarAST;
const elementNode = labelOpNode.getChild(1) as GrammarAST;
const lp = new LabelElementPair(this.g, labelNode, elementNode, labelOpNode.getType());
r.alt[1].labelDefs.map(labelNode.getText(), lp);
const lp = new LabelElementPair(this.g, ast, elementNode, labelOpNode.getType());
r.alt[1].labelDefs.set(ast.getText()!, [lp]);
}
// copy to rule from walker
r.leftRecursiveRuleRefLabels = leftRecursiveRuleWalker.leftRecursiveRuleRefLabels;

this.tool.logInfo("grammar", "added: " + t.toStringTree());
this.tool.logInfo({ component: "grammar", msg: "added: " + t.toStringTree() });

return true;
}

public parseArtificialRule(/* final */ g: Grammar, ruleText: string): RuleAST {
const lexer = new ANTLRLexer(new ANTLRStringStream(ruleText));
const adaptor = new GrammarASTAdaptor(lexer.getCharStream());
public parseArtificialRule(g: Grammar, ruleText: string): RuleAST | undefined {
const stream = CharStream.fromString(ruleText);
const lexer = new ANTLRv4Lexer(stream);
const adaptor = new GrammarASTAdaptor(stream);
const tokens = new CommonTokenStream(lexer);
lexer.tokens = tokens;
//lexer.tokens = tokens;
const p = new ToolANTLRParser(tokens, this.tool);
p.setTreeAdaptor(adaptor);
let ruleStart = null;
//p.setTreeAdaptor(adaptor);
const ruleStart = null;
try {
const r = p.rule();
/*const r = p.rule();
const tree = r.getTree() as RuleAST;
ruleStart = r.getStart() as Token;
GrammarTransformPipeline.setGrammarPtr(g, tree);
GrammarTransformPipeline.augmentTokensWithOriginalPosition(g, tree);
return tree;
return tree;*/
} catch (e) {
if (e instanceof Exception) {
this.tool.errMgr.toolError(ErrorType.INTERNAL_ERROR,
e,
ruleStart,
"error parsing rule created during left-recursion detection: " + ruleText);
} else {
throw e;
}
this.tool.errMgr.toolError(ErrorType.INTERNAL_ERROR, e, ruleStart,
"error parsing rule created during left-recursion detection: " + ruleText);
}

return null;
return undefined;
}

/**
Expand All @@ -233,26 +241,22 @@ export class LeftRecursiveRuleTransformer {
* </pre>
*/
public setAltASTPointers(r: LeftRecursiveRule, t: RuleAST): void {
// System.out.println("RULE: "+t.toStringTree());
const ruleBlk = t.getFirstChildWithType(ANTLRv4Parser.BLOCK) as BlockAST;
const ruleBlk = t.getFirstChildWithType(ANTLRv4Parser.LPAREN) as BlockAST;
const mainAlt = ruleBlk.getChild(0) as AltAST;
const primaryBlk = mainAlt.getChild(0) as BlockAST;
const opsBlk = mainAlt.getChild(1).getChild(0) as BlockAST; // (* BLOCK ...)
for (let i = 0; i < r.recPrimaryAlts.size(); i++) {
const altInfo = r.recPrimaryAlts.get(i);
const opsBlk = mainAlt.getChild(1)!.getChild(0) as BlockAST; // (* BLOCK ...)
for (let i = 0; i < r.recPrimaryAlts.length; i++) {
const altInfo = r.recPrimaryAlts[i];
altInfo.altAST = primaryBlk.getChild(i) as AltAST;
altInfo.altAST.leftRecursiveAltInfo = altInfo;
altInfo.originalAltAST.leftRecursiveAltInfo = altInfo;
// altInfo.originalAltAST.parent = altInfo.altAST.parent;
// System.out.println(altInfo.altAST.toStringTree());
altInfo.originalAltAST!.leftRecursiveAltInfo = altInfo;
}
for (let i = 0; i < r.recOpAlts.size(); i++) {
const altInfo = r.recOpAlts.getElement(i);

for (let i = 0; i < r.recOpAlts.size; i++) {
const altInfo = r.recOpAlts.get(i)!;
altInfo.altAST = opsBlk.getChild(i) as AltAST;
altInfo.altAST.leftRecursiveAltInfo = altInfo;
altInfo.originalAltAST.leftRecursiveAltInfo = altInfo;
// altInfo.originalAltAST.parent = altInfo.altAST.parent;
// System.out.println(altInfo.altAST.toStringTree());
altInfo.originalAltAST!.leftRecursiveAltInfo = altInfo;
}
}

Expand Down
Loading

0 comments on commit 8415551

Please sign in to comment.