Skip to content

Commit

Permalink
All target classes are done now.
Browse files Browse the repository at this point in the history 
- More cleanup.
- Added MurmurHash implementation from my package jree.
- Worked on target classes.
- Updated 3rd party libs.

Less than 7000 errors left \o/
  • Loading branch information
@mike-lischke
mike-lischke committed Oct 4, 2024
1 parent bfad77c commit 35f9b2f
Show file tree
Hide file tree
Showing 31 changed files with 1,748 additions and 1,553 deletions.
1 change: 1 addition & 0 deletions cspell.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@
"nintervals",
"nmodes",
"nonambig",
"nongreedy",
"nqry",
"nrules",
"nsets",
Expand Down
165 changes: 89 additions & 76 deletions package-lock.json
View file

Large diffs are not rendered by default.

18 changes: 9 additions & 9 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,31 +10,31 @@
"author": "Mike Lischke",
"license": "MIT",
"dependencies": {
"antlr4ng": "3.0.5",
"antlr4ng": "3.0.7",
"commander": "12.1.0",
"fast-printf": "1.6.9",
"stringtemplate4ts": "1.0.1",
"unicode-properties": "1.4.1"
},
"devDependencies": {
"@eslint/js": "9.11.0",
"@eslint/js": "9.11.1",
"@stylistic/eslint-plugin": "2.8.0",
"@stylistic/eslint-plugin-ts": "2.8.0",
"@types/eslint__js": "8.42.3",
"@types/jest": "29.5.11",
"@types/node": "22.5.5",
"@types/jest": "29.5.13",
"@types/node": "22.7.4",
"@types/unicode-properties": "1.3.2",
"@typescript-eslint/eslint-plugin": "8.6.0",
"@typescript-eslint/parser": "8.6.0",
"@typescript-eslint/eslint-plugin": "8.7.0",
"@typescript-eslint/parser": "8.7.0",
"@unicode/unicode-15.1.0": "1.6.0",
"eslint": "9.11.0",
"eslint-plugin-jsdoc": "50.2.4",
"eslint": "9.11.1",
"eslint-plugin-jsdoc": "50.3.0",
"eslint-plugin-prefer-arrow": "1.2.3",
"jest": "29.7.0",
"ts-jest": "29.2.5",
"ts-node": "10.9.2",
"typescript": "5.6.2",
"typescript-eslint": "8.6.0"
"typescript-eslint": "8.7.0"
},
"scripts": {
"run": "node --no-warnings --loader ts-node/esm src/runner.ts",
Expand Down
2 changes: 1 addition & 1 deletion src/tree-walkers/GrammarTreeVisitor.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5582,7 +5582,7 @@ export class GrammarTreeVisitor extends TreeParser {
protected exitElementOption(tree: GrammarAST): void { }
}

// eslint-disable-next-line @typescript-eslint/no-namespace, no-redeclare

export namespace GrammarTreeVisitor {
export type grammarSpec_return = InstanceType<typeof GrammarTreeVisitor.grammarSpec_return>;
export type prequelConstructs_return = InstanceType<typeof GrammarTreeVisitor.prequelConstructs_return>;
Expand Down
4 changes: 2 additions & 2 deletions tool/src/org/antlr/v4/Tool.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import { IATNFactory } from "./automata/ATNFactory.js";
import { LexerATNFactory } from "./automata/LexerATNFactory.js";
import { ParserATNFactory } from "./automata/ParserATNFactory.js";
import { CodeGenPipeline } from "./codegen/CodeGenPipeline.js";
import { CodeGenerator } from "./codegen/CodeGenerator.js";
import { CodeGenerator, targetLanguages, type SupportedLanguage } from "./codegen/CodeGenerator.js";
import { grammarOptions } from "./grammar-options.js";
import { Graph } from "./misc/Graph.js";
import { GrammarASTAdaptor } from "./parse/GrammarASTAdaptor.js";
Expand Down Expand Up @@ -217,7 +217,7 @@ export class Tool {
return;
}

const codeGenerator = CodeGenerator.fromGrammar(g);
const codeGenerator = new CodeGenerator(this, g, g.getLanguage());

// BUILD ATN FROM AST
let factory: IATNFactory;
Expand Down
39 changes: 17 additions & 22 deletions tool/src/org/antlr/v4/analysis/AnalysisPipeline.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,7 @@ export class AnalysisPipeline {
public static disjoint(altLook: IntervalSet[]): boolean {
let collision = false;
const combined = new IntervalSet();
if (altLook === null) {
return false;
}

for (const look of altLook) {
if (!look) {
return false;
}

// lookahead must've computation failed
if (look.and(combined).length >= 0) {
collision = true;
Expand All @@ -46,7 +38,7 @@ export class AnalysisPipeline {

public process(): void {
// LEFT-RECURSION CHECK
const lr = new LeftRecursionDetector(this.g, this.g.#atn);
const lr = new LeftRecursionDetector(this.g, this.g.atn);
lr.check();
if (lr.listOfRecursiveCycles.length !== 0) {
return;
Expand All @@ -69,7 +61,7 @@ export class AnalysisPipeline {
}

const analyzer = new LL1Analyzer();
const look = analyzer.look(this.g.#atn, this.g.#atn.ruleToStartState[rule.index]!, undefined);
const look = analyzer.look(this.g.atn, this.g.atn.ruleToStartState[rule.index]!, undefined);
if (look.contains(Token.EPSILON)) {
this.g.tool.errMgr.grammarError(ErrorType.EPSILON_TOKEN, this.g.fileName,
(rule.ast.getChild(0) as GrammarAST).getToken(), rule.name);
Expand All @@ -78,23 +70,26 @@ export class AnalysisPipeline {
}

protected processParser(): void {
this.g.decisionLOOK = new Array<IntervalSet[]>(this.g.#atn.getNumberOfDecisions() + 1);
for (const s of this.g.#atn.decisionToState) {
this.g.tool.logInfo("LL1", "\nDECISION " + s.decision + " in rule " + this.g.getRule(s.ruleIndex).name);
let look: IntervalSet[];
this.g.decisionLOOK = new Array<IntervalSet[]>(this.g.atn.getNumberOfDecisions() + 1);
for (const s of this.g.atn.decisionToState) {
this.g.tool.logInfo({
component: "LL1",
msg: "\nDECISION " + s.decision + " in rule " + this.g.getRule(s.ruleIndex)?.name
});

let look: IntervalSet[] | undefined;
if (s.nonGreedy) { // nongreedy decisions can't be LL(1)
look = new Array<IntervalSet>(s.getNumberOfTransitions() + 1);
}
else {
const anal = new LL1Analyzer(this.g.#atn);
look = anal.getDecisionLookahead(s);
this.g.tool.logInfo("LL1", "look=" + Arrays.toString(look));
look = new Array<IntervalSet>(s.transitions.length + 1);
} else {
const anal = new LL1Analyzer();
look = anal.getDecisionLookahead(s) as IntervalSet[];
this.g.tool.logInfo({ component: "LL1", msg: "look=" + look });
}

/* assert s.decision + 1 >= g.decisionLOOK.size(); */
Utils.setSize(this.g.decisionLOOK, s.decision + 1);
this.g.decisionLOOK.set(s.decision, look);
this.g.tool.logInfo("LL1", "LL(1)? " + AnalysisPipeline.disjoint(look));
this.g.decisionLOOK[s.decision] = look;
this.g.tool.logInfo({ component: "LL1", msg: "LL(1)? " + AnalysisPipeline.disjoint(look) });
}
}
}
142 changes: 65 additions & 77 deletions tool/src/org/antlr/v4/analysis/LeftRecursionDetector.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,18 @@
* can be found in the LICENSE.txt file in the project root.
*/

import { ATN, ATNState, RuleStartState, RuleStopState, RuleTransition, Transition, HashSet, OrderedHashSet } from "antlr4ng";
import {
ATN, ATNState, HashSet, RuleStartState, RuleStopState, RuleTransition
} from "antlr4ng";

import { Grammar } from "../tool/Grammar.js";
import { Rule } from "../tool/Rule.js";

export class LeftRecursionDetector {
public atn: ATN;

/** Holds a list of cycles (sets of rule names). */
public listOfRecursiveCycles = new Array<Set<Rule>>();
public listOfRecursiveCycles = new Array<Rule[]>();
protected g: Grammar;

/**
Expand All @@ -27,7 +30,6 @@ export class LeftRecursionDetector {
}

public check(): void;

/**
* From state s, look for any transition to a rule that is currently
* being traced. When tracing r, visitedPerRuleCheck has r
Expand All @@ -40,82 +42,67 @@ export class LeftRecursionDetector {
* filling the cycles in listOfRecursiveCycles and also, as a
* side-effect, set leftRecursiveRules.
*/
public check(enclosingRule: Rule, s: ATNState, visitedStates: Set<ATNState>): boolean;
public check(...args: unknown[]): void | boolean {
switch (args.length) {
case 0: {

for (const start of this.atn.ruleToStartState) {
//System.out.print("check "+start.rule.name);
this.rulesVisitedPerRuleCheck.clear();
this.rulesVisitedPerRuleCheck.add(start);
//FASerializer ser = new FASerializer(atn.g, start);
//System.out.print(":\n"+ser+"\n");

this.check(this.g.getRule(start.ruleIndex), start, new HashSet<ATNState>());
}
//System.out.println("cycles="+listOfRecursiveCycles);
if (!this.listOfRecursiveCycles.isEmpty()) {
this.g.tool.errMgr.leftRecursionCycles(this.g.fileName, this.listOfRecursiveCycles);
}
public check(enclosingRule: Rule, s: ATNState, visitedStates: HashSet<ATNState>): boolean;
public check(...args: unknown[]): undefined | boolean {
if (args.length === 0) {
for (const start of this.atn.ruleToStartState) {
this.rulesVisitedPerRuleCheck.clear();
this.rulesVisitedPerRuleCheck.add(start!);

this.check(this.g.getRule(start!.ruleIndex)!, start!, new HashSet<ATNState>());
}

break;
if (this.listOfRecursiveCycles.length > 0) {
this.g.tool.errMgr.leftRecursionCycles(this.g.fileName, this.listOfRecursiveCycles);
}

case 3: {
const [enclosingRule, s, visitedStates] = args as [Rule, ATNState, Set<ATNState>];
return;
}

if (s instanceof RuleStopState) {
return true;
}
const [enclosingRule, s, visitedStates] = args as [Rule, ATNState, HashSet<ATNState>];

if (visitedStates.contains(s)) {
return false;
}
if (s instanceof RuleStopState) {
return true;
}

visitedStates.add(s);

//System.out.println("visit "+s);
const n = s.getNumberOfTransitions();
let stateReachesStopState = false;
for (let i = 0; i < n; i++) {
const t = s.transition(i);
if (t instanceof RuleTransition) {
const rt = t;
const r = this.g.getRule(rt.ruleIndex);
if (this.rulesVisitedPerRuleCheck.contains(t.target as RuleStartState)) {
this.addRulesToCycle(enclosingRule, r);
}
else {
// must visit if not already visited; mark target, pop when done
this.rulesVisitedPerRuleCheck.add(t.target as RuleStartState);
// send new visitedStates set per rule invocation
const nullable = this.check(r, t.target, new HashSet<ATNState>());
// we're back from visiting that rule
this.rulesVisitedPerRuleCheck.remove(t.target as RuleStartState);
if (nullable) {
stateReachesStopState |= this.check(enclosingRule, rt.followState, visitedStates);
}
}
}
else {
if (t.isEpsilon()) {
stateReachesStopState |= this.check(enclosingRule, t.target, visitedStates);
}
}
if (visitedStates.contains(s)) {
return false;
}

// else ignore non-epsilon transitions
visitedStates.add(s);

const n = s.transitions.length;
let stateReachesStopState = false;
for (let i = 0; i < n; i++) {
const t = s.transitions[i];
if (t instanceof RuleTransition) {
const rt = t;
const r = this.g.getRule(rt.ruleIndex)!;
if (this.rulesVisitedPerRuleCheck.contains(t.target as RuleStartState)) {
this.addRulesToCycle(enclosingRule, r);
} else {
// must visit if not already visited; mark target, pop when done
this.rulesVisitedPerRuleCheck.add(t.target as RuleStartState);

// send new visitedStates set per rule invocation
const nullable = this.check(r, t.target, new HashSet<ATNState>());

// we're back from visiting that rule
this.rulesVisitedPerRuleCheck.remove(t.target as RuleStartState);
if (nullable) {
stateReachesStopState ||= this.check(enclosingRule, rt.followState, visitedStates);
}
}
} else {
if (t.isEpsilon) {
stateReachesStopState ||= this.check(enclosingRule, t.target, visitedStates);
}

return stateReachesStopState;

break;
}

default: {
throw new java.lang.IllegalArgumentException(S`Invalid number of arguments`);
}
// else ignore non-epsilon transitions
}

return stateReachesStopState;
}

/**
Expand All @@ -125,24 +112,25 @@ export class LeftRecursionDetector {
* cycle.
*/
protected addRulesToCycle(enclosingRule: Rule, targetRule: Rule): void {
//System.err.println("left-recursion to "+targetRule.name+" from "+enclosingRule.name);
let foundCycle = false;
for (const rulesInCycle of this.listOfRecursiveCycles) {
// ensure both rules are in same cycle
if (rulesInCycle.contains(targetRule)) {
rulesInCycle.add(enclosingRule);
if (rulesInCycle.includes(targetRule)) {
rulesInCycle.push(enclosingRule);
foundCycle = true;
}
if (rulesInCycle.contains(enclosingRule)) {
rulesInCycle.add(targetRule);

if (rulesInCycle.includes(enclosingRule)) {
rulesInCycle.push(targetRule);
foundCycle = true;
}
}

if (!foundCycle) {
const cycle = new OrderedHashSet<Rule>();
cycle.add(targetRule);
cycle.add(enclosingRule);
this.listOfRecursiveCycles.add(cycle);
const cycle = new Array<Rule>();
cycle.push(targetRule);
cycle.push(enclosingRule);
this.listOfRecursiveCycles.push(cycle);
}
}
}
Loading

0 comments on commit 35f9b2f

Please sign in to comment.