Skip to content

Commit eef6441

Browse files
gregshgregsh
committed
Parser/Generator: introduce "upper" rules: change AST type on the go
1 parent bd5d401 commit eef6441

File tree

15 files changed

+285
-23
lines changed

15 files changed

+285
-23
lines changed

‎gen/org/intellij/grammar/parser/GrammarParser.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -550,7 +550,7 @@ public static boolean literal_expression(PsiBuilder builder, int level) {
550550

551551
/* ********************************************************** */
552552
// 'private' | 'external' | 'meta'
553-
// | 'inner' | 'left' | 'fake'
553+
// | 'inner' | 'left' | 'upper' | 'fake'
554554
public static boolean modifier(PsiBuilder builder, int level) {
555555
if (!recursion_guard_(builder, level, "modifier")) return false;
556556
boolean result;
@@ -560,6 +560,7 @@ public static boolean modifier(PsiBuilder builder, int level) {
560560
if (!result) result = consumeToken(builder, "meta");
561561
if (!result) result = consumeToken(builder, "inner");
562562
if (!result) result = consumeToken(builder, "left");
563+
if (!result) result = consumeToken(builder, "upper");
563564
if (!result) result = consumeToken(builder, "fake");
564565
exit_section_(builder, level, marker, BNF_MODIFIER, result, false, null);
565566
return result;

‎grammars/Grammar.bnf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ private grammar_element_recover::=!('{'|rule_start)
6464
rule ::= rule_start expression attrs? ';'? {pin=2}
6565
private rule_start ::= modifier* id '::='
6666
modifier ::= 'private' | 'external' | 'meta'
67-
| 'inner' | 'left' | 'fake'
67+
| 'inner' | 'left' | 'upper' | 'fake'
6868

6969
attrs ::= '{' attr * '}' {pin=1}
7070
attr ::= attr_start attr_value ';'? {

‎support/org/intellij/grammar/generator/ParserGenerator.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -573,6 +573,7 @@ void generateNode(BnfRule rule, BnfExpression initialNode, String funcName, Set<
573573
boolean isPrivate = !(isRule || firstNonTrivial) || Rule.isPrivate(rule) || myGrammarRoot.equals(rule.getName());
574574
boolean isLeft = firstNonTrivial && Rule.isLeft(rule);
575575
boolean isLeftInner = isLeft && (isPrivate || Rule.isInner(rule));
576+
boolean isBranch = !isPrivate && Rule.isUpper(rule);
576577
final String recoverWhile = firstNonTrivial ? ObjectUtils.chooseNotNull(
577578
getAttribute(rule, KnownAttribute.RECOVER_WHILE.alias("recoverUntil")),
578579
getAttribute(rule, KnownAttribute.RECOVER_WHILE)) : null;
@@ -642,6 +643,7 @@ void generateNode(BnfRule rule, BnfExpression initialNode, String funcName, Set<
642643
else if (isLeft) modifierList.add("_LEFT_");
643644
if (type == BNF_OP_AND) modifierList.add("_AND_");
644645
else if (type == BNF_OP_NOT) modifierList.add("_NOT_");
646+
if (isBranch) modifierList.add("_UPPER_");
645647
if (modifierList.isEmpty() && (pinned || frameName != null)) modifierList.add("_NONE_");
646648
boolean sectionRequired = !alwaysTrue || !isPrivate || isLeft || recoverWhile != null;
647649
boolean sectionRequiredSimple = sectionRequired && modifierList.isEmpty() && recoverWhile == null;

‎support/org/intellij/grammar/generator/ParserGeneratorUtil.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -615,6 +615,10 @@ public static boolean isFake(BnfRule node) {
615615
return hasModifier(node, "fake");
616616
}
617617

618+
public static boolean isUpper(BnfRule node) {
619+
return hasModifier(node, "upper");
620+
}
621+
618622
private static boolean hasModifier(BnfRule node, String s) {
619623
for (BnfModifier modifier : node.getModifierList()) {
620624
if (s.equals(modifier.getText())) return true;

‎support/org/intellij/grammar/livePreview/LivePreviewParser.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ protected boolean expression(PsiBuilder builder,
163163
boolean isPrivate = !(isRule || firstNonTrivial) || ParserGeneratorUtil.Rule.isPrivate(rule) || myGrammarRoot == rule;
164164
boolean isLeft = firstNonTrivial && ParserGeneratorUtil.Rule.isLeft(rule);
165165
boolean isLeftInner = isLeft && (isPrivate || ParserGeneratorUtil.Rule.isInner(rule));
166+
boolean isBranch = !isPrivate && Rule.isUpper(rule);
166167
String recoverWhile = firstNonTrivial ? getAttribute(rule, KnownAttribute.RECOVER_WHILE) : null;
167168
boolean canCollapse = !isPrivate && (!isLeft || isLeftInner) && firstNonTrivial && myGraphHelper.canCollapse(rule);
168169

@@ -211,6 +212,7 @@ protected boolean expression(PsiBuilder builder,
211212
else if (isLeft) modifiers |= _LEFT_;
212213
if (type == BNF_OP_AND) modifiers |= _AND_;
213214
else if (type == BNF_OP_NOT) modifiers |= _NOT_;
215+
if (isBranch) modifiers |= _UPPER_;
214216

215217
PsiBuilder.Marker marker_ = null;
216218
boolean sectionRequired = !alwaysTrue || !isPrivate || isLeft || recoverWhile != null;

‎support/org/intellij/grammar/parser/GeneratedParserUtilBase.java

Lines changed: 28 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ public static boolean invalid_left_marker_guard_(PsiBuilder builder, PsiBuilder.
113113
if (!goodMarker) return false;
114114
ErrorState state = ErrorState.get(builder);
115115

116-
return !state.frameStack.isEmpty();
116+
return state.currentFrame != null;
117117
}
118118

119119
public static TokenSet create_token_set_(IElementType... tokenTypes) {
@@ -393,6 +393,7 @@ public static boolean isWhitespaceOrComment(@NotNull PsiBuilder builder, @Nullab
393393
public static final int _LEFT_INNER_ = 0x4;
394394
public static final int _AND_ = 0x8;
395395
public static final int _NOT_ = 0x10;
396+
public static final int _UPPER_ = 0x20;
396397

397398
// simple enter/exit methods pair that doesn't require frame object
398399
public static PsiBuilder.Marker enter_section_(PsiBuilder builder) {
@@ -403,7 +404,7 @@ public static void exit_section_(PsiBuilder builder,
403404
PsiBuilder.Marker marker,
404405
@Nullable IElementType elementType,
405406
boolean result) {
406-
close_marker_impl_(ErrorState.get(builder).frameStack.peekLast(), marker, elementType, result);
407+
close_marker_impl_(ErrorState.get(builder).currentFrame, marker, elementType, result);
407408
}
408409

409410
// complex enter/exit methods pair with frame object
@@ -416,7 +417,7 @@ public static PsiBuilder.Marker enter_section_(PsiBuilder builder, int level, in
416417
private static void enter_section_impl_(PsiBuilder builder, int level, int modifiers, @Nullable String frameName) {
417418
ErrorState state = ErrorState.get(builder);
418419
Frame frame = state.FRAMES.alloc().init(builder, state, level, modifiers, frameName);
419-
Frame prevFrame = state.frameStack.peekLast();
420+
Frame prevFrame = state.currentFrame;
420421
if (prevFrame != null && prevFrame.errorReportedAt > frame.position) {
421422
// report error for previous unsuccessful frame
422423
reportError(builder, state, frame, null, true, false);
@@ -427,7 +428,7 @@ private static void enter_section_impl_(PsiBuilder builder, int level, int modif
427428
frame.leftMarker = left;
428429
}
429430
}
430-
state.frameStack.add(frame);
431+
state.currentFrame = frame;
431432
if ((modifiers & _AND_) != 0) {
432433
if (state.predicateCount == 0 && !state.predicateSign) {
433434
throw new AssertionError("Incorrect false predicate sign");
@@ -454,7 +455,9 @@ public static void exit_section_(PsiBuilder builder,
454455
@Nullable Parser eatMore) {
455456
ErrorState state = ErrorState.get(builder);
456457

457-
Frame frame = state.frameStack.pollLast();
458+
Frame frame = state.currentFrame;
459+
state.currentFrame = frame == null ? null : frame.parentFrame;
460+
if (frame != null && frame.branchType != null) elementType = frame.branchType;
458461
if (frame == null || level != frame.level) {
459462
LOG.error("Unbalanced error section: got " + frame + ", expected level " + level);
460463
if (frame != null) state.FRAMES.recycle(frame);
@@ -466,11 +469,11 @@ public static void exit_section_(PsiBuilder builder,
466469
close_marker_impl_(frame, marker, null, false);
467470
state.predicateCount--;
468471
if ((frame.modifiers & _NOT_) != 0) state.predicateSign = !state.predicateSign;
469-
state.FRAMES.recycle(frame);
470-
return;
471472
}
472-
close_frame_impl_(state, frame, builder, marker, elementType, result, pinned);
473-
exit_section_impl_(state, frame, builder, elementType, result, pinned, eatMore);
473+
else {
474+
close_frame_impl_(state, frame, builder, marker, elementType, result, pinned);
475+
exit_section_impl_(state, frame, builder, elementType, result, pinned, eatMore);
476+
}
474477
state.FRAMES.recycle(frame);
475478
}
476479

@@ -554,7 +557,7 @@ else if (lastErrorPos > initialPos) {
554557
}
555558
}
556559
// propagate errorReportedAt up the stack to avoid duplicate reporting
557-
Frame prevFrame = willFail && eatMore == null ? null : state.frameStack.peekLast();
560+
Frame prevFrame = willFail && eatMore == null ? null : state.currentFrame;
558561
if (prevFrame != null && prevFrame.errorReportedAt < frame.errorReportedAt) {
559562
prevFrame.errorReportedAt = frame.errorReportedAt;
560563
}
@@ -579,7 +582,11 @@ private static void close_frame_impl_(ErrorState state,
579582
}
580583
}
581584
if (result || pinned) {
582-
if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) {
585+
if ((frame.modifiers & _UPPER_) != 0) {
586+
marker.drop();
587+
frame.parentFrame.branchType = elementType;
588+
}
589+
else if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) {
583590
marker.done(elementType);
584591
frame.leftMarker.precede().done(((LighterASTNode)frame.leftMarker).getTokenType());
585592
frame.leftMarker.drop();
@@ -622,8 +629,8 @@ private static void close_marker_impl_(Frame frame, PsiBuilder.Marker marker, IE
622629
else {
623630
if (frame != null) {
624631
int position = ((PsiBuilderImpl.ProductionMarker)marker).getStartIndex();
625-
if (frame.errorReportedAt > position) {
626-
frame.errorReportedAt = frame.errorReportedAtPrev;
632+
if (frame.errorReportedAt > position && frame.parentFrame != null) {
633+
frame.errorReportedAt = frame.parentFrame.errorReportedAt;
627634
}
628635
}
629636
marker.rollbackTo();
@@ -636,7 +643,7 @@ public static boolean report_error_(PsiBuilder builder, boolean result) {
636643
}
637644

638645
public static void report_error_(PsiBuilder builder, ErrorState state, boolean advance) {
639-
Frame frame = state.frameStack.isEmpty()? null : state.frameStack.getLast();
646+
Frame frame = state.currentFrame;
640647
if (frame == null) {
641648
LOG.error("unbalanced enter/exit section call: got null");
642649
return;
@@ -755,7 +762,7 @@ public static class ErrorState {
755762
int predicateCount;
756763
boolean predicateSign = true;
757764
boolean suppressErrors;
758-
public final LinkedList<Frame> frameStack = new LinkedList<Frame>();
765+
public Frame currentFrame;
759766
public CompletionState completionState;
760767

761768
private boolean caseSensitive;
@@ -876,20 +883,22 @@ boolean typeExtends(IElementType child, IElementType parent) {
876883
}
877884

878885
public static class Frame {
886+
public Frame parentFrame;
879887
public int offset;
880888
public int position;
881889
public int level;
882890
public int modifiers;
883891
public String name;
884892
public int variantCount;
885893
public int errorReportedAt;
886-
public int errorReportedAtPrev;
887894
public PsiBuilder.Marker leftMarker;
895+
public IElementType branchType;
888896

889897
public Frame() {
890898
}
891899

892900
public Frame init(PsiBuilder builder, ErrorState state, int level_, int modifiers_, String name_) {
901+
parentFrame = state.currentFrame;
893902
offset = builder.getCurrentOffset();
894903
position = builder.rawTokenIndex();
895904
level = level_;
@@ -898,9 +907,8 @@ public Frame init(PsiBuilder builder, ErrorState state, int level_, int modifier
898907
variantCount = state.variants.size();
899908
errorReportedAt = -1;
900909

901-
Frame prev = state.frameStack.peekLast();
902-
errorReportedAtPrev = prev == null? -1 : prev.errorReportedAt;
903910
leftMarker = null;
911+
branchType = null;
904912
return this;
905913
}
906914

@@ -911,7 +919,8 @@ public String toString() {
911919
((modifiers & _LEFT_) != 0? "_LEFT_, ": "") +
912920
((modifiers & _LEFT_INNER_) != 0? "_LEFT_INNER_, ": "") +
913921
((modifiers & _AND_) != 0? "_AND_, ": "") +
914-
((modifiers & _NOT_) != 0? "_NOT_, ": "");
922+
((modifiers & _NOT_) != 0? "_NOT_, ": "") +
923+
((modifiers & _UPPER_) != 0 ? "_UPPER_, " : "");
915924
return String.format("{%s:%s:%d, %d, %s%s}", offset, position, level, errorReportedAt, mod, name);
916925
}
917926
}

‎testData/generator/BnfGrammar.expected.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -537,7 +537,7 @@ public static boolean literal_expression(PsiBuilder builder, int level) {
537537

538538
/* ********************************************************** */
539539
// 'private' | 'external' | 'meta'
540-
// | 'inner' | 'left' | 'fake'
540+
// | 'inner' | 'left' | 'upper' | 'fake'
541541
public static boolean modifier(PsiBuilder builder, int level) {
542542
if (!recursion_guard_(builder, level, "modifier")) return false;
543543
boolean result;
@@ -547,6 +547,7 @@ public static boolean modifier(PsiBuilder builder, int level) {
547547
if (!result) result = consumeToken(builder, "meta");
548548
if (!result) result = consumeToken(builder, "inner");
549549
if (!result) result = consumeToken(builder, "left");
550+
if (!result) result = consumeToken(builder, "upper");
550551
if (!result) result = consumeToken(builder, "fake");
551552
exit_section_(builder, level, marker, BNF_MODIFIER, result, false, null);
552553
return result;

‎testData/generator/UpperRules.PSI.expected.java

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
// ---- GeneratedTypes.java -----------------
2+
//header.txt
3+
package generated;
4+
5+
import com.intellij.psi.tree.IElementType;
6+
import com.intellij.psi.PsiElement;
7+
import com.intellij.lang.ASTNode;
8+
import generated.psi.impl.*;
9+
10+
public interface GeneratedTypes {
11+
12+
IElementType ABC = new IElementType("ABC", null);
13+
IElementType ABC_ONE = new IElementType("ABC_ONE", null);
14+
IElementType ABC_PIN = new IElementType("ABC_PIN", null);
15+
IElementType ABC_SEQ = new IElementType("ABC_SEQ", null);
16+
IElementType ABC_TWO = new IElementType("ABC_TWO", null);
17+
IElementType PREFIX = new IElementType("PREFIX", null);
18+
IElementType ROOT = new IElementType("ROOT", null);
19+
20+
IElementType A = new IElementType("A", null);
21+
IElementType B = new IElementType("B", null);
22+
IElementType C = new IElementType("C", null);
23+
}
24+
// ---- Abc.java -----------------
25+
//header.txt
26+
package generated.psi;
27+
28+
import java.util.List;
29+
import org.jetbrains.annotations.*;
30+
import com.intellij.psi.PsiElement;
31+
32+
public interface Abc extends PsiElement {
33+
34+
}
35+
// ---- AbcOne.java -----------------
36+
//header.txt
37+
package generated.psi;
38+
39+
import java.util.List;
40+
import org.jetbrains.annotations.*;
41+
import com.intellij.psi.PsiElement;
42+
43+
public interface AbcOne extends PsiElement {
44+
45+
}
46+
// ---- AbcPin.java -----------------
47+
//header.txt
48+
package generated.psi;
49+
50+
import java.util.List;
51+
import org.jetbrains.annotations.*;
52+
import com.intellij.psi.PsiElement;
53+
54+
public interface AbcPin extends PsiElement {
55+
56+
@NotNull
57+
Prefix getPrefix();
58+
59+
}
60+
// ---- AbcSeq.java -----------------
61+
//header.txt
62+
package generated.psi;
63+
64+
import java.util.List;
65+
import org.jetbrains.annotations.*;
66+
import com.intellij.psi.PsiElement;
67+
68+
public interface AbcSeq extends PsiElement {
69+
70+
}
71+
// ---- AbcTwo.java -----------------
72+
//header.txt
73+
package generated.psi;
74+
75+
import java.util.List;
76+
import org.jetbrains.annotations.*;
77+
import com.intellij.psi.PsiElement;
78+
79+
public interface AbcTwo extends PsiElement {
80+
81+
}
82+
// ---- Prefix.java -----------------
83+
//header.txt
84+
package generated.psi;
85+
86+
import java.util.List;
87+
import org.jetbrains.annotations.*;
88+
import com.intellij.psi.PsiElement;
89+
90+
public interface Prefix extends PsiElement {
91+
92+
}
93+
// ---- Root.java -----------------
94+
//header.txt
95+
package generated.psi;
96+
97+
import java.util.List;
98+
import org.jetbrains.annotations.*;
99+
import com.intellij.psi.PsiElement;
100+
101+
public interface Root extends PsiElement {
102+
103+
@NotNull
104+
List<Abc> getAbcList();
105+
106+
@NotNull
107+
List<AbcPin> getAbcPinList();
108+
109+
@NotNull
110+
List<AbcSeq> getAbcSeqList();
111+
112+
}

‎testData/generator/UpperRules.bnf

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
generate=[visitor="no" root-rules="" psi-factory="no"]
3+
classHeader="//header.txt"
4+
tokens=[
5+
space='regexp:\s+'
6+
]
7+
8+
}
9+
10+
file ::= root
11+
12+
root ::= (abc | abc_pin | abc_seq) *
13+
14+
prefix ::= A
15+
16+
abc ::= prefix (abc_one | abc_two)
17+
upper abc_one ::= B
18+
upper abc_two ::= C
19+
20+
abc_pin ::= prefix (abc_one | abc_two) {pin=1}
21+
abc_seq ::= prefix abc_one abc_two

0 commit comments

Comments
 (0)