diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index 8ac7ecbec..68b395014 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -18,7 +18,7 @@ from ..utils import logger, OrderedSet, dedup_list from .grammar_analysis import GrammarAnalyzer from ..grammar import NonTerminal -from .earley_common import Item +from .earley_common import Item, TransitiveItem from .earley_forest import ForestSumVisitor, SymbolNode, StableSymbolNode, TokenNode, ForestToParseTree if TYPE_CHECKING: @@ -75,7 +75,7 @@ def __init__(self, lexer_conf: 'LexerConf', parser_conf: 'ParserConf', term_matc self.term_matcher = term_matcher - def predict_and_complete(self, i, to_scan, columns, transitives, node_cache): + def predict_and_complete(self, i, to_scan, columns, transitives, node_cache, start_symbol=None): """The core Earley Predictor and Completer. At each stage of the input, we handling any completed items (things @@ -86,6 +86,70 @@ def predict_and_complete(self, i, to_scan, columns, transitives, node_cache): # Held Completions (H in E.Scotts paper). held_completions = {} + def is_quasi_complete(item): + quasi = item.advance() + while not quasi.is_complete: + if quasi.expect not in self.NULLABLE: + return False + if quasi.rule.origin == start_symbol and quasi.expect == start_symbol: + return False + quasi = quasi.advance() + return True + + def create_leo_transitives(origin, start): + visited = set() + to_create = [] + trule = None + previous = None + + ### Recursively walk backwards through the Earley sets until we find the + # first transitive candidate. If this is done continuously, we shouldn't + # have to walk more than 1 hop. + while True: + if origin in transitives[start]: + previous = trule = transitives[start][origin] + break + + if not self.FIRST[origin]: + break + + if origin in self.NULLABLE: + break + + candidates = [c for c in columns[start] if c.expect is not None and c.expect == origin] + if len(candidates) != 1: + break + originator = candidates[0] + + if originator in visited: + break + + visited.add(originator) + if not is_quasi_complete(originator): + break + + trule = originator.advance() + if originator.start != start: + visited.clear() + + to_create.append((origin, start, originator)) + origin = originator.rule.origin + start = originator.start + + # If a suitable Transitive candidate is not found, bail. + if trule is None: + return + + ### Now walk forwards and create Transitive Items in each set we walked through; + # and link each transitive item to the next set forwards. + while to_create: + origin, start, originator = to_create.pop() + if previous is not None: + titem = previous.next_titem = TransitiveItem(origin, trule, originator, previous.column) + else: + titem = TransitiveItem(origin, trule, originator, start) + previous = transitives[start][origin] = titem + column = columns[i] # R (items) = Ei (column.items) items = deque(column) @@ -99,24 +163,23 @@ def predict_and_complete(self, i, to_scan, columns, transitives, node_cache): item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label)) item.node.add_family(item.s, item.rule, item.start, None, None) - # create_leo_transitives(item.rule.origin, item.start) + create_leo_transitives(item.rule.origin, item.start) ###R Joop Leo right recursion Completer if item.rule.origin in transitives[item.start]: - transitive = transitives[item.start][item.s] - if transitive.previous in transitives[transitive.column]: - root_transitive = transitives[transitive.column][transitive.previous] - else: - root_transitive = transitive + transitive = transitives[item.start][item.rule.origin] + # Every titem is inserted into transitives[col][previous], + # so this lookup always finds the chain's root (or transitive + # itself when it is the root). + root_transitive = transitives[transitive.column][transitive.previous] new_item = Item(transitive.rule, transitive.ptr, transitive.start) label = (root_transitive.s, root_transitive.start, i) new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label)) new_item.node.add_path(root_transitive, item.node) - if new_item.expect in self.TERMINALS: - # Add (B :: aC.B, h, y) to Q - to_scan.add(new_item) - elif new_item not in column: + # new_item.expect cannot be a terminal: is_quasi_complete + # required the post-recursion suffix to be all-nullable. + if new_item not in column: # Add (B :: aC.B, h, y) to Ei and R column.add(new_item) items.append(new_item) @@ -127,8 +190,7 @@ def predict_and_complete(self, i, to_scan, columns, transitives, node_cache): # any predictions that result, that themselves require empty. Avoids # infinite recursion on empty symbols. # held_completions is 'H' in E.Scott's paper. - is_empty_item = item.start == i - if is_empty_item: + if item.start == i: held_completions[item.rule.origin] = item.node originators = [originator for originator in columns[item.start] if originator.expect is not None and originator.expect == item.s] @@ -169,22 +231,6 @@ def predict_and_complete(self, i, to_scan, columns, transitives, node_cache): def _parse(self, lexer, columns, to_scan, start_symbol=None): - def is_quasi_complete(item): - if item.is_complete: - return True - - quasi = item.advance() - while not quasi.is_complete: - if quasi.expect not in self.NULLABLE: - return False - if quasi.rule.origin == start_symbol and quasi.expect == start_symbol: - return False - quasi = quasi.advance() - return True - - # def create_leo_transitives(origin, start): - # ... # removed at commit 4c1cfb2faf24e8f8bff7112627a00b94d261b420 - def scan(i, token, to_scan): """The core Earley Scanner. @@ -246,7 +292,7 @@ def scan(i, token, to_scan): i = 0 node_cache = {} for token in lexer.lex(expects): - self.predict_and_complete(i, to_scan, columns, transitives, node_cache) + self.predict_and_complete(i, to_scan, columns, transitives, node_cache, start_symbol) to_scan, node_cache = scan(i, token, to_scan) i += 1 @@ -254,7 +300,7 @@ def scan(i, token, to_scan): expects.clear() expects |= {i.expect for i in to_scan} - self.predict_and_complete(i, to_scan, columns, transitives, node_cache) + self.predict_and_complete(i, to_scan, columns, transitives, node_cache, start_symbol) ## Column is now the final column in the parse. assert i == len(columns)-1 diff --git a/lark/parsers/earley_common.py b/lark/parsers/earley_common.py index 0ea2d4fa9..faf00f175 100644 --- a/lark/parsers/earley_common.py +++ b/lark/parsers/earley_common.py @@ -38,5 +38,22 @@ def __repr__(self): return '%s (%d)' % (symbol, self.start) -# class TransitiveItem(Item): -# ... # removed at commit 4c1cfb2faf24e8f8bff7112627a00b94d261b420 +class TransitiveItem(Item): + """A Joop Leo transitive item, for shortcutting right-recursive completions.""" + __slots__ = ('recognized', 'reduction', 'column', 'next_titem') + + def __init__(self, recognized, trule, originator, start): + super(TransitiveItem, self).__init__(trule.rule, trule.ptr, trule.start) + self.recognized = recognized + self.reduction = originator + self.column = start + self.next_titem = None + self._hash = hash((self.s, self.start, self.recognized)) + + def __eq__(self, other): + if not isinstance(other, TransitiveItem): + return False + return self is other or (type(self.s) == type(other.s) and self.s == other.s and self.start == other.start and self.recognized == other.recognized) + + def __hash__(self): + return self._hash diff --git a/lark/parsers/earley_forest.py b/lark/parsers/earley_forest.py index c60f3a6b5..39ceff10f 100644 --- a/lark/parsers/earley_forest.py +++ b/lark/parsers/earley_forest.py @@ -67,13 +67,27 @@ def add_path(self, transitive, node): self.paths.add((transitive, node)) def load_paths(self): + # Share canonical intermediate symbol nodes across multi-hop Leo chains: + # SPPF identity is (s, start, end), so any path whose node matches the + # expected intermediate's label is reused as the canonical node, and other + # paths targeting the same label merge into it via add_path. + node_cache = {(node.s, node.start, node.end): node for (_, node) in self.paths} for transitive, node in self.paths: + next_node = node if transitive.next_titem is not None: - vn = type(self)(transitive.next_titem.s, transitive.next_titem.start, self.end) - vn.add_path(transitive.next_titem, node) - self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, transitive.reduction.start, transitive.reduction.node, vn) - else: - self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, transitive.reduction.start, transitive.reduction.node, node) + label = (transitive.next_titem.reduction.rule.origin, + transitive.next_titem.reduction.start, + self.end) + if label != (node.s, node.start, node.end): + if label in node_cache: + # Canonical already added as right-side by an earlier path; just merge. + node_cache[label].add_path(transitive.next_titem, node) + continue + next_node = node_cache[label] = type(self)(*label) + next_node.add_path(transitive.next_titem, node) + + self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, + transitive.reduction.start, transitive.reduction.node, next_node) self.paths_loaded = True @property diff --git a/lark/parsers/xearley.py b/lark/parsers/xearley.py index f334ed199..0894c5630 100644 --- a/lark/parsers/xearley.py +++ b/lark/parsers/xearley.py @@ -156,7 +156,7 @@ def scan(i, to_scan): i = 0 node_cache = {} for token in stream: - self.predict_and_complete(i, to_scan, columns, transitives, node_cache) + self.predict_and_complete(i, to_scan, columns, transitives, node_cache, start_symbol) to_scan, node_cache = scan(i, to_scan) @@ -167,7 +167,7 @@ def scan(i, to_scan): text_column += 1 i += 1 - self.predict_and_complete(i, to_scan, columns, transitives, node_cache) + self.predict_and_complete(i, to_scan, columns, transitives, node_cache, start_symbol) ## Column is now the final column in the parse. assert i == len(columns)-1 diff --git a/tests/test_parser.py b/tests/test_parser.py index 533ae7890..9cb25dac4 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1070,6 +1070,75 @@ def test_consistent_derivation_order1(self): n = Tree('a', []) assert tree == Tree('start', [n, n]) + def test_earley_jeo_right_recursion(self): + """Joop Leo optimization should correctly handle right-recursive grammars""" + grammar = r""" + start: "x" start | "x" + """ + l = Lark(grammar, parser='earley', lexer=LEXER) + res = l.parse('xxx') + self.assertEqual(res, Tree('start', [Tree('start', [Tree('start', [])])])) + + def test_earley_jeo_multi_start_right_recursion(self): + """Joop Leo optimization should work correctly with multiple start symbols (issue #397)""" + grammar = r""" + a: "x" a | "x" + b: "y" b | "y" + """ + l = Lark(grammar, parser='earley', lexer=LEXER, start=['a', 'b']) + + res_a = l.parse('xxx', start='a') + self.assertEqual(res_a, Tree('a', [Tree('a', [Tree('a', [])])])) + + res_b = l.parse('yyy', start='b') + self.assertEqual(res_b, Tree('b', [Tree('b', [Tree('b', [])])])) + + def test_earley_jeo_chain_stops_at_nonnullable_suffix(self): + """Leo chain walks outward until the outer rule has a non-nullable + suffix after the recursive position, at which point is_quasi_complete + returns False (covers the non-NULLABLE check + break in + create_leo_transitives).""" + grammar = r""" + start: a "z" + a: "x" a | "y" + """ + l = Lark(grammar, parser='earley', lexer=LEXER) + tree = l.parse('xxyz') + self.assertEqual(tree, Tree('start', [Tree('a', [Tree('a', [Tree('a', [])])])])) + + def test_earley_jeo_chain_extends_through_nullable_suffix(self): + """Leo chain extends into an outer rule when the trailing symbols + are all nullable: is_quasi_complete traverses them via the + NULLABLE check + advance loop.""" + grammar = r""" + start: a opt + a: "x" a | "y" + opt: "q" | + """ + l = Lark(grammar, parser='earley', lexer=LEXER) + tree_no_opt = l.parse('xxy') + self.assertEqual(tree_no_opt, Tree('start', [Tree('a', [Tree('a', [Tree('a', [])])]), Tree('opt', [])])) + tree_with_opt = l.parse('xxyq') + self.assertEqual(tree_with_opt, Tree('start', [Tree('a', [Tree('a', [Tree('a', [])])]), Tree('opt', [])])) + + def test_earley_jeo_nullable_start_self_recursion(self): + """is_quasi_complete's start-symbol self-reference guard: when the + Leo chain walks into a rule of the form `start: ... start ...` + with `start` itself nullable, the guard returns False so the chain + doesn't pretend the start rule is quasi-complete.""" + grammar = r""" + start: a x start | + x: + a: "x" a | "y" + """ + l = Lark(grammar, parser='earley', lexer=LEXER) + self.assertEqual(l.parse(''), Tree('start', [])) + tree = l.parse('xxy') + self.assertEqual( + tree, + Tree('start', [Tree('a', [Tree('a', [Tree('a', [])])]), Tree('x', []), Tree('start', [])]) + ) + _NAME = "TestFullEarley" + LEXER.capitalize() _TestFullEarley.__name__ = _NAME globals()[_NAME] = _TestFullEarley