From 8b2013ebdcc8a8fc9ab6eb1299e0fd5fe3b4b4ac Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Mon, 1 Jun 2026 23:38:46 +0200 Subject: [PATCH 1/5] Re-enable Joop Leo right-recursion optimization (fixes #397) Leo was disabled when predict_and_complete was refactored into a method, breaking is_quasi_complete's access to start_symbol. This caused incorrect behavior with self-referential start symbols and multiple start symbols. - Restore TransitiveItem in earley_common.py - Add start_symbol parameter to predict_and_complete so is_quasi_complete can correctly guard against self-referential cycles - Move held_completions update before the Leo/regular branch so empty items populate it even when Leo fires (fixes nullable symbol parsing) - Guard create_leo_transitives with NULLABLE check: Leo must not shortcut nullable right-recursive symbols, as that bypasses the intermediate SPPF nodes where priority/disambiguation lives - Fix load_paths for multi-hop chains: use reduction.rule.origin instead of next_titem.s (which carries the outer rule's origin, not the intermediate symbol), and merge paths sharing the same intermediate symbol into one canonical SymbolNode to preserve SPPF sharing - Pass start_symbol through in xearley.py - Add tests for right-recursive Leo and multi-start-symbol correctness --- lark/parsers/earley.py | 104 ++++++++++++++++++++++++---------- lark/parsers/earley_common.py | 21 ++++++- lark/parsers/earley_forest.py | 32 ++++++++++- lark/parsers/xearley.py | 4 +- tests/test_parser.py | 23 ++++++++ 5 files changed, 147 insertions(+), 37 deletions(-) diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index 8ac7ecbec..93e76485d 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -18,7 +18,7 @@ from ..utils import logger, OrderedSet, dedup_list from .grammar_analysis import GrammarAnalyzer from ..grammar import NonTerminal -from .earley_common import Item +from .earley_common import Item, TransitiveItem from .earley_forest import ForestSumVisitor, SymbolNode, StableSymbolNode, TokenNode, ForestToParseTree if TYPE_CHECKING: @@ -75,7 +75,7 @@ def __init__(self, lexer_conf: 'LexerConf', parser_conf: 'ParserConf', term_matc self.term_matcher = term_matcher - def predict_and_complete(self, i, to_scan, columns, transitives, node_cache): + def predict_and_complete(self, i, to_scan, columns, transitives, node_cache, start_symbol=None): """The core Earley Predictor and Completer. At each stage of the input, we handling any completed items (things @@ -86,6 +86,66 @@ def predict_and_complete(self, i, to_scan, columns, transitives, node_cache): # Held Completions (H in E.Scotts paper). held_completions = {} + def is_quasi_complete(item): + if item.is_complete: + return True + quasi = item.advance() + while not quasi.is_complete: + if quasi.expect not in self.NULLABLE: + return False + if quasi.rule.origin == start_symbol and quasi.expect == start_symbol: + return False + quasi = quasi.advance() + return True + + def create_leo_transitives(origin, start): + visited = set() + to_create = [] + trule = None + previous = None + + while True: + if origin in transitives[start]: + previous = trule = transitives[start][origin] + break + + if not self.FIRST[origin]: + break + + if origin in self.NULLABLE: + break + + candidates = [c for c in columns[start] if c.expect is not None and c.expect == origin] + if len(candidates) != 1: + break + originator = candidates[0] + + if originator in visited: + break + + visited.add(originator) + if not is_quasi_complete(originator): + break + + trule = originator.advance() + if originator.start != start: + visited.clear() + + to_create.append((origin, start, originator)) + origin = originator.rule.origin + start = originator.start + + if trule is None: + return + + while to_create: + origin, start, originator = to_create.pop() + if previous is not None: + titem = previous.next_titem = TransitiveItem(origin, trule, originator, previous.column) + else: + titem = TransitiveItem(origin, trule, originator, start) + previous = transitives[start][origin] = titem + column = columns[i] # R (items) = Ei (column.items) items = deque(column) @@ -99,7 +159,16 @@ def predict_and_complete(self, i, to_scan, columns, transitives, node_cache): item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label)) item.node.add_family(item.s, item.rule, item.start, None, None) - # create_leo_transitives(item.rule.origin, item.start) + # Empty has 0 length. If we complete an empty symbol in a particular + # parse step, we need to be able to use that same empty symbol to complete + # any predictions that result, that themselves require empty. Avoids + # infinite recursion on empty symbols. + # held_completions is 'H' in E.Scott's paper. + # Must be updated before Leo fires, since Leo skips the regular completer. + if item.start == i: + held_completions[item.rule.origin] = item.node + + create_leo_transitives(item.rule.origin, item.start) ###R Joop Leo right recursion Completer if item.rule.origin in transitives[item.start]: @@ -122,15 +191,6 @@ def predict_and_complete(self, i, to_scan, columns, transitives, node_cache): items.append(new_item) ###R Regular Earley completer else: - # Empty has 0 length. If we complete an empty symbol in a particular - # parse step, we need to be able to use that same empty symbol to complete - # any predictions that result, that themselves require empty. Avoids - # infinite recursion on empty symbols. - # held_completions is 'H' in E.Scott's paper. - is_empty_item = item.start == i - if is_empty_item: - held_completions[item.rule.origin] = item.node - originators = [originator for originator in columns[item.start] if originator.expect is not None and originator.expect == item.s] for originator in originators: new_item = originator.advance() @@ -169,22 +229,6 @@ def predict_and_complete(self, i, to_scan, columns, transitives, node_cache): def _parse(self, lexer, columns, to_scan, start_symbol=None): - def is_quasi_complete(item): - if item.is_complete: - return True - - quasi = item.advance() - while not quasi.is_complete: - if quasi.expect not in self.NULLABLE: - return False - if quasi.rule.origin == start_symbol and quasi.expect == start_symbol: - return False - quasi = quasi.advance() - return True - - # def create_leo_transitives(origin, start): - # ... # removed at commit 4c1cfb2faf24e8f8bff7112627a00b94d261b420 - def scan(i, token, to_scan): """The core Earley Scanner. @@ -246,7 +290,7 @@ def scan(i, token, to_scan): i = 0 node_cache = {} for token in lexer.lex(expects): - self.predict_and_complete(i, to_scan, columns, transitives, node_cache) + self.predict_and_complete(i, to_scan, columns, transitives, node_cache, start_symbol) to_scan, node_cache = scan(i, token, to_scan) i += 1 @@ -254,7 +298,7 @@ def scan(i, token, to_scan): expects.clear() expects |= {i.expect for i in to_scan} - self.predict_and_complete(i, to_scan, columns, transitives, node_cache) + self.predict_and_complete(i, to_scan, columns, transitives, node_cache, start_symbol) ## Column is now the final column in the parse. assert i == len(columns)-1 diff --git a/lark/parsers/earley_common.py b/lark/parsers/earley_common.py index 0ea2d4fa9..faf00f175 100644 --- a/lark/parsers/earley_common.py +++ b/lark/parsers/earley_common.py @@ -38,5 +38,22 @@ def __repr__(self): return '%s (%d)' % (symbol, self.start) -# class TransitiveItem(Item): -# ... # removed at commit 4c1cfb2faf24e8f8bff7112627a00b94d261b420 +class TransitiveItem(Item): + """A Joop Leo transitive item, for shortcutting right-recursive completions.""" + __slots__ = ('recognized', 'reduction', 'column', 'next_titem') + + def __init__(self, recognized, trule, originator, start): + super(TransitiveItem, self).__init__(trule.rule, trule.ptr, trule.start) + self.recognized = recognized + self.reduction = originator + self.column = start + self.next_titem = None + self._hash = hash((self.s, self.start, self.recognized)) + + def __eq__(self, other): + if not isinstance(other, TransitiveItem): + return False + return self is other or (type(self.s) == type(other.s) and self.s == other.s and self.start == other.start and self.recognized == other.recognized) + + def __hash__(self): + return self._hash diff --git a/lark/parsers/earley_forest.py b/lark/parsers/earley_forest.py index c60f3a6b5..27afd52d6 100644 --- a/lark/parsers/earley_forest.py +++ b/lark/parsers/earley_forest.py @@ -67,11 +67,37 @@ def add_path(self, transitive, node): self.paths.add((transitive, node)) def load_paths(self): + # Collect canonical intermediate symbol nodes for multi-hop Leo chains. + # When a node already IS the intermediate symbol (from a direct Leo completion), + # use it as the canonical node so all other paths to the same symbol merge into it, + # preserving SPPF sharing and rule-order-based disambiguation. + canonical_vn = {} for transitive, node in self.paths: if transitive.next_titem is not None: - vn = type(self)(transitive.next_titem.s, transitive.next_titem.start, self.end) - vn.add_path(transitive.next_titem, node) - self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, transitive.reduction.start, transitive.reduction.node, vn) + vn_s = transitive.next_titem.reduction.rule.origin + vn_start = transitive.next_titem.reduction.start + if getattr(node, 's', None) == vn_s and getattr(node, 'start', None) == vn_start: + canonical_vn[(vn_s, vn_start)] = node + + for transitive, node in self.paths: + if transitive.next_titem is not None: + vn_s = transitive.next_titem.reduction.rule.origin + vn_start = transitive.next_titem.reduction.start + key = (vn_s, vn_start) + if getattr(node, 's', None) == vn_s and getattr(node, 'start', None) == vn_start: + # node already represents the intermediate symbol; use it directly + self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, + transitive.reduction.start, transitive.reduction.node, node) + elif key in canonical_vn: + # Merge this derivation into the existing canonical intermediate node + canonical_vn[key].add_path(transitive.next_titem, node) + else: + # No existing node; create one and record it + vn = type(self)(vn_s, vn_start, self.end) + canonical_vn[key] = vn + vn.add_path(transitive.next_titem, node) + self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, + transitive.reduction.start, transitive.reduction.node, vn) else: self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, transitive.reduction.start, transitive.reduction.node, node) self.paths_loaded = True diff --git a/lark/parsers/xearley.py b/lark/parsers/xearley.py index f334ed199..0894c5630 100644 --- a/lark/parsers/xearley.py +++ b/lark/parsers/xearley.py @@ -156,7 +156,7 @@ def scan(i, to_scan): i = 0 node_cache = {} for token in stream: - self.predict_and_complete(i, to_scan, columns, transitives, node_cache) + self.predict_and_complete(i, to_scan, columns, transitives, node_cache, start_symbol) to_scan, node_cache = scan(i, to_scan) @@ -167,7 +167,7 @@ def scan(i, to_scan): text_column += 1 i += 1 - self.predict_and_complete(i, to_scan, columns, transitives, node_cache) + self.predict_and_complete(i, to_scan, columns, transitives, node_cache, start_symbol) ## Column is now the final column in the parse. assert i == len(columns)-1 diff --git a/tests/test_parser.py b/tests/test_parser.py index 533ae7890..72bb9343b 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1070,6 +1070,29 @@ def test_consistent_derivation_order1(self): n = Tree('a', []) assert tree == Tree('start', [n, n]) + def test_earley_jeo_right_recursion(self): + """Joop Leo optimization should correctly handle right-recursive grammars""" + grammar = r""" + start: "x" start | "x" + """ + l = Lark(grammar, parser='earley', lexer=LEXER) + res = l.parse('xxx') + self.assertEqual(res, Tree('start', [Tree('start', [Tree('start', [])])])) + + def test_earley_jeo_multi_start_right_recursion(self): + """Joop Leo optimization should work correctly with multiple start symbols (issue #397)""" + grammar = r""" + a: "x" a | "x" + b: "y" b | "y" + """ + l = Lark(grammar, parser='earley', lexer=LEXER, start=['a', 'b']) + + res_a = l.parse('xxx', start='a') + self.assertEqual(res_a, Tree('a', [Tree('a', [Tree('a', [])])])) + + res_b = l.parse('yyy', start='b') + self.assertEqual(res_b, Tree('b', [Tree('b', [Tree('b', [])])])) + _NAME = "TestFullEarley" + LEXER.capitalize() _TestFullEarley.__name__ = _NAME globals()[_NAME] = _TestFullEarley From a0f980f82185bf4df1695fee97332aa1974b378a Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Sat, 6 Jun 2026 00:59:47 +0200 Subject: [PATCH 2/5] Simplify SymbolNode.load_paths: single pass with unified node_cache As suggested by @chanicpanic, with small updates. Collapses the two-pass canonical_vn dance into one loop keyed on the full (s, start, end) SPPF identity. Paths whose node matches the expected intermediate label are reused as the canonical node; later paths to the same label merge via add_path and skip the redundant add_family. --- lark/parsers/earley_forest.py | 50 +++++++++++++---------------------- 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/lark/parsers/earley_forest.py b/lark/parsers/earley_forest.py index 27afd52d6..39ceff10f 100644 --- a/lark/parsers/earley_forest.py +++ b/lark/parsers/earley_forest.py @@ -67,39 +67,27 @@ def add_path(self, transitive, node): self.paths.add((transitive, node)) def load_paths(self): - # Collect canonical intermediate symbol nodes for multi-hop Leo chains. - # When a node already IS the intermediate symbol (from a direct Leo completion), - # use it as the canonical node so all other paths to the same symbol merge into it, - # preserving SPPF sharing and rule-order-based disambiguation. - canonical_vn = {} + # Share canonical intermediate symbol nodes across multi-hop Leo chains: + # SPPF identity is (s, start, end), so any path whose node matches the + # expected intermediate's label is reused as the canonical node, and other + # paths targeting the same label merge into it via add_path. + node_cache = {(node.s, node.start, node.end): node for (_, node) in self.paths} for transitive, node in self.paths: + next_node = node if transitive.next_titem is not None: - vn_s = transitive.next_titem.reduction.rule.origin - vn_start = transitive.next_titem.reduction.start - if getattr(node, 's', None) == vn_s and getattr(node, 'start', None) == vn_start: - canonical_vn[(vn_s, vn_start)] = node - - for transitive, node in self.paths: - if transitive.next_titem is not None: - vn_s = transitive.next_titem.reduction.rule.origin - vn_start = transitive.next_titem.reduction.start - key = (vn_s, vn_start) - if getattr(node, 's', None) == vn_s and getattr(node, 'start', None) == vn_start: - # node already represents the intermediate symbol; use it directly - self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, - transitive.reduction.start, transitive.reduction.node, node) - elif key in canonical_vn: - # Merge this derivation into the existing canonical intermediate node - canonical_vn[key].add_path(transitive.next_titem, node) - else: - # No existing node; create one and record it - vn = type(self)(vn_s, vn_start, self.end) - canonical_vn[key] = vn - vn.add_path(transitive.next_titem, node) - self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, - transitive.reduction.start, transitive.reduction.node, vn) - else: - self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, transitive.reduction.start, transitive.reduction.node, node) + label = (transitive.next_titem.reduction.rule.origin, + transitive.next_titem.reduction.start, + self.end) + if label != (node.s, node.start, node.end): + if label in node_cache: + # Canonical already added as right-side by an earlier path; just merge. + node_cache[label].add_path(transitive.next_titem, node) + continue + next_node = node_cache[label] = type(self)(*label) + next_node.add_path(transitive.next_titem, node) + + self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, + transitive.reduction.start, transitive.reduction.node, next_node) self.paths_loaded = True @property From 7f367ffe01625b8b5a4ba3a52c6fd5ec42ee3ba4 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Sat, 6 Jun 2026 01:16:37 +0200 Subject: [PATCH 3/5] Add tests for more coverage 1) Add Joop Leo regression tests for chain-extension corner cases Two new tests under the Joop Leo section in TestFullEarley: - Chain stops at non-nullable suffix: is_quasi_complete returns False when the outer rule has a non-nullable symbol after the recursive position. - Chain extends through nullable suffix: is_quasi_complete traverses trailing nullable non-terminals via the NULLABLE check + advance loop. These cover the previously-untested NULLABLE-suffix branch of is_quasi_complete and the break-at-non-nullable-suffix path of create_leo_transitives. 2) Add Joop Leo test for nullable-start self-reference guard Covers the is_quasi_complete branch that returns False when the chain encounters a state in start_symbol's rule expecting start_symbol itself as a nullable non-terminal. Without this guard, the Leo shortcut would treat the start rule as quasi-complete and produce an incorrect chain. --- tests/test_parser.py | 46 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/test_parser.py b/tests/test_parser.py index 72bb9343b..9cb25dac4 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1093,6 +1093,52 @@ def test_earley_jeo_multi_start_right_recursion(self): res_b = l.parse('yyy', start='b') self.assertEqual(res_b, Tree('b', [Tree('b', [Tree('b', [])])])) + def test_earley_jeo_chain_stops_at_nonnullable_suffix(self): + """Leo chain walks outward until the outer rule has a non-nullable + suffix after the recursive position, at which point is_quasi_complete + returns False (covers the non-NULLABLE check + break in + create_leo_transitives).""" + grammar = r""" + start: a "z" + a: "x" a | "y" + """ + l = Lark(grammar, parser='earley', lexer=LEXER) + tree = l.parse('xxyz') + self.assertEqual(tree, Tree('start', [Tree('a', [Tree('a', [Tree('a', [])])])])) + + def test_earley_jeo_chain_extends_through_nullable_suffix(self): + """Leo chain extends into an outer rule when the trailing symbols + are all nullable: is_quasi_complete traverses them via the + NULLABLE check + advance loop.""" + grammar = r""" + start: a opt + a: "x" a | "y" + opt: "q" | + """ + l = Lark(grammar, parser='earley', lexer=LEXER) + tree_no_opt = l.parse('xxy') + self.assertEqual(tree_no_opt, Tree('start', [Tree('a', [Tree('a', [Tree('a', [])])]), Tree('opt', [])])) + tree_with_opt = l.parse('xxyq') + self.assertEqual(tree_with_opt, Tree('start', [Tree('a', [Tree('a', [Tree('a', [])])]), Tree('opt', [])])) + + def test_earley_jeo_nullable_start_self_recursion(self): + """is_quasi_complete's start-symbol self-reference guard: when the + Leo chain walks into a rule of the form `start: ... start ...` + with `start` itself nullable, the guard returns False so the chain + doesn't pretend the start rule is quasi-complete.""" + grammar = r""" + start: a x start | + x: + a: "x" a | "y" + """ + l = Lark(grammar, parser='earley', lexer=LEXER) + self.assertEqual(l.parse(''), Tree('start', [])) + tree = l.parse('xxy') + self.assertEqual( + tree, + Tree('start', [Tree('a', [Tree('a', [Tree('a', [])])]), Tree('x', []), Tree('start', [])]) + ) + _NAME = "TestFullEarley" + LEXER.capitalize() _TestFullEarley.__name__ = _NAME globals()[_NAME] = _TestFullEarley From 583bac0751f42facdab41dde0bfca1e02495d39f Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Sat, 6 Jun 2026 01:42:54 +0200 Subject: [PATCH 4/5] Remove three unreachable defensive branches in Joop Leo completer - is_quasi_complete's complete-item fast-path: caller in create_leo_transitives only passes originators with expect != None. - root_transitive's else branch: every titem is registered at transitives[col][previous], so the lookup always succeeds (the root titem self-resolves through it). - Leo + terminal scan branch: is_quasi_complete requires the post- recursion suffix to be all-nullable, so new_item.expect is never a terminal. The first two are Python-level defensive code; the third is residue from a more paper-faithful structure that the current titem representation makes redundant. Cycle detection (visited) and the empty-FIRST guard are retained as cheap protection against unusual grammars. --- lark/parsers/earley.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index 93e76485d..cdba70b39 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -87,8 +87,6 @@ def predict_and_complete(self, i, to_scan, columns, transitives, node_cache, sta held_completions = {} def is_quasi_complete(item): - if item.is_complete: - return True quasi = item.advance() while not quasi.is_complete: if quasi.expect not in self.NULLABLE: @@ -173,19 +171,18 @@ def create_leo_transitives(origin, start): ###R Joop Leo right recursion Completer if item.rule.origin in transitives[item.start]: transitive = transitives[item.start][item.s] - if transitive.previous in transitives[transitive.column]: - root_transitive = transitives[transitive.column][transitive.previous] - else: - root_transitive = transitive + # Every titem is inserted into transitives[col][previous], + # so this lookup always finds the chain's root (or transitive + # itself when it is the root). + root_transitive = transitives[transitive.column][transitive.previous] new_item = Item(transitive.rule, transitive.ptr, transitive.start) label = (root_transitive.s, root_transitive.start, i) new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label)) new_item.node.add_path(root_transitive, item.node) - if new_item.expect in self.TERMINALS: - # Add (B :: aC.B, h, y) to Q - to_scan.add(new_item) - elif new_item not in column: + # new_item.expect cannot be a terminal: is_quasi_complete + # required the post-recursion suffix to be all-nullable. + if new_item not in column: # Add (B :: aC.B, h, y) to Ei and R column.add(new_item) items.append(new_item) From 25347a5d75bee9104c8e5125e3a90a821651de3e Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Sat, 6 Jun 2026 02:15:07 +0200 Subject: [PATCH 5/5] Address PR review comments on Joop Leo re-enable - Move held_completions update back into the regular completer branch. Empty (nullable) items can never trigger Leo (create_leo_transitives breaks at the NULLABLE check, so transitives never contains nullable origins as keys), so the unconditional placement was unnecessary. Reviewer's observation that no test failed without the move is correct. - Use item.rule.origin instead of item.s when looking up the transitive, matching the preceding `in transitives[item.start]` check. Both are equivalent for complete items, but item.rule.origin is less surprising. - Restore the original walk-backwards/walk-forwards comments in create_leo_transitives so the two phases are immediately legible. Co-Authored-By: Claude Opus 4.7 (1M context) --- lark/parsers/earley.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index cdba70b39..68b395014 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -102,6 +102,9 @@ def create_leo_transitives(origin, start): trule = None previous = None + ### Recursively walk backwards through the Earley sets until we find the + # first transitive candidate. If this is done continuously, we shouldn't + # have to walk more than 1 hop. while True: if origin in transitives[start]: previous = trule = transitives[start][origin] @@ -133,9 +136,12 @@ def create_leo_transitives(origin, start): origin = originator.rule.origin start = originator.start + # If a suitable Transitive candidate is not found, bail. if trule is None: return + ### Now walk forwards and create Transitive Items in each set we walked through; + # and link each transitive item to the next set forwards. while to_create: origin, start, originator = to_create.pop() if previous is not None: @@ -157,20 +163,11 @@ def create_leo_transitives(origin, start): item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label)) item.node.add_family(item.s, item.rule, item.start, None, None) - # Empty has 0 length. If we complete an empty symbol in a particular - # parse step, we need to be able to use that same empty symbol to complete - # any predictions that result, that themselves require empty. Avoids - # infinite recursion on empty symbols. - # held_completions is 'H' in E.Scott's paper. - # Must be updated before Leo fires, since Leo skips the regular completer. - if item.start == i: - held_completions[item.rule.origin] = item.node - create_leo_transitives(item.rule.origin, item.start) ###R Joop Leo right recursion Completer if item.rule.origin in transitives[item.start]: - transitive = transitives[item.start][item.s] + transitive = transitives[item.start][item.rule.origin] # Every titem is inserted into transitives[col][previous], # so this lookup always finds the chain's root (or transitive # itself when it is the root). @@ -188,6 +185,14 @@ def create_leo_transitives(origin, start): items.append(new_item) ###R Regular Earley completer else: + # Empty has 0 length. If we complete an empty symbol in a particular + # parse step, we need to be able to use that same empty symbol to complete + # any predictions that result, that themselves require empty. Avoids + # infinite recursion on empty symbols. + # held_completions is 'H' in E.Scott's paper. + if item.start == i: + held_completions[item.rule.origin] = item.node + originators = [originator for originator in columns[item.start] if originator.expect is not None and originator.expect == item.s] for originator in originators: new_item = originator.advance()