Reject cyclic grammars instead of looping forever (#1585)

lark/parsers/cyk.py

@@ -12,6 +12,7 @@
 from ..lexer import Token
 from ..tree import Tree
 from ..grammar import Terminal as T, NonTerminal as NT, Symbol
+from .grammar_analysis import calculate_sets, check_cyclic_grammar
 
 def match(t, s):
     assert isinstance(t, T)
@@ -81,6 +82,8 @@ class Parser:
 
     def __init__(self, rules):
         super(Parser, self).__init__()
+        _, _, nullable = calculate_sets(rules)
+        check_cyclic_grammar(rules, nullable)
         self.orig_rules = {rule: rule for rule in rules}
         rules = [self._to_rule(rule) for rule in rules]
         self.grammar = to_cnf(Grammar(rules))

lark/parsers/grammar_analysis.py

@@ -137,6 +137,48 @@ def calculate_sets(rules):
     return FIRST, FOLLOW, NULLABLE
 
 
+def check_cyclic_grammar(rules, nullable):
+    """Raise GrammarError if a non-terminal can derive itself without consuming input.
+
+    A grammar is cyclic when some rule allows ``A => A`` (directly, or through other
+    symbols that are all nullable), e.g. ``a: a b*`` where ``b*`` may be empty. Such a
+    rule has no base case to terminate on, so the LALR and CYK parsers loop forever on
+    it (Earley merely produces an arbitrary parse). Reject it at build time instead.
+    """
+    # A -> B whenever a rule expands A to B with every other symbol nullable.
+    derives = defaultdict(set)
+    for rule in rules:
+        expansion = rule.expansion
+        for i, sym in enumerate(expansion):
+            if sym.is_term:
+                continue
+            if all(s in nullable for j, s in enumerate(expansion) if j != i):
+                derives[rule.origin].add(sym)
+
+    # Iterative DFS that stops at the first non-terminal found on a cycle.
+    UNVISITED, ON_STACK, DONE = 0, 1, 2
+    state = defaultdict(int)
+    for start in list(derives):
+        if state[start] != UNVISITED:
+            continue
+        path = [(start, iter(derives[start]))]
+        state[start] = ON_STACK
+        while path:
+            origin, children = path[-1]
+            for nxt in children:
+                if state[nxt] == ON_STACK:
+                    raise GrammarError("Rule '%s' is cyclic: it can derive itself without "
+                                       "consuming any input, which makes the parser loop "
+                                       "forever (e.g. `a: a b*`)." % nxt.name)
+                if state[nxt] == UNVISITED:
+                    state[nxt] = ON_STACK
+                    path.append((nxt, iter(derives[nxt])))
+                    break
+            else:
+                state[origin] = DONE
+                path.pop()
+
+
 class GrammarAnalyzer:
     def __init__(self, parser_conf: ParserConf, debug: bool=False, strict: bool=False):
         self.debug = debug

lark/parsers/lalr_analysis.py

@@ -12,7 +12,7 @@
 from ..utils import classify, classify_bool, bfs, fzset, Enumerator, logger
 from ..exceptions import GrammarError
 
-from .grammar_analysis import GrammarAnalyzer, Terminal, LR0ItemSet, RulePtr, State
+from .grammar_analysis import GrammarAnalyzer, Terminal, LR0ItemSet, RulePtr, State, check_cyclic_grammar
 from ..grammar import Rule, Symbol
 from ..common import ParserConf
 
@@ -156,6 +156,7 @@ class LALR_Analyzer(GrammarAnalyzer):
 
     def __init__(self, parser_conf: ParserConf, debug: bool=False, strict: bool=False):
         GrammarAnalyzer.__init__(self, parser_conf, debug, strict)
+        check_cyclic_grammar(parser_conf.rules, self.NULLABLE)
         self.nonterminal_transitions = []
         self.directly_reads = defaultdict(set)
         self.reads = defaultdict(set)

tests/test_grammar.py

@@ -315,6 +315,22 @@ def test_symbol_eq(self):
 
         self.assertNotEqual(a, b)
 
+    def test_cyclic_rule(self):
+        # A rule that can derive itself without consuming any input makes the
+        # LALR and CYK parsers loop forever. Reject such grammars at build time
+        # with a clear error instead of hanging (issue #1585).
+        self.assertRaises(GrammarError, Lark, 'start.1: "a" | start start*', parser='lalr')
+        self.assertRaises(GrammarError, Lark, 'start: x\nx.1: "a" | x x*', parser='lalr')
+        self.assertRaises(GrammarError, Lark, 'start: a\na: b\nb: a | "x"', parser='lalr')
+        self.assertRaises(GrammarError, Lark, 'start.1: "a" | start start*', parser='cyk')
+
+        # Earley resolves cyclic grammars (see test_many_cycles), so it is left as-is.
+        Lark('start.1: "a" | start start*', parser='earley').parse('aa')
+
+        # Left recursion and optional repetition are not cyclic and still build.
+        Lark('start: e\ne: e "+" t | t\nt: "a"', parser='lalr')
+        Lark('start: "a"*', parser='lalr')
+
 
 if __name__ == '__main__':
     main()