diff --git a/lark/parsers/cyk.py b/lark/parsers/cyk.py index b5334f907..db94dc5ba 100644 --- a/lark/parsers/cyk.py +++ b/lark/parsers/cyk.py @@ -12,6 +12,7 @@ from ..lexer import Token from ..tree import Tree from ..grammar import Terminal as T, NonTerminal as NT, Symbol +from .grammar_analysis import calculate_sets, check_cyclic_grammar def match(t, s): assert isinstance(t, T) @@ -81,6 +82,8 @@ class Parser: def __init__(self, rules): super(Parser, self).__init__() + _, _, nullable = calculate_sets(rules) + check_cyclic_grammar(rules, nullable) self.orig_rules = {rule: rule for rule in rules} rules = [self._to_rule(rule) for rule in rules] self.grammar = to_cnf(Grammar(rules)) diff --git a/lark/parsers/grammar_analysis.py b/lark/parsers/grammar_analysis.py index 28d3cb637..c27f63b03 100644 --- a/lark/parsers/grammar_analysis.py +++ b/lark/parsers/grammar_analysis.py @@ -137,6 +137,48 @@ def calculate_sets(rules): return FIRST, FOLLOW, NULLABLE +def check_cyclic_grammar(rules, nullable): + """Raise GrammarError if a non-terminal can derive itself without consuming input. + + A grammar is cyclic when some rule allows ``A => A`` (directly, or through other + symbols that are all nullable), e.g. ``a: a b*`` where ``b*`` may be empty. Such a + rule has no base case to terminate on, so the LALR and CYK parsers loop forever on + it (Earley merely produces an arbitrary parse). Reject it at build time instead. + """ + # A -> B whenever a rule expands A to B with every other symbol nullable. + derives = defaultdict(set) + for rule in rules: + expansion = rule.expansion + for i, sym in enumerate(expansion): + if sym.is_term: + continue + if all(s in nullable for j, s in enumerate(expansion) if j != i): + derives[rule.origin].add(sym) + + # Iterative DFS that stops at the first non-terminal found on a cycle. + UNVISITED, ON_STACK, DONE = 0, 1, 2 + state = defaultdict(int) + for start in list(derives): + if state[start] != UNVISITED: + continue + path = [(start, iter(derives[start]))] + state[start] = ON_STACK + while path: + origin, children = path[-1] + for nxt in children: + if state[nxt] == ON_STACK: + raise GrammarError("Rule '%s' is cyclic: it can derive itself without " + "consuming any input, which makes the parser loop " + "forever (e.g. `a: a b*`)." % nxt.name) + if state[nxt] == UNVISITED: + state[nxt] = ON_STACK + path.append((nxt, iter(derives[nxt]))) + break + else: + state[origin] = DONE + path.pop() + + class GrammarAnalyzer: def __init__(self, parser_conf: ParserConf, debug: bool=False, strict: bool=False): self.debug = debug diff --git a/lark/parsers/lalr_analysis.py b/lark/parsers/lalr_analysis.py index 4ea8f643a..891c9888c 100644 --- a/lark/parsers/lalr_analysis.py +++ b/lark/parsers/lalr_analysis.py @@ -12,7 +12,7 @@ from ..utils import classify, classify_bool, bfs, fzset, Enumerator, logger from ..exceptions import GrammarError -from .grammar_analysis import GrammarAnalyzer, Terminal, LR0ItemSet, RulePtr, State +from .grammar_analysis import GrammarAnalyzer, Terminal, LR0ItemSet, RulePtr, State, check_cyclic_grammar from ..grammar import Rule, Symbol from ..common import ParserConf @@ -156,6 +156,7 @@ class LALR_Analyzer(GrammarAnalyzer): def __init__(self, parser_conf: ParserConf, debug: bool=False, strict: bool=False): GrammarAnalyzer.__init__(self, parser_conf, debug, strict) + check_cyclic_grammar(parser_conf.rules, self.NULLABLE) self.nonterminal_transitions = [] self.directly_reads = defaultdict(set) self.reads = defaultdict(set) diff --git a/tests/test_grammar.py b/tests/test_grammar.py index 9524eb4c8..586de32b4 100644 --- a/tests/test_grammar.py +++ b/tests/test_grammar.py @@ -315,6 +315,22 @@ def test_symbol_eq(self): self.assertNotEqual(a, b) + def test_cyclic_rule(self): + # A rule that can derive itself without consuming any input makes the + # LALR and CYK parsers loop forever. Reject such grammars at build time + # with a clear error instead of hanging (issue #1585). + self.assertRaises(GrammarError, Lark, 'start.1: "a" | start start*', parser='lalr') + self.assertRaises(GrammarError, Lark, 'start: x\nx.1: "a" | x x*', parser='lalr') + self.assertRaises(GrammarError, Lark, 'start: a\na: b\nb: a | "x"', parser='lalr') + self.assertRaises(GrammarError, Lark, 'start.1: "a" | start start*', parser='cyk') + + # Earley resolves cyclic grammars (see test_many_cycles), so it is left as-is. + Lark('start.1: "a" | start start*', parser='earley').parse('aa') + + # Left recursion and optional repetition are not cyclic and still build. + Lark('start: e\ne: e "+" t | t\nt: "a"', parser='lalr') + Lark('start: "a"*', parser='lalr') + if __name__ == '__main__': main()