diff --git a/lark/lexer.py b/lark/lexer.py index 9aa10c2e2..20afddb48 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -253,6 +253,17 @@ def _future_update(self, type: Optional[str] = None, value: Optional[Any] = None def new_borrow_pos(cls: Type[_T], type_: str, value: Any, borrow_t: 'Token') -> _T: return cls(type_, value, borrow_t.start_pos, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos) + def as_str(self) -> str: + """Return the token's value as a ``str``. + + When the parser is created with ``use_bytes=True``, ``value`` is ``bytes``, + so ``str(token)`` yields its repr rather than its text. This decodes it, + letting transformer callbacks work the same regardless of ``use_bytes``. + """ + if isinstance(self.value, bytes): + return self.value.decode('latin-1') + return self.value + def __reduce__(self): return (self.__class__, (self.type, self.value, self.start_pos, self.line, self.column)) diff --git a/tests/test_parser.py b/tests/test_parser.py index 721e8693a..e2c9f0ff5 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1337,6 +1337,18 @@ def test_bytes_utf8(self): s = j.encode(enc) self.assertRaises(UnexpectedCharacters, g.parse, s) + def test_bytes_token_as_str(self): + # With use_bytes=True the token value is bytes, so float(token) + # or str ops that work under use_bytes=False fail. as_str() decodes + # it, letting a callback do float(token.as_str()) either way. (#1526) + g = _Lark(r"""start: NUMBER + NUMBER: /[0-9]+/ + """, use_bytes=True) + num, = g.parse(b"42").children + self.assertEqual(num.value, b"42") + self.assertEqual(num.as_str(), "42") + self.assertEqual(float(num.as_str()), 42.0) + @unittest.skipIf(PARSER == 'cyk', "Takes forever") def test_stack_for_ebnf(self): """Verify that stack depth isn't an issue for EBNF grammars"""