Add Token.as_str() to decode bytes token values

lark/lexer.py

@@ -253,6 +253,17 @@ def _future_update(self, type: Optional[str] = None, value: Optional[Any] = None
     def new_borrow_pos(cls: Type[_T], type_: str, value: Any, borrow_t: 'Token') -> _T:
         return cls(type_, value, borrow_t.start_pos, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos)
 
+    def as_str(self) -> str:
+        """Return the token's value as a ``str``.
+
+        When the parser is created with ``use_bytes=True``, ``value`` is ``bytes``,
+        so ``str(token)`` yields its repr rather than its text. This decodes it,
+        letting transformer callbacks work the same regardless of ``use_bytes``.
+        """
+        if isinstance(self.value, bytes):
+            return self.value.decode('latin-1')
+        return self.value
+
     def __reduce__(self):
         return (self.__class__, (self.type, self.value, self.start_pos, self.line, self.column))
 

tests/test_parser.py

@@ -1337,6 +1337,18 @@ def test_bytes_utf8(self):
                 s = j.encode(enc)
                 self.assertRaises(UnexpectedCharacters, g.parse, s)
 
+        def test_bytes_token_as_str(self):
+            # With use_bytes=True the token value is bytes, so float(token)
+            # or str ops that work under use_bytes=False fail. as_str() decodes
+            # it, letting a callback do float(token.as_str()) either way. (#1526)
+            g = _Lark(r"""start: NUMBER
+                          NUMBER: /[0-9]+/
+                       """, use_bytes=True)
+            num, = g.parse(b"42").children
+            self.assertEqual(num.value, b"42")
+            self.assertEqual(num.as_str(), "42")
+            self.assertEqual(float(num.as_str()), 42.0)
+
         @unittest.skipIf(PARSER == 'cyk', "Takes forever")
         def test_stack_for_ebnf(self):
             """Verify that stack depth isn't an issue for EBNF grammars"""