Improve output of tokens in Parse Errors

IMSoP · nikic · commit 55a15f32ced0 · 2020-07-13T11:07:40.000+02:00
Currently, unexpected tokens in the parser are shown as the text
found, plus the internal token name, including the notorious
"unexpected '::' (T_PAAMAYIM_NEKUDOTAYIM)".

This commit replaces that with a more user-friendly format, with
two main types of token:

* Tokens which always represent the same text are shown like
  'unexpected token "::"' and 'expected "::"'
* Tokens which have variable text are given a user-friendly
  name, and show like 'unexpected identifier "foo"', and
  'expected identifer'.

A few tokens have special cases:

* unexpected token """ -&gt; unexpected double-quote mark
* unexpected quoted string "'foo'" -&gt; unexpected single-quoted
  string "foo"
* unexpected quoted string ""foo"" -&gt; unexpected double-quoted
  string "foo"
* unexpected illegal character "_" -&gt; unexpected character 0xNN
  (where _ is almost certainly a control character, and NN is the
   hexadecimal value of the byte)

The \ token has a special case in the implementation just to stop
bison making a mess of escaping it and it coming out as \\
diff --git a/Zend/tests/attributes/019_variable_attribute_name.phpt b/Zend/tests/attributes/019_variable_attribute_name.phpt
@@ -8,4 +8,4 @@ class A {}
 
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected '$x' (T_VARIABLE), expecting identifier (T_STRING) or static (T_STATIC) or namespace (T_NAMESPACE) or \\ (T_NS_SEPARATOR) in %s on line %d
+Parse error: syntax error, unexpected variable "$x", expecting identifier or "static" or "namespace" or "\" in %s on line %d
diff --git a/Zend/tests/bug43343.phpt b/Zend/tests/bug43343.phpt
@@ -8,4 +8,4 @@ $foo = 'bar';
 var_dump(new namespace::$foo);
 ?>
 --EXPECTF--
-Parse error: %s error%sexpecting%sT_NS_SEPARATOR%sin %sbug43343.php on line 5
+Parse error: %s error%sexpecting%s"\"%sin %sbug43343.php on line 5
diff --git a/Zend/tests/bug70430.phpt b/Zend/tests/bug70430.phpt
@@ -7,4 +7,4 @@ $"*** Testing function() :  ***\n";
 
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected '"*** Testing function() :  ***' (T_CONSTANT_ENCAPSED_STRING), expecting variable (T_VARIABLE) or '{' or '$' in %s on line %d
+Parse error: syntax error, unexpected double-quoted string "*** Testing function() :  ***\n", expecting variable or "{" or "$" in %s on line %d
diff --git a/Zend/tests/bug75252.phpt b/Zend/tests/bug75252.phpt
@@ -24,5 +24,5 @@ try {
 
 ?>
 --EXPECT--
-string(41) "syntax error, unexpected 'FOO' (T_STRING)"
-string(41) "syntax error, unexpected 'FOO' (T_STRING)"
+string(41) "syntax error, unexpected identifier "FOO""
+string(41) "syntax error, unexpected identifier "FOO""
diff --git a/Zend/tests/bug77993.phpt b/Zend/tests/bug77993.phpt
@@ -4,4 +4,4 @@ Bug #77993 (Wrong parse error for invalid hex literal on Windows)
 <?php
 0xg10;
 --EXPECTF--
-Parse error: syntax error, unexpected 'xg10' (T_STRING) in %s on line %d
+Parse error: syntax error, unexpected identifier "xg10" in %s on line %d
diff --git a/Zend/tests/bug78454_1.phpt b/Zend/tests/bug78454_1.phpt
@@ -4,4 +4,4 @@ Invalid consecutive numeric separators after hex literal
 <?php
 0x0__F;
 --EXPECTF--
-Parse error: syntax error, unexpected '__F' (T_STRING) in %s on line %d
+Parse error: syntax error, unexpected identifier "__F" in %s on line %d
diff --git a/Zend/tests/bug78454_2.phpt b/Zend/tests/bug78454_2.phpt
@@ -4,4 +4,4 @@ Invalid consecutive numeric separators after binary literal
 <?php
 0b0__1
 --EXPECTF--
-Parse error: syntax error, unexpected '__1' (T_STRING) in %s on line %d
+Parse error: syntax error, unexpected identifier "__1" in %s on line %d
diff --git a/Zend/tests/ctor_promotion_additional_modifiers.phpt b/Zend/tests/ctor_promotion_additional_modifiers.phpt
@@ -9,4 +9,4 @@ class Test {
 
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected 'static' (T_STATIC), expecting variable (T_VARIABLE) in %s on line %d
+Parse error: syntax error, unexpected token "static", expecting variable in %s on line %d
diff --git a/Zend/tests/flexible-heredoc-error7.phpt b/Zend/tests/flexible-heredoc-error7.phpt
@@ -8,4 +8,4 @@ Note: the closing ?> has been deliberately elided.
 echo <<<END
 
 --EXPECTF--
-Parse error: syntax error, unexpected end of file, expecting variable (T_VARIABLE) or heredoc end (T_END_HEREDOC) or ${ (T_DOLLAR_OPEN_CURLY_BRACES) or {$ (T_CURLY_OPEN) in %s on line %d
+Parse error: syntax error, unexpected end of file, expecting variable or heredoc end or "${" or "{$" in %s on line %d
diff --git a/Zend/tests/flexible-nowdoc-error7.phpt b/Zend/tests/flexible-nowdoc-error7.phpt
@@ -8,4 +8,4 @@ Note: the closing ?> has been deliberately elided.
 echo <<<'END'
 
 --EXPECTF--
-Parse error: syntax error, unexpected end of file, expecting variable (T_VARIABLE) or heredoc end (T_END_HEREDOC) or ${ (T_DOLLAR_OPEN_CURLY_BRACES) or {$ (T_CURLY_OPEN) in %s on line %d
+Parse error: syntax error, unexpected end of file, expecting variable or heredoc end or "${" or "{$" in %s on line %d
diff --git a/Zend/tests/flexible-nowdoc-error8.phpt b/Zend/tests/flexible-nowdoc-error8.phpt
@@ -8,4 +8,4 @@ eval('<<<\'end\'
 
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected end of file, expecting variable (T_VARIABLE) or heredoc end (T_END_HEREDOC) or ${ (T_DOLLAR_OPEN_CURLY_BRACES) or {$ (T_CURLY_OPEN) in %s on line %d
+Parse error: syntax error, unexpected end of file, expecting variable or heredoc end or "${" or "{$" in %s on line %d
diff --git a/Zend/tests/function_arguments/call_with_leading_comma_error.phpt b/Zend/tests/function_arguments/call_with_leading_comma_error.phpt
@@ -5,4 +5,4 @@ Leading commas in function calls is not allowed
 foo(,$foo);
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected ',' in %s on line %d
+Parse error: syntax error, unexpected token "," in %s on line %d
diff --git a/Zend/tests/function_arguments/call_with_multi_inner_comma_error.phpt b/Zend/tests/function_arguments/call_with_multi_inner_comma_error.phpt
@@ -5,4 +5,4 @@ Multiple inner commas in function calls is not allowed
 foo($foo,,$bar);
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected ',', expecting ')' in %s on line %d
+Parse error: syntax error, unexpected token ",", expecting ")" in %s on line %d
diff --git a/Zend/tests/function_arguments/call_with_multi_trailing_comma_error.phpt b/Zend/tests/function_arguments/call_with_multi_trailing_comma_error.phpt
@@ -5,4 +5,4 @@ Multiple trailing commas in function calls is not allowed
 foo($foo,,);
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected ',', expecting ')' in %s on line %d
+Parse error: syntax error, unexpected token ",", expecting ")" in %s on line %d
diff --git a/Zend/tests/function_arguments/call_with_only_comma_error.phpt b/Zend/tests/function_arguments/call_with_only_comma_error.phpt
@@ -5,4 +5,4 @@ Single comma in function calls is not allowed
 foo(,);
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected ',' in %s on line %d
+Parse error: syntax error, unexpected token "," in %s on line %d
diff --git a/Zend/tests/grammar/regression_004.phpt b/Zend/tests/grammar/regression_004.phpt
@@ -11,4 +11,4 @@ class Obj
 
 function echo(){} // not valid
 --EXPECTF--
-Parse error: syntax error, unexpected 'echo' (T_ECHO), expecting %s in %s on line 9
+Parse error: syntax error, unexpected token "echo", expecting "(" in %s on line %d
diff --git a/Zend/tests/grammar/regression_005.phpt b/Zend/tests/grammar/regression_005.phpt
@@ -10,4 +10,4 @@ class Obj
 
 const return = 'nope';
 --EXPECTF--
-Parse error: syntax error, unexpected 'return' (T_RETURN), expecting identifier (T_STRING) in %s on line 8
+Parse error: syntax error, unexpected token "return", expecting identifier in %s on line %d
diff --git a/Zend/tests/grammar/regression_012.phpt b/Zend/tests/grammar/regression_012.phpt
@@ -9,4 +9,4 @@ class A {
 
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected 'FOREACH' (T_FOREACH), expecting ']' in %s on line %d
+Parse error: syntax error, unexpected token "foreach", expecting "]" in %s on line %d
diff --git a/Zend/tests/list_011.phpt b/Zend/tests/list_011.phpt
@@ -7,4 +7,4 @@ var_dump(list(1, 2, 3));
 
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected ')', expecting '=' in %s on line %d
+Parse error: syntax error, unexpected token ")", expecting "=" in %s on line %d
diff --git a/Zend/tests/ns_088.phpt b/Zend/tests/ns_088.phpt
@@ -14,4 +14,4 @@ namespace Fiz\Biz\Buz {
 }
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected '{', expecting '}' in %sns_088.php on line 5
+Parse error: syntax error, unexpected token "{", expecting "}" in %s on line %d
diff --git a/Zend/tests/ns_094.phpt b/Zend/tests/ns_094.phpt
@@ -11,4 +11,4 @@ use const Foo\Bar\{
     function C
 };
 --EXPECTF--
-Parse error: syntax error, unexpected 'const' (T_CONST), expecting '}' in %s on line 7
+Parse error: syntax error, unexpected token "const", expecting "}" in %s on line %d
diff --git a/Zend/tests/ns_096.phpt b/Zend/tests/ns_096.phpt
@@ -7,4 +7,4 @@ use Foo\Bar\{\Baz};
 
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected '\' (T_NS_SEPARATOR), expecting identifier (T_STRING) or function (T_FUNCTION) or const (T_CONST) in %s on line 3
+Parse error: syntax error, unexpected token "\", expecting identifier or "function" or "const" in %s on line %d
diff --git a/Zend/tests/ns_trailing_comma_error_01.phpt b/Zend/tests/ns_trailing_comma_error_01.phpt
@@ -5,4 +5,4 @@ Group use declarations mustn't be empty
 use Baz\{};
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected '}', expecting identifier (T_STRING) or function (T_FUNCTION) or const (T_CONST) in %s on line %d
+Parse error: syntax error, unexpected token "}", expecting identifier or "function" or "const" in %s on line %d
diff --git a/Zend/tests/ns_trailing_comma_error_02.phpt b/Zend/tests/ns_trailing_comma_error_02.phpt
@@ -5,4 +5,4 @@ Group use declarations mustn't contain just a comma
 use Baz\{,};
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected ',', expecting identifier (T_STRING) or function (T_FUNCTION) or const (T_CONST) in %s on line %d
+Parse error: syntax error, unexpected token ",", expecting identifier or "function" or "const" in %s on line %d
diff --git a/Zend/tests/ns_trailing_comma_error_03.phpt b/Zend/tests/ns_trailing_comma_error_03.phpt
@@ -5,4 +5,4 @@ Group use declarations mustn't allow more than one comma
 use Baz\{Foo,,};
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected ',', expecting '}' in %s on line %d
+Parse error: syntax error, unexpected token ",", expecting "}" in %s on line %d
diff --git a/Zend/tests/ns_trailing_comma_error_04.phpt b/Zend/tests/ns_trailing_comma_error_04.phpt
@@ -5,4 +5,4 @@ Group use declarations mustn't begin with a comma
 use Baz\{,Foo};
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected ',', expecting identifier (T_STRING) or function (T_FUNCTION) or const (T_CONST) in %s on line %d
+Parse error: syntax error, unexpected token ",", expecting identifier or "function" or "const" in %s on line %d
diff --git a/Zend/tests/ns_trailing_comma_error_05.phpt b/Zend/tests/ns_trailing_comma_error_05.phpt
@@ -5,4 +5,4 @@ Group use declarations mustn't contain two commas mid-list
 use Baz\{Foo,,Bar};
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected ',', expecting '}' in %s on line %d
+Parse error: syntax error, unexpected token ",", expecting "}" in %s on line %d
diff --git a/Zend/tests/ns_trailing_comma_error_06.phpt b/Zend/tests/ns_trailing_comma_error_06.phpt
@@ -5,4 +5,4 @@ Unmixed group use declarations mustn't allow more than one comma
 use const Baz\{Foo,,};
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected ',', expecting '}' in %s on line %d
+Parse error: syntax error, unexpected token ",", expecting "}" in %s on line %d
diff --git a/Zend/tests/ns_trailing_comma_error_07.phpt b/Zend/tests/ns_trailing_comma_error_07.phpt
@@ -5,4 +5,4 @@ Unmixed group use declarations mustn't begin with a comma
 use function Baz\{,Foo};
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected ',', expecting identifier (T_STRING) in %s on line %d
+Parse error: syntax error, unexpected token ",", expecting identifier in %s on line %d
diff --git a/Zend/tests/ns_trailing_comma_error_08.phpt b/Zend/tests/ns_trailing_comma_error_08.phpt
@@ -5,4 +5,4 @@ Unmixed group use declarations mustn't contain two commas mid-list
 use const Baz\{Foo,,Bar};
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected ',', expecting '}' in %s on line %d
+Parse error: syntax error, unexpected token ",", expecting "}" in %s on line %d
diff --git a/Zend/tests/numeric_literal_separator_002.phpt b/Zend/tests/numeric_literal_separator_002.phpt
@@ -4,4 +4,4 @@ Invalid use: trailing underscore
 <?php
 100_;
 --EXPECTF--
-Parse error: syntax error, unexpected '_' (T_STRING) in %s on line %d
+Parse error: syntax error, unexpected identifier "_" in %s on line %d
diff --git a/Zend/tests/numeric_literal_separator_003.phpt b/Zend/tests/numeric_literal_separator_003.phpt
@@ -4,4 +4,4 @@ Invalid use: adjacent underscores
 <?php
 10__0;
 --EXPECTF--
-Parse error: syntax error, unexpected '__0' (T_STRING) in %s on line %d
+Parse error: syntax error, unexpected identifier "__0" in %s on line %d
diff --git a/Zend/tests/numeric_literal_separator_004.phpt b/Zend/tests/numeric_literal_separator_004.phpt
@@ -4,4 +4,4 @@ Invalid use: underscore left of period
 <?php
 100_.0;
 --EXPECTF--
-Parse error: syntax error, unexpected '_' (T_STRING) in %s on line %d
+Parse error: syntax error, unexpected identifier "_" in %s on line %d
diff --git a/Zend/tests/numeric_literal_separator_005.phpt b/Zend/tests/numeric_literal_separator_005.phpt
@@ -4,4 +4,4 @@ Invalid use: underscore right of period
 <?php
 100._0;
 --EXPECTF--
-Parse error: syntax error, unexpected '_0' (T_STRING) in %s on line %d
+Parse error: syntax error, unexpected identifier "_0" in %s on line %d
diff --git a/Zend/tests/numeric_literal_separator_006.phpt b/Zend/tests/numeric_literal_separator_006.phpt
@@ -4,4 +4,4 @@ Invalid use: underscore next to 0x
 <?php
 0x_0123;
 --EXPECTF--
-Parse error: syntax error, unexpected 'x_0123' (T_STRING) in %s on line %d
+Parse error: syntax error, unexpected identifier "x_0123" in %s on line %d
diff --git a/Zend/tests/numeric_literal_separator_007.phpt b/Zend/tests/numeric_literal_separator_007.phpt
@@ -4,4 +4,4 @@ Invalid use: underscore next to 0b
 <?php
 0b_0101;
 --EXPECTF--
-Parse error: syntax error, unexpected 'b_0101' (T_STRING) in %s on line %d
+Parse error: syntax error, unexpected identifier "b_0101" in %s on line %d
diff --git a/Zend/tests/numeric_literal_separator_008.phpt b/Zend/tests/numeric_literal_separator_008.phpt
@@ -4,4 +4,4 @@ Invalid use: underscore left of e
 <?php
 1_e2;
 --EXPECTF--
-Parse error: syntax error, unexpected '_e2' (T_STRING) in %s on line %d
+Parse error: syntax error, unexpected identifier "_e2" in %s on line %d
diff --git a/Zend/tests/numeric_literal_separator_009.phpt b/Zend/tests/numeric_literal_separator_009.phpt
@@ -4,4 +4,4 @@ Invalid use: underscore right of e
 <?php
 1e_2;
 --EXPECTF--
-Parse error: syntax error, unexpected 'e_2' (T_STRING) in %s on line %d
+Parse error: syntax error, unexpected identifier "e_2" in %s on line %d
diff --git a/Zend/tests/require_parse_exception.phpt b/Zend/tests/require_parse_exception.phpt
@@ -45,7 +45,7 @@ var_dump("\u{ffffff}");');
 Deprecated: Directive 'allow_url_include' is deprecated in Unknown on line 0
 Unclosed '{' on line 2
 Unclosed '{' on line 3
-syntax error, unexpected end of file, expecting '(' on line 2
+syntax error, unexpected end of file, expecting "(" on line 2
 Invalid numeric literal on line 2
 Invalid UTF-8 codepoint escape sequence on line 2
 Invalid UTF-8 codepoint escape sequence: Codepoint too large on line 2
diff --git a/Zend/tests/traits/bug55524.phpt b/Zend/tests/traits/bug55524.phpt
@@ -12,4 +12,4 @@ trait Foo extends Base {
 echo 'DONE';
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected 'extends' (T_EXTENDS), expecting '{' in %s on line %d
+Parse error: syntax error, unexpected token "extends", expecting "{" in %s on line %d
diff --git a/Zend/tests/traits/bugs/interfaces.phpt b/Zend/tests/traits/bugs/interfaces.phpt
@@ -16,4 +16,4 @@ trait THello implements MyInterface {
 
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected 'implements' (T_IMPLEMENTS), expecting '{' in %s on line %d
+Parse error: syntax error, unexpected token "implements", expecting "{" in %s on line %d
diff --git a/Zend/tests/type_declarations/mixed/casting/mixed_cast_error.phpt b/Zend/tests/type_declarations/mixed/casting/mixed_cast_error.phpt
@@ -7,4 +7,4 @@ $foo = (mixed) 12;
 
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected '12' (T_LNUMBER) in %s on line %d
+Parse error: syntax error, unexpected integer "12" in %s on line %d
diff --git a/Zend/tests/type_declarations/static_type_param.phpt b/Zend/tests/type_declarations/static_type_param.phpt
@@ -12,4 +12,4 @@ class Test {
 
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected 'static' (T_STATIC), expecting variable (T_VARIABLE) in %s on line %d
+Parse error: syntax error, unexpected token "static", expecting variable in %s on line %d
diff --git a/Zend/tests/type_declarations/static_type_property.phpt b/Zend/tests/type_declarations/static_type_property.phpt
@@ -10,4 +10,4 @@ class Test {
 
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected 'static' (T_STATIC) in %s on line %d
+Parse error: syntax error, unexpected token "static" in %s on line %d
diff --git a/Zend/tests/type_declarations/typed_properties_025.phpt b/Zend/tests/type_declarations/typed_properties_025.phpt
@@ -8,4 +8,4 @@ class Foo {
 }
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected 'int' (T_STRING), expecting variable (T_VARIABLE) in %s on line 4
+Parse error: syntax error, unexpected identifier "int", expecting variable in %s on line %d
diff --git a/Zend/tests/varSyntax/globalNonSimpleVariableError.phpt b/Zend/tests/varSyntax/globalNonSimpleVariableError.phpt
@@ -7,4 +7,4 @@ global $$foo->bar;
 
 ?>
 --EXPECTF--
-Parse error: syntax error, unexpected '->' (T_OBJECT_OPERATOR), expecting ';' or ',' in %s on line %d
+Parse error: syntax error, unexpected token "->", expecting ";" or "," in %s on line %d
diff --git a/Zend/zend_language_parser.y b/Zend/zend_language_parser.y
diff --git a/ext/reflection/tests/bug74454.phpt b/ext/reflection/tests/bug74454.phpt
diff --git a/ext/spl/tests/bug74372.phpt b/ext/spl/tests/bug74372.phpt
diff --git a/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_000.phpt b/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_000.phpt
diff --git a/ext/tokenizer/tests/token_get_all_heredoc_nowdoc.phpt b/ext/tokenizer/tests/token_get_all_heredoc_nowdoc.phpt
diff --git a/tests/lang/bug21820.phpt b/tests/lang/bug21820.phpt
diff --git a/tests/lang/bug71897.phpt b/tests/lang/bug71897.phpt
