Correct token encoding: wide string literals L"" and -> shared the same enum value

- This resulted in wrong error messages when an unexpected -> was found in the input.
- Remove the magic 265 (used explicitly by ->) by reordering the tokens.

diff --git a/tokens.inc b/tokens.inc
index dff544f..686adf0 100644 (file)
--- a/tokens.inc
+++ b/tokens.inc
@@ -2,7 +2,14 @@
 #define TS(x,str,val)
 #endif
 
-TS(IDENTIFIER,                "identifier", = 256)
+/* These must go first. Some set explicit values. */
+#define ALTERNATE(name, val)          T(_CXX, name, #name,  val)
+#define PUNCTUATOR(name, string, val) T(_ALL, name, string, val)
+#include "tokens_punctuator.inc"
+#undef PUNCTUATOR
+#undef ALTERNATE
+
+TS(IDENTIFIER,                "identifier",)
 TS(INTEGER,                   "integer number",)
 TS(INTEGER_OCTAL,             "octal integer number",)
 TS(INTEGER_HEXADECIMAL,       "hexadecimal integer number",)
@@ -13,12 +20,6 @@ TS(WIDE_CHARACTER_CONSTANT,   "wide character constant",)
 TS(STRING_LITERAL,            "string literal",)
 TS(WIDE_STRING_LITERAL,       "wide string literal",)
 
-#define ALTERNATE(name, val)          T(_CXX, name, #name,  val)
-#define PUNCTUATOR(name, string, val) T(_ALL, name, string, val)
-#include "tokens_punctuator.inc"
-#undef PUNCTUATOR
-#undef ALTERNATE
-
 #define S(mode,x)   T(mode,x,#x,)
 S(_ALL, auto)
 S(_ALL, break)

diff --git a/tokens_punctuator.inc b/tokens_punctuator.inc
index 4646357..c01706e 100644 (file)
--- a/tokens_punctuator.inc
+++ b/tokens_punctuator.inc
@@ -36,7 +36,8 @@ PUNCTUATOR(EQUAL,           "=", = '=')
 PUNCTUATOR(COMMA,           ",", = ',')
 PUNCTUATOR(HASH,            "#", = '#')
 
-PUNCTUATOR(MINUSGREATER,             "->", = 265)
+/* Start after extended ASCII. */
+PUNCTUATOR(MINUSGREATER,             "->", = 256)
 PUNCTUATOR(PLUSPLUS,                 "++",)
 PUNCTUATOR(MINUSMINUS,               "--",)
 PUNCTUATOR(LESSLESS,                 "<<",)