#define TS(x,str,val)
#endif
-TS(IDENTIFIER, "identifier", = 256)
-TS(INTEGER, "integer number",)
-TS(FLOATINGPOINT, "floatingpoint number",)
-TS(STRING_LITERAL, "string literal",)
-TS(WIDE_STRING_LITERAL, "wide string literal",)
+TS(IDENTIFIER, "identifier", = 256)
+TS(INTEGER, "integer number",)
+TS(CHARACTER_CONSTANT, "character constant",)
+TS(WIDE_CHARACTER_CONSTANT, "wide character constant",)
+TS(FLOATINGPOINT, "floatingpoint number",)
+TS(STRING_LITERAL, "string literal",)
+TS(WIDE_STRING_LITERAL, "wide string literal",)
+
+#define PUNCTUATOR(name,string,val) T(_ALL,name,string,val)
+#include "tokens_punctuator.inc"
+#undef PUNCTUATOR
#define S(mode,x) T(mode,x,#x,)
S(_ALL, auto)
S(_ALL, register)
S(_ALL, return)
S(_ALL, short)
-S(_ANSI|_C99, signed)
S(_ALL, sizeof)
S(_ALL, static)
S(_ALL, struct)
S(_ALL, while)
S(_C99|_GNUC, _Bool)
-S(_C99|_GNUC, _Complex)
-S(_C99|_GNUC, _Imaginary)
-S(_GNUC, __thread)
-S(_GNUC, __extension__)
-S(_GNUC, __builtin_classify_type)
-S(_GNUC, __builtin_va_list)
-S(_GNUC, __builtin_expect)
-S(_GNUC, __builtin_offsetof)
-S(_GNUC, __builtin_va_arg)
-S(_GNUC, __builtin_va_end)
-S(_GNUC, __builtin_alloca)
-S(_GNUC, __builtin_nanf)
-S(_GNUC, __builtin_nan)
-S(_GNUC, __builtin_nand)
-S(_GNUC, __builtin_isgreater)
-S(_GNUC, __builtin_isgreaterequal)
-S(_GNUC, __builtin_isless)
-S(_GNUC, __builtin_islessequal)
-S(_GNUC, __builtin_islessgreater)
-S(_GNUC, __builtin_isunordered)
-S(_GNUC, __builtin_constant_p)
-S(_GNUC, __builtin_prefetch)
-S(_C99, __PRETTY_FUNCTION__)
+S(_ALL, __thread)
+S(_ALL, __extension__)
+S(_ALL, __builtin_classify_type)
+S(_ALL, __builtin_va_list)
+S(_ALL, __builtin_expect)
+S(_ALL, __builtin_offsetof)
+S(_ALL, __builtin_va_arg)
+S(_ALL, __builtin_va_end)
+S(_ALL, __builtin_alloca)
+S(_ALL, __builtin_inf)
+S(_ALL, __builtin_inff)
+S(_ALL, __builtin_infl)
+S(_ALL, __builtin_nan)
+S(_ALL, __builtin_nanf)
+S(_ALL, __builtin_nanl)
+S(_ALL, __builtin_isgreater)
+S(_ALL, __builtin_isgreaterequal)
+S(_ALL, __builtin_isless)
+S(_ALL, __builtin_islessequal)
+S(_ALL, __builtin_islessgreater)
+S(_ALL, __builtin_isunordered)
+S(_ALL, __builtin_constant_p)
+S(_ALL, __builtin_prefetch)
+S(_ALL, __builtin_huge_val)
+S(_ALL, __PRETTY_FUNCTION__)
S(_ALL, __FUNCTION__)
+S(_ALL, __label__)
S(_C99, __func__)
-S(_C99, __alignof__)
+S(_MS, __FUNCSIG__)
+S(_MS, __FUNCDNAME__)
#undef S
-T(_GNUC, real, "__real__",)
-T(_GNUC, _real, "__real", = T_real)
-T(_GNUC, imag, "__imag__",)
-T(_GNUC, _imag, "__imag", = T_imag)
-T(_GNUC|_MS, __alignof, "__alignof", = T___alignof__)
-T(_MS, _alignof, "_alignof", = T___alignof__)
-T(_ANSI|_C99, const, "const",)
-T(_GNUC, _const, "__const", = T_const)
-T(_GNUC|_MS, restrict, "__restrict",)
-T(_GNUC, _restrict_, "__restrict__", = T_restrict)
-T(_C99, _restrict, "restrict", = T_restrict)
-T(_ALL, asm, "asm",)
-T(_GNUC, __asm__, "__asm__", = T_asm)
-T(_ANSI|_C99, volatile, "volatile",)
-T(_GNUC, __volatile__, "__volatile__", = T_volatile)
-T(_C99, inline, "inline",)
-T(_GNUC|_MS, __inline, "__inline", = T_inline)
-T(_GNUC, __inline__, "__inline__", = T_inline)
-T(_GNUC, typeof, "typeof",)
-T(_GNUC, __typeof, "__typeof", = T_typeof)
-T(_GNUC, __typeof__, "__typeof__", = T_typeof)
-T(_GNUC, attribute, "__attribute",)
-T(_GNUC, __attribute__, "__attribute__", = T_attribute)
-
-T(_GNUC, __builtin_va_start, "__builtin_va_start",)
-T(_GNUC, __builtin_stdarg_start, "__builtin_stdarg_start", = T___builtin_va_start)
-
-T(_MS, near, "_near",)
-T(_MS, _near, "__near", = T_near)
-T(_MS, far, "_far",)
-T(_MS, _far, "__far", = T_far)
-T(_MS, _asm, "_asm", = T_asm)
-T(_MS, __asm, "__asm", = T_asm)
-T(_MS, cdecl, "cdecl",)
-T(_MS, _cdecl, "_cdecl", = T_cdecl)
-T(_MS, __cdecl, "__cdecl", = T_cdecl)
-T(_MS, stdcall, "_stdcall",)
-T(_MS, __stdcall, "__stdcall", = T_stdcall)
-T(_MS, fastcall, "_fastcall",)
-T(_MS, __fastcall, "__fastcall", = T_fastcall)
-T(_MS, thiscall, "_thiscall",)
-T(_MS, __thiscall, "__thiscall", = T_thiscall)
-T(_MS, forceinline, "_forceinline",)
-T(_MS, __forceinline, "__forceinline", = T_forceinline)
-T(_MS, unaligned, "_unaligned",)
-T(_MS, __unaligned, "__unaligned", = T_unaligned)
-T(_MS, assume, "_assume",)
-T(_MS, __assume, "__assume", = T_assume)
-T(_MS, try, "_try",)
-T(_MS, __try, "__try", = T_try)
-T(_MS, finally, "_finally",)
-T(_MS, __finally, "__finally", = T_finally)
-T(_MS, leave, "_leave",)
-T(_MS, __leave, "__leave", = T_leave)
-T(_MS, except, "_except",)
-T(_MS, __except, "__except", = T_except)
-T(_MS, declspec, "_declspec",)
-T(_MS, __declspec, "__declspec", = T_declspec)
-T(_MS, based, "_based",)
-T(_MS, __based, "__based", = T_based)
-
-T(_MS, int8, "_int8",)
-T(_MS, __int8, "__int8", = T_int8)
-T(_MS, int16, "_int16",)
-T(_MS, __int16, "__int16", = T_int16)
-T(_MS, int32, "_int32",)
-T(_MS, __int32, "__int32", = T_int32)
-T(_MS, int64, "_int64",)
-T(_MS, __int64, "__int64", = T_int64)
-
-#define _T(x,str,val) T(_ALL,x,str,val)
+/* needed on Windows */
+#ifdef __w64
+#undef __w64
+#endif
+#ifdef __ptr64
+#undef __ptr64
+#endif
-_T(MINUSGREATER, "->",)
-_T(PLUSPLUS, "++",)
-_T(MINUSMINUS, "--",)
-_T(LESSLESS, "<<",)
-_T(GREATERGREATER, ">>",)
-_T(LESSEQUAL, "<=",)
-_T(GREATEREQUAL, ">=",)
-_T(EQUALEQUAL, "==",)
-_T(EXCLAMATIONMARKEQUAL, "!=",)
-_T(ANDAND, "&&",)
-_T(PIPEPIPE, "||",)
-_T(DOTDOTDOT, "...",)
-_T(ASTERISKEQUAL, "*=",)
-_T(SLASHEQUAL, "/=",)
-_T(PERCENTEQUAL, "%=",)
-_T(PLUSEQUAL, "+=",)
-_T(MINUSEQUAL, "-=",)
-_T(LESSLESSEQUAL, "<<=",)
-_T(GREATERGREATEREQUAL, ">>=",)
-_T(ANDEQUAL, "&=",)
-_T(CARETEQUAL, "^=",)
-_T(PIPEEQUAL, "|=",)
-_T(HASHHASH, "##",)
+/* needed on darwin... */
+#ifdef __signed
+#undef __signed
+#endif
+#ifdef __volatile
+#undef __volatile
+#endif
+#ifdef __const
+#undef __const
+#endif
+#ifdef __restrict
+#undef __restrict
+#endif
-#define T_LAST_TOKEN (T_HASHHASH+1)
+#define S(mode, x, val) T(mode, x, #x, val)
+S(_ANSI|_C99, signed, )
+S(_ALL, __signed, = T_signed)
+S(_ALL, __signed__, = T_signed)
+S(_C99|_GNUC, _Complex, )
+S(_ALL, __complex__, = T__Complex)
+S(_ALL, __complex, = T__Complex)
+S(_C99|_GNUC, _Imaginary, )
+S(_ALL, __real__, )
+S(_ALL, __real, = T___real__)
+S(_ALL, __imag__, )
+S(_ALL, __imag, = T___imag__)
+S(_ALL, __alignof__, )
+S(_ALL, __alignof, = T___alignof__)
+S(_MS, _alignof, = T___alignof__)
+S(_ANSI|_C99, const, )
+S(_ALL, __const, = T_const)
+S(_ALL, __const__, = T_const)
+S(_C99, restrict, )
+S(_ALL, __restrict__, = T_restrict)
+S(_ALL, __restrict, = T_restrict)
+S(_MS, _restrict, = T_restrict)
+S(_ALL, asm, )
+S(_ALL, __asm__, = T_asm)
+S(_MS, _asm, = T_asm)
+S(_ALL, __asm, = T_asm)
+S(_ANSI|_C99, volatile, )
+S(_ALL, __volatile, = T_volatile)
+S(_ALL, __volatile__, = T_volatile)
+S(_C99, inline, )
+S(_ALL, __inline, = T_inline)
+S(_ALL, __inline__, = T_inline)
+S(_GNUC, typeof, )
+S(_ALL, __typeof, = T_typeof)
+S(_ALL, __typeof__, = T_typeof)
+S(_ALL, __attribute__, )
-_T(LESSCOLON, "<:", = '[')
-_T(COLONGREATER, ":>", = ']')
-_T(LESSPERCENT, "<%", = '{')
-_T(PERCENTGREATER, "%>", = '}')
-_T(PERCENTCOLON, "%:", = '#')
-_T(PERCENTCOLONPERCENTCOLON, "%:%:", = T_HASHHASH)
+S(_ALL, __builtin_va_start, )
+S(_ALL, __builtin_stdarg_start, = T___builtin_va_start)
-_T(RBRACK, "[", = '[')
-_T(LBRACK, "]", = ']')
-_T(LBRACE, "(", = '(')
-_T(RBRACE, ")", = ')')
-_T(RCURLY, "{", = '{')
-_T(LCURLY, "}", = '}')
-_T(DOT, ".", = '.')
-_T(AND, "&", = '&')
-_T(ASTERISK, "*", = '*')
-_T(PLUS, "+", = '+')
-_T(MINUS, "-", = '-')
-_T(TILDE, "~", = '~')
-_T(EXCLAMATIONMARK, "!", = '!')
-_T(SLASH, "/", = '/')
-_T(PERCENT, "%", = '%')
-_T(LESS, "<", = '<')
-_T(GREATER, ">", = '>')
-_T(CARET, "^", = '^')
-_T(PIPE, "|", = '|')
-_T(QUESTIONMARK, "?", = '?')
-_T(COLON, ":", = ':')
-_T(SEMICOLON, ";", = ';')
-_T(EQUAL, "=", = '=')
-_T(COMMA, ",", = ',')
-_T(HASH, "#", = '#')
+S(_MS, _near, )
+S(_MS, __near, = T__near)
+S(_MS, _far, )
+S(_MS, __far, = T__far)
+S(_MS, cdecl, )
+S(_MS, _cdecl, = T_cdecl)
+S(_MS, __cdecl, = T_cdecl)
+S(_MS, _stdcall, )
+S(_MS, __stdcall, = T__stdcall)
+S(_MS, _fastcall, )
+S(_MS, __fastcall, = T__fastcall)
+S(_MS, __thiscall, )
+S(_MS, _forceinline, )
+S(_MS, __forceinline, = T__forceinline)
+S(_MS, __unaligned, )
+S(_MS, _assume, )
+S(_MS, __assume, = T__assume)
+S(_MS, _try, )
+S(_MS, __try, = T__try)
+S(_MS, _finally, )
+S(_MS, __finally, = T__finally)
+S(_MS, _leave, )
+S(_MS, __leave, = T__leave)
+S(_MS, _except, )
+S(_MS, __except, = T__except)
+S(_MS, _declspec, )
+S(_MS, __declspec, = T__declspec)
+S(_MS, _based, )
+S(_MS, __based, = T__based)
+S(_MS, __noop, )
-#undef _T
+S(_MS, __ptr32, )
+S(_MS, __ptr64, )
+S(_MS, __sptr, )
+S(_MS, __uptr, )
+S(_MS, _w64, )
+S(_MS, __w64, = T__w64)
-TS(NEWLINE, "newline", = '\n')
+S(_MS, _int8, )
+S(_MS, __int8, = T__int8)
+S(_MS, _int16, )
+S(_MS, __int16, = T__int16)
+S(_MS, _int32, )
+S(_MS, __int32, = T__int32)
+S(_MS, _int64, )
+S(_MS, __int64, = T__int64)
+S(_MS, _int128, )
+S(_MS, __int128, = T__int128)
+#undef S