some refactoring in preparation for a preprocessor

[cparser] / tokens.inc

diff --git a/tokens.inc b/tokens.inc
index e01ffd4..7074783 100644 (file)
--- a/tokens.inc
+++ b/tokens.inc
@@ -2,11 +2,17 @@
 #define TS(x,str,val)
 #endif
 
-TS(IDENTIFIER,          "identifier", = 256)
-TS(INTEGER,             "integer number",)
-TS(FLOATINGPOINT,       "floatingpoint number",)
-TS(STRING_LITERAL,      "string literal",)
-TS(WIDE_STRING_LITERAL, "wide string literal",)
+TS(IDENTIFIER,              "identifier", = 256)
+TS(INTEGER,                 "integer number",)
+TS(CHARACTER_CONSTANT,      "character constant",)
+TS(WIDE_CHARACTER_CONSTANT, "wide character constant",)
+TS(FLOATINGPOINT,           "floatingpoint number",)
+TS(STRING_LITERAL,          "string literal",)
+TS(WIDE_STRING_LITERAL,     "wide string literal",)
+
+#define PUNCTUATOR(name,string,val)   T(_ALL,name,string,val)
+#include "tokens_punctuator.inc"
+#undef PUNCTUATOR
 
 #define S(mode,x)   T(mode,x,#x,)
 S(_ALL, auto)
@@ -61,25 +67,30 @@ S(_GNUC, __builtin_isless)
 S(_GNUC, __builtin_islessequal)
 S(_GNUC, __builtin_islessgreater)
 S(_GNUC, __builtin_isunordered)
-S(_C99, __PRETTY_FUNCTION__)
+S(_GNUC, __builtin_constant_p)
+S(_GNUC, __builtin_prefetch)
+S(_GNUC, __PRETTY_FUNCTION__)
 S(_ALL, __FUNCTION__)
 S(_C99, __func__)
-S(_C99, __alignof__)
 #undef S
 
-T(_GNUC,      real,           "__real__",)
-T(_GNUC,      _real,          "__real",                 = T_real)
-T(_GNUC,      imag,           "__imag__",)
-T(_GNUC,      _imag,          "__imag",                 = T_imag)
+T(_GNUC,      __real__,       "__real__",)
+T(_GNUC,      __real,         "__real",                 = T___real__)
+T(_GNUC,      __imag__,       "__imag__",)
+T(_GNUC,      __imag,         "__imag",                 = T___imag__)
+T(_GNUC|_MS,  __alignof__,    "__alignof__",)
 T(_GNUC|_MS,  __alignof,      "__alignof",              = T___alignof__)
 T(_MS,        _alignof,       "_alignof",               = T___alignof__)
 T(_ANSI|_C99, const,          "const",)
-T(_GNUC,      _const,         "__const",                = T_const)
-T(_GNUC|_MS,  restrict,       "__restrict",)
-T(_GNUC,      _restrict_,     "__restrict__",           = T_restrict)
-T(_C99,       _restrict,      "restrict",               = T_restrict)
+T(_GNUC,      __const,        "__const",                = T_const)
+T(_C99,       restrict,       "restrict",)
+T(_GNUC,      __restrict__,   "__restrict__",           = T_restrict)
+T(_GNUC,      __restrict,     "__restrict",             = T_restrict)
+T(_MS,        _restrict,      "_restrict",              = T_restrict)
 T(_ALL,       asm,            "asm",)
 T(_GNUC,      __asm__,        "__asm__",                = T_asm)
+T(_MS,        _asm,           "_asm",                   = T_asm)
+T(_GNUC|_MS,  __asm,          "__asm",                  = T_asm)
 T(_ANSI|_C99, volatile,       "volatile",)
 T(_GNUC,      __volatile__,   "__volatile__",           = T_volatile)
 T(_C99,       inline,         "inline",)
@@ -88,8 +99,7 @@ T(_GNUC,      __inline__,     "__inline__",             = T_inline)
 T(_GNUC,      typeof,         "typeof",)
 T(_GNUC,      __typeof,       "__typeof",               = T_typeof)
 T(_GNUC,      __typeof__,     "__typeof__",             = T_typeof)
-T(_GNUC,      attribute,      "__attribute",)
-T(_GNUC,      __attribute__,  "__attribute__",          = T_attribute)
+T(_GNUC,      __attribute__,  "__attribute__",)
 
 T(_GNUC,     __builtin_va_start,     "__builtin_va_start",)
 T(_GNUC,     __builtin_stdarg_start, "__builtin_stdarg_start", = T___builtin_va_start)
@@ -98,8 +108,6 @@ T(_MS,       near,            "_near",)
 T(_MS,       _near,           "__near",                  = T_near)
 T(_MS,       far,             "_far",)
 T(_MS,       _far,            "__far",                   = T_far)
-T(_MS,      _asm,             "_asm",                    = T_asm)
-T(_MS,      __asm,            "__asm",                   = T_asm)
 T(_MS,      cdecl,            "cdecl",)
 T(_MS,      _cdecl,           "_cdecl",                  = T_cdecl)
 T(_MS,      __cdecl,          "__cdecl",                 = T_cdecl)
@@ -137,67 +145,6 @@ T(_MS,      __int32,          "__int32",                 = T_int32)
 T(_MS,      int64,            "_int64",)
 T(_MS,      __int64,          "__int64",                 = T_int64)
 
-#define _T(x,str,val) T(_ALL,x,str,val)
-
-_T(MINUSGREATER,             "->",)
-_T(PLUSPLUS,                 "++",)
-_T(MINUSMINUS,               "--",)
-_T(LESSLESS,                 "<<",)
-_T(GREATERGREATER,           ">>",)
-_T(LESSEQUAL,                "<=",)
-_T(GREATEREQUAL,             ">=",)
-_T(EQUALEQUAL,               "==",)
-_T(EXCLAMATIONMARKEQUAL,     "!=",)
-_T(ANDAND,                   "&&",)
-_T(PIPEPIPE,                 "||",)
-_T(DOTDOTDOT,                "...",)
-_T(ASTERISKEQUAL,            "*=",)
-_T(SLASHEQUAL,               "/=",)
-_T(PERCENTEQUAL,             "%=",)
-_T(PLUSEQUAL,                "+=",)
-_T(MINUSEQUAL,               "-=",)
-_T(LESSLESSEQUAL,            "<<=",)
-_T(GREATERGREATEREQUAL,      ">>=",)
-_T(ANDEQUAL,                 "&=",)
-_T(CARETEQUAL,               "^=",)
-_T(PIPEEQUAL,                "|=",)
-_T(HASHHASH,                 "##",)
-
-#define T_LAST_TOKEN  (T_HASHHASH+1)
-
-_T(LESSCOLON,                "<:",   = '[')
-_T(COLONGREATER,             ":>",   = ']')
-_T(LESSPERCENT,              "<%",   = '{')
-_T(PERCENTGREATER,           "%>",   = '}')
-_T(PERCENTCOLON,             "%:",   = '#')
-_T(PERCENTCOLONPERCENTCOLON, "%:%:", = T_HASHHASH)
-
-_T(RBRACK,          "[", = '[')
-_T(LBRACK,          "]", = ']')
-_T(LBRACE,          "(", = '(')
-_T(RBRACE,          ")", = ')')
-_T(RCURLY,          "{", = '{')
-_T(LCURLY,          "}", = '}')
-_T(DOT,             ".", = '.')
-_T(AND,             "&", = '&')
-_T(ASTERISK,        "*", = '*')
-_T(PLUS,            "+", = '+')
-_T(MINUS,           "-", = '-')
-_T(TILDE,           "~", = '~')
-_T(EXCLAMATIONMARK, "!", = '!')
-_T(SLASH,           "/", = '/')
-_T(PERCENT,         "%", = '%')
-_T(LESS,            "<", = '<')
-_T(GREATER,         ">", = '>')
-_T(CARET,           "^", = '^')
-_T(PIPE,            "|", = '|')
-_T(QUESTIONMARK,    "?", = '?')
-_T(COLON,           ":", = ':')
-_T(SEMICOLON,       ";", = ';')
-_T(EQUAL,           "=", = '=')
-_T(COMMA,           ",", = ',')
-_T(HASH,            "#", = '#')
+#define T_LAST_TOKEN  (T___int64+1)
 
 #undef _T
-
-TS(NEWLINE,        "newline", = '\n')