nsz Git - cparser/blob - lexer.c

   1 #include <config.h>
   2
   3 #include "lexer.h"
   4 #include "token_t.h"
   5 #include "symbol_table_t.h"
   6 #include "adt/error.h"
   7 #include "adt/strset.h"
   8 #include "adt/util.h"
   9 #include "type_t.h"
  10 #include "target_architecture.h"
  11
  12 #include <assert.h>
  13 #include <errno.h>
  14 #include <string.h>
  15 #include <stdbool.h>
  16 #include <ctype.h>
  17
  18 //#define DEBUG_CHARS
  19 #define MAX_PUTBACK 3
  20
  21 #ifdef _WIN32
  22 /* No strtold on windows and no replacement yet */
  23 #define strtold(s, e) strtod(s, e)
  24 #endif
  25
  26 #if defined HAS_SIGNED_CHAR
  27 typedef signed char char_type;
  28 #elif defined HAS_UNSIGNED_CHAR
  29 typedef unsigned char char_type;
  30 #else
  31 #       error signedness of char not determined
  32 #endif
  33
  34 static int         c;
  35 token_t            lexer_token;
  36 symbol_t          *symbol_L;
  37 static FILE       *input;
  38 static char        buf[1024 + MAX_PUTBACK];
  39 static const char *bufend;
  40 static const char *bufpos;
  41 static strset_t    stringset;
  42
  43 static type_t     *type_int        = NULL;
  44 static type_t     *type_uint       = NULL;
  45 static type_t     *type_long       = NULL;
  46 static type_t     *type_ulong      = NULL;
  47 static type_t     *type_longlong   = NULL;
  48 static type_t     *type_ulonglong  = NULL;
  49 static type_t     *type_float      = NULL;
  50 static type_t     *type_double     = NULL;
  51 static type_t     *type_longdouble = NULL;
  52
  53 static void error_prefix_at(const char *input_name, unsigned linenr)
  54 {
  55         fprintf(stderr, "%s:%u: Error: ", input_name, linenr);
  56 }
  57
  58 static void error_prefix(void)
  59 {
  60         error_prefix_at(lexer_token.source_position.input_name,
  61                         lexer_token.source_position.linenr);
  62 }
  63
  64 static void parse_error(const char *msg)
  65 {
  66         error_prefix();
  67         fprintf(stderr, "%s\n", msg);
  68 }
  69
  70 static inline void next_real_char(void)
  71 {
  72         bufpos++;
  73         if(bufpos >= bufend) {
  74                 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
  75                                  input);
  76                 if(s == 0) {
  77                         c = EOF;
  78                         return;
  79                 }
  80                 bufpos = buf + MAX_PUTBACK;
  81                 bufend = buf + MAX_PUTBACK + s;
  82         }
  83         c = *(bufpos);
  84 }
  85
  86 static inline void put_back(int pc)
  87 {
  88         assert(bufpos >= buf);
  89         //assert(bufpos < buf+MAX_PUTBACK || *bufpos == pc);
  90
  91         char *p = buf + (bufpos - buf);
  92         *p = (char) pc;
  93
  94         /* going backwards in the buffer is legal as long as it's not more often
  95          * than MAX_PUTBACK */
  96         bufpos--;
  97
  98 #ifdef DEBUG_CHARS
  99         printf("putback '%c'\n", pc);
 100 #endif
 101 }
 102
 103 static inline void next_char(void);
 104
 105 #define MATCH_NEWLINE(code)                   \
 106         case '\r':                                \
 107                 next_char();                          \
 108                 if(c == '\n') {                       \
 109                         next_char();                      \
 110                 }                                     \
 111                 lexer_token.source_position.linenr++; \
 112                 code;                                 \
 113         case '\n':                                \
 114                 next_char();                          \
 115                 lexer_token.source_position.linenr++; \
 116                 code;
 117
 118 #define eat(c_type)  do { assert(c == c_type); next_char(); } while(0)
 119
 120 static void maybe_concat_lines(void)
 121 {
 122         eat('\\');
 123
 124         switch(c) {
 125         MATCH_NEWLINE(return;)
 126
 127         default:
 128                 break;
 129         }
 130
 131         put_back(c);
 132         c = '\\';
 133 }
 134
 135 static inline void next_char(void)
 136 {
 137         next_real_char();
 138
 139         /* filter trigraphs */
 140         if(UNLIKELY(c == '\\')) {
 141                 maybe_concat_lines();
 142                 goto end_of_next_char;
 143         }
 144
 145         if(LIKELY(c != '?'))
 146                 goto end_of_next_char;
 147
 148         next_real_char();
 149         if(LIKELY(c != '?')) {
 150                 put_back(c);
 151                 c = '?';
 152                 goto end_of_next_char;
 153         }
 154
 155         next_real_char();
 156         switch(c) {
 157         case '=': c = '#'; break;
 158         case '(': c = '['; break;
 159         case '/': c = '\\'; maybe_concat_lines(); break;
 160         case ')': c = ']'; break;
 161         case '\'': c = '^'; break;
 162         case '<': c = '{'; break;
 163         case '!': c = '|'; break;
 164         case '>': c = '}'; break;
 165         case '-': c = '~'; break;
 166         default:
 167                 put_back('?');
 168                 put_back(c);
 169                 c = '?';
 170                 break;
 171         }
 172
 173 end_of_next_char:;
 174 #ifdef DEBUG_CHARS
 175         printf("nchar '%c'\n", c);
 176 #endif
 177 }
 178
 179 #define SYMBOL_CHARS  \
 180         case 'a':         \
 181         case 'b':         \
 182         case 'c':         \
 183         case 'd':         \
 184         case 'e':         \
 185         case 'f':         \
 186         case 'g':         \
 187         case 'h':         \
 188         case 'i':         \
 189         case 'j':         \
 190         case 'k':         \
 191         case 'l':         \
 192         case 'm':         \
 193         case 'n':         \
 194         case 'o':         \
 195         case 'p':         \
 196         case 'q':         \
 197         case 'r':         \
 198         case 's':         \
 199         case 't':         \
 200         case 'u':         \
 201         case 'v':         \
 202         case 'w':         \
 203         case 'x':         \
 204         case 'y':         \
 205         case 'z':         \
 206         case 'A':         \
 207         case 'B':         \
 208         case 'C':         \
 209         case 'D':         \
 210         case 'E':         \
 211         case 'F':         \
 212         case 'G':         \
 213         case 'H':         \
 214         case 'I':         \
 215         case 'J':         \
 216         case 'K':         \
 217         case 'L':         \
 218         case 'M':         \
 219         case 'N':         \
 220         case 'O':         \
 221         case 'P':         \
 222         case 'Q':         \
 223         case 'R':         \
 224         case 'S':         \
 225         case 'T':         \
 226         case 'U':         \
 227         case 'V':         \
 228         case 'W':         \
 229         case 'X':         \
 230         case 'Y':         \
 231         case 'Z':         \
 232         case '_':
 233
 234 #define DIGITS        \
 235         case '0':         \
 236         case '1':         \
 237         case '2':         \
 238         case '3':         \
 239         case '4':         \
 240         case '5':         \
 241         case '6':         \
 242         case '7':         \
 243         case '8':         \
 244         case '9':
 245
 246 static void parse_symbol(void)
 247 {
 248         symbol_t *symbol;
 249         char     *string;
 250
 251         obstack_1grow(&symbol_obstack, (char) c);
 252         next_char();
 253
 254         while(1) {
 255                 switch(c) {
 256                 DIGITS
 257                 SYMBOL_CHARS
 258                         obstack_1grow(&symbol_obstack, (char) c);
 259                         next_char();
 260                         break;
 261
 262                 default:
 263                         goto end_symbol;
 264                 }
 265         }
 266
 267 end_symbol:
 268         obstack_1grow(&symbol_obstack, '\0');
 269
 270         string = obstack_finish(&symbol_obstack);
 271         symbol = symbol_table_insert(string);
 272
 273         lexer_token.type     = symbol->ID;
 274         lexer_token.v.symbol = symbol;
 275
 276         if(symbol->string != string) {
 277                 obstack_free(&symbol_obstack, string);
 278         }
 279 }
 280
 281 static void parse_integer_suffix(bool is_oct_hex)
 282 {
 283         bool is_unsigned  = false;
 284         bool min_long     = false;
 285         bool min_longlong = false;
 286
 287         if(c == 'U' || c == 'u') {
 288                 is_unsigned = true;
 289                 next_char();
 290                 if(c == 'L' || c == 'l') {
 291                         min_long = true;
 292                         next_char();
 293                         if(c == 'L' || c == 'l') {
 294                                 min_longlong = true;
 295                                 next_char();
 296                         }
 297                 }
 298         } else if(c == 'l' || c == 'L') {
 299                 min_long = true;
 300                 next_char();
 301                 if(c == 'l' || c == 'L') {
 302                         min_longlong = true;
 303                         next_char();
 304                         if(c == 'u' || c == 'U') {
 305                                 is_unsigned = true;
 306                                 next_char();
 307                         }
 308                 } else if(c == 'u' || c == 'U') {
 309                         is_unsigned = true;
 310                         next_char();
 311                         lexer_token.datatype = type_ulong;
 312                 }
 313         }
 314
 315         if(!is_unsigned) {
 316                 long long v = lexer_token.v.intvalue;
 317                 if(!min_long) {
 318                         if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
 319                                 lexer_token.datatype = type_int;
 320                                 return;
 321                         } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
 322                                 lexer_token.datatype = type_uint;
 323                                 return;
 324                         }
 325                 }
 326                 if(!min_longlong) {
 327                         if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
 328                                 lexer_token.datatype = type_long;
 329                                 return;
 330                         } else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) {
 331                                 lexer_token.datatype = type_ulong;
 332                                 return;
 333                         }
 334                 }
 335                 unsigned long long uv = (unsigned long long) v;
 336                 if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
 337                         lexer_token.datatype = type_ulonglong;
 338                         return;
 339                 }
 340
 341                 lexer_token.datatype = type_longlong;
 342         } else {
 343                 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
 344                 if(!min_long && v <= TARGET_UINT_MAX) {
 345                         lexer_token.datatype = type_uint;
 346                         return;
 347                 }
 348                 if(!min_longlong && v <= TARGET_ULONG_MAX) {
 349                         lexer_token.datatype = type_ulong;
 350                         return;
 351                 }
 352                 lexer_token.datatype = type_ulonglong;
 353         }
 354 }
 355
 356 static void parse_floating_suffix(void)
 357 {
 358         switch(c) {
 359         /* TODO: do something usefull with the suffixes... */
 360         case 'f':
 361         case 'F':
 362                 next_char();
 363                 lexer_token.datatype = type_float;
 364                 break;
 365         case 'l':
 366         case 'L':
 367                 next_char();
 368                 lexer_token.datatype = type_longdouble;
 369                 break;
 370         default:
 371                 lexer_token.datatype = type_double;
 372                 break;
 373         }
 374 }
 375
 376 /**
 377  * A replacement for strtoull. Only those parts needed for
 378  * our parser are implemented.
 379  */
 380 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
 381         unsigned long long v = 0;
 382
 383         switch (base) {
 384         case 16:
 385                 for (;; ++s) {
 386                         /* check for overrun */
 387                         if (v >= 0x1000000000000000ULL)
 388                                 break;
 389                         switch (tolower(*s)) {
 390                         case '0': v <<= 4; break;
 391                         case '1': v <<= 4; v |= 0x1; break;
 392                         case '2': v <<= 4; v |= 0x2; break;
 393                         case '3': v <<= 4; v |= 0x3; break;
 394                         case '4': v <<= 4; v |= 0x4; break;
 395                         case '5': v <<= 4; v |= 0x5; break;
 396                         case '6': v <<= 4; v |= 0x6; break;
 397                         case '7': v <<= 4; v |= 0x7; break;
 398                         case '8': v <<= 4; v |= 0x8; break;
 399                         case '9': v <<= 4; v |= 0x9; break;
 400                         case 'a': v <<= 4; v |= 0xa; break;
 401                         case 'b': v <<= 4; v |= 0xb; break;
 402                         case 'c': v <<= 4; v |= 0xc; break;
 403                         case 'd': v <<= 4; v |= 0xd; break;
 404                         case 'e': v <<= 4; v |= 0xe; break;
 405                         case 'f': v <<= 4; v |= 0xf; break;
 406                         default:
 407                                 goto end;
 408                         }
 409                 }
 410                 break;
 411         case 8:
 412                 for (;; ++s) {
 413                         /* check for overrun */
 414                         if (v >= 0x2000000000000000ULL)
 415                                 break;
 416                         switch (tolower(*s)) {
 417                         case '0': v <<= 3; break;
 418                         case '1': v <<= 3; v |= 1; break;
 419                         case '2': v <<= 3; v |= 2; break;
 420                         case '3': v <<= 3; v |= 3; break;
 421                         case '4': v <<= 3; v |= 4; break;
 422                         case '5': v <<= 3; v |= 5; break;
 423                         case '6': v <<= 3; v |= 6; break;
 424                         case '7': v <<= 3; v |= 7; break;
 425                         default:
 426                                 goto end;
 427                         }
 428                 }
 429                 break;
 430         case 10:
 431                 for (;; ++s) {
 432                         /* check for overrun */
 433                         if (v > 0x1999999999999999ULL)
 434                                 break;
 435                         switch (tolower(*s)) {
 436                         case '0': v *= 10; break;
 437                         case '1': v *= 10; v += 1; break;
 438                         case '2': v *= 10; v += 2; break;
 439                         case '3': v *= 10; v += 3; break;
 440                         case '4': v *= 10; v += 4; break;
 441                         case '5': v *= 10; v += 5; break;
 442                         case '6': v *= 10; v += 6; break;
 443                         case '7': v *= 10; v += 7; break;
 444                         case '8': v *= 10; v += 8; break;
 445                         case '9': v *= 10; v += 9; break;
 446                         default:
 447                                 goto end;
 448                         }
 449                 }
 450                 break;
 451         default:
 452                 assert(0);
 453                 break;
 454         }
 455 end:
 456         *endptr = s;
 457         return v;
 458 }
 459
 460 static void parse_number_hex(void)
 461 {
 462         assert(c == 'x' || c == 'X');
 463         next_char();
 464
 465         while(isxdigit(c)) {
 466                 obstack_1grow(&symbol_obstack, (char) c);
 467                 next_char();
 468         }
 469         obstack_1grow(&symbol_obstack, '\0');
 470         char *string = obstack_finish(&symbol_obstack);
 471
 472         if(c == '.' || c == 'p' || c == 'P') {
 473                 next_char();
 474                 panic("Hex floating point numbers not implemented yet");
 475         }
 476         if(*string == '\0') {
 477                 parse_error("invalid hex number");
 478                 lexer_token.type = T_ERROR;
 479         }
 480
 481         const char *endptr;
 482         lexer_token.type       = T_INTEGER;
 483         lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
 484         if(*endptr != '\0') {
 485                 parse_error("hex number literal too long");
 486         }
 487
 488         obstack_free(&symbol_obstack, string);
 489         parse_integer_suffix(true);
 490 }
 491
 492 static inline bool is_octal_digit(int chr)
 493 {
 494         return '0' <= chr && chr <= '7';
 495 }
 496
 497 static void parse_number_oct(void)
 498 {
 499         while(is_octal_digit(c)) {
 500                 obstack_1grow(&symbol_obstack, (char) c);
 501                 next_char();
 502         }
 503         obstack_1grow(&symbol_obstack, '\0');
 504         char *string = obstack_finish(&symbol_obstack);
 505
 506         const char *endptr;
 507         lexer_token.type       = T_INTEGER;
 508         lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
 509         if(*endptr != '\0') {
 510                 parse_error("octal number literal too long");
 511         }
 512
 513         obstack_free(&symbol_obstack, string);
 514         parse_integer_suffix(true);
 515 }
 516
 517 static void parse_number_dec(void)
 518 {
 519         bool is_float = false;
 520         while(isdigit(c)) {
 521                 obstack_1grow(&symbol_obstack, (char) c);
 522                 next_char();
 523         }
 524
 525         if(c == '.') {
 526                 obstack_1grow(&symbol_obstack, '.');
 527                 next_char();
 528
 529                 while(isdigit(c)) {
 530                         obstack_1grow(&symbol_obstack, (char) c);
 531                         next_char();
 532                 }
 533                 is_float = true;
 534         }
 535         if(c == 'e' || c == 'E') {
 536                 obstack_1grow(&symbol_obstack, 'e');
 537                 next_char();
 538
 539                 if(c == '-' || c == '+') {
 540                         obstack_1grow(&symbol_obstack, (char) c);
 541                         next_char();
 542                 }
 543
 544                 while(isdigit(c)) {
 545                         obstack_1grow(&symbol_obstack, (char) c);
 546                         next_char();
 547                 }
 548                 is_float = true;
 549         }
 550
 551         obstack_1grow(&symbol_obstack, '\0');
 552         char *string = obstack_finish(&symbol_obstack);
 553
 554         if(is_float) {
 555                 char *endptr;
 556                 lexer_token.type         = T_FLOATINGPOINT;
 557                 lexer_token.v.floatvalue = strtold(string, &endptr);
 558
 559                 if(*endptr != '\0') {
 560                         parse_error("invalid number literal");
 561                 }
 562
 563                 parse_floating_suffix();
 564         } else {
 565                 const char *endptr;
 566                 lexer_token.type       = T_INTEGER;
 567                 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
 568
 569                 if(*endptr != '\0') {
 570                         parse_error("invalid number literal");
 571                 }
 572
 573                 parse_integer_suffix(false);
 574         }
 575         obstack_free(&symbol_obstack, string);
 576 }
 577
 578 static void parse_number(void)
 579 {
 580         if (c == '0') {
 581                 next_char();
 582                 switch (c) {
 583                         case 'X':
 584                         case 'x':
 585                                 parse_number_hex();
 586                                 break;
 587                         case '0':
 588                         case '1':
 589                         case '2':
 590                         case '3':
 591                         case '4':
 592                         case '5':
 593                         case '6':
 594                         case '7':
 595                                 parse_number_oct();
 596                                 break;
 597                         case '8':
 598                         case '9':
 599                                 next_char();
 600                                 parse_error("invalid octal number");
 601                                 lexer_token.type = T_ERROR;
 602                                 return;
 603                         case '.':
 604                         case 'e':
 605                         case 'E':
 606                         default:
 607                                 obstack_1grow(&symbol_obstack, '0');
 608                                 parse_number_dec();
 609                                 return;
 610                 }
 611         } else {
 612                 parse_number_dec();
 613         }
 614 }
 615
 616 static int parse_octal_sequence(const int first_digit)
 617 {
 618         assert(is_octal_digit(first_digit));
 619         int value = first_digit - '0';
 620         if (!is_octal_digit(c)) return value;
 621         value = 8 * value + c - '0';
 622         next_char();
 623         if (!is_octal_digit(c)) return value;
 624         value = 8 * value + c - '0';
 625         next_char();
 626         return (char_type)value;
 627 }
 628
 629 static int parse_hex_sequence(void)
 630 {
 631         int value = 0;
 632         while(1) {
 633                 if (c >= '0' && c <= '9') {
 634                         value = 16 * value + c - '0';
 635                 } else if ('A' <= c && c <= 'F') {
 636                         value = 16 * value + c - 'A' + 10;
 637                 } else if ('a' <= c && c <= 'f') {
 638                         value = 16 * value + c - 'a' + 10;
 639                 } else {
 640                         break;
 641                 }
 642                 next_char();
 643         }
 644
 645         return (char_type)value;
 646 }
 647
 648 static int parse_escape_sequence(void)
 649 {
 650         eat('\\');
 651
 652         int ec = c;
 653         next_char();
 654
 655         switch(ec) {
 656         case '"':  return '"';
 657         case '\'': return '\'';
 658         case '\\': return '\\';
 659         case '?': return '\?';
 660         case 'a': return '\a';
 661         case 'b': return '\b';
 662         case 'f': return '\f';
 663         case 'n': return '\n';
 664         case 'r': return '\r';
 665         case 't': return '\t';
 666         case 'v': return '\v';
 667         case 'x':
 668                 return parse_hex_sequence();
 669         case '0':
 670         case '1':
 671         case '2':
 672         case '3':
 673         case '4':
 674         case '5':
 675         case '6':
 676         case '7':
 677                 return parse_octal_sequence(ec);
 678         case EOF:
 679                 parse_error("reached end of file while parsing escape sequence");
 680                 return EOF;
 681         default:
 682                 parse_error("unknown escape sequence");
 683                 return EOF;
 684         }
 685 }
 686
 687 const char *concat_strings(const char *s1, const char *s2)
 688 {
 689         size_t  len1   = strlen(s1);
 690         size_t  len2   = strlen(s2);
 691
 692         char   *concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
 693         memcpy(concat, s1, len1);
 694         memcpy(concat + len1, s2, len2 + 1);
 695
 696         const char *result = strset_insert(&stringset, concat);
 697         if(result != concat) {
 698                 obstack_free(&symbol_obstack, concat);
 699         }
 700
 701         return result;
 702 }
 703
 704 static void parse_string_literal(void)
 705 {
 706         unsigned    start_linenr = lexer_token.source_position.linenr;
 707         char       *string;
 708         const char *result;
 709
 710         assert(c == '"');
 711         next_char();
 712
 713         int tc;
 714         while(1) {
 715                 switch(c) {
 716                 case '\\':
 717                         tc = parse_escape_sequence();
 718                         obstack_1grow(&symbol_obstack, (char) tc);
 719                         break;
 720
 721                 case EOF:
 722                         error_prefix_at(lexer_token.source_position.input_name,
 723                                         start_linenr);
 724                         fprintf(stderr, "string has no end\n");
 725                         lexer_token.type = T_ERROR;
 726                         return;
 727
 728                 case '"':
 729                         next_char();
 730                         goto end_of_string;
 731
 732                 default:
 733                         obstack_1grow(&symbol_obstack, (char) c);
 734                         next_char();
 735                         break;
 736                 }
 737         }
 738
 739 end_of_string:
 740
 741         /* TODO: concatenate multiple strings separated by whitespace... */
 742
 743         /* add finishing 0 to the string */
 744         obstack_1grow(&symbol_obstack, '\0');
 745         string = obstack_finish(&symbol_obstack);
 746
 747         /* check if there is already a copy of the string */
 748         result = strset_insert(&stringset, string);
 749         if(result != string) {
 750                 obstack_free(&symbol_obstack, string);
 751         }
 752
 753         lexer_token.type     = T_STRING_LITERAL;
 754         lexer_token.v.string = result;
 755 }
 756
 757 static void parse_wide_string_literal(void)
 758 {
 759         const unsigned start_linenr = lexer_token.source_position.linenr;
 760
 761         assert(c == '"');
 762         next_char();
 763
 764         while(1) {
 765                 switch(c) {
 766                         case '\\': {
 767                                 wchar_rep_t tc = parse_escape_sequence();
 768                                 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 769                                 break;
 770                         }
 771
 772                         case EOF:
 773                                 error_prefix_at(lexer_token.source_position.input_name,
 774                                                 start_linenr);
 775                                 fprintf(stderr, "string has no end\n");
 776                                 lexer_token.type = T_ERROR;
 777                                 return;
 778
 779                         case '"':
 780                                 next_char();
 781                                 goto end_of_string;
 782
 783                         default: {
 784                                 wchar_rep_t tc = c;
 785                                 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 786                                 next_char();
 787                                 break;
 788                         }
 789                 }
 790         }
 791
 792 end_of_string:;
 793
 794         /* TODO: concatenate multiple strings separated by whitespace... */
 795
 796         /* add finishing 0 to the string */
 797         wchar_rep_t nul = L'\0';
 798         obstack_grow(&symbol_obstack, &nul, sizeof(nul));
 799         const size_t             size   = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
 800         const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
 801
 802 #if 0 /* TODO hash */
 803         /* check if there is already a copy of the string */
 804         const wchar_rep_t *const result = strset_insert(&stringset, string);
 805         if(result != string) {
 806                 obstack_free(&symbol_obstack, string);
 807         }
 808 #else
 809         const wchar_rep_t *const result = string;
 810 #endif
 811
 812         lexer_token.type                = T_WIDE_STRING_LITERAL;
 813         lexer_token.v.wide_string.begin = result;
 814         lexer_token.v.wide_string.size  = size;
 815 }
 816
 817 static void parse_character_constant(void)
 818 {
 819         eat('\'');
 820
 821         int found_char = 0;
 822         while(1) {
 823                 switch(c) {
 824                 case '\\':
 825                         found_char = parse_escape_sequence();
 826                         break;
 827
 828                 MATCH_NEWLINE(
 829                         parse_error("newline while parsing character constant");
 830                         break;
 831                 )
 832
 833                 case '\'':
 834                         next_char();
 835                         goto end_of_char_constant;
 836
 837                 case EOF:
 838                         parse_error("EOF while parsing character constant");
 839                         lexer_token.type = T_ERROR;
 840                         return;
 841
 842                 default:
 843                         if(found_char != 0) {
 844                                 parse_error("more than 1 characters in character "
 845                                             "constant");
 846                                 goto end_of_char_constant;
 847                         } else {
 848                                 found_char = c;
 849                                 next_char();
 850                         }
 851                         break;
 852                 }
 853         }
 854
 855 end_of_char_constant:
 856         lexer_token.type       = T_INTEGER;
 857         lexer_token.v.intvalue = found_char;
 858 }
 859
 860 static void skip_multiline_comment(void)
 861 {
 862         unsigned start_linenr = lexer_token.source_position.linenr;
 863
 864         while(1) {
 865                 switch(c) {
 866                 case '*':
 867                         next_char();
 868                         if(c == '/') {
 869                                 next_char();
 870                                 return;
 871                         }
 872                         break;
 873
 874                 MATCH_NEWLINE(break;)
 875
 876                 case EOF:
 877                         error_prefix_at(lexer_token.source_position.input_name,
 878                                         start_linenr);
 879                         fprintf(stderr, "at end of file while looking for comment end\n");
 880                         return;
 881
 882                 default:
 883                         next_char();
 884                         break;
 885                 }
 886         }
 887 }
 888
 889 static void skip_line_comment(void)
 890 {
 891         while(1) {
 892                 switch(c) {
 893                 case EOF:
 894                         return;
 895
 896                 case '\n':
 897                 case '\r':
 898                         return;
 899
 900                 default:
 901                         next_char();
 902                         break;
 903                 }
 904         }
 905 }
 906
 907 static token_t pp_token;
 908
 909 static inline void next_pp_token(void)
 910 {
 911         lexer_next_preprocessing_token();
 912         pp_token = lexer_token;
 913 }
 914
 915 static void eat_until_newline(void)
 916 {
 917         while(pp_token.type != '\n' && pp_token.type != T_EOF) {
 918                 next_pp_token();
 919         }
 920 }
 921
 922 static void error_directive(void)
 923 {
 924         error_prefix();
 925         fprintf(stderr, "#error directive: \n");
 926
 927         /* parse pp-tokens until new-line */
 928 }
 929
 930 static void define_directive(void)
 931 {
 932         lexer_next_preprocessing_token();
 933         if(lexer_token.type != T_IDENTIFIER) {
 934                 parse_error("expected identifier after #define\n");
 935                 eat_until_newline();
 936         }
 937 }
 938
 939 static void ifdef_directive(int is_ifndef)
 940 {
 941         (void) is_ifndef;
 942         lexer_next_preprocessing_token();
 943         //expect_identifier();
 944         //extect_newline();
 945 }
 946
 947 static void endif_directive(void)
 948 {
 949         //expect_newline();
 950 }
 951
 952 static void parse_line_directive(void)
 953 {
 954         if(pp_token.type != T_INTEGER) {
 955                 parse_error("expected integer");
 956         } else {
 957                 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
 958                 next_pp_token();
 959         }
 960         if(pp_token.type == T_STRING_LITERAL) {
 961                 lexer_token.source_position.input_name = pp_token.v.string;
 962                 next_pp_token();
 963         }
 964
 965         eat_until_newline();
 966 }
 967
 968 static void parse_preprocessor_identifier(void)
 969 {
 970         assert(pp_token.type == T_IDENTIFIER);
 971         symbol_t *symbol = pp_token.v.symbol;
 972
 973         switch(symbol->pp_ID) {
 974         case TP_include:
 975                 printf("include - enable header name parsing!\n");
 976                 break;
 977         case TP_define:
 978                 define_directive();
 979                 break;
 980         case TP_ifdef:
 981                 ifdef_directive(0);
 982                 break;
 983         case TP_ifndef:
 984                 ifdef_directive(1);
 985                 break;
 986         case TP_endif:
 987                 endif_directive();
 988                 break;
 989         case TP_line:
 990                 next_pp_token();
 991                 parse_line_directive();
 992                 break;
 993         case TP_if:
 994         case TP_else:
 995         case TP_elif:
 996         case TP_undef:
 997         case TP_error:
 998                 error_directive();
 999                 break;
1000         case TP_pragma:
1001                 break;
1002         }
1003 }
1004
1005 static void parse_preprocessor_directive(void)
1006 {
1007         next_pp_token();
1008
1009         switch(pp_token.type) {
1010         case T_IDENTIFIER:
1011                 parse_preprocessor_identifier();
1012                 break;
1013         case T_INTEGER:
1014                 parse_line_directive();
1015                 break;
1016         default:
1017                 parse_error("invalid preprocessor directive");
1018                 eat_until_newline();
1019                 break;
1020         }
1021 }
1022
1023 #define MAYBE_PROLOG                                       \
1024                         next_char();                                   \
1025                         while(1) {                                     \
1026                                 switch(c) {
1027
1028 #define MAYBE(ch, set_type)                                \
1029                                 case ch:                                   \
1030                                         next_char();                           \
1031                                         lexer_token.type = set_type;           \
1032                                         return;
1033
1034 #define ELSE_CODE(code)                                    \
1035                                 default:                                   \
1036                                         code;                                  \
1037                                 }                                          \
1038                         } /* end of while(1) */                        \
1039                         break;
1040
1041 #define ELSE(set_type)                                     \
1042                 ELSE_CODE(                                         \
1043                         lexer_token.type = set_type;                   \
1044                         return;                                        \
1045                 )
1046
1047 void lexer_next_preprocessing_token(void)
1048 {
1049         while(1) {
1050                 switch(c) {
1051                 case ' ':
1052                 case '\t':
1053                         next_char();
1054                         break;
1055
1056                 MATCH_NEWLINE(
1057                         lexer_token.type = '\n';
1058                         return;
1059                 )
1060
1061                 SYMBOL_CHARS
1062                         parse_symbol();
1063                         /* might be a wide string ( L"string" ) */
1064                         if(c == '"' && (lexer_token.type == T_IDENTIFIER &&
1065                            lexer_token.v.symbol == symbol_L)) {
1066                                 parse_wide_string_literal();
1067                         }
1068                         return;
1069
1070                 DIGITS
1071                         parse_number();
1072                         return;
1073
1074                 case '"':
1075                         parse_string_literal();
1076                         return;
1077
1078                 case '\'':
1079                         parse_character_constant();
1080                         return;
1081
1082                 case '.':
1083                         MAYBE_PROLOG
1084                                 case '.':
1085                                         MAYBE_PROLOG
1086                                         MAYBE('.', T_DOTDOTDOT)
1087                                         ELSE_CODE(
1088                                                 put_back(c);
1089                                                 c = '.';
1090                                                 lexer_token.type = '.';
1091                                                 return;
1092                                         )
1093                         ELSE('.')
1094                 case '&':
1095                         MAYBE_PROLOG
1096                         MAYBE('&', T_ANDAND)
1097                         MAYBE('=', T_ANDEQUAL)
1098                         ELSE('&')
1099                 case '*':
1100                         MAYBE_PROLOG
1101                         MAYBE('=', T_ASTERISKEQUAL)
1102                         ELSE('*')
1103                 case '+':
1104                         MAYBE_PROLOG
1105                         MAYBE('+', T_PLUSPLUS)
1106                         MAYBE('=', T_PLUSEQUAL)
1107                         ELSE('+')
1108                 case '-':
1109                         MAYBE_PROLOG
1110                         MAYBE('>', T_MINUSGREATER)
1111                         MAYBE('-', T_MINUSMINUS)
1112                         MAYBE('=', T_MINUSEQUAL)
1113                         ELSE('-')
1114                 case '!':
1115                         MAYBE_PROLOG
1116                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1117                         ELSE('!')
1118                 case '/':
1119                         MAYBE_PROLOG
1120                         MAYBE('=', T_SLASHEQUAL)
1121                                 case '*':
1122                                         next_char();
1123                                         skip_multiline_comment();
1124                                         lexer_next_preprocessing_token();
1125                                         return;
1126                                 case '/':
1127                                         next_char();
1128                                         skip_line_comment();
1129                                         lexer_next_preprocessing_token();
1130                                         return;
1131                         ELSE('/')
1132                 case '%':
1133                         MAYBE_PROLOG
1134                         MAYBE('>', T_PERCENTGREATER)
1135                         MAYBE('=', T_PERCENTEQUAL)
1136                                 case ':':
1137                                         MAYBE_PROLOG
1138                                                 case '%':
1139                                                         MAYBE_PROLOG
1140                                                         MAYBE(':', T_PERCENTCOLONPERCENTCOLON)
1141                                                         ELSE_CODE(
1142                                                                 put_back(c);
1143                                                                 c = '%';
1144                                                                 lexer_token.type = T_PERCENTCOLON;
1145                                                                 return;
1146                                                         )
1147                                         ELSE(T_PERCENTCOLON)
1148                         ELSE('%')
1149                 case '<':
1150                         MAYBE_PROLOG
1151                         MAYBE(':', T_LESSCOLON)
1152                         MAYBE('%', T_LESSPERCENT)
1153                         MAYBE('=', T_LESSEQUAL)
1154                                 case '<':
1155                                         MAYBE_PROLOG
1156                                         MAYBE('=', T_LESSLESSEQUAL)
1157                                         ELSE(T_LESSLESS)
1158                         ELSE('<')
1159                 case '>':
1160                         MAYBE_PROLOG
1161                         MAYBE('=', T_GREATEREQUAL)
1162                                 case '>':
1163                                         MAYBE_PROLOG
1164                                         MAYBE('=', T_GREATERGREATEREQUAL)
1165                                         ELSE(T_GREATERGREATER)
1166                         ELSE('>')
1167                 case '^':
1168                         MAYBE_PROLOG
1169                         MAYBE('=', T_CARETEQUAL)
1170                         ELSE('^')
1171                 case '|':
1172                         MAYBE_PROLOG
1173                         MAYBE('=', T_PIPEEQUAL)
1174                         MAYBE('|', T_PIPEPIPE)
1175                         ELSE('|')
1176                 case ':':
1177                         MAYBE_PROLOG
1178                         MAYBE('>', T_COLONGREATER)
1179                         ELSE(':')
1180                 case '=':
1181                         MAYBE_PROLOG
1182                         MAYBE('=', T_EQUALEQUAL)
1183                         ELSE('=')
1184                 case '#':
1185                         MAYBE_PROLOG
1186                         MAYBE('#', T_HASHHASH)
1187                         ELSE('#')
1188
1189                 case '?':
1190                 case '[':
1191                 case ']':
1192                 case '(':
1193                 case ')':
1194                 case '{':
1195                 case '}':
1196                 case '~':
1197                 case ';':
1198                 case ',':
1199                 case '\\':
1200                         lexer_token.type = c;
1201                         next_char();
1202                         return;
1203
1204                 case EOF:
1205                         lexer_token.type = T_EOF;
1206                         return;
1207
1208                 default:
1209                         next_char();
1210                         error_prefix();
1211                         fprintf(stderr, "unknown character '%c' found\n", c);
1212                         lexer_token.type = T_ERROR;
1213                         return;
1214                 }
1215         }
1216 }
1217
1218 void lexer_next_token(void)
1219 {
1220         lexer_next_preprocessing_token();
1221         if(lexer_token.type != '\n')
1222                 return;
1223
1224 newline_found:
1225         do {
1226                 lexer_next_preprocessing_token();
1227         } while(lexer_token.type == '\n');
1228
1229         if(lexer_token.type == '#') {
1230                 parse_preprocessor_directive();
1231                 goto newline_found;
1232         }
1233 }
1234
1235 void init_lexer(void)
1236 {
1237         strset_init(&stringset);
1238
1239         type_int       = make_atomic_type(ATOMIC_TYPE_INT, TYPE_QUALIFIER_NONE);
1240         type_uint      = make_atomic_type(ATOMIC_TYPE_UINT, TYPE_QUALIFIER_NONE);
1241         type_long      = make_atomic_type(ATOMIC_TYPE_LONG, TYPE_QUALIFIER_NONE);
1242         type_ulong     = make_atomic_type(ATOMIC_TYPE_ULONG, TYPE_QUALIFIER_NONE);
1243         type_longlong  = make_atomic_type(ATOMIC_TYPE_LONGLONG,
1244                                           TYPE_QUALIFIER_NONE);
1245         type_ulonglong = make_atomic_type(ATOMIC_TYPE_ULONGLONG,
1246                                           TYPE_QUALIFIER_NONE);
1247
1248         type_float      = make_atomic_type(ATOMIC_TYPE_FLOAT, TYPE_QUALIFIER_CONST);
1249         type_double     = make_atomic_type(ATOMIC_TYPE_DOUBLE,
1250                                            TYPE_QUALIFIER_CONST);
1251         type_longdouble = make_atomic_type(ATOMIC_TYPE_LONG_DOUBLE,
1252                                            TYPE_QUALIFIER_CONST);
1253 }
1254
1255 void lexer_open_stream(FILE *stream, const char *input_name)
1256 {
1257         input                                  = stream;
1258         lexer_token.source_position.linenr     = 0;
1259         lexer_token.source_position.input_name = input_name;
1260
1261         symbol_L = symbol_table_insert("L");
1262
1263         /* place a virtual \n at the beginning so the lexer knows that we're
1264          * at the beginning of a line */
1265         c = '\n';
1266 }
1267
1268 void exit_lexer(void)
1269 {
1270         strset_destroy(&stringset);
1271 }
1272
1273 static __attribute__((unused))
1274 void dbg_pos(const source_position_t source_position)
1275 {
1276         fprintf(stdout, "%s:%u\n", source_position.input_name,
1277                 source_position.linenr);
1278         fflush(stdout);
1279 }