nsz Git - cparser/blob - lexer.c

   1 #include <config.h>
   2
   3 #include "lexer.h"
   4 #include "token_t.h"
   5 #include "symbol_table_t.h"
   6 #include "adt/error.h"
   7 #include "adt/strset.h"
   8 #include "adt/util.h"
   9 #include "type_t.h"
  10 #include "target_architecture.h"
  11
  12 #include <assert.h>
  13 #include <errno.h>
  14 #include <string.h>
  15 #include <stdbool.h>
  16 #include <ctype.h>
  17
  18 //#define DEBUG_CHARS
  19 #define MAX_PUTBACK 3
  20
  21 #ifdef _WIN32
  22 /* No strtold on windows and no replacement yet */
  23 #define strtold(s, e) strtod(s, e)
  24 #endif
  25
  26 #if defined HAS_SIGNED_CHAR
  27 typedef signed char char_type;
  28 #elif defined HAS_UNSIGNED_CHAR
  29 typedef unsigned char char_type;
  30 #else
  31 #       error signedness of char not determined
  32 #endif
  33
  34 static int         c;
  35 token_t            lexer_token;
  36 symbol_t          *symbol_L;
  37 static FILE       *input;
  38 static char        buf[1024 + MAX_PUTBACK];
  39 static const char *bufend;
  40 static const char *bufpos;
  41 static strset_t    stringset;
  42
  43 static type_t     *type_int        = NULL;
  44 static type_t     *type_uint       = NULL;
  45 static type_t     *type_long       = NULL;
  46 static type_t     *type_ulong      = NULL;
  47 static type_t     *type_longlong   = NULL;
  48 static type_t     *type_ulonglong  = NULL;
  49 static type_t     *type_float      = NULL;
  50 static type_t     *type_double     = NULL;
  51 static type_t     *type_longdouble = NULL;
  52
  53 static void error_prefix_at(const char *input_name, unsigned linenr)
  54 {
  55         fprintf(stderr, "%s:%u: Error: ", input_name, linenr);
  56 }
  57
  58 static void error_prefix(void)
  59 {
  60         error_prefix_at(lexer_token.source_position.input_name,
  61                         lexer_token.source_position.linenr);
  62 }
  63
  64 static void parse_error(const char *msg)
  65 {
  66         error_prefix();
  67         fprintf(stderr, "%s\n", msg);
  68 }
  69
  70 static inline void next_real_char(void)
  71 {
  72         bufpos++;
  73         if(bufpos >= bufend) {
  74                 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
  75                                  input);
  76                 if(s == 0) {
  77                         c = EOF;
  78                         return;
  79                 }
  80                 bufpos = buf + MAX_PUTBACK;
  81                 bufend = buf + MAX_PUTBACK + s;
  82         }
  83         c = *(bufpos);
  84 }
  85
  86 static inline void put_back(int pc)
  87 {
  88         assert(bufpos >= buf);
  89         //assert(bufpos < buf+MAX_PUTBACK || *bufpos == pc);
  90
  91         char *p = buf + (bufpos - buf);
  92         *p = (char) pc;
  93
  94         /* going backwards in the buffer is legal as long as it's not more often
  95          * than MAX_PUTBACK */
  96         bufpos--;
  97
  98 #ifdef DEBUG_CHARS
  99         printf("putback '%c'\n", pc);
 100 #endif
 101 }
 102
 103 static inline void next_char(void);
 104
 105 #define MATCH_NEWLINE(code)                   \
 106         case '\r':                                \
 107                 next_char();                          \
 108                 if(c == '\n') {                       \
 109                         next_char();                      \
 110                 }                                     \
 111                 lexer_token.source_position.linenr++; \
 112                 code;                                 \
 113         case '\n':                                \
 114                 next_char();                          \
 115                 lexer_token.source_position.linenr++; \
 116                 code;
 117
 118 #define eat(c_type)  do { assert(c == c_type); next_char(); } while(0)
 119
 120 static void maybe_concat_lines(void)
 121 {
 122         eat('\\');
 123
 124         switch(c) {
 125         MATCH_NEWLINE(return;)
 126
 127         default:
 128                 break;
 129         }
 130
 131         put_back(c);
 132         c = '\\';
 133 }
 134
 135 static inline void next_char(void)
 136 {
 137         next_real_char();
 138
 139         /* filter trigraphs */
 140         if(UNLIKELY(c == '\\')) {
 141                 maybe_concat_lines();
 142                 goto end_of_next_char;
 143         }
 144
 145         if(LIKELY(c != '?'))
 146                 goto end_of_next_char;
 147
 148         next_real_char();
 149         if(LIKELY(c != '?')) {
 150                 put_back(c);
 151                 c = '?';
 152                 goto end_of_next_char;
 153         }
 154
 155         next_real_char();
 156         switch(c) {
 157         case '=': c = '#'; break;
 158         case '(': c = '['; break;
 159         case '/': c = '\\'; maybe_concat_lines(); break;
 160         case ')': c = ']'; break;
 161         case '\'': c = '^'; break;
 162         case '<': c = '{'; break;
 163         case '!': c = '|'; break;
 164         case '>': c = '}'; break;
 165         case '-': c = '~'; break;
 166         default:
 167                 put_back('?');
 168                 put_back(c);
 169                 c = '?';
 170                 break;
 171         }
 172
 173 end_of_next_char:;
 174 #ifdef DEBUG_CHARS
 175         printf("nchar '%c'\n", c);
 176 #endif
 177 }
 178
 179 #define SYMBOL_CHARS  \
 180         case 'a':         \
 181         case 'b':         \
 182         case 'c':         \
 183         case 'd':         \
 184         case 'e':         \
 185         case 'f':         \
 186         case 'g':         \
 187         case 'h':         \
 188         case 'i':         \
 189         case 'j':         \
 190         case 'k':         \
 191         case 'l':         \
 192         case 'm':         \
 193         case 'n':         \
 194         case 'o':         \
 195         case 'p':         \
 196         case 'q':         \
 197         case 'r':         \
 198         case 's':         \
 199         case 't':         \
 200         case 'u':         \
 201         case 'v':         \
 202         case 'w':         \
 203         case 'x':         \
 204         case 'y':         \
 205         case 'z':         \
 206         case 'A':         \
 207         case 'B':         \
 208         case 'C':         \
 209         case 'D':         \
 210         case 'E':         \
 211         case 'F':         \
 212         case 'G':         \
 213         case 'H':         \
 214         case 'I':         \
 215         case 'J':         \
 216         case 'K':         \
 217         case 'L':         \
 218         case 'M':         \
 219         case 'N':         \
 220         case 'O':         \
 221         case 'P':         \
 222         case 'Q':         \
 223         case 'R':         \
 224         case 'S':         \
 225         case 'T':         \
 226         case 'U':         \
 227         case 'V':         \
 228         case 'W':         \
 229         case 'X':         \
 230         case 'Y':         \
 231         case 'Z':         \
 232         case '_':
 233
 234 #define DIGITS        \
 235         case '0':         \
 236         case '1':         \
 237         case '2':         \
 238         case '3':         \
 239         case '4':         \
 240         case '5':         \
 241         case '6':         \
 242         case '7':         \
 243         case '8':         \
 244         case '9':
 245
 246 static void parse_symbol(void)
 247 {
 248         symbol_t *symbol;
 249         char     *string;
 250
 251         obstack_1grow(&symbol_obstack, (char) c);
 252         next_char();
 253
 254         while(1) {
 255                 switch(c) {
 256                 DIGITS
 257                 SYMBOL_CHARS
 258                         obstack_1grow(&symbol_obstack, (char) c);
 259                         next_char();
 260                         break;
 261
 262                 default:
 263                         goto end_symbol;
 264                 }
 265         }
 266
 267 end_symbol:
 268         obstack_1grow(&symbol_obstack, '\0');
 269
 270         string = obstack_finish(&symbol_obstack);
 271         symbol = symbol_table_insert(string);
 272
 273         lexer_token.type     = symbol->ID;
 274         lexer_token.v.symbol = symbol;
 275
 276         if(symbol->string != string) {
 277                 obstack_free(&symbol_obstack, string);
 278         }
 279 }
 280
 281 static void parse_integer_suffix(bool is_oct_hex)
 282 {
 283         bool is_unsigned  = false;
 284         bool min_long     = false;
 285         bool min_longlong = false;
 286
 287         if(c == 'U' || c == 'u') {
 288                 is_unsigned = true;
 289                 next_char();
 290                 if(c == 'L' || c == 'l') {
 291                         min_long = true;
 292                         next_char();
 293                         if(c == 'L' || c == 'l') {
 294                                 min_longlong = true;
 295                                 next_char();
 296                         }
 297                 }
 298         } else if(c == 'l' || c == 'L') {
 299                 min_long = true;
 300                 next_char();
 301                 if(c == 'l' || c == 'L') {
 302                         min_longlong = true;
 303                         next_char();
 304                         if(c == 'u' || c == 'U') {
 305                                 is_unsigned = true;
 306                                 next_char();
 307                         }
 308                 } else if(c == 'u' || c == 'U') {
 309                         is_unsigned = true;
 310                         next_char();
 311                         lexer_token.datatype = type_ulong;
 312                 }
 313         }
 314
 315         if(!is_unsigned) {
 316                 long long v = lexer_token.v.intvalue;
 317                 if(!min_long) {
 318                         if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
 319                                 lexer_token.datatype = type_int;
 320                                 return;
 321                         } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
 322                                 lexer_token.datatype = type_uint;
 323                                 return;
 324                         }
 325                 }
 326                 if(!min_longlong) {
 327                         if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
 328                                 lexer_token.datatype = type_long;
 329                                 return;
 330                         } else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) {
 331                                 lexer_token.datatype = type_ulong;
 332                                 return;
 333                         }
 334                 }
 335                 unsigned long long uv = (unsigned long long) v;
 336                 if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
 337                         lexer_token.datatype = type_ulonglong;
 338                         return;
 339                 }
 340
 341                 lexer_token.datatype = type_longlong;
 342         } else {
 343                 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
 344                 if(!min_long && v <= TARGET_UINT_MAX) {
 345                         lexer_token.datatype = type_uint;
 346                         return;
 347                 }
 348                 if(!min_longlong && v <= TARGET_ULONG_MAX) {
 349                         lexer_token.datatype = type_ulong;
 350                         return;
 351                 }
 352                 lexer_token.datatype = type_ulonglong;
 353         }
 354 }
 355
 356 static void parse_floating_suffix(void)
 357 {
 358         switch(c) {
 359         /* TODO: do something usefull with the suffixes... */
 360         case 'f':
 361         case 'F':
 362                 next_char();
 363                 lexer_token.datatype = type_float;
 364                 break;
 365         case 'l':
 366         case 'L':
 367                 next_char();
 368                 lexer_token.datatype = type_longdouble;
 369                 break;
 370         default:
 371                 lexer_token.datatype = type_double;
 372                 break;
 373         }
 374 }
 375
 376 /**
 377  * A replacement for strtoull. Only those parts needed for
 378  * our parser are implemented.
 379  */
 380 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
 381         unsigned long long v = 0;
 382
 383         switch (base) {
 384         case 16:
 385                 for (;; ++s) {
 386                         /* check for overrun */
 387                         if (v >= 0x1000000000000000ULL)
 388                                 break;
 389                         switch (tolower(*s)) {
 390                         case '0': v <<= 4; break;
 391                         case '1': v <<= 4; v |= 0x1; break;
 392                         case '2': v <<= 4; v |= 0x2; break;
 393                         case '3': v <<= 4; v |= 0x3; break;
 394                         case '4': v <<= 4; v |= 0x4; break;
 395                         case '5': v <<= 4; v |= 0x5; break;
 396                         case '6': v <<= 4; v |= 0x6; break;
 397                         case '7': v <<= 4; v |= 0x7; break;
 398                         case '8': v <<= 4; v |= 0x8; break;
 399                         case '9': v <<= 4; v |= 0x9; break;
 400                         case 'a': v <<= 4; v |= 0xa; break;
 401                         case 'b': v <<= 4; v |= 0xb; break;
 402                         case 'c': v <<= 4; v |= 0xc; break;
 403                         case 'd': v <<= 4; v |= 0xd; break;
 404                         case 'e': v <<= 4; v |= 0xe; break;
 405                         case 'f': v <<= 4; v |= 0xf; break;
 406                         default:
 407                                 goto end;
 408                         }
 409                 }
 410                 break;
 411         case 8:
 412                 for (;; ++s) {
 413                         /* check for overrun */
 414                         if (v >= 0x2000000000000000ULL)
 415                                 break;
 416                         switch (tolower(*s)) {
 417                         case '0': v <<= 3; break;
 418                         case '1': v <<= 3; v |= 1; break;
 419                         case '2': v <<= 3; v |= 2; break;
 420                         case '3': v <<= 3; v |= 3; break;
 421                         case '4': v <<= 3; v |= 4; break;
 422                         case '5': v <<= 3; v |= 5; break;
 423                         case '6': v <<= 3; v |= 6; break;
 424                         case '7': v <<= 3; v |= 7; break;
 425                         default:
 426                                 goto end;
 427                         }
 428                 }
 429                 break;
 430         case 10:
 431                 for (;; ++s) {
 432                         /* check for overrun */
 433                         if (v > 0x1999999999999999ULL)
 434                                 break;
 435                         switch (tolower(*s)) {
 436                         case '0': v *= 10; break;
 437                         case '1': v *= 10; v += 1; break;
 438                         case '2': v *= 10; v += 2; break;
 439                         case '3': v *= 10; v += 3; break;
 440                         case '4': v *= 10; v += 4; break;
 441                         case '5': v *= 10; v += 5; break;
 442                         case '6': v *= 10; v += 6; break;
 443                         case '7': v *= 10; v += 7; break;
 444                         case '8': v *= 10; v += 8; break;
 445                         case '9': v *= 10; v += 9; break;
 446                         default:
 447                                 goto end;
 448                         }
 449                 }
 450                 break;
 451         default:
 452                 assert(0);
 453                 break;
 454         }
 455 end:
 456         *endptr = s;
 457         return v;
 458 }
 459
 460 static void parse_number_hex(void)
 461 {
 462         assert(c == 'x' || c == 'X');
 463         next_char();
 464
 465         while(isxdigit(c)) {
 466                 obstack_1grow(&symbol_obstack, (char) c);
 467                 next_char();
 468         }
 469         obstack_1grow(&symbol_obstack, '\0');
 470         char *string = obstack_finish(&symbol_obstack);
 471
 472         if(c == '.' || c == 'p' || c == 'P') {
 473                 next_char();
 474                 panic("Hex floating point numbers not implemented yet");
 475         }
 476         if(*string == '\0') {
 477                 parse_error("invalid hex number");
 478                 lexer_token.type = T_ERROR;
 479         }
 480
 481         const char *endptr;
 482         lexer_token.type       = T_INTEGER;
 483         lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
 484         if(*endptr != '\0') {
 485                 parse_error("hex number literal too long");
 486         }
 487
 488         obstack_free(&symbol_obstack, string);
 489         parse_integer_suffix(true);
 490 }
 491
 492 static inline bool is_octal_digit(int chr)
 493 {
 494         return '0' <= chr && chr <= '7';
 495 }
 496
 497 static void parse_number_oct(void)
 498 {
 499         while(is_octal_digit(c)) {
 500                 obstack_1grow(&symbol_obstack, (char) c);
 501                 next_char();
 502         }
 503         obstack_1grow(&symbol_obstack, '\0');
 504         char *string = obstack_finish(&symbol_obstack);
 505
 506         const char *endptr;
 507         lexer_token.type       = T_INTEGER;
 508         lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
 509         if(*endptr != '\0') {
 510                 parse_error("octal number literal too long");
 511         }
 512
 513         obstack_free(&symbol_obstack, string);
 514         parse_integer_suffix(true);
 515 }
 516
 517 static void parse_number_dec(void)
 518 {
 519         bool is_float = false;
 520         while(isdigit(c)) {
 521                 obstack_1grow(&symbol_obstack, (char) c);
 522                 next_char();
 523         }
 524
 525         if(c == '.') {
 526                 obstack_1grow(&symbol_obstack, '.');
 527                 next_char();
 528
 529                 while(isdigit(c)) {
 530                         obstack_1grow(&symbol_obstack, (char) c);
 531                         next_char();
 532                 }
 533                 is_float = true;
 534         }
 535         if(c == 'e' || c == 'E') {
 536                 obstack_1grow(&symbol_obstack, 'e');
 537                 next_char();
 538
 539                 if(c == '-' || c == '+') {
 540                         obstack_1grow(&symbol_obstack, (char) c);
 541                         next_char();
 542                 }
 543
 544                 while(isdigit(c)) {
 545                         obstack_1grow(&symbol_obstack, (char) c);
 546                         next_char();
 547                 }
 548                 is_float = true;
 549         }
 550
 551         obstack_1grow(&symbol_obstack, '\0');
 552         char *string = obstack_finish(&symbol_obstack);
 553
 554         if(is_float) {
 555                 char *endptr;
 556                 lexer_token.type         = T_FLOATINGPOINT;
 557                 lexer_token.v.floatvalue = strtold(string, &endptr);
 558
 559                 if(*endptr != '\0') {
 560                         parse_error("invalid number literal");
 561                 }
 562
 563                 parse_floating_suffix();
 564         } else {
 565                 const char *endptr;
 566                 lexer_token.type       = T_INTEGER;
 567                 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
 568
 569                 if(*endptr != '\0') {
 570                         parse_error("invalid number literal");
 571                 }
 572
 573                 parse_integer_suffix(false);
 574         }
 575         obstack_free(&symbol_obstack, string);
 576 }
 577
 578 static void parse_number(void)
 579 {
 580         if (c == '0') {
 581                 next_char();
 582                 switch (c) {
 583                         case 'X':
 584                         case 'x':
 585                                 parse_number_hex();
 586                                 break;
 587                         case '0':
 588                         case '1':
 589                         case '2':
 590                         case '3':
 591                         case '4':
 592                         case '5':
 593                         case '6':
 594                         case '7':
 595                                 parse_number_oct();
 596                                 break;
 597                         case '8':
 598                         case '9':
 599                                 next_char();
 600                                 parse_error("invalid octal number");
 601                                 lexer_token.type = T_ERROR;
 602                                 return;
 603                         case '.':
 604                         case 'e':
 605                         case 'E':
 606                         default:
 607                                 obstack_1grow(&symbol_obstack, '0');
 608                                 parse_number_dec();
 609                                 return;
 610                 }
 611         } else {
 612                 parse_number_dec();
 613         }
 614 }
 615
 616 static int parse_octal_sequence(const int first_digit)
 617 {
 618         assert(is_octal_digit(first_digit));
 619         int value = first_digit - '0';
 620         if (!is_octal_digit(c)) return value;
 621         value = 8 * value + c - '0';
 622         next_char();
 623         if (!is_octal_digit(c)) return value;
 624         value = 8 * value + c - '0';
 625         next_char();
 626         return (char_type)value;
 627 }
 628
 629 static int parse_hex_sequence(void)
 630 {
 631         int value = 0;
 632         while(1) {
 633                 if (c >= '0' && c <= '9') {
 634                         value = 16 * value + c - '0';
 635                 } else if ('A' <= c && c <= 'F') {
 636                         value = 16 * value + c - 'A' + 10;
 637                 } else if ('a' <= c && c <= 'f') {
 638                         value = 16 * value + c - 'a' + 10;
 639                 } else {
 640                         break;
 641                 }
 642                 next_char();
 643         }
 644
 645         return (char_type)value;
 646 }
 647
 648 static int parse_escape_sequence(void)
 649 {
 650         eat('\\');
 651
 652         int ec = c;
 653         next_char();
 654
 655         switch(ec) {
 656         case '"':  return '"';
 657         case '\'': return '\'';
 658         case '\\': return '\\';
 659         case '?': return '\?';
 660         case 'a': return '\a';
 661         case 'b': return '\b';
 662         case 'f': return '\f';
 663         case 'n': return '\n';
 664         case 'r': return '\r';
 665         case 't': return '\t';
 666         case 'v': return '\v';
 667         case 'x':
 668                 return parse_hex_sequence();
 669         case '0':
 670         case '1':
 671         case '2':
 672         case '3':
 673         case '4':
 674         case '5':
 675         case '6':
 676         case '7':
 677                 return parse_octal_sequence(ec);
 678         case EOF:
 679                 parse_error("reached end of file while parsing escape sequence");
 680                 return EOF;
 681         default:
 682                 parse_error("unknown escape sequence");
 683                 return EOF;
 684         }
 685 }
 686
 687 const char *concat_strings(const char *s1, const char *s2)
 688 {
 689         size_t  len1   = strlen(s1);
 690         size_t  len2   = strlen(s2);
 691
 692         char   *concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
 693         memcpy(concat, s1, len1);
 694         memcpy(concat + len1, s2, len2 + 1);
 695
 696         const char *result = strset_insert(&stringset, concat);
 697         if(result != concat) {
 698                 obstack_free(&symbol_obstack, concat);
 699         }
 700
 701         return result;
 702 }
 703
 704 static void parse_string_literal(void)
 705 {
 706         unsigned    start_linenr = lexer_token.source_position.linenr;
 707         char       *string;
 708         const char *result;
 709
 710         assert(c == '"');
 711         next_char();
 712
 713         int tc;
 714         while(1) {
 715                 switch(c) {
 716                 case '\\':
 717                         tc = parse_escape_sequence();
 718                         obstack_1grow(&symbol_obstack, (char) tc);
 719                         break;
 720
 721                 case EOF:
 722                         error_prefix_at(lexer_token.source_position.input_name,
 723                                         start_linenr);
 724                         fprintf(stderr, "string has no end\n");
 725                         lexer_token.type = T_ERROR;
 726                         return;
 727
 728                 case '"':
 729                         next_char();
 730                         goto end_of_string;
 731
 732                 default:
 733                         obstack_1grow(&symbol_obstack, (char) c);
 734                         next_char();
 735                         break;
 736                 }
 737         }
 738
 739 end_of_string:
 740
 741         /* TODO: concatenate multiple strings separated by whitespace... */
 742
 743         /* add finishing 0 to the string */
 744         obstack_1grow(&symbol_obstack, '\0');
 745         string = obstack_finish(&symbol_obstack);
 746
 747         /* check if there is already a copy of the string */
 748         result = strset_insert(&stringset, string);
 749         if(result != string) {
 750                 obstack_free(&symbol_obstack, string);
 751         }
 752
 753         lexer_token.type     = T_STRING_LITERAL;
 754         lexer_token.v.string = result;
 755 }
 756
 757 static void parse_character_constant(void)
 758 {
 759         eat('\'');
 760
 761         int found_char = 0;
 762         while(1) {
 763                 switch(c) {
 764                 case '\\':
 765                         found_char = parse_escape_sequence();
 766                         break;
 767
 768                 MATCH_NEWLINE(
 769                         parse_error("newline while parsing character constant");
 770                         break;
 771                 )
 772
 773                 case '\'':
 774                         next_char();
 775                         goto end_of_char_constant;
 776
 777                 case EOF:
 778                         parse_error("EOF while parsing character constant");
 779                         lexer_token.type = T_ERROR;
 780                         return;
 781
 782                 default:
 783                         if(found_char != 0) {
 784                                 parse_error("more than 1 characters in character "
 785                                             "constant");
 786                                 goto end_of_char_constant;
 787                         } else {
 788                                 found_char = c;
 789                                 next_char();
 790                         }
 791                         break;
 792                 }
 793         }
 794
 795 end_of_char_constant:
 796         lexer_token.type       = T_INTEGER;
 797         lexer_token.v.intvalue = found_char;
 798 }
 799
 800 static void skip_multiline_comment(void)
 801 {
 802         unsigned start_linenr = lexer_token.source_position.linenr;
 803
 804         while(1) {
 805                 switch(c) {
 806                 case '*':
 807                         next_char();
 808                         if(c == '/') {
 809                                 next_char();
 810                                 return;
 811                         }
 812                         break;
 813
 814                 MATCH_NEWLINE(break;)
 815
 816                 case EOF:
 817                         error_prefix_at(lexer_token.source_position.input_name,
 818                                         start_linenr);
 819                         fprintf(stderr, "at end of file while looking for comment end\n");
 820                         return;
 821
 822                 default:
 823                         next_char();
 824                         break;
 825                 }
 826         }
 827 }
 828
 829 static void skip_line_comment(void)
 830 {
 831         while(1) {
 832                 switch(c) {
 833                 case EOF:
 834                         return;
 835
 836                 case '\n':
 837                 case '\r':
 838                         return;
 839
 840                 default:
 841                         next_char();
 842                         break;
 843                 }
 844         }
 845 }
 846
 847 static token_t pp_token;
 848
 849 static inline void next_pp_token(void)
 850 {
 851         lexer_next_preprocessing_token();
 852         pp_token = lexer_token;
 853 }
 854
 855 static void eat_until_newline(void)
 856 {
 857         while(pp_token.type != '\n' && pp_token.type != T_EOF) {
 858                 next_pp_token();
 859         }
 860 }
 861
 862 static void error_directive(void)
 863 {
 864         error_prefix();
 865         fprintf(stderr, "#error directive: \n");
 866
 867         /* parse pp-tokens until new-line */
 868 }
 869
 870 static void define_directive(void)
 871 {
 872         lexer_next_preprocessing_token();
 873         if(lexer_token.type != T_IDENTIFIER) {
 874                 parse_error("expected identifier after #define\n");
 875                 eat_until_newline();
 876         }
 877 }
 878
 879 static void ifdef_directive(int is_ifndef)
 880 {
 881         (void) is_ifndef;
 882         lexer_next_preprocessing_token();
 883         //expect_identifier();
 884         //extect_newline();
 885 }
 886
 887 static void endif_directive(void)
 888 {
 889         //expect_newline();
 890 }
 891
 892 static void parse_line_directive(void)
 893 {
 894         if(pp_token.type != T_INTEGER) {
 895                 parse_error("expected integer");
 896         } else {
 897                 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
 898                 next_pp_token();
 899         }
 900         if(pp_token.type == T_STRING_LITERAL) {
 901                 lexer_token.source_position.input_name = pp_token.v.string;
 902                 next_pp_token();
 903         }
 904
 905         eat_until_newline();
 906 }
 907
 908 static void parse_preprocessor_identifier(void)
 909 {
 910         assert(pp_token.type == T_IDENTIFIER);
 911         symbol_t *symbol = pp_token.v.symbol;
 912
 913         switch(symbol->pp_ID) {
 914         case TP_include:
 915                 printf("include - enable header name parsing!\n");
 916                 break;
 917         case TP_define:
 918                 define_directive();
 919                 break;
 920         case TP_ifdef:
 921                 ifdef_directive(0);
 922                 break;
 923         case TP_ifndef:
 924                 ifdef_directive(1);
 925                 break;
 926         case TP_endif:
 927                 endif_directive();
 928                 break;
 929         case TP_line:
 930                 next_pp_token();
 931                 parse_line_directive();
 932                 break;
 933         case TP_if:
 934         case TP_else:
 935         case TP_elif:
 936         case TP_undef:
 937         case TP_error:
 938                 error_directive();
 939                 break;
 940         case TP_pragma:
 941                 break;
 942         }
 943 }
 944
 945 static void parse_preprocessor_directive(void)
 946 {
 947         next_pp_token();
 948
 949         switch(pp_token.type) {
 950         case T_IDENTIFIER:
 951                 parse_preprocessor_identifier();
 952                 break;
 953         case T_INTEGER:
 954                 parse_line_directive();
 955                 break;
 956         default:
 957                 parse_error("invalid preprocessor directive");
 958                 eat_until_newline();
 959                 break;
 960         }
 961 }
 962
 963 #define MAYBE_PROLOG                                       \
 964                         next_char();                                   \
 965                         while(1) {                                     \
 966                                 switch(c) {
 967
 968 #define MAYBE(ch, set_type)                                \
 969                                 case ch:                                   \
 970                                         next_char();                           \
 971                                         lexer_token.type = set_type;           \
 972                                         return;
 973
 974 #define ELSE_CODE(code)                                    \
 975                                 default:                                   \
 976                                         code;                                  \
 977                                 }                                          \
 978                         } /* end of while(1) */                        \
 979                         break;
 980
 981 #define ELSE(set_type)                                     \
 982                 ELSE_CODE(                                         \
 983                         lexer_token.type = set_type;                   \
 984                         return;                                        \
 985                 )
 986
 987 void lexer_next_preprocessing_token(void)
 988 {
 989         while(1) {
 990                 switch(c) {
 991                 case ' ':
 992                 case '\t':
 993                         next_char();
 994                         break;
 995
 996                 MATCH_NEWLINE(
 997                         lexer_token.type = '\n';
 998                         return;
 999                 )
1000
1001                 SYMBOL_CHARS
1002                         parse_symbol();
1003                         /* might be a wide string ( L"string" ) */
1004                         if(c == '"' && (lexer_token.type == T_IDENTIFIER &&
1005                            lexer_token.v.symbol == symbol_L)) {
1006                                 parse_string_literal();
1007                                 return;
1008                         }
1009                         return;
1010
1011                 DIGITS
1012                         parse_number();
1013                         return;
1014
1015                 case '"':
1016                         parse_string_literal();
1017                         return;
1018
1019                 case '\'':
1020                         parse_character_constant();
1021                         return;
1022
1023                 case '.':
1024                         MAYBE_PROLOG
1025                                 case '.':
1026                                         MAYBE_PROLOG
1027                                         MAYBE('.', T_DOTDOTDOT)
1028                                         ELSE_CODE(
1029                                                 put_back(c);
1030                                                 c = '.';
1031                                                 lexer_token.type = '.';
1032                                                 return;
1033                                         )
1034                         ELSE('.')
1035                 case '&':
1036                         MAYBE_PROLOG
1037                         MAYBE('&', T_ANDAND)
1038                         MAYBE('=', T_ANDEQUAL)
1039                         ELSE('&')
1040                 case '*':
1041                         MAYBE_PROLOG
1042                         MAYBE('=', T_ASTERISKEQUAL)
1043                         ELSE('*')
1044                 case '+':
1045                         MAYBE_PROLOG
1046                         MAYBE('+', T_PLUSPLUS)
1047                         MAYBE('=', T_PLUSEQUAL)
1048                         ELSE('+')
1049                 case '-':
1050                         MAYBE_PROLOG
1051                         MAYBE('>', T_MINUSGREATER)
1052                         MAYBE('-', T_MINUSMINUS)
1053                         MAYBE('=', T_MINUSEQUAL)
1054                         ELSE('-')
1055                 case '!':
1056                         MAYBE_PROLOG
1057                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1058                         ELSE('!')
1059                 case '/':
1060                         MAYBE_PROLOG
1061                         MAYBE('=', T_SLASHEQUAL)
1062                                 case '*':
1063                                         next_char();
1064                                         skip_multiline_comment();
1065                                         lexer_next_preprocessing_token();
1066                                         return;
1067                                 case '/':
1068                                         next_char();
1069                                         skip_line_comment();
1070                                         lexer_next_preprocessing_token();
1071                                         return;
1072                         ELSE('/')
1073                 case '%':
1074                         MAYBE_PROLOG
1075                         MAYBE('>', T_PERCENTGREATER)
1076                         MAYBE('=', T_PERCENTEQUAL)
1077                                 case ':':
1078                                         MAYBE_PROLOG
1079                                                 case '%':
1080                                                         MAYBE_PROLOG
1081                                                         MAYBE(':', T_PERCENTCOLONPERCENTCOLON)
1082                                                         ELSE_CODE(
1083                                                                 put_back(c);
1084                                                                 c = '%';
1085                                                                 lexer_token.type = T_PERCENTCOLON;
1086                                                                 return;
1087                                                         )
1088                                         ELSE(T_PERCENTCOLON)
1089                         ELSE('%')
1090                 case '<':
1091                         MAYBE_PROLOG
1092                         MAYBE(':', T_LESSCOLON)
1093                         MAYBE('%', T_LESSPERCENT)
1094                         MAYBE('=', T_LESSEQUAL)
1095                                 case '<':
1096                                         MAYBE_PROLOG
1097                                         MAYBE('=', T_LESSLESSEQUAL)
1098                                         ELSE(T_LESSLESS)
1099                         ELSE('<')
1100                 case '>':
1101                         MAYBE_PROLOG
1102                         MAYBE('=', T_GREATEREQUAL)
1103                                 case '>':
1104                                         MAYBE_PROLOG
1105                                         MAYBE('=', T_GREATERGREATEREQUAL)
1106                                         ELSE(T_GREATERGREATER)
1107                         ELSE('>')
1108                 case '^':
1109                         MAYBE_PROLOG
1110                         MAYBE('=', T_CARETEQUAL)
1111                         ELSE('^')
1112                 case '|':
1113                         MAYBE_PROLOG
1114                         MAYBE('=', T_PIPEEQUAL)
1115                         MAYBE('|', T_PIPEPIPE)
1116                         ELSE('|')
1117                 case ':':
1118                         MAYBE_PROLOG
1119                         MAYBE('>', T_COLONGREATER)
1120                         ELSE(':')
1121                 case '=':
1122                         MAYBE_PROLOG
1123                         MAYBE('=', T_EQUALEQUAL)
1124                         ELSE('=')
1125                 case '#':
1126                         MAYBE_PROLOG
1127                         MAYBE('#', T_HASHHASH)
1128                         ELSE('#')
1129
1130                 case '?':
1131                 case '[':
1132                 case ']':
1133                 case '(':
1134                 case ')':
1135                 case '{':
1136                 case '}':
1137                 case '~':
1138                 case ';':
1139                 case ',':
1140                 case '\\':
1141                         lexer_token.type = c;
1142                         next_char();
1143                         return;
1144
1145                 case EOF:
1146                         lexer_token.type = T_EOF;
1147                         return;
1148
1149                 default:
1150                         next_char();
1151                         error_prefix();
1152                         fprintf(stderr, "unknown character '%c' found\n", c);
1153                         lexer_token.type = T_ERROR;
1154                         return;
1155                 }
1156         }
1157 }
1158
1159 void lexer_next_token(void)
1160 {
1161         lexer_next_preprocessing_token();
1162         if(lexer_token.type != '\n')
1163                 return;
1164
1165 newline_found:
1166         do {
1167                 lexer_next_preprocessing_token();
1168         } while(lexer_token.type == '\n');
1169
1170         if(lexer_token.type == '#') {
1171                 parse_preprocessor_directive();
1172                 goto newline_found;
1173         }
1174 }
1175
1176 void init_lexer(void)
1177 {
1178         strset_init(&stringset);
1179
1180         type_int       = make_atomic_type(ATOMIC_TYPE_INT, TYPE_QUALIFIER_NONE);
1181         type_uint      = make_atomic_type(ATOMIC_TYPE_UINT, TYPE_QUALIFIER_NONE);
1182         type_long      = make_atomic_type(ATOMIC_TYPE_LONG, TYPE_QUALIFIER_NONE);
1183         type_ulong     = make_atomic_type(ATOMIC_TYPE_ULONG, TYPE_QUALIFIER_NONE);
1184         type_longlong  = make_atomic_type(ATOMIC_TYPE_LONGLONG,
1185                                           TYPE_QUALIFIER_NONE);
1186         type_ulonglong = make_atomic_type(ATOMIC_TYPE_ULONGLONG,
1187                                           TYPE_QUALIFIER_NONE);
1188
1189         type_float      = make_atomic_type(ATOMIC_TYPE_FLOAT, TYPE_QUALIFIER_CONST);
1190         type_double     = make_atomic_type(ATOMIC_TYPE_DOUBLE,
1191                                            TYPE_QUALIFIER_CONST);
1192         type_longdouble = make_atomic_type(ATOMIC_TYPE_LONG_DOUBLE,
1193                                            TYPE_QUALIFIER_CONST);
1194 }
1195
1196 void lexer_open_stream(FILE *stream, const char *input_name)
1197 {
1198         input                                  = stream;
1199         lexer_token.source_position.linenr     = 0;
1200         lexer_token.source_position.input_name = input_name;
1201
1202         symbol_L = symbol_table_insert("L");
1203
1204         /* place a virtual \n at the beginning so the lexer knows that we're
1205          * at the beginning of a line */
1206         c = '\n';
1207 }
1208
1209 void exit_lexer(void)
1210 {
1211         strset_destroy(&stringset);
1212 }
1213
1214 static __attribute__((unused))
1215 void dbg_pos(const source_position_t source_position)
1216 {
1217         fprintf(stdout, "%s:%u\n", source_position.input_name,
1218                 source_position.linenr);
1219         fflush(stdout);
1220 }