nsz Git - cparser/blob - lexer.c

   1 #include <config.h>
   2
   3 #include "diagnostic.h"
   4 #include "lexer.h"
   5 #include "token_t.h"
   6 #include "symbol_table_t.h"
   7 #include "adt/error.h"
   8 #include "adt/strset.h"
   9 #include "adt/util.h"
  10 #include "types.h"
  11 #include "type_t.h"
  12 #include "target_architecture.h"
  13 #include "parser.h"
  14
  15 #include <assert.h>
  16 #include <errno.h>
  17 #include <string.h>
  18 #include <stdbool.h>
  19 #include <ctype.h>
  20
  21 //#define DEBUG_CHARS
  22 #define MAX_PUTBACK 3
  23
  24 #ifdef _WIN32
  25 /* No strtold on windows and no replacement yet */
  26 #define strtold(s, e) strtod(s, e)
  27 #endif
  28
  29 #if defined HAS_SIGNED_CHAR
  30 typedef signed char char_type;
  31 #elif defined HAS_UNSIGNED_CHAR
  32 typedef unsigned char char_type;
  33 #else
  34 #       error signedness of char not determined
  35 #endif
  36
  37 static int         c;
  38 token_t            lexer_token;
  39 symbol_t          *symbol_L;
  40 static FILE       *input;
  41 static char        buf[1024 + MAX_PUTBACK];
  42 static const char *bufend;
  43 static const char *bufpos;
  44 static strset_t    stringset;
  45
  46 static void error_prefix_at(const char *input_name, unsigned linenr)
  47 {
  48         fprintf(stderr, "%s:%u: Error: ", input_name, linenr);
  49 }
  50
  51 static void error_prefix(void)
  52 {
  53         error_prefix_at(lexer_token.source_position.input_name,
  54                         lexer_token.source_position.linenr);
  55 }
  56
  57 static void parse_error(const char *msg)
  58 {
  59         error_prefix();
  60         fprintf(stderr, "%s\n", msg);
  61 }
  62
  63 static inline void next_real_char(void)
  64 {
  65         assert(bufpos <= bufend);
  66         if (bufpos >= bufend) {
  67                 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
  68                                  input);
  69                 if(s == 0) {
  70                         c = EOF;
  71                         return;
  72                 }
  73                 bufpos = buf + MAX_PUTBACK;
  74                 bufend = buf + MAX_PUTBACK + s;
  75         }
  76         c = *bufpos++;
  77 }
  78
  79 static inline void put_back(int pc)
  80 {
  81         assert(bufpos > buf);
  82         *(--bufpos - buf + buf) = (char) pc;
  83
  84 #ifdef DEBUG_CHARS
  85         printf("putback '%c'\n", pc);
  86 #endif
  87 }
  88
  89 static inline void next_char(void);
  90
  91 #define MATCH_NEWLINE(code)                   \
  92         case '\r':                                \
  93                 next_char();                          \
  94                 if(c == '\n') {                       \
  95                         next_char();                      \
  96                 }                                     \
  97                 lexer_token.source_position.linenr++; \
  98                 code;                                 \
  99         case '\n':                                \
 100                 next_char();                          \
 101                 lexer_token.source_position.linenr++; \
 102                 code;
 103
 104 #define eat(c_type)  do { assert(c == c_type); next_char(); } while(0)
 105
 106 static void maybe_concat_lines(void)
 107 {
 108         eat('\\');
 109
 110         switch(c) {
 111         MATCH_NEWLINE(return;)
 112
 113         default:
 114                 break;
 115         }
 116
 117         put_back(c);
 118         c = '\\';
 119 }
 120
 121 static inline void next_char(void)
 122 {
 123         next_real_char();
 124
 125         /* filter trigraphs */
 126         if(UNLIKELY(c == '\\')) {
 127                 maybe_concat_lines();
 128                 goto end_of_next_char;
 129         }
 130
 131         if(LIKELY(c != '?'))
 132                 goto end_of_next_char;
 133
 134         next_real_char();
 135         if(LIKELY(c != '?')) {
 136                 put_back(c);
 137                 c = '?';
 138                 goto end_of_next_char;
 139         }
 140
 141         next_real_char();
 142         switch(c) {
 143         case '=': c = '#'; break;
 144         case '(': c = '['; break;
 145         case '/': c = '\\'; maybe_concat_lines(); break;
 146         case ')': c = ']'; break;
 147         case '\'': c = '^'; break;
 148         case '<': c = '{'; break;
 149         case '!': c = '|'; break;
 150         case '>': c = '}'; break;
 151         case '-': c = '~'; break;
 152         default:
 153                 put_back(c);
 154                 put_back('?');
 155                 c = '?';
 156                 break;
 157         }
 158
 159 end_of_next_char:;
 160 #ifdef DEBUG_CHARS
 161         printf("nchar '%c'\n", c);
 162 #endif
 163 }
 164
 165 #define SYMBOL_CHARS  \
 166         case 'a':         \
 167         case 'b':         \
 168         case 'c':         \
 169         case 'd':         \
 170         case 'e':         \
 171         case 'f':         \
 172         case 'g':         \
 173         case 'h':         \
 174         case 'i':         \
 175         case 'j':         \
 176         case 'k':         \
 177         case 'l':         \
 178         case 'm':         \
 179         case 'n':         \
 180         case 'o':         \
 181         case 'p':         \
 182         case 'q':         \
 183         case 'r':         \
 184         case 's':         \
 185         case 't':         \
 186         case 'u':         \
 187         case 'v':         \
 188         case 'w':         \
 189         case 'x':         \
 190         case 'y':         \
 191         case 'z':         \
 192         case 'A':         \
 193         case 'B':         \
 194         case 'C':         \
 195         case 'D':         \
 196         case 'E':         \
 197         case 'F':         \
 198         case 'G':         \
 199         case 'H':         \
 200         case 'I':         \
 201         case 'J':         \
 202         case 'K':         \
 203         case 'L':         \
 204         case 'M':         \
 205         case 'N':         \
 206         case 'O':         \
 207         case 'P':         \
 208         case 'Q':         \
 209         case 'R':         \
 210         case 'S':         \
 211         case 'T':         \
 212         case 'U':         \
 213         case 'V':         \
 214         case 'W':         \
 215         case 'X':         \
 216         case 'Y':         \
 217         case 'Z':         \
 218         case '_':
 219
 220 #define DIGITS        \
 221         case '0':         \
 222         case '1':         \
 223         case '2':         \
 224         case '3':         \
 225         case '4':         \
 226         case '5':         \
 227         case '6':         \
 228         case '7':         \
 229         case '8':         \
 230         case '9':
 231
 232 static void parse_symbol(void)
 233 {
 234         symbol_t *symbol;
 235         char     *string;
 236
 237         obstack_1grow(&symbol_obstack, (char) c);
 238         next_char();
 239
 240         while(1) {
 241                 switch(c) {
 242                 DIGITS
 243                 SYMBOL_CHARS
 244                         obstack_1grow(&symbol_obstack, (char) c);
 245                         next_char();
 246                         break;
 247
 248                 default:
 249                         goto end_symbol;
 250                 }
 251         }
 252
 253 end_symbol:
 254         obstack_1grow(&symbol_obstack, '\0');
 255
 256         string = obstack_finish(&symbol_obstack);
 257         symbol = symbol_table_insert(string);
 258
 259         lexer_token.type     = symbol->ID;
 260         lexer_token.v.symbol = symbol;
 261
 262         if(symbol->string != string) {
 263                 obstack_free(&symbol_obstack, string);
 264         }
 265 }
 266
 267 static void parse_integer_suffix(bool is_oct_hex)
 268 {
 269         bool is_unsigned  = false;
 270         bool min_long     = false;
 271         bool min_longlong = false;
 272
 273         if(c == 'U' || c == 'u') {
 274                 is_unsigned = true;
 275                 next_char();
 276                 if(c == 'L' || c == 'l') {
 277                         min_long = true;
 278                         next_char();
 279                         if(c == 'L' || c == 'l') {
 280                                 min_longlong = true;
 281                                 next_char();
 282                         }
 283                 }
 284         } else if(c == 'l' || c == 'L') {
 285                 min_long = true;
 286                 next_char();
 287                 if(c == 'l' || c == 'L') {
 288                         min_longlong = true;
 289                         next_char();
 290                         if(c == 'u' || c == 'U') {
 291                                 is_unsigned = true;
 292                                 next_char();
 293                         }
 294                 } else if(c == 'u' || c == 'U') {
 295                         is_unsigned = true;
 296                         next_char();
 297                         lexer_token.datatype = type_unsigned_long;
 298                 }
 299         }
 300
 301         if(!is_unsigned) {
 302                 long long v = lexer_token.v.intvalue;
 303                 if(!min_long) {
 304                         if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
 305                                 lexer_token.datatype = type_int;
 306                                 return;
 307                         } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
 308                                 lexer_token.datatype = type_unsigned_int;
 309                                 return;
 310                         }
 311                 }
 312                 if(!min_longlong) {
 313                         if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
 314                                 lexer_token.datatype = type_long;
 315                                 return;
 316                         } else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) {
 317                                 lexer_token.datatype = type_unsigned_long;
 318                                 return;
 319                         }
 320                 }
 321                 unsigned long long uv = (unsigned long long) v;
 322                 if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
 323                         lexer_token.datatype = type_unsigned_long_long;
 324                         return;
 325                 }
 326
 327                 lexer_token.datatype = type_long_long;
 328         } else {
 329                 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
 330                 if(!min_long && v <= TARGET_UINT_MAX) {
 331                         lexer_token.datatype = type_unsigned_int;
 332                         return;
 333                 }
 334                 if(!min_longlong && v <= TARGET_ULONG_MAX) {
 335                         lexer_token.datatype = type_unsigned_long;
 336                         return;
 337                 }
 338                 lexer_token.datatype = type_unsigned_long_long;
 339         }
 340 }
 341
 342 static void parse_floating_suffix(void)
 343 {
 344         switch(c) {
 345         /* TODO: do something usefull with the suffixes... */
 346         case 'f':
 347         case 'F':
 348                 next_char();
 349                 lexer_token.datatype = type_float;
 350                 break;
 351         case 'l':
 352         case 'L':
 353                 next_char();
 354                 lexer_token.datatype = type_long_double;
 355                 break;
 356         default:
 357                 lexer_token.datatype = type_double;
 358                 break;
 359         }
 360 }
 361
 362 /**
 363  * A replacement for strtoull. Only those parts needed for
 364  * our parser are implemented.
 365  */
 366 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
 367         unsigned long long v = 0;
 368
 369         switch (base) {
 370         case 16:
 371                 for (;; ++s) {
 372                         /* check for overrun */
 373                         if (v >= 0x1000000000000000ULL)
 374                                 break;
 375                         switch (tolower(*s)) {
 376                         case '0': v <<= 4; break;
 377                         case '1': v <<= 4; v |= 0x1; break;
 378                         case '2': v <<= 4; v |= 0x2; break;
 379                         case '3': v <<= 4; v |= 0x3; break;
 380                         case '4': v <<= 4; v |= 0x4; break;
 381                         case '5': v <<= 4; v |= 0x5; break;
 382                         case '6': v <<= 4; v |= 0x6; break;
 383                         case '7': v <<= 4; v |= 0x7; break;
 384                         case '8': v <<= 4; v |= 0x8; break;
 385                         case '9': v <<= 4; v |= 0x9; break;
 386                         case 'a': v <<= 4; v |= 0xa; break;
 387                         case 'b': v <<= 4; v |= 0xb; break;
 388                         case 'c': v <<= 4; v |= 0xc; break;
 389                         case 'd': v <<= 4; v |= 0xd; break;
 390                         case 'e': v <<= 4; v |= 0xe; break;
 391                         case 'f': v <<= 4; v |= 0xf; break;
 392                         default:
 393                                 goto end;
 394                         }
 395                 }
 396                 break;
 397         case 8:
 398                 for (;; ++s) {
 399                         /* check for overrun */
 400                         if (v >= 0x2000000000000000ULL)
 401                                 break;
 402                         switch (tolower(*s)) {
 403                         case '0': v <<= 3; break;
 404                         case '1': v <<= 3; v |= 1; break;
 405                         case '2': v <<= 3; v |= 2; break;
 406                         case '3': v <<= 3; v |= 3; break;
 407                         case '4': v <<= 3; v |= 4; break;
 408                         case '5': v <<= 3; v |= 5; break;
 409                         case '6': v <<= 3; v |= 6; break;
 410                         case '7': v <<= 3; v |= 7; break;
 411                         default:
 412                                 goto end;
 413                         }
 414                 }
 415                 break;
 416         case 10:
 417                 for (;; ++s) {
 418                         /* check for overrun */
 419                         if (v > 0x1999999999999999ULL)
 420                                 break;
 421                         switch (tolower(*s)) {
 422                         case '0': v *= 10; break;
 423                         case '1': v *= 10; v += 1; break;
 424                         case '2': v *= 10; v += 2; break;
 425                         case '3': v *= 10; v += 3; break;
 426                         case '4': v *= 10; v += 4; break;
 427                         case '5': v *= 10; v += 5; break;
 428                         case '6': v *= 10; v += 6; break;
 429                         case '7': v *= 10; v += 7; break;
 430                         case '8': v *= 10; v += 8; break;
 431                         case '9': v *= 10; v += 9; break;
 432                         default:
 433                                 goto end;
 434                         }
 435                 }
 436                 break;
 437         default:
 438                 assert(0);
 439                 break;
 440         }
 441 end:
 442         *endptr = s;
 443         return v;
 444 }
 445
 446 static void parse_number_hex(void)
 447 {
 448         assert(c == 'x' || c == 'X');
 449         next_char();
 450
 451         while(isxdigit(c)) {
 452                 obstack_1grow(&symbol_obstack, (char) c);
 453                 next_char();
 454         }
 455         obstack_1grow(&symbol_obstack, '\0');
 456         char *string = obstack_finish(&symbol_obstack);
 457
 458         if(c == '.' || c == 'p' || c == 'P') {
 459                 next_char();
 460                 panic("Hex floating point numbers not implemented yet");
 461         }
 462         if(*string == '\0') {
 463                 parse_error("invalid hex number");
 464                 lexer_token.type = T_ERROR;
 465         }
 466
 467         const char *endptr;
 468         lexer_token.type       = T_INTEGER;
 469         lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
 470         if(*endptr != '\0') {
 471                 parse_error("hex number literal too long");
 472         }
 473
 474         obstack_free(&symbol_obstack, string);
 475         parse_integer_suffix(true);
 476 }
 477
 478 static inline bool is_octal_digit(int chr)
 479 {
 480         return '0' <= chr && chr <= '7';
 481 }
 482
 483 static void parse_number_oct(void)
 484 {
 485         while(is_octal_digit(c)) {
 486                 obstack_1grow(&symbol_obstack, (char) c);
 487                 next_char();
 488         }
 489         obstack_1grow(&symbol_obstack, '\0');
 490         char *string = obstack_finish(&symbol_obstack);
 491
 492         const char *endptr;
 493         lexer_token.type       = T_INTEGER;
 494         lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
 495         if(*endptr != '\0') {
 496                 parse_error("octal number literal too long");
 497         }
 498
 499         obstack_free(&symbol_obstack, string);
 500         parse_integer_suffix(true);
 501 }
 502
 503 static void parse_number_dec(void)
 504 {
 505         bool is_float = false;
 506         while(isdigit(c)) {
 507                 obstack_1grow(&symbol_obstack, (char) c);
 508                 next_char();
 509         }
 510
 511         if(c == '.') {
 512                 obstack_1grow(&symbol_obstack, '.');
 513                 next_char();
 514
 515                 while(isdigit(c)) {
 516                         obstack_1grow(&symbol_obstack, (char) c);
 517                         next_char();
 518                 }
 519                 is_float = true;
 520         }
 521         if(c == 'e' || c == 'E') {
 522                 obstack_1grow(&symbol_obstack, 'e');
 523                 next_char();
 524
 525                 if(c == '-' || c == '+') {
 526                         obstack_1grow(&symbol_obstack, (char) c);
 527                         next_char();
 528                 }
 529
 530                 while(isdigit(c)) {
 531                         obstack_1grow(&symbol_obstack, (char) c);
 532                         next_char();
 533                 }
 534                 is_float = true;
 535         }
 536
 537         obstack_1grow(&symbol_obstack, '\0');
 538         char *string = obstack_finish(&symbol_obstack);
 539
 540         if(is_float) {
 541                 char *endptr;
 542                 lexer_token.type         = T_FLOATINGPOINT;
 543                 lexer_token.v.floatvalue = strtold(string, &endptr);
 544
 545                 if(*endptr != '\0') {
 546                         parse_error("invalid number literal");
 547                 }
 548
 549                 parse_floating_suffix();
 550         } else {
 551                 const char *endptr;
 552                 lexer_token.type       = T_INTEGER;
 553                 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
 554
 555                 if(*endptr != '\0') {
 556                         parse_error("invalid number literal");
 557                 }
 558
 559                 parse_integer_suffix(false);
 560         }
 561         obstack_free(&symbol_obstack, string);
 562 }
 563
 564 static void parse_number(void)
 565 {
 566         if (c == '0') {
 567                 next_char();
 568                 switch (c) {
 569                         case 'X':
 570                         case 'x':
 571                                 parse_number_hex();
 572                                 break;
 573                         case '0':
 574                         case '1':
 575                         case '2':
 576                         case '3':
 577                         case '4':
 578                         case '5':
 579                         case '6':
 580                         case '7':
 581                                 parse_number_oct();
 582                                 break;
 583                         case '8':
 584                         case '9':
 585                                 next_char();
 586                                 parse_error("invalid octal number");
 587                                 lexer_token.type = T_ERROR;
 588                                 return;
 589                         case '.':
 590                         case 'e':
 591                         case 'E':
 592                         default:
 593                                 obstack_1grow(&symbol_obstack, '0');
 594                                 parse_number_dec();
 595                                 return;
 596                 }
 597         } else {
 598                 parse_number_dec();
 599         }
 600 }
 601
 602 static int parse_octal_sequence(const int first_digit)
 603 {
 604         assert(is_octal_digit(first_digit));
 605         int value = first_digit - '0';
 606         if (!is_octal_digit(c)) return value;
 607         value = 8 * value + c - '0';
 608         next_char();
 609         if (!is_octal_digit(c)) return value;
 610         value = 8 * value + c - '0';
 611         next_char();
 612         return (char_type)value;
 613 }
 614
 615 static int parse_hex_sequence(void)
 616 {
 617         int value = 0;
 618         while(1) {
 619                 if (c >= '0' && c <= '9') {
 620                         value = 16 * value + c - '0';
 621                 } else if ('A' <= c && c <= 'F') {
 622                         value = 16 * value + c - 'A' + 10;
 623                 } else if ('a' <= c && c <= 'f') {
 624                         value = 16 * value + c - 'a' + 10;
 625                 } else {
 626                         break;
 627                 }
 628                 next_char();
 629         }
 630
 631         return (char_type)value;
 632 }
 633
 634 static int parse_escape_sequence(void)
 635 {
 636         eat('\\');
 637
 638         int ec = c;
 639         next_char();
 640
 641         switch(ec) {
 642         case '"':  return '"';
 643         case '\'': return '\'';
 644         case '\\': return '\\';
 645         case '?': return '\?';
 646         case 'a': return '\a';
 647         case 'b': return '\b';
 648         case 'f': return '\f';
 649         case 'n': return '\n';
 650         case 'r': return '\r';
 651         case 't': return '\t';
 652         case 'v': return '\v';
 653         case 'x':
 654                 return parse_hex_sequence();
 655         case '0':
 656         case '1':
 657         case '2':
 658         case '3':
 659         case '4':
 660         case '5':
 661         case '6':
 662         case '7':
 663                 return parse_octal_sequence(ec);
 664         case EOF:
 665                 parse_error("reached end of file while parsing escape sequence");
 666                 return EOF;
 667         default:
 668                 parse_error("unknown escape sequence");
 669                 return EOF;
 670         }
 671 }
 672
 673 const char *concat_strings(const char *s1, const char *s2)
 674 {
 675         size_t  len1   = strlen(s1);
 676         size_t  len2   = strlen(s2);
 677
 678         char   *concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
 679         memcpy(concat, s1, len1);
 680         memcpy(concat + len1, s2, len2 + 1);
 681
 682         const char *result = strset_insert(&stringset, concat);
 683         if(result != concat) {
 684                 obstack_free(&symbol_obstack, concat);
 685         }
 686
 687         return result;
 688 }
 689
 690 static void parse_string_literal(void)
 691 {
 692         unsigned    start_linenr = lexer_token.source_position.linenr;
 693         char       *string;
 694         const char *result;
 695
 696         assert(c == '"');
 697         next_char();
 698
 699         int tc;
 700         while(1) {
 701                 switch(c) {
 702                 case '\\':
 703                         tc = parse_escape_sequence();
 704                         obstack_1grow(&symbol_obstack, (char) tc);
 705                         break;
 706
 707                 case EOF:
 708                         error_prefix_at(lexer_token.source_position.input_name,
 709                                         start_linenr);
 710                         fprintf(stderr, "string has no end\n");
 711                         lexer_token.type = T_ERROR;
 712                         return;
 713
 714                 case '"':
 715                         next_char();
 716                         goto end_of_string;
 717
 718                 default:
 719                         obstack_1grow(&symbol_obstack, (char) c);
 720                         next_char();
 721                         break;
 722                 }
 723         }
 724
 725 end_of_string:
 726
 727         /* TODO: concatenate multiple strings separated by whitespace... */
 728
 729         /* add finishing 0 to the string */
 730         obstack_1grow(&symbol_obstack, '\0');
 731         string = obstack_finish(&symbol_obstack);
 732
 733         /* check if there is already a copy of the string */
 734         result = strset_insert(&stringset, string);
 735         if(result != string) {
 736                 obstack_free(&symbol_obstack, string);
 737         }
 738
 739         lexer_token.type     = T_STRING_LITERAL;
 740         lexer_token.v.string = result;
 741 }
 742
 743 static void parse_wide_character_constant(void)
 744 {
 745         eat('\'');
 746
 747         int found_char = 0;
 748         while(1) {
 749                 switch(c) {
 750                 case '\\':
 751                         found_char = parse_escape_sequence();
 752                         break;
 753
 754                 MATCH_NEWLINE(
 755                         parse_error("newline while parsing character constant");
 756                         break;
 757                 )
 758
 759                 case '\'':
 760                         next_char();
 761                         goto end_of_wide_char_constant;
 762
 763                 case EOF:
 764                         parse_error("EOF while parsing character constant");
 765                         lexer_token.type = T_ERROR;
 766                         return;
 767
 768                 default:
 769                         if(found_char != 0) {
 770                                 parse_error("more than 1 characters in character "
 771                                             "constant");
 772                                 goto end_of_wide_char_constant;
 773                         } else {
 774                                 found_char = c;
 775                                 next_char();
 776                         }
 777                         break;
 778                 }
 779         }
 780
 781 end_of_wide_char_constant:
 782         lexer_token.type       = T_INTEGER;
 783         lexer_token.v.intvalue = found_char;
 784         lexer_token.datatype   = type_wchar_t;
 785 }
 786
 787 static void parse_wide_string_literal(void)
 788 {
 789         const unsigned start_linenr = lexer_token.source_position.linenr;
 790
 791         assert(c == '"');
 792         next_char();
 793
 794         while(1) {
 795                 switch(c) {
 796                         case '\\': {
 797                                 wchar_rep_t tc = parse_escape_sequence();
 798                                 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 799                                 break;
 800                         }
 801
 802                         case EOF:
 803                                 error_prefix_at(lexer_token.source_position.input_name,
 804                                                 start_linenr);
 805                                 fprintf(stderr, "string has no end\n");
 806                                 lexer_token.type = T_ERROR;
 807                                 return;
 808
 809                         case '"':
 810                                 next_char();
 811                                 goto end_of_string;
 812
 813                         default: {
 814                                 wchar_rep_t tc = c;
 815                                 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 816                                 next_char();
 817                                 break;
 818                         }
 819                 }
 820         }
 821
 822 end_of_string:;
 823
 824         /* TODO: concatenate multiple strings separated by whitespace... */
 825
 826         /* add finishing 0 to the string */
 827         wchar_rep_t nul = L'\0';
 828         obstack_grow(&symbol_obstack, &nul, sizeof(nul));
 829         const size_t             size   = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
 830         const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
 831
 832 #if 0 /* TODO hash */
 833         /* check if there is already a copy of the string */
 834         const wchar_rep_t *const result = strset_insert(&stringset, string);
 835         if(result != string) {
 836                 obstack_free(&symbol_obstack, string);
 837         }
 838 #else
 839         const wchar_rep_t *const result = string;
 840 #endif
 841
 842         lexer_token.type                = T_WIDE_STRING_LITERAL;
 843         lexer_token.v.wide_string.begin = result;
 844         lexer_token.v.wide_string.size  = size;
 845 }
 846
 847 static void parse_character_constant(void)
 848 {
 849         eat('\'');
 850
 851         int found_char = 0;
 852         while(1) {
 853                 switch(c) {
 854                 case '\\':
 855                         found_char = parse_escape_sequence();
 856                         break;
 857
 858                 MATCH_NEWLINE(
 859                         parse_error("newline while parsing character constant");
 860                         break;
 861                 )
 862
 863                 case '\'':
 864                         next_char();
 865                         goto end_of_char_constant;
 866
 867                 case EOF:
 868                         parse_error("EOF while parsing character constant");
 869                         lexer_token.type = T_ERROR;
 870                         return;
 871
 872                 default:
 873                         if(found_char != 0) {
 874                                 parse_error("more than 1 characters in character "
 875                                             "constant");
 876                                 goto end_of_char_constant;
 877                         } else {
 878                                 found_char = c;
 879                                 next_char();
 880                         }
 881                         break;
 882                 }
 883         }
 884
 885 end_of_char_constant:
 886         lexer_token.type       = T_INTEGER;
 887         lexer_token.v.intvalue = found_char;
 888         lexer_token.datatype   = type_int;
 889 }
 890
 891 static void skip_multiline_comment(void)
 892 {
 893         unsigned start_linenr = lexer_token.source_position.linenr;
 894
 895         while(1) {
 896                 switch(c) {
 897                 case '*':
 898                         next_char();
 899                         if(c == '/') {
 900                                 next_char();
 901                                 return;
 902                         }
 903                         break;
 904
 905                 MATCH_NEWLINE(break;)
 906
 907                 case EOF:
 908                         error_prefix_at(lexer_token.source_position.input_name,
 909                                         start_linenr);
 910                         fprintf(stderr, "at end of file while looking for comment end\n");
 911                         return;
 912
 913                 default:
 914                         next_char();
 915                         break;
 916                 }
 917         }
 918 }
 919
 920 static void skip_line_comment(void)
 921 {
 922         while(1) {
 923                 switch(c) {
 924                 case EOF:
 925                         return;
 926
 927                 case '\n':
 928                 case '\r':
 929                         return;
 930
 931                 default:
 932                         next_char();
 933                         break;
 934                 }
 935         }
 936 }
 937
 938 static token_t pp_token;
 939
 940 static inline void next_pp_token(void)
 941 {
 942         lexer_next_preprocessing_token();
 943         pp_token = lexer_token;
 944 }
 945
 946 static void eat_until_newline(void)
 947 {
 948         while(pp_token.type != '\n' && pp_token.type != T_EOF) {
 949                 next_pp_token();
 950         }
 951 }
 952
 953 static void error_directive(void)
 954 {
 955         error_prefix();
 956         fprintf(stderr, "#error directive: \n");
 957
 958         /* parse pp-tokens until new-line */
 959 }
 960
 961 static void define_directive(void)
 962 {
 963         lexer_next_preprocessing_token();
 964         if(lexer_token.type != T_IDENTIFIER) {
 965                 parse_error("expected identifier after #define\n");
 966                 eat_until_newline();
 967         }
 968 }
 969
 970 static void ifdef_directive(int is_ifndef)
 971 {
 972         (void) is_ifndef;
 973         lexer_next_preprocessing_token();
 974         //expect_identifier();
 975         //extect_newline();
 976 }
 977
 978 static void endif_directive(void)
 979 {
 980         //expect_newline();
 981 }
 982
 983 static void parse_line_directive(void)
 984 {
 985         if(pp_token.type != T_INTEGER) {
 986                 parse_error("expected integer");
 987         } else {
 988                 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
 989                 next_pp_token();
 990         }
 991         if(pp_token.type == T_STRING_LITERAL) {
 992                 lexer_token.source_position.input_name = pp_token.v.string;
 993                 next_pp_token();
 994         }
 995
 996         eat_until_newline();
 997 }
 998
 999 static void parse_preprocessor_identifier(void)
1000 {
1001         assert(pp_token.type == T_IDENTIFIER);
1002         symbol_t *symbol = pp_token.v.symbol;
1003
1004         switch(symbol->pp_ID) {
1005         case TP_include:
1006                 printf("include - enable header name parsing!\n");
1007                 break;
1008         case TP_define:
1009                 define_directive();
1010                 break;
1011         case TP_ifdef:
1012                 ifdef_directive(0);
1013                 break;
1014         case TP_ifndef:
1015                 ifdef_directive(1);
1016                 break;
1017         case TP_endif:
1018                 endif_directive();
1019                 break;
1020         case TP_line:
1021                 next_pp_token();
1022                 parse_line_directive();
1023                 break;
1024         case TP_if:
1025         case TP_else:
1026         case TP_elif:
1027         case TP_undef:
1028         case TP_error:
1029                 error_directive();
1030                 break;
1031         case TP_pragma:
1032                 warningf(lexer_token.source_position, "encountered unknown #pragma");
1033                 eat_until_newline();
1034                 break;
1035         }
1036 }
1037
1038 static void parse_preprocessor_directive(void)
1039 {
1040         next_pp_token();
1041
1042         switch(pp_token.type) {
1043         case T_IDENTIFIER:
1044                 parse_preprocessor_identifier();
1045                 break;
1046         case T_INTEGER:
1047                 parse_line_directive();
1048                 break;
1049         default:
1050                 parse_error("invalid preprocessor directive");
1051                 eat_until_newline();
1052                 break;
1053         }
1054 }
1055
1056 #define MAYBE_PROLOG                                       \
1057                         next_char();                                   \
1058                         while(1) {                                     \
1059                                 switch(c) {
1060
1061 #define MAYBE(ch, set_type)                                \
1062                                 case ch:                                   \
1063                                         next_char();                           \
1064                                         lexer_token.type = set_type;           \
1065                                         return;
1066
1067 #define ELSE_CODE(code)                                    \
1068                                 default:                                   \
1069                                         code;                                  \
1070                                 }                                          \
1071                         } /* end of while(1) */                        \
1072                         break;
1073
1074 #define ELSE(set_type)                                     \
1075                 ELSE_CODE(                                         \
1076                         lexer_token.type = set_type;                   \
1077                         return;                                        \
1078                 )
1079
1080 void lexer_next_preprocessing_token(void)
1081 {
1082         while(1) {
1083                 switch(c) {
1084                 case ' ':
1085                 case '\t':
1086                         next_char();
1087                         break;
1088
1089                 MATCH_NEWLINE(
1090                         lexer_token.type = '\n';
1091                         return;
1092                 )
1093
1094                 SYMBOL_CHARS
1095                         parse_symbol();
1096                         /* might be a wide string ( L"string" ) */
1097                         if(lexer_token.type == T_IDENTIFIER &&
1098                             lexer_token.v.symbol == symbol_L) {
1099                             if(c == '"') {
1100                                         parse_wide_string_literal();
1101                                 } else if(c == '\'') {
1102                                         parse_wide_character_constant();
1103                                 }
1104                         }
1105                         return;
1106
1107                 DIGITS
1108                         parse_number();
1109                         return;
1110
1111                 case '"':
1112                         parse_string_literal();
1113                         return;
1114
1115                 case '\'':
1116                         parse_character_constant();
1117                         return;
1118
1119                 case '.':
1120                         MAYBE_PROLOG
1121                                 case '0':
1122                                 case '1':
1123                                 case '2':
1124                                 case '3':
1125                                 case '4':
1126                                 case '5':
1127                                 case '6':
1128                                 case '7':
1129                                 case '8':
1130                                 case '9':
1131                                         put_back(c);
1132                                         c = '.';
1133                                         parse_number_dec();
1134                                         return;
1135
1136                                 case '.':
1137                                         MAYBE_PROLOG
1138                                         MAYBE('.', T_DOTDOTDOT)
1139                                         ELSE_CODE(
1140                                                 put_back(c);
1141                                                 c = '.';
1142                                                 lexer_token.type = '.';
1143                                                 return;
1144                                         )
1145                         ELSE('.')
1146                 case '&':
1147                         MAYBE_PROLOG
1148                         MAYBE('&', T_ANDAND)
1149                         MAYBE('=', T_ANDEQUAL)
1150                         ELSE('&')
1151                 case '*':
1152                         MAYBE_PROLOG
1153                         MAYBE('=', T_ASTERISKEQUAL)
1154                         ELSE('*')
1155                 case '+':
1156                         MAYBE_PROLOG
1157                         MAYBE('+', T_PLUSPLUS)
1158                         MAYBE('=', T_PLUSEQUAL)
1159                         ELSE('+')
1160                 case '-':
1161                         MAYBE_PROLOG
1162                         MAYBE('>', T_MINUSGREATER)
1163                         MAYBE('-', T_MINUSMINUS)
1164                         MAYBE('=', T_MINUSEQUAL)
1165                         ELSE('-')
1166                 case '!':
1167                         MAYBE_PROLOG
1168                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1169                         ELSE('!')
1170                 case '/':
1171                         MAYBE_PROLOG
1172                         MAYBE('=', T_SLASHEQUAL)
1173                                 case '*':
1174                                         next_char();
1175                                         skip_multiline_comment();
1176                                         lexer_next_preprocessing_token();
1177                                         return;
1178                                 case '/':
1179                                         next_char();
1180                                         skip_line_comment();
1181                                         lexer_next_preprocessing_token();
1182                                         return;
1183                         ELSE('/')
1184                 case '%':
1185                         MAYBE_PROLOG
1186                         MAYBE('>', T_PERCENTGREATER)
1187                         MAYBE('=', T_PERCENTEQUAL)
1188                                 case ':':
1189                                         MAYBE_PROLOG
1190                                                 case '%':
1191                                                         MAYBE_PROLOG
1192                                                         MAYBE(':', T_PERCENTCOLONPERCENTCOLON)
1193                                                         ELSE_CODE(
1194                                                                 put_back(c);
1195                                                                 c = '%';
1196                                                                 lexer_token.type = T_PERCENTCOLON;
1197                                                                 return;
1198                                                         )
1199                                         ELSE(T_PERCENTCOLON)
1200                         ELSE('%')
1201                 case '<':
1202                         MAYBE_PROLOG
1203                         MAYBE(':', T_LESSCOLON)
1204                         MAYBE('%', T_LESSPERCENT)
1205                         MAYBE('=', T_LESSEQUAL)
1206                                 case '<':
1207                                         MAYBE_PROLOG
1208                                         MAYBE('=', T_LESSLESSEQUAL)
1209                                         ELSE(T_LESSLESS)
1210                         ELSE('<')
1211                 case '>':
1212                         MAYBE_PROLOG
1213                         MAYBE('=', T_GREATEREQUAL)
1214                                 case '>':
1215                                         MAYBE_PROLOG
1216                                         MAYBE('=', T_GREATERGREATEREQUAL)
1217                                         ELSE(T_GREATERGREATER)
1218                         ELSE('>')
1219                 case '^':
1220                         MAYBE_PROLOG
1221                         MAYBE('=', T_CARETEQUAL)
1222                         ELSE('^')
1223                 case '|':
1224                         MAYBE_PROLOG
1225                         MAYBE('=', T_PIPEEQUAL)
1226                         MAYBE('|', T_PIPEPIPE)
1227                         ELSE('|')
1228                 case ':':
1229                         MAYBE_PROLOG
1230                         MAYBE('>', T_COLONGREATER)
1231                         ELSE(':')
1232                 case '=':
1233                         MAYBE_PROLOG
1234                         MAYBE('=', T_EQUALEQUAL)
1235                         ELSE('=')
1236                 case '#':
1237                         MAYBE_PROLOG
1238                         MAYBE('#', T_HASHHASH)
1239                         ELSE('#')
1240
1241                 case '?':
1242                 case '[':
1243                 case ']':
1244                 case '(':
1245                 case ')':
1246                 case '{':
1247                 case '}':
1248                 case '~':
1249                 case ';':
1250                 case ',':
1251                 case '\\':
1252                         lexer_token.type = c;
1253                         next_char();
1254                         return;
1255
1256                 case EOF:
1257                         lexer_token.type = T_EOF;
1258                         return;
1259
1260                 default:
1261                         next_char();
1262                         error_prefix();
1263                         fprintf(stderr, "unknown character '%c' found\n", c);
1264                         lexer_token.type = T_ERROR;
1265                         return;
1266                 }
1267         }
1268 }
1269
1270 void lexer_next_token(void)
1271 {
1272         lexer_next_preprocessing_token();
1273         if(lexer_token.type != '\n')
1274                 return;
1275
1276 newline_found:
1277         do {
1278                 lexer_next_preprocessing_token();
1279         } while(lexer_token.type == '\n');
1280
1281         if(lexer_token.type == '#') {
1282                 parse_preprocessor_directive();
1283                 goto newline_found;
1284         }
1285 }
1286
1287 void init_lexer(void)
1288 {
1289         strset_init(&stringset);
1290 }
1291
1292 void lexer_open_stream(FILE *stream, const char *input_name)
1293 {
1294         input                                  = stream;
1295         lexer_token.source_position.linenr     = 0;
1296         lexer_token.source_position.input_name = input_name;
1297
1298         symbol_L = symbol_table_insert("L");
1299         bufpos = NULL;
1300         bufend = NULL;
1301
1302         /* place a virtual \n at the beginning so the lexer knows that we're
1303          * at the beginning of a line */
1304         c = '\n';
1305 }
1306
1307 void exit_lexer(void)
1308 {
1309         strset_destroy(&stringset);
1310 }
1311
1312 static __attribute__((unused))
1313 void dbg_pos(const source_position_t source_position)
1314 {
1315         fprintf(stdout, "%s:%u\n", source_position.input_name,
1316                 source_position.linenr);
1317         fflush(stdout);
1318 }