nsz Git - cparser/blob - lexer.c

   1 #include <config.h>
   2
   3 #include "diagnostic.h"
   4 #include "lexer.h"
   5 #include "token_t.h"
   6 #include "symbol_table_t.h"
   7 #include "adt/error.h"
   8 #include "adt/strset.h"
   9 #include "adt/util.h"
  10 #include "types.h"
  11 #include "type_t.h"
  12 #include "target_architecture.h"
  13 #include "parser.h"
  14
  15 #include <assert.h>
  16 #include <errno.h>
  17 #include <string.h>
  18 #include <stdbool.h>
  19 #include <ctype.h>
  20
  21 //#define DEBUG_CHARS
  22 #define MAX_PUTBACK 3
  23
  24 #ifdef _WIN32
  25 /* No strtold on windows and no replacement yet */
  26 #define strtold(s, e) strtod(s, e)
  27 #endif
  28
  29 #if defined HAS_SIGNED_CHAR
  30 typedef signed char char_type;
  31 #elif defined HAS_UNSIGNED_CHAR
  32 typedef unsigned char char_type;
  33 #else
  34 #       error signedness of char not determined
  35 #endif
  36
  37 static int         c;
  38 token_t            lexer_token;
  39 symbol_t          *symbol_L;
  40 static FILE       *input;
  41 static char        buf[1024 + MAX_PUTBACK];
  42 static const char *bufend;
  43 static const char *bufpos;
  44 static strset_t    stringset;
  45
  46 static void error_prefix_at(const char *input_name, unsigned linenr)
  47 {
  48         fprintf(stderr, "%s:%u: Error: ", input_name, linenr);
  49 }
  50
  51 static void error_prefix(void)
  52 {
  53         error_prefix_at(lexer_token.source_position.input_name,
  54                         lexer_token.source_position.linenr);
  55 }
  56
  57 static void parse_error(const char *msg)
  58 {
  59         error_prefix();
  60         fprintf(stderr, "%s\n", msg);
  61 }
  62
  63 static inline void next_real_char(void)
  64 {
  65         assert(bufpos <= bufend);
  66         if (bufpos >= bufend) {
  67                 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
  68                                  input);
  69                 if(s == 0) {
  70                         c = EOF;
  71                         return;
  72                 }
  73                 bufpos = buf + MAX_PUTBACK;
  74                 bufend = buf + MAX_PUTBACK + s;
  75         }
  76         c = *bufpos++;
  77 }
  78
  79 static inline void put_back(int pc)
  80 {
  81         assert(bufpos > buf);
  82         *(--bufpos - buf + buf) = (char) pc;
  83
  84 #ifdef DEBUG_CHARS
  85         printf("putback '%c'\n", pc);
  86 #endif
  87 }
  88
  89 static inline void next_char(void);
  90
  91 #define MATCH_NEWLINE(code)                   \
  92         case '\r':                                \
  93                 next_char();                          \
  94                 if(c == '\n') {                       \
  95                         next_char();                      \
  96                 }                                     \
  97                 lexer_token.source_position.linenr++; \
  98                 code;                                 \
  99         case '\n':                                \
 100                 next_char();                          \
 101                 lexer_token.source_position.linenr++; \
 102                 code;
 103
 104 #define eat(c_type)  do { assert(c == c_type); next_char(); } while(0)
 105
 106 static void maybe_concat_lines(void)
 107 {
 108         eat('\\');
 109
 110         switch(c) {
 111         MATCH_NEWLINE(return;)
 112
 113         default:
 114                 break;
 115         }
 116
 117         put_back(c);
 118         c = '\\';
 119 }
 120
 121 static inline void next_char(void)
 122 {
 123         next_real_char();
 124
 125         /* filter trigraphs */
 126         if(UNLIKELY(c == '\\')) {
 127                 maybe_concat_lines();
 128                 goto end_of_next_char;
 129         }
 130
 131         if(LIKELY(c != '?'))
 132                 goto end_of_next_char;
 133
 134         next_real_char();
 135         if(LIKELY(c != '?')) {
 136                 put_back(c);
 137                 c = '?';
 138                 goto end_of_next_char;
 139         }
 140
 141         next_real_char();
 142         switch(c) {
 143         case '=': c = '#'; break;
 144         case '(': c = '['; break;
 145         case '/': c = '\\'; maybe_concat_lines(); break;
 146         case ')': c = ']'; break;
 147         case '\'': c = '^'; break;
 148         case '<': c = '{'; break;
 149         case '!': c = '|'; break;
 150         case '>': c = '}'; break;
 151         case '-': c = '~'; break;
 152         default:
 153                 put_back(c);
 154                 put_back('?');
 155                 c = '?';
 156                 break;
 157         }
 158
 159 end_of_next_char:;
 160 #ifdef DEBUG_CHARS
 161         printf("nchar '%c'\n", c);
 162 #endif
 163 }
 164
 165 #define SYMBOL_CHARS  \
 166         case 'a':         \
 167         case 'b':         \
 168         case 'c':         \
 169         case 'd':         \
 170         case 'e':         \
 171         case 'f':         \
 172         case 'g':         \
 173         case 'h':         \
 174         case 'i':         \
 175         case 'j':         \
 176         case 'k':         \
 177         case 'l':         \
 178         case 'm':         \
 179         case 'n':         \
 180         case 'o':         \
 181         case 'p':         \
 182         case 'q':         \
 183         case 'r':         \
 184         case 's':         \
 185         case 't':         \
 186         case 'u':         \
 187         case 'v':         \
 188         case 'w':         \
 189         case 'x':         \
 190         case 'y':         \
 191         case 'z':         \
 192         case 'A':         \
 193         case 'B':         \
 194         case 'C':         \
 195         case 'D':         \
 196         case 'E':         \
 197         case 'F':         \
 198         case 'G':         \
 199         case 'H':         \
 200         case 'I':         \
 201         case 'J':         \
 202         case 'K':         \
 203         case 'L':         \
 204         case 'M':         \
 205         case 'N':         \
 206         case 'O':         \
 207         case 'P':         \
 208         case 'Q':         \
 209         case 'R':         \
 210         case 'S':         \
 211         case 'T':         \
 212         case 'U':         \
 213         case 'V':         \
 214         case 'W':         \
 215         case 'X':         \
 216         case 'Y':         \
 217         case 'Z':         \
 218         case '_':
 219
 220 #define DIGITS        \
 221         case '0':         \
 222         case '1':         \
 223         case '2':         \
 224         case '3':         \
 225         case '4':         \
 226         case '5':         \
 227         case '6':         \
 228         case '7':         \
 229         case '8':         \
 230         case '9':
 231
 232 static void parse_symbol(void)
 233 {
 234         symbol_t *symbol;
 235         char     *string;
 236
 237         obstack_1grow(&symbol_obstack, (char) c);
 238         next_char();
 239
 240         while(1) {
 241                 switch(c) {
 242                 DIGITS
 243                 SYMBOL_CHARS
 244                         obstack_1grow(&symbol_obstack, (char) c);
 245                         next_char();
 246                         break;
 247
 248                 default:
 249                         goto end_symbol;
 250                 }
 251         }
 252
 253 end_symbol:
 254         obstack_1grow(&symbol_obstack, '\0');
 255
 256         string = obstack_finish(&symbol_obstack);
 257         symbol = symbol_table_insert(string);
 258
 259         lexer_token.type     = symbol->ID;
 260         lexer_token.v.symbol = symbol;
 261
 262         if(symbol->string != string) {
 263                 obstack_free(&symbol_obstack, string);
 264         }
 265 }
 266
 267 static void parse_integer_suffix(bool is_oct_hex)
 268 {
 269         bool is_unsigned  = false;
 270         bool min_long     = false;
 271         bool min_longlong = false;
 272
 273         if(c == 'U' || c == 'u') {
 274                 is_unsigned = true;
 275                 next_char();
 276                 if(c == 'L' || c == 'l') {
 277                         min_long = true;
 278                         next_char();
 279                         if(c == 'L' || c == 'l') {
 280                                 min_longlong = true;
 281                                 next_char();
 282                         }
 283                 }
 284         } else if(c == 'l' || c == 'L') {
 285                 min_long = true;
 286                 next_char();
 287                 if(c == 'l' || c == 'L') {
 288                         min_longlong = true;
 289                         next_char();
 290                         if(c == 'u' || c == 'U') {
 291                                 is_unsigned = true;
 292                                 next_char();
 293                         }
 294                 } else if(c == 'u' || c == 'U') {
 295                         is_unsigned = true;
 296                         next_char();
 297                         lexer_token.datatype = type_unsigned_long;
 298                 }
 299         }
 300
 301         if(!is_unsigned) {
 302                 long long v = lexer_token.v.intvalue;
 303                 if(!min_long) {
 304                         if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
 305                                 lexer_token.datatype = type_int;
 306                                 return;
 307                         } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
 308                                 lexer_token.datatype = type_unsigned_int;
 309                                 return;
 310                         }
 311                 }
 312                 if(!min_longlong) {
 313                         if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
 314                                 lexer_token.datatype = type_long;
 315                                 return;
 316                         } else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) {
 317                                 lexer_token.datatype = type_unsigned_long;
 318                                 return;
 319                         }
 320                 }
 321                 unsigned long long uv = (unsigned long long) v;
 322                 if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
 323                         lexer_token.datatype = type_unsigned_long_long;
 324                         return;
 325                 }
 326
 327                 lexer_token.datatype = type_long_long;
 328         } else {
 329                 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
 330                 if(!min_long && v <= TARGET_UINT_MAX) {
 331                         lexer_token.datatype = type_unsigned_int;
 332                         return;
 333                 }
 334                 if(!min_longlong && v <= TARGET_ULONG_MAX) {
 335                         lexer_token.datatype = type_unsigned_long;
 336                         return;
 337                 }
 338                 lexer_token.datatype = type_unsigned_long_long;
 339         }
 340 }
 341
 342 static void parse_floating_suffix(void)
 343 {
 344         switch(c) {
 345         /* TODO: do something usefull with the suffixes... */
 346         case 'f':
 347         case 'F':
 348                 next_char();
 349                 lexer_token.datatype = type_float;
 350                 break;
 351         case 'l':
 352         case 'L':
 353                 next_char();
 354                 lexer_token.datatype = type_long_double;
 355                 break;
 356         default:
 357                 lexer_token.datatype = type_double;
 358                 break;
 359         }
 360 }
 361
 362 /**
 363  * A replacement for strtoull. Only those parts needed for
 364  * our parser are implemented.
 365  */
 366 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
 367         unsigned long long v = 0;
 368
 369         switch (base) {
 370         case 16:
 371                 for (;; ++s) {
 372                         /* check for overrun */
 373                         if (v >= 0x1000000000000000ULL)
 374                                 break;
 375                         switch (tolower(*s)) {
 376                         case '0': v <<= 4; break;
 377                         case '1': v <<= 4; v |= 0x1; break;
 378                         case '2': v <<= 4; v |= 0x2; break;
 379                         case '3': v <<= 4; v |= 0x3; break;
 380                         case '4': v <<= 4; v |= 0x4; break;
 381                         case '5': v <<= 4; v |= 0x5; break;
 382                         case '6': v <<= 4; v |= 0x6; break;
 383                         case '7': v <<= 4; v |= 0x7; break;
 384                         case '8': v <<= 4; v |= 0x8; break;
 385                         case '9': v <<= 4; v |= 0x9; break;
 386                         case 'a': v <<= 4; v |= 0xa; break;
 387                         case 'b': v <<= 4; v |= 0xb; break;
 388                         case 'c': v <<= 4; v |= 0xc; break;
 389                         case 'd': v <<= 4; v |= 0xd; break;
 390                         case 'e': v <<= 4; v |= 0xe; break;
 391                         case 'f': v <<= 4; v |= 0xf; break;
 392                         default:
 393                                 goto end;
 394                         }
 395                 }
 396                 break;
 397         case 8:
 398                 for (;; ++s) {
 399                         /* check for overrun */
 400                         if (v >= 0x2000000000000000ULL)
 401                                 break;
 402                         switch (tolower(*s)) {
 403                         case '0': v <<= 3; break;
 404                         case '1': v <<= 3; v |= 1; break;
 405                         case '2': v <<= 3; v |= 2; break;
 406                         case '3': v <<= 3; v |= 3; break;
 407                         case '4': v <<= 3; v |= 4; break;
 408                         case '5': v <<= 3; v |= 5; break;
 409                         case '6': v <<= 3; v |= 6; break;
 410                         case '7': v <<= 3; v |= 7; break;
 411                         default:
 412                                 goto end;
 413                         }
 414                 }
 415                 break;
 416         case 10:
 417                 for (;; ++s) {
 418                         /* check for overrun */
 419                         if (v > 0x1999999999999999ULL)
 420                                 break;
 421                         switch (tolower(*s)) {
 422                         case '0': v *= 10; break;
 423                         case '1': v *= 10; v += 1; break;
 424                         case '2': v *= 10; v += 2; break;
 425                         case '3': v *= 10; v += 3; break;
 426                         case '4': v *= 10; v += 4; break;
 427                         case '5': v *= 10; v += 5; break;
 428                         case '6': v *= 10; v += 6; break;
 429                         case '7': v *= 10; v += 7; break;
 430                         case '8': v *= 10; v += 8; break;
 431                         case '9': v *= 10; v += 9; break;
 432                         default:
 433                                 goto end;
 434                         }
 435                 }
 436                 break;
 437         default:
 438                 assert(0);
 439                 break;
 440         }
 441 end:
 442         *endptr = s;
 443         return v;
 444 }
 445
 446 static void parse_number_hex(void)
 447 {
 448         assert(c == 'x' || c == 'X');
 449         next_char();
 450
 451         while(isxdigit(c)) {
 452                 obstack_1grow(&symbol_obstack, (char) c);
 453                 next_char();
 454         }
 455         obstack_1grow(&symbol_obstack, '\0');
 456         char *string = obstack_finish(&symbol_obstack);
 457
 458         if(c == '.' || c == 'p' || c == 'P') {
 459                 next_char();
 460                 panic("Hex floating point numbers not implemented yet");
 461         }
 462         if(*string == '\0') {
 463                 parse_error("invalid hex number");
 464                 lexer_token.type = T_ERROR;
 465         }
 466
 467         const char *endptr;
 468         lexer_token.type       = T_INTEGER;
 469         lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
 470         if(*endptr != '\0') {
 471                 parse_error("hex number literal too long");
 472         }
 473
 474         obstack_free(&symbol_obstack, string);
 475         parse_integer_suffix(true);
 476 }
 477
 478 static inline bool is_octal_digit(int chr)
 479 {
 480         return '0' <= chr && chr <= '7';
 481 }
 482
 483 static void parse_number_oct(void)
 484 {
 485         while(is_octal_digit(c)) {
 486                 obstack_1grow(&symbol_obstack, (char) c);
 487                 next_char();
 488         }
 489         obstack_1grow(&symbol_obstack, '\0');
 490         char *string = obstack_finish(&symbol_obstack);
 491
 492         const char *endptr;
 493         lexer_token.type       = T_INTEGER;
 494         lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
 495         if(*endptr != '\0') {
 496                 parse_error("octal number literal too long");
 497         }
 498
 499         obstack_free(&symbol_obstack, string);
 500         parse_integer_suffix(true);
 501 }
 502
 503 static void parse_number_dec(void)
 504 {
 505         bool is_float = false;
 506         while(isdigit(c)) {
 507                 obstack_1grow(&symbol_obstack, (char) c);
 508                 next_char();
 509         }
 510
 511         if(c == '.') {
 512                 obstack_1grow(&symbol_obstack, '.');
 513                 next_char();
 514
 515                 while(isdigit(c)) {
 516                         obstack_1grow(&symbol_obstack, (char) c);
 517                         next_char();
 518                 }
 519                 is_float = true;
 520         }
 521         if(c == 'e' || c == 'E') {
 522                 obstack_1grow(&symbol_obstack, 'e');
 523                 next_char();
 524
 525                 if(c == '-' || c == '+') {
 526                         obstack_1grow(&symbol_obstack, (char) c);
 527                         next_char();
 528                 }
 529
 530                 while(isdigit(c)) {
 531                         obstack_1grow(&symbol_obstack, (char) c);
 532                         next_char();
 533                 }
 534                 is_float = true;
 535         }
 536
 537         obstack_1grow(&symbol_obstack, '\0');
 538         char *string = obstack_finish(&symbol_obstack);
 539
 540         if(is_float) {
 541                 char *endptr;
 542                 lexer_token.type         = T_FLOATINGPOINT;
 543                 lexer_token.v.floatvalue = strtold(string, &endptr);
 544
 545                 if(*endptr != '\0') {
 546                         parse_error("invalid number literal");
 547                 }
 548
 549                 parse_floating_suffix();
 550         } else {
 551                 const char *endptr;
 552                 lexer_token.type       = T_INTEGER;
 553                 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
 554
 555                 if(*endptr != '\0') {
 556                         parse_error("invalid number literal");
 557                 }
 558
 559                 parse_integer_suffix(false);
 560         }
 561         obstack_free(&symbol_obstack, string);
 562 }
 563
 564 static void parse_number(void)
 565 {
 566         if (c == '0') {
 567                 next_char();
 568                 switch (c) {
 569                         case 'X':
 570                         case 'x':
 571                                 parse_number_hex();
 572                                 break;
 573                         case '0':
 574                         case '1':
 575                         case '2':
 576                         case '3':
 577                         case '4':
 578                         case '5':
 579                         case '6':
 580                         case '7':
 581                                 parse_number_oct();
 582                                 break;
 583                         case '8':
 584                         case '9':
 585                                 next_char();
 586                                 parse_error("invalid octal number");
 587                                 lexer_token.type = T_ERROR;
 588                                 return;
 589                         case '.':
 590                         case 'e':
 591                         case 'E':
 592                         default:
 593                                 obstack_1grow(&symbol_obstack, '0');
 594                                 parse_number_dec();
 595                                 return;
 596                 }
 597         } else {
 598                 parse_number_dec();
 599         }
 600 }
 601
 602 static int parse_octal_sequence(const int first_digit)
 603 {
 604         assert(is_octal_digit(first_digit));
 605         int value = first_digit - '0';
 606         if (!is_octal_digit(c)) return value;
 607         value = 8 * value + c - '0';
 608         next_char();
 609         if (!is_octal_digit(c)) return value;
 610         value = 8 * value + c - '0';
 611         next_char();
 612         return (char_type)value;
 613 }
 614
 615 static int parse_hex_sequence(void)
 616 {
 617         int value = 0;
 618         while(1) {
 619                 if (c >= '0' && c <= '9') {
 620                         value = 16 * value + c - '0';
 621                 } else if ('A' <= c && c <= 'F') {
 622                         value = 16 * value + c - 'A' + 10;
 623                 } else if ('a' <= c && c <= 'f') {
 624                         value = 16 * value + c - 'a' + 10;
 625                 } else {
 626                         break;
 627                 }
 628                 next_char();
 629         }
 630
 631         return (char_type)value;
 632 }
 633
 634 static int parse_escape_sequence(void)
 635 {
 636         eat('\\');
 637
 638         int ec = c;
 639         next_char();
 640
 641         switch(ec) {
 642         case '"':  return '"';
 643         case '\'': return '\'';
 644         case '\\': return '\\';
 645         case '?': return '\?';
 646         case 'a': return '\a';
 647         case 'b': return '\b';
 648         case 'f': return '\f';
 649         case 'n': return '\n';
 650         case 'r': return '\r';
 651         case 't': return '\t';
 652         case 'v': return '\v';
 653         case 'x':
 654                 return parse_hex_sequence();
 655         case '0':
 656         case '1':
 657         case '2':
 658         case '3':
 659         case '4':
 660         case '5':
 661         case '6':
 662         case '7':
 663                 return parse_octal_sequence(ec);
 664         case EOF:
 665                 parse_error("reached end of file while parsing escape sequence");
 666                 return EOF;
 667         default:
 668                 parse_error("unknown escape sequence");
 669                 return EOF;
 670         }
 671 }
 672
 673 string_t concat_strings(const string_t *const s1, const string_t *const s2)
 674 {
 675         const size_t len1 = s1->size - 1;
 676         const size_t len2 = s2->size - 1;
 677
 678         char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
 679         memcpy(concat, s1->begin, len1);
 680         memcpy(concat + len1, s2->begin, len2 + 1);
 681
 682 #if 0 /* TODO hash */
 683         const char *result = strset_insert(&stringset, concat);
 684         if(result != concat) {
 685                 obstack_free(&symbol_obstack, concat);
 686         }
 687
 688         return result;
 689 #else
 690         return (string_t){ concat, len1 + len2 + 1 };
 691 #endif
 692 }
 693
 694 static void parse_string_literal(void)
 695 {
 696         const unsigned start_linenr = lexer_token.source_position.linenr;
 697
 698         assert(c == '"');
 699         next_char();
 700
 701         int tc;
 702         while(1) {
 703                 switch(c) {
 704                 case '\\':
 705                         tc = parse_escape_sequence();
 706                         obstack_1grow(&symbol_obstack, (char) tc);
 707                         break;
 708
 709                 case EOF:
 710                         error_prefix_at(lexer_token.source_position.input_name,
 711                                         start_linenr);
 712                         fprintf(stderr, "string has no end\n");
 713                         lexer_token.type = T_ERROR;
 714                         return;
 715
 716                 case '"':
 717                         next_char();
 718                         goto end_of_string;
 719
 720                 default:
 721                         obstack_1grow(&symbol_obstack, (char) c);
 722                         next_char();
 723                         break;
 724                 }
 725         }
 726
 727 end_of_string:
 728
 729         /* TODO: concatenate multiple strings separated by whitespace... */
 730
 731         /* add finishing 0 to the string */
 732         obstack_1grow(&symbol_obstack, '\0');
 733         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
 734         const char *const string = obstack_finish(&symbol_obstack);
 735
 736 #if 0 /* TODO hash */
 737         /* check if there is already a copy of the string */
 738         result = strset_insert(&stringset, string);
 739         if(result != string) {
 740                 obstack_free(&symbol_obstack, string);
 741         }
 742 #else
 743         const char *const result = string;
 744 #endif
 745
 746         lexer_token.type           = T_STRING_LITERAL;
 747         lexer_token.v.string.begin = result;
 748         lexer_token.v.string.size  = size;
 749 }
 750
 751 static void parse_wide_character_constant(void)
 752 {
 753         eat('\'');
 754
 755         int found_char = 0;
 756         while(1) {
 757                 switch(c) {
 758                 case '\\':
 759                         found_char = parse_escape_sequence();
 760                         break;
 761
 762                 MATCH_NEWLINE(
 763                         parse_error("newline while parsing character constant");
 764                         break;
 765                 )
 766
 767                 case '\'':
 768                         next_char();
 769                         goto end_of_wide_char_constant;
 770
 771                 case EOF:
 772                         parse_error("EOF while parsing character constant");
 773                         lexer_token.type = T_ERROR;
 774                         return;
 775
 776                 default:
 777                         if(found_char != 0) {
 778                                 parse_error("more than 1 characters in character "
 779                                             "constant");
 780                                 goto end_of_wide_char_constant;
 781                         } else {
 782                                 found_char = c;
 783                                 next_char();
 784                         }
 785                         break;
 786                 }
 787         }
 788
 789 end_of_wide_char_constant:
 790         lexer_token.type       = T_INTEGER;
 791         lexer_token.v.intvalue = found_char;
 792         lexer_token.datatype   = type_wchar_t;
 793 }
 794
 795 static void parse_wide_string_literal(void)
 796 {
 797         const unsigned start_linenr = lexer_token.source_position.linenr;
 798
 799         assert(c == '"');
 800         next_char();
 801
 802         while(1) {
 803                 switch(c) {
 804                         case '\\': {
 805                                 wchar_rep_t tc = parse_escape_sequence();
 806                                 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 807                                 break;
 808                         }
 809
 810                         case EOF:
 811                                 error_prefix_at(lexer_token.source_position.input_name,
 812                                                 start_linenr);
 813                                 fprintf(stderr, "string has no end\n");
 814                                 lexer_token.type = T_ERROR;
 815                                 return;
 816
 817                         case '"':
 818                                 next_char();
 819                                 goto end_of_string;
 820
 821                         default: {
 822                                 wchar_rep_t tc = c;
 823                                 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 824                                 next_char();
 825                                 break;
 826                         }
 827                 }
 828         }
 829
 830 end_of_string:;
 831
 832         /* TODO: concatenate multiple strings separated by whitespace... */
 833
 834         /* add finishing 0 to the string */
 835         wchar_rep_t nul = L'\0';
 836         obstack_grow(&symbol_obstack, &nul, sizeof(nul));
 837         const size_t             size   = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
 838         const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
 839
 840 #if 0 /* TODO hash */
 841         /* check if there is already a copy of the string */
 842         const wchar_rep_t *const result = strset_insert(&stringset, string);
 843         if(result != string) {
 844                 obstack_free(&symbol_obstack, string);
 845         }
 846 #else
 847         const wchar_rep_t *const result = string;
 848 #endif
 849
 850         lexer_token.type                = T_WIDE_STRING_LITERAL;
 851         lexer_token.v.wide_string.begin = result;
 852         lexer_token.v.wide_string.size  = size;
 853 }
 854
 855 static void parse_character_constant(void)
 856 {
 857         eat('\'');
 858
 859         int found_char = 0;
 860         while(1) {
 861                 switch(c) {
 862                 case '\\':
 863                         found_char = parse_escape_sequence();
 864                         break;
 865
 866                 MATCH_NEWLINE(
 867                         parse_error("newline while parsing character constant");
 868                         break;
 869                 )
 870
 871                 case '\'':
 872                         next_char();
 873                         goto end_of_char_constant;
 874
 875                 case EOF:
 876                         parse_error("EOF while parsing character constant");
 877                         lexer_token.type = T_ERROR;
 878                         return;
 879
 880                 default:
 881                         if(found_char != 0) {
 882                                 parse_error("more than 1 characters in character "
 883                                             "constant");
 884                                 goto end_of_char_constant;
 885                         } else {
 886                                 found_char = c;
 887                                 next_char();
 888                         }
 889                         break;
 890                 }
 891         }
 892
 893 end_of_char_constant:
 894         lexer_token.type       = T_INTEGER;
 895         lexer_token.v.intvalue = found_char;
 896         lexer_token.datatype   = type_int;
 897 }
 898
 899 static void skip_multiline_comment(void)
 900 {
 901         unsigned start_linenr = lexer_token.source_position.linenr;
 902
 903         while(1) {
 904                 switch(c) {
 905                 case '*':
 906                         next_char();
 907                         if(c == '/') {
 908                                 next_char();
 909                                 return;
 910                         }
 911                         break;
 912
 913                 MATCH_NEWLINE(break;)
 914
 915                 case EOF:
 916                         error_prefix_at(lexer_token.source_position.input_name,
 917                                         start_linenr);
 918                         fprintf(stderr, "at end of file while looking for comment end\n");
 919                         return;
 920
 921                 default:
 922                         next_char();
 923                         break;
 924                 }
 925         }
 926 }
 927
 928 static void skip_line_comment(void)
 929 {
 930         while(1) {
 931                 switch(c) {
 932                 case EOF:
 933                         return;
 934
 935                 case '\n':
 936                 case '\r':
 937                         return;
 938
 939                 default:
 940                         next_char();
 941                         break;
 942                 }
 943         }
 944 }
 945
 946 static token_t pp_token;
 947
 948 static inline void next_pp_token(void)
 949 {
 950         lexer_next_preprocessing_token();
 951         pp_token = lexer_token;
 952 }
 953
 954 static void eat_until_newline(void)
 955 {
 956         while(pp_token.type != '\n' && pp_token.type != T_EOF) {
 957                 next_pp_token();
 958         }
 959 }
 960
 961 static void error_directive(void)
 962 {
 963         error_prefix();
 964         fprintf(stderr, "#error directive: \n");
 965
 966         /* parse pp-tokens until new-line */
 967 }
 968
 969 static void define_directive(void)
 970 {
 971         lexer_next_preprocessing_token();
 972         if(lexer_token.type != T_IDENTIFIER) {
 973                 parse_error("expected identifier after #define\n");
 974                 eat_until_newline();
 975         }
 976 }
 977
 978 static void ifdef_directive(int is_ifndef)
 979 {
 980         (void) is_ifndef;
 981         lexer_next_preprocessing_token();
 982         //expect_identifier();
 983         //extect_newline();
 984 }
 985
 986 static void endif_directive(void)
 987 {
 988         //expect_newline();
 989 }
 990
 991 static void parse_line_directive(void)
 992 {
 993         if(pp_token.type != T_INTEGER) {
 994                 parse_error("expected integer");
 995         } else {
 996                 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
 997                 next_pp_token();
 998         }
 999         if(pp_token.type == T_STRING_LITERAL) {
1000                 lexer_token.source_position.input_name = pp_token.v.string.begin;
1001                 next_pp_token();
1002         }
1003
1004         eat_until_newline();
1005 }
1006
1007 static void parse_preprocessor_identifier(void)
1008 {
1009         assert(pp_token.type == T_IDENTIFIER);
1010         symbol_t *symbol = pp_token.v.symbol;
1011
1012         switch(symbol->pp_ID) {
1013         case TP_include:
1014                 printf("include - enable header name parsing!\n");
1015                 break;
1016         case TP_define:
1017                 define_directive();
1018                 break;
1019         case TP_ifdef:
1020                 ifdef_directive(0);
1021                 break;
1022         case TP_ifndef:
1023                 ifdef_directive(1);
1024                 break;
1025         case TP_endif:
1026                 endif_directive();
1027                 break;
1028         case TP_line:
1029                 next_pp_token();
1030                 parse_line_directive();
1031                 break;
1032         case TP_if:
1033         case TP_else:
1034         case TP_elif:
1035         case TP_undef:
1036         case TP_error:
1037                 error_directive();
1038                 break;
1039         case TP_pragma:
1040                 warningf(lexer_token.source_position, "encountered unknown #pragma");
1041                 eat_until_newline();
1042                 break;
1043         }
1044 }
1045
1046 static void parse_preprocessor_directive(void)
1047 {
1048         next_pp_token();
1049
1050         switch(pp_token.type) {
1051         case T_IDENTIFIER:
1052                 parse_preprocessor_identifier();
1053                 break;
1054         case T_INTEGER:
1055                 parse_line_directive();
1056                 break;
1057         default:
1058                 parse_error("invalid preprocessor directive");
1059                 eat_until_newline();
1060                 break;
1061         }
1062 }
1063
1064 #define MAYBE_PROLOG                                       \
1065                         next_char();                                   \
1066                         while(1) {                                     \
1067                                 switch(c) {
1068
1069 #define MAYBE(ch, set_type)                                \
1070                                 case ch:                                   \
1071                                         next_char();                           \
1072                                         lexer_token.type = set_type;           \
1073                                         return;
1074
1075 #define ELSE_CODE(code)                                    \
1076                                 default:                                   \
1077                                         code;                                  \
1078                                 }                                          \
1079                         } /* end of while(1) */                        \
1080                         break;
1081
1082 #define ELSE(set_type)                                     \
1083                 ELSE_CODE(                                         \
1084                         lexer_token.type = set_type;                   \
1085                         return;                                        \
1086                 )
1087
1088 void lexer_next_preprocessing_token(void)
1089 {
1090         while(1) {
1091                 switch(c) {
1092                 case ' ':
1093                 case '\t':
1094                         next_char();
1095                         break;
1096
1097                 MATCH_NEWLINE(
1098                         lexer_token.type = '\n';
1099                         return;
1100                 )
1101
1102                 SYMBOL_CHARS
1103                         parse_symbol();
1104                         /* might be a wide string ( L"string" ) */
1105                         if(lexer_token.type == T_IDENTIFIER &&
1106                             lexer_token.v.symbol == symbol_L) {
1107                             if(c == '"') {
1108                                         parse_wide_string_literal();
1109                                 } else if(c == '\'') {
1110                                         parse_wide_character_constant();
1111                                 }
1112                         }
1113                         return;
1114
1115                 DIGITS
1116                         parse_number();
1117                         return;
1118
1119                 case '"':
1120                         parse_string_literal();
1121                         return;
1122
1123                 case '\'':
1124                         parse_character_constant();
1125                         return;
1126
1127                 case '.':
1128                         MAYBE_PROLOG
1129                                 case '0':
1130                                 case '1':
1131                                 case '2':
1132                                 case '3':
1133                                 case '4':
1134                                 case '5':
1135                                 case '6':
1136                                 case '7':
1137                                 case '8':
1138                                 case '9':
1139                                         put_back(c);
1140                                         c = '.';
1141                                         parse_number_dec();
1142                                         return;
1143
1144                                 case '.':
1145                                         MAYBE_PROLOG
1146                                         MAYBE('.', T_DOTDOTDOT)
1147                                         ELSE_CODE(
1148                                                 put_back(c);
1149                                                 c = '.';
1150                                                 lexer_token.type = '.';
1151                                                 return;
1152                                         )
1153                         ELSE('.')
1154                 case '&':
1155                         MAYBE_PROLOG
1156                         MAYBE('&', T_ANDAND)
1157                         MAYBE('=', T_ANDEQUAL)
1158                         ELSE('&')
1159                 case '*':
1160                         MAYBE_PROLOG
1161                         MAYBE('=', T_ASTERISKEQUAL)
1162                         ELSE('*')
1163                 case '+':
1164                         MAYBE_PROLOG
1165                         MAYBE('+', T_PLUSPLUS)
1166                         MAYBE('=', T_PLUSEQUAL)
1167                         ELSE('+')
1168                 case '-':
1169                         MAYBE_PROLOG
1170                         MAYBE('>', T_MINUSGREATER)
1171                         MAYBE('-', T_MINUSMINUS)
1172                         MAYBE('=', T_MINUSEQUAL)
1173                         ELSE('-')
1174                 case '!':
1175                         MAYBE_PROLOG
1176                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1177                         ELSE('!')
1178                 case '/':
1179                         MAYBE_PROLOG
1180                         MAYBE('=', T_SLASHEQUAL)
1181                                 case '*':
1182                                         next_char();
1183                                         skip_multiline_comment();
1184                                         lexer_next_preprocessing_token();
1185                                         return;
1186                                 case '/':
1187                                         next_char();
1188                                         skip_line_comment();
1189                                         lexer_next_preprocessing_token();
1190                                         return;
1191                         ELSE('/')
1192                 case '%':
1193                         MAYBE_PROLOG
1194                         MAYBE('>', T_PERCENTGREATER)
1195                         MAYBE('=', T_PERCENTEQUAL)
1196                                 case ':':
1197                                         MAYBE_PROLOG
1198                                                 case '%':
1199                                                         MAYBE_PROLOG
1200                                                         MAYBE(':', T_PERCENTCOLONPERCENTCOLON)
1201                                                         ELSE_CODE(
1202                                                                 put_back(c);
1203                                                                 c = '%';
1204                                                                 lexer_token.type = T_PERCENTCOLON;
1205                                                                 return;
1206                                                         )
1207                                         ELSE(T_PERCENTCOLON)
1208                         ELSE('%')
1209                 case '<':
1210                         MAYBE_PROLOG
1211                         MAYBE(':', T_LESSCOLON)
1212                         MAYBE('%', T_LESSPERCENT)
1213                         MAYBE('=', T_LESSEQUAL)
1214                                 case '<':
1215                                         MAYBE_PROLOG
1216                                         MAYBE('=', T_LESSLESSEQUAL)
1217                                         ELSE(T_LESSLESS)
1218                         ELSE('<')
1219                 case '>':
1220                         MAYBE_PROLOG
1221                         MAYBE('=', T_GREATEREQUAL)
1222                                 case '>':
1223                                         MAYBE_PROLOG
1224                                         MAYBE('=', T_GREATERGREATEREQUAL)
1225                                         ELSE(T_GREATERGREATER)
1226                         ELSE('>')
1227                 case '^':
1228                         MAYBE_PROLOG
1229                         MAYBE('=', T_CARETEQUAL)
1230                         ELSE('^')
1231                 case '|':
1232                         MAYBE_PROLOG
1233                         MAYBE('=', T_PIPEEQUAL)
1234                         MAYBE('|', T_PIPEPIPE)
1235                         ELSE('|')
1236                 case ':':
1237                         MAYBE_PROLOG
1238                         MAYBE('>', T_COLONGREATER)
1239                         ELSE(':')
1240                 case '=':
1241                         MAYBE_PROLOG
1242                         MAYBE('=', T_EQUALEQUAL)
1243                         ELSE('=')
1244                 case '#':
1245                         MAYBE_PROLOG
1246                         MAYBE('#', T_HASHHASH)
1247                         ELSE('#')
1248
1249                 case '?':
1250                 case '[':
1251                 case ']':
1252                 case '(':
1253                 case ')':
1254                 case '{':
1255                 case '}':
1256                 case '~':
1257                 case ';':
1258                 case ',':
1259                 case '\\':
1260                         lexer_token.type = c;
1261                         next_char();
1262                         return;
1263
1264                 case EOF:
1265                         lexer_token.type = T_EOF;
1266                         return;
1267
1268                 default:
1269                         next_char();
1270                         error_prefix();
1271                         fprintf(stderr, "unknown character '%c' found\n", c);
1272                         lexer_token.type = T_ERROR;
1273                         return;
1274                 }
1275         }
1276 }
1277
1278 void lexer_next_token(void)
1279 {
1280         lexer_next_preprocessing_token();
1281         if(lexer_token.type != '\n')
1282                 return;
1283
1284 newline_found:
1285         do {
1286                 lexer_next_preprocessing_token();
1287         } while(lexer_token.type == '\n');
1288
1289         if(lexer_token.type == '#') {
1290                 parse_preprocessor_directive();
1291                 goto newline_found;
1292         }
1293 }
1294
1295 void init_lexer(void)
1296 {
1297         strset_init(&stringset);
1298 }
1299
1300 void lexer_open_stream(FILE *stream, const char *input_name)
1301 {
1302         input                                  = stream;
1303         lexer_token.source_position.linenr     = 0;
1304         lexer_token.source_position.input_name = input_name;
1305
1306         symbol_L = symbol_table_insert("L");
1307         bufpos = NULL;
1308         bufend = NULL;
1309
1310         /* place a virtual \n at the beginning so the lexer knows that we're
1311          * at the beginning of a line */
1312         c = '\n';
1313 }
1314
1315 void exit_lexer(void)
1316 {
1317         strset_destroy(&stringset);
1318 }
1319
1320 static __attribute__((unused))
1321 void dbg_pos(const source_position_t source_position)
1322 {
1323         fprintf(stdout, "%s:%u\n", source_position.input_name,
1324                 source_position.linenr);
1325         fflush(stdout);
1326 }