nsz Git - cparser/blob - lexer.c

   1 #include <config.h>
   2
   3 #include "diagnostic.h"
   4 #include "lexer.h"
   5 #include "token_t.h"
   6 #include "symbol_table_t.h"
   7 #include "adt/error.h"
   8 #include "adt/strset.h"
   9 #include "adt/util.h"
  10 #include "types.h"
  11 #include "type_t.h"
  12 #include "target_architecture.h"
  13 #include "parser.h"
  14 #include "warning.h"
  15 #include "lang_features.h"
  16
  17 #include <assert.h>
  18 #include <errno.h>
  19 #include <string.h>
  20 #include <stdbool.h>
  21 #include <ctype.h>
  22
  23 //#define DEBUG_CHARS
  24 #define MAX_PUTBACK 3
  25
  26 #ifdef _WIN32
  27 /* No strtold on windows and no replacement yet */
  28 #define strtold(s, e) strtod(s, e)
  29 #endif
  30
  31 #if defined HAS_SIGNED_CHAR
  32 typedef signed char char_type;
  33 #elif defined HAS_UNSIGNED_CHAR
  34 typedef unsigned char char_type;
  35 #else
  36 #       error signedness of char not determined
  37 #endif
  38
  39 static int         c;
  40 token_t            lexer_token;
  41 symbol_t          *symbol_L;
  42 static FILE       *input;
  43 static char        buf[1024 + MAX_PUTBACK];
  44 static const char *bufend;
  45 static const char *bufpos;
  46 static strset_t    stringset;
  47
  48 /**
  49  * Prints a parse error message at the current token.
  50  *
  51  * @param msg   the error message
  52  */
  53 static void parse_error(const char *msg)
  54 {
  55         errorf(lexer_token.source_position,  "%s", msg);
  56 }
  57
  58 static inline void next_real_char(void)
  59 {
  60         assert(bufpos <= bufend);
  61         if (bufpos >= bufend) {
  62                 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
  63                                  input);
  64                 if(s == 0) {
  65                         c = EOF;
  66                         return;
  67                 }
  68                 bufpos = buf + MAX_PUTBACK;
  69                 bufend = buf + MAX_PUTBACK + s;
  70         }
  71         c = *bufpos++;
  72 }
  73
  74 /**
  75  * Put a character back into the buffer.
  76  *
  77  * @param pc  the character to put back
  78  */
  79 static inline void put_back(int pc)
  80 {
  81         assert(bufpos > buf);
  82         *(--bufpos - buf + buf) = (char) pc;
  83
  84 #ifdef DEBUG_CHARS
  85         printf("putback '%c'\n", pc);
  86 #endif
  87 }
  88
  89 static inline void next_char(void);
  90
  91 #define MATCH_NEWLINE(code)                   \
  92         case '\r':                                \
  93                 next_char();                          \
  94                 if(c == '\n') {                       \
  95                         next_char();                      \
  96                 }                                     \
  97                 lexer_token.source_position.linenr++; \
  98                 code                                  \
  99         case '\n':                                \
 100                 next_char();                          \
 101                 lexer_token.source_position.linenr++; \
 102                 code
 103
 104 #define eat(c_type)  do { assert(c == c_type); next_char(); } while(0)
 105
 106 static void maybe_concat_lines(void)
 107 {
 108         eat('\\');
 109
 110         switch(c) {
 111         MATCH_NEWLINE(return;)
 112
 113         default:
 114                 break;
 115         }
 116
 117         put_back(c);
 118         c = '\\';
 119 }
 120
 121 /**
 122  * Set c to the next input character, ie.
 123  * after expanding trigraphs.
 124  */
 125 static inline void next_char(void)
 126 {
 127         next_real_char();
 128
 129         /* filter trigraphs */
 130         if(UNLIKELY(c == '\\')) {
 131                 maybe_concat_lines();
 132                 goto end_of_next_char;
 133         }
 134
 135         if(LIKELY(c != '?'))
 136                 goto end_of_next_char;
 137
 138         next_real_char();
 139         if(LIKELY(c != '?')) {
 140                 put_back(c);
 141                 c = '?';
 142                 goto end_of_next_char;
 143         }
 144
 145         next_real_char();
 146         switch(c) {
 147         case '=': c = '#'; break;
 148         case '(': c = '['; break;
 149         case '/': c = '\\'; maybe_concat_lines(); break;
 150         case ')': c = ']'; break;
 151         case '\'': c = '^'; break;
 152         case '<': c = '{'; break;
 153         case '!': c = '|'; break;
 154         case '>': c = '}'; break;
 155         case '-': c = '~'; break;
 156         default:
 157                 put_back(c);
 158                 put_back('?');
 159                 c = '?';
 160                 break;
 161         }
 162
 163 end_of_next_char:;
 164 #ifdef DEBUG_CHARS
 165         printf("nchar '%c'\n", c);
 166 #endif
 167 }
 168
 169 #define SYMBOL_CHARS  \
 170         case 'a':         \
 171         case 'b':         \
 172         case 'c':         \
 173         case 'd':         \
 174         case 'e':         \
 175         case 'f':         \
 176         case 'g':         \
 177         case 'h':         \
 178         case 'i':         \
 179         case 'j':         \
 180         case 'k':         \
 181         case 'l':         \
 182         case 'm':         \
 183         case 'n':         \
 184         case 'o':         \
 185         case 'p':         \
 186         case 'q':         \
 187         case 'r':         \
 188         case 's':         \
 189         case 't':         \
 190         case 'u':         \
 191         case 'v':         \
 192         case 'w':         \
 193         case 'x':         \
 194         case 'y':         \
 195         case 'z':         \
 196         case 'A':         \
 197         case 'B':         \
 198         case 'C':         \
 199         case 'D':         \
 200         case 'E':         \
 201         case 'F':         \
 202         case 'G':         \
 203         case 'H':         \
 204         case 'I':         \
 205         case 'J':         \
 206         case 'K':         \
 207         case 'L':         \
 208         case 'M':         \
 209         case 'N':         \
 210         case 'O':         \
 211         case 'P':         \
 212         case 'Q':         \
 213         case 'R':         \
 214         case 'S':         \
 215         case 'T':         \
 216         case 'U':         \
 217         case 'V':         \
 218         case 'W':         \
 219         case 'X':         \
 220         case 'Y':         \
 221         case 'Z':         \
 222         case '_':
 223
 224 #define DIGITS        \
 225         case '0':         \
 226         case '1':         \
 227         case '2':         \
 228         case '3':         \
 229         case '4':         \
 230         case '5':         \
 231         case '6':         \
 232         case '7':         \
 233         case '8':         \
 234         case '9':
 235
 236 /**
 237  * Read a symbol from the input and build
 238  * the lexer_token.
 239  */
 240 static void parse_symbol(void)
 241 {
 242         symbol_t *symbol;
 243         char     *string;
 244
 245         obstack_1grow(&symbol_obstack, (char) c);
 246         next_char();
 247
 248         while(1) {
 249                 switch(c) {
 250                 DIGITS
 251                 SYMBOL_CHARS
 252                         obstack_1grow(&symbol_obstack, (char) c);
 253                         next_char();
 254                         break;
 255
 256                 default:
 257                         goto end_symbol;
 258                 }
 259         }
 260
 261 end_symbol:
 262         obstack_1grow(&symbol_obstack, '\0');
 263
 264         string = obstack_finish(&symbol_obstack);
 265         symbol = symbol_table_insert(string);
 266
 267         lexer_token.type     = symbol->ID;
 268         lexer_token.v.symbol = symbol;
 269
 270         if(symbol->string != string) {
 271                 obstack_free(&symbol_obstack, string);
 272         }
 273 }
 274
 275 static void parse_integer_suffix(bool is_oct_hex)
 276 {
 277         bool is_unsigned  = false;
 278         bool min_long     = false;
 279         bool min_longlong = false;
 280
 281         if(c == 'U' || c == 'u') {
 282                 is_unsigned = true;
 283                 next_char();
 284                 if(c == 'L' || c == 'l') {
 285                         min_long = true;
 286                         next_char();
 287                         if(c == 'L' || c == 'l') {
 288                                 min_longlong = true;
 289                                 next_char();
 290                         }
 291                 }
 292         } else if(c == 'l' || c == 'L') {
 293                 min_long = true;
 294                 next_char();
 295                 if(c == 'l' || c == 'L') {
 296                         min_longlong = true;
 297                         next_char();
 298                         if(c == 'u' || c == 'U') {
 299                                 is_unsigned = true;
 300                                 next_char();
 301                         }
 302                 } else if(c == 'u' || c == 'U') {
 303                         is_unsigned = true;
 304                         next_char();
 305                         lexer_token.datatype = type_unsigned_long;
 306                 }
 307         }
 308
 309         if(!is_unsigned) {
 310                 long long v = lexer_token.v.intvalue;
 311                 if(!min_long) {
 312                         if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
 313                                 lexer_token.datatype = type_int;
 314                                 return;
 315                         } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
 316                                 lexer_token.datatype = type_unsigned_int;
 317                                 return;
 318                         }
 319                 }
 320                 if(!min_longlong) {
 321                         if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
 322                                 lexer_token.datatype = type_long;
 323                                 return;
 324                         } else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) {
 325                                 lexer_token.datatype = type_unsigned_long;
 326                                 return;
 327                         }
 328                 }
 329                 unsigned long long uv = (unsigned long long) v;
 330                 if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
 331                         lexer_token.datatype = type_unsigned_long_long;
 332                         return;
 333                 }
 334
 335                 lexer_token.datatype = type_long_long;
 336         } else {
 337                 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
 338                 if(!min_long && v <= TARGET_UINT_MAX) {
 339                         lexer_token.datatype = type_unsigned_int;
 340                         return;
 341                 }
 342                 if(!min_longlong && v <= TARGET_ULONG_MAX) {
 343                         lexer_token.datatype = type_unsigned_long;
 344                         return;
 345                 }
 346                 lexer_token.datatype = type_unsigned_long_long;
 347         }
 348 }
 349
 350 static void parse_floating_suffix(void)
 351 {
 352         switch(c) {
 353         /* TODO: do something useful with the suffixes... */
 354         case 'f':
 355         case 'F':
 356                 next_char();
 357                 lexer_token.datatype = type_float;
 358                 break;
 359         case 'l':
 360         case 'L':
 361                 next_char();
 362                 lexer_token.datatype = type_long_double;
 363                 break;
 364         default:
 365                 lexer_token.datatype = type_double;
 366                 break;
 367         }
 368 }
 369
 370 /**
 371  * A replacement for strtoull. Only those parts needed for
 372  * our parser are implemented.
 373  */
 374 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
 375         unsigned long long v = 0;
 376
 377         switch (base) {
 378         case 16:
 379                 for (;; ++s) {
 380                         /* check for overrun */
 381                         if (v >= 0x1000000000000000ULL)
 382                                 break;
 383                         switch (tolower(*s)) {
 384                         case '0': v <<= 4; break;
 385                         case '1': v <<= 4; v |= 0x1; break;
 386                         case '2': v <<= 4; v |= 0x2; break;
 387                         case '3': v <<= 4; v |= 0x3; break;
 388                         case '4': v <<= 4; v |= 0x4; break;
 389                         case '5': v <<= 4; v |= 0x5; break;
 390                         case '6': v <<= 4; v |= 0x6; break;
 391                         case '7': v <<= 4; v |= 0x7; break;
 392                         case '8': v <<= 4; v |= 0x8; break;
 393                         case '9': v <<= 4; v |= 0x9; break;
 394                         case 'a': v <<= 4; v |= 0xa; break;
 395                         case 'b': v <<= 4; v |= 0xb; break;
 396                         case 'c': v <<= 4; v |= 0xc; break;
 397                         case 'd': v <<= 4; v |= 0xd; break;
 398                         case 'e': v <<= 4; v |= 0xe; break;
 399                         case 'f': v <<= 4; v |= 0xf; break;
 400                         default:
 401                                 goto end;
 402                         }
 403                 }
 404                 break;
 405         case 8:
 406                 for (;; ++s) {
 407                         /* check for overrun */
 408                         if (v >= 0x2000000000000000ULL)
 409                                 break;
 410                         switch (tolower(*s)) {
 411                         case '0': v <<= 3; break;
 412                         case '1': v <<= 3; v |= 1; break;
 413                         case '2': v <<= 3; v |= 2; break;
 414                         case '3': v <<= 3; v |= 3; break;
 415                         case '4': v <<= 3; v |= 4; break;
 416                         case '5': v <<= 3; v |= 5; break;
 417                         case '6': v <<= 3; v |= 6; break;
 418                         case '7': v <<= 3; v |= 7; break;
 419                         default:
 420                                 goto end;
 421                         }
 422                 }
 423                 break;
 424         case 10:
 425                 for (;; ++s) {
 426                         /* check for overrun */
 427                         if (v > 0x1999999999999999ULL)
 428                                 break;
 429                         switch (tolower(*s)) {
 430                         case '0': v *= 10; break;
 431                         case '1': v *= 10; v += 1; break;
 432                         case '2': v *= 10; v += 2; break;
 433                         case '3': v *= 10; v += 3; break;
 434                         case '4': v *= 10; v += 4; break;
 435                         case '5': v *= 10; v += 5; break;
 436                         case '6': v *= 10; v += 6; break;
 437                         case '7': v *= 10; v += 7; break;
 438                         case '8': v *= 10; v += 8; break;
 439                         case '9': v *= 10; v += 9; break;
 440                         default:
 441                                 goto end;
 442                         }
 443                 }
 444                 break;
 445         default:
 446                 assert(0);
 447                 break;
 448         }
 449 end:
 450         *endptr = s;
 451         return v;
 452 }
 453
 454 /**
 455  * Parses a hex number including hex floats and set the
 456  * lexer_token.
 457  */
 458 static void parse_number_hex(void)
 459 {
 460         assert(c == 'x' || c == 'X');
 461         next_char();
 462
 463         while(isxdigit(c)) {
 464                 obstack_1grow(&symbol_obstack, (char) c);
 465                 next_char();
 466         }
 467         obstack_1grow(&symbol_obstack, '\0');
 468         char *string = obstack_finish(&symbol_obstack);
 469
 470         if(c == '.' || c == 'p' || c == 'P') {
 471                 next_char();
 472                 panic("Hex floating point numbers not implemented yet");
 473         }
 474         if(*string == '\0') {
 475                 parse_error("invalid hex number");
 476                 lexer_token.type = T_ERROR;
 477         }
 478
 479         const char *endptr;
 480         lexer_token.type       = T_INTEGER;
 481         lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
 482         if(*endptr != '\0') {
 483                 parse_error("hex number literal too long");
 484         }
 485
 486         obstack_free(&symbol_obstack, string);
 487         parse_integer_suffix(true);
 488 }
 489
 490 /**
 491  * Returns true if the given char is a octal digit.
 492  *
 493  * @param char  the character to check
 494  */
 495 static inline bool is_octal_digit(int chr)
 496 {
 497         switch(chr) {
 498         case '0':
 499         case '1':
 500         case '2':
 501         case '3':
 502         case '4':
 503         case '5':
 504         case '6':
 505         case '7':
 506                 return true;
 507         default:
 508                 return false;
 509         }
 510 }
 511
 512 /**
 513  * Parses a octal number and set the lexer_token.
 514  */
 515 static void parse_number_oct(void)
 516 {
 517         while(is_octal_digit(c)) {
 518                 obstack_1grow(&symbol_obstack, (char) c);
 519                 next_char();
 520         }
 521         obstack_1grow(&symbol_obstack, '\0');
 522         char *string = obstack_finish(&symbol_obstack);
 523
 524         const char *endptr;
 525         lexer_token.type       = T_INTEGER;
 526         lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
 527         if(*endptr != '\0') {
 528                 parse_error("octal number literal too long");
 529         }
 530
 531         obstack_free(&symbol_obstack, string);
 532         parse_integer_suffix(true);
 533 }
 534
 535 /**
 536  * Parses a decimal including float number and set the
 537  * lexer_token.
 538  */
 539 static void parse_number_dec(void)
 540 {
 541         bool is_float = false;
 542         while(isdigit(c)) {
 543                 obstack_1grow(&symbol_obstack, (char) c);
 544                 next_char();
 545         }
 546
 547         if(c == '.') {
 548                 obstack_1grow(&symbol_obstack, '.');
 549                 next_char();
 550
 551                 while(isdigit(c)) {
 552                         obstack_1grow(&symbol_obstack, (char) c);
 553                         next_char();
 554                 }
 555                 is_float = true;
 556         }
 557         if(c == 'e' || c == 'E') {
 558                 obstack_1grow(&symbol_obstack, 'e');
 559                 next_char();
 560
 561                 if(c == '-' || c == '+') {
 562                         obstack_1grow(&symbol_obstack, (char) c);
 563                         next_char();
 564                 }
 565
 566                 while(isdigit(c)) {
 567                         obstack_1grow(&symbol_obstack, (char) c);
 568                         next_char();
 569                 }
 570                 is_float = true;
 571         }
 572
 573         obstack_1grow(&symbol_obstack, '\0');
 574         char *string = obstack_finish(&symbol_obstack);
 575
 576         if(is_float) {
 577                 char *endptr;
 578                 lexer_token.type         = T_FLOATINGPOINT;
 579                 lexer_token.v.floatvalue = strtold(string, &endptr);
 580
 581                 if(*endptr != '\0') {
 582                         parse_error("invalid number literal");
 583                 }
 584
 585                 parse_floating_suffix();
 586         } else {
 587                 const char *endptr;
 588                 lexer_token.type       = T_INTEGER;
 589                 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
 590
 591                 if(*endptr != '\0') {
 592                         parse_error("invalid number literal");
 593                 }
 594
 595                 parse_integer_suffix(false);
 596         }
 597         obstack_free(&symbol_obstack, string);
 598 }
 599
 600 /**
 601  * Parses a number and sets the lexer_token.
 602  */
 603 static void parse_number(void)
 604 {
 605         if (c == '0') {
 606                 next_char();
 607                 switch (c) {
 608                         case 'X':
 609                         case 'x':
 610                                 parse_number_hex();
 611                                 break;
 612                         case '0':
 613                         case '1':
 614                         case '2':
 615                         case '3':
 616                         case '4':
 617                         case '5':
 618                         case '6':
 619                         case '7':
 620                                 parse_number_oct();
 621                                 break;
 622                         case '8':
 623                         case '9':
 624                                 next_char();
 625                                 parse_error("invalid octal number");
 626                                 lexer_token.type = T_ERROR;
 627                                 return;
 628                         case '.':
 629                         case 'e':
 630                         case 'E':
 631                         default:
 632                                 obstack_1grow(&symbol_obstack, '0');
 633                                 parse_number_dec();
 634                                 return;
 635                 }
 636         } else {
 637                 parse_number_dec();
 638         }
 639 }
 640
 641 /**
 642  * Returns the value of a digit.
 643  * The only portable way to do it ...
 644  */
 645 static int digit_value(int digit) {
 646         switch (digit) {
 647         case '0': return 0;
 648         case '1': return 1;
 649         case '2': return 2;
 650         case '3': return 3;
 651         case '4': return 4;
 652         case '5': return 5;
 653         case '6': return 6;
 654         case '7': return 7;
 655         case '8': return 8;
 656         case '9': return 9;
 657         case 'a':
 658         case 'A': return 10;
 659         case 'b':
 660         case 'B': return 11;
 661         case 'c':
 662         case 'C': return 12;
 663         case 'd':
 664         case 'D': return 13;
 665         case 'e':
 666         case 'E': return 14;
 667         case 'f':
 668         case 'F': return 15;
 669         default:
 670                 panic("wrong character given");
 671         }
 672 }
 673
 674 /**
 675  * Parses an octal character sequence.
 676  *
 677  * @param first_digit  the already read first digit
 678  */
 679 static int parse_octal_sequence(const int first_digit)
 680 {
 681         assert(is_octal_digit(first_digit));
 682         int value = digit_value(first_digit);
 683         if (!is_octal_digit(c)) return value;
 684         value = 8 * value + digit_value(c);
 685         next_char();
 686         if (!is_octal_digit(c)) return value;
 687         value = 8 * value + digit_value(c);
 688         next_char();
 689         return (char_type)value;
 690 }
 691
 692 /**
 693  * Parses a hex character sequence.
 694  */
 695 static int parse_hex_sequence(void)
 696 {
 697         int value = 0;
 698         while(isxdigit(c)) {
 699                 value = 16 * value + digit_value(c);
 700                 next_char();
 701         }
 702
 703         return (char_type)value;
 704 }
 705
 706 /**
 707  * Parse an escape sequence.
 708  */
 709 static int parse_escape_sequence(void)
 710 {
 711         eat('\\');
 712
 713         int ec = c;
 714         next_char();
 715
 716         switch(ec) {
 717         case '"':  return '"';
 718         case '\'': return '\'';
 719         case '\\': return '\\';
 720         case '?': return '\?';
 721         case 'a': return '\a';
 722         case 'b': return '\b';
 723         case 'f': return '\f';
 724         case 'n': return '\n';
 725         case 'r': return '\r';
 726         case 't': return '\t';
 727         case 'v': return '\v';
 728         case 'x':
 729                 return parse_hex_sequence();
 730         case '0':
 731         case '1':
 732         case '2':
 733         case '3':
 734         case '4':
 735         case '5':
 736         case '6':
 737         case '7':
 738                 return parse_octal_sequence(ec);
 739         case EOF:
 740                 parse_error("reached end of file while parsing escape sequence");
 741                 return EOF;
 742         default:
 743                 parse_error("unknown escape sequence");
 744                 return EOF;
 745         }
 746 }
 747
 748 /**
 749  * Concatenate two strings.
 750  */
 751 string_t concat_strings(const string_t *const s1, const string_t *const s2)
 752 {
 753         const size_t len1 = s1->size - 1;
 754         const size_t len2 = s2->size - 1;
 755
 756         char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
 757         memcpy(concat, s1->begin, len1);
 758         memcpy(concat + len1, s2->begin, len2 + 1);
 759
 760 #if 0 /* TODO hash */
 761         const char *result = strset_insert(&stringset, concat);
 762         if(result != concat) {
 763                 obstack_free(&symbol_obstack, concat);
 764         }
 765
 766         return result;
 767 #else
 768         return (string_t){ concat, len1 + len2 + 1 };
 769 #endif
 770 }
 771
 772 /**
 773  * Concatenate a string and a wide string.
 774  */
 775 wide_string_t concat_string_wide_string(const string_t *const s1, const wide_string_t *const s2)
 776 {
 777         const size_t len1 = s1->size - 1;
 778         const size_t len2 = s2->size - 1;
 779
 780         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 781         const char *const src = s1->begin;
 782         for (size_t i = 0; i != len1; ++i) {
 783                 concat[i] = src[i];
 784         }
 785         memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
 786
 787         return (wide_string_t){ concat, len1 + len2 + 1 };
 788 }
 789
 790 /**
 791  * Concatenate two wide strings.
 792  */
 793 wide_string_t concat_wide_strings(const wide_string_t *const s1, const wide_string_t *const s2)
 794 {
 795         const size_t len1 = s1->size - 1;
 796         const size_t len2 = s2->size - 1;
 797
 798         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 799         memcpy(concat,        s1->begin, len1       * sizeof(*concat));
 800         memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
 801
 802         return (wide_string_t){ concat, len1 + len2 + 1 };
 803 }
 804
 805 /**
 806  * Concatenate a wide string and a string.
 807  */
 808 wide_string_t concat_wide_string_string(const wide_string_t *const s1, const string_t *const s2)
 809 {
 810         const size_t len1 = s1->size - 1;
 811         const size_t len2 = s2->size - 1;
 812
 813         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 814         memcpy(concat, s1->begin, len1 * sizeof(*concat));
 815         const char *const src = s2->begin;
 816         for (size_t i = 0; i != len2 + 1; ++i) {
 817                 concat[i] = src[i];
 818         }
 819
 820         return (wide_string_t){ concat, len1 + len2 + 1 };
 821 }
 822
 823 /**
 824  * Parse a string literal and set lexer_token.
 825  */
 826 static void parse_string_literal(void)
 827 {
 828         const unsigned start_linenr = lexer_token.source_position.linenr;
 829
 830         eat('"');
 831
 832         int tc;
 833         while(1) {
 834                 switch(c) {
 835                 case '\\':
 836                         tc = parse_escape_sequence();
 837                         obstack_1grow(&symbol_obstack, (char) tc);
 838                         break;
 839
 840                 case EOF: {
 841                         source_position_t source_position;
 842                         source_position.input_name = lexer_token.source_position.input_name;
 843                         source_position.linenr     = start_linenr;
 844                         errorf(source_position, "string has no end");
 845                         lexer_token.type = T_ERROR;
 846                         return;
 847                 }
 848
 849                 case '"':
 850                         next_char();
 851                         goto end_of_string;
 852
 853                 default:
 854                         obstack_1grow(&symbol_obstack, (char) c);
 855                         next_char();
 856                         break;
 857                 }
 858         }
 859
 860 end_of_string:
 861
 862         /* TODO: concatenate multiple strings separated by whitespace... */
 863
 864         /* add finishing 0 to the string */
 865         obstack_1grow(&symbol_obstack, '\0');
 866         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
 867         const char *const string = obstack_finish(&symbol_obstack);
 868
 869 #if 0 /* TODO hash */
 870         /* check if there is already a copy of the string */
 871         result = strset_insert(&stringset, string);
 872         if(result != string) {
 873                 obstack_free(&symbol_obstack, string);
 874         }
 875 #else
 876         const char *const result = string;
 877 #endif
 878
 879         lexer_token.type           = T_STRING_LITERAL;
 880         lexer_token.v.string.begin = result;
 881         lexer_token.v.string.size  = size;
 882 }
 883
 884 /**
 885  * Parse a wide character constant and set lexer_token.
 886  */
 887 static void parse_wide_character_constant(void)
 888 {
 889         eat('\'');
 890
 891         int found_char = 0;
 892         while(1) {
 893                 switch(c) {
 894                 case '\\':
 895                         found_char = parse_escape_sequence();
 896                         break;
 897
 898                 MATCH_NEWLINE(
 899                         parse_error("newline while parsing character constant");
 900                         break;
 901                 )
 902
 903                 case '\'':
 904                         next_char();
 905                         goto end_of_wide_char_constant;
 906
 907                 case EOF:
 908                         parse_error("EOF while parsing character constant");
 909                         lexer_token.type = T_ERROR;
 910                         return;
 911
 912                 default:
 913                         if(found_char != 0) {
 914                                 parse_error("more than 1 characters in character "
 915                                             "constant");
 916                                 goto end_of_wide_char_constant;
 917                         } else {
 918                                 found_char = c;
 919                                 next_char();
 920                         }
 921                         break;
 922                 }
 923         }
 924
 925 end_of_wide_char_constant:
 926         lexer_token.type       = T_INTEGER;
 927         lexer_token.v.intvalue = found_char;
 928         lexer_token.datatype   = type_wchar_t;
 929 }
 930
 931 /**
 932  * Parse a wide string literal and set lexer_token.
 933  */
 934 static void parse_wide_string_literal(void)
 935 {
 936         const unsigned start_linenr = lexer_token.source_position.linenr;
 937
 938         assert(c == '"');
 939         next_char();
 940
 941         while(1) {
 942                 switch(c) {
 943                 case '\\': {
 944                         wchar_rep_t tc = parse_escape_sequence();
 945                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 946                         break;
 947                 }
 948
 949                 case EOF: {
 950                         source_position_t source_position;
 951                         source_position.input_name = lexer_token.source_position.input_name;
 952                         source_position.linenr     = start_linenr;
 953                         errorf(source_position, "string has no end");
 954                         lexer_token.type = T_ERROR;
 955                         return;
 956                 }
 957
 958                 case '"':
 959                         next_char();
 960                         goto end_of_string;
 961
 962                 default: {
 963                         wchar_rep_t tc = c;
 964                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 965                         next_char();
 966                         break;
 967                 }
 968                 }
 969         }
 970
 971 end_of_string:;
 972
 973         /* TODO: concatenate multiple strings separated by whitespace... */
 974
 975         /* add finishing 0 to the string */
 976         wchar_rep_t nul = L'\0';
 977         obstack_grow(&symbol_obstack, &nul, sizeof(nul));
 978         const size_t             size   = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
 979         const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
 980
 981 #if 0 /* TODO hash */
 982         /* check if there is already a copy of the string */
 983         const wchar_rep_t *const result = strset_insert(&stringset, string);
 984         if(result != string) {
 985                 obstack_free(&symbol_obstack, string);
 986         }
 987 #else
 988         const wchar_rep_t *const result = string;
 989 #endif
 990
 991         lexer_token.type                = T_WIDE_STRING_LITERAL;
 992         lexer_token.v.wide_string.begin = result;
 993         lexer_token.v.wide_string.size  = size;
 994 }
 995
 996 /**
 997  * Parse a character constant and set lexer_token.
 998  */
 999 static void parse_character_constant(void)
1000 {
1001         const unsigned start_linenr = lexer_token.source_position.linenr;
1002
1003         eat('\'');
1004
1005         int tc;
1006         while(1) {
1007                 switch(c) {
1008                 case '\\':
1009                         tc = parse_escape_sequence();
1010                         obstack_1grow(&symbol_obstack, (char) tc);
1011                         break;
1012
1013                 MATCH_NEWLINE(
1014                         parse_error("newline while parsing character constant");
1015                         break;
1016                 )
1017
1018                 case EOF: {
1019                         source_position_t source_position;
1020                         source_position.input_name = lexer_token.source_position.input_name;
1021                         source_position.linenr     = start_linenr;
1022                         errorf(source_position, "EOF while parsing character constant");
1023                         lexer_token.type = T_ERROR;
1024                         return;
1025                 }
1026
1027                 case '\'':
1028                         next_char();
1029                         goto end_of_char_constant;
1030
1031                 default:
1032                         obstack_1grow(&symbol_obstack, (char) c);
1033                         next_char();
1034                         break;
1035
1036                 }
1037         }
1038
1039 end_of_char_constant:;
1040         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
1041         const char *const string = obstack_finish(&symbol_obstack);
1042
1043         lexer_token.type           = T_CHARS;
1044         lexer_token.v.string.begin = string;
1045         lexer_token.v.string.size  = size;
1046         lexer_token.datatype       = type_int;
1047 }
1048
1049 /**
1050  * Skip a multiline comment.
1051  */
1052 static void skip_multiline_comment(void)
1053 {
1054         unsigned start_linenr = lexer_token.source_position.linenr;
1055
1056         while(1) {
1057                 switch(c) {
1058                 case '/':
1059                         next_char();
1060                         if (c == '*') {
1061                                 /* TODO: nested comment, warn here */
1062                         }
1063                         break;
1064                 case '*':
1065                         next_char();
1066                         if(c == '/') {
1067                                 next_char();
1068                                 return;
1069                         }
1070                         break;
1071
1072                 MATCH_NEWLINE(break;)
1073
1074                 case EOF: {
1075                         source_position_t source_position;
1076                         source_position.input_name = lexer_token.source_position.input_name;
1077                         source_position.linenr     = start_linenr;
1078                         errorf(source_position, "at end of file while looking for comment end");
1079                         return;
1080                 }
1081
1082                 default:
1083                         next_char();
1084                         break;
1085                 }
1086         }
1087 }
1088
1089 /**
1090  * Skip a single line comment.
1091  */
1092 static void skip_line_comment(void)
1093 {
1094         while(1) {
1095                 switch(c) {
1096                 case EOF:
1097                         return;
1098
1099                 case '\n':
1100                 case '\r':
1101                         return;
1102
1103                 default:
1104                         next_char();
1105                         break;
1106                 }
1107         }
1108 }
1109
1110 /** The current preprocessor token. */
1111 static token_t pp_token;
1112
1113 /**
1114  * Read the next preprocessor token.
1115  */
1116 static inline void next_pp_token(void)
1117 {
1118         lexer_next_preprocessing_token();
1119         pp_token = lexer_token;
1120 }
1121
1122 /**
1123  * Eat all preprocessor tokens until newline.
1124  */
1125 static void eat_until_newline(void)
1126 {
1127         while(pp_token.type != '\n' && pp_token.type != T_EOF) {
1128                 next_pp_token();
1129         }
1130 }
1131
1132 /**
1133  * Handle the define directive.
1134  */
1135 static void define_directive(void)
1136 {
1137         lexer_next_preprocessing_token();
1138         if(lexer_token.type != T_IDENTIFIER) {
1139                 parse_error("expected identifier after #define\n");
1140                 eat_until_newline();
1141         }
1142 }
1143
1144 /**
1145  * Handle the ifdef directive.
1146  */
1147 static void ifdef_directive(int is_ifndef)
1148 {
1149         (void) is_ifndef;
1150         lexer_next_preprocessing_token();
1151         //expect_identifier();
1152         //extect_newline();
1153 }
1154
1155 /**
1156  * Handle the endif directive.
1157  */
1158 static void endif_directive(void)
1159 {
1160         //expect_newline();
1161 }
1162
1163 /**
1164  * Parse the line directive.
1165  */
1166 static void parse_line_directive(void)
1167 {
1168         if(pp_token.type != T_INTEGER) {
1169                 parse_error("expected integer");
1170         } else {
1171                 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
1172                 next_pp_token();
1173         }
1174         if(pp_token.type == T_STRING_LITERAL) {
1175                 lexer_token.source_position.input_name = pp_token.v.string.begin;
1176                 next_pp_token();
1177         }
1178
1179         eat_until_newline();
1180 }
1181
1182 /**
1183  * STDC pragmas.
1184  */
1185 typedef enum {
1186         STDC_UNKNOWN,
1187         STDC_FP_CONTRACT,
1188         STDC_FENV_ACCESS,
1189         STDC_CX_LIMITED_RANGE
1190 } stdc_pragma_kind_t;
1191
1192 /**
1193  * STDC pragma values.
1194  */
1195 typedef enum {
1196         STDC_VALUE_UNKNOWN,
1197         STDC_VALUE_ON,
1198         STDC_VALUE_OFF,
1199         STDC_VALUE_DEFAULT
1200 } stdc_pragma_value_kind_t;
1201
1202 /**
1203  * Parse a pragma directive.
1204  */
1205 static void parse_pragma(void) {
1206         bool unknown_pragma = true;
1207
1208         next_pp_token();
1209         if (pp_token.v.symbol->pp_ID == TP_STDC) {
1210                 stdc_pragma_kind_t kind = STDC_UNKNOWN;
1211                 /* a STDC pragma */
1212                 if (c_mode & _C99) {
1213                         next_pp_token();
1214
1215                         switch (pp_token.v.symbol->pp_ID) {
1216                         case TP_FP_CONTRACT:
1217                                 kind = STDC_FP_CONTRACT;
1218                                 break;
1219                         case TP_FENV_ACCESS:
1220                                 kind = STDC_FENV_ACCESS;
1221                                 break;
1222                         case TP_CX_LIMITED_RANGE:
1223                                 kind = STDC_CX_LIMITED_RANGE;
1224                                 break;
1225                         default:
1226                                 break;
1227                         }
1228                         if (kind != STDC_UNKNOWN) {
1229                                 stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN;
1230                                 next_pp_token();
1231                                 switch (pp_token.v.symbol->pp_ID) {
1232                                 case TP_ON:
1233                                         value = STDC_VALUE_ON;
1234                                         break;
1235                                 case TP_OFF:
1236                                         value = STDC_VALUE_OFF;
1237                                         break;
1238                                 case TP_DEFAULT:
1239                                         value = STDC_VALUE_DEFAULT;
1240                                         break;
1241                                 default:
1242                                         break;
1243                                 }
1244                                 if (value != STDC_VALUE_UNKNOWN) {
1245                                         unknown_pragma = false;
1246                                 } else {
1247                                         errorf(pp_token.source_position, "bad STDC pragma argument");
1248                                 }
1249                         }
1250                 }
1251         } else {
1252                 unknown_pragma = true;
1253         }
1254         eat_until_newline();
1255         if (unknown_pragma && warning.unknown_pragmas) {
1256                 warningf(pp_token.source_position, "encountered unknown #pragma");
1257         }
1258 }
1259
1260 /**
1261  * Parse a preprocessor non-null directive.
1262  */
1263 static void parse_preprocessor_identifier(void)
1264 {
1265         assert(pp_token.type == T_IDENTIFIER);
1266         symbol_t *symbol = pp_token.v.symbol;
1267
1268         switch(symbol->pp_ID) {
1269         case TP_include:
1270                 printf("include - enable header name parsing!\n");
1271                 break;
1272         case TP_define:
1273                 define_directive();
1274                 break;
1275         case TP_ifdef:
1276                 ifdef_directive(0);
1277                 break;
1278         case TP_ifndef:
1279                 ifdef_directive(1);
1280                 break;
1281         case TP_endif:
1282                 endif_directive();
1283                 break;
1284         case TP_line:
1285                 next_pp_token();
1286                 parse_line_directive();
1287                 break;
1288         case TP_if:
1289         case TP_else:
1290         case TP_elif:
1291         case TP_undef:
1292         case TP_error:
1293                 /* TODO; output the rest of the line */
1294                 parse_error("#error directive: ");
1295                 break;
1296         case TP_pragma:
1297                 parse_pragma();
1298                 break;
1299         }
1300 }
1301
1302 /**
1303  * Parse a preprocessor directive.
1304  */
1305 static void parse_preprocessor_directive(void)
1306 {
1307         next_pp_token();
1308
1309         switch(pp_token.type) {
1310         case T_IDENTIFIER:
1311                 parse_preprocessor_identifier();
1312                 break;
1313         case T_INTEGER:
1314                 parse_line_directive();
1315                 break;
1316         case '\n':
1317                 /* NULL directive, see § 6.10.7 */
1318                 break;
1319         default:
1320                 parse_error("invalid preprocessor directive");
1321                 eat_until_newline();
1322                 break;
1323         }
1324 }
1325
1326 #define MAYBE_PROLOG                                       \
1327                         next_char();                                   \
1328                         while(1) {                                     \
1329                                 switch(c) {
1330
1331 #define MAYBE(ch, set_type)                                \
1332                                 case ch:                                   \
1333                                         next_char();                           \
1334                                         lexer_token.type = set_type;           \
1335                                         return;
1336
1337 #define ELSE_CODE(code)                                    \
1338                                 default:                                   \
1339                                         code;                                  \
1340                                 }                                          \
1341                         } /* end of while(1) */                        \
1342                         break;
1343
1344 #define ELSE(set_type)                                     \
1345                 ELSE_CODE(                                         \
1346                         lexer_token.type = set_type;                   \
1347                         return;                                        \
1348                 )
1349
1350 void lexer_next_preprocessing_token(void)
1351 {
1352         while(1) {
1353                 switch(c) {
1354                 case ' ':
1355                 case '\t':
1356                         next_char();
1357                         break;
1358
1359                 MATCH_NEWLINE(
1360                         lexer_token.type = '\n';
1361                         return;
1362                 )
1363
1364                 SYMBOL_CHARS
1365                         parse_symbol();
1366                         /* might be a wide string ( L"string" ) */
1367                         if(lexer_token.type == T_IDENTIFIER &&
1368                             lexer_token.v.symbol == symbol_L) {
1369                             if(c == '"') {
1370                                         parse_wide_string_literal();
1371                                 } else if(c == '\'') {
1372                                         parse_wide_character_constant();
1373                                 }
1374                         }
1375                         return;
1376
1377                 DIGITS
1378                         parse_number();
1379                         return;
1380
1381                 case '"':
1382                         parse_string_literal();
1383                         return;
1384
1385                 case '\'':
1386                         parse_character_constant();
1387                         return;
1388
1389                 case '.':
1390                         MAYBE_PROLOG
1391                                 case '0':
1392                                 case '1':
1393                                 case '2':
1394                                 case '3':
1395                                 case '4':
1396                                 case '5':
1397                                 case '6':
1398                                 case '7':
1399                                 case '8':
1400                                 case '9':
1401                                         put_back(c);
1402                                         c = '.';
1403                                         parse_number_dec();
1404                                         return;
1405
1406                                 case '.':
1407                                         MAYBE_PROLOG
1408                                         MAYBE('.', T_DOTDOTDOT)
1409                                         ELSE_CODE(
1410                                                 put_back(c);
1411                                                 c = '.';
1412                                                 lexer_token.type = '.';
1413                                                 return;
1414                                         )
1415                         ELSE('.')
1416                 case '&':
1417                         MAYBE_PROLOG
1418                         MAYBE('&', T_ANDAND)
1419                         MAYBE('=', T_ANDEQUAL)
1420                         ELSE('&')
1421                 case '*':
1422                         MAYBE_PROLOG
1423                         MAYBE('=', T_ASTERISKEQUAL)
1424                         ELSE('*')
1425                 case '+':
1426                         MAYBE_PROLOG
1427                         MAYBE('+', T_PLUSPLUS)
1428                         MAYBE('=', T_PLUSEQUAL)
1429                         ELSE('+')
1430                 case '-':
1431                         MAYBE_PROLOG
1432                         MAYBE('>', T_MINUSGREATER)
1433                         MAYBE('-', T_MINUSMINUS)
1434                         MAYBE('=', T_MINUSEQUAL)
1435                         ELSE('-')
1436                 case '!':
1437                         MAYBE_PROLOG
1438                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1439                         ELSE('!')
1440                 case '/':
1441                         MAYBE_PROLOG
1442                         MAYBE('=', T_SLASHEQUAL)
1443                                 case '*':
1444                                         next_char();
1445                                         skip_multiline_comment();
1446                                         lexer_next_preprocessing_token();
1447                                         return;
1448                                 case '/':
1449                                         next_char();
1450                                         skip_line_comment();
1451                                         lexer_next_preprocessing_token();
1452                                         return;
1453                         ELSE('/')
1454                 case '%':
1455                         MAYBE_PROLOG
1456                         MAYBE('>', T_PERCENTGREATER)
1457                         MAYBE('=', T_PERCENTEQUAL)
1458                                 case ':':
1459                                         MAYBE_PROLOG
1460                                                 case '%':
1461                                                         MAYBE_PROLOG
1462                                                         MAYBE(':', T_PERCENTCOLONPERCENTCOLON)
1463                                                         ELSE_CODE(
1464                                                                 put_back(c);
1465                                                                 c = '%';
1466                                                                 lexer_token.type = T_PERCENTCOLON;
1467                                                                 return;
1468                                                         )
1469                                         ELSE(T_PERCENTCOLON)
1470                         ELSE('%')
1471                 case '<':
1472                         MAYBE_PROLOG
1473                         MAYBE(':', T_LESSCOLON)
1474                         MAYBE('%', T_LESSPERCENT)
1475                         MAYBE('=', T_LESSEQUAL)
1476                                 case '<':
1477                                         MAYBE_PROLOG
1478                                         MAYBE('=', T_LESSLESSEQUAL)
1479                                         ELSE(T_LESSLESS)
1480                         ELSE('<')
1481                 case '>':
1482                         MAYBE_PROLOG
1483                         MAYBE('=', T_GREATEREQUAL)
1484                                 case '>':
1485                                         MAYBE_PROLOG
1486                                         MAYBE('=', T_GREATERGREATEREQUAL)
1487                                         ELSE(T_GREATERGREATER)
1488                         ELSE('>')
1489                 case '^':
1490                         MAYBE_PROLOG
1491                         MAYBE('=', T_CARETEQUAL)
1492                         ELSE('^')
1493                 case '|':
1494                         MAYBE_PROLOG
1495                         MAYBE('=', T_PIPEEQUAL)
1496                         MAYBE('|', T_PIPEPIPE)
1497                         ELSE('|')
1498                 case ':':
1499                         MAYBE_PROLOG
1500                         MAYBE('>', T_COLONGREATER)
1501                         ELSE(':')
1502                 case '=':
1503                         MAYBE_PROLOG
1504                         MAYBE('=', T_EQUALEQUAL)
1505                         ELSE('=')
1506                 case '#':
1507                         MAYBE_PROLOG
1508                         MAYBE('#', T_HASHHASH)
1509                         ELSE('#')
1510
1511                 case '?':
1512                 case '[':
1513                 case ']':
1514                 case '(':
1515                 case ')':
1516                 case '{':
1517                 case '}':
1518                 case '~':
1519                 case ';':
1520                 case ',':
1521                 case '\\':
1522                         lexer_token.type = c;
1523                         next_char();
1524                         return;
1525
1526                 case EOF:
1527                         lexer_token.type = T_EOF;
1528                         return;
1529
1530                 default:
1531                         next_char();
1532                         errorf(lexer_token.source_position, "unknown character '%c' found\n", c);
1533                         lexer_token.type = T_ERROR;
1534                         return;
1535                 }
1536         }
1537 }
1538
1539 void lexer_next_token(void)
1540 {
1541         lexer_next_preprocessing_token();
1542         if(lexer_token.type != '\n')
1543                 return;
1544
1545 newline_found:
1546         do {
1547                 lexer_next_preprocessing_token();
1548         } while(lexer_token.type == '\n');
1549
1550         if(lexer_token.type == '#') {
1551                 parse_preprocessor_directive();
1552                 goto newline_found;
1553         }
1554 }
1555
1556 void init_lexer(void)
1557 {
1558         strset_init(&stringset);
1559 }
1560
1561 void lexer_open_stream(FILE *stream, const char *input_name)
1562 {
1563         input                                  = stream;
1564         lexer_token.source_position.linenr     = 0;
1565         lexer_token.source_position.input_name = input_name;
1566
1567         symbol_L = symbol_table_insert("L");
1568         bufpos = NULL;
1569         bufend = NULL;
1570
1571         /* place a virtual \n at the beginning so the lexer knows that we're
1572          * at the beginning of a line */
1573         c = '\n';
1574 }
1575
1576 void exit_lexer(void)
1577 {
1578         strset_destroy(&stringset);
1579 }
1580
1581 static __attribute__((unused))
1582 void dbg_pos(const source_position_t source_position)
1583 {
1584         fprintf(stdout, "%s:%u\n", source_position.input_name,
1585                 source_position.linenr);
1586         fflush(stdout);
1587 }