nsz Git - cparser/blob - lexer.c

   1 #include <config.h>
   2
   3 #include "diagnostic.h"
   4 #include "lexer.h"
   5 #include "token_t.h"
   6 #include "symbol_table_t.h"
   7 #include "adt/error.h"
   8 #include "adt/strset.h"
   9 #include "adt/util.h"
  10 #include "types.h"
  11 #include "type_t.h"
  12 #include "target_architecture.h"
  13 #include "parser.h"
  14 #include "warning.h"
  15 #include "lang_features.h"
  16
  17 #include <assert.h>
  18 #include <errno.h>
  19 #include <string.h>
  20 #include <stdbool.h>
  21 #include <ctype.h>
  22
  23 //#define DEBUG_CHARS
  24 #define MAX_PUTBACK 3
  25
  26 #ifdef _WIN32
  27 /* No strtold on windows and no replacement yet */
  28 #define strtold(s, e) strtod(s, e)
  29 #endif
  30
  31 static int         c;
  32 token_t            lexer_token;
  33 symbol_t          *symbol_L;
  34 static FILE       *input;
  35 static char        buf[1024 + MAX_PUTBACK];
  36 static const char *bufend;
  37 static const char *bufpos;
  38 static strset_t    stringset;
  39
  40 /**
  41  * Prints a parse error message at the current token.
  42  *
  43  * @param msg   the error message
  44  */
  45 static void parse_error(const char *msg)
  46 {
  47         errorf(lexer_token.source_position,  "%s", msg);
  48 }
  49
  50 static inline void next_real_char(void)
  51 {
  52         assert(bufpos <= bufend);
  53         if (bufpos >= bufend) {
  54                 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
  55                                  input);
  56                 if(s == 0) {
  57                         c = EOF;
  58                         return;
  59                 }
  60                 bufpos = buf + MAX_PUTBACK;
  61                 bufend = buf + MAX_PUTBACK + s;
  62         }
  63         c = *bufpos++;
  64 }
  65
  66 /**
  67  * Put a character back into the buffer.
  68  *
  69  * @param pc  the character to put back
  70  */
  71 static inline void put_back(int pc)
  72 {
  73         assert(bufpos > buf);
  74         *(--bufpos - buf + buf) = (char) pc;
  75
  76 #ifdef DEBUG_CHARS
  77         printf("putback '%c'\n", pc);
  78 #endif
  79 }
  80
  81 static inline void next_char(void);
  82
  83 #define MATCH_NEWLINE(code)                   \
  84         case '\r':                                \
  85                 next_char();                          \
  86                 if(c == '\n') {                       \
  87                         next_char();                      \
  88                 }                                     \
  89                 lexer_token.source_position.linenr++; \
  90                 code                                  \
  91         case '\n':                                \
  92                 next_char();                          \
  93                 lexer_token.source_position.linenr++; \
  94                 code
  95
  96 #define eat(c_type)  do { assert(c == c_type); next_char(); } while(0)
  97
  98 static void maybe_concat_lines(void)
  99 {
 100         eat('\\');
 101
 102         switch(c) {
 103         MATCH_NEWLINE(return;)
 104
 105         default:
 106                 break;
 107         }
 108
 109         put_back(c);
 110         c = '\\';
 111 }
 112
 113 /**
 114  * Set c to the next input character, ie.
 115  * after expanding trigraphs.
 116  */
 117 static inline void next_char(void)
 118 {
 119         next_real_char();
 120
 121         /* filter trigraphs */
 122         if(UNLIKELY(c == '\\')) {
 123                 maybe_concat_lines();
 124                 goto end_of_next_char;
 125         }
 126
 127         if(LIKELY(c != '?'))
 128                 goto end_of_next_char;
 129
 130         next_real_char();
 131         if(LIKELY(c != '?')) {
 132                 put_back(c);
 133                 c = '?';
 134                 goto end_of_next_char;
 135         }
 136
 137         next_real_char();
 138         switch(c) {
 139         case '=': c = '#'; break;
 140         case '(': c = '['; break;
 141         case '/': c = '\\'; maybe_concat_lines(); break;
 142         case ')': c = ']'; break;
 143         case '\'': c = '^'; break;
 144         case '<': c = '{'; break;
 145         case '!': c = '|'; break;
 146         case '>': c = '}'; break;
 147         case '-': c = '~'; break;
 148         default:
 149                 put_back(c);
 150                 put_back('?');
 151                 c = '?';
 152                 break;
 153         }
 154
 155 end_of_next_char:;
 156 #ifdef DEBUG_CHARS
 157         printf("nchar '%c'\n", c);
 158 #endif
 159 }
 160
 161 #define SYMBOL_CHARS  \
 162         case 'a':         \
 163         case 'b':         \
 164         case 'c':         \
 165         case 'd':         \
 166         case 'e':         \
 167         case 'f':         \
 168         case 'g':         \
 169         case 'h':         \
 170         case 'i':         \
 171         case 'j':         \
 172         case 'k':         \
 173         case 'l':         \
 174         case 'm':         \
 175         case 'n':         \
 176         case 'o':         \
 177         case 'p':         \
 178         case 'q':         \
 179         case 'r':         \
 180         case 's':         \
 181         case 't':         \
 182         case 'u':         \
 183         case 'v':         \
 184         case 'w':         \
 185         case 'x':         \
 186         case 'y':         \
 187         case 'z':         \
 188         case 'A':         \
 189         case 'B':         \
 190         case 'C':         \
 191         case 'D':         \
 192         case 'E':         \
 193         case 'F':         \
 194         case 'G':         \
 195         case 'H':         \
 196         case 'I':         \
 197         case 'J':         \
 198         case 'K':         \
 199         case 'L':         \
 200         case 'M':         \
 201         case 'N':         \
 202         case 'O':         \
 203         case 'P':         \
 204         case 'Q':         \
 205         case 'R':         \
 206         case 'S':         \
 207         case 'T':         \
 208         case 'U':         \
 209         case 'V':         \
 210         case 'W':         \
 211         case 'X':         \
 212         case 'Y':         \
 213         case 'Z':         \
 214         case '_':
 215
 216 #define DIGITS        \
 217         case '0':         \
 218         case '1':         \
 219         case '2':         \
 220         case '3':         \
 221         case '4':         \
 222         case '5':         \
 223         case '6':         \
 224         case '7':         \
 225         case '8':         \
 226         case '9':
 227
 228 /**
 229  * Read a symbol from the input and build
 230  * the lexer_token.
 231  */
 232 static void parse_symbol(void)
 233 {
 234         symbol_t *symbol;
 235         char     *string;
 236
 237         obstack_1grow(&symbol_obstack, (char) c);
 238         next_char();
 239
 240         while(1) {
 241                 switch(c) {
 242                 DIGITS
 243                 SYMBOL_CHARS
 244                         obstack_1grow(&symbol_obstack, (char) c);
 245                         next_char();
 246                         break;
 247
 248                 default:
 249                         goto end_symbol;
 250                 }
 251         }
 252
 253 end_symbol:
 254         obstack_1grow(&symbol_obstack, '\0');
 255
 256         string = obstack_finish(&symbol_obstack);
 257         symbol = symbol_table_insert(string);
 258
 259         lexer_token.type     = symbol->ID;
 260         lexer_token.v.symbol = symbol;
 261
 262         if(symbol->string != string) {
 263                 obstack_free(&symbol_obstack, string);
 264         }
 265 }
 266
 267 static void parse_integer_suffix(bool is_oct_hex)
 268 {
 269         bool is_unsigned  = false;
 270         bool min_long     = false;
 271         bool min_longlong = false;
 272
 273         if(c == 'U' || c == 'u') {
 274                 is_unsigned = true;
 275                 next_char();
 276                 if(c == 'L' || c == 'l') {
 277                         min_long = true;
 278                         next_char();
 279                         if(c == 'L' || c == 'l') {
 280                                 min_longlong = true;
 281                                 next_char();
 282                         }
 283                 }
 284         } else if(c == 'l' || c == 'L') {
 285                 min_long = true;
 286                 next_char();
 287                 if(c == 'l' || c == 'L') {
 288                         min_longlong = true;
 289                         next_char();
 290                         if(c == 'u' || c == 'U') {
 291                                 is_unsigned = true;
 292                                 next_char();
 293                         }
 294                 } else if(c == 'u' || c == 'U') {
 295                         is_unsigned = true;
 296                         next_char();
 297                         lexer_token.datatype = type_unsigned_long;
 298                 }
 299         }
 300
 301         if(!is_unsigned) {
 302                 long long v = lexer_token.v.intvalue;
 303                 if(!min_long) {
 304                         if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
 305                                 lexer_token.datatype = type_int;
 306                                 return;
 307                         } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
 308                                 lexer_token.datatype = type_unsigned_int;
 309                                 return;
 310                         }
 311                 }
 312                 if(!min_longlong) {
 313                         if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
 314                                 lexer_token.datatype = type_long;
 315                                 return;
 316                         } else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) {
 317                                 lexer_token.datatype = type_unsigned_long;
 318                                 return;
 319                         }
 320                 }
 321                 unsigned long long uv = (unsigned long long) v;
 322                 if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
 323                         lexer_token.datatype = type_unsigned_long_long;
 324                         return;
 325                 }
 326
 327                 lexer_token.datatype = type_long_long;
 328         } else {
 329                 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
 330                 if(!min_long && v <= TARGET_UINT_MAX) {
 331                         lexer_token.datatype = type_unsigned_int;
 332                         return;
 333                 }
 334                 if(!min_longlong && v <= TARGET_ULONG_MAX) {
 335                         lexer_token.datatype = type_unsigned_long;
 336                         return;
 337                 }
 338                 lexer_token.datatype = type_unsigned_long_long;
 339         }
 340 }
 341
 342 static void parse_floating_suffix(void)
 343 {
 344         switch(c) {
 345         /* TODO: do something useful with the suffixes... */
 346         case 'f':
 347         case 'F':
 348                 next_char();
 349                 lexer_token.datatype = type_float;
 350                 break;
 351         case 'l':
 352         case 'L':
 353                 next_char();
 354                 lexer_token.datatype = type_long_double;
 355                 break;
 356         default:
 357                 lexer_token.datatype = type_double;
 358                 break;
 359         }
 360 }
 361
 362 /**
 363  * A replacement for strtoull. Only those parts needed for
 364  * our parser are implemented.
 365  */
 366 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
 367         unsigned long long v = 0;
 368
 369         switch (base) {
 370         case 16:
 371                 for (;; ++s) {
 372                         /* check for overrun */
 373                         if (v >= 0x1000000000000000ULL)
 374                                 break;
 375                         switch (tolower(*s)) {
 376                         case '0': v <<= 4; break;
 377                         case '1': v <<= 4; v |= 0x1; break;
 378                         case '2': v <<= 4; v |= 0x2; break;
 379                         case '3': v <<= 4; v |= 0x3; break;
 380                         case '4': v <<= 4; v |= 0x4; break;
 381                         case '5': v <<= 4; v |= 0x5; break;
 382                         case '6': v <<= 4; v |= 0x6; break;
 383                         case '7': v <<= 4; v |= 0x7; break;
 384                         case '8': v <<= 4; v |= 0x8; break;
 385                         case '9': v <<= 4; v |= 0x9; break;
 386                         case 'a': v <<= 4; v |= 0xa; break;
 387                         case 'b': v <<= 4; v |= 0xb; break;
 388                         case 'c': v <<= 4; v |= 0xc; break;
 389                         case 'd': v <<= 4; v |= 0xd; break;
 390                         case 'e': v <<= 4; v |= 0xe; break;
 391                         case 'f': v <<= 4; v |= 0xf; break;
 392                         default:
 393                                 goto end;
 394                         }
 395                 }
 396                 break;
 397         case 8:
 398                 for (;; ++s) {
 399                         /* check for overrun */
 400                         if (v >= 0x2000000000000000ULL)
 401                                 break;
 402                         switch (tolower(*s)) {
 403                         case '0': v <<= 3; break;
 404                         case '1': v <<= 3; v |= 1; break;
 405                         case '2': v <<= 3; v |= 2; break;
 406                         case '3': v <<= 3; v |= 3; break;
 407                         case '4': v <<= 3; v |= 4; break;
 408                         case '5': v <<= 3; v |= 5; break;
 409                         case '6': v <<= 3; v |= 6; break;
 410                         case '7': v <<= 3; v |= 7; break;
 411                         default:
 412                                 goto end;
 413                         }
 414                 }
 415                 break;
 416         case 10:
 417                 for (;; ++s) {
 418                         /* check for overrun */
 419                         if (v > 0x1999999999999999ULL)
 420                                 break;
 421                         switch (tolower(*s)) {
 422                         case '0': v *= 10; break;
 423                         case '1': v *= 10; v += 1; break;
 424                         case '2': v *= 10; v += 2; break;
 425                         case '3': v *= 10; v += 3; break;
 426                         case '4': v *= 10; v += 4; break;
 427                         case '5': v *= 10; v += 5; break;
 428                         case '6': v *= 10; v += 6; break;
 429                         case '7': v *= 10; v += 7; break;
 430                         case '8': v *= 10; v += 8; break;
 431                         case '9': v *= 10; v += 9; break;
 432                         default:
 433                                 goto end;
 434                         }
 435                 }
 436                 break;
 437         default:
 438                 assert(0);
 439                 break;
 440         }
 441 end:
 442         *endptr = s;
 443         return v;
 444 }
 445
 446 /**
 447  * Parses a hex number including hex floats and set the
 448  * lexer_token.
 449  */
 450 static void parse_number_hex(void)
 451 {
 452         assert(c == 'x' || c == 'X');
 453         next_char();
 454
 455         while(isxdigit(c)) {
 456                 obstack_1grow(&symbol_obstack, (char) c);
 457                 next_char();
 458         }
 459         obstack_1grow(&symbol_obstack, '\0');
 460         char *string = obstack_finish(&symbol_obstack);
 461
 462         if(c == '.' || c == 'p' || c == 'P') {
 463                 next_char();
 464                 panic("Hex floating point numbers not implemented yet");
 465         }
 466         if(*string == '\0') {
 467                 parse_error("invalid hex number");
 468                 lexer_token.type = T_ERROR;
 469         }
 470
 471         const char *endptr;
 472         lexer_token.type       = T_INTEGER;
 473         lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
 474         if(*endptr != '\0') {
 475                 parse_error("hex number literal too long");
 476         }
 477
 478         obstack_free(&symbol_obstack, string);
 479         parse_integer_suffix(true);
 480 }
 481
 482 /**
 483  * Returns true if the given char is a octal digit.
 484  *
 485  * @param char  the character to check
 486  */
 487 static inline bool is_octal_digit(int chr)
 488 {
 489         switch(chr) {
 490         case '0':
 491         case '1':
 492         case '2':
 493         case '3':
 494         case '4':
 495         case '5':
 496         case '6':
 497         case '7':
 498                 return true;
 499         default:
 500                 return false;
 501         }
 502 }
 503
 504 /**
 505  * Parses a octal number and set the lexer_token.
 506  */
 507 static void parse_number_oct(void)
 508 {
 509         while(is_octal_digit(c)) {
 510                 obstack_1grow(&symbol_obstack, (char) c);
 511                 next_char();
 512         }
 513         obstack_1grow(&symbol_obstack, '\0');
 514         char *string = obstack_finish(&symbol_obstack);
 515
 516         const char *endptr;
 517         lexer_token.type       = T_INTEGER;
 518         lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
 519         if(*endptr != '\0') {
 520                 parse_error("octal number literal too long");
 521         }
 522
 523         obstack_free(&symbol_obstack, string);
 524         parse_integer_suffix(true);
 525 }
 526
 527 /**
 528  * Parses a decimal including float number and set the
 529  * lexer_token.
 530  */
 531 static void parse_number_dec(void)
 532 {
 533         bool is_float = false;
 534         while(isdigit(c)) {
 535                 obstack_1grow(&symbol_obstack, (char) c);
 536                 next_char();
 537         }
 538
 539         if(c == '.') {
 540                 obstack_1grow(&symbol_obstack, '.');
 541                 next_char();
 542
 543                 while(isdigit(c)) {
 544                         obstack_1grow(&symbol_obstack, (char) c);
 545                         next_char();
 546                 }
 547                 is_float = true;
 548         }
 549         if(c == 'e' || c == 'E') {
 550                 obstack_1grow(&symbol_obstack, 'e');
 551                 next_char();
 552
 553                 if(c == '-' || c == '+') {
 554                         obstack_1grow(&symbol_obstack, (char) c);
 555                         next_char();
 556                 }
 557
 558                 while(isdigit(c)) {
 559                         obstack_1grow(&symbol_obstack, (char) c);
 560                         next_char();
 561                 }
 562                 is_float = true;
 563         }
 564
 565         obstack_1grow(&symbol_obstack, '\0');
 566         char *string = obstack_finish(&symbol_obstack);
 567
 568         if(is_float) {
 569                 char *endptr;
 570                 lexer_token.type         = T_FLOATINGPOINT;
 571                 lexer_token.v.floatvalue = strtold(string, &endptr);
 572
 573                 if(*endptr != '\0') {
 574                         parse_error("invalid number literal");
 575                 }
 576
 577                 parse_floating_suffix();
 578         } else {
 579                 const char *endptr;
 580                 lexer_token.type       = T_INTEGER;
 581                 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
 582
 583                 if(*endptr != '\0') {
 584                         parse_error("invalid number literal");
 585                 }
 586
 587                 parse_integer_suffix(false);
 588         }
 589         obstack_free(&symbol_obstack, string);
 590 }
 591
 592 /**
 593  * Parses a number and sets the lexer_token.
 594  */
 595 static void parse_number(void)
 596 {
 597         if (c == '0') {
 598                 next_char();
 599                 switch (c) {
 600                         case 'X':
 601                         case 'x':
 602                                 parse_number_hex();
 603                                 break;
 604                         case '0':
 605                         case '1':
 606                         case '2':
 607                         case '3':
 608                         case '4':
 609                         case '5':
 610                         case '6':
 611                         case '7':
 612                                 parse_number_oct();
 613                                 break;
 614                         case '8':
 615                         case '9':
 616                                 next_char();
 617                                 parse_error("invalid octal number");
 618                                 lexer_token.type = T_ERROR;
 619                                 return;
 620                         case '.':
 621                         case 'e':
 622                         case 'E':
 623                         default:
 624                                 obstack_1grow(&symbol_obstack, '0');
 625                                 parse_number_dec();
 626                                 return;
 627                 }
 628         } else {
 629                 parse_number_dec();
 630         }
 631 }
 632
 633 /**
 634  * Returns the value of a digit.
 635  * The only portable way to do it ...
 636  */
 637 static int digit_value(int digit) {
 638         switch (digit) {
 639         case '0': return 0;
 640         case '1': return 1;
 641         case '2': return 2;
 642         case '3': return 3;
 643         case '4': return 4;
 644         case '5': return 5;
 645         case '6': return 6;
 646         case '7': return 7;
 647         case '8': return 8;
 648         case '9': return 9;
 649         case 'a':
 650         case 'A': return 10;
 651         case 'b':
 652         case 'B': return 11;
 653         case 'c':
 654         case 'C': return 12;
 655         case 'd':
 656         case 'D': return 13;
 657         case 'e':
 658         case 'E': return 14;
 659         case 'f':
 660         case 'F': return 15;
 661         default:
 662                 panic("wrong character given");
 663         }
 664 }
 665
 666 /**
 667  * Parses an octal character sequence.
 668  *
 669  * @param first_digit  the already read first digit
 670  */
 671 static int parse_octal_sequence(const int first_digit)
 672 {
 673         assert(is_octal_digit(first_digit));
 674         int value = digit_value(first_digit);
 675         if (!is_octal_digit(c)) return value;
 676         value = 8 * value + digit_value(c);
 677         next_char();
 678         if (!is_octal_digit(c)) return value;
 679         value = 8 * value + digit_value(c);
 680         next_char();
 681
 682         if(char_is_signed) {
 683                 return (signed char) value;
 684         } else {
 685                 return (unsigned char) value;
 686         }
 687 }
 688
 689 /**
 690  * Parses a hex character sequence.
 691  */
 692 static int parse_hex_sequence(void)
 693 {
 694         int value = 0;
 695         while(isxdigit(c)) {
 696                 value = 16 * value + digit_value(c);
 697                 next_char();
 698         }
 699
 700         if(char_is_signed) {
 701                 return (signed char) value;
 702         } else {
 703                 return (unsigned char) value;
 704         }
 705 }
 706
 707 /**
 708  * Parse an escape sequence.
 709  */
 710 static int parse_escape_sequence(void)
 711 {
 712         eat('\\');
 713
 714         int ec = c;
 715         next_char();
 716
 717         switch(ec) {
 718         case '"':  return '"';
 719         case '\'': return '\'';
 720         case '\\': return '\\';
 721         case '?': return '\?';
 722         case 'a': return '\a';
 723         case 'b': return '\b';
 724         case 'f': return '\f';
 725         case 'n': return '\n';
 726         case 'r': return '\r';
 727         case 't': return '\t';
 728         case 'v': return '\v';
 729         case 'x':
 730                 return parse_hex_sequence();
 731         case '0':
 732         case '1':
 733         case '2':
 734         case '3':
 735         case '4':
 736         case '5':
 737         case '6':
 738         case '7':
 739                 return parse_octal_sequence(ec);
 740         case EOF:
 741                 parse_error("reached end of file while parsing escape sequence");
 742                 return EOF;
 743         default:
 744                 parse_error("unknown escape sequence");
 745                 return EOF;
 746         }
 747 }
 748
 749 /**
 750  * Concatenate two strings.
 751  */
 752 string_t concat_strings(const string_t *const s1, const string_t *const s2)
 753 {
 754         const size_t len1 = s1->size - 1;
 755         const size_t len2 = s2->size - 1;
 756
 757         char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
 758         memcpy(concat, s1->begin, len1);
 759         memcpy(concat + len1, s2->begin, len2 + 1);
 760
 761 #if 0 /* TODO hash */
 762         const char *result = strset_insert(&stringset, concat);
 763         if(result != concat) {
 764                 obstack_free(&symbol_obstack, concat);
 765         }
 766
 767         return result;
 768 #else
 769         return (string_t){ concat, len1 + len2 + 1 };
 770 #endif
 771 }
 772
 773 /**
 774  * Concatenate a string and a wide string.
 775  */
 776 wide_string_t concat_string_wide_string(const string_t *const s1, const wide_string_t *const s2)
 777 {
 778         const size_t len1 = s1->size - 1;
 779         const size_t len2 = s2->size - 1;
 780
 781         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 782         const char *const src = s1->begin;
 783         for (size_t i = 0; i != len1; ++i) {
 784                 concat[i] = src[i];
 785         }
 786         memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
 787
 788         return (wide_string_t){ concat, len1 + len2 + 1 };
 789 }
 790
 791 /**
 792  * Concatenate two wide strings.
 793  */
 794 wide_string_t concat_wide_strings(const wide_string_t *const s1, const wide_string_t *const s2)
 795 {
 796         const size_t len1 = s1->size - 1;
 797         const size_t len2 = s2->size - 1;
 798
 799         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 800         memcpy(concat,        s1->begin, len1       * sizeof(*concat));
 801         memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
 802
 803         return (wide_string_t){ concat, len1 + len2 + 1 };
 804 }
 805
 806 /**
 807  * Concatenate a wide string and a string.
 808  */
 809 wide_string_t concat_wide_string_string(const wide_string_t *const s1, const string_t *const s2)
 810 {
 811         const size_t len1 = s1->size - 1;
 812         const size_t len2 = s2->size - 1;
 813
 814         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 815         memcpy(concat, s1->begin, len1 * sizeof(*concat));
 816         const char *const src = s2->begin;
 817         for (size_t i = 0; i != len2 + 1; ++i) {
 818                 concat[i] = src[i];
 819         }
 820
 821         return (wide_string_t){ concat, len1 + len2 + 1 };
 822 }
 823
 824 /**
 825  * Parse a string literal and set lexer_token.
 826  */
 827 static void parse_string_literal(void)
 828 {
 829         const unsigned start_linenr = lexer_token.source_position.linenr;
 830
 831         eat('"');
 832
 833         int tc;
 834         while(1) {
 835                 switch(c) {
 836                 case '\\':
 837                         tc = parse_escape_sequence();
 838                         obstack_1grow(&symbol_obstack, (char) tc);
 839                         break;
 840
 841                 case EOF: {
 842                         source_position_t source_position;
 843                         source_position.input_name = lexer_token.source_position.input_name;
 844                         source_position.linenr     = start_linenr;
 845                         errorf(source_position, "string has no end");
 846                         lexer_token.type = T_ERROR;
 847                         return;
 848                 }
 849
 850                 case '"':
 851                         next_char();
 852                         goto end_of_string;
 853
 854                 default:
 855                         obstack_1grow(&symbol_obstack, (char) c);
 856                         next_char();
 857                         break;
 858                 }
 859         }
 860
 861 end_of_string:
 862
 863         /* TODO: concatenate multiple strings separated by whitespace... */
 864
 865         /* add finishing 0 to the string */
 866         obstack_1grow(&symbol_obstack, '\0');
 867         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
 868         const char *const string = obstack_finish(&symbol_obstack);
 869
 870 #if 0 /* TODO hash */
 871         /* check if there is already a copy of the string */
 872         result = strset_insert(&stringset, string);
 873         if(result != string) {
 874                 obstack_free(&symbol_obstack, string);
 875         }
 876 #else
 877         const char *const result = string;
 878 #endif
 879
 880         lexer_token.type           = T_STRING_LITERAL;
 881         lexer_token.v.string.begin = result;
 882         lexer_token.v.string.size  = size;
 883 }
 884
 885 /**
 886  * Parse a wide character constant and set lexer_token.
 887  */
 888 static void parse_wide_character_constant(void)
 889 {
 890         eat('\'');
 891
 892         int found_char = 0;
 893         while(1) {
 894                 switch(c) {
 895                 case '\\':
 896                         found_char = parse_escape_sequence();
 897                         break;
 898
 899                 MATCH_NEWLINE(
 900                         parse_error("newline while parsing character constant");
 901                         break;
 902                 )
 903
 904                 case '\'':
 905                         next_char();
 906                         goto end_of_wide_char_constant;
 907
 908                 case EOF:
 909                         parse_error("EOF while parsing character constant");
 910                         lexer_token.type = T_ERROR;
 911                         return;
 912
 913                 default:
 914                         if(found_char != 0) {
 915                                 parse_error("more than 1 characters in character "
 916                                             "constant");
 917                                 goto end_of_wide_char_constant;
 918                         } else {
 919                                 found_char = c;
 920                                 next_char();
 921                         }
 922                         break;
 923                 }
 924         }
 925
 926 end_of_wide_char_constant:
 927         lexer_token.type       = T_INTEGER;
 928         lexer_token.v.intvalue = found_char;
 929         lexer_token.datatype   = type_wchar_t;
 930 }
 931
 932 /**
 933  * Parse a wide string literal and set lexer_token.
 934  */
 935 static void parse_wide_string_literal(void)
 936 {
 937         const unsigned start_linenr = lexer_token.source_position.linenr;
 938
 939         assert(c == '"');
 940         next_char();
 941
 942         while(1) {
 943                 switch(c) {
 944                 case '\\': {
 945                         wchar_rep_t tc = parse_escape_sequence();
 946                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 947                         break;
 948                 }
 949
 950                 case EOF: {
 951                         source_position_t source_position;
 952                         source_position.input_name = lexer_token.source_position.input_name;
 953                         source_position.linenr     = start_linenr;
 954                         errorf(source_position, "string has no end");
 955                         lexer_token.type = T_ERROR;
 956                         return;
 957                 }
 958
 959                 case '"':
 960                         next_char();
 961                         goto end_of_string;
 962
 963                 default: {
 964                         wchar_rep_t tc = c;
 965                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 966                         next_char();
 967                         break;
 968                 }
 969                 }
 970         }
 971
 972 end_of_string:;
 973
 974         /* TODO: concatenate multiple strings separated by whitespace... */
 975
 976         /* add finishing 0 to the string */
 977         wchar_rep_t nul = L'\0';
 978         obstack_grow(&symbol_obstack, &nul, sizeof(nul));
 979         const size_t             size   = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
 980         const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
 981
 982 #if 0 /* TODO hash */
 983         /* check if there is already a copy of the string */
 984         const wchar_rep_t *const result = strset_insert(&stringset, string);
 985         if(result != string) {
 986                 obstack_free(&symbol_obstack, string);
 987         }
 988 #else
 989         const wchar_rep_t *const result = string;
 990 #endif
 991
 992         lexer_token.type                = T_WIDE_STRING_LITERAL;
 993         lexer_token.v.wide_string.begin = result;
 994         lexer_token.v.wide_string.size  = size;
 995 }
 996
 997 /**
 998  * Parse a character constant and set lexer_token.
 999  */
1000 static void parse_character_constant(void)
1001 {
1002         const unsigned start_linenr = lexer_token.source_position.linenr;
1003
1004         eat('\'');
1005
1006         int tc;
1007         while(1) {
1008                 switch(c) {
1009                 case '\\':
1010                         tc = parse_escape_sequence();
1011                         obstack_1grow(&symbol_obstack, (char) tc);
1012                         break;
1013
1014                 MATCH_NEWLINE(
1015                         parse_error("newline while parsing character constant");
1016                         break;
1017                 )
1018
1019                 case EOF: {
1020                         source_position_t source_position;
1021                         source_position.input_name = lexer_token.source_position.input_name;
1022                         source_position.linenr     = start_linenr;
1023                         errorf(source_position, "EOF while parsing character constant");
1024                         lexer_token.type = T_ERROR;
1025                         return;
1026                 }
1027
1028                 case '\'':
1029                         next_char();
1030                         goto end_of_char_constant;
1031
1032                 default:
1033                         obstack_1grow(&symbol_obstack, (char) c);
1034                         next_char();
1035                         break;
1036
1037                 }
1038         }
1039
1040 end_of_char_constant:;
1041         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
1042         const char *const string = obstack_finish(&symbol_obstack);
1043
1044         lexer_token.type           = T_CHARS;
1045         lexer_token.v.string.begin = string;
1046         lexer_token.v.string.size  = size;
1047         lexer_token.datatype       = type_int;
1048 }
1049
1050 /**
1051  * Skip a multiline comment.
1052  */
1053 static void skip_multiline_comment(void)
1054 {
1055         unsigned start_linenr = lexer_token.source_position.linenr;
1056
1057         while(1) {
1058                 switch(c) {
1059                 case '/':
1060                         next_char();
1061                         if (c == '*') {
1062                                 /* TODO: nested comment, warn here */
1063                         }
1064                         break;
1065                 case '*':
1066                         next_char();
1067                         if(c == '/') {
1068                                 next_char();
1069                                 return;
1070                         }
1071                         break;
1072
1073                 MATCH_NEWLINE(break;)
1074
1075                 case EOF: {
1076                         source_position_t source_position;
1077                         source_position.input_name = lexer_token.source_position.input_name;
1078                         source_position.linenr     = start_linenr;
1079                         errorf(source_position, "at end of file while looking for comment end");
1080                         return;
1081                 }
1082
1083                 default:
1084                         next_char();
1085                         break;
1086                 }
1087         }
1088 }
1089
1090 /**
1091  * Skip a single line comment.
1092  */
1093 static void skip_line_comment(void)
1094 {
1095         while(1) {
1096                 switch(c) {
1097                 case EOF:
1098                         return;
1099
1100                 case '\n':
1101                 case '\r':
1102                         return;
1103
1104                 default:
1105                         next_char();
1106                         break;
1107                 }
1108         }
1109 }
1110
1111 /** The current preprocessor token. */
1112 static token_t pp_token;
1113
1114 /**
1115  * Read the next preprocessor token.
1116  */
1117 static inline void next_pp_token(void)
1118 {
1119         lexer_next_preprocessing_token();
1120         pp_token = lexer_token;
1121 }
1122
1123 /**
1124  * Eat all preprocessor tokens until newline.
1125  */
1126 static void eat_until_newline(void)
1127 {
1128         while(pp_token.type != '\n' && pp_token.type != T_EOF) {
1129                 next_pp_token();
1130         }
1131 }
1132
1133 /**
1134  * Handle the define directive.
1135  */
1136 static void define_directive(void)
1137 {
1138         lexer_next_preprocessing_token();
1139         if(lexer_token.type != T_IDENTIFIER) {
1140                 parse_error("expected identifier after #define\n");
1141                 eat_until_newline();
1142         }
1143 }
1144
1145 /**
1146  * Handle the ifdef directive.
1147  */
1148 static void ifdef_directive(int is_ifndef)
1149 {
1150         (void) is_ifndef;
1151         lexer_next_preprocessing_token();
1152         //expect_identifier();
1153         //extect_newline();
1154 }
1155
1156 /**
1157  * Handle the endif directive.
1158  */
1159 static void endif_directive(void)
1160 {
1161         //expect_newline();
1162 }
1163
1164 /**
1165  * Parse the line directive.
1166  */
1167 static void parse_line_directive(void)
1168 {
1169         if(pp_token.type != T_INTEGER) {
1170                 parse_error("expected integer");
1171         } else {
1172                 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
1173                 next_pp_token();
1174         }
1175         if(pp_token.type == T_STRING_LITERAL) {
1176                 lexer_token.source_position.input_name = pp_token.v.string.begin;
1177                 next_pp_token();
1178         }
1179
1180         eat_until_newline();
1181 }
1182
1183 /**
1184  * STDC pragmas.
1185  */
1186 typedef enum {
1187         STDC_UNKNOWN,
1188         STDC_FP_CONTRACT,
1189         STDC_FENV_ACCESS,
1190         STDC_CX_LIMITED_RANGE
1191 } stdc_pragma_kind_t;
1192
1193 /**
1194  * STDC pragma values.
1195  */
1196 typedef enum {
1197         STDC_VALUE_UNKNOWN,
1198         STDC_VALUE_ON,
1199         STDC_VALUE_OFF,
1200         STDC_VALUE_DEFAULT
1201 } stdc_pragma_value_kind_t;
1202
1203 /**
1204  * Parse a pragma directive.
1205  */
1206 static void parse_pragma(void) {
1207         bool unknown_pragma = true;
1208
1209         next_pp_token();
1210         if (pp_token.v.symbol->pp_ID == TP_STDC) {
1211                 stdc_pragma_kind_t kind = STDC_UNKNOWN;
1212                 /* a STDC pragma */
1213                 if (c_mode & _C99) {
1214                         next_pp_token();
1215
1216                         switch (pp_token.v.symbol->pp_ID) {
1217                         case TP_FP_CONTRACT:
1218                                 kind = STDC_FP_CONTRACT;
1219                                 break;
1220                         case TP_FENV_ACCESS:
1221                                 kind = STDC_FENV_ACCESS;
1222                                 break;
1223                         case TP_CX_LIMITED_RANGE:
1224                                 kind = STDC_CX_LIMITED_RANGE;
1225                                 break;
1226                         default:
1227                                 break;
1228                         }
1229                         if (kind != STDC_UNKNOWN) {
1230                                 stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN;
1231                                 next_pp_token();
1232                                 switch (pp_token.v.symbol->pp_ID) {
1233                                 case TP_ON:
1234                                         value = STDC_VALUE_ON;
1235                                         break;
1236                                 case TP_OFF:
1237                                         value = STDC_VALUE_OFF;
1238                                         break;
1239                                 case TP_DEFAULT:
1240                                         value = STDC_VALUE_DEFAULT;
1241                                         break;
1242                                 default:
1243                                         break;
1244                                 }
1245                                 if (value != STDC_VALUE_UNKNOWN) {
1246                                         unknown_pragma = false;
1247                                 } else {
1248                                         errorf(pp_token.source_position, "bad STDC pragma argument");
1249                                 }
1250                         }
1251                 }
1252         } else {
1253                 unknown_pragma = true;
1254         }
1255         eat_until_newline();
1256         if (unknown_pragma && warning.unknown_pragmas) {
1257                 warningf(pp_token.source_position, "encountered unknown #pragma");
1258         }
1259 }
1260
1261 /**
1262  * Parse a preprocessor non-null directive.
1263  */
1264 static void parse_preprocessor_identifier(void)
1265 {
1266         assert(pp_token.type == T_IDENTIFIER);
1267         symbol_t *symbol = pp_token.v.symbol;
1268
1269         switch(symbol->pp_ID) {
1270         case TP_include:
1271                 printf("include - enable header name parsing!\n");
1272                 break;
1273         case TP_define:
1274                 define_directive();
1275                 break;
1276         case TP_ifdef:
1277                 ifdef_directive(0);
1278                 break;
1279         case TP_ifndef:
1280                 ifdef_directive(1);
1281                 break;
1282         case TP_endif:
1283                 endif_directive();
1284                 break;
1285         case TP_line:
1286                 next_pp_token();
1287                 parse_line_directive();
1288                 break;
1289         case TP_if:
1290         case TP_else:
1291         case TP_elif:
1292         case TP_undef:
1293         case TP_error:
1294                 /* TODO; output the rest of the line */
1295                 parse_error("#error directive: ");
1296                 break;
1297         case TP_pragma:
1298                 parse_pragma();
1299                 break;
1300         }
1301 }
1302
1303 /**
1304  * Parse a preprocessor directive.
1305  */
1306 static void parse_preprocessor_directive(void)
1307 {
1308         next_pp_token();
1309
1310         switch(pp_token.type) {
1311         case T_IDENTIFIER:
1312                 parse_preprocessor_identifier();
1313                 break;
1314         case T_INTEGER:
1315                 parse_line_directive();
1316                 break;
1317         case '\n':
1318                 /* NULL directive, see § 6.10.7 */
1319                 break;
1320         default:
1321                 parse_error("invalid preprocessor directive");
1322                 eat_until_newline();
1323                 break;
1324         }
1325 }
1326
1327 #define MAYBE_PROLOG                                       \
1328                         next_char();                                   \
1329                         while(1) {                                     \
1330                                 switch(c) {
1331
1332 #define MAYBE(ch, set_type)                                \
1333                                 case ch:                                   \
1334                                         next_char();                           \
1335                                         lexer_token.type = set_type;           \
1336                                         return;
1337
1338 #define ELSE_CODE(code)                                    \
1339                                 default:                                   \
1340                                         code;                                  \
1341                                 }                                          \
1342                         } /* end of while(1) */                        \
1343                         break;
1344
1345 #define ELSE(set_type)                                     \
1346                 ELSE_CODE(                                         \
1347                         lexer_token.type = set_type;                   \
1348                         return;                                        \
1349                 )
1350
1351 void lexer_next_preprocessing_token(void)
1352 {
1353         while(1) {
1354                 switch(c) {
1355                 case ' ':
1356                 case '\t':
1357                         next_char();
1358                         break;
1359
1360                 MATCH_NEWLINE(
1361                         lexer_token.type = '\n';
1362                         return;
1363                 )
1364
1365                 SYMBOL_CHARS
1366                         parse_symbol();
1367                         /* might be a wide string ( L"string" ) */
1368                         if(lexer_token.type == T_IDENTIFIER &&
1369                             lexer_token.v.symbol == symbol_L) {
1370                             if(c == '"') {
1371                                         parse_wide_string_literal();
1372                                 } else if(c == '\'') {
1373                                         parse_wide_character_constant();
1374                                 }
1375                         }
1376                         return;
1377
1378                 DIGITS
1379                         parse_number();
1380                         return;
1381
1382                 case '"':
1383                         parse_string_literal();
1384                         return;
1385
1386                 case '\'':
1387                         parse_character_constant();
1388                         return;
1389
1390                 case '.':
1391                         MAYBE_PROLOG
1392                                 case '0':
1393                                 case '1':
1394                                 case '2':
1395                                 case '3':
1396                                 case '4':
1397                                 case '5':
1398                                 case '6':
1399                                 case '7':
1400                                 case '8':
1401                                 case '9':
1402                                         put_back(c);
1403                                         c = '.';
1404                                         parse_number_dec();
1405                                         return;
1406
1407                                 case '.':
1408                                         MAYBE_PROLOG
1409                                         MAYBE('.', T_DOTDOTDOT)
1410                                         ELSE_CODE(
1411                                                 put_back(c);
1412                                                 c = '.';
1413                                                 lexer_token.type = '.';
1414                                                 return;
1415                                         )
1416                         ELSE('.')
1417                 case '&':
1418                         MAYBE_PROLOG
1419                         MAYBE('&', T_ANDAND)
1420                         MAYBE('=', T_ANDEQUAL)
1421                         ELSE('&')
1422                 case '*':
1423                         MAYBE_PROLOG
1424                         MAYBE('=', T_ASTERISKEQUAL)
1425                         ELSE('*')
1426                 case '+':
1427                         MAYBE_PROLOG
1428                         MAYBE('+', T_PLUSPLUS)
1429                         MAYBE('=', T_PLUSEQUAL)
1430                         ELSE('+')
1431                 case '-':
1432                         MAYBE_PROLOG
1433                         MAYBE('>', T_MINUSGREATER)
1434                         MAYBE('-', T_MINUSMINUS)
1435                         MAYBE('=', T_MINUSEQUAL)
1436                         ELSE('-')
1437                 case '!':
1438                         MAYBE_PROLOG
1439                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1440                         ELSE('!')
1441                 case '/':
1442                         MAYBE_PROLOG
1443                         MAYBE('=', T_SLASHEQUAL)
1444                                 case '*':
1445                                         next_char();
1446                                         skip_multiline_comment();
1447                                         lexer_next_preprocessing_token();
1448                                         return;
1449                                 case '/':
1450                                         next_char();
1451                                         skip_line_comment();
1452                                         lexer_next_preprocessing_token();
1453                                         return;
1454                         ELSE('/')
1455                 case '%':
1456                         MAYBE_PROLOG
1457                         MAYBE('>', T_PERCENTGREATER)
1458                         MAYBE('=', T_PERCENTEQUAL)
1459                                 case ':':
1460                                         MAYBE_PROLOG
1461                                                 case '%':
1462                                                         MAYBE_PROLOG
1463                                                         MAYBE(':', T_PERCENTCOLONPERCENTCOLON)
1464                                                         ELSE_CODE(
1465                                                                 put_back(c);
1466                                                                 c = '%';
1467                                                                 lexer_token.type = T_PERCENTCOLON;
1468                                                                 return;
1469                                                         )
1470                                         ELSE(T_PERCENTCOLON)
1471                         ELSE('%')
1472                 case '<':
1473                         MAYBE_PROLOG
1474                         MAYBE(':', T_LESSCOLON)
1475                         MAYBE('%', T_LESSPERCENT)
1476                         MAYBE('=', T_LESSEQUAL)
1477                                 case '<':
1478                                         MAYBE_PROLOG
1479                                         MAYBE('=', T_LESSLESSEQUAL)
1480                                         ELSE(T_LESSLESS)
1481                         ELSE('<')
1482                 case '>':
1483                         MAYBE_PROLOG
1484                         MAYBE('=', T_GREATEREQUAL)
1485                                 case '>':
1486                                         MAYBE_PROLOG
1487                                         MAYBE('=', T_GREATERGREATEREQUAL)
1488                                         ELSE(T_GREATERGREATER)
1489                         ELSE('>')
1490                 case '^':
1491                         MAYBE_PROLOG
1492                         MAYBE('=', T_CARETEQUAL)
1493                         ELSE('^')
1494                 case '|':
1495                         MAYBE_PROLOG
1496                         MAYBE('=', T_PIPEEQUAL)
1497                         MAYBE('|', T_PIPEPIPE)
1498                         ELSE('|')
1499                 case ':':
1500                         MAYBE_PROLOG
1501                         MAYBE('>', T_COLONGREATER)
1502                         ELSE(':')
1503                 case '=':
1504                         MAYBE_PROLOG
1505                         MAYBE('=', T_EQUALEQUAL)
1506                         ELSE('=')
1507                 case '#':
1508                         MAYBE_PROLOG
1509                         MAYBE('#', T_HASHHASH)
1510                         ELSE('#')
1511
1512                 case '?':
1513                 case '[':
1514                 case ']':
1515                 case '(':
1516                 case ')':
1517                 case '{':
1518                 case '}':
1519                 case '~':
1520                 case ';':
1521                 case ',':
1522                 case '\\':
1523                         lexer_token.type = c;
1524                         next_char();
1525                         return;
1526
1527                 case EOF:
1528                         lexer_token.type = T_EOF;
1529                         return;
1530
1531                 default:
1532                         next_char();
1533                         errorf(lexer_token.source_position, "unknown character '%c' found\n", c);
1534                         lexer_token.type = T_ERROR;
1535                         return;
1536                 }
1537         }
1538 }
1539
1540 void lexer_next_token(void)
1541 {
1542         lexer_next_preprocessing_token();
1543         if(lexer_token.type != '\n')
1544                 return;
1545
1546 newline_found:
1547         do {
1548                 lexer_next_preprocessing_token();
1549         } while(lexer_token.type == '\n');
1550
1551         if(lexer_token.type == '#') {
1552                 parse_preprocessor_directive();
1553                 goto newline_found;
1554         }
1555 }
1556
1557 void init_lexer(void)
1558 {
1559         strset_init(&stringset);
1560 }
1561
1562 void lexer_open_stream(FILE *stream, const char *input_name)
1563 {
1564         input                                  = stream;
1565         lexer_token.source_position.linenr     = 0;
1566         lexer_token.source_position.input_name = input_name;
1567
1568         symbol_L = symbol_table_insert("L");
1569         bufpos = NULL;
1570         bufend = NULL;
1571
1572         /* place a virtual \n at the beginning so the lexer knows that we're
1573          * at the beginning of a line */
1574         c = '\n';
1575 }
1576
1577 void exit_lexer(void)
1578 {
1579         strset_destroy(&stringset);
1580 }
1581
1582 static __attribute__((unused))
1583 void dbg_pos(const source_position_t source_position)
1584 {
1585         fprintf(stdout, "%s:%u\n", source_position.input_name,
1586                 source_position.linenr);
1587         fflush(stdout);
1588 }