nsz Git - cparser/blob - preprocessor.c

   1 #include <config.h>
   2
   3 #include "token_t.h"
   4 #include "symbol_t.h"
   5 #include "adt/util.h"
   6 #include "adt/error.h"
   7 #include "lang_features.h"
   8 #include "diagnostic.h"
   9 #include "string_rep.h"
  10
  11 #include <assert.h>
  12 #include <errno.h>
  13 #include <string.h>
  14 #include <stdbool.h>
  15 #include <ctype.h>
  16
  17 //#define DEBUG_CHARS
  18 #define MAX_PUTBACK 3
  19 #define INCLUDE_LIMIT 199  /* 199 is for gcc "compatibility" */
  20
  21 struct pp_definition_t {
  22         symbol_t          *symbol;
  23         source_position_t  source_position;
  24         pp_definition_t   *parent_expansion;
  25         size_t             expand_pos;
  26         bool               is_variadic  : 1;
  27         bool               is_expanding : 1;
  28         size_t             argument_count;
  29         token_t           *arguments;
  30         size_t             list_len;
  31         token_t           *replacement_list;
  32 };
  33
  34 typedef struct pp_input_t pp_input_t;
  35 struct pp_input_t {
  36         FILE              *file;
  37         int                c;
  38         char               buf[1024+MAX_PUTBACK];
  39         const char        *bufend;
  40         const char        *bufpos;
  41         source_position_t  position;
  42         bool               had_non_space;
  43         pp_input_t        *parent;
  44 };
  45
  46 pp_input_t input;
  47 #define CC input.c
  48
  49 static pp_input_t     *input_stack;
  50 static unsigned        n_inputs;
  51 static struct obstack  input_obstack;
  52
  53 token_t                   pp_token;
  54 static bool               resolve_escape_sequences = false;
  55 static bool               do_print_spaces          = true;
  56 static FILE              *out;
  57 static struct obstack     pp_obstack;
  58 static unsigned           counted_newlines;
  59 static unsigned           counted_spaces;
  60 static const char        *printed_input_name = NULL;
  61 static pp_definition_t   *current_expansion  = NULL;
  62 static bool               do_expansions;
  63
  64 static inline void next_char(void);
  65 static void next_preprocessing_token(void);
  66 static void print_line_directive(const source_position_t *pos, const char *add);
  67 static void print_spaces(void);
  68
  69 static bool open_input(const char *filename)
  70 {
  71         FILE *file = fopen(filename, "r");
  72         if (file == NULL)
  73                 return false;
  74
  75         input.file                = file;
  76         input.bufend              = NULL;
  77         input.bufpos              = NULL;
  78         input.had_non_space       = false;
  79         input.position.input_name = filename;
  80         input.position.linenr     = 1;
  81
  82         /* indicate that we're at a new input */
  83         print_line_directive(&input.position, input_stack != NULL ? "1" : NULL);
  84
  85         counted_newlines = 0;
  86         counted_spaces   = 0;
  87
  88         /* read first char and first token */
  89         next_char();
  90         next_preprocessing_token();
  91
  92         return true;
  93 }
  94
  95 static void close_input(void)
  96 {
  97         /* ensure we have a newline at EOF */
  98         if (input.had_non_space) {
  99                 fputc('\n', out);
 100         }
 101
 102         assert(input.file != NULL);
 103
 104         fclose(input.file);
 105         input.file   = NULL;
 106         input.bufend = NULL;
 107         input.bufpos = NULL;
 108         input.c      = EOF;
 109 }
 110
 111 static void push_input(void)
 112 {
 113         pp_input_t *saved_input
 114                 = obstack_alloc(&input_obstack, sizeof(*saved_input));
 115
 116         memcpy(saved_input, &input, sizeof(*saved_input));
 117
 118         /* adjust buffer positions */
 119         if (input.bufpos != NULL)
 120                 saved_input->bufpos = saved_input->buf + (input.bufpos - input.buf);
 121         if (input.bufend != NULL)
 122                 saved_input->bufend = saved_input->buf + (input.bufend - input.buf);
 123
 124         saved_input->parent = input_stack;
 125         input_stack         = saved_input;
 126         ++n_inputs;
 127 }
 128
 129 static void pop_restore_input(void)
 130 {
 131         assert(n_inputs > 0);
 132         assert(input_stack != NULL);
 133
 134         pp_input_t *saved_input = input_stack;
 135
 136         memcpy(&input, saved_input, sizeof(input));
 137         input.parent = NULL;
 138
 139         /* adjust buffer positions */
 140         if (saved_input->bufpos != NULL)
 141                 input.bufpos = input.buf + (saved_input->bufpos - saved_input->buf);
 142         if (saved_input->bufend != NULL)
 143                 input.bufend = input.buf + (saved_input->bufend - saved_input->buf);
 144
 145         input_stack = saved_input->parent;
 146         obstack_free(&input_obstack, saved_input);
 147         --n_inputs;
 148 }
 149
 150 /**
 151  * Prints a parse error message at the current token.
 152  *
 153  * @param msg   the error message
 154  */
 155 static void parse_error(const char *msg)
 156 {
 157         errorf(&pp_token.source_position,  "%s", msg);
 158 }
 159
 160 static inline void next_real_char(void)
 161 {
 162         assert(input.bufpos <= input.bufend);
 163         if (input.bufpos >= input.bufend) {
 164                 size_t s = fread(input.buf + MAX_PUTBACK, 1,
 165                                  sizeof(input.buf) - MAX_PUTBACK, input.file);
 166                 if(s == 0) {
 167                         CC = EOF;
 168                         return;
 169                 }
 170                 input.bufpos = input.buf + MAX_PUTBACK;
 171                 input.bufend = input.buf + MAX_PUTBACK + s;
 172         }
 173         CC = *input.bufpos++;
 174 }
 175
 176 /**
 177  * Put a character back into the buffer.
 178  *
 179  * @param pc  the character to put back
 180  */
 181 static inline void put_back(int pc)
 182 {
 183         assert(input.bufpos > input.buf);
 184         *(--input.bufpos - input.buf + input.buf) = (char) pc;
 185
 186 #ifdef DEBUG_CHARS
 187         printf("putback '%c'\n", pc);
 188 #endif
 189 }
 190
 191 #define MATCH_NEWLINE(code)                   \
 192         case '\r':                                \
 193                 next_char();                          \
 194                 if(CC == '\n') {                      \
 195                         next_char();                      \
 196                 }                                     \
 197                 ++input.position.linenr;              \
 198                 code                                  \
 199         case '\n':                                \
 200                 next_char();                          \
 201                 ++input.position.linenr;              \
 202                 code
 203
 204 #define eat(c_type)  do { assert(CC == c_type); next_char(); } while(0)
 205
 206 static void maybe_concat_lines(void)
 207 {
 208         eat('\\');
 209
 210         switch(CC) {
 211         MATCH_NEWLINE(return;)
 212
 213         default:
 214                 break;
 215         }
 216
 217         put_back(CC);
 218         CC = '\\';
 219 }
 220
 221 /**
 222  * Set c to the next input character, ie.
 223  * after expanding trigraphs.
 224  */
 225 static inline void next_char(void)
 226 {
 227         next_real_char();
 228
 229         /* filter trigraphs and concatenated lines */
 230         if(UNLIKELY(CC == '\\')) {
 231                 maybe_concat_lines();
 232                 goto end_of_next_char;
 233         }
 234
 235         if(LIKELY(CC != '?'))
 236                 goto end_of_next_char;
 237
 238         next_real_char();
 239         if(LIKELY(CC != '?')) {
 240                 put_back(CC);
 241                 CC = '?';
 242                 goto end_of_next_char;
 243         }
 244
 245         next_real_char();
 246         switch(CC) {
 247         case '=': CC = '#'; break;
 248         case '(': CC = '['; break;
 249         case '/': CC = '\\'; maybe_concat_lines(); break;
 250         case ')': CC = ']'; break;
 251         case '\'': CC = '^'; break;
 252         case '<': CC = '{'; break;
 253         case '!': CC = '|'; break;
 254         case '>': CC = '}'; break;
 255         case '-': CC = '~'; break;
 256         default:
 257                 put_back(CC);
 258                 put_back('?');
 259                 CC = '?';
 260                 break;
 261         }
 262
 263 end_of_next_char:;
 264 #ifdef DEBUG_CHARS
 265         printf("nchar '%c'\n", CC);
 266 #endif
 267 }
 268
 269
 270
 271 /**
 272  * Returns true if the given char is a octal digit.
 273  *
 274  * @param char  the character to check
 275  */
 276 static inline bool is_octal_digit(int chr)
 277 {
 278         switch(chr) {
 279         case '0':
 280         case '1':
 281         case '2':
 282         case '3':
 283         case '4':
 284         case '5':
 285         case '6':
 286         case '7':
 287                 return true;
 288         default:
 289                 return false;
 290         }
 291 }
 292
 293 /**
 294  * Returns the value of a digit.
 295  * The only portable way to do it ...
 296  */
 297 static int digit_value(int digit) {
 298         switch (digit) {
 299         case '0': return 0;
 300         case '1': return 1;
 301         case '2': return 2;
 302         case '3': return 3;
 303         case '4': return 4;
 304         case '5': return 5;
 305         case '6': return 6;
 306         case '7': return 7;
 307         case '8': return 8;
 308         case '9': return 9;
 309         case 'a':
 310         case 'A': return 10;
 311         case 'b':
 312         case 'B': return 11;
 313         case 'c':
 314         case 'C': return 12;
 315         case 'd':
 316         case 'D': return 13;
 317         case 'e':
 318         case 'E': return 14;
 319         case 'f':
 320         case 'F': return 15;
 321         default:
 322                 panic("wrong character given");
 323         }
 324 }
 325
 326 /**
 327  * Parses an octal character sequence.
 328  *
 329  * @param first_digit  the already read first digit
 330  */
 331 static int parse_octal_sequence(const int first_digit)
 332 {
 333         assert(is_octal_digit(first_digit));
 334         int value = digit_value(first_digit);
 335         if (!is_octal_digit(CC)) return value;
 336         value = 8 * value + digit_value(CC);
 337         next_char();
 338         if (!is_octal_digit(CC)) return value;
 339         value = 8 * value + digit_value(CC);
 340         next_char();
 341
 342         if(char_is_signed) {
 343                 return (signed char) value;
 344         } else {
 345                 return (unsigned char) value;
 346         }
 347 }
 348
 349 /**
 350  * Parses a hex character sequence.
 351  */
 352 static int parse_hex_sequence(void)
 353 {
 354         int value = 0;
 355         while(isxdigit(CC)) {
 356                 value = 16 * value + digit_value(CC);
 357                 next_char();
 358         }
 359
 360         if(char_is_signed) {
 361                 return (signed char) value;
 362         } else {
 363                 return (unsigned char) value;
 364         }
 365 }
 366
 367 /**
 368  * Parse an escape sequence.
 369  */
 370 static int parse_escape_sequence(void)
 371 {
 372         eat('\\');
 373
 374         int ec = CC;
 375         next_char();
 376
 377         switch(ec) {
 378         case '"':  return '"';
 379         case '\'': return '\'';
 380         case '\\': return '\\';
 381         case '?': return '\?';
 382         case 'a': return '\a';
 383         case 'b': return '\b';
 384         case 'f': return '\f';
 385         case 'n': return '\n';
 386         case 'r': return '\r';
 387         case 't': return '\t';
 388         case 'v': return '\v';
 389         case 'x':
 390                 return parse_hex_sequence();
 391         case '0':
 392         case '1':
 393         case '2':
 394         case '3':
 395         case '4':
 396         case '5':
 397         case '6':
 398         case '7':
 399                 return parse_octal_sequence(ec);
 400         case EOF:
 401                 parse_error("reached end of file while parsing escape sequence");
 402                 return EOF;
 403         default:
 404                 parse_error("unknown escape sequence");
 405                 return EOF;
 406         }
 407 }
 408
 409 static void parse_string_literal(void)
 410 {
 411         const unsigned start_linenr = input.position.linenr;
 412
 413         eat('"');
 414
 415         int tc;
 416         while(1) {
 417                 switch(CC) {
 418                 case '\\':
 419                         if(resolve_escape_sequences) {
 420                                 tc = parse_escape_sequence();
 421                                 obstack_1grow(&symbol_obstack, (char) tc);
 422                         } else {
 423                                 obstack_1grow(&symbol_obstack, (char) CC);
 424                                 next_char();
 425                                 obstack_1grow(&symbol_obstack, (char) CC);
 426                                 next_char();
 427                         }
 428                         break;
 429
 430                 case EOF: {
 431                         source_position_t source_position;
 432                         source_position.input_name = pp_token.source_position.input_name;
 433                         source_position.linenr     = start_linenr;
 434                         errorf(&source_position, "string has no end");
 435                         pp_token.type = TP_ERROR;
 436                         return;
 437                 }
 438
 439                 case '"':
 440                         next_char();
 441                         goto end_of_string;
 442
 443                 default:
 444                         obstack_1grow(&symbol_obstack, (char) CC);
 445                         next_char();
 446                         break;
 447                 }
 448         }
 449
 450 end_of_string:
 451         /* add finishing 0 to the string */
 452         obstack_1grow(&symbol_obstack, '\0');
 453         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
 454         const char *const string = obstack_finish(&symbol_obstack);
 455
 456 #if 0 /* TODO hash */
 457         /* check if there is already a copy of the string */
 458         result = strset_insert(&stringset, string);
 459         if(result != string) {
 460                 obstack_free(&symbol_obstack, string);
 461         }
 462 #else
 463         const char *const result = string;
 464 #endif
 465
 466         pp_token.type           = TP_STRING_LITERAL;
 467         pp_token.v.string.begin = result;
 468         pp_token.v.string.size  = size;
 469 }
 470
 471 static void parse_wide_character_constant(void)
 472 {
 473         eat('\'');
 474
 475         int found_char = 0;
 476         while(1) {
 477                 switch(CC) {
 478                 case '\\':
 479                         found_char = parse_escape_sequence();
 480                         break;
 481
 482                 MATCH_NEWLINE(
 483                         parse_error("newline while parsing character constant");
 484                         break;
 485                 )
 486
 487                 case '\'':
 488                         next_char();
 489                         goto end_of_wide_char_constant;
 490
 491                 case EOF:
 492                         parse_error("EOF while parsing character constant");
 493                         pp_token.type = TP_ERROR;
 494                         return;
 495
 496                 default:
 497                         if(found_char != 0) {
 498                                 parse_error("more than 1 characters in character "
 499                                             "constant");
 500                                 goto end_of_wide_char_constant;
 501                         } else {
 502                                 found_char = CC;
 503                                 next_char();
 504                         }
 505                         break;
 506                 }
 507         }
 508
 509 end_of_wide_char_constant:
 510         pp_token.type       = TP_WIDE_CHARACTER_CONSTANT;
 511         /* TODO... */
 512 }
 513
 514 static void parse_wide_string_literal(void)
 515 {
 516         const unsigned start_linenr = input.position.linenr;
 517
 518         assert(CC == '"');
 519         next_char();
 520
 521         while(1) {
 522                 switch(CC) {
 523                 case '\\': {
 524                         wchar_rep_t tc = parse_escape_sequence();
 525                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 526                         break;
 527                 }
 528
 529                 case EOF: {
 530                         source_position_t source_position;
 531                         source_position.input_name = pp_token.source_position.input_name;
 532                         source_position.linenr     = start_linenr;
 533                         errorf(&source_position, "string has no end");
 534                         pp_token.type = TP_ERROR;
 535                         return;
 536                 }
 537
 538                 case '"':
 539                         next_char();
 540                         goto end_of_string;
 541
 542                 default: {
 543                         wchar_rep_t tc = CC;
 544                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 545                         next_char();
 546                         break;
 547                 }
 548                 }
 549         }
 550
 551 end_of_string:;
 552         /* add finishing 0 to the string */
 553         static const wchar_rep_t nul = L'\0';
 554         obstack_grow(&symbol_obstack, &nul, sizeof(nul));
 555
 556         const size_t size
 557                 = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
 558         const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
 559
 560 #if 0 /* TODO hash */
 561         /* check if there is already a copy of the string */
 562         const wchar_rep_t *const result = strset_insert(&stringset, string);
 563         if(result != string) {
 564                 obstack_free(&symbol_obstack, string);
 565         }
 566 #else
 567         const wchar_rep_t *const result = string;
 568 #endif
 569
 570         pp_token.type                = TP_WIDE_STRING_LITERAL;
 571         pp_token.v.wide_string.begin = result;
 572         pp_token.v.wide_string.size  = size;
 573 }
 574
 575 static void parse_character_constant(void)
 576 {
 577         const unsigned start_linenr = input.position.linenr;
 578
 579         eat('\'');
 580
 581         int tc;
 582         while(1) {
 583                 switch(CC) {
 584                 case '\\':
 585                         tc = parse_escape_sequence();
 586                         obstack_1grow(&symbol_obstack, (char) tc);
 587                         break;
 588
 589                 MATCH_NEWLINE(
 590                         parse_error("newline while parsing character constant");
 591                         break;
 592                 )
 593
 594                 case EOF: {
 595                         source_position_t source_position;
 596                         source_position.input_name = pp_token.source_position.input_name;
 597                         source_position.linenr     = start_linenr;
 598                         errorf(&source_position, "EOF while parsing character constant");
 599                         pp_token.type = TP_ERROR;
 600                         return;
 601                 }
 602
 603                 case '\'':
 604                         next_char();
 605                         goto end_of_char_constant;
 606
 607                 default:
 608                         obstack_1grow(&symbol_obstack, (char) CC);
 609                         next_char();
 610                         break;
 611
 612                 }
 613         }
 614
 615 end_of_char_constant:;
 616         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
 617         const char *const string = obstack_finish(&symbol_obstack);
 618
 619         pp_token.type           = TP_CHARACTER_CONSTANT;
 620         pp_token.v.string.begin = string;
 621         pp_token.v.string.size  = size;
 622 }
 623
 624 #define SYMBOL_CHARS_WITHOUT_E_P \
 625         case 'a': \
 626         case 'b': \
 627         case 'c': \
 628         case 'd': \
 629         case 'f': \
 630         case 'g': \
 631         case 'h': \
 632         case 'i': \
 633         case 'j': \
 634         case 'k': \
 635         case 'l': \
 636         case 'm': \
 637         case 'n': \
 638         case 'o': \
 639         case 'q': \
 640         case 'r': \
 641         case 's': \
 642         case 't': \
 643         case 'u': \
 644         case 'v': \
 645         case 'w': \
 646         case 'x': \
 647         case 'y': \
 648         case 'z': \
 649         case 'A': \
 650         case 'B': \
 651         case 'C': \
 652         case 'D': \
 653         case 'F': \
 654         case 'G': \
 655         case 'H': \
 656         case 'I': \
 657         case 'J': \
 658         case 'K': \
 659         case 'L': \
 660         case 'M': \
 661         case 'N': \
 662         case 'O': \
 663         case 'Q': \
 664         case 'R': \
 665         case 'S': \
 666         case 'T': \
 667         case 'U': \
 668         case 'V': \
 669         case 'W': \
 670         case 'X': \
 671         case 'Y': \
 672         case 'Z': \
 673         case '_':
 674
 675 #define SYMBOL_CHARS \
 676         SYMBOL_CHARS_WITHOUT_E_P \
 677         case 'e': \
 678         case 'p': \
 679         case 'E': \
 680         case 'P':
 681
 682 #define DIGITS \
 683         case '0':  \
 684         case '1':  \
 685         case '2':  \
 686         case '3':  \
 687         case '4':  \
 688         case '5':  \
 689         case '6':  \
 690         case '7':  \
 691         case '8':  \
 692         case '9':
 693
 694 /**
 695  * returns next final token from a preprocessor macro expansion
 696  */
 697 static void expand_next(void)
 698 {
 699         assert(current_expansion != NULL);
 700
 701         pp_definition_t *definition = current_expansion;
 702
 703 restart:
 704         if(definition->list_len == 0
 705                         || definition->expand_pos >= definition->list_len) {
 706                 /* we're finished with the current macro, move up 1 level in the
 707                  * expansion stack */
 708                 pp_definition_t *parent = definition->parent_expansion;
 709                 definition->parent_expansion = NULL;
 710                 definition->is_expanding     = false;
 711
 712                 /* it was the outermost expansion, parse normal pptoken */
 713                 if(parent == NULL) {
 714                         current_expansion = NULL;
 715                         next_preprocessing_token();
 716                         return;
 717                 }
 718                 definition        = parent;
 719                 current_expansion = definition;
 720                 goto restart;
 721         }
 722         pp_token = definition->replacement_list[definition->expand_pos];
 723         ++definition->expand_pos;
 724
 725         if(pp_token.type != TP_IDENTIFIER)
 726                 return;
 727
 728         /* if it was an identifier then we might need to expand again */
 729         pp_definition_t *symbol_definition = pp_token.v.symbol->pp_definition;
 730         if(symbol_definition != NULL && !symbol_definition->is_expanding) {
 731                 symbol_definition->parent_expansion = definition;
 732                 symbol_definition->expand_pos       = 0;
 733                 symbol_definition->is_expanding     = true;
 734                 definition                          = symbol_definition;
 735                 current_expansion                   = definition;
 736                 goto restart;
 737         }
 738 }
 739
 740 static void parse_symbol(void)
 741 {
 742         obstack_1grow(&symbol_obstack, (char) CC);
 743         next_char();
 744
 745         while(1) {
 746                 switch(CC) {
 747                 DIGITS
 748                 SYMBOL_CHARS
 749                         obstack_1grow(&symbol_obstack, (char) CC);
 750                         next_char();
 751                         break;
 752
 753                 default:
 754                         goto end_symbol;
 755                 }
 756         }
 757
 758 end_symbol:
 759         obstack_1grow(&symbol_obstack, '\0');
 760         char *string = obstack_finish(&symbol_obstack);
 761
 762         /* might be a wide string or character constant ( L"string"/L'c' ) */
 763         if(CC == '"' && string[0] == 'L' && string[1] == '\0') {
 764                 obstack_free(&symbol_obstack, string);
 765                 parse_wide_string_literal();
 766                 return;
 767         } else if(CC == '\'' && string[0] == 'L' && string[1] == '\0') {
 768                 obstack_free(&symbol_obstack, string);
 769                 parse_wide_character_constant();
 770                 return;
 771         }
 772
 773         symbol_t *symbol = symbol_table_insert(string);
 774
 775         pp_token.type     = symbol->pp_ID;
 776         pp_token.v.symbol = symbol;
 777
 778         /* we can free the memory from symbol obstack if we already had an entry in
 779          * the symbol table */
 780         if(symbol->string != string) {
 781                 obstack_free(&symbol_obstack, string);
 782         }
 783
 784         pp_definition_t *pp_definition = symbol->pp_definition;
 785         if(do_expansions && pp_definition != NULL) {
 786                 pp_definition->expand_pos   = 0;
 787                 pp_definition->is_expanding = true,
 788                 current_expansion           = pp_definition;
 789                 expand_next();
 790         }
 791 }
 792
 793 static void parse_number(void)
 794 {
 795         obstack_1grow(&symbol_obstack, (char) CC);
 796         next_char();
 797
 798         while(1) {
 799                 switch(CC) {
 800                 case '.':
 801                 DIGITS
 802                 SYMBOL_CHARS_WITHOUT_E_P
 803                         obstack_1grow(&symbol_obstack, (char) CC);
 804                         next_char();
 805                         break;
 806
 807                 case 'e':
 808                 case 'p':
 809                 case 'E':
 810                 case 'P':
 811                         obstack_1grow(&symbol_obstack, (char) CC);
 812                         next_char();
 813                         if(CC == '+' || CC == '-') {
 814                                 obstack_1grow(&symbol_obstack, (char) CC);
 815                                 next_char();
 816                         }
 817                         break;
 818
 819                 default:
 820                         goto end_number;
 821                 }
 822         }
 823
 824 end_number:
 825         obstack_1grow(&symbol_obstack, '\0');
 826         size_t  size   = obstack_object_size(&symbol_obstack);
 827         char   *string = obstack_finish(&symbol_obstack);
 828
 829         pp_token.type           = TP_NUMBER;
 830         pp_token.v.string.begin = string;
 831         pp_token.v.string.size  = size;
 832 }
 833
 834 static void skip_multiline_comment(void)
 835 {
 836         unsigned start_linenr = input.position.linenr;
 837
 838         while(1) {
 839                 switch(CC) {
 840                 case '/':
 841                         next_char();
 842                         if (CC == '*') {
 843                                 /* TODO: nested comment, warn here */
 844                         }
 845                         break;
 846                 case '*':
 847                         next_char();
 848                         if(CC == '/') {
 849                                 next_char();
 850                                 return;
 851                         }
 852                         break;
 853
 854                 MATCH_NEWLINE(
 855                         if(do_print_spaces) {
 856                                 counted_newlines++;
 857                                 counted_spaces = 0;
 858                         }
 859                         break;
 860                 )
 861
 862                 case EOF: {
 863                         source_position_t source_position;
 864                         source_position.input_name = pp_token.source_position.input_name;
 865                         source_position.linenr     = start_linenr;
 866                         errorf(&source_position, "at end of file while looking for comment end");
 867                         return;
 868                 }
 869
 870                 default:
 871                         next_char();
 872                         break;
 873                 }
 874         }
 875 }
 876
 877 static void skip_line_comment(void)
 878 {
 879         while(1) {
 880                 switch(CC) {
 881                 case EOF:
 882                         return;
 883
 884                 case '\n':
 885                 case '\r':
 886                         return;
 887
 888                 default:
 889                         next_char();
 890                         break;
 891                 }
 892         }
 893 }
 894
 895
 896
 897 #define MAYBE_PROLOG                                       \
 898                         next_char();                                   \
 899                         while(1) {                                     \
 900                                 switch(CC) {
 901
 902 #define MAYBE(ch, set_type)                                \
 903                                 case ch:                                   \
 904                                         next_char();                           \
 905                                         pp_token.type = set_type;              \
 906                                         return;
 907
 908 #define ELSE_CODE(code)                                    \
 909                                 default:                                   \
 910                                         code;                                  \
 911                                 }                                          \
 912                         } /* end of while(1) */                        \
 913                         break;
 914
 915 #define ELSE(set_type)                                     \
 916                 ELSE_CODE(                                         \
 917                         pp_token.type = set_type;                      \
 918                         return;                                        \
 919                 )
 920
 921 static void next_preprocessing_token(void)
 922 {
 923         if(current_expansion != NULL) {
 924                 expand_next();
 925                 return;
 926         }
 927
 928         pp_token.source_position = input.position;
 929
 930 restart:
 931         switch(CC) {
 932         case ' ':
 933         case '\t':
 934                 if(do_print_spaces)
 935                         counted_spaces++;
 936                 next_char();
 937                 goto restart;
 938
 939         MATCH_NEWLINE(
 940                 counted_newlines++;
 941                 counted_spaces = 0;
 942                 pp_token.type = '\n';
 943                 return;
 944         )
 945
 946         SYMBOL_CHARS
 947                 parse_symbol();
 948                 return;
 949
 950         DIGITS
 951                 parse_number();
 952                 return;
 953
 954         case '"':
 955                 parse_string_literal();
 956                 return;
 957
 958         case '\'':
 959                 parse_character_constant();
 960                 return;
 961
 962         case '.':
 963                 MAYBE_PROLOG
 964                         case '0':
 965                         case '1':
 966                         case '2':
 967                         case '3':
 968                         case '4':
 969                         case '5':
 970                         case '6':
 971                         case '7':
 972                         case '8':
 973                         case '9':
 974                                 put_back(CC);
 975                                 CC = '.';
 976                                 parse_number();
 977                                 return;
 978
 979                         case '.':
 980                                 MAYBE_PROLOG
 981                                 MAYBE('.', TP_DOTDOTDOT)
 982                                 ELSE_CODE(
 983                                         put_back(CC);
 984                                         CC = '.';
 985                                         pp_token.type = '.';
 986                                         return;
 987                                 )
 988                 ELSE('.')
 989         case '&':
 990                 MAYBE_PROLOG
 991                 MAYBE('&', TP_ANDAND)
 992                 MAYBE('=', TP_ANDEQUAL)
 993                 ELSE('&')
 994         case '*':
 995                 MAYBE_PROLOG
 996                 MAYBE('=', TP_ASTERISKEQUAL)
 997                 ELSE('*')
 998         case '+':
 999                 MAYBE_PROLOG
1000                 MAYBE('+', TP_PLUSPLUS)
1001                 MAYBE('=', TP_PLUSEQUAL)
1002                 ELSE('+')
1003         case '-':
1004                 MAYBE_PROLOG
1005                 MAYBE('>', TP_MINUSGREATER)
1006                 MAYBE('-', TP_MINUSMINUS)
1007                 MAYBE('=', TP_MINUSEQUAL)
1008                 ELSE('-')
1009         case '!':
1010                 MAYBE_PROLOG
1011                 MAYBE('=', TP_EXCLAMATIONMARKEQUAL)
1012                 ELSE('!')
1013         case '/':
1014                 MAYBE_PROLOG
1015                 MAYBE('=', TP_SLASHEQUAL)
1016                         case '*':
1017                                 next_char();
1018                                 skip_multiline_comment();
1019                                 if(do_print_spaces)
1020                                         counted_spaces++;
1021                                 goto restart;
1022                         case '/':
1023                                 next_char();
1024                                 skip_line_comment();
1025                                 if(do_print_spaces)
1026                                         counted_spaces++;
1027                                 goto restart;
1028                 ELSE('/')
1029         case '%':
1030                 MAYBE_PROLOG
1031                 MAYBE('>', '}')
1032                 MAYBE('=', TP_PERCENTEQUAL)
1033                         case ':':
1034                                 MAYBE_PROLOG
1035                                         case '%':
1036                                                 MAYBE_PROLOG
1037                                                 MAYBE(':', TP_HASHHASH)
1038                                                 ELSE_CODE(
1039                                                         put_back(CC);
1040                                                         CC = '%';
1041                                                         pp_token.type = '#';
1042                                                         return;
1043                                                 )
1044                                 ELSE('#')
1045                 ELSE('%')
1046         case '<':
1047                 MAYBE_PROLOG
1048                 MAYBE(':', '[')
1049                 MAYBE('%', '{')
1050                 MAYBE('=', TP_LESSEQUAL)
1051                         case '<':
1052                                 MAYBE_PROLOG
1053                                 MAYBE('=', TP_LESSLESSEQUAL)
1054                                 ELSE(TP_LESSLESS)
1055                 ELSE('<')
1056         case '>':
1057                 MAYBE_PROLOG
1058                 MAYBE('=', TP_GREATEREQUAL)
1059                         case '>':
1060                                 MAYBE_PROLOG
1061                                 MAYBE('=', TP_GREATERGREATEREQUAL)
1062                                 ELSE(TP_GREATERGREATER)
1063                 ELSE('>')
1064         case '^':
1065                 MAYBE_PROLOG
1066                 MAYBE('=', TP_CARETEQUAL)
1067                 ELSE('^')
1068         case '|':
1069                 MAYBE_PROLOG
1070                 MAYBE('=', TP_PIPEEQUAL)
1071                 MAYBE('|', TP_PIPEPIPE)
1072                 ELSE('|')
1073         case ':':
1074                 MAYBE_PROLOG
1075                 MAYBE('>', ']')
1076                 ELSE(':')
1077         case '=':
1078                 MAYBE_PROLOG
1079                 MAYBE('=', TP_EQUALEQUAL)
1080                 ELSE('=')
1081         case '#':
1082                 MAYBE_PROLOG
1083                 MAYBE('#', TP_HASHHASH)
1084                 ELSE('#')
1085
1086         case '?':
1087         case '[':
1088         case ']':
1089         case '(':
1090         case ')':
1091         case '{':
1092         case '}':
1093         case '~':
1094         case ';':
1095         case ',':
1096         case '\\':
1097                 pp_token.type = CC;
1098                 next_char();
1099                 return;
1100
1101         case EOF:
1102                 if (input_stack != NULL) {
1103                         close_input();
1104                         pop_restore_input();
1105                         counted_newlines = 0;
1106                         counted_spaces   = 0;
1107                         /* hack to output correct line number */
1108                         print_line_directive(&input.position, "2");
1109                         next_preprocessing_token();
1110                 } else {
1111                         pp_token.type = TP_EOF;
1112                 }
1113                 return;
1114
1115         default:
1116                 next_char();
1117                 errorf(&pp_token.source_position, "unknown character '%c' found\n", CC);
1118                 pp_token.type = TP_ERROR;
1119                 return;
1120         }
1121 }
1122
1123 static void print_quoted_string(const char *const string)
1124 {
1125         fputc('"', out);
1126         for (const char *c = string; *c != 0; ++c) {
1127                 switch(*c) {
1128                 case '"': fputs("\\\"", out); break;
1129                 case '\\':  fputs("\\\\", out); break;
1130                 case '\a':  fputs("\\a", out); break;
1131                 case '\b':  fputs("\\b", out); break;
1132                 case '\f':  fputs("\\f", out); break;
1133                 case '\n':  fputs("\\n", out); break;
1134                 case '\r':  fputs("\\r", out); break;
1135                 case '\t':  fputs("\\t", out); break;
1136                 case '\v':  fputs("\\v", out); break;
1137                 case '\?':  fputs("\\?", out); break;
1138                 default:
1139                         if(!isprint(*c)) {
1140                                 fprintf(out, "\\%03o", *c);
1141                                 break;
1142                         }
1143                         fputc(*c, out);
1144                         break;
1145                 }
1146         }
1147         fputc('"', out);
1148 }
1149
1150 static void print_line_directive(const source_position_t *pos, const char *add)
1151 {
1152         fprintf(out, "# %d ", pos->linenr);
1153         print_quoted_string(pos->input_name);
1154         if (add != NULL) {
1155                 fputc(' ', out);
1156                 fputs(add, out);
1157         }
1158         fputc('\n', out);
1159
1160         printed_input_name = pos->input_name;
1161 }
1162
1163 static void print_spaces(void)
1164 {
1165         if (counted_newlines >= 9) {
1166                 if (input.had_non_space) {
1167                         fputc('\n', out);
1168                 }
1169                 print_line_directive(&pp_token.source_position, NULL);
1170                 counted_newlines = 0;
1171         } else {
1172                 for (unsigned i = 0; i < counted_newlines; ++i)
1173                         fputc('\n', out);
1174                 counted_newlines = 0;
1175         }
1176         for (unsigned i = 0; i < counted_spaces; ++i)
1177                 fputc(' ', out);
1178         counted_spaces = 0;
1179 }
1180
1181 static void emit_pp_token(void)
1182 {
1183         if (pp_token.type != '\n') {
1184                 print_spaces();
1185                 input.had_non_space = true;
1186         }
1187
1188         switch(pp_token.type) {
1189         case TP_IDENTIFIER:
1190                 fputs(pp_token.v.symbol->string, out);
1191                 break;
1192         case TP_NUMBER:
1193                 fputs(pp_token.v.string.begin, out);
1194                 break;
1195         case TP_STRING_LITERAL:
1196                 fputc('"', out);
1197                 fputs(pp_token.v.string.begin, out);
1198                 fputc('"', out);
1199                 break;
1200         case '\n':
1201                 break;
1202         default:
1203                 print_pp_token_type(out, pp_token.type);
1204                 break;
1205         }
1206 }
1207
1208 static void eat_pp(preprocessor_token_type_t type)
1209 {
1210         (void) type;
1211         assert(pp_token.type == type);
1212         next_preprocessing_token();
1213 }
1214
1215 static void eat_pp_directive(void)
1216 {
1217         while(pp_token.type != '\n' && pp_token.type != TP_EOF) {
1218                 next_preprocessing_token();
1219         }
1220 }
1221
1222 static bool strings_equal(const string_t *string1, const string_t *string2)
1223 {
1224         size_t size = string1->size;
1225         if(size != string2->size)
1226                 return false;
1227
1228         const char *c1 = string1->begin;
1229         const char *c2 = string2->begin;
1230         for(size_t i = 0; i < size; ++i, ++c1, ++c2) {
1231                 if(*c1 != *c2)
1232                         return false;
1233         }
1234         return true;
1235 }
1236
1237 static bool wide_strings_equal(const wide_string_t *string1,
1238                                const wide_string_t *string2)
1239 {
1240         size_t size = string1->size;
1241         if(size != string2->size)
1242                 return false;
1243
1244         const wchar_rep_t *c1 = string1->begin;
1245         const wchar_rep_t *c2 = string2->begin;
1246         for(size_t i = 0; i < size; ++i, ++c1, ++c2) {
1247                 if(*c1 != *c2)
1248                         return false;
1249         }
1250         return true;
1251 }
1252
1253 static bool pp_tokens_equal(const token_t *token1, const token_t *token2)
1254 {
1255         if(token1->type != token2->type)
1256                 return false;
1257
1258         switch(token1->type) {
1259         case TP_HEADERNAME:
1260                 /* TODO */
1261                 return false;
1262         case TP_IDENTIFIER:
1263                 return token1->v.symbol == token2->v.symbol;
1264         case TP_NUMBER:
1265         case TP_CHARACTER_CONSTANT:
1266         case TP_STRING_LITERAL:
1267                 return strings_equal(&token1->v.string, &token2->v.string);
1268
1269         case TP_WIDE_CHARACTER_CONSTANT:
1270         case TP_WIDE_STRING_LITERAL:
1271                 return wide_strings_equal(&token1->v.wide_string,
1272                                           &token2->v.wide_string);
1273         default:
1274                 return true;
1275         }
1276 }
1277
1278 static bool pp_definitions_equal(const pp_definition_t *definition1,
1279                                  const pp_definition_t *definition2)
1280 {
1281         if(definition1->list_len != definition2->list_len)
1282                 return false;
1283
1284         size_t         len = definition1->list_len;
1285         const token_t *t1  = definition1->replacement_list;
1286         const token_t *t2  = definition2->replacement_list;
1287         for(size_t i = 0; i < len; ++i, ++t1, ++t2) {
1288                 if(!pp_tokens_equal(t1, t2))
1289                         return false;
1290         }
1291         return true;
1292 }
1293
1294 static void parse_define_directive(void)
1295 {
1296         eat_pp(TP_define);
1297
1298         if(pp_token.type != TP_IDENTIFIER) {
1299                 errorf(&pp_token.source_position,
1300                        "expected identifier after #define, got '%T'", &pp_token);
1301                 eat_pp_directive();
1302                 return;
1303         }
1304         symbol_t *symbol = pp_token.v.symbol;
1305
1306         pp_definition_t *new_definition
1307                 = obstack_alloc(&pp_obstack, sizeof(new_definition[0]));
1308         memset(new_definition, 0, sizeof(new_definition[0]));
1309         new_definition->source_position = input.position;
1310
1311         /* this is probably the only place where spaces are significant in the
1312          * lexer (except for the fact that they separate tokens). #define b(x)
1313          * is something else than #define b (x) */
1314         //token_t *arguments = NULL;
1315         if(CC == '(') {
1316                 next_preprocessing_token();
1317                 while(pp_token.type != ')') {
1318                         if(pp_token.type == TP_DOTDOTDOT) {
1319                                 new_definition->is_variadic = true;
1320                                 next_preprocessing_token();
1321                                 if(pp_token.type != ')') {
1322                                         errorf(&input.position,
1323                                                         "'...' not at end of macro argument list");
1324                                         continue;
1325                                 }
1326                         } else if(pp_token.type != TP_IDENTIFIER) {
1327                                 next_preprocessing_token();
1328                         }
1329                 }
1330         } else {
1331                 next_preprocessing_token();
1332         }
1333
1334         /* construct a new pp_definition on the obstack */
1335         assert(obstack_object_size(&pp_obstack) == 0);
1336         size_t list_len = 0;
1337         while(pp_token.type != '\n' && pp_token.type != TP_EOF) {
1338                 obstack_grow(&pp_obstack, &pp_token, sizeof(pp_token));
1339                 ++list_len;
1340                 next_preprocessing_token();
1341         }
1342
1343         new_definition->list_len         = list_len;
1344         new_definition->replacement_list = obstack_finish(&pp_obstack);
1345
1346         pp_definition_t *old_definition = symbol->pp_definition;
1347         if(old_definition != NULL) {
1348                 if(!pp_definitions_equal(old_definition, new_definition)) {
1349                         warningf(&input.position, "multiple definition of macro '%Y' (first defined %P)",
1350                                  symbol, &old_definition->source_position);
1351                 } else {
1352                         /* reuse the old definition */
1353                         obstack_free(&pp_obstack, new_definition);
1354                         new_definition = old_definition;
1355                 }
1356         }
1357
1358         symbol->pp_definition = new_definition;
1359 }
1360
1361 static void parse_undef_directive(void)
1362 {
1363         eat_pp(TP_undef);
1364
1365         if(pp_token.type != TP_IDENTIFIER) {
1366                 errorf(&input.position,
1367                        "expected identifier after #undef, got '%T'", &pp_token);
1368                 eat_pp_directive();
1369                 return;
1370         }
1371
1372         symbol_t *symbol = pp_token.v.symbol;
1373         symbol->pp_definition = NULL;
1374         next_preprocessing_token();
1375
1376         if(pp_token.type != '\n') {
1377                 warningf(&input.position, "extra tokens at end of #undef directive");
1378         }
1379         /* eat until '\n' */
1380         eat_pp_directive();
1381 }
1382
1383 static const char *parse_headername(void)
1384 {
1385         /* behind an #include we can have the special headername lexems, check
1386          * for them here */
1387
1388         /* skip spaces */
1389         while (CC == ' ' || CC == '\t') {
1390                 next_char();
1391         }
1392
1393         assert(obstack_object_size(&input_obstack) == 0);
1394
1395         switch (CC) {
1396         case '<':
1397                 /* for now until we have proper searchpath handling */
1398                 obstack_1grow(&input_obstack, '.');
1399                 obstack_1grow(&input_obstack, '/');
1400
1401                 next_char();
1402                 while (true) {
1403                         switch (CC) {
1404                         case EOF:
1405                                 /* fallthrough */
1406                         MATCH_NEWLINE(
1407                                 parse_error("header name without closing '>'");
1408                                 return NULL;
1409                         )
1410                         case '>':
1411                                 next_char();
1412                                 goto finished_headername;
1413                         }
1414                         obstack_1grow(&input_obstack, (char) CC);
1415                         next_char();
1416                 }
1417                 /* we should never be here */
1418
1419         case '"':
1420                 /* for now until we have proper searchpath handling */
1421                 obstack_1grow(&input_obstack, '.');
1422                 obstack_1grow(&input_obstack, '/');
1423
1424                 next_char();
1425                 while (true) {
1426                         switch (CC) {
1427                         case EOF:
1428                                 /* fallthrough */
1429                         MATCH_NEWLINE(
1430                                 parse_error("header name without closing '>'");
1431                                 return NULL;
1432                         )
1433                         case '"':
1434                                 next_char();
1435                                 goto finished_headername;
1436                         }
1437                         obstack_1grow(&input_obstack, (char) CC);
1438                         next_char();
1439                 }
1440                 /* we should never be here */
1441
1442         default:
1443                 /* TODO: do normale pp_token parsing and concatenate results */
1444                 panic("pp_token concat include not implemented yet");
1445         }
1446
1447 finished_headername:
1448         obstack_1grow(&input_obstack, '\0');
1449         char *headername = obstack_finish(&input_obstack);
1450
1451         /* TODO: iterate search-path to find the file */
1452
1453         next_preprocessing_token();
1454
1455         return headername;
1456 }
1457
1458 static void parse_include_directive(void)
1459 {
1460         /* don't eat the TP_include here!
1461          * we need an alternative parsing for the next token */
1462
1463         print_spaces();
1464
1465         const char *headername = parse_headername();
1466         if (headername == NULL) {
1467                 eat_pp_directive();
1468                 return;
1469         }
1470
1471         if (pp_token.type != '\n' && pp_token.type != TP_EOF) {
1472                 warningf(&pp_token.source_position,
1473                          "extra tokens at end of #include directive");
1474                 eat_pp_directive();
1475         }
1476
1477         if (n_inputs > INCLUDE_LIMIT) {
1478                 errorf(&pp_token.source_position, "#include nested too deeply");
1479                 /* eat \n or EOF */
1480                 next_preprocessing_token();
1481                 return;
1482         }
1483
1484         /* switch inputs */
1485         push_input();
1486         bool res = open_input(headername);
1487         if (!res) {
1488                 errorf(&pp_token.source_position,
1489                        "failed including '%s': %s", headername, strerror(errno));
1490                 pop_restore_input();
1491                 return;
1492         }
1493 }
1494
1495 static void parse_preprocessing_directive(void)
1496 {
1497         do_print_spaces = false;
1498         do_expansions   = false;
1499         eat_pp('#');
1500
1501         switch(pp_token.type) {
1502         case TP_define:
1503                 parse_define_directive();
1504                 break;
1505         case TP_undef:
1506                 parse_undef_directive();
1507                 break;
1508         case TP_include:
1509                 parse_include_directive();
1510                 /* no need to parse ending '\n' */
1511                 do_print_spaces = true;
1512                 do_expansions   = true;
1513                 return;
1514         default:
1515                 errorf(&pp_token.source_position,
1516                        "invalid preprocessing directive #%T", &pp_token);
1517                 eat_pp_directive();
1518                 break;
1519         }
1520
1521         do_print_spaces = true;
1522         do_expansions   = true;
1523
1524         /* eat '\n' */
1525         assert(pp_token.type == '\n' || pp_token.type == TP_EOF);
1526         next_preprocessing_token();
1527 }
1528
1529 #define GCC_COMPAT_MODE
1530
1531 int pptest_main(int argc, char **argv);
1532 int pptest_main(int argc, char **argv)
1533 {
1534         init_symbol_table();
1535         init_tokens();
1536
1537         obstack_init(&pp_obstack);
1538         obstack_init(&input_obstack);
1539
1540         const char *filename = "t.c";
1541         if (argc > 1)
1542                 filename = argv[1];
1543
1544         out = stdout;
1545
1546 #ifdef GCC_COMPAT_MODE
1547         /* this is here so we can directly compare "gcc -E" output and our output */
1548         fprintf(out, "# 1 \"%s\"\n", filename);
1549         fputs("# 1 \"<built-in>\"\n", out);
1550         fputs("# 1 \"<command-line>\"\n", out);
1551 #endif
1552
1553         bool ok = open_input(filename);
1554         assert(ok);
1555
1556         while(true) {
1557                 /* we're at a line begin */
1558                 if(pp_token.type == '#') {
1559                         parse_preprocessing_directive();
1560                 } else {
1561                         /* parse+emit a line */
1562                         while(pp_token.type != '\n') {
1563                                 if(pp_token.type == TP_EOF)
1564                                         goto end_of_main_loop;
1565                                 emit_pp_token();
1566                                 next_preprocessing_token();
1567                         }
1568                         emit_pp_token();
1569                         next_preprocessing_token();
1570                 }
1571         }
1572 end_of_main_loop:
1573
1574         close_input();
1575
1576         obstack_free(&input_obstack, NULL);
1577         obstack_free(&pp_obstack, NULL);
1578
1579         exit_tokens();
1580         exit_symbol_table();
1581
1582         return 0;
1583 }