nsz Git - cparser/blob - preprocessor.c

   1 #include <config.h>
   2
   3 #include <assert.h>
   4 #include <errno.h>
   5 #include <string.h>
   6 #include <stdbool.h>
   7 #include <ctype.h>
   8
   9 #include "preprocessor.h"
  10 #include "token_t.h"
  11 #include "symbol_t.h"
  12 #include "adt/util.h"
  13 #include "adt/error.h"
  14 #include "adt/strutil.h"
  15 #include "adt/strset.h"
  16 #include "lang_features.h"
  17 #include "diagnostic.h"
  18 #include "string_rep.h"
  19 #include "input.h"
  20
  21 #define MAX_PUTBACK 3
  22 #define INCLUDE_LIMIT 199  /* 199 is for gcc "compatibility" */
  23
  24 typedef struct saved_token_t {
  25         token_t token;
  26         bool    had_whitespace;
  27 } saved_token_t;
  28
  29 typedef struct whitespace_info_t {
  30         /** current token had whitespace in front of it */
  31         bool     had_whitespace;
  32         /** current token is at the beginning of a line.
  33          * => a "#" at line begin starts a preprocessing directive. */
  34         bool     at_line_begin;
  35         /** number of spaces before the first token in a line */
  36         unsigned whitespace_at_line_begin;
  37 } whitespace_info_t;
  38
  39 struct pp_definition_t {
  40         symbol_t          *symbol;
  41         source_position_t  source_position;
  42         pp_definition_t   *parent_expansion;
  43         size_t             expand_pos;
  44         whitespace_info_t  expand_info;
  45         bool               is_variadic    : 1;
  46         bool               is_expanding   : 1;
  47         bool               has_parameters : 1;
  48         bool               is_parameter   : 1;
  49         pp_definition_t   *function_definition;
  50         size_t             n_parameters;
  51         pp_definition_t   *parameters;
  52
  53         /* replacement */
  54         size_t             list_len;
  55         saved_token_t     *token_list;
  56 };
  57
  58 typedef struct pp_conditional_t pp_conditional_t;
  59 struct pp_conditional_t {
  60         source_position_t  source_position;
  61         bool               condition;
  62         bool               in_else;
  63         /** conditional in skip mode (then+else gets skipped) */
  64         bool               skip;
  65         pp_conditional_t  *parent;
  66 };
  67
  68 typedef struct pp_input_t pp_input_t;
  69 struct pp_input_t {
  70         FILE               *file;
  71         input_t            *input;
  72         utf32               c;
  73         utf32               buf[1024+MAX_PUTBACK];
  74         const utf32        *bufend;
  75         const utf32        *bufpos;
  76         source_position_t   position;
  77         pp_input_t         *parent;
  78         unsigned            output_line;
  79         searchpath_entry_t *path;
  80 };
  81
  82 struct searchpath_entry_t {
  83         const char         *path;
  84         searchpath_entry_t *next;
  85 };
  86
  87 static pp_input_t      input;
  88
  89 static pp_input_t     *input_stack;
  90 static unsigned        n_inputs;
  91 static struct obstack  input_obstack;
  92
  93 static pp_conditional_t *conditional_stack;
  94
  95 token_t                  pp_token;
  96 bool                     allow_dollar_in_symbol   = true;
  97 static bool              resolve_escape_sequences = true;
  98 static bool              error_on_unknown_chars   = true;
  99 static bool              skip_mode;
 100 static FILE             *out;
 101 static struct obstack    pp_obstack;
 102 static struct obstack    config_obstack;
 103 static const char       *printed_input_name = NULL;
 104 static source_position_t expansion_pos;
 105 static pp_definition_t  *current_expansion  = NULL;
 106 static pp_definition_t  *current_call       = NULL;
 107 static pp_definition_t  *current_argument   = NULL;
 108 static pp_definition_t  *argument_expanding = NULL;
 109 static unsigned          argument_brace_count;
 110 static strset_t          stringset;
 111 static token_kind_t      last_token;
 112
 113 static searchpath_entry_t *searchpath;
 114
 115 static whitespace_info_t next_info; /* valid if had_whitespace is true */
 116 static whitespace_info_t info;
 117
 118 static inline void next_char(void);
 119 static void next_input_token(void);
 120 static void print_line_directive(const source_position_t *pos, const char *add);
 121
 122 static symbol_t *symbol_colongreater;
 123 static symbol_t *symbol_lesscolon;
 124 static symbol_t *symbol_lesspercent;
 125 static symbol_t *symbol_percentcolon;
 126 static symbol_t *symbol_percentcolonpercentcolon;
 127 static symbol_t *symbol_percentgreater;
 128
 129 static void init_symbols(void)
 130 {
 131         symbol_colongreater             = symbol_table_insert(":>");
 132         symbol_lesscolon                = symbol_table_insert("<:");
 133         symbol_lesspercent              = symbol_table_insert("<%");
 134         symbol_percentcolon             = symbol_table_insert("%:");
 135         symbol_percentcolonpercentcolon = symbol_table_insert("%:%:");
 136         symbol_percentgreater           = symbol_table_insert("%>");
 137 }
 138
 139 void switch_pp_input(FILE *const file, char const *const filename, searchpath_entry_t *const path)
 140 {
 141         input.file                = file;
 142         input.input               = input_from_stream(file, NULL);
 143         input.bufend              = NULL;
 144         input.bufpos              = NULL;
 145         input.output_line         = 0;
 146         input.position.input_name = filename;
 147         input.position.lineno     = 1;
 148         input.path                = path;
 149
 150         /* indicate that we're at a new input */
 151         print_line_directive(&input.position, input_stack != NULL ? "1" : NULL);
 152
 153         /* place a virtual '\n' so we realize we're at line begin */
 154         input.position.lineno = 0;
 155         input.c               = '\n';
 156 }
 157
 158 FILE *close_pp_input(void)
 159 {
 160         input_free(input.input);
 161
 162         FILE* const file = input.file;
 163         assert(file);
 164
 165         input.input  = NULL;
 166         input.file   = NULL;
 167         input.bufend = NULL;
 168         input.bufpos = NULL;
 169         input.c      = EOF;
 170
 171         return file;
 172 }
 173
 174 static void push_input(void)
 175 {
 176         pp_input_t *const saved_input = obstack_copy(&input_obstack, &input, sizeof(input));
 177
 178         /* adjust buffer positions */
 179         if (input.bufpos != NULL)
 180                 saved_input->bufpos = saved_input->buf + (input.bufpos - input.buf);
 181         if (input.bufend != NULL)
 182                 saved_input->bufend = saved_input->buf + (input.bufend - input.buf);
 183
 184         saved_input->parent = input_stack;
 185         input_stack         = saved_input;
 186         ++n_inputs;
 187 }
 188
 189 static void pop_restore_input(void)
 190 {
 191         assert(n_inputs > 0);
 192         assert(input_stack != NULL);
 193
 194         pp_input_t *saved_input = input_stack;
 195
 196         memcpy(&input, saved_input, sizeof(input));
 197         input.parent = NULL;
 198
 199         /* adjust buffer positions */
 200         if (saved_input->bufpos != NULL)
 201                 input.bufpos = input.buf + (saved_input->bufpos - saved_input->buf);
 202         if (saved_input->bufend != NULL)
 203                 input.bufend = input.buf + (saved_input->bufend - saved_input->buf);
 204
 205         input_stack = saved_input->parent;
 206         obstack_free(&input_obstack, saved_input);
 207         --n_inputs;
 208 }
 209
 210 /**
 211  * Prints a parse error message at the current token.
 212  *
 213  * @param msg   the error message
 214  */
 215 static void parse_error(const char *msg)
 216 {
 217         errorf(&pp_token.base.source_position,  "%s", msg);
 218 }
 219
 220 static inline void next_real_char(void)
 221 {
 222         assert(input.bufpos <= input.bufend);
 223         if (input.bufpos >= input.bufend) {
 224                 size_t const n = decode(input.input, input.buf + MAX_PUTBACK, lengthof(input.buf) - MAX_PUTBACK);
 225                 if (n == 0) {
 226                         input.c = EOF;
 227                         return;
 228                 }
 229                 input.bufpos = input.buf + MAX_PUTBACK;
 230                 input.bufend = input.bufpos + n;
 231         }
 232         input.c = *input.bufpos++;
 233         ++input.position.colno;
 234 }
 235
 236 /**
 237  * Put a character back into the buffer.
 238  *
 239  * @param pc  the character to put back
 240  */
 241 static inline void put_back(utf32 const pc)
 242 {
 243         assert(input.bufpos > input.buf);
 244         *(--input.bufpos - input.buf + input.buf) = (char) pc;
 245         --input.position.colno;
 246 }
 247
 248 #define NEWLINE \
 249         '\r': \
 250                 next_char(); \
 251                 if (input.c == '\n') { \
 252         case '\n': \
 253                         next_char(); \
 254                 } \
 255                 ++input.position.lineno; \
 256                 input.position.colno = 1; \
 257                 goto newline; \
 258                 newline // Let it look like an ordinary case label.
 259
 260 #define eat(c_type) (assert(input.c == c_type), next_char())
 261
 262 static void maybe_concat_lines(void)
 263 {
 264         eat('\\');
 265
 266         switch (input.c) {
 267         case NEWLINE:
 268                 info.whitespace_at_line_begin = 0;
 269                 return;
 270
 271         default:
 272                 break;
 273         }
 274
 275         put_back(input.c);
 276         input.c = '\\';
 277 }
 278
 279 /**
 280  * Set c to the next input character, ie.
 281  * after expanding trigraphs.
 282  */
 283 static inline void next_char(void)
 284 {
 285         next_real_char();
 286
 287         /* filter trigraphs and concatenated lines */
 288         if (UNLIKELY(input.c == '\\')) {
 289                 maybe_concat_lines();
 290                 goto end_of_next_char;
 291         }
 292
 293         if (LIKELY(input.c != '?'))
 294                 goto end_of_next_char;
 295
 296         next_real_char();
 297         if (LIKELY(input.c != '?')) {
 298                 put_back(input.c);
 299                 input.c = '?';
 300                 goto end_of_next_char;
 301         }
 302
 303         next_real_char();
 304         switch (input.c) {
 305         case '=': input.c = '#'; break;
 306         case '(': input.c = '['; break;
 307         case '/': input.c = '\\'; maybe_concat_lines(); break;
 308         case ')': input.c = ']'; break;
 309         case '\'': input.c = '^'; break;
 310         case '<': input.c = '{'; break;
 311         case '!': input.c = '|'; break;
 312         case '>': input.c = '}'; break;
 313         case '-': input.c = '~'; break;
 314         default:
 315                 put_back(input.c);
 316                 put_back('?');
 317                 input.c = '?';
 318                 break;
 319         }
 320
 321 end_of_next_char:;
 322 #ifdef DEBUG_CHARS
 323         printf("nchar '%c'\n", input.c);
 324 #endif
 325 }
 326
 327
 328
 329 /**
 330  * Returns true if the given char is a octal digit.
 331  *
 332  * @param char  the character to check
 333  */
 334 static inline bool is_octal_digit(int chr)
 335 {
 336         switch (chr) {
 337         case '0':
 338         case '1':
 339         case '2':
 340         case '3':
 341         case '4':
 342         case '5':
 343         case '6':
 344         case '7':
 345                 return true;
 346         default:
 347                 return false;
 348         }
 349 }
 350
 351 /**
 352  * Returns the value of a digit.
 353  * The only portable way to do it ...
 354  */
 355 static int digit_value(int digit)
 356 {
 357         switch (digit) {
 358         case '0': return 0;
 359         case '1': return 1;
 360         case '2': return 2;
 361         case '3': return 3;
 362         case '4': return 4;
 363         case '5': return 5;
 364         case '6': return 6;
 365         case '7': return 7;
 366         case '8': return 8;
 367         case '9': return 9;
 368         case 'a':
 369         case 'A': return 10;
 370         case 'b':
 371         case 'B': return 11;
 372         case 'c':
 373         case 'C': return 12;
 374         case 'd':
 375         case 'D': return 13;
 376         case 'e':
 377         case 'E': return 14;
 378         case 'f':
 379         case 'F': return 15;
 380         default:
 381                 panic("wrong character given");
 382         }
 383 }
 384
 385 /**
 386  * Parses an octal character sequence.
 387  *
 388  * @param first_digit  the already read first digit
 389  */
 390 static utf32 parse_octal_sequence(const utf32 first_digit)
 391 {
 392         assert(is_octal_digit(first_digit));
 393         utf32 value = digit_value(first_digit);
 394         if (!is_octal_digit(input.c)) return value;
 395         value = 8 * value + digit_value(input.c);
 396         next_char();
 397         if (!is_octal_digit(input.c)) return value;
 398         value = 8 * value + digit_value(input.c);
 399         next_char();
 400         return value;
 401
 402 }
 403
 404 /**
 405  * Parses a hex character sequence.
 406  */
 407 static utf32 parse_hex_sequence(void)
 408 {
 409         utf32 value = 0;
 410         while (isxdigit(input.c)) {
 411                 value = 16 * value + digit_value(input.c);
 412                 next_char();
 413         }
 414         return value;
 415 }
 416
 417 static bool is_universal_char_valid(utf32 const v)
 418 {
 419         /* C11 §6.4.3:2 */
 420         if (v < 0xA0U && v != 0x24 && v != 0x40 && v != 0x60)
 421                 return false;
 422         if (0xD800 <= v && v <= 0xDFFF)
 423                 return false;
 424         return true;
 425 }
 426
 427 static utf32 parse_universal_char(unsigned const n_digits)
 428 {
 429         utf32 v = 0;
 430         for (unsigned k = n_digits; k != 0; --k) {
 431                 if (isxdigit(input.c)) {
 432                         v = 16 * v + digit_value(input.c);
 433                         if (!resolve_escape_sequences)
 434                                 obstack_1grow(&symbol_obstack, input.c);
 435                         next_char();
 436                 } else {
 437                         errorf(&input.position,
 438                                "short universal character name, expected %u more digits",
 439                                    k);
 440                         break;
 441                 }
 442         }
 443         if (!is_universal_char_valid(v)) {
 444                 errorf(&input.position,
 445                        "\\%c%0*X is not a valid universal character name",
 446                        n_digits == 4 ? 'u' : 'U', (int)n_digits, v);
 447         }
 448         return v;
 449 }
 450
 451 static bool is_universal_char_valid_identifier(utf32 const v)
 452 {
 453         /* C11 Annex D.1 */
 454         if (                v == 0x000A8) return true;
 455         if (                v == 0x000AA) return true;
 456         if (                v == 0x000AD) return true;
 457         if (                v == 0x000AF) return true;
 458         if (0x000B2 <= v && v <= 0x000B5) return true;
 459         if (0x000B7 <= v && v <= 0x000BA) return true;
 460         if (0x000BC <= v && v <= 0x000BE) return true;
 461         if (0x000C0 <= v && v <= 0x000D6) return true;
 462         if (0x000D8 <= v && v <= 0x000F6) return true;
 463         if (0x000F8 <= v && v <= 0x000FF) return true;
 464         if (0x00100 <= v && v <= 0x0167F) return true;
 465         if (0x01681 <= v && v <= 0x0180D) return true;
 466         if (0x0180F <= v && v <= 0x01FFF) return true;
 467         if (0x0200B <= v && v <= 0x0200D) return true;
 468         if (0x0202A <= v && v <= 0x0202E) return true;
 469         if (0x0203F <= v && v <= 0x02040) return true;
 470         if (                v == 0x02054) return true;
 471         if (0x02060 <= v && v <= 0x0206F) return true;
 472         if (0x02070 <= v && v <= 0x0218F) return true;
 473         if (0x02460 <= v && v <= 0x024FF) return true;
 474         if (0x02776 <= v && v <= 0x02793) return true;
 475         if (0x02C00 <= v && v <= 0x02DFF) return true;
 476         if (0x02E80 <= v && v <= 0x02FFF) return true;
 477         if (0x03004 <= v && v <= 0x03007) return true;
 478         if (0x03021 <= v && v <= 0x0302F) return true;
 479         if (0x03031 <= v && v <= 0x0303F) return true;
 480         if (0x03040 <= v && v <= 0x0D7FF) return true;
 481         if (0x0F900 <= v && v <= 0x0FD3D) return true;
 482         if (0x0FD40 <= v && v <= 0x0FDCF) return true;
 483         if (0x0FDF0 <= v && v <= 0x0FE44) return true;
 484         if (0x0FE47 <= v && v <= 0x0FFFD) return true;
 485         if (0x10000 <= v && v <= 0x1FFFD) return true;
 486         if (0x20000 <= v && v <= 0x2FFFD) return true;
 487         if (0x30000 <= v && v <= 0x3FFFD) return true;
 488         if (0x40000 <= v && v <= 0x4FFFD) return true;
 489         if (0x50000 <= v && v <= 0x5FFFD) return true;
 490         if (0x60000 <= v && v <= 0x6FFFD) return true;
 491         if (0x70000 <= v && v <= 0x7FFFD) return true;
 492         if (0x80000 <= v && v <= 0x8FFFD) return true;
 493         if (0x90000 <= v && v <= 0x9FFFD) return true;
 494         if (0xA0000 <= v && v <= 0xAFFFD) return true;
 495         if (0xB0000 <= v && v <= 0xBFFFD) return true;
 496         if (0xC0000 <= v && v <= 0xCFFFD) return true;
 497         if (0xD0000 <= v && v <= 0xDFFFD) return true;
 498         if (0xE0000 <= v && v <= 0xEFFFD) return true;
 499         return false;
 500 }
 501
 502 static bool is_universal_char_valid_identifier_start(utf32 const v)
 503 {
 504         /* C11 Annex D.2 */
 505         if (0x0300 <= v && v <= 0x036F) return false;
 506         if (0x1DC0 <= v && v <= 0x1DFF) return false;
 507         if (0x20D0 <= v && v <= 0x20FF) return false;
 508         if (0xFE20 <= v && v <= 0xFE2F) return false;
 509         return true;
 510 }
 511
 512 /**
 513  * Parse an escape sequence.
 514  */
 515 static utf32 parse_escape_sequence(void)
 516 {
 517         eat('\\');
 518
 519         utf32 const ec = input.c;
 520         next_char();
 521
 522         switch (ec) {
 523         case '"':  return '"';
 524         case '\'': return '\'';
 525         case '\\': return '\\';
 526         case '?': return '\?';
 527         case 'a': return '\a';
 528         case 'b': return '\b';
 529         case 'f': return '\f';
 530         case 'n': return '\n';
 531         case 'r': return '\r';
 532         case 't': return '\t';
 533         case 'v': return '\v';
 534         case 'x':
 535                 return parse_hex_sequence();
 536         case '0':
 537         case '1':
 538         case '2':
 539         case '3':
 540         case '4':
 541         case '5':
 542         case '6':
 543         case '7':
 544                 return parse_octal_sequence(ec);
 545         case EOF:
 546                 parse_error("reached end of file while parsing escape sequence");
 547                 return EOF;
 548         /* \E is not documented, but handled, by GCC.  It is acceptable according
 549          * to §6.11.4, whereas \e is not. */
 550         case 'E':
 551         case 'e':
 552                 if (c_mode & _GNUC)
 553                         return 27;   /* hopefully 27 is ALWAYS the code for ESCAPE */
 554                 break;
 555
 556         case 'U': return parse_universal_char(8);
 557         case 'u': return parse_universal_char(4);
 558
 559         default:
 560                 break;
 561         }
 562         /* §6.4.4.4:8 footnote 64 */
 563         parse_error("unknown escape sequence");
 564         return EOF;
 565 }
 566
 567 static const char *identify_string(char *string)
 568 {
 569         const char *result = strset_insert(&stringset, string);
 570         if (result != string) {
 571                 obstack_free(&symbol_obstack, string);
 572         }
 573         return result;
 574 }
 575
 576 static string_t sym_make_string(string_encoding_t const enc)
 577 {
 578         obstack_1grow(&symbol_obstack, '\0');
 579         size_t      const len    = obstack_object_size(&symbol_obstack) - 1;
 580         char       *const string = obstack_finish(&symbol_obstack);
 581         char const *const result = identify_string(string);
 582         return (string_t){ result, len, enc };
 583 }
 584
 585 string_t make_string(char const *const string)
 586 {
 587         obstack_grow(&symbol_obstack, string, strlen(string));
 588         return sym_make_string(STRING_ENCODING_CHAR);
 589 }
 590
 591 static void parse_string(utf32 const delimiter, token_kind_t const kind,
 592                          string_encoding_t const enc,
 593                          char const *const context)
 594 {
 595         const unsigned start_linenr = input.position.lineno;
 596
 597         eat(delimiter);
 598
 599         while (true) {
 600                 switch (input.c) {
 601                 case '\\': {
 602                         if (resolve_escape_sequences) {
 603                                 utf32 const tc = parse_escape_sequence();
 604                                 if (enc == STRING_ENCODING_CHAR) {
 605                                         if (tc >= 0x100) {
 606                                                 warningf(WARN_OTHER, &pp_token.base.source_position, "escape sequence out of range");
 607                                         }
 608                                         obstack_1grow(&symbol_obstack, tc);
 609                                 } else {
 610                                         obstack_grow_utf8(&symbol_obstack, tc);
 611                                 }
 612                         } else {
 613                                 obstack_1grow(&symbol_obstack, (char)input.c);
 614                                 next_char();
 615                                 obstack_1grow(&symbol_obstack, (char)input.c);
 616                                 next_char();
 617                         }
 618                         break;
 619                 }
 620
 621                 case NEWLINE:
 622                         errorf(&pp_token.base.source_position, "newline while parsing %s", context);
 623                         break;
 624
 625                 case EOF: {
 626                         source_position_t source_position;
 627                         source_position.input_name = pp_token.base.source_position.input_name;
 628                         source_position.lineno     = start_linenr;
 629                         errorf(&source_position, "EOF while parsing %s", context);
 630                         goto end_of_string;
 631                 }
 632
 633                 default:
 634                         if (input.c == delimiter) {
 635                                 next_char();
 636                                 goto end_of_string;
 637                         } else {
 638                                 obstack_grow_utf8(&symbol_obstack, input.c);
 639                                 next_char();
 640                                 break;
 641                         }
 642                 }
 643         }
 644
 645 end_of_string:
 646         pp_token.kind           = kind;
 647         pp_token.literal.string = sym_make_string(enc);
 648 }
 649
 650 static void parse_string_literal(string_encoding_t const enc)
 651 {
 652         parse_string('"', T_STRING_LITERAL, enc, "string literal");
 653 }
 654
 655 static void parse_character_constant(string_encoding_t const enc)
 656 {
 657         parse_string('\'', T_CHARACTER_CONSTANT, enc, "character constant");
 658         if (pp_token.literal.string.size == 0) {
 659                 parse_error("empty character constant");
 660         }
 661 }
 662
 663 #define SYMBOL_CASES_WITHOUT_E_P \
 664              '$': if (!allow_dollar_in_symbol) goto dollar_sign; \
 665         case 'a': \
 666         case 'b': \
 667         case 'c': \
 668         case 'd': \
 669         case 'f': \
 670         case 'g': \
 671         case 'h': \
 672         case 'i': \
 673         case 'j': \
 674         case 'k': \
 675         case 'l': \
 676         case 'm': \
 677         case 'n': \
 678         case 'o': \
 679         case 'q': \
 680         case 'r': \
 681         case 's': \
 682         case 't': \
 683         case 'u': \
 684         case 'v': \
 685         case 'w': \
 686         case 'x': \
 687         case 'y': \
 688         case 'z': \
 689         case 'A': \
 690         case 'B': \
 691         case 'C': \
 692         case 'D': \
 693         case 'F': \
 694         case 'G': \
 695         case 'H': \
 696         case 'I': \
 697         case 'J': \
 698         case 'K': \
 699         case 'L': \
 700         case 'M': \
 701         case 'N': \
 702         case 'O': \
 703         case 'Q': \
 704         case 'R': \
 705         case 'S': \
 706         case 'T': \
 707         case 'U': \
 708         case 'V': \
 709         case 'W': \
 710         case 'X': \
 711         case 'Y': \
 712         case 'Z': \
 713         case '_'
 714
 715 #define SYMBOL_CASES \
 716              SYMBOL_CASES_WITHOUT_E_P: \
 717         case 'e': \
 718         case 'p': \
 719         case 'E': \
 720         case 'P'
 721
 722 #define DIGIT_CASES \
 723              '0':  \
 724         case '1':  \
 725         case '2':  \
 726         case '3':  \
 727         case '4':  \
 728         case '5':  \
 729         case '6':  \
 730         case '7':  \
 731         case '8':  \
 732         case '9'
 733
 734 static void start_expanding(pp_definition_t *definition)
 735 {
 736         definition->parent_expansion = current_expansion;
 737         definition->expand_pos       = 0;
 738         definition->is_expanding     = true;
 739         if (definition->list_len > 0) {
 740                 definition->token_list[0].had_whitespace
 741                         = info.had_whitespace;
 742         }
 743         current_expansion = definition;
 744 }
 745
 746 static void finished_expanding(pp_definition_t *definition)
 747 {
 748         assert(definition->is_expanding);
 749         pp_definition_t *parent = definition->parent_expansion;
 750         definition->parent_expansion = NULL;
 751         definition->is_expanding     = false;
 752
 753         /* stop further expanding once we expanded a parameter used in a
 754          * sub macro-call */
 755         if (definition == argument_expanding)
 756                 argument_expanding = NULL;
 757
 758         assert(current_expansion == definition);
 759         current_expansion = parent;
 760 }
 761
 762 static void grow_string_escaped(struct obstack *obst, const string_t *string, char const *delimiter)
 763 {
 764         char const *prefix = get_string_encoding_prefix(string->encoding);
 765         obstack_printf(obst, "%s%s", prefix, delimiter);
 766         size_t      size = string->size;
 767         const char *str  = string->begin;
 768         if (resolve_escape_sequences) {
 769                 obstack_grow(obst, str, size);
 770         } else {
 771                 for (size_t i = 0; i < size; ++i) {
 772                         const char c = str[i];
 773                         if (c == '\\' || c == '"')
 774                                 obstack_1grow(obst, '\\');
 775                         obstack_1grow(obst, c);
 776                 }
 777         }
 778         obstack_printf(obst, "%s", delimiter);
 779 }
 780
 781 static void grow_token(struct obstack *obst, const token_t *token)
 782 {
 783         switch (token->kind) {
 784         case T_NUMBER:
 785                 obstack_grow(obst, token->literal.string.begin, token->literal.string.size);
 786                 break;
 787
 788         case T_STRING_LITERAL: {
 789                 char const *const delimiter = resolve_escape_sequences ? "\"" : "\\\"";
 790                 grow_string_escaped(obst, &token->literal.string, delimiter);
 791                 break;
 792         }
 793
 794         case T_CHARACTER_CONSTANT:
 795                 grow_string_escaped(obst, &token->literal.string, "'");
 796                 break;
 797
 798         case T_IDENTIFIER:
 799         default: {
 800                 const char *str = token->base.symbol->string;
 801                 size_t      len = strlen(str);
 802                 obstack_grow(obst, str, len);
 803                 break;
 804         }
 805         }
 806 }
 807
 808 static void stringify(const pp_definition_t *definition)
 809 {
 810         assert(obstack_object_size(&symbol_obstack) == 0);
 811
 812         size_t list_len = definition->list_len;
 813         for (size_t p = 0; p < list_len; ++p) {
 814                 const saved_token_t *saved = &definition->token_list[p];
 815                 if (p > 0 && saved->had_whitespace)
 816                         obstack_1grow(&symbol_obstack, ' ');
 817                 grow_token(&symbol_obstack, &saved->token);
 818         }
 819         pp_token.kind           = T_STRING_LITERAL;
 820         pp_token.literal.string = sym_make_string(STRING_ENCODING_CHAR);
 821 }
 822
 823 static inline void set_punctuator(token_kind_t const kind)
 824 {
 825         pp_token.kind        = kind;
 826         pp_token.base.symbol = token_symbols[kind];
 827 }
 828
 829 static inline void set_digraph(token_kind_t const kind, symbol_t *const symbol)
 830 {
 831         pp_token.kind        = kind;
 832         pp_token.base.symbol = symbol;
 833 }
 834
 835 /**
 836  * returns next final token from a preprocessor macro expansion
 837  */
 838 static bool expand_next(void)
 839 {
 840         if (current_expansion == NULL)
 841                 return false;
 842
 843 restart:;
 844         size_t pos = current_expansion->expand_pos;
 845         if (pos >= current_expansion->list_len) {
 846                 finished_expanding(current_expansion);
 847                 /* it was the outermost expansion, parse pptoken normally */
 848                 if (current_expansion == NULL) {
 849                         return false;
 850                 }
 851                 goto restart;
 852         }
 853         const saved_token_t *saved = &current_expansion->token_list[pos++];
 854         pp_token = saved->token;
 855         if (pp_token.kind == '#') {
 856                 if (pos < current_expansion->list_len) {
 857                         const saved_token_t *next = &current_expansion->token_list[pos];
 858                         if (next->token.kind == T_MACRO_PARAMETER) {
 859                                 pp_definition_t *def = next->token.macro_parameter.def;
 860                                 assert(def != NULL && def->is_parameter);
 861                                 stringify(def);
 862                                 ++pos;
 863                         }
 864                 }
 865         }
 866
 867         if (current_expansion->expand_pos > 0)
 868                 info.had_whitespace = saved->had_whitespace;
 869         current_expansion->expand_pos = pos;
 870         pp_token.base.source_position = expansion_pos;
 871
 872         return true;
 873 }
 874
 875 /**
 876  * Returns the next token kind found when continuing the current expansions
 877  * without starting new sub-expansions.
 878  */
 879 static token_kind_t peek_expansion(void)
 880 {
 881         for (pp_definition_t *e = current_expansion; e; e = e->parent_expansion) {
 882                 if (e->expand_pos < e->list_len)
 883                         return e->token_list[e->expand_pos].token.kind;
 884         }
 885         return T_EOF;
 886 }
 887
 888 static void skip_line_comment(void)
 889 {
 890         info.had_whitespace = true;
 891         while (true) {
 892                 switch (input.c) {
 893                 case EOF:
 894                         return;
 895
 896                 case '\r':
 897                 case '\n':
 898                         return;
 899
 900                 default:
 901                         next_char();
 902                         break;
 903                 }
 904         }
 905 }
 906
 907 static void skip_multiline_comment(void)
 908 {
 909         info.had_whitespace = true;
 910
 911         unsigned start_linenr = input.position.lineno;
 912         while (true) {
 913                 switch (input.c) {
 914                 case '/':
 915                         next_char();
 916                         if (input.c == '*') {
 917                                 /* TODO: nested comment, warn here */
 918                         }
 919                         break;
 920                 case '*':
 921                         next_char();
 922                         if (input.c == '/') {
 923                                 if (input.position.lineno != input.output_line)
 924                                         info.whitespace_at_line_begin = input.position.colno;
 925                                 next_char();
 926                                 return;
 927                         }
 928                         break;
 929
 930                 case NEWLINE:
 931                         break;
 932
 933                 case EOF: {
 934                         source_position_t source_position;
 935                         source_position.input_name = pp_token.base.source_position.input_name;
 936                         source_position.lineno     = start_linenr;
 937                         errorf(&source_position, "at end of file while looking for comment end");
 938                         return;
 939                 }
 940
 941                 default:
 942                         next_char();
 943                         break;
 944                 }
 945         }
 946 }
 947
 948 static bool skip_till_newline(bool stop_at_non_whitespace)
 949 {
 950         bool res = false;
 951         while (true) {
 952                 switch (input.c) {
 953                 case ' ':
 954                 case '\t':
 955                         next_char();
 956                         continue;
 957
 958                 case '/':
 959                         next_char();
 960                         if (input.c == '/') {
 961                                 next_char();
 962                                 skip_line_comment();
 963                                 continue;
 964                         } else if (input.c == '*') {
 965                                 next_char();
 966                                 skip_multiline_comment();
 967                                 continue;
 968                         } else {
 969                                 put_back(input.c);
 970                                 input.c = '/';
 971                         }
 972                         return true;
 973
 974                 case NEWLINE:
 975                         return res;
 976
 977                 default:
 978                         if (stop_at_non_whitespace)
 979                                 return false;
 980                         res = true;
 981                         next_char();
 982                         continue;
 983                 }
 984         }
 985 }
 986
 987 static void skip_whitespace(void)
 988 {
 989         while (true) {
 990                 switch (input.c) {
 991                 case ' ':
 992                 case '\t':
 993                         ++info.whitespace_at_line_begin;
 994                         info.had_whitespace = true;
 995                         next_char();
 996                         continue;
 997
 998                 case NEWLINE:
 999                         info.at_line_begin  = true;
1000                         info.had_whitespace = true;
1001                         info.whitespace_at_line_begin = 0;
1002                         continue;
1003
1004                 case '/':
1005                         next_char();
1006                         if (input.c == '/') {
1007                                 next_char();
1008                                 skip_line_comment();
1009                                 continue;
1010                         } else if (input.c == '*') {
1011                                 next_char();
1012                                 skip_multiline_comment();
1013                                 continue;
1014                         } else {
1015                                 put_back(input.c);
1016                                 input.c = '/';
1017                         }
1018                         return;
1019
1020                 default:
1021                         return;
1022                 }
1023         }
1024 }
1025
1026 static inline void eat_pp(pp_token_kind_t const kind)
1027 {
1028         assert(pp_token.base.symbol->pp_ID == kind);
1029         (void) kind;
1030         next_input_token();
1031 }
1032
1033 static inline void eat_token(token_kind_t const kind)
1034 {
1035         assert(pp_token.kind == kind);
1036         (void)kind;
1037         next_input_token();
1038 }
1039
1040 static void parse_symbol(void)
1041 {
1042         assert(obstack_object_size(&symbol_obstack) == 0);
1043         while (true) {
1044                 switch (input.c) {
1045                 case DIGIT_CASES:
1046                 case SYMBOL_CASES:
1047                         obstack_1grow(&symbol_obstack, (char) input.c);
1048                         next_char();
1049                         break;
1050
1051                 case '\\':
1052                         next_char();
1053                         switch (input.c) {
1054                         {
1055                                 unsigned n;
1056                         case 'U': n = 8; goto universal;
1057                         case 'u': n = 4; goto universal;
1058 universal:
1059                                 if (!resolve_escape_sequences) {
1060                                         obstack_1grow(&symbol_obstack, '\\');
1061                                         obstack_1grow(&symbol_obstack, input.c);
1062                                 }
1063                                 next_char();
1064                                 utf32 const v = parse_universal_char(n);
1065                                 if (!is_universal_char_valid_identifier(v)) {
1066                                         if (is_universal_char_valid(v)) {
1067                                                 errorf(&input.position,
1068                                                            "universal character \\%c%0*X is not valid in an identifier",
1069                                                            n == 4 ? 'u' : 'U', (int)n, v);
1070                                         }
1071                                 } else if (obstack_object_size(&symbol_obstack) == 0 && !is_universal_char_valid_identifier_start(v)) {
1072                                         errorf(&input.position,
1073                                                    "universal character \\%c%0*X is not valid as start of an identifier",
1074                                                    n == 4 ? 'u' : 'U', (int)n, v);
1075                                 } else if (resolve_escape_sequences) {
1076                                         obstack_grow_utf8(&symbol_obstack, v);
1077                                 }
1078                                 break;
1079                         }
1080
1081                         default:
1082                                 put_back(input.c);
1083                                 input.c = '\\';
1084                                 goto end_symbol;
1085                         }
1086
1087                 default:
1088 dollar_sign:
1089                         goto end_symbol;
1090                 }
1091         }
1092
1093 end_symbol:
1094         obstack_1grow(&symbol_obstack, '\0');
1095         char *string = obstack_finish(&symbol_obstack);
1096
1097         /* might be a wide string or character constant ( L"string"/L'c' ) */
1098         if (input.c == '"' && string[0] == 'L' && string[1] == '\0') {
1099                 obstack_free(&symbol_obstack, string);
1100                 parse_string_literal(STRING_ENCODING_WIDE);
1101                 return;
1102         } else if (input.c == '\'' && string[0] == 'L' && string[1] == '\0') {
1103                 obstack_free(&symbol_obstack, string);
1104                 parse_character_constant(STRING_ENCODING_WIDE);
1105                 return;
1106         }
1107
1108         symbol_t *symbol = symbol_table_insert(string);
1109
1110         pp_token.kind        = symbol->ID;
1111         pp_token.base.symbol = symbol;
1112
1113         /* we can free the memory from symbol obstack if we already had an entry in
1114          * the symbol table */
1115         if (symbol->string != string) {
1116                 obstack_free(&symbol_obstack, string);
1117         }
1118 }
1119
1120 static void parse_number(void)
1121 {
1122         obstack_1grow(&symbol_obstack, (char) input.c);
1123         next_char();
1124
1125         while (true) {
1126                 switch (input.c) {
1127                 case '.':
1128                 case DIGIT_CASES:
1129                 case SYMBOL_CASES_WITHOUT_E_P:
1130                         obstack_1grow(&symbol_obstack, (char) input.c);
1131                         next_char();
1132                         break;
1133
1134                 case 'e':
1135                 case 'p':
1136                 case 'E':
1137                 case 'P':
1138                         obstack_1grow(&symbol_obstack, (char) input.c);
1139                         next_char();
1140                         if (input.c == '+' || input.c == '-') {
1141                                 obstack_1grow(&symbol_obstack, (char) input.c);
1142                                 next_char();
1143                         }
1144                         break;
1145
1146                 default:
1147 dollar_sign:
1148                         goto end_number;
1149                 }
1150         }
1151
1152 end_number:
1153         pp_token.kind           = T_NUMBER;
1154         pp_token.literal.string = sym_make_string(STRING_ENCODING_CHAR);
1155 }
1156
1157 #define MAYBE_PROLOG \
1158         next_char(); \
1159         switch (input.c) {
1160
1161 #define MAYBE(ch, kind) \
1162         case ch: \
1163                 next_char(); \
1164                 set_punctuator(kind); \
1165                 return;
1166
1167 #define MAYBE_DIGRAPH(ch, kind, symbol) \
1168         case ch: \
1169                 next_char(); \
1170                 set_digraph(kind, symbol); \
1171                 return;
1172
1173 #define ELSE_CODE(code) \
1174         default: \
1175                 code \
1176         }
1177
1178 #define ELSE(kind) ELSE_CODE(set_punctuator(kind); return;)
1179
1180 /** identifies and returns the next preprocessing token contained in the
1181  * input stream. No macro expansion is performed. */
1182 static void next_input_token(void)
1183 {
1184         if (next_info.had_whitespace) {
1185                 info = next_info;
1186                 next_info.had_whitespace = false;
1187         } else {
1188                 info.at_line_begin  = false;
1189                 info.had_whitespace = false;
1190         }
1191 restart:
1192         pp_token.base.source_position = input.position;
1193         pp_token.base.symbol          = NULL;
1194
1195         switch (input.c) {
1196         case ' ':
1197         case '\t':
1198                 info.whitespace_at_line_begin++;
1199                 info.had_whitespace = true;
1200                 next_char();
1201                 goto restart;
1202
1203         case NEWLINE:
1204                 info.at_line_begin            = true;
1205                 info.had_whitespace           = true;
1206                 info.whitespace_at_line_begin = 0;
1207                 goto restart;
1208
1209         case SYMBOL_CASES:
1210                 parse_symbol();
1211                 return;
1212
1213         case DIGIT_CASES:
1214                 parse_number();
1215                 return;
1216
1217         case '"':
1218                 parse_string_literal(STRING_ENCODING_CHAR);
1219                 return;
1220
1221         case '\'':
1222                 parse_character_constant(STRING_ENCODING_CHAR);
1223                 return;
1224
1225         case '.':
1226                 MAYBE_PROLOG
1227                         case '0':
1228                         case '1':
1229                         case '2':
1230                         case '3':
1231                         case '4':
1232                         case '5':
1233                         case '6':
1234                         case '7':
1235                         case '8':
1236                         case '9':
1237                                 put_back(input.c);
1238                                 input.c = '.';
1239                                 parse_number();
1240                                 return;
1241
1242                         case '.':
1243                                 MAYBE_PROLOG
1244                                 MAYBE('.', T_DOTDOTDOT)
1245                                 ELSE_CODE(
1246                                         put_back(input.c);
1247                                         input.c = '.';
1248                                         set_punctuator('.');
1249                                         return;
1250                                 )
1251                 ELSE('.')
1252         case '&':
1253                 MAYBE_PROLOG
1254                 MAYBE('&', T_ANDAND)
1255                 MAYBE('=', T_ANDEQUAL)
1256                 ELSE('&')
1257         case '*':
1258                 MAYBE_PROLOG
1259                 MAYBE('=', T_ASTERISKEQUAL)
1260                 ELSE('*')
1261         case '+':
1262                 MAYBE_PROLOG
1263                 MAYBE('+', T_PLUSPLUS)
1264                 MAYBE('=', T_PLUSEQUAL)
1265                 ELSE('+')
1266         case '-':
1267                 MAYBE_PROLOG
1268                 MAYBE('>', T_MINUSGREATER)
1269                 MAYBE('-', T_MINUSMINUS)
1270                 MAYBE('=', T_MINUSEQUAL)
1271                 ELSE('-')
1272         case '!':
1273                 MAYBE_PROLOG
1274                 MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1275                 ELSE('!')
1276         case '/':
1277                 MAYBE_PROLOG
1278                 MAYBE('=', T_SLASHEQUAL)
1279                 case '*':
1280                         next_char();
1281                         skip_multiline_comment();
1282                         goto restart;
1283                 case '/':
1284                         next_char();
1285                         skip_line_comment();
1286                         goto restart;
1287                 ELSE('/')
1288         case '%':
1289                 MAYBE_PROLOG
1290                 MAYBE_DIGRAPH('>', '}', symbol_percentgreater)
1291                 MAYBE('=', T_PERCENTEQUAL)
1292                 case ':':
1293                         MAYBE_PROLOG
1294                         case '%':
1295                                 MAYBE_PROLOG
1296                                 MAYBE_DIGRAPH(':', T_HASHHASH, symbol_percentcolonpercentcolon)
1297                                 ELSE_CODE(
1298                                         put_back(input.c);
1299                                         input.c = '%';
1300                                         goto digraph_percentcolon;
1301                                 )
1302                         ELSE_CODE(
1303 digraph_percentcolon:
1304                                 set_digraph('#', symbol_percentcolon);
1305                                 return;
1306                         )
1307                 ELSE('%')
1308         case '<':
1309                 MAYBE_PROLOG
1310                 MAYBE_DIGRAPH(':', '[', symbol_lesscolon)
1311                 MAYBE_DIGRAPH('%', '{', symbol_lesspercent)
1312                 MAYBE('=', T_LESSEQUAL)
1313                 case '<':
1314                         MAYBE_PROLOG
1315                         MAYBE('=', T_LESSLESSEQUAL)
1316                         ELSE(T_LESSLESS)
1317                 ELSE('<')
1318         case '>':
1319                 MAYBE_PROLOG
1320                 MAYBE('=', T_GREATEREQUAL)
1321                 case '>':
1322                         MAYBE_PROLOG
1323                         MAYBE('=', T_GREATERGREATEREQUAL)
1324                         ELSE(T_GREATERGREATER)
1325                 ELSE('>')
1326         case '^':
1327                 MAYBE_PROLOG
1328                 MAYBE('=', T_CARETEQUAL)
1329                 ELSE('^')
1330         case '|':
1331                 MAYBE_PROLOG
1332                 MAYBE('=', T_PIPEEQUAL)
1333                 MAYBE('|', T_PIPEPIPE)
1334                 ELSE('|')
1335         case ':':
1336                 MAYBE_PROLOG
1337                 MAYBE_DIGRAPH('>', ']', symbol_colongreater)
1338                 case ':':
1339                         if (c_mode & _CXX) {
1340                                 next_char();
1341                                 set_punctuator(T_COLONCOLON);
1342                                 return;
1343                         }
1344                         /* FALLTHROUGH */
1345                 ELSE(':')
1346         case '=':
1347                 MAYBE_PROLOG
1348                 MAYBE('=', T_EQUALEQUAL)
1349                 ELSE('=')
1350         case '#':
1351                 MAYBE_PROLOG
1352                 MAYBE('#', T_HASHHASH)
1353                 ELSE('#')
1354
1355         case '?':
1356         case '[':
1357         case ']':
1358         case '(':
1359         case ')':
1360         case '{':
1361         case '}':
1362         case '~':
1363         case ';':
1364         case ',':
1365                 set_punctuator(input.c);
1366                 next_char();
1367                 return;
1368
1369         case EOF:
1370                 if (input_stack != NULL) {
1371                         fclose(close_pp_input());
1372                         pop_restore_input();
1373                         fputc('\n', out);
1374                         if (input.c == (utf32)EOF)
1375                                 --input.position.lineno;
1376                         print_line_directive(&input.position, "2");
1377                         goto restart;
1378                 } else {
1379                         info.at_line_begin = true;
1380                         set_punctuator(T_EOF);
1381                 }
1382                 return;
1383
1384         case '\\':
1385                 next_char();
1386                 int next_c = input.c;
1387                 put_back(input.c);
1388                 input.c = '\\';
1389                 if (next_c == 'U' || next_c == 'u') {
1390                         parse_symbol();
1391                         return;
1392                 }
1393                 /* FALLTHROUGH */
1394         default:
1395 dollar_sign:
1396                 if (error_on_unknown_chars) {
1397                         errorf(&pp_token.base.source_position,
1398                                "unknown character '%lc' found\n", input.c);
1399                         next_char();
1400                         goto restart;
1401                 } else {
1402                         assert(obstack_object_size(&symbol_obstack) == 0);
1403                         obstack_grow_utf8(&symbol_obstack, input.c);
1404                         obstack_1grow(&symbol_obstack, '\0');
1405                         char     *const string = obstack_finish(&symbol_obstack);
1406                         symbol_t *const symbol = symbol_table_insert(string);
1407                         if (symbol->string != string)
1408                                 obstack_free(&symbol_obstack, string);
1409
1410                         pp_token.kind        = T_UNKNOWN_CHAR;
1411                         pp_token.base.symbol = symbol;
1412                         next_char();
1413                         return;
1414                 }
1415         }
1416 }
1417
1418 static void print_quoted_string(const char *const string)
1419 {
1420         fputc('"', out);
1421         for (const char *c = string; *c != 0; ++c) {
1422                 switch (*c) {
1423                 case '"': fputs("\\\"", out); break;
1424                 case '\\':  fputs("\\\\", out); break;
1425                 case '\a':  fputs("\\a", out); break;
1426                 case '\b':  fputs("\\b", out); break;
1427                 case '\f':  fputs("\\f", out); break;
1428                 case '\n':  fputs("\\n", out); break;
1429                 case '\r':  fputs("\\r", out); break;
1430                 case '\t':  fputs("\\t", out); break;
1431                 case '\v':  fputs("\\v", out); break;
1432                 case '\?':  fputs("\\?", out); break;
1433                 default:
1434                         if (!isprint(*c)) {
1435                                 fprintf(out, "\\%03o", (unsigned)*c);
1436                                 break;
1437                         }
1438                         fputc(*c, out);
1439                         break;
1440                 }
1441         }
1442         fputc('"', out);
1443 }
1444
1445 static void print_line_directive(const source_position_t *pos, const char *add)
1446 {
1447         if (!out)
1448                 return;
1449
1450         fprintf(out, "# %u ", pos->lineno);
1451         print_quoted_string(pos->input_name);
1452         if (add != NULL) {
1453                 fputc(' ', out);
1454                 fputs(add, out);
1455         }
1456         if (pos->is_system_header) {
1457                 fputs(" 3", out);
1458         }
1459
1460         printed_input_name = pos->input_name;
1461         input.output_line  = pos->lineno-1;
1462 }
1463
1464 static bool emit_newlines(void)
1465 {
1466         unsigned delta = pp_token.base.source_position.lineno - input.output_line;
1467         if (delta == 0)
1468                 return false;
1469
1470         if (delta >= 9) {
1471                 fputc('\n', out);
1472                 print_line_directive(&pp_token.base.source_position, NULL);
1473                 fputc('\n', out);
1474         } else {
1475                 for (unsigned i = 0; i < delta; ++i) {
1476                         fputc('\n', out);
1477                 }
1478         }
1479         input.output_line = pp_token.base.source_position.lineno;
1480
1481         unsigned whitespace = info.whitespace_at_line_begin;
1482         /* make sure there is at least 1 whitespace before a (macro-expanded)
1483          * '#' at line begin. I'm not sure why this is good, but gcc does it. */
1484         if (pp_token.kind == '#' && whitespace == 0)
1485                 ++whitespace;
1486         for (unsigned i = 0; i < whitespace; ++i)
1487                 fputc(' ', out);
1488
1489         return true;
1490 }
1491
1492 void set_preprocessor_output(FILE *output)
1493 {
1494         out = output;
1495         if (out != NULL) {
1496                 error_on_unknown_chars   = false;
1497                 resolve_escape_sequences = false;
1498         } else {
1499                 error_on_unknown_chars   = true;
1500                 resolve_escape_sequences = true;
1501         }
1502 }
1503
1504 void emit_pp_token(void)
1505 {
1506         if (!emit_newlines() &&
1507             (info.had_whitespace || tokens_would_paste(last_token, pp_token.kind)))
1508                 fputc(' ', out);
1509
1510         switch (pp_token.kind) {
1511         case T_NUMBER:
1512                 fputs(pp_token.literal.string.begin, out);
1513                 break;
1514
1515         case T_STRING_LITERAL:
1516                 fputs(get_string_encoding_prefix(pp_token.literal.string.encoding), out);
1517                 fputc('"', out);
1518                 fputs(pp_token.literal.string.begin, out);
1519                 fputc('"', out);
1520                 break;
1521
1522         case T_CHARACTER_CONSTANT:
1523                 fputs(get_string_encoding_prefix(pp_token.literal.string.encoding), out);
1524                 fputc('\'', out);
1525                 fputs(pp_token.literal.string.begin, out);
1526                 fputc('\'', out);
1527                 break;
1528
1529         case T_MACRO_PARAMETER:
1530                 panic("macro parameter not expanded");
1531
1532         default:
1533                 fputs(pp_token.base.symbol->string, out);
1534                 break;
1535         }
1536         last_token = pp_token.kind;
1537 }
1538
1539 static void eat_pp_directive(void)
1540 {
1541         while (!info.at_line_begin) {
1542                 next_input_token();
1543         }
1544 }
1545
1546 static bool strings_equal(const string_t *string1, const string_t *string2)
1547 {
1548         size_t size = string1->size;
1549         if (size != string2->size)
1550                 return false;
1551
1552         const char *c1 = string1->begin;
1553         const char *c2 = string2->begin;
1554         for (size_t i = 0; i < size; ++i, ++c1, ++c2) {
1555                 if (*c1 != *c2)
1556                         return false;
1557         }
1558         return true;
1559 }
1560
1561 static bool pp_tokens_equal(const token_t *token1, const token_t *token2)
1562 {
1563         if (token1->kind != token2->kind)
1564                 return false;
1565
1566         switch (token1->kind) {
1567         case T_NUMBER:
1568         case T_CHARACTER_CONSTANT:
1569         case T_STRING_LITERAL:
1570                 return strings_equal(&token1->literal.string, &token2->literal.string);
1571
1572         case T_MACRO_PARAMETER:
1573                 return token1->macro_parameter.def->symbol
1574                     == token2->macro_parameter.def->symbol;
1575
1576         default:
1577                 return token1->base.symbol == token2->base.symbol;
1578         }
1579 }
1580
1581 static bool pp_definitions_equal(const pp_definition_t *definition1,
1582                                  const pp_definition_t *definition2)
1583 {
1584         if (definition1->list_len != definition2->list_len)
1585                 return false;
1586
1587         size_t               len = definition1->list_len;
1588         const saved_token_t *t1  = definition1->token_list;
1589         const saved_token_t *t2  = definition2->token_list;
1590         for (size_t i = 0; i < len; ++i, ++t1, ++t2) {
1591                 if (!pp_tokens_equal(&t1->token, &t2->token))
1592                         return false;
1593         }
1594         return true;
1595 }
1596
1597 static void missing_macro_param_error(void)
1598 {
1599         errorf(&pp_token.base.source_position,
1600                "'#' is not followed by a macro parameter");
1601 }
1602
1603 static bool is_defineable_token(char const *const context)
1604 {
1605         if (info.at_line_begin) {
1606                 errorf(&pp_token.base.source_position, "unexpected end of line after %s", context);
1607         }
1608
1609         symbol_t *const symbol = pp_token.base.symbol;
1610         if (!symbol)
1611                 goto no_ident;
1612
1613         if (pp_token.kind != T_IDENTIFIER) {
1614                 switch (symbol->string[0]) {
1615                 case SYMBOL_CASES:
1616 dollar_sign:
1617                         break;
1618
1619                 default:
1620 no_ident:
1621                         errorf(&pp_token.base.source_position, "expected identifier after %s, got %K", context, &pp_token);
1622                         return false;
1623                 }
1624         }
1625
1626         /* TODO turn this into a flag in pp_def. */
1627         switch (symbol->pp_ID) {
1628         /* §6.10.8:4 */
1629         case TP_defined:
1630                 errorf(&pp_token.base.source_position, "%K cannot be used as macro name in %s", &pp_token, context);
1631                 return false;
1632
1633         default:
1634                 return true;
1635         }
1636 }
1637
1638 static void parse_define_directive(void)
1639 {
1640         eat_pp(TP_define);
1641         if (skip_mode) {
1642                 eat_pp_directive();
1643                 return;
1644         }
1645
1646         assert(obstack_object_size(&pp_obstack) == 0);
1647
1648         if (!is_defineable_token("#define"))
1649                 goto error_out;
1650         symbol_t *const symbol = pp_token.base.symbol;
1651
1652         pp_definition_t *new_definition
1653                 = obstack_alloc(&pp_obstack, sizeof(new_definition[0]));
1654         memset(new_definition, 0, sizeof(new_definition[0]));
1655         new_definition->symbol          = symbol;
1656         new_definition->source_position = input.position;
1657
1658         /* this is probably the only place where spaces are significant in the
1659          * lexer (except for the fact that they separate tokens). #define b(x)
1660          * is something else than #define b (x) */
1661         if (input.c == '(') {
1662                 next_input_token();
1663                 eat_token('(');
1664
1665                 while (true) {
1666                         switch (pp_token.kind) {
1667                         case T_DOTDOTDOT:
1668                                 new_definition->is_variadic = true;
1669                                 eat_token(T_DOTDOTDOT);
1670                                 if (pp_token.kind != ')') {
1671                                         errorf(&input.position,
1672                                                         "'...' not at end of macro argument list");
1673                                         goto error_out;
1674                                 }
1675                                 break;
1676
1677                         case T_IDENTIFIER: {
1678                                 pp_definition_t parameter;
1679                                 memset(&parameter, 0, sizeof(parameter));
1680                                 parameter.source_position = pp_token.base.source_position;
1681                                 parameter.symbol          = pp_token.base.symbol;
1682                                 parameter.is_parameter    = true;
1683                                 obstack_grow(&pp_obstack, &parameter, sizeof(parameter));
1684                                 eat_token(T_IDENTIFIER);
1685
1686                                 if (pp_token.kind == ',') {
1687                                         eat_token(',');
1688                                         break;
1689                                 }
1690
1691                                 if (pp_token.kind != ')') {
1692                                         errorf(&pp_token.base.source_position,
1693                                                "expected ',' or ')' after identifier, got %K",
1694                                                &pp_token);
1695                                         goto error_out;
1696                                 }
1697                                 break;
1698                         }
1699
1700                         case ')':
1701                                 eat_token(')');
1702                                 goto finish_argument_list;
1703
1704                         default:
1705                                 errorf(&pp_token.base.source_position,
1706                                        "expected identifier, '...' or ')' in #define argument list, got %K",
1707                                        &pp_token);
1708                                 goto error_out;
1709                         }
1710                 }
1711
1712         finish_argument_list:
1713                 new_definition->has_parameters = true;
1714                 size_t size = obstack_object_size(&pp_obstack);
1715                 new_definition->n_parameters
1716                         = size / sizeof(new_definition->parameters[0]);
1717                 new_definition->parameters = obstack_finish(&pp_obstack);
1718                 for (size_t i = 0; i < new_definition->n_parameters; ++i) {
1719                         pp_definition_t *param    = &new_definition->parameters[i];
1720                         symbol_t        *symbol   = param->symbol;
1721                         pp_definition_t *previous = symbol->pp_definition;
1722                         if (previous != NULL
1723                             && previous->function_definition == new_definition) {
1724                                 errorf(&param->source_position,
1725                                        "duplicate macro parameter '%Y'", symbol);
1726                                 param->symbol = sym_anonymous;
1727                                 continue;
1728                         }
1729                         param->parent_expansion    = previous;
1730                         param->function_definition = new_definition;
1731                         symbol->pp_definition      = param;
1732                 }
1733         } else {
1734                 next_input_token();
1735         }
1736
1737         /* construct token list */
1738         assert(obstack_object_size(&pp_obstack) == 0);
1739         bool next_must_be_param = false;
1740         while (!info.at_line_begin) {
1741                 if (pp_token.kind == T_IDENTIFIER) {
1742                         const symbol_t  *symbol     = pp_token.base.symbol;
1743                         pp_definition_t *definition = symbol->pp_definition;
1744                         if (definition != NULL
1745                             && definition->function_definition == new_definition) {
1746                             pp_token.kind                = T_MACRO_PARAMETER;
1747                             pp_token.macro_parameter.def = definition;
1748                         }
1749                 }
1750                 if (next_must_be_param && pp_token.kind != T_MACRO_PARAMETER) {
1751                         missing_macro_param_error();
1752                 }
1753                 saved_token_t saved_token;
1754                 saved_token.token = pp_token;
1755                 saved_token.had_whitespace = info.had_whitespace;
1756                 obstack_grow(&pp_obstack, &saved_token, sizeof(saved_token));
1757                 next_must_be_param
1758                         = new_definition->has_parameters && pp_token.kind == '#';
1759                 next_input_token();
1760         }
1761         if (next_must_be_param)
1762                 missing_macro_param_error();
1763
1764         new_definition->list_len   = obstack_object_size(&pp_obstack)
1765                 / sizeof(new_definition->token_list[0]);
1766         new_definition->token_list = obstack_finish(&pp_obstack);
1767
1768         if (new_definition->has_parameters) {
1769                 for (size_t i = 0; i < new_definition->n_parameters; ++i) {
1770                         pp_definition_t *param      = &new_definition->parameters[i];
1771                         symbol_t        *symbol     = param->symbol;
1772                         if (symbol == sym_anonymous)
1773                                 continue;
1774                         assert(symbol->pp_definition == param);
1775                         assert(param->function_definition == new_definition);
1776                         symbol->pp_definition   = param->parent_expansion;
1777                         param->parent_expansion = NULL;
1778                 }
1779         }
1780
1781         pp_definition_t *old_definition = symbol->pp_definition;
1782         if (old_definition != NULL) {
1783                 if (!pp_definitions_equal(old_definition, new_definition)) {
1784                         warningf(WARN_OTHER, &input.position, "multiple definition of macro '%Y' (first defined %P)", symbol, &old_definition->source_position);
1785                 } else {
1786                         /* reuse the old definition */
1787                         obstack_free(&pp_obstack, new_definition);
1788                         new_definition = old_definition;
1789                 }
1790         }
1791
1792         symbol->pp_definition = new_definition;
1793         return;
1794
1795 error_out:
1796         if (obstack_object_size(&pp_obstack) > 0) {
1797                 char *ptr = obstack_finish(&pp_obstack);
1798                 obstack_free(&pp_obstack, ptr);
1799         }
1800         eat_pp_directive();
1801 }
1802
1803 static void parse_undef_directive(void)
1804 {
1805         eat_pp(TP_undef);
1806         if (skip_mode) {
1807                 eat_pp_directive();
1808                 return;
1809         }
1810
1811         if (!is_defineable_token("#undef")) {
1812                 eat_pp_directive();
1813                 return;
1814         }
1815
1816         pp_token.base.symbol->pp_definition = NULL;
1817         next_input_token();
1818
1819         if (!info.at_line_begin) {
1820                 warningf(WARN_OTHER, &input.position, "extra tokens at end of #undef directive");
1821         }
1822         eat_pp_directive();
1823 }
1824
1825 /** behind an #include we can have the special headername lexems.
1826  * They're only allowed behind an #include so they're not recognized
1827  * by the normal next_preprocessing_token. We handle them as a special
1828  * exception here */
1829 static const char *parse_headername(bool *system_include)
1830 {
1831         if (info.at_line_begin) {
1832                 parse_error("expected headername after #include");
1833                 return NULL;
1834         }
1835
1836         /* check wether we have a "... or <... headername */
1837         source_position_t position = input.position;
1838         switch (input.c) {
1839         {
1840                 utf32 delimiter;
1841         case '<': delimiter = '>'; *system_include = true;  goto parse_name;
1842         case '"': delimiter = '"'; *system_include = false; goto parse_name;
1843 parse_name:
1844                 assert(obstack_object_size(&symbol_obstack) == 0);
1845                 next_char();
1846                 while (true) {
1847                         switch (input.c) {
1848                         case NEWLINE:
1849                         case EOF:
1850                                 {
1851                                         char *dummy = obstack_finish(&symbol_obstack);
1852                                         obstack_free(&symbol_obstack, dummy);
1853                                 }
1854                                 errorf(&pp_token.base.source_position,
1855                                        "header name without closing '%c'", (char)delimiter);
1856                                 return NULL;
1857
1858                         default:
1859                                 if (input.c == delimiter) {
1860                                         next_char();
1861                                         goto finish_headername;
1862                                 } else {
1863                                         obstack_1grow(&symbol_obstack, (char)input.c);
1864                                         next_char();
1865                                 }
1866                                 break;
1867                         }
1868                 }
1869                 /* we should never be here */
1870         }
1871
1872         default:
1873                 next_preprocessing_token();
1874                 if (info.at_line_begin) {
1875                         /* TODO: if we are already in the new line then we parsed more than
1876                          * wanted. We reuse the token, but could produce following errors
1877                          * misbehaviours... */
1878                         goto error_invalid_input;
1879                 }
1880                 if (pp_token.kind == T_STRING_LITERAL) {
1881                         *system_include = false;
1882                         return pp_token.literal.string.begin;
1883                 } else if (pp_token.kind == '<') {
1884                         *system_include = true;
1885                         assert(obstack_object_size(&pp_obstack) == 0);
1886                         while (true) {
1887                                 next_preprocessing_token();
1888                                 if (info.at_line_begin) {
1889                                         /* TODO: we shouldn't have parsed/expanded something on the
1890                                          * next line yet... */
1891                                         char *dummy = obstack_finish(&pp_obstack);
1892                                         obstack_free(&pp_obstack, dummy);
1893                                         goto error_invalid_input;
1894                                 }
1895                                 if (pp_token.kind == '>')
1896                                         break;
1897
1898                                 saved_token_t saved;
1899                                 saved.token          = pp_token;
1900                                 saved.had_whitespace = info.had_whitespace;
1901                                 obstack_grow(&pp_obstack, &saved, sizeof(saved));
1902                         }
1903                         size_t size = obstack_object_size(&pp_obstack);
1904                         assert(size % sizeof(saved_token_t) == 0);
1905                         size_t n_tokens = size / sizeof(saved_token_t);
1906                         saved_token_t *tokens = obstack_finish(&pp_obstack);
1907                         assert(obstack_object_size(&symbol_obstack) == 0);
1908                         for (size_t i = 0; i < n_tokens; ++i) {
1909                                 const saved_token_t *saved = &tokens[i];
1910                                 if (i > 0 && saved->had_whitespace)
1911                                         obstack_1grow(&symbol_obstack, ' ');
1912                                 grow_token(&symbol_obstack, &saved->token);
1913                         }
1914                         obstack_free(&pp_obstack, tokens);
1915                         goto finish_headername;
1916                 } else {
1917 error_invalid_input:
1918                         {
1919                                 char *dummy = obstack_finish(&symbol_obstack);
1920                                 obstack_free(&symbol_obstack, dummy);
1921                         }
1922
1923                         errorf(&pp_token.base.source_position,
1924                                "expected \"FILENAME\" or <FILENAME> after #include");
1925                         return NULL;
1926                 }
1927         }
1928
1929 finish_headername:
1930         obstack_1grow(&symbol_obstack, '\0');
1931         char *const  headername = obstack_finish(&symbol_obstack);
1932         const char  *identified = identify_string(headername);
1933         pp_token.base.source_position = position;
1934         return identified;
1935 }
1936
1937 static bool do_include(bool const system_include, bool const include_next, char const *const headername)
1938 {
1939         size_t const        headername_len = strlen(headername);
1940         searchpath_entry_t *entry;
1941         if (include_next) {
1942                 entry = input.path ? input.path->next : searchpath;
1943         } else {
1944                 if (!system_include) {
1945                         /* put dirname of current input on obstack */
1946                         const char *filename   = input.position.input_name;
1947                         const char *last_slash = strrchr(filename, '/');
1948                         const char *full_name;
1949                         if (last_slash != NULL) {
1950                                 size_t len = last_slash - filename;
1951                                 obstack_grow(&symbol_obstack, filename, len + 1);
1952                                 obstack_grow0(&symbol_obstack, headername, headername_len);
1953                                 char *complete_path = obstack_finish(&symbol_obstack);
1954                                 full_name = identify_string(complete_path);
1955                         } else {
1956                                 full_name = headername;
1957                         }
1958
1959                         FILE *file = fopen(full_name, "r");
1960                         if (file != NULL) {
1961                                 switch_pp_input(file, full_name, NULL);
1962                                 return true;
1963                         }
1964                 }
1965
1966                 entry = searchpath;
1967         }
1968
1969         assert(obstack_object_size(&symbol_obstack) == 0);
1970         /* check searchpath */
1971         for (; entry; entry = entry->next) {
1972             const char *path = entry->path;
1973             size_t      len  = strlen(path);
1974                 obstack_grow(&symbol_obstack, path, len);
1975                 if (path[len-1] != '/')
1976                         obstack_1grow(&symbol_obstack, '/');
1977                 obstack_grow(&symbol_obstack, headername, headername_len+1);
1978
1979                 char *complete_path = obstack_finish(&symbol_obstack);
1980                 FILE *file          = fopen(complete_path, "r");
1981                 if (file != NULL) {
1982                         const char *filename = identify_string(complete_path);
1983                         switch_pp_input(file, filename, entry);
1984                         return true;
1985                 } else {
1986                         obstack_free(&symbol_obstack, complete_path);
1987                 }
1988         }
1989
1990         return false;
1991 }
1992
1993 static void parse_include_directive(bool const include_next)
1994 {
1995         if (skip_mode) {
1996                 eat_pp_directive();
1997                 return;
1998         }
1999
2000         /* do not eat the TP_include, since it would already parse the next token
2001          * which needs special handling here. */
2002         skip_till_newline(true);
2003         bool system_include;
2004         const char *headername = parse_headername(&system_include);
2005         if (headername == NULL) {
2006                 eat_pp_directive();
2007                 return;
2008         }
2009
2010         bool had_nonwhitespace = skip_till_newline(false);
2011         if (had_nonwhitespace) {
2012                 warningf(WARN_OTHER, &input.position,
2013                          "extra tokens at end of #include directive");
2014         }
2015
2016         if (n_inputs > INCLUDE_LIMIT) {
2017                 errorf(&pp_token.base.source_position, "#include nested too deeply");
2018                 /* eat \n or EOF */
2019                 next_input_token();
2020                 return;
2021         }
2022
2023         /* switch inputs */
2024         info.whitespace_at_line_begin = 0;
2025         info.had_whitespace           = false;
2026         info.at_line_begin            = true;
2027         emit_newlines();
2028         push_input();
2029         bool res = do_include(system_include, include_next, headername);
2030         if (res) {
2031                 next_input_token();
2032         } else {
2033                 errorf(&pp_token.base.source_position, "failed including '%s': %s", headername, strerror(errno));
2034                 pop_restore_input();
2035         }
2036 }
2037
2038 static pp_conditional_t *push_conditional(void)
2039 {
2040         pp_conditional_t *conditional
2041                 = obstack_alloc(&pp_obstack, sizeof(*conditional));
2042         memset(conditional, 0, sizeof(*conditional));
2043
2044         conditional->parent = conditional_stack;
2045         conditional_stack   = conditional;
2046
2047         return conditional;
2048 }
2049
2050 static void pop_conditional(void)
2051 {
2052         assert(conditional_stack != NULL);
2053         conditional_stack = conditional_stack->parent;
2054 }
2055
2056 void check_unclosed_conditionals(void)
2057 {
2058         while (conditional_stack != NULL) {
2059                 pp_conditional_t *conditional = conditional_stack;
2060
2061                 if (conditional->in_else) {
2062                         errorf(&conditional->source_position, "unterminated #else");
2063                 } else {
2064                         errorf(&conditional->source_position, "unterminated condition");
2065                 }
2066                 pop_conditional();
2067         }
2068 }
2069
2070 static void parse_ifdef_ifndef_directive(bool const is_ifdef)
2071 {
2072         bool condition;
2073         eat_pp(is_ifdef ? TP_ifdef : TP_ifndef);
2074
2075         if (skip_mode) {
2076                 eat_pp_directive();
2077                 pp_conditional_t *conditional = push_conditional();
2078                 conditional->source_position  = pp_token.base.source_position;
2079                 conditional->skip             = true;
2080                 return;
2081         }
2082
2083         if (pp_token.kind != T_IDENTIFIER || info.at_line_begin) {
2084                 errorf(&pp_token.base.source_position,
2085                        "expected identifier after #%s, got %K",
2086                        is_ifdef ? "ifdef" : "ifndef", &pp_token);
2087                 eat_pp_directive();
2088
2089                 /* just take the true case in the hope to avoid further errors */
2090                 condition = true;
2091         } else {
2092                 /* evaluate wether we are in true or false case */
2093                 condition = (bool)pp_token.base.symbol->pp_definition == is_ifdef;
2094                 eat_token(T_IDENTIFIER);
2095
2096                 if (!info.at_line_begin) {
2097                         errorf(&pp_token.base.source_position,
2098                                "extra tokens at end of #%s",
2099                                is_ifdef ? "ifdef" : "ifndef");
2100                         eat_pp_directive();
2101                 }
2102         }
2103
2104         pp_conditional_t *conditional = push_conditional();
2105         conditional->source_position  = pp_token.base.source_position;
2106         conditional->condition        = condition;
2107
2108         if (!condition) {
2109                 skip_mode = true;
2110         }
2111 }
2112
2113 static void parse_else_directive(void)
2114 {
2115         eat_pp(TP_else);
2116
2117         if (!info.at_line_begin) {
2118                 if (!skip_mode) {
2119                         warningf(WARN_OTHER, &pp_token.base.source_position, "extra tokens at end of #else");
2120                 }
2121                 eat_pp_directive();
2122         }
2123
2124         pp_conditional_t *conditional = conditional_stack;
2125         if (conditional == NULL) {
2126                 errorf(&pp_token.base.source_position, "#else without prior #if");
2127                 return;
2128         }
2129
2130         if (conditional->in_else) {
2131                 errorf(&pp_token.base.source_position,
2132                        "#else after #else (condition started %P)",
2133                        &conditional->source_position);
2134                 skip_mode = true;
2135                 return;
2136         }
2137
2138         conditional->in_else = true;
2139         if (!conditional->skip) {
2140                 skip_mode = conditional->condition;
2141         }
2142         conditional->source_position = pp_token.base.source_position;
2143 }
2144
2145 static void parse_endif_directive(void)
2146 {
2147         eat_pp(TP_endif);
2148
2149         if (!info.at_line_begin) {
2150                 if (!skip_mode) {
2151                         warningf(WARN_OTHER, &pp_token.base.source_position, "extra tokens at end of #endif");
2152                 }
2153                 eat_pp_directive();
2154         }
2155
2156         pp_conditional_t *conditional = conditional_stack;
2157         if (conditional == NULL) {
2158                 errorf(&pp_token.base.source_position, "#endif without prior #if");
2159                 return;
2160         }
2161
2162         if (!conditional->skip) {
2163                 skip_mode = false;
2164         }
2165         pop_conditional();
2166 }
2167
2168 typedef enum stdc_pragma_kind_t {
2169         STDC_UNKNOWN,
2170         STDC_FP_CONTRACT,
2171         STDC_FENV_ACCESS,
2172         STDC_CX_LIMITED_RANGE
2173 } stdc_pragma_kind_t;
2174
2175 typedef enum stdc_pragma_value_kind_t {
2176         STDC_VALUE_UNKNOWN,
2177         STDC_VALUE_ON,
2178         STDC_VALUE_OFF,
2179         STDC_VALUE_DEFAULT
2180 } stdc_pragma_value_kind_t;
2181
2182 static void parse_pragma_directive(void)
2183 {
2184         eat_pp(TP_pragma);
2185         if (skip_mode) {
2186                 eat_pp_directive();
2187                 return;
2188         }
2189
2190         if (pp_token.kind != T_IDENTIFIER) {
2191                 warningf(WARN_UNKNOWN_PRAGMAS, &pp_token.base.source_position,
2192                          "expected identifier after #pragma");
2193                 eat_pp_directive();
2194                 return;
2195         }
2196
2197         stdc_pragma_kind_t kind = STDC_UNKNOWN;
2198         if (pp_token.base.symbol->pp_ID == TP_STDC && c_mode & _C99) {
2199                 /* a STDC pragma */
2200                 next_input_token();
2201
2202                 switch (pp_token.base.symbol->pp_ID) {
2203                 case TP_FP_CONTRACT:      kind = STDC_FP_CONTRACT;      break;
2204                 case TP_FENV_ACCESS:      kind = STDC_FENV_ACCESS;      break;
2205                 case TP_CX_LIMITED_RANGE: kind = STDC_CX_LIMITED_RANGE; break;
2206                 default:                  break;
2207                 }
2208                 if (kind != STDC_UNKNOWN) {
2209                         next_input_token();
2210                         stdc_pragma_value_kind_t value;
2211                         switch (pp_token.base.symbol->pp_ID) {
2212                         case TP_ON:      value = STDC_VALUE_ON;      break;
2213                         case TP_OFF:     value = STDC_VALUE_OFF;     break;
2214                         case TP_DEFAULT: value = STDC_VALUE_DEFAULT; break;
2215                         default:         value = STDC_VALUE_UNKNOWN; break;
2216                         }
2217                         if (value == STDC_VALUE_UNKNOWN) {
2218                                 kind = STDC_UNKNOWN;
2219                                 errorf(&pp_token.base.source_position, "bad STDC pragma argument");
2220                         }
2221                 }
2222         }
2223         eat_pp_directive();
2224         if (kind == STDC_UNKNOWN) {
2225                 warningf(WARN_UNKNOWN_PRAGMAS, &pp_token.base.source_position,
2226                          "encountered unknown #pragma");
2227         }
2228 }
2229
2230 static void parse_line_directive(void)
2231 {
2232         if (pp_token.kind != T_NUMBER) {
2233                 if (!skip_mode)
2234                         parse_error("expected integer");
2235         } else {
2236                 char      *end;
2237                 long const line = strtol(pp_token.literal.string.begin, &end, 0);
2238                 if (*end == '\0') {
2239                         /* use offset -1 as this is about the next line */
2240                         input.position.lineno = line - 1;
2241                         /* force output of line */
2242                         input.output_line = input.position.lineno - 20;
2243                 } else {
2244                         if (!skip_mode) {
2245                                 errorf(&input.position, "'%S' is not a valid line number",
2246                                            &pp_token.literal.string);
2247                         }
2248                 }
2249                 next_input_token();
2250                 if (info.at_line_begin)
2251                         return;
2252         }
2253         if (pp_token.kind == T_STRING_LITERAL
2254             && pp_token.literal.string.encoding == STRING_ENCODING_CHAR) {
2255                 input.position.input_name       = pp_token.literal.string.begin;
2256                 input.position.is_system_header = false;
2257                 next_input_token();
2258
2259                 /* attempt to parse numeric flags as outputted by gcc preprocessor */
2260                 while (!info.at_line_begin && pp_token.kind == T_NUMBER) {
2261                         /* flags:
2262                          * 1 - indicates start of a new file
2263                          * 2 - indicates return from a file
2264                          * 3 - indicates system header
2265                          * 4 - indicates implicit extern "C" in C++ mode
2266                          *
2267                          * currently we're only interested in "3"
2268                          */
2269                         if (streq(pp_token.literal.string.begin, "3")) {
2270                                 input.position.is_system_header = true;
2271                         }
2272                         next_input_token();
2273                 }
2274         }
2275
2276         eat_pp_directive();
2277 }
2278
2279 static void parse_error_directive(void)
2280 {
2281         if (skip_mode) {
2282                 eat_pp_directive();
2283                 return;
2284         }
2285
2286         bool const old_resolve_escape_sequences = resolve_escape_sequences;
2287         resolve_escape_sequences = false;
2288
2289         source_position_t const pos = pp_token.base.source_position;
2290         do {
2291                 if (info.had_whitespace && obstack_object_size(&pp_obstack) != 0)
2292                         obstack_1grow(&pp_obstack, ' ');
2293
2294                 switch (pp_token.kind) {
2295                 case T_NUMBER: {
2296                         string_t const *const str = &pp_token.literal.string;
2297                         obstack_grow(&pp_obstack, str->begin, str->size);
2298                         break;
2299                 }
2300
2301                 {
2302                         char delim;
2303                 case T_STRING_LITERAL:     delim =  '"'; goto string;
2304                 case T_CHARACTER_CONSTANT: delim = '\''; goto string;
2305 string:;
2306                         string_t const *const str = &pp_token.literal.string;
2307                         char     const *const enc = get_string_encoding_prefix(str->encoding);
2308                         obstack_printf(&pp_obstack, "%s%c%s%c", enc, delim, str->begin, delim);
2309                         break;
2310                 }
2311
2312                 default: {
2313                         char const *const str = pp_token.base.symbol->string;
2314                         obstack_grow(&pp_obstack, str, strlen(str));
2315                         break;
2316                 }
2317                 }
2318
2319                 next_input_token();
2320         } while (!info.at_line_begin);
2321
2322         resolve_escape_sequences = old_resolve_escape_sequences;
2323
2324         obstack_1grow(&pp_obstack, '\0');
2325         char *const str = obstack_finish(&pp_obstack);
2326         errorf(&pos, "#%s", str);
2327         obstack_free(&pp_obstack, str);
2328 }
2329
2330 static void parse_preprocessing_directive(void)
2331 {
2332         eat_token('#');
2333
2334         if (info.at_line_begin) {
2335                 /* empty directive */
2336                 return;
2337         }
2338
2339         if (pp_token.base.symbol) {
2340                 switch (pp_token.base.symbol->pp_ID) {
2341                 case TP_define:       parse_define_directive();            break;
2342                 case TP_else:         parse_else_directive();              break;
2343                 case TP_endif:        parse_endif_directive();             break;
2344                 case TP_error:        parse_error_directive();             break;
2345                 case TP_ifdef:        parse_ifdef_ifndef_directive(true);  break;
2346                 case TP_ifndef:       parse_ifdef_ifndef_directive(false); break;
2347                 case TP_include:      parse_include_directive(false);      break;
2348                 case TP_include_next: parse_include_directive(true);       break;
2349                 case TP_line:         next_input_token(); goto line_directive;
2350                 case TP_pragma:       parse_pragma_directive();            break;
2351                 case TP_undef:        parse_undef_directive();             break;
2352                 default:              goto skip;
2353                 }
2354         } else if (pp_token.kind == T_NUMBER) {
2355 line_directive:
2356                 parse_line_directive();
2357         } else {
2358 skip:
2359                 if (!skip_mode) {
2360                         errorf(&pp_token.base.source_position, "invalid preprocessing directive #%K", &pp_token);
2361                 }
2362                 eat_pp_directive();
2363         }
2364
2365         assert(info.at_line_begin);
2366 }
2367
2368 static void finish_current_argument(void)
2369 {
2370         if (current_argument == NULL)
2371                 return;
2372         size_t size = obstack_object_size(&pp_obstack);
2373         current_argument->list_len   = size/sizeof(current_argument->token_list[0]);
2374         current_argument->token_list = obstack_finish(&pp_obstack);
2375 }
2376
2377 void next_preprocessing_token(void)
2378 {
2379 restart:
2380         if (!expand_next()) {
2381                 do {
2382                         next_input_token();
2383                         while (pp_token.kind == '#' && info.at_line_begin) {
2384                                 parse_preprocessing_directive();
2385                         }
2386                 } while (skip_mode && pp_token.kind != T_EOF);
2387         }
2388
2389         const token_kind_t kind = pp_token.kind;
2390         if (current_call == NULL || argument_expanding != NULL) {
2391                 symbol_t *const symbol = pp_token.base.symbol;
2392                 if (symbol) {
2393                         if (kind == T_MACRO_PARAMETER) {
2394                                 assert(current_expansion != NULL);
2395                                 start_expanding(pp_token.macro_parameter.def);
2396                                 goto restart;
2397                         }
2398
2399                         pp_definition_t *const pp_definition = symbol->pp_definition;
2400                         if (pp_definition != NULL && !pp_definition->is_expanding) {
2401                                 if (pp_definition->has_parameters) {
2402
2403                                         /* check if next token is a '(' */
2404                                         whitespace_info_t old_info   = info;
2405                                         token_kind_t      next_token = peek_expansion();
2406                                         if (next_token == T_EOF) {
2407                                                 info.at_line_begin  = false;
2408                                                 info.had_whitespace = false;
2409                                                 skip_whitespace();
2410                                                 if (input.c == '(') {
2411                                                         next_token = '(';
2412                                                 }
2413                                         }
2414
2415                                         if (next_token == '(') {
2416                                                 if (current_expansion == NULL)
2417                                                         expansion_pos = pp_token.base.source_position;
2418                                                 next_preprocessing_token();
2419                                                 assert(pp_token.kind == '(');
2420
2421                                                 pp_definition->parent_expansion = current_expansion;
2422                                                 current_call              = pp_definition;
2423                                                 current_call->expand_pos  = 0;
2424                                                 current_call->expand_info = old_info;
2425                                                 if (current_call->n_parameters > 0) {
2426                                                         current_argument = &current_call->parameters[0];
2427                                                         assert(argument_brace_count == 0);
2428                                                 }
2429                                                 goto restart;
2430                                         } else {
2431                                                 /* skip_whitespaces() skipped newlines and whitespace,
2432                                                  * remember results for next token */
2433                                                 next_info = info;
2434                                                 info      = old_info;
2435                                                 return;
2436                                         }
2437                                 } else {
2438                                         if (current_expansion == NULL)
2439                                                 expansion_pos = pp_token.base.source_position;
2440                                         start_expanding(pp_definition);
2441                                         goto restart;
2442                                 }
2443                         }
2444                 }
2445         }
2446
2447         if (current_call != NULL) {
2448                 /* current_call != NULL */
2449                 if (kind == '(') {
2450                         ++argument_brace_count;
2451                 } else if (kind == ')') {
2452                         if (argument_brace_count > 0) {
2453                                 --argument_brace_count;
2454                         } else {
2455                                 finish_current_argument();
2456                                 assert(kind == ')');
2457                                 start_expanding(current_call);
2458                                 info = current_call->expand_info;
2459                                 current_call     = NULL;
2460                                 current_argument = NULL;
2461                                 goto restart;
2462                         }
2463                 } else if (kind == ',' && argument_brace_count == 0) {
2464                         finish_current_argument();
2465                         current_call->expand_pos++;
2466                         if (current_call->expand_pos >= current_call->n_parameters) {
2467                                 errorf(&pp_token.base.source_position,
2468                                            "too many arguments passed for macro '%Y'",
2469                                            current_call->symbol);
2470                                 current_argument = NULL;
2471                         } else {
2472                                 current_argument
2473                                         = &current_call->parameters[current_call->expand_pos];
2474                         }
2475                         goto restart;
2476                 } else if (kind == T_MACRO_PARAMETER) {
2477                         /* parameters have to be fully expanded before being used as
2478                          * parameters for another macro-call */
2479                         assert(current_expansion != NULL);
2480                         pp_definition_t *argument = pp_token.macro_parameter.def;
2481                         argument_expanding = argument;
2482                         start_expanding(argument);
2483                         goto restart;
2484                 } else if (kind == T_EOF) {
2485                         errorf(&expansion_pos,
2486                                "reached end of file while parsing arguments for '%Y'",
2487                                current_call->symbol);
2488                         return;
2489                 }
2490                 if (current_argument != NULL) {
2491                         saved_token_t saved;
2492                         saved.token = pp_token;
2493                         saved.had_whitespace = info.had_whitespace;
2494                         obstack_grow(&pp_obstack, &saved, sizeof(saved));
2495                 }
2496                 goto restart;
2497         }
2498 }
2499
2500
2501 static void prepend_include_path(const char *path)
2502 {
2503         searchpath_entry_t *entry = OALLOCZ(&config_obstack, searchpath_entry_t);
2504         entry->path = path;
2505         entry->next = searchpath;
2506         searchpath  = entry;
2507 }
2508
2509 static void setup_include_path(void)
2510 {
2511         /* built-in paths */
2512         prepend_include_path("/usr/include");
2513
2514         /* parse environment variable */
2515         const char *cpath = getenv("CPATH");
2516         if (cpath != NULL && *cpath != '\0') {
2517                 const char *begin = cpath;
2518                 const char *c;
2519                 do {
2520                         c = begin;
2521                         while (*c != '\0' && *c != ':')
2522                                 ++c;
2523
2524                         size_t len = c-begin;
2525                         if (len == 0) {
2526                                 /* for gcc compatibility (Matze: I would expect that
2527                                  * nothing happens for an empty entry...) */
2528                                 prepend_include_path(".");
2529                         } else {
2530                                 char *const string = obstack_copy0(&config_obstack, begin, len);
2531                                 prepend_include_path(string);
2532                         }
2533
2534                         begin = c+1;
2535                         /* skip : */
2536                         if (*begin == ':')
2537                                 ++begin;
2538                 } while(*c != '\0');
2539         }
2540 }
2541
2542 static void input_error(unsigned const delta_lines, unsigned const delta_cols, char const *const message)
2543 {
2544         source_position_t pos = pp_token.base.source_position;
2545         pos.lineno += delta_lines;
2546         pos.colno  += delta_cols;
2547         errorf(&pos, "%s", message);
2548 }
2549
2550 void init_preprocessor(void)
2551 {
2552         init_symbols();
2553
2554         obstack_init(&config_obstack);
2555         obstack_init(&pp_obstack);
2556         obstack_init(&input_obstack);
2557         strset_init(&stringset);
2558
2559         setup_include_path();
2560
2561         set_input_error_callback(input_error);
2562 }
2563
2564 void exit_preprocessor(void)
2565 {
2566         obstack_free(&input_obstack, NULL);
2567         obstack_free(&pp_obstack, NULL);
2568         obstack_free(&config_obstack, NULL);
2569
2570         strset_destroy(&stringset);
2571 }
2572
2573 int pptest_main(int argc, char **argv);
2574 int pptest_main(int argc, char **argv)
2575 {
2576         init_symbol_table();
2577         init_preprocessor();
2578         init_tokens();
2579
2580         error_on_unknown_chars   = false;
2581         resolve_escape_sequences = false;
2582
2583         /* simplistic commandline parser */
2584         const char *filename = NULL;
2585         const char *output = NULL;
2586         for (int i = 1; i < argc; ++i) {
2587                 const char *opt = argv[i];
2588                 if (streq(opt, "-I")) {
2589                         prepend_include_path(argv[++i]);
2590                         continue;
2591                 } else if (streq(opt, "-E")) {
2592                         /* ignore */
2593                 } else if (streq(opt, "-o")) {
2594                         output = argv[++i];
2595                         continue;
2596                 } else if (opt[0] == '-') {
2597                         fprintf(stderr, "Unknown option '%s'\n", opt);
2598                 } else {
2599                         if (filename != NULL)
2600                                 fprintf(stderr, "Multiple inputs not supported\n");
2601                         filename = argv[i];
2602                 }
2603         }
2604         if (filename == NULL) {
2605                 fprintf(stderr, "No input specified\n");
2606                 return 1;
2607         }
2608
2609         if (output == NULL) {
2610                 out = stdout;
2611         } else {
2612                 out = fopen(output, "w");
2613                 if (out == NULL) {
2614                         fprintf(stderr, "Couldn't open output '%s'\n", output);
2615                         return 1;
2616                 }
2617         }
2618
2619         /* just here for gcc compatibility */
2620         fprintf(out, "# 1 \"%s\"\n", filename);
2621         fprintf(out, "# 1 \"<built-in>\"\n");
2622         fprintf(out, "# 1 \"<command-line>\"\n");
2623
2624         FILE *file = fopen(filename, "r");
2625         if (file == NULL) {
2626                 fprintf(stderr, "Couldn't open input '%s'\n", filename);
2627                 return 1;
2628         }
2629         switch_pp_input(file, filename, NULL);
2630
2631         for (;;) {
2632                 next_preprocessing_token();
2633                 if (pp_token.kind == T_EOF)
2634                         break;
2635                 emit_pp_token();
2636         }
2637
2638         fputc('\n', out);
2639         check_unclosed_conditionals();
2640         fclose(close_pp_input());
2641         if (out != stdout)
2642                 fclose(out);
2643
2644         exit_tokens();
2645         exit_preprocessor();
2646         exit_symbol_table();
2647
2648         return 0;
2649 }