Improve error recovery by adding anchor tokens earlier.
[cparser] / preprocessor.c
index 48825af..f829db4 100644 (file)
@@ -91,7 +91,6 @@ static pp_conditional_t *conditional_stack;
 static token_t           pp_token;
 static bool              resolve_escape_sequences = false;
 static bool              ignore_unknown_chars     = true;
-static bool              in_pp_directive;
 static bool              skip_mode;
 static FILE             *out;
 static struct obstack    pp_obstack;
@@ -100,7 +99,7 @@ static const char       *printed_input_name = NULL;
 static source_position_t expansion_pos;
 static pp_definition_t  *current_expansion  = NULL;
 static strset_t          stringset;
-static preprocessor_token_type_t last_token = TP_ERROR;
+static preprocessor_token_kind_t last_token;
 
 static searchpath_entry_t *searchpath;
 
@@ -188,15 +187,14 @@ static void pop_restore_input(void)
  */
 static void parse_error(const char *msg)
 {
-       errorf(&pp_token.source_position,  "%s", msg);
+       errorf(&pp_token.base.source_position,  "%s", msg);
 }
 
 static inline void next_real_char(void)
 {
        assert(input.bufpos <= input.bufend);
        if (input.bufpos >= input.bufend) {
-               size_t n = decode(input.input, input.buf + MAX_PUTBACK,
-                                 sizeof(input.buf)/sizeof(input.buf[0]) - MAX_PUTBACK);
+               size_t const n = decode(input.input, input.buf + MAX_PUTBACK, lengthof(input.buf) - MAX_PUTBACK);
                if (n == 0) {
                        input.c = EOF;
                        return;
@@ -362,50 +360,41 @@ static int digit_value(int digit)
  *
  * @param first_digit  the already read first digit
  */
-static int parse_octal_sequence(const int first_digit)
+static utf32 parse_octal_sequence(const utf32 first_digit)
 {
        assert(is_octal_digit(first_digit));
-       int value = digit_value(first_digit);
+       utf32 value = digit_value(first_digit);
        if (!is_octal_digit(input.c)) return value;
        value = 8 * value + digit_value(input.c);
        next_char();
        if (!is_octal_digit(input.c)) return value;
        value = 8 * value + digit_value(input.c);
        next_char();
+       return value;
 
-       if (char_is_signed) {
-               return (signed char) value;
-       } else {
-               return (unsigned char) value;
-       }
 }
 
 /**
  * Parses a hex character sequence.
  */
-static int parse_hex_sequence(void)
+static utf32 parse_hex_sequence(void)
 {
-       int value = 0;
+       utf32 value = 0;
        while (isxdigit(input.c)) {
                value = 16 * value + digit_value(input.c);
                next_char();
        }
-
-       if (char_is_signed) {
-               return (signed char) value;
-       } else {
-               return (unsigned char) value;
-       }
+       return value;
 }
 
 /**
  * Parse an escape sequence.
  */
-static int parse_escape_sequence(void)
+static utf32 parse_escape_sequence(void)
 {
        eat('\\');
 
-       int ec = input.c;
+       utf32 const ec = input.c;
        next_char();
 
        switch (ec) {
@@ -434,30 +423,23 @@ static int parse_escape_sequence(void)
        case EOF:
                parse_error("reached end of file while parsing escape sequence");
                return EOF;
-       default:
-               parse_error("unknown escape sequence");
+       /* \E is not documented, but handled, by GCC.  It is acceptable according
+        * to §6.11.4, whereas \e is not. */
+       case 'E':
+       case 'e':
+               if (c_mode & _GNUC)
+                       return 27;   /* hopefully 27 is ALWAYS the code for ESCAPE */
+               break;
+       case 'u':
+       case 'U':
+               parse_error("universal character parsing not implemented yet");
                return EOF;
+       default:
+               break;
        }
-}
-
-static void grow_symbol(utf32 const tc)
-{
-       struct obstack *const o  = &symbol_obstack;
-       if (tc < 0x80U) {
-               obstack_1grow(o, tc);
-       } else if (tc < 0x800) {
-               obstack_1grow(o, 0xC0 | (tc >> 6));
-               obstack_1grow(o, 0x80 | (tc & 0x3F));
-       } else if (tc < 0x10000) {
-               obstack_1grow(o, 0xE0 | ( tc >> 12));
-               obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
-               obstack_1grow(o, 0x80 | ( tc        & 0x3F));
-       } else {
-               obstack_1grow(o, 0xF0 | ( tc >> 18));
-               obstack_1grow(o, 0x80 | ((tc >> 12) & 0x3F));
-               obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
-               obstack_1grow(o, 0x80 | ( tc        & 0x3F));
-       }
+       /* §6.4.4.4:8 footnote 64 */
+       parse_error("unknown escape sequence");
+       return EOF;
 }
 
 static const char *identify_string(char *string)
@@ -499,11 +481,10 @@ static void parse_string_literal(void)
 
                case EOF: {
                        source_position_t source_position;
-                       source_position.input_name = pp_token.source_position.input_name;
+                       source_position.input_name = pp_token.base.source_position.input_name;
                        source_position.lineno     = start_linenr;
                        errorf(&source_position, "string has no end");
-                       pp_token.type = TP_ERROR;
-                       return;
+                       goto end_of_string;
                }
 
                case '"':
@@ -511,7 +492,7 @@ static void parse_string_literal(void)
                        goto end_of_string;
 
                default:
-                       grow_symbol(input.c);
+                       obstack_grow_symbol(&symbol_obstack, input.c);
                        next_char();
                        break;
                }
@@ -523,8 +504,8 @@ end_of_string:
        const size_t size   = (size_t)obstack_object_size(&symbol_obstack);
        char *const  string = obstack_finish(&symbol_obstack);
 
-       pp_token.type    = TP_STRING_LITERAL;
-       pp_token.literal = make_string(string, size);
+       pp_token.kind          = TP_STRING_LITERAL;
+       pp_token.string.string = make_string(string, size);
 }
 
 /**
@@ -533,8 +514,8 @@ end_of_string:
 static void parse_wide_string_literal(void)
 {
        parse_string_literal();
-       if (pp_token.type == TP_STRING_LITERAL)
-               pp_token.type = TP_WIDE_STRING_LITERAL;
+       if (pp_token.kind == TP_STRING_LITERAL)
+               pp_token.kind = TP_WIDE_STRING_LITERAL;
 }
 
 static void parse_wide_character_constant(void)
@@ -545,7 +526,7 @@ static void parse_wide_character_constant(void)
                switch (input.c) {
                case '\\': {
                        const utf32 tc = parse_escape_sequence();
-                       grow_symbol(tc);
+                       obstack_grow_symbol(&symbol_obstack, tc);
                        break;
                }
 
@@ -560,11 +541,10 @@ static void parse_wide_character_constant(void)
 
                case EOF:
                        parse_error("EOF while parsing character constant");
-                       pp_token.type = TP_ERROR;
-                       return;
+                       goto end_of_wide_char_constant;
 
                default:
-                       grow_symbol(input.c);
+                       obstack_grow_symbol(&symbol_obstack, input.c);
                        next_char();
                        break;
                }
@@ -574,8 +554,8 @@ end_of_wide_char_constant:
        obstack_1grow(&symbol_obstack, '\0');
        size_t  size = (size_t) obstack_object_size(&symbol_obstack)-1;
        char   *string = obstack_finish(&symbol_obstack);
-       pp_token.type    = TP_WIDE_CHARACTER_CONSTANT;
-       pp_token.literal = make_string(string, size);
+       pp_token.kind          = TP_WIDE_CHARACTER_CONSTANT;
+       pp_token.string.string = make_string(string, size);
 
        if (size == 0) {
                parse_error("empty character constant");
@@ -603,11 +583,10 @@ static void parse_character_constant(void)
 
                case EOF: {
                        source_position_t source_position;
-                       source_position.input_name = pp_token.source_position.input_name;
+                       source_position.input_name = pp_token.base.source_position.input_name;
                        source_position.lineno     = start_linenr;
                        errorf(&source_position, "EOF while parsing character constant");
-                       pp_token.type = TP_ERROR;
-                       return;
+                       goto end_of_char_constant;
                }
 
                case '\'':
@@ -623,12 +602,16 @@ static void parse_character_constant(void)
        }
 
 end_of_char_constant:;
-       const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
-       const char *const string = obstack_finish(&symbol_obstack);
+       obstack_1grow(&symbol_obstack, '\0');
+       const size_t size   = (size_t)obstack_object_size(&symbol_obstack);
+       char *const  string = obstack_finish(&symbol_obstack);
+
+       pp_token.kind          = TP_CHARACTER_CONSTANT;
+       pp_token.string.string = make_string(string, size);
 
-       pp_token.type          = TP_CHARACTER_CONSTANT;
-       pp_token.literal.begin = string;
-       pp_token.literal.size  = size;
+       if (size == 0) {
+               parse_error("empty character constant");
+       }
 }
 
 #define SYMBOL_CHARS_WITHOUT_E_P \
@@ -730,14 +713,14 @@ restart:
                goto restart;
        }
        pp_token = definition->token_list[definition->expand_pos];
-       pp_token.source_position = expansion_pos;
+       pp_token.base.source_position = expansion_pos;
        ++definition->expand_pos;
 
-       if (pp_token.type != TP_IDENTIFIER)
+       if (pp_token.kind != TP_IDENTIFIER)
                return;
 
        /* if it was an identifier then we might need to expand again */
-       pp_definition_t *symbol_definition = pp_token.symbol->pp_definition;
+       pp_definition_t *const symbol_definition = pp_token.base.symbol->pp_definition;
        if (symbol_definition != NULL && !symbol_definition->is_expanding) {
                symbol_definition->parent_expansion = definition;
                symbol_definition->expand_pos       = 0;
@@ -780,20 +763,20 @@ static void skip_multiline_comment(void)
                case '*':
                        next_char();
                        if (input.c == '/') {
+                               if (input.position.lineno != input.output_line)
+                                       info.whitespace = input.position.colno;
                                next_char();
-                               info.whitespace += input.position.colno-1;
                                return;
                        }
                        break;
 
                MATCH_NEWLINE(
-                       info.at_line_begin |= !in_pp_directive;
                        break;
                )
 
                case EOF: {
                        source_position_t source_position;
-                       source_position.input_name = pp_token.source_position.input_name;
+                       source_position.input_name = pp_token.base.source_position.input_name;
                        source_position.lineno     = start_linenr;
                        errorf(&source_position, "at end of file while looking for comment end");
                        return;
@@ -841,10 +824,10 @@ static void skip_whitespace(void)
        }
 }
 
-static void eat_pp(int type)
+static void eat_pp(preprocessor_token_kind_t const type)
 {
        (void) type;
-       assert(pp_token.type == type);
+       assert(pp_token.kind == type);
        next_preprocessing_token();
 }
 
@@ -883,8 +866,8 @@ end_symbol:
 
        symbol_t *symbol = symbol_table_insert(string);
 
-       pp_token.type   = symbol->pp_ID;
-       pp_token.symbol = symbol;
+       pp_token.kind        = symbol->pp_ID;
+       pp_token.base.symbol = symbol;
 
        /* we can free the memory from symbol obstack if we already had an entry in
         * the symbol table */
@@ -929,9 +912,8 @@ end_number:
        size_t  size   = obstack_object_size(&symbol_obstack);
        char   *string = obstack_finish(&symbol_obstack);
 
-       pp_token.type          = TP_NUMBER;
-       pp_token.literal.begin = string;
-       pp_token.literal.size  = size;
+       pp_token.kind          = TP_NUMBER;
+       pp_token.number.number = make_string(string, size);
 }
 
 
@@ -943,7 +925,7 @@ end_number:
 #define MAYBE(ch, set_type)                                \
                                case ch:                                   \
                                        next_char();                           \
-                                       pp_token.type = set_type;              \
+                                       pp_token.kind = set_type;              \
                                        return;
 
 #define ELSE_CODE(code)                                    \
@@ -955,7 +937,7 @@ end_number:
 
 #define ELSE(set_type)                                     \
                ELSE_CODE(                                         \
-                       pp_token.type = set_type;                      \
+                       pp_token.kind = set_type;                      \
                )
 
 static void next_preprocessing_token(void)
@@ -968,7 +950,9 @@ static void next_preprocessing_token(void)
        info.at_line_begin  = false;
        info.had_whitespace = false;
 restart:
-       pp_token.source_position = input.position;
+       pp_token.base.source_position = input.position;
+       pp_token.base.symbol          = NULL;
+
        switch (input.c) {
        case ' ':
        case '\t':
@@ -1022,7 +1006,7 @@ restart:
                                ELSE_CODE(
                                        put_back(input.c);
                                        input.c = '.';
-                                       pp_token.type = '.';
+                                       pp_token.kind = '.';
                                )
                ELSE('.')
        case '&':
@@ -1075,7 +1059,7 @@ restart:
                                                ELSE_CODE(
                                                        put_back(input.c);
                                                        input.c = '%';
-                                                       pp_token.type = '#';
+                                                       pp_token.kind = '#';
                                                )
                                ELSE('#')
                ELSE('%')
@@ -1118,7 +1102,7 @@ restart:
                MAYBE_PROLOG
                MAYBE('#', TP_HASHHASH)
                ELSE_CODE(
-                       pp_token.type = '#';
+                       pp_token.kind = '#';
                )
 
        case '?':
@@ -1132,7 +1116,7 @@ restart:
        case ';':
        case ',':
        case '\\':
-               pp_token.type = input.c;
+               pp_token.kind = input.c;
                next_char();
                return;
 
@@ -1144,22 +1128,22 @@ restart:
                        print_line_directive(&input.position, "2");
                        goto restart;
                } else {
-                       pp_token.source_position.lineno++;
+                       pp_token.base.source_position.lineno++;
                        info.at_line_begin = true;
-                       pp_token.type = TP_EOF;
+                       pp_token.kind = TP_EOF;
                }
                return;
 
        default:
                next_char();
                if (!ignore_unknown_chars) {
-                       errorf(&pp_token.source_position, "unknown character '%c' found\n",
-                              input.c);
-                       pp_token.type = TP_ERROR;
+                       errorf(&pp_token.base.source_position,
+                              "unknown character '%c' found\n", input.c);
+                       goto restart;
                } else {
-                       pp_token.type = input.c;
+                       pp_token.kind = input.c;
+                       return;
                }
-               return;
        }
 }
 
@@ -1203,20 +1187,27 @@ static void print_line_directive(const source_position_t *pos, const char *add)
        input.output_line  = pos->lineno-1;
 }
 
-static void emit_newlines(void)
+static bool emit_newlines(void)
 {
-       unsigned delta = pp_token.source_position.lineno - input.output_line;
+       unsigned delta = pp_token.base.source_position.lineno - input.output_line;
+       if (delta == 0)
+               return false;
 
        if (delta >= 9) {
                fputc('\n', out);
-               print_line_directive(&pp_token.source_position, NULL);
+               print_line_directive(&pp_token.base.source_position, NULL);
                fputc('\n', out);
        } else {
                for (unsigned i = 0; i < delta; ++i) {
                        fputc('\n', out);
                }
        }
-       input.output_line = pp_token.source_position.lineno;
+       input.output_line = pp_token.base.source_position.lineno;
+
+       for (unsigned i = 0; i < info.whitespace; ++i)
+               fputc(' ', out);
+
+       return true;
 }
 
 static void emit_pp_token(void)
@@ -1224,43 +1215,36 @@ static void emit_pp_token(void)
        if (skip_mode)
                return;
 
-       if (info.at_line_begin) {
-               emit_newlines();
-
-               for (unsigned i = 0; i < info.whitespace; ++i)
-                       fputc(' ', out);
-
-       } else if (info.had_whitespace ||
-                          tokens_would_paste(last_token, pp_token.type)) {
+       if (!emit_newlines() &&
+           (info.had_whitespace || tokens_would_paste(last_token, pp_token.kind)))
                fputc(' ', out);
-       }
 
-       switch (pp_token.type) {
+       switch (pp_token.kind) {
        case TP_IDENTIFIER:
-               fputs(pp_token.symbol->string, out);
+               fputs(pp_token.base.symbol->string, out);
                break;
        case TP_NUMBER:
-               fputs(pp_token.literal.begin, out);
+               fputs(pp_token.number.number.begin, out);
                break;
        case TP_WIDE_STRING_LITERAL:
                fputc('L', out);
        case TP_STRING_LITERAL:
                fputc('"', out);
-               fputs(pp_token.literal.begin, out);
+               fputs(pp_token.string.string.begin, out);
                fputc('"', out);
                break;
        case TP_WIDE_CHARACTER_CONSTANT:
                fputc('L', out);
        case TP_CHARACTER_CONSTANT:
                fputc('\'', out);
-               fputs(pp_token.literal.begin, out);
+               fputs(pp_token.string.string.begin, out);
                fputc('\'', out);
                break;
        default:
-               print_pp_token_type(out, pp_token.type);
+               print_pp_token_kind(out, pp_token.kind);
                break;
        }
-       last_token = pp_token.type;
+       last_token = pp_token.kind;
 }
 
 static void eat_pp_directive(void)
@@ -1287,19 +1271,17 @@ static bool strings_equal(const string_t *string1, const string_t *string2)
 
 static bool pp_tokens_equal(const token_t *token1, const token_t *token2)
 {
-       if (token1->type != token2->type)
+       if (token1->kind != token2->kind)
                return false;
 
-       switch (token1->type) {
-       case TP_HEADERNAME:
-               /* TODO */
-               return false;
+       switch (token1->kind) {
        case TP_IDENTIFIER:
-               return token1->symbol == token2->symbol;
+               return token1->base.symbol == token2->base.symbol;
+
        case TP_NUMBER:
        case TP_CHARACTER_CONSTANT:
        case TP_STRING_LITERAL:
-               return strings_equal(&token1->literal, &token2->literal);
+               return strings_equal(&token1->string.string, &token2->string.string);
 
        default:
                return true;
@@ -1327,12 +1309,12 @@ static void parse_define_directive(void)
        eat_pp(TP_define);
        assert(obstack_object_size(&pp_obstack) == 0);
 
-       if (pp_token.type != TP_IDENTIFIER || info.at_line_begin) {
-               errorf(&pp_token.source_position,
+       if (pp_token.kind != TP_IDENTIFIER || info.at_line_begin) {
+               errorf(&pp_token.base.source_position,
                       "expected identifier after #define, got '%t'", &pp_token);
                goto error_out;
        }
-       symbol_t *symbol = pp_token.symbol;
+       symbol_t *const symbol = pp_token.base.symbol;
 
        pp_definition_t *new_definition
                = obstack_alloc(&pp_obstack, sizeof(new_definition[0]));
@@ -1349,27 +1331,27 @@ static void parse_define_directive(void)
                next_preprocessing_token();
 
                while (true) {
-                       switch (pp_token.type) {
+                       switch (pp_token.kind) {
                        case TP_DOTDOTDOT:
                                new_definition->is_variadic = true;
                                next_preprocessing_token();
-                               if (pp_token.type != ')') {
+                               if (pp_token.kind != ')') {
                                        errorf(&input.position,
                                                        "'...' not at end of macro argument list");
                                        goto error_out;
                                }
                                break;
                        case TP_IDENTIFIER:
-                               obstack_ptr_grow(&pp_obstack, pp_token.symbol);
+                               obstack_ptr_grow(&pp_obstack, pp_token.base.symbol);
                                next_preprocessing_token();
 
-                               if (pp_token.type == ',') {
+                               if (pp_token.kind == ',') {
                                        next_preprocessing_token();
                                        break;
                                }
 
-                               if (pp_token.type != ')') {
-                                       errorf(&pp_token.source_position,
+                               if (pp_token.kind != ')') {
+                                       errorf(&pp_token.base.source_position,
                                               "expected ',' or ')' after identifier, got '%t'",
                                               &pp_token);
                                        goto error_out;
@@ -1379,7 +1361,7 @@ static void parse_define_directive(void)
                                next_preprocessing_token();
                                goto finish_argument_list;
                        default:
-                               errorf(&pp_token.source_position,
+                               errorf(&pp_token.base.source_position,
                                       "expected identifier, '...' or ')' in #define argument list, got '%t'",
                                       &pp_token);
                                goto error_out;
@@ -1433,15 +1415,14 @@ static void parse_undef_directive(void)
 {
        eat_pp(TP_undef);
 
-       if (pp_token.type != TP_IDENTIFIER) {
+       if (pp_token.kind != TP_IDENTIFIER) {
                errorf(&input.position,
                       "expected identifier after #undef, got '%t'", &pp_token);
                eat_pp_directive();
                return;
        }
 
-       symbol_t *symbol = pp_token.symbol;
-       symbol->pp_definition = NULL;
+       pp_token.base.symbol->pp_definition = NULL;
        next_preprocessing_token();
 
        if (!info.at_line_begin) {
@@ -1450,19 +1431,21 @@ static void parse_undef_directive(void)
        eat_pp_directive();
 }
 
-static const char *parse_headername(void)
+static void parse_headername(void)
 {
+       const source_position_t start_position = input.position;
+       string_t                string         = {NULL, 0};
+       assert(obstack_object_size(&symbol_obstack) == 0);
+
        /* behind an #include we can have the special headername lexems.
         * They're only allowed behind an #include so they're not recognized
         * by the normal next_preprocessing_token. We handle them as a special
         * exception here */
        if (info.at_line_begin) {
                parse_error("expected headername after #include");
-               return NULL;
+               goto finish_error;
        }
 
-       assert(obstack_object_size(&symbol_obstack) == 0);
-
        /* check wether we have a "... or <... headername */
        switch (input.c) {
        case '<':
@@ -1473,7 +1456,7 @@ static const char *parse_headername(void)
                                /* fallthrough */
                        MATCH_NEWLINE(
                                parse_error("header name without closing '>'");
-                               return NULL;
+                               goto finish_error;
                        )
                        case '>':
                                next_char();
@@ -1492,7 +1475,7 @@ static const char *parse_headername(void)
                                /* fallthrough */
                        MATCH_NEWLINE(
                                parse_error("header name without closing '>'");
-                               return NULL;
+                               goto finish_error;
                        )
                        case '"':
                                next_char();
@@ -1510,21 +1493,31 @@ static const char *parse_headername(void)
 
 finished_headername:
        obstack_1grow(&symbol_obstack, '\0');
-       char *headername = obstack_finish(&symbol_obstack);
-
-       /* TODO: iterate search-path to find the file */
-
-       skip_whitespace();
-
-       return identify_string(headername);
+       const size_t size       = (size_t)obstack_object_size(&symbol_obstack);
+       char *const  headername = obstack_finish(&symbol_obstack);
+       string                  = make_string(headername, size);
+
+finish_error:
+       pp_token.base.source_position = start_position;
+       pp_token.kind                 = TP_HEADERNAME;
+       pp_token.string.string        = string;
 }
 
 static bool do_include(bool system_include, const char *headername)
 {
+       size_t headername_len = strlen(headername);
        if (!system_include) {
-               /* for "bla" includes first try current dir
-                * TODO: this isn't correct, should be the directory of the source file
-                */
+               /* put dirname of current input on obstack */
+               const char *filename   = input.position.input_name;
+               const char *last_slash = strrchr(filename, '/');
+               if (last_slash != NULL) {
+                       size_t len = last_slash - filename;
+                       obstack_grow(&symbol_obstack, filename, len + 1);
+                       obstack_grow0(&symbol_obstack, headername, headername_len);
+                       char *complete_path = obstack_finish(&symbol_obstack);
+                       headername = identify_string(complete_path);
+               }
+
                FILE *file = fopen(headername, "r");
                if (file != NULL) {
                        switch_input(file, headername);
@@ -1532,68 +1525,82 @@ static bool do_include(bool system_include, const char *headername)
                }
        }
 
-       size_t headername_len = strlen(headername);
-       assert(obstack_object_size(&pp_obstack) == 0);
+       assert(obstack_object_size(&symbol_obstack) == 0);
        /* check searchpath */
        for (searchpath_entry_t *entry = searchpath; entry != NULL;
             entry = entry->next) {
            const char *path = entry->path;
            size_t      len  = strlen(path);
-               obstack_grow(&pp_obstack, path, len);
+               obstack_grow(&symbol_obstack, path, len);
                if (path[len-1] != '/')
-                       obstack_1grow(&pp_obstack, '/');
-               obstack_grow(&pp_obstack, headername, headername_len+1);
+                       obstack_1grow(&symbol_obstack, '/');
+               obstack_grow(&symbol_obstack, headername, headername_len+1);
 
-               char *complete_path = obstack_finish(&pp_obstack);
+               char *complete_path = obstack_finish(&symbol_obstack);
                FILE *file          = fopen(complete_path, "r");
                if (file != NULL) {
                        const char *filename = identify_string(complete_path);
                        switch_input(file, filename);
                        return true;
+               } else {
+                       obstack_free(&symbol_obstack, complete_path);
                }
-               obstack_free(&pp_obstack, complete_path);
        }
 
        return false;
 }
 
+/* read till next newline character, only for parse_include_directive(),
+ * use eat_pp_directive() in all other cases */
+static void skip_till_newline(void)
+{
+       /* skip till newline */
+       while (true) {
+               switch (input.c) {
+               MATCH_NEWLINE(
+                       return;
+               )
+               case EOF:
+                       return;
+               }
+               next_char();
+       }
+}
+
 static bool parse_include_directive(void)
 {
        /* don't eat the TP_include here!
         * we need an alternative parsing for the next token */
        skip_whitespace();
        bool system_include = input.c == '<';
-       const char *headername = parse_headername();
-       if (headername == NULL) {
+       parse_headername();
+       string_t headername = pp_token.string.string;
+       if (headername.begin == NULL) {
                eat_pp_directive();
                return false;
        }
 
+       skip_whitespace();
        if (!info.at_line_begin) {
-               warningf(WARN_OTHER, &pp_token.source_position, "extra tokens at end of #include directive");
-               eat_pp_directive();
+               warningf(WARN_OTHER, &pp_token.base.source_position,
+                        "extra tokens at end of #include directive");
+               skip_till_newline();
        }
 
        if (n_inputs > INCLUDE_LIMIT) {
-               errorf(&pp_token.source_position, "#include nested too deeply");
+               errorf(&pp_token.base.source_position, "#include nested too deeply");
                /* eat \n or EOF */
                next_preprocessing_token();
                return false;
        }
 
-       /* we have to reenable space counting and macro expansion here,
-        * because it is still disabled in directive parsing,
-        * but we will trigger a preprocessing token reading of the new file
-        * now and need expansions/space counting */
-       in_pp_directive = false;
-
        /* switch inputs */
        emit_newlines();
        push_input();
-       bool res = do_include(system_include, headername);
+       bool res = do_include(system_include, pp_token.string.string.begin);
        if (!res) {
-               errorf(&pp_token.source_position,
-                      "failed including '%s': %s", headername, strerror(errno));
+               errorf(&pp_token.base.source_position,
+                      "failed including '%S': %s", pp_token.string, strerror(errno));
                pop_restore_input();
                return false;
        }
@@ -1635,20 +1642,20 @@ static void check_unclosed_conditionals(void)
 
 static void parse_ifdef_ifndef_directive(void)
 {
-       bool is_ifndef = (pp_token.type == TP_ifndef);
+       bool is_ifndef = (pp_token.kind == TP_ifndef);
        bool condition;
        next_preprocessing_token();
 
        if (skip_mode) {
                eat_pp_directive();
                pp_conditional_t *conditional = push_conditional();
-               conditional->source_position  = pp_token.source_position;
+               conditional->source_position  = pp_token.base.source_position;
                conditional->skip             = true;
                return;
        }
 
-       if (pp_token.type != TP_IDENTIFIER || info.at_line_begin) {
-               errorf(&pp_token.source_position,
+       if (pp_token.kind != TP_IDENTIFIER || info.at_line_begin) {
+               errorf(&pp_token.base.source_position,
                       "expected identifier after #%s, got '%t'",
                       is_ifndef ? "ifndef" : "ifdef", &pp_token);
                eat_pp_directive();
@@ -1656,23 +1663,21 @@ static void parse_ifdef_ifndef_directive(void)
                /* just take the true case in the hope to avoid further errors */
                condition = true;
        } else {
-               symbol_t        *symbol        = pp_token.symbol;
-               pp_definition_t *pp_definition = symbol->pp_definition;
+               /* evaluate wether we are in true or false case */
+               condition = !pp_token.base.symbol->pp_definition == is_ifndef;
+
                next_preprocessing_token();
 
                if (!info.at_line_begin) {
-                       errorf(&pp_token.source_position,
+                       errorf(&pp_token.base.source_position,
                               "extra tokens at end of #%s",
                               is_ifndef ? "ifndef" : "ifdef");
                        eat_pp_directive();
                }
-
-               /* evaluate wether we are in true or false case */
-               condition = is_ifndef ? pp_definition == NULL : pp_definition != NULL;
        }
 
        pp_conditional_t *conditional = push_conditional();
-       conditional->source_position  = pp_token.source_position;
+       conditional->source_position  = pp_token.base.source_position;
        conditional->condition        = condition;
 
        if (!condition) {
@@ -1686,19 +1691,19 @@ static void parse_else_directive(void)
 
        if (!info.at_line_begin) {
                if (!skip_mode) {
-                       warningf(WARN_OTHER, &pp_token.source_position, "extra tokens at end of #else");
+                       warningf(WARN_OTHER, &pp_token.base.source_position, "extra tokens at end of #else");
                }
                eat_pp_directive();
        }
 
        pp_conditional_t *conditional = conditional_stack;
        if (conditional == NULL) {
-               errorf(&pp_token.source_position, "#else without prior #if");
+               errorf(&pp_token.base.source_position, "#else without prior #if");
                return;
        }
 
        if (conditional->in_else) {
-               errorf(&pp_token.source_position,
+               errorf(&pp_token.base.source_position,
                       "#else after #else (condition started %P)",
                       conditional->source_position);
                skip_mode = true;
@@ -1709,7 +1714,7 @@ static void parse_else_directive(void)
        if (!conditional->skip) {
                skip_mode = conditional->condition;
        }
-       conditional->source_position = pp_token.source_position;
+       conditional->source_position = pp_token.base.source_position;
 }
 
 static void parse_endif_directive(void)
@@ -1718,14 +1723,14 @@ static void parse_endif_directive(void)
 
        if (!info.at_line_begin) {
                if (!skip_mode) {
-                       warningf(WARN_OTHER, &pp_token.source_position, "extra tokens at end of #endif");
+                       warningf(WARN_OTHER, &pp_token.base.source_position, "extra tokens at end of #endif");
                }
                eat_pp_directive();
        }
 
        pp_conditional_t *conditional = conditional_stack;
        if (conditional == NULL) {
-               errorf(&pp_token.source_position, "#endif without prior #if");
+               errorf(&pp_token.base.source_position, "#endif without prior #if");
                return;
        }
 
@@ -1737,11 +1742,15 @@ static void parse_endif_directive(void)
 
 static void parse_preprocessing_directive(void)
 {
-       in_pp_directive = true;
        eat_pp('#');
 
+       if (info.at_line_begin) {
+               /* empty directive */
+               return;
+       }
+
        if (skip_mode) {
-               switch (pp_token.type) {
+               switch (pp_token.kind) {
                case TP_ifdef:
                case TP_ifndef:
                        parse_ifdef_ifndef_directive();
@@ -1757,7 +1766,7 @@ static void parse_preprocessing_directive(void)
                        break;
                }
        } else {
-               switch (pp_token.type) {
+               switch (pp_token.kind) {
                case TP_define:
                        parse_define_directive();
                        break;
@@ -1782,14 +1791,13 @@ static void parse_preprocessing_directive(void)
                                /* the nop directive "#" */
                                break;
                        }
-                       errorf(&pp_token.source_position,
+                       errorf(&pp_token.base.source_position,
                                   "invalid preprocessing directive #%t", &pp_token);
                        eat_pp_directive();
                        break;
                }
        }
 
-       in_pp_directive = false;
        assert(info.at_line_begin);
 }
 
@@ -1852,6 +1860,7 @@ int pptest_main(int argc, char **argv)
 
        /* simplistic commandline parser */
        const char *filename = NULL;
+       const char *output = NULL;
        for (int i = 1; i < argc; ++i) {
                const char *opt = argv[i];
                if (streq(opt, "-I")) {
@@ -1859,6 +1868,9 @@ int pptest_main(int argc, char **argv)
                        continue;
                } else if (streq(opt, "-E")) {
                        /* ignore */
+               } else if (streq(opt, "-o")) {
+                       output = argv[++i];
+                       continue;
                } else if (opt[0] == '-') {
                        fprintf(stderr, "Unknown option '%s'\n", opt);
                } else {
@@ -1872,7 +1884,15 @@ int pptest_main(int argc, char **argv)
                return 1;
        }
 
-       out = stdout;
+       if (output == NULL) {
+               out = stdout;
+       } else {
+               out = fopen(output, "w");
+               if (out == NULL) {
+                       fprintf(stderr, "Couldn't open output '%s'\n", output);
+                       return 1;
+               }
+       }
 
        /* just here for gcc compatibility */
        fprintf(out, "# 1 \"%s\"\n", filename);
@@ -1887,35 +1907,35 @@ int pptest_main(int argc, char **argv)
        switch_input(file, filename);
 
        while (true) {
-               if (pp_token.type == '#' && info.at_line_begin) {
+               if (pp_token.kind == '#' && info.at_line_begin) {
                        parse_preprocessing_directive();
                        continue;
-               } else if (pp_token.type == TP_EOF) {
+               } else if (pp_token.kind == TP_EOF) {
                        goto end_of_main_loop;
-               } else if (pp_token.type == TP_IDENTIFIER && !in_pp_directive) {
-                       symbol_t *symbol = pp_token.symbol;
-                       pp_definition_t *pp_definition = symbol->pp_definition;
+               } else if (pp_token.kind == TP_IDENTIFIER) {
+                       symbol_t        *const symbol        = pp_token.base.symbol;
+                       pp_definition_t *const pp_definition = symbol->pp_definition;
                        if (pp_definition != NULL && !pp_definition->is_expanding) {
-                               expansion_pos = pp_token.source_position;
+                               expansion_pos = pp_token.base.source_position;
                                if (pp_definition->has_parameters) {
-                                       source_position_t position = pp_token.source_position;
+                                       source_position_t position = pp_token.base.source_position;
                                        add_token_info_t old_info = info;
                                        next_preprocessing_token();
                                        add_token_info_t new_info = info;
 
                                        /* no opening brace -> no expansion */
-                                       if (pp_token.type == '(') {
+                                       if (pp_token.kind == '(') {
                                                eat_pp('(');
 
                                                /* parse arguments (TODO) */
-                                               while (pp_token.type != TP_EOF && pp_token.type != ')')
+                                               while (pp_token.kind != TP_EOF && pp_token.kind != ')')
                                                        next_preprocessing_token();
                                        } else {
                                                token_t next_token = pp_token;
                                                /* restore identifier token */
-                                               pp_token.type            = TP_IDENTIFIER;
-                                               pp_token.symbol          = symbol;
-                                               pp_token.source_position = position;
+                                               pp_token.kind                 = TP_IDENTIFIER;
+                                               pp_token.base.symbol          = symbol;
+                                               pp_token.base.source_position = position;
                                                info = old_info;
                                                emit_pp_token();
 
@@ -1941,6 +1961,8 @@ end_of_main_loop:
        fputc('\n', out);
        check_unclosed_conditionals();
        close_input();
+       if (out != stdout)
+               fclose(out);
 
        obstack_free(&input_obstack, NULL);
        obstack_free(&pp_obstack, NULL);