X-Git-Url: http://nsz.repo.hu/git/?a=blobdiff_plain;f=preprocessor.c;h=f829db422ca4591c471cc4dac8235b9b8af04896;hb=ce8747dcdb81cc4c5000092807102283a1e62b68;hp=48825afa449342992272187ad3b8a7620e9dbebd;hpb=15c8bcaebaca1cdebe3744c172c820fbe0b3bba6;p=cparser

diff --git a/preprocessor.c b/preprocessor.c
index 48825af..f829db4 100644
--- a/preprocessor.c
+++ b/preprocessor.c
@@ -91,7 +91,6 @@ static pp_conditional_t *conditional_stack;
 static token_t           pp_token;
 static bool              resolve_escape_sequences = false;
 static bool              ignore_unknown_chars     = true;
-static bool              in_pp_directive;
 static bool              skip_mode;
 static FILE             *out;
 static struct obstack    pp_obstack;
@@ -100,7 +99,7 @@ static const char       *printed_input_name = NULL;
 static source_position_t expansion_pos;
 static pp_definition_t  *current_expansion  = NULL;
 static strset_t          stringset;
-static preprocessor_token_type_t last_token = TP_ERROR;
+static preprocessor_token_kind_t last_token;
 
 static searchpath_entry_t *searchpath;
 
@@ -188,15 +187,14 @@ static void pop_restore_input(void)
  */
 static void parse_error(const char *msg)
 {
-	errorf(&pp_token.source_position,  "%s", msg);
+	errorf(&pp_token.base.source_position,  "%s", msg);
 }
 
 static inline void next_real_char(void)
 {
 	assert(input.bufpos <= input.bufend);
 	if (input.bufpos >= input.bufend) {
-		size_t n = decode(input.input, input.buf + MAX_PUTBACK,
-		                  sizeof(input.buf)/sizeof(input.buf[0]) - MAX_PUTBACK);
+		size_t const n = decode(input.input, input.buf + MAX_PUTBACK, lengthof(input.buf) - MAX_PUTBACK);
 		if (n == 0) {
 			input.c = EOF;
 			return;
@@ -362,50 +360,41 @@ static int digit_value(int digit)
  *
  * @param first_digit  the already read first digit
  */
-static int parse_octal_sequence(const int first_digit)
+static utf32 parse_octal_sequence(const utf32 first_digit)
 {
 	assert(is_octal_digit(first_digit));
-	int value = digit_value(first_digit);
+	utf32 value = digit_value(first_digit);
 	if (!is_octal_digit(input.c)) return value;
 	value = 8 * value + digit_value(input.c);
 	next_char();
 	if (!is_octal_digit(input.c)) return value;
 	value = 8 * value + digit_value(input.c);
 	next_char();
+	return value;
 
-	if (char_is_signed) {
-		return (signed char) value;
-	} else {
-		return (unsigned char) value;
-	}
 }
 
 /**
  * Parses a hex character sequence.
  */
-static int parse_hex_sequence(void)
+static utf32 parse_hex_sequence(void)
 {
-	int value = 0;
+	utf32 value = 0;
 	while (isxdigit(input.c)) {
 		value = 16 * value + digit_value(input.c);
 		next_char();
 	}
-
-	if (char_is_signed) {
-		return (signed char) value;
-	} else {
-		return (unsigned char) value;
-	}
+	return value;
 }
 
 /**
  * Parse an escape sequence.
  */
-static int parse_escape_sequence(void)
+static utf32 parse_escape_sequence(void)
 {
 	eat('\\');
 
-	int ec = input.c;
+	utf32 const ec = input.c;
 	next_char();
 
 	switch (ec) {
@@ -434,30 +423,23 @@ static int parse_escape_sequence(void)
 	case EOF:
 		parse_error("reached end of file while parsing escape sequence");
 		return EOF;
-	default:
-		parse_error("unknown escape sequence");
+	/* \E is not documented, but handled, by GCC.  It is acceptable according
+	 * to §6.11.4, whereas \e is not. */
+	case 'E':
+	case 'e':
+		if (c_mode & _GNUC)
+			return 27;   /* hopefully 27 is ALWAYS the code for ESCAPE */
+		break;
+	case 'u':
+	case 'U':
+		parse_error("universal character parsing not implemented yet");
 		return EOF;
+	default:
+		break;
 	}
-}
-
-static void grow_symbol(utf32 const tc)
-{
-	struct obstack *const o  = &symbol_obstack;
-	if (tc < 0x80U) {
-		obstack_1grow(o, tc);
-	} else if (tc < 0x800) {
-		obstack_1grow(o, 0xC0 | (tc >> 6));
-		obstack_1grow(o, 0x80 | (tc & 0x3F));
-	} else if (tc < 0x10000) {
-		obstack_1grow(o, 0xE0 | ( tc >> 12));
-		obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
-		obstack_1grow(o, 0x80 | ( tc        & 0x3F));
-	} else {
-		obstack_1grow(o, 0xF0 | ( tc >> 18));
-		obstack_1grow(o, 0x80 | ((tc >> 12) & 0x3F));
-		obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
-		obstack_1grow(o, 0x80 | ( tc        & 0x3F));
-	}
+	/* §6.4.4.4:8 footnote 64 */
+	parse_error("unknown escape sequence");
+	return EOF;
 }
 
 static const char *identify_string(char *string)
@@ -499,11 +481,10 @@ static void parse_string_literal(void)
 
 		case EOF: {
 			source_position_t source_position;
-			source_position.input_name = pp_token.source_position.input_name;
+			source_position.input_name = pp_token.base.source_position.input_name;
 			source_position.lineno     = start_linenr;
 			errorf(&source_position, "string has no end");
-			pp_token.type = TP_ERROR;
-			return;
+			goto end_of_string;
 		}
 
 		case '"':
@@ -511,7 +492,7 @@ static void parse_string_literal(void)
 			goto end_of_string;
 
 		default:
-			grow_symbol(input.c);
+			obstack_grow_symbol(&symbol_obstack, input.c);
 			next_char();
 			break;
 		}
@@ -523,8 +504,8 @@ end_of_string:
 	const size_t size   = (size_t)obstack_object_size(&symbol_obstack);
 	char *const  string = obstack_finish(&symbol_obstack);
 
-	pp_token.type    = TP_STRING_LITERAL;
-	pp_token.literal = make_string(string, size);
+	pp_token.kind          = TP_STRING_LITERAL;
+	pp_token.string.string = make_string(string, size);
 }
 
 /**
@@ -533,8 +514,8 @@ end_of_string:
 static void parse_wide_string_literal(void)
 {
 	parse_string_literal();
-	if (pp_token.type == TP_STRING_LITERAL)
-		pp_token.type = TP_WIDE_STRING_LITERAL;
+	if (pp_token.kind == TP_STRING_LITERAL)
+		pp_token.kind = TP_WIDE_STRING_LITERAL;
 }
 
 static void parse_wide_character_constant(void)
@@ -545,7 +526,7 @@ static void parse_wide_character_constant(void)
 		switch (input.c) {
 		case '\\': {
 			const utf32 tc = parse_escape_sequence();
-			grow_symbol(tc);
+			obstack_grow_symbol(&symbol_obstack, tc);
 			break;
 		}
 
@@ -560,11 +541,10 @@ static void parse_wide_character_constant(void)
 
 		case EOF:
 			parse_error("EOF while parsing character constant");
-			pp_token.type = TP_ERROR;
-			return;
+			goto end_of_wide_char_constant;
 
 		default:
-			grow_symbol(input.c);
+			obstack_grow_symbol(&symbol_obstack, input.c);
 			next_char();
 			break;
 		}
@@ -574,8 +554,8 @@ end_of_wide_char_constant:
 	obstack_1grow(&symbol_obstack, '\0');
 	size_t  size = (size_t) obstack_object_size(&symbol_obstack)-1;
 	char   *string = obstack_finish(&symbol_obstack);
-	pp_token.type    = TP_WIDE_CHARACTER_CONSTANT;
-	pp_token.literal = make_string(string, size);
+	pp_token.kind          = TP_WIDE_CHARACTER_CONSTANT;
+	pp_token.string.string = make_string(string, size);
 
 	if (size == 0) {
 		parse_error("empty character constant");
@@ -603,11 +583,10 @@ static void parse_character_constant(void)
 
 		case EOF: {
 			source_position_t source_position;
-			source_position.input_name = pp_token.source_position.input_name;
+			source_position.input_name = pp_token.base.source_position.input_name;
 			source_position.lineno     = start_linenr;
 			errorf(&source_position, "EOF while parsing character constant");
-			pp_token.type = TP_ERROR;
-			return;
+			goto end_of_char_constant;
 		}
 
 		case '\'':
@@ -623,12 +602,16 @@ static void parse_character_constant(void)
 	}
 
 end_of_char_constant:;
-	const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
-	const char *const string = obstack_finish(&symbol_obstack);
+	obstack_1grow(&symbol_obstack, '\0');
+	const size_t size   = (size_t)obstack_object_size(&symbol_obstack);
+	char *const  string = obstack_finish(&symbol_obstack);
+
+	pp_token.kind          = TP_CHARACTER_CONSTANT;
+	pp_token.string.string = make_string(string, size);
 
-	pp_token.type          = TP_CHARACTER_CONSTANT;
-	pp_token.literal.begin = string;
-	pp_token.literal.size  = size;
+	if (size == 0) {
+		parse_error("empty character constant");
+	}
 }
 
 #define SYMBOL_CHARS_WITHOUT_E_P \
@@ -730,14 +713,14 @@ restart:
 		goto restart;
 	}
 	pp_token = definition->token_list[definition->expand_pos];
-	pp_token.source_position = expansion_pos;
+	pp_token.base.source_position = expansion_pos;
 	++definition->expand_pos;
 
-	if (pp_token.type != TP_IDENTIFIER)
+	if (pp_token.kind != TP_IDENTIFIER)
 		return;
 
 	/* if it was an identifier then we might need to expand again */
-	pp_definition_t *symbol_definition = pp_token.symbol->pp_definition;
+	pp_definition_t *const symbol_definition = pp_token.base.symbol->pp_definition;
 	if (symbol_definition != NULL && !symbol_definition->is_expanding) {
 		symbol_definition->parent_expansion = definition;
 		symbol_definition->expand_pos       = 0;
@@ -780,20 +763,20 @@ static void skip_multiline_comment(void)
 		case '*':
 			next_char();
 			if (input.c == '/') {
+				if (input.position.lineno != input.output_line)
+					info.whitespace = input.position.colno;
 				next_char();
-				info.whitespace += input.position.colno-1;
 				return;
 			}
 			break;
 
 		MATCH_NEWLINE(
-			info.at_line_begin |= !in_pp_directive;
 			break;
 		)
 
 		case EOF: {
 			source_position_t source_position;
-			source_position.input_name = pp_token.source_position.input_name;
+			source_position.input_name = pp_token.base.source_position.input_name;
 			source_position.lineno     = start_linenr;
 			errorf(&source_position, "at end of file while looking for comment end");
 			return;
@@ -841,10 +824,10 @@ static void skip_whitespace(void)
 	}
 }
 
-static void eat_pp(int type)
+static void eat_pp(preprocessor_token_kind_t const type)
 {
 	(void) type;
-	assert(pp_token.type == type);
+	assert(pp_token.kind == type);
 	next_preprocessing_token();
 }
 
@@ -883,8 +866,8 @@ end_symbol:
 
 	symbol_t *symbol = symbol_table_insert(string);
 
-	pp_token.type   = symbol->pp_ID;
-	pp_token.symbol = symbol;
+	pp_token.kind        = symbol->pp_ID;
+	pp_token.base.symbol = symbol;
 
 	/* we can free the memory from symbol obstack if we already had an entry in
 	 * the symbol table */
@@ -929,9 +912,8 @@ end_number:
 	size_t  size   = obstack_object_size(&symbol_obstack);
 	char   *string = obstack_finish(&symbol_obstack);
 
-	pp_token.type          = TP_NUMBER;
-	pp_token.literal.begin = string;
-	pp_token.literal.size  = size;
+	pp_token.kind          = TP_NUMBER;
+	pp_token.number.number = make_string(string, size);
 }
 
 
@@ -943,7 +925,7 @@ end_number:
 #define MAYBE(ch, set_type)                                \
 				case ch:                                   \
 					next_char();                           \
-					pp_token.type = set_type;              \
+					pp_token.kind = set_type;              \
 					return;
 
 #define ELSE_CODE(code)                                    \
@@ -955,7 +937,7 @@ end_number:
 
 #define ELSE(set_type)                                     \
 		ELSE_CODE(                                         \
-			pp_token.type = set_type;                      \
+			pp_token.kind = set_type;                      \
 		)
 
 static void next_preprocessing_token(void)
@@ -968,7 +950,9 @@ static void next_preprocessing_token(void)
 	info.at_line_begin  = false;
 	info.had_whitespace = false;
 restart:
-	pp_token.source_position = input.position;
+	pp_token.base.source_position = input.position;
+	pp_token.base.symbol          = NULL;
+
 	switch (input.c) {
 	case ' ':
 	case '\t':
@@ -1022,7 +1006,7 @@ restart:
 				ELSE_CODE(
 					put_back(input.c);
 					input.c = '.';
-					pp_token.type = '.';
+					pp_token.kind = '.';
 				)
 		ELSE('.')
 	case '&':
@@ -1075,7 +1059,7 @@ restart:
 						ELSE_CODE(
 							put_back(input.c);
 							input.c = '%';
-							pp_token.type = '#';
+							pp_token.kind = '#';
 						)
 				ELSE('#')
 		ELSE('%')
@@ -1118,7 +1102,7 @@ restart:
 		MAYBE_PROLOG
 		MAYBE('#', TP_HASHHASH)
 		ELSE_CODE(
-			pp_token.type = '#';
+			pp_token.kind = '#';
 		)
 
 	case '?':
@@ -1132,7 +1116,7 @@ restart:
 	case ';':
 	case ',':
 	case '\\':
-		pp_token.type = input.c;
+		pp_token.kind = input.c;
 		next_char();
 		return;
 
@@ -1144,22 +1128,22 @@ restart:
 			print_line_directive(&input.position, "2");
 			goto restart;
 		} else {
-			pp_token.source_position.lineno++;
+			pp_token.base.source_position.lineno++;
 			info.at_line_begin = true;
-			pp_token.type = TP_EOF;
+			pp_token.kind = TP_EOF;
 		}
 		return;
 
 	default:
 		next_char();
 		if (!ignore_unknown_chars) {
-			errorf(&pp_token.source_position, "unknown character '%c' found\n",
-			       input.c);
-			pp_token.type = TP_ERROR;
+			errorf(&pp_token.base.source_position,
+			       "unknown character '%c' found\n", input.c);
+			goto restart;
 		} else {
-			pp_token.type = input.c;
+			pp_token.kind = input.c;
+			return;
 		}
-		return;
 	}
 }
 
@@ -1203,20 +1187,27 @@ static void print_line_directive(const source_position_t *pos, const char *add)
 	input.output_line  = pos->lineno-1;
 }
 
-static void emit_newlines(void)
+static bool emit_newlines(void)
 {
-	unsigned delta = pp_token.source_position.lineno - input.output_line;
+	unsigned delta = pp_token.base.source_position.lineno - input.output_line;
+	if (delta == 0)
+		return false;
 
 	if (delta >= 9) {
 		fputc('\n', out);
-		print_line_directive(&pp_token.source_position, NULL);
+		print_line_directive(&pp_token.base.source_position, NULL);
 		fputc('\n', out);
 	} else {
 		for (unsigned i = 0; i < delta; ++i) {
 			fputc('\n', out);
 		}
 	}
-	input.output_line = pp_token.source_position.lineno;
+	input.output_line = pp_token.base.source_position.lineno;
+
+	for (unsigned i = 0; i < info.whitespace; ++i)
+		fputc(' ', out);
+
+	return true;
 }
 
 static void emit_pp_token(void)
@@ -1224,43 +1215,36 @@ static void emit_pp_token(void)
 	if (skip_mode)
 		return;
 
-	if (info.at_line_begin) {
-		emit_newlines();
-
-		for (unsigned i = 0; i < info.whitespace; ++i)
-			fputc(' ', out);
-
-	} else if (info.had_whitespace ||
-			   tokens_would_paste(last_token, pp_token.type)) {
+	if (!emit_newlines() &&
+	    (info.had_whitespace || tokens_would_paste(last_token, pp_token.kind)))
 		fputc(' ', out);
-	}
 
-	switch (pp_token.type) {
+	switch (pp_token.kind) {
 	case TP_IDENTIFIER:
-		fputs(pp_token.symbol->string, out);
+		fputs(pp_token.base.symbol->string, out);
 		break;
 	case TP_NUMBER:
-		fputs(pp_token.literal.begin, out);
+		fputs(pp_token.number.number.begin, out);
 		break;
 	case TP_WIDE_STRING_LITERAL:
 		fputc('L', out);
 	case TP_STRING_LITERAL:
 		fputc('"', out);
-		fputs(pp_token.literal.begin, out);
+		fputs(pp_token.string.string.begin, out);
 		fputc('"', out);
 		break;
 	case TP_WIDE_CHARACTER_CONSTANT:
 		fputc('L', out);
 	case TP_CHARACTER_CONSTANT:
 		fputc('\'', out);
-		fputs(pp_token.literal.begin, out);
+		fputs(pp_token.string.string.begin, out);
 		fputc('\'', out);
 		break;
 	default:
-		print_pp_token_type(out, pp_token.type);
+		print_pp_token_kind(out, pp_token.kind);
 		break;
 	}
-	last_token = pp_token.type;
+	last_token = pp_token.kind;
 }
 
 static void eat_pp_directive(void)
@@ -1287,19 +1271,17 @@ static bool strings_equal(const string_t *string1, const string_t *string2)
 
 static bool pp_tokens_equal(const token_t *token1, const token_t *token2)
 {
-	if (token1->type != token2->type)
+	if (token1->kind != token2->kind)
 		return false;
 
-	switch (token1->type) {
-	case TP_HEADERNAME:
-		/* TODO */
-		return false;
+	switch (token1->kind) {
 	case TP_IDENTIFIER:
-		return token1->symbol == token2->symbol;
+		return token1->base.symbol == token2->base.symbol;
+
 	case TP_NUMBER:
 	case TP_CHARACTER_CONSTANT:
 	case TP_STRING_LITERAL:
-		return strings_equal(&token1->literal, &token2->literal);
+		return strings_equal(&token1->string.string, &token2->string.string);
 
 	default:
 		return true;
@@ -1327,12 +1309,12 @@ static void parse_define_directive(void)
 	eat_pp(TP_define);
 	assert(obstack_object_size(&pp_obstack) == 0);
 
-	if (pp_token.type != TP_IDENTIFIER || info.at_line_begin) {
-		errorf(&pp_token.source_position,
+	if (pp_token.kind != TP_IDENTIFIER || info.at_line_begin) {
+		errorf(&pp_token.base.source_position,
 		       "expected identifier after #define, got '%t'", &pp_token);
 		goto error_out;
 	}
-	symbol_t *symbol = pp_token.symbol;
+	symbol_t *const symbol = pp_token.base.symbol;
 
 	pp_definition_t *new_definition
 		= obstack_alloc(&pp_obstack, sizeof(new_definition[0]));
@@ -1349,27 +1331,27 @@ static void parse_define_directive(void)
 		next_preprocessing_token();
 
 		while (true) {
-			switch (pp_token.type) {
+			switch (pp_token.kind) {
 			case TP_DOTDOTDOT:
 				new_definition->is_variadic = true;
 				next_preprocessing_token();
-				if (pp_token.type != ')') {
+				if (pp_token.kind != ')') {
 					errorf(&input.position,
 							"'...' not at end of macro argument list");
 					goto error_out;
 				}
 				break;
 			case TP_IDENTIFIER:
-				obstack_ptr_grow(&pp_obstack, pp_token.symbol);
+				obstack_ptr_grow(&pp_obstack, pp_token.base.symbol);
 				next_preprocessing_token();
 
-				if (pp_token.type == ',') {
+				if (pp_token.kind == ',') {
 					next_preprocessing_token();
 					break;
 				}
 
-				if (pp_token.type != ')') {
-					errorf(&pp_token.source_position,
+				if (pp_token.kind != ')') {
+					errorf(&pp_token.base.source_position,
 					       "expected ',' or ')' after identifier, got '%t'",
 					       &pp_token);
 					goto error_out;
@@ -1379,7 +1361,7 @@ static void parse_define_directive(void)
 				next_preprocessing_token();
 				goto finish_argument_list;
 			default:
-				errorf(&pp_token.source_position,
+				errorf(&pp_token.base.source_position,
 				       "expected identifier, '...' or ')' in #define argument list, got '%t'",
 				       &pp_token);
 				goto error_out;
@@ -1433,15 +1415,14 @@ static void parse_undef_directive(void)
 {
 	eat_pp(TP_undef);
 
-	if (pp_token.type != TP_IDENTIFIER) {
+	if (pp_token.kind != TP_IDENTIFIER) {
 		errorf(&input.position,
 		       "expected identifier after #undef, got '%t'", &pp_token);
 		eat_pp_directive();
 		return;
 	}
 
-	symbol_t *symbol = pp_token.symbol;
-	symbol->pp_definition = NULL;
+	pp_token.base.symbol->pp_definition = NULL;
 	next_preprocessing_token();
 
 	if (!info.at_line_begin) {
@@ -1450,19 +1431,21 @@ static void parse_undef_directive(void)
 	eat_pp_directive();
 }
 
-static const char *parse_headername(void)
+static void parse_headername(void)
 {
+	const source_position_t start_position = input.position;
+	string_t                string         = {NULL, 0};
+	assert(obstack_object_size(&symbol_obstack) == 0);
+
 	/* behind an #include we can have the special headername lexems.
 	 * They're only allowed behind an #include so they're not recognized
 	 * by the normal next_preprocessing_token. We handle them as a special
 	 * exception here */
 	if (info.at_line_begin) {
 		parse_error("expected headername after #include");
-		return NULL;
+		goto finish_error;
 	}
 
-	assert(obstack_object_size(&symbol_obstack) == 0);
-
 	/* check wether we have a "... or <... headername */
 	switch (input.c) {
 	case '<':
@@ -1473,7 +1456,7 @@ static const char *parse_headername(void)
 				/* fallthrough */
 			MATCH_NEWLINE(
 				parse_error("header name without closing '>'");
-				return NULL;
+				goto finish_error;
 			)
 			case '>':
 				next_char();
@@ -1492,7 +1475,7 @@ static const char *parse_headername(void)
 				/* fallthrough */
 			MATCH_NEWLINE(
 				parse_error("header name without closing '>'");
-				return NULL;
+				goto finish_error;
 			)
 			case '"':
 				next_char();
@@ -1510,21 +1493,31 @@ static const char *parse_headername(void)
 
 finished_headername:
 	obstack_1grow(&symbol_obstack, '\0');
-	char *headername = obstack_finish(&symbol_obstack);
-
-	/* TODO: iterate search-path to find the file */
-
-	skip_whitespace();
-
-	return identify_string(headername);
+	const size_t size       = (size_t)obstack_object_size(&symbol_obstack);
+	char *const  headername = obstack_finish(&symbol_obstack);
+	string                  = make_string(headername, size);
+
+finish_error:
+	pp_token.base.source_position = start_position;
+	pp_token.kind                 = TP_HEADERNAME;
+	pp_token.string.string        = string;
 }
 
 static bool do_include(bool system_include, const char *headername)
 {
+	size_t headername_len = strlen(headername);
 	if (!system_include) {
-		/* for "bla" includes first try current dir
-		 * TODO: this isn't correct, should be the directory of the source file
-		 */
+		/* put dirname of current input on obstack */
+		const char *filename   = input.position.input_name;
+		const char *last_slash = strrchr(filename, '/');
+		if (last_slash != NULL) {
+			size_t len = last_slash - filename;
+			obstack_grow(&symbol_obstack, filename, len + 1);
+			obstack_grow0(&symbol_obstack, headername, headername_len);
+			char *complete_path = obstack_finish(&symbol_obstack);
+			headername = identify_string(complete_path);
+		}
+
 		FILE *file = fopen(headername, "r");
 		if (file != NULL) {
 			switch_input(file, headername);
@@ -1532,68 +1525,82 @@ static bool do_include(bool system_include, const char *headername)
 		}
 	}
 
-	size_t headername_len = strlen(headername);
-	assert(obstack_object_size(&pp_obstack) == 0);
+	assert(obstack_object_size(&symbol_obstack) == 0);
 	/* check searchpath */
 	for (searchpath_entry_t *entry = searchpath; entry != NULL;
 	     entry = entry->next) {
 	    const char *path = entry->path;
 	    size_t      len  = strlen(path);
-		obstack_grow(&pp_obstack, path, len);
+		obstack_grow(&symbol_obstack, path, len);
 		if (path[len-1] != '/')
-			obstack_1grow(&pp_obstack, '/');
-		obstack_grow(&pp_obstack, headername, headername_len+1);
+			obstack_1grow(&symbol_obstack, '/');
+		obstack_grow(&symbol_obstack, headername, headername_len+1);
 
-		char *complete_path = obstack_finish(&pp_obstack);
+		char *complete_path = obstack_finish(&symbol_obstack);
 		FILE *file          = fopen(complete_path, "r");
 		if (file != NULL) {
 			const char *filename = identify_string(complete_path);
 			switch_input(file, filename);
 			return true;
+		} else {
+			obstack_free(&symbol_obstack, complete_path);
 		}
-		obstack_free(&pp_obstack, complete_path);
 	}
 
 	return false;
 }
 
+/* read till next newline character, only for parse_include_directive(),
+ * use eat_pp_directive() in all other cases */
+static void skip_till_newline(void)
+{
+	/* skip till newline */
+	while (true) {
+		switch (input.c) {
+		MATCH_NEWLINE(
+			return;
+		)
+		case EOF:
+			return;
+		}
+		next_char();
+	}
+}
+
 static bool parse_include_directive(void)
 {
 	/* don't eat the TP_include here!
 	 * we need an alternative parsing for the next token */
 	skip_whitespace();
 	bool system_include = input.c == '<';
-	const char *headername = parse_headername();
-	if (headername == NULL) {
+	parse_headername();
+	string_t headername = pp_token.string.string;
+	if (headername.begin == NULL) {
 		eat_pp_directive();
 		return false;
 	}
 
+	skip_whitespace();
 	if (!info.at_line_begin) {
-		warningf(WARN_OTHER, &pp_token.source_position, "extra tokens at end of #include directive");
-		eat_pp_directive();
+		warningf(WARN_OTHER, &pp_token.base.source_position,
+		         "extra tokens at end of #include directive");
+		skip_till_newline();
 	}
 
 	if (n_inputs > INCLUDE_LIMIT) {
-		errorf(&pp_token.source_position, "#include nested too deeply");
+		errorf(&pp_token.base.source_position, "#include nested too deeply");
 		/* eat \n or EOF */
 		next_preprocessing_token();
 		return false;
 	}
 
-	/* we have to reenable space counting and macro expansion here,
-	 * because it is still disabled in directive parsing,
-	 * but we will trigger a preprocessing token reading of the new file
-	 * now and need expansions/space counting */
-	in_pp_directive = false;
-
 	/* switch inputs */
 	emit_newlines();
 	push_input();
-	bool res = do_include(system_include, headername);
+	bool res = do_include(system_include, pp_token.string.string.begin);
 	if (!res) {
-		errorf(&pp_token.source_position,
-		       "failed including '%s': %s", headername, strerror(errno));
+		errorf(&pp_token.base.source_position,
+		       "failed including '%S': %s", pp_token.string, strerror(errno));
 		pop_restore_input();
 		return false;
 	}
@@ -1635,20 +1642,20 @@ static void check_unclosed_conditionals(void)
 
 static void parse_ifdef_ifndef_directive(void)
 {
-	bool is_ifndef = (pp_token.type == TP_ifndef);
+	bool is_ifndef = (pp_token.kind == TP_ifndef);
 	bool condition;
 	next_preprocessing_token();
 
 	if (skip_mode) {
 		eat_pp_directive();
 		pp_conditional_t *conditional = push_conditional();
-		conditional->source_position  = pp_token.source_position;
+		conditional->source_position  = pp_token.base.source_position;
 		conditional->skip             = true;
 		return;
 	}
 
-	if (pp_token.type != TP_IDENTIFIER || info.at_line_begin) {
-		errorf(&pp_token.source_position,
+	if (pp_token.kind != TP_IDENTIFIER || info.at_line_begin) {
+		errorf(&pp_token.base.source_position,
 		       "expected identifier after #%s, got '%t'",
 		       is_ifndef ? "ifndef" : "ifdef", &pp_token);
 		eat_pp_directive();
@@ -1656,23 +1663,21 @@ static void parse_ifdef_ifndef_directive(void)
 		/* just take the true case in the hope to avoid further errors */
 		condition = true;
 	} else {
-		symbol_t        *symbol        = pp_token.symbol;
-		pp_definition_t *pp_definition = symbol->pp_definition;
+		/* evaluate wether we are in true or false case */
+		condition = !pp_token.base.symbol->pp_definition == is_ifndef;
+
 		next_preprocessing_token();
 
 		if (!info.at_line_begin) {
-			errorf(&pp_token.source_position,
+			errorf(&pp_token.base.source_position,
 			       "extra tokens at end of #%s",
 			       is_ifndef ? "ifndef" : "ifdef");
 			eat_pp_directive();
 		}
-
-		/* evaluate wether we are in true or false case */
-		condition = is_ifndef ? pp_definition == NULL : pp_definition != NULL;
 	}
 
 	pp_conditional_t *conditional = push_conditional();
-	conditional->source_position  = pp_token.source_position;
+	conditional->source_position  = pp_token.base.source_position;
 	conditional->condition        = condition;
 
 	if (!condition) {
@@ -1686,19 +1691,19 @@ static void parse_else_directive(void)
 
 	if (!info.at_line_begin) {
 		if (!skip_mode) {
-			warningf(WARN_OTHER, &pp_token.source_position, "extra tokens at end of #else");
+			warningf(WARN_OTHER, &pp_token.base.source_position, "extra tokens at end of #else");
 		}
 		eat_pp_directive();
 	}
 
 	pp_conditional_t *conditional = conditional_stack;
 	if (conditional == NULL) {
-		errorf(&pp_token.source_position, "#else without prior #if");
+		errorf(&pp_token.base.source_position, "#else without prior #if");
 		return;
 	}
 
 	if (conditional->in_else) {
-		errorf(&pp_token.source_position,
+		errorf(&pp_token.base.source_position,
 		       "#else after #else (condition started %P)",
 		       conditional->source_position);
 		skip_mode = true;
@@ -1709,7 +1714,7 @@ static void parse_else_directive(void)
 	if (!conditional->skip) {
 		skip_mode = conditional->condition;
 	}
-	conditional->source_position = pp_token.source_position;
+	conditional->source_position = pp_token.base.source_position;
 }
 
 static void parse_endif_directive(void)
@@ -1718,14 +1723,14 @@ static void parse_endif_directive(void)
 
 	if (!info.at_line_begin) {
 		if (!skip_mode) {
-			warningf(WARN_OTHER, &pp_token.source_position, "extra tokens at end of #endif");
+			warningf(WARN_OTHER, &pp_token.base.source_position, "extra tokens at end of #endif");
 		}
 		eat_pp_directive();
 	}
 
 	pp_conditional_t *conditional = conditional_stack;
 	if (conditional == NULL) {
-		errorf(&pp_token.source_position, "#endif without prior #if");
+		errorf(&pp_token.base.source_position, "#endif without prior #if");
 		return;
 	}
 
@@ -1737,11 +1742,15 @@ static void parse_endif_directive(void)
 
 static void parse_preprocessing_directive(void)
 {
-	in_pp_directive = true;
 	eat_pp('#');
 
+	if (info.at_line_begin) {
+		/* empty directive */
+		return;
+	}
+
 	if (skip_mode) {
-		switch (pp_token.type) {
+		switch (pp_token.kind) {
 		case TP_ifdef:
 		case TP_ifndef:
 			parse_ifdef_ifndef_directive();
@@ -1757,7 +1766,7 @@ static void parse_preprocessing_directive(void)
 			break;
 		}
 	} else {
-		switch (pp_token.type) {
+		switch (pp_token.kind) {
 		case TP_define:
 			parse_define_directive();
 			break;
@@ -1782,14 +1791,13 @@ static void parse_preprocessing_directive(void)
 				/* the nop directive "#" */
 				break;
 			}
-			errorf(&pp_token.source_position,
+			errorf(&pp_token.base.source_position,
 				   "invalid preprocessing directive #%t", &pp_token);
 			eat_pp_directive();
 			break;
 		}
 	}
 
-	in_pp_directive = false;
 	assert(info.at_line_begin);
 }
 
@@ -1852,6 +1860,7 @@ int pptest_main(int argc, char **argv)
 
 	/* simplistic commandline parser */
 	const char *filename = NULL;
+	const char *output = NULL;
 	for (int i = 1; i < argc; ++i) {
 		const char *opt = argv[i];
 		if (streq(opt, "-I")) {
@@ -1859,6 +1868,9 @@ int pptest_main(int argc, char **argv)
 			continue;
 		} else if (streq(opt, "-E")) {
 			/* ignore */
+		} else if (streq(opt, "-o")) {
+			output = argv[++i];
+			continue;
 		} else if (opt[0] == '-') {
 			fprintf(stderr, "Unknown option '%s'\n", opt);
 		} else {
@@ -1872,7 +1884,15 @@ int pptest_main(int argc, char **argv)
 		return 1;
 	}
 
-	out = stdout;
+	if (output == NULL) {
+		out = stdout;
+	} else {
+		out = fopen(output, "w");
+		if (out == NULL) {
+			fprintf(stderr, "Couldn't open output '%s'\n", output);
+			return 1;
+		}
+	}
 
 	/* just here for gcc compatibility */
 	fprintf(out, "# 1 \"%s\"\n", filename);
@@ -1887,35 +1907,35 @@ int pptest_main(int argc, char **argv)
 	switch_input(file, filename);
 
 	while (true) {
-		if (pp_token.type == '#' && info.at_line_begin) {
+		if (pp_token.kind == '#' && info.at_line_begin) {
 			parse_preprocessing_directive();
 			continue;
-		} else if (pp_token.type == TP_EOF) {
+		} else if (pp_token.kind == TP_EOF) {
 			goto end_of_main_loop;
-		} else if (pp_token.type == TP_IDENTIFIER && !in_pp_directive) {
-			symbol_t *symbol = pp_token.symbol;
-			pp_definition_t *pp_definition = symbol->pp_definition;
+		} else if (pp_token.kind == TP_IDENTIFIER) {
+			symbol_t        *const symbol        = pp_token.base.symbol;
+			pp_definition_t *const pp_definition = symbol->pp_definition;
 			if (pp_definition != NULL && !pp_definition->is_expanding) {
-				expansion_pos = pp_token.source_position;
+				expansion_pos = pp_token.base.source_position;
 				if (pp_definition->has_parameters) {
-					source_position_t position = pp_token.source_position;
+					source_position_t position = pp_token.base.source_position;
 					add_token_info_t old_info = info;
 					next_preprocessing_token();
 					add_token_info_t new_info = info;
 
 					/* no opening brace -> no expansion */
-					if (pp_token.type == '(') {
+					if (pp_token.kind == '(') {
 						eat_pp('(');
 
 						/* parse arguments (TODO) */
-						while (pp_token.type != TP_EOF && pp_token.type != ')')
+						while (pp_token.kind != TP_EOF && pp_token.kind != ')')
 							next_preprocessing_token();
 					} else {
 						token_t next_token = pp_token;
 						/* restore identifier token */
-						pp_token.type            = TP_IDENTIFIER;
-						pp_token.symbol          = symbol;
-						pp_token.source_position = position;
+						pp_token.kind                 = TP_IDENTIFIER;
+						pp_token.base.symbol          = symbol;
+						pp_token.base.source_position = position;
 						info = old_info;
 						emit_pp_token();
 
@@ -1941,6 +1961,8 @@ end_of_main_loop:
 	fputc('\n', out);
 	check_unclosed_conditionals();
 	close_input();
+	if (out != stdout)
+		fclose(out);
 
 	obstack_free(&input_obstack, NULL);
 	obstack_free(&pp_obstack, NULL);