Do not strip the 0x prefix from the textual representation of hexadecimal constants.
[cparser] / preprocessor.c
index 21c2fc5..2140145 100644 (file)
@@ -10,6 +10,8 @@
 #include "symbol_t.h"
 #include "adt/util.h"
 #include "adt/error.h"
+#include "adt/strutil.h"
+#include "adt/strset.h"
 #include "lang_features.h"
 #include "diagnostic.h"
 #include "string_rep.h"
@@ -72,7 +74,13 @@ typedef struct add_token_info_t {
        bool     at_line_begin;
 } add_token_info_t;
 
-static pp_input_t input;
+typedef struct searchpath_entry_t searchpath_entry_t;
+struct searchpath_entry_t {
+       const char         *path;
+       searchpath_entry_t *next;
+};
+
+static pp_input_t      input;
 
 static pp_input_t     *input_stack;
 static unsigned        n_inputs;
@@ -87,10 +95,14 @@ static bool              in_pp_directive;
 static bool              skip_mode;
 static FILE             *out;
 static struct obstack    pp_obstack;
+static struct obstack    config_obstack;
 static const char       *printed_input_name = NULL;
 static source_position_t expansion_pos;
 static pp_definition_t  *current_expansion  = NULL;
-static preprocessor_token_type_t last_token = TP_ERROR;
+static strset_t          stringset;
+static preprocessor_token_kind_t last_token;
+
+static searchpath_entry_t *searchpath;
 
 static add_token_info_t  info;
 
@@ -98,12 +110,8 @@ static inline void next_char(void);
 static void next_preprocessing_token(void);
 static void print_line_directive(const source_position_t *pos, const char *add);
 
-static bool open_input(const char *filename)
+static void switch_input(FILE *file, const char *filename)
 {
-       FILE *file = fopen(filename, "r");
-       if (file == NULL)
-               return false;
-
        input.file                = file;
        input.input               = input_from_stream(file, NULL);
        input.bufend              = NULL;
@@ -116,11 +124,9 @@ static bool open_input(const char *filename)
        print_line_directive(&input.position, input_stack != NULL ? "1" : NULL);
 
        /* place a virtual '\n' so we realize we're at line begin */
-       input.position.lineno     = 0;
-       input.c = '\n';
+       input.position.lineno = 0;
+       input.c               = '\n';
        next_preprocessing_token();
-
-       return true;
 }
 
 static void close_input(void)
@@ -182,15 +188,14 @@ static void pop_restore_input(void)
  */
 static void parse_error(const char *msg)
 {
-       errorf(&pp_token.source_position,  "%s", msg);
+       errorf(&pp_token.base.source_position,  "%s", msg);
 }
 
 static inline void next_real_char(void)
 {
        assert(input.bufpos <= input.bufend);
        if (input.bufpos >= input.bufend) {
-               size_t n = decode(input.input, input.buf + MAX_PUTBACK,
-                                 sizeof(input.buf)/sizeof(input.buf[0]) - MAX_PUTBACK);
+               size_t const n = decode(input.input, input.buf + MAX_PUTBACK, lengthof(input.buf) - MAX_PUTBACK);
                if (n == 0) {
                        input.c = EOF;
                        return;
@@ -356,50 +361,41 @@ static int digit_value(int digit)
  *
  * @param first_digit  the already read first digit
  */
-static int parse_octal_sequence(const int first_digit)
+static utf32 parse_octal_sequence(const utf32 first_digit)
 {
        assert(is_octal_digit(first_digit));
-       int value = digit_value(first_digit);
+       utf32 value = digit_value(first_digit);
        if (!is_octal_digit(input.c)) return value;
        value = 8 * value + digit_value(input.c);
        next_char();
        if (!is_octal_digit(input.c)) return value;
        value = 8 * value + digit_value(input.c);
        next_char();
+       return value;
 
-       if (char_is_signed) {
-               return (signed char) value;
-       } else {
-               return (unsigned char) value;
-       }
 }
 
 /**
  * Parses a hex character sequence.
  */
-static int parse_hex_sequence(void)
+static utf32 parse_hex_sequence(void)
 {
-       int value = 0;
+       utf32 value = 0;
        while (isxdigit(input.c)) {
                value = 16 * value + digit_value(input.c);
                next_char();
        }
-
-       if (char_is_signed) {
-               return (signed char) value;
-       } else {
-               return (unsigned char) value;
-       }
+       return value;
 }
 
 /**
  * Parse an escape sequence.
  */
-static int parse_escape_sequence(void)
+static utf32 parse_escape_sequence(void)
 {
        eat('\\');
 
-       int ec = input.c;
+       utf32 const ec = input.c;
        next_char();
 
        switch (ec) {
@@ -428,43 +424,37 @@ static int parse_escape_sequence(void)
        case EOF:
                parse_error("reached end of file while parsing escape sequence");
                return EOF;
-       default:
-               parse_error("unknown escape sequence");
+       /* \E is not documented, but handled, by GCC.  It is acceptable according
+        * to §6.11.4, whereas \e is not. */
+       case 'E':
+       case 'e':
+               if (c_mode & _GNUC)
+                       return 27;   /* hopefully 27 is ALWAYS the code for ESCAPE */
+               break;
+       case 'u':
+       case 'U':
+               parse_error("universal character parsing not implemented yet");
                return EOF;
+       default:
+               break;
        }
+       /* §6.4.4.4:8 footnote 64 */
+       parse_error("unknown escape sequence");
+       return EOF;
 }
 
-static void grow_symbol(utf32 const tc)
+static const char *identify_string(char *string)
 {
-       struct obstack *const o  = &symbol_obstack;
-       if (tc < 0x80U) {
-               obstack_1grow(o, tc);
-       } else if (tc < 0x800) {
-               obstack_1grow(o, 0xC0 | (tc >> 6));
-               obstack_1grow(o, 0x80 | (tc & 0x3F));
-       } else if (tc < 0x10000) {
-               obstack_1grow(o, 0xE0 | ( tc >> 12));
-               obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
-               obstack_1grow(o, 0x80 | ( tc        & 0x3F));
-       } else {
-               obstack_1grow(o, 0xF0 | ( tc >> 18));
-               obstack_1grow(o, 0x80 | ((tc >> 12) & 0x3F));
-               obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
-               obstack_1grow(o, 0x80 | ( tc        & 0x3F));
+       const char *result = strset_insert(&stringset, string);
+       if (result != string) {
+               obstack_free(&symbol_obstack, string);
        }
+       return result;
 }
 
-static string_t identify_string(char *string, size_t len)
+static string_t make_string(char *string, size_t len)
 {
-       /* TODO hash */
-#if 0
-       const char *result = strset_insert(&stringset, concat);
-       if (result != concat) {
-               obstack_free(&symbol_obstack, concat);
-       }
-#else
-       const char *result = string;
-#endif
+       const char *result = identify_string(string);
        return (string_t) {result, len};
 }
 
@@ -492,11 +482,10 @@ static void parse_string_literal(void)
 
                case EOF: {
                        source_position_t source_position;
-                       source_position.input_name = pp_token.source_position.input_name;
+                       source_position.input_name = pp_token.base.source_position.input_name;
                        source_position.lineno     = start_linenr;
                        errorf(&source_position, "string has no end");
-                       pp_token.type = TP_ERROR;
-                       return;
+                       goto end_of_string;
                }
 
                case '"':
@@ -504,7 +493,7 @@ static void parse_string_literal(void)
                        goto end_of_string;
 
                default:
-                       grow_symbol(input.c);
+                       obstack_grow_symbol(&symbol_obstack, input.c);
                        next_char();
                        break;
                }
@@ -516,8 +505,8 @@ end_of_string:
        const size_t size   = (size_t)obstack_object_size(&symbol_obstack);
        char *const  string = obstack_finish(&symbol_obstack);
 
-       pp_token.type    = TP_STRING_LITERAL;
-       pp_token.literal = identify_string(string, size);
+       pp_token.kind          = TP_STRING_LITERAL;
+       pp_token.string.string = make_string(string, size);
 }
 
 /**
@@ -526,8 +515,8 @@ end_of_string:
 static void parse_wide_string_literal(void)
 {
        parse_string_literal();
-       if (pp_token.type == TP_STRING_LITERAL)
-               pp_token.type = TP_WIDE_STRING_LITERAL;
+       if (pp_token.kind == TP_STRING_LITERAL)
+               pp_token.kind = TP_WIDE_STRING_LITERAL;
 }
 
 static void parse_wide_character_constant(void)
@@ -538,7 +527,7 @@ static void parse_wide_character_constant(void)
                switch (input.c) {
                case '\\': {
                        const utf32 tc = parse_escape_sequence();
-                       grow_symbol(tc);
+                       obstack_grow_symbol(&symbol_obstack, tc);
                        break;
                }
 
@@ -553,11 +542,10 @@ static void parse_wide_character_constant(void)
 
                case EOF:
                        parse_error("EOF while parsing character constant");
-                       pp_token.type = TP_ERROR;
-                       return;
+                       goto end_of_wide_char_constant;
 
                default:
-                       grow_symbol(input.c);
+                       obstack_grow_symbol(&symbol_obstack, input.c);
                        next_char();
                        break;
                }
@@ -567,8 +555,8 @@ end_of_wide_char_constant:
        obstack_1grow(&symbol_obstack, '\0');
        size_t  size = (size_t) obstack_object_size(&symbol_obstack)-1;
        char   *string = obstack_finish(&symbol_obstack);
-       pp_token.type       = TP_WIDE_CHARACTER_CONSTANT;
-       pp_token.literal = identify_string(string, size);
+       pp_token.kind          = TP_WIDE_CHARACTER_CONSTANT;
+       pp_token.string.string = make_string(string, size);
 
        if (size == 0) {
                parse_error("empty character constant");
@@ -596,11 +584,10 @@ static void parse_character_constant(void)
 
                case EOF: {
                        source_position_t source_position;
-                       source_position.input_name = pp_token.source_position.input_name;
+                       source_position.input_name = pp_token.base.source_position.input_name;
                        source_position.lineno     = start_linenr;
                        errorf(&source_position, "EOF while parsing character constant");
-                       pp_token.type = TP_ERROR;
-                       return;
+                       goto end_of_char_constant;
                }
 
                case '\'':
@@ -616,12 +603,16 @@ static void parse_character_constant(void)
        }
 
 end_of_char_constant:;
-       const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
-       const char *const string = obstack_finish(&symbol_obstack);
+       obstack_1grow(&symbol_obstack, '\0');
+       const size_t size   = (size_t)obstack_object_size(&symbol_obstack);
+       char *const  string = obstack_finish(&symbol_obstack);
+
+       pp_token.kind          = TP_CHARACTER_CONSTANT;
+       pp_token.string.string = make_string(string, size);
 
-       pp_token.type          = TP_CHARACTER_CONSTANT;
-       pp_token.literal.begin = string;
-       pp_token.literal.size  = size;
+       if (size == 0) {
+               parse_error("empty character constant");
+       }
 }
 
 #define SYMBOL_CHARS_WITHOUT_E_P \
@@ -723,14 +714,14 @@ restart:
                goto restart;
        }
        pp_token = definition->token_list[definition->expand_pos];
-       pp_token.source_position = expansion_pos;
+       pp_token.base.source_position = expansion_pos;
        ++definition->expand_pos;
 
-       if (pp_token.type != TP_IDENTIFIER)
+       if (pp_token.kind != TP_IDENTIFIER)
                return;
 
        /* if it was an identifier then we might need to expand again */
-       pp_definition_t *symbol_definition = pp_token.symbol->pp_definition;
+       pp_definition_t *symbol_definition = pp_token.identifier.symbol->pp_definition;
        if (symbol_definition != NULL && !symbol_definition->is_expanding) {
                symbol_definition->parent_expansion = definition;
                symbol_definition->expand_pos       = 0;
@@ -786,7 +777,7 @@ static void skip_multiline_comment(void)
 
                case EOF: {
                        source_position_t source_position;
-                       source_position.input_name = pp_token.source_position.input_name;
+                       source_position.input_name = pp_token.base.source_position.input_name;
                        source_position.lineno     = start_linenr;
                        errorf(&source_position, "at end of file while looking for comment end");
                        return;
@@ -834,10 +825,10 @@ static void skip_whitespace(void)
        }
 }
 
-static void eat_pp(int type)
+static void eat_pp(preprocessor_token_kind_t const type)
 {
        (void) type;
-       assert(pp_token.type == type);
+       assert(pp_token.kind == type);
        next_preprocessing_token();
 }
 
@@ -876,8 +867,8 @@ end_symbol:
 
        symbol_t *symbol = symbol_table_insert(string);
 
-       pp_token.type   = symbol->pp_ID;
-       pp_token.symbol = symbol;
+       pp_token.kind              = symbol->pp_ID;
+       pp_token.identifier.symbol = symbol;
 
        /* we can free the memory from symbol obstack if we already had an entry in
         * the symbol table */
@@ -922,9 +913,8 @@ end_number:
        size_t  size   = obstack_object_size(&symbol_obstack);
        char   *string = obstack_finish(&symbol_obstack);
 
-       pp_token.type          = TP_NUMBER;
-       pp_token.literal.begin = string;
-       pp_token.literal.size  = size;
+       pp_token.kind          = TP_NUMBER;
+       pp_token.number.number = make_string(string, size);
 }
 
 
@@ -936,7 +926,7 @@ end_number:
 #define MAYBE(ch, set_type)                                \
                                case ch:                                   \
                                        next_char();                           \
-                                       pp_token.type = set_type;              \
+                                       pp_token.kind = set_type;              \
                                        return;
 
 #define ELSE_CODE(code)                                    \
@@ -948,7 +938,7 @@ end_number:
 
 #define ELSE(set_type)                                     \
                ELSE_CODE(                                         \
-                       pp_token.type = set_type;                      \
+                       pp_token.kind = set_type;                      \
                )
 
 static void next_preprocessing_token(void)
@@ -961,7 +951,7 @@ static void next_preprocessing_token(void)
        info.at_line_begin  = false;
        info.had_whitespace = false;
 restart:
-       pp_token.source_position = input.position;
+       pp_token.base.source_position = input.position;
        switch (input.c) {
        case ' ':
        case '\t':
@@ -1015,7 +1005,7 @@ restart:
                                ELSE_CODE(
                                        put_back(input.c);
                                        input.c = '.';
-                                       pp_token.type = '.';
+                                       pp_token.kind = '.';
                                )
                ELSE('.')
        case '&':
@@ -1068,7 +1058,7 @@ restart:
                                                ELSE_CODE(
                                                        put_back(input.c);
                                                        input.c = '%';
-                                                       pp_token.type = '#';
+                                                       pp_token.kind = '#';
                                                )
                                ELSE('#')
                ELSE('%')
@@ -1111,7 +1101,7 @@ restart:
                MAYBE_PROLOG
                MAYBE('#', TP_HASHHASH)
                ELSE_CODE(
-                       pp_token.type = '#';
+                       pp_token.kind = '#';
                )
 
        case '?':
@@ -1125,7 +1115,7 @@ restart:
        case ';':
        case ',':
        case '\\':
-               pp_token.type = input.c;
+               pp_token.kind = input.c;
                next_char();
                return;
 
@@ -1137,22 +1127,22 @@ restart:
                        print_line_directive(&input.position, "2");
                        goto restart;
                } else {
-                       pp_token.source_position.lineno++;
+                       pp_token.base.source_position.lineno++;
                        info.at_line_begin = true;
-                       pp_token.type = TP_EOF;
+                       pp_token.kind = TP_EOF;
                }
                return;
 
        default:
                next_char();
                if (!ignore_unknown_chars) {
-                       errorf(&pp_token.source_position, "unknown character '%c' found\n",
-                              input.c);
-                       pp_token.type = TP_ERROR;
+                       errorf(&pp_token.base.source_position,
+                              "unknown character '%c' found\n", input.c);
+                       goto restart;
                } else {
-                       pp_token.type = input.c;
+                       pp_token.kind = input.c;
+                       return;
                }
-               return;
        }
 }
 
@@ -1198,18 +1188,18 @@ static void print_line_directive(const source_position_t *pos, const char *add)
 
 static void emit_newlines(void)
 {
-       unsigned delta = pp_token.source_position.lineno - input.output_line;
+       unsigned delta = pp_token.base.source_position.lineno - input.output_line;
 
        if (delta >= 9) {
                fputc('\n', out);
-               print_line_directive(&pp_token.source_position, NULL);
+               print_line_directive(&pp_token.base.source_position, NULL);
                fputc('\n', out);
        } else {
                for (unsigned i = 0; i < delta; ++i) {
                        fputc('\n', out);
                }
        }
-       input.output_line = pp_token.source_position.lineno;
+       input.output_line = pp_token.base.source_position.lineno;
 }
 
 static void emit_pp_token(void)
@@ -1224,36 +1214,36 @@ static void emit_pp_token(void)
                        fputc(' ', out);
 
        } else if (info.had_whitespace ||
-                          tokens_would_paste(last_token, pp_token.type)) {
+                          tokens_would_paste(last_token, pp_token.kind)) {
                fputc(' ', out);
        }
 
-       switch (pp_token.type) {
+       switch (pp_token.kind) {
        case TP_IDENTIFIER:
-               fputs(pp_token.symbol->string, out);
+               fputs(pp_token.identifier.symbol->string, out);
                break;
        case TP_NUMBER:
-               fputs(pp_token.literal.begin, out);
+               fputs(pp_token.number.number.begin, out);
                break;
        case TP_WIDE_STRING_LITERAL:
                fputc('L', out);
        case TP_STRING_LITERAL:
                fputc('"', out);
-               fputs(pp_token.literal.begin, out);
+               fputs(pp_token.string.string.begin, out);
                fputc('"', out);
                break;
        case TP_WIDE_CHARACTER_CONSTANT:
                fputc('L', out);
        case TP_CHARACTER_CONSTANT:
                fputc('\'', out);
-               fputs(pp_token.literal.begin, out);
+               fputs(pp_token.string.string.begin, out);
                fputc('\'', out);
                break;
        default:
-               print_pp_token_type(out, pp_token.type);
+               print_pp_token_kind(out, pp_token.kind);
                break;
        }
-       last_token = pp_token.type;
+       last_token = pp_token.kind;
 }
 
 static void eat_pp_directive(void)
@@ -1280,19 +1270,16 @@ static bool strings_equal(const string_t *string1, const string_t *string2)
 
 static bool pp_tokens_equal(const token_t *token1, const token_t *token2)
 {
-       if (token1->type != token2->type)
+       if (token1->kind != token2->kind)
                return false;
 
-       switch (token1->type) {
-       case TP_HEADERNAME:
-               /* TODO */
-               return false;
+       switch (token1->kind) {
        case TP_IDENTIFIER:
-               return token1->symbol == token2->symbol;
+               return token1->identifier.symbol == token2->identifier.symbol;
        case TP_NUMBER:
        case TP_CHARACTER_CONSTANT:
        case TP_STRING_LITERAL:
-               return strings_equal(&token1->literal, &token2->literal);
+               return strings_equal(&token1->string.string, &token2->string.string);
 
        default:
                return true;
@@ -1320,12 +1307,12 @@ static void parse_define_directive(void)
        eat_pp(TP_define);
        assert(obstack_object_size(&pp_obstack) == 0);
 
-       if (pp_token.type != TP_IDENTIFIER || info.at_line_begin) {
-               errorf(&pp_token.source_position,
+       if (pp_token.kind != TP_IDENTIFIER || info.at_line_begin) {
+               errorf(&pp_token.base.source_position,
                       "expected identifier after #define, got '%t'", &pp_token);
                goto error_out;
        }
-       symbol_t *symbol = pp_token.symbol;
+       symbol_t *symbol = pp_token.identifier.symbol;
 
        pp_definition_t *new_definition
                = obstack_alloc(&pp_obstack, sizeof(new_definition[0]));
@@ -1342,27 +1329,27 @@ static void parse_define_directive(void)
                next_preprocessing_token();
 
                while (true) {
-                       switch (pp_token.type) {
+                       switch (pp_token.kind) {
                        case TP_DOTDOTDOT:
                                new_definition->is_variadic = true;
                                next_preprocessing_token();
-                               if (pp_token.type != ')') {
+                               if (pp_token.kind != ')') {
                                        errorf(&input.position,
                                                        "'...' not at end of macro argument list");
                                        goto error_out;
                                }
                                break;
                        case TP_IDENTIFIER:
-                               obstack_ptr_grow(&pp_obstack, pp_token.symbol);
+                               obstack_ptr_grow(&pp_obstack, pp_token.identifier.symbol);
                                next_preprocessing_token();
 
-                               if (pp_token.type == ',') {
+                               if (pp_token.kind == ',') {
                                        next_preprocessing_token();
                                        break;
                                }
 
-                               if (pp_token.type != ')') {
-                                       errorf(&pp_token.source_position,
+                               if (pp_token.kind != ')') {
+                                       errorf(&pp_token.base.source_position,
                                               "expected ',' or ')' after identifier, got '%t'",
                                               &pp_token);
                                        goto error_out;
@@ -1372,7 +1359,7 @@ static void parse_define_directive(void)
                                next_preprocessing_token();
                                goto finish_argument_list;
                        default:
-                               errorf(&pp_token.source_position,
+                               errorf(&pp_token.base.source_position,
                                       "expected identifier, '...' or ')' in #define argument list, got '%t'",
                                       &pp_token);
                                goto error_out;
@@ -1426,14 +1413,14 @@ static void parse_undef_directive(void)
 {
        eat_pp(TP_undef);
 
-       if (pp_token.type != TP_IDENTIFIER) {
+       if (pp_token.kind != TP_IDENTIFIER) {
                errorf(&input.position,
                       "expected identifier after #undef, got '%t'", &pp_token);
                eat_pp_directive();
                return;
        }
 
-       symbol_t *symbol = pp_token.symbol;
+       symbol_t *symbol = pp_token.identifier.symbol;
        symbol->pp_definition = NULL;
        next_preprocessing_token();
 
@@ -1449,14 +1436,12 @@ static const char *parse_headername(void)
         * They're only allowed behind an #include so they're not recognized
         * by the normal next_preprocessing_token. We handle them as a special
         * exception here */
-       skip_whitespace();
-
        if (info.at_line_begin) {
                parse_error("expected headername after #include");
                return NULL;
        }
 
-       assert(obstack_object_size(&input_obstack) == 0);
+       assert(obstack_object_size(&symbol_obstack) == 0);
 
        /* check wether we have a "... or <... headername */
        switch (input.c) {
@@ -1474,7 +1459,7 @@ static const char *parse_headername(void)
                                next_char();
                                goto finished_headername;
                        }
-                       obstack_1grow(&input_obstack, (char) input.c);
+                       obstack_1grow(&symbol_obstack, (char) input.c);
                        next_char();
                }
                /* we should never be here */
@@ -1493,7 +1478,7 @@ static const char *parse_headername(void)
                                next_char();
                                goto finished_headername;
                        }
-                       obstack_1grow(&input_obstack, (char) input.c);
+                       obstack_1grow(&symbol_obstack, (char) input.c);
                        next_char();
                }
                /* we should never be here */
@@ -1504,21 +1489,60 @@ static const char *parse_headername(void)
        }
 
 finished_headername:
-       obstack_1grow(&input_obstack, '\0');
-       char *headername = obstack_finish(&input_obstack);
+       obstack_1grow(&symbol_obstack, '\0');
+       char *headername = obstack_finish(&symbol_obstack);
 
        /* TODO: iterate search-path to find the file */
 
        skip_whitespace();
 
-       return headername;
+       return identify_string(headername);
+}
+
+static bool do_include(bool system_include, const char *headername)
+{
+       if (!system_include) {
+               /* for "bla" includes first try current dir
+                * TODO: this isn't correct, should be the directory of the source file
+                */
+               FILE *file = fopen(headername, "r");
+               if (file != NULL) {
+                       switch_input(file, headername);
+                       return true;
+               }
+       }
+
+       size_t headername_len = strlen(headername);
+       assert(obstack_object_size(&pp_obstack) == 0);
+       /* check searchpath */
+       for (searchpath_entry_t *entry = searchpath; entry != NULL;
+            entry = entry->next) {
+           const char *path = entry->path;
+           size_t      len  = strlen(path);
+               obstack_grow(&pp_obstack, path, len);
+               if (path[len-1] != '/')
+                       obstack_1grow(&pp_obstack, '/');
+               obstack_grow(&pp_obstack, headername, headername_len+1);
+
+               char *complete_path = obstack_finish(&pp_obstack);
+               FILE *file          = fopen(complete_path, "r");
+               if (file != NULL) {
+                       const char *filename = identify_string(complete_path);
+                       switch_input(file, filename);
+                       return true;
+               }
+               obstack_free(&pp_obstack, complete_path);
+       }
+
+       return false;
 }
 
 static bool parse_include_directive(void)
 {
        /* don't eat the TP_include here!
         * we need an alternative parsing for the next token */
-
+       skip_whitespace();
+       bool system_include = input.c == '<';
        const char *headername = parse_headername();
        if (headername == NULL) {
                eat_pp_directive();
@@ -1526,12 +1550,13 @@ static bool parse_include_directive(void)
        }
 
        if (!info.at_line_begin) {
-               warningf(WARN_OTHER, &pp_token.source_position, "extra tokens at end of #include directive");
+               warningf(WARN_OTHER, &pp_token.base.source_position,
+                        "extra tokens at end of #include directive");
                eat_pp_directive();
        }
 
        if (n_inputs > INCLUDE_LIMIT) {
-               errorf(&pp_token.source_position, "#include nested too deeply");
+               errorf(&pp_token.base.source_position, "#include nested too deeply");
                /* eat \n or EOF */
                next_preprocessing_token();
                return false;
@@ -1546,9 +1571,9 @@ static bool parse_include_directive(void)
        /* switch inputs */
        emit_newlines();
        push_input();
-       bool res = open_input(headername);
+       bool res = do_include(system_include, headername);
        if (!res) {
-               errorf(&pp_token.source_position,
+               errorf(&pp_token.base.source_position,
                       "failed including '%s': %s", headername, strerror(errno));
                pop_restore_input();
                return false;
@@ -1591,20 +1616,20 @@ static void check_unclosed_conditionals(void)
 
 static void parse_ifdef_ifndef_directive(void)
 {
-       bool is_ifndef = (pp_token.type == TP_ifndef);
+       bool is_ifndef = (pp_token.kind == TP_ifndef);
        bool condition;
        next_preprocessing_token();
 
        if (skip_mode) {
                eat_pp_directive();
                pp_conditional_t *conditional = push_conditional();
-               conditional->source_position  = pp_token.source_position;
+               conditional->source_position  = pp_token.base.source_position;
                conditional->skip             = true;
                return;
        }
 
-       if (pp_token.type != TP_IDENTIFIER || info.at_line_begin) {
-               errorf(&pp_token.source_position,
+       if (pp_token.kind != TP_IDENTIFIER || info.at_line_begin) {
+               errorf(&pp_token.base.source_position,
                       "expected identifier after #%s, got '%t'",
                       is_ifndef ? "ifndef" : "ifdef", &pp_token);
                eat_pp_directive();
@@ -1612,12 +1637,12 @@ static void parse_ifdef_ifndef_directive(void)
                /* just take the true case in the hope to avoid further errors */
                condition = true;
        } else {
-               symbol_t        *symbol        = pp_token.symbol;
+               symbol_t        *symbol        = pp_token.identifier.symbol;
                pp_definition_t *pp_definition = symbol->pp_definition;
                next_preprocessing_token();
 
                if (!info.at_line_begin) {
-                       errorf(&pp_token.source_position,
+                       errorf(&pp_token.base.source_position,
                               "extra tokens at end of #%s",
                               is_ifndef ? "ifndef" : "ifdef");
                        eat_pp_directive();
@@ -1628,7 +1653,7 @@ static void parse_ifdef_ifndef_directive(void)
        }
 
        pp_conditional_t *conditional = push_conditional();
-       conditional->source_position  = pp_token.source_position;
+       conditional->source_position  = pp_token.base.source_position;
        conditional->condition        = condition;
 
        if (!condition) {
@@ -1642,19 +1667,19 @@ static void parse_else_directive(void)
 
        if (!info.at_line_begin) {
                if (!skip_mode) {
-                       warningf(WARN_OTHER, &pp_token.source_position, "extra tokens at end of #else");
+                       warningf(WARN_OTHER, &pp_token.base.source_position, "extra tokens at end of #else");
                }
                eat_pp_directive();
        }
 
        pp_conditional_t *conditional = conditional_stack;
        if (conditional == NULL) {
-               errorf(&pp_token.source_position, "#else without prior #if");
+               errorf(&pp_token.base.source_position, "#else without prior #if");
                return;
        }
 
        if (conditional->in_else) {
-               errorf(&pp_token.source_position,
+               errorf(&pp_token.base.source_position,
                       "#else after #else (condition started %P)",
                       conditional->source_position);
                skip_mode = true;
@@ -1665,7 +1690,7 @@ static void parse_else_directive(void)
        if (!conditional->skip) {
                skip_mode = conditional->condition;
        }
-       conditional->source_position = pp_token.source_position;
+       conditional->source_position = pp_token.base.source_position;
 }
 
 static void parse_endif_directive(void)
@@ -1674,14 +1699,14 @@ static void parse_endif_directive(void)
 
        if (!info.at_line_begin) {
                if (!skip_mode) {
-                       warningf(WARN_OTHER, &pp_token.source_position, "extra tokens at end of #endif");
+                       warningf(WARN_OTHER, &pp_token.base.source_position, "extra tokens at end of #endif");
                }
                eat_pp_directive();
        }
 
        pp_conditional_t *conditional = conditional_stack;
        if (conditional == NULL) {
-               errorf(&pp_token.source_position, "#endif without prior #if");
+               errorf(&pp_token.base.source_position, "#endif without prior #if");
                return;
        }
 
@@ -1697,7 +1722,7 @@ static void parse_preprocessing_directive(void)
        eat_pp('#');
 
        if (skip_mode) {
-               switch (pp_token.type) {
+               switch (pp_token.kind) {
                case TP_ifdef:
                case TP_ifndef:
                        parse_ifdef_ifndef_directive();
@@ -1713,7 +1738,7 @@ static void parse_preprocessing_directive(void)
                        break;
                }
        } else {
-               switch (pp_token.type) {
+               switch (pp_token.kind) {
                case TP_define:
                        parse_define_directive();
                        break;
@@ -1738,7 +1763,7 @@ static void parse_preprocessing_directive(void)
                                /* the nop directive "#" */
                                break;
                        }
-                       errorf(&pp_token.source_position,
+                       errorf(&pp_token.base.source_position,
                                   "invalid preprocessing directive #%t", &pp_token);
                        eat_pp_directive();
                        break;
@@ -1749,18 +1774,84 @@ static void parse_preprocessing_directive(void)
        assert(info.at_line_begin);
 }
 
+static void prepend_include_path(const char *path)
+{
+       searchpath_entry_t *entry = OALLOCZ(&config_obstack, searchpath_entry_t);
+       entry->path = path;
+       entry->next = searchpath;
+       searchpath  = entry;
+}
+
+static void setup_include_path(void)
+{
+       /* built-in paths */
+       prepend_include_path("/usr/include");
+
+       /* parse environment variable */
+       const char *cpath = getenv("CPATH");
+       if (cpath != NULL && *cpath != '\0') {
+               const char *begin = cpath;
+               const char *c;
+               do {
+                       c = begin;
+                       while (*c != '\0' && *c != ':')
+                               ++c;
+
+                       size_t len = c-begin;
+                       if (len == 0) {
+                               /* for gcc compatibility (Matze: I would expect that
+                                * nothing happens for an empty entry...) */
+                               prepend_include_path(".");
+                       } else {
+                               char *string = obstack_alloc(&config_obstack, len+1);
+                               memcpy(string, begin, len);
+                               string[len] = '\0';
+
+                               prepend_include_path(string);
+                       }
+
+                       begin = c+1;
+                       /* skip : */
+                       if (*begin == ':')
+                               ++begin;
+               } while(*c != '\0');
+       }
+}
+
 int pptest_main(int argc, char **argv);
 int pptest_main(int argc, char **argv)
 {
        init_symbol_table();
        init_tokens();
 
+       obstack_init(&config_obstack);
        obstack_init(&pp_obstack);
        obstack_init(&input_obstack);
+       strset_init(&stringset);
 
-       const char *filename = "t.c";
-       if (argc > 1)
-               filename = argv[1];
+       setup_include_path();
+
+       /* simplistic commandline parser */
+       const char *filename = NULL;
+       for (int i = 1; i < argc; ++i) {
+               const char *opt = argv[i];
+               if (streq(opt, "-I")) {
+                       prepend_include_path(argv[++i]);
+                       continue;
+               } else if (streq(opt, "-E")) {
+                       /* ignore */
+               } else if (opt[0] == '-') {
+                       fprintf(stderr, "Unknown option '%s'\n", opt);
+               } else {
+                       if (filename != NULL)
+                               fprintf(stderr, "Multiple inputs not supported\n");
+                       filename = argv[i];
+               }
+       }
+       if (filename == NULL) {
+               fprintf(stderr, "No input specified\n");
+               return 1;
+       }
 
        out = stdout;
 
@@ -1769,39 +1860,43 @@ int pptest_main(int argc, char **argv)
        fprintf(out, "# 1 \"<built-in>\"\n");
        fprintf(out, "# 1 \"<command-line>\"\n");
 
-       bool ok = open_input(filename);
-       assert(ok);
+       FILE *file = fopen(filename, "r");
+       if (file == NULL) {
+               fprintf(stderr, "Couldn't open input '%s'\n", filename);
+               return 1;
+       }
+       switch_input(file, filename);
 
        while (true) {
-               if (pp_token.type == '#' && info.at_line_begin) {
+               if (pp_token.kind == '#' && info.at_line_begin) {
                        parse_preprocessing_directive();
                        continue;
-               } else if (pp_token.type == TP_EOF) {
+               } else if (pp_token.kind == TP_EOF) {
                        goto end_of_main_loop;
-               } else if (pp_token.type == TP_IDENTIFIER && !in_pp_directive) {
-                       symbol_t *symbol = pp_token.symbol;
+               } else if (pp_token.kind == TP_IDENTIFIER && !in_pp_directive) {
+                       symbol_t *symbol = pp_token.identifier.symbol;
                        pp_definition_t *pp_definition = symbol->pp_definition;
                        if (pp_definition != NULL && !pp_definition->is_expanding) {
-                               expansion_pos = pp_token.source_position;
+                               expansion_pos = pp_token.base.source_position;
                                if (pp_definition->has_parameters) {
-                                       source_position_t position = pp_token.source_position;
+                                       source_position_t position = pp_token.base.source_position;
                                        add_token_info_t old_info = info;
                                        next_preprocessing_token();
                                        add_token_info_t new_info = info;
 
                                        /* no opening brace -> no expansion */
-                                       if (pp_token.type == '(') {
+                                       if (pp_token.kind == '(') {
                                                eat_pp('(');
 
                                                /* parse arguments (TODO) */
-                                               while (pp_token.type != TP_EOF && pp_token.type != ')')
+                                               while (pp_token.kind != TP_EOF && pp_token.kind != ')')
                                                        next_preprocessing_token();
                                        } else {
                                                token_t next_token = pp_token;
                                                /* restore identifier token */
-                                               pp_token.type            = TP_IDENTIFIER;
-                                               pp_token.symbol          = symbol;
-                                               pp_token.source_position = position;
+                                               pp_token.kind                 = TP_IDENTIFIER;
+                                               pp_token.identifier.symbol    = symbol;
+                                               pp_token.base.source_position = position;
                                                info = old_info;
                                                emit_pp_token();
 
@@ -1830,6 +1925,9 @@ end_of_main_loop:
 
        obstack_free(&input_obstack, NULL);
        obstack_free(&pp_obstack, NULL);
+       obstack_free(&config_obstack, NULL);
+
+       strset_destroy(&stringset);
 
        exit_tokens();
        exit_symbol_table();