X-Git-Url: http://nsz.repo.hu/git/?a=blobdiff_plain;f=preprocessor.c;h=47275a1b91899ea99543aac8fe900629adec2511;hb=3c9d1105cdebe8a419b5e8a2b53e52ef4fa386a0;hp=48825afa449342992272187ad3b8a7620e9dbebd;hpb=15c8bcaebaca1cdebe3744c172c820fbe0b3bba6;p=cparser diff --git a/preprocessor.c b/preprocessor.c index 48825af..47275a1 100644 --- a/preprocessor.c +++ b/preprocessor.c @@ -90,8 +90,7 @@ static pp_conditional_t *conditional_stack; static token_t pp_token; static bool resolve_escape_sequences = false; -static bool ignore_unknown_chars = true; -static bool in_pp_directive; +static bool error_on_unknown_chars = true; static bool skip_mode; static FILE *out; static struct obstack pp_obstack; @@ -100,7 +99,7 @@ static const char *printed_input_name = NULL; static source_position_t expansion_pos; static pp_definition_t *current_expansion = NULL; static strset_t stringset; -static preprocessor_token_type_t last_token = TP_ERROR; +static token_kind_t last_token; static searchpath_entry_t *searchpath; @@ -188,15 +187,14 @@ static void pop_restore_input(void) */ static void parse_error(const char *msg) { - errorf(&pp_token.source_position, "%s", msg); + errorf(&pp_token.base.source_position, "%s", msg); } static inline void next_real_char(void) { assert(input.bufpos <= input.bufend); if (input.bufpos >= input.bufend) { - size_t n = decode(input.input, input.buf + MAX_PUTBACK, - sizeof(input.buf)/sizeof(input.buf[0]) - MAX_PUTBACK); + size_t const n = decode(input.input, input.buf + MAX_PUTBACK, lengthof(input.buf) - MAX_PUTBACK); if (n == 0) { input.c = EOF; return; @@ -220,17 +218,18 @@ static inline void put_back(utf32 const pc) --input.position.colno; } -#define MATCH_NEWLINE(code) \ - case '\r': \ - next_char(); \ - if (input.c == '\n') { \ - case '\n': \ - next_char(); \ - } \ - info.whitespace = 0; \ - ++input.position.lineno; \ - input.position.colno = 1; \ - code +#define NEWLINE \ + '\r': \ + next_char(); \ + if (input.c == '\n') { \ + case '\n': \ + next_char(); \ + } \ + info.whitespace = 0; \ + ++input.position.lineno; \ + input.position.colno = 1; \ + goto newline; \ + newline // Let it look like an ordinary case label. #define eat(c_type) (assert(input.c == c_type), next_char()) @@ -239,9 +238,8 @@ static void maybe_concat_lines(void) eat('\\'); switch (input.c) { - MATCH_NEWLINE( + case NEWLINE: return; - ) default: break; @@ -362,50 +360,41 @@ static int digit_value(int digit) * * @param first_digit the already read first digit */ -static int parse_octal_sequence(const int first_digit) +static utf32 parse_octal_sequence(const utf32 first_digit) { assert(is_octal_digit(first_digit)); - int value = digit_value(first_digit); + utf32 value = digit_value(first_digit); if (!is_octal_digit(input.c)) return value; value = 8 * value + digit_value(input.c); next_char(); if (!is_octal_digit(input.c)) return value; value = 8 * value + digit_value(input.c); next_char(); + return value; - if (char_is_signed) { - return (signed char) value; - } else { - return (unsigned char) value; - } } /** * Parses a hex character sequence. */ -static int parse_hex_sequence(void) +static utf32 parse_hex_sequence(void) { - int value = 0; + utf32 value = 0; while (isxdigit(input.c)) { value = 16 * value + digit_value(input.c); next_char(); } - - if (char_is_signed) { - return (signed char) value; - } else { - return (unsigned char) value; - } + return value; } /** * Parse an escape sequence. */ -static int parse_escape_sequence(void) +static utf32 parse_escape_sequence(void) { eat('\\'); - int ec = input.c; + utf32 const ec = input.c; next_char(); switch (ec) { @@ -434,30 +423,23 @@ static int parse_escape_sequence(void) case EOF: parse_error("reached end of file while parsing escape sequence"); return EOF; - default: - parse_error("unknown escape sequence"); + /* \E is not documented, but handled, by GCC. It is acceptable according + * to §6.11.4, whereas \e is not. */ + case 'E': + case 'e': + if (c_mode & _GNUC) + return 27; /* hopefully 27 is ALWAYS the code for ESCAPE */ + break; + case 'u': + case 'U': + parse_error("universal character parsing not implemented yet"); return EOF; + default: + break; } -} - -static void grow_symbol(utf32 const tc) -{ - struct obstack *const o = &symbol_obstack; - if (tc < 0x80U) { - obstack_1grow(o, tc); - } else if (tc < 0x800) { - obstack_1grow(o, 0xC0 | (tc >> 6)); - obstack_1grow(o, 0x80 | (tc & 0x3F)); - } else if (tc < 0x10000) { - obstack_1grow(o, 0xE0 | ( tc >> 12)); - obstack_1grow(o, 0x80 | ((tc >> 6) & 0x3F)); - obstack_1grow(o, 0x80 | ( tc & 0x3F)); - } else { - obstack_1grow(o, 0xF0 | ( tc >> 18)); - obstack_1grow(o, 0x80 | ((tc >> 12) & 0x3F)); - obstack_1grow(o, 0x80 | ((tc >> 6) & 0x3F)); - obstack_1grow(o, 0x80 | ( tc & 0x3F)); - } + /* §6.4.4.4:8 footnote 64 */ + parse_error("unknown escape sequence"); + return EOF; } static const char *identify_string(char *string) @@ -469,170 +451,89 @@ static const char *identify_string(char *string) return result; } -static string_t make_string(char *string, size_t len) +static string_t sym_make_string(string_encoding_t const enc) { - const char *result = identify_string(string); - return (string_t) {result, len}; + obstack_1grow(&symbol_obstack, '\0'); + size_t const len = obstack_object_size(&symbol_obstack) - 1; + char *const string = obstack_finish(&symbol_obstack); + char const *const result = identify_string(string); + return (string_t){ result, len, enc }; } -static void parse_string_literal(void) +static void parse_string(utf32 const delimiter, token_kind_t const kind, + string_encoding_t const enc, + char const *const context) { const unsigned start_linenr = input.position.lineno; - eat('"'); + eat(delimiter); while (true) { switch (input.c) { case '\\': { - utf32 tc; if (resolve_escape_sequences) { - tc = parse_escape_sequence(); - obstack_1grow(&symbol_obstack, (char) tc); + utf32 const tc = parse_escape_sequence(); + if (enc == STRING_ENCODING_CHAR) { + if (tc >= 0x100) { + warningf(WARN_OTHER, &pp_token.base.source_position, "escape sequence out of range"); + } + obstack_1grow(&symbol_obstack, tc); + } else { + obstack_grow_utf8(&symbol_obstack, tc); + } } else { - obstack_1grow(&symbol_obstack, (char) input.c); + obstack_1grow(&symbol_obstack, (char)input.c); next_char(); - obstack_1grow(&symbol_obstack, (char) input.c); + obstack_1grow(&symbol_obstack, (char)input.c); next_char(); } break; } + case NEWLINE: + errorf(&pp_token.base.source_position, "newline while parsing %s", context); + break; + case EOF: { source_position_t source_position; - source_position.input_name = pp_token.source_position.input_name; + source_position.input_name = pp_token.base.source_position.input_name; source_position.lineno = start_linenr; - errorf(&source_position, "string has no end"); - pp_token.type = TP_ERROR; - return; - } - - case '"': - next_char(); + errorf(&source_position, "EOF while parsing %s", context); goto end_of_string; + } default: - grow_symbol(input.c); - next_char(); - break; + if (input.c == delimiter) { + next_char(); + goto end_of_string; + } else { + obstack_grow_utf8(&symbol_obstack, input.c); + next_char(); + break; + } } } end_of_string: - /* add finishing 0 to the string */ - obstack_1grow(&symbol_obstack, '\0'); - const size_t size = (size_t)obstack_object_size(&symbol_obstack); - char *const string = obstack_finish(&symbol_obstack); - - pp_token.type = TP_STRING_LITERAL; - pp_token.literal = make_string(string, size); + pp_token.kind = kind; + pp_token.literal.string = sym_make_string(enc); } -/** - * Parse a wide string literal and set lexer_token. - */ -static void parse_wide_string_literal(void) +static void parse_string_literal(string_encoding_t const enc) { - parse_string_literal(); - if (pp_token.type == TP_STRING_LITERAL) - pp_token.type = TP_WIDE_STRING_LITERAL; + parse_string('"', T_STRING_LITERAL, enc, "string literal"); } -static void parse_wide_character_constant(void) +static void parse_character_constant(string_encoding_t const enc) { - eat('\''); - - while (true) { - switch (input.c) { - case '\\': { - const utf32 tc = parse_escape_sequence(); - grow_symbol(tc); - break; - } - - MATCH_NEWLINE( - parse_error("newline while parsing character constant"); - break; - ) - - case '\'': - next_char(); - goto end_of_wide_char_constant; - - case EOF: - parse_error("EOF while parsing character constant"); - pp_token.type = TP_ERROR; - return; - - default: - grow_symbol(input.c); - next_char(); - break; - } - } - -end_of_wide_char_constant: - obstack_1grow(&symbol_obstack, '\0'); - size_t size = (size_t) obstack_object_size(&symbol_obstack)-1; - char *string = obstack_finish(&symbol_obstack); - pp_token.type = TP_WIDE_CHARACTER_CONSTANT; - pp_token.literal = make_string(string, size); - - if (size == 0) { + parse_string('\'', T_CHARACTER_CONSTANT, enc, "character constant"); + if (pp_token.literal.string.size == 0) { parse_error("empty character constant"); } } -static void parse_character_constant(void) -{ - const unsigned start_linenr = input.position.lineno; - - eat('\''); - - int tc; - while (true) { - switch (input.c) { - case '\\': - tc = parse_escape_sequence(); - obstack_1grow(&symbol_obstack, (char) tc); - break; - - MATCH_NEWLINE( - parse_error("newline while parsing character constant"); - break; - ) - - case EOF: { - source_position_t source_position; - source_position.input_name = pp_token.source_position.input_name; - source_position.lineno = start_linenr; - errorf(&source_position, "EOF while parsing character constant"); - pp_token.type = TP_ERROR; - return; - } - - case '\'': - next_char(); - goto end_of_char_constant; - - default: - obstack_1grow(&symbol_obstack, (char) input.c); - next_char(); - break; - - } - } - -end_of_char_constant:; - const size_t size = (size_t)obstack_object_size(&symbol_obstack); - const char *const string = obstack_finish(&symbol_obstack); - - pp_token.type = TP_CHARACTER_CONSTANT; - pp_token.literal.begin = string; - pp_token.literal.size = size; -} - -#define SYMBOL_CHARS_WITHOUT_E_P \ - case 'a': \ +#define SYMBOL_CASES_WITHOUT_E_P \ + 'a': \ case 'b': \ case 'c': \ case 'd': \ @@ -680,17 +581,17 @@ end_of_char_constant:; case 'X': \ case 'Y': \ case 'Z': \ - case '_': + case '_' -#define SYMBOL_CHARS \ - SYMBOL_CHARS_WITHOUT_E_P \ +#define SYMBOL_CASES \ + SYMBOL_CASES_WITHOUT_E_P: \ case 'e': \ case 'p': \ case 'E': \ - case 'P': + case 'P' -#define DIGITS \ - case '0': \ +#define DIGIT_CASES \ + '0': \ case '1': \ case '2': \ case '3': \ @@ -699,7 +600,7 @@ end_of_char_constant:; case '6': \ case '7': \ case '8': \ - case '9': + case '9' /** * returns next final token from a preprocessor macro expansion @@ -730,14 +631,14 @@ restart: goto restart; } pp_token = definition->token_list[definition->expand_pos]; - pp_token.source_position = expansion_pos; + pp_token.base.source_position = expansion_pos; ++definition->expand_pos; - if (pp_token.type != TP_IDENTIFIER) + if (pp_token.kind != T_IDENTIFIER) return; /* if it was an identifier then we might need to expand again */ - pp_definition_t *symbol_definition = pp_token.symbol->pp_definition; + pp_definition_t *const symbol_definition = pp_token.base.symbol->pp_definition; if (symbol_definition != NULL && !symbol_definition->is_expanding) { symbol_definition->parent_expansion = definition; symbol_definition->expand_pos = 0; @@ -780,20 +681,19 @@ static void skip_multiline_comment(void) case '*': next_char(); if (input.c == '/') { + if (input.position.lineno != input.output_line) + info.whitespace = input.position.colno; next_char(); - info.whitespace += input.position.colno-1; return; } break; - MATCH_NEWLINE( - info.at_line_begin |= !in_pp_directive; + case NEWLINE: break; - ) case EOF: { source_position_t source_position; - source_position.input_name = pp_token.source_position.input_name; + source_position.input_name = pp_token.base.source_position.input_name; source_position.lineno = start_linenr; errorf(&source_position, "at end of file while looking for comment end"); return; @@ -815,10 +715,9 @@ static void skip_whitespace(void) next_char(); continue; - MATCH_NEWLINE( + case NEWLINE: info.at_line_begin = true; return; - ) case '/': next_char(); @@ -841,10 +740,17 @@ static void skip_whitespace(void) } } -static void eat_pp(int type) +static inline void eat_pp(pp_token_kind_t const kind) { - (void) type; - assert(pp_token.type == type); + assert(pp_token.base.symbol->pp_ID == kind); + (void) kind; + next_preprocessing_token(); +} + +static inline void eat_token(token_kind_t const kind) +{ + assert(pp_token.kind == kind); + (void)kind; next_preprocessing_token(); } @@ -855,8 +761,8 @@ static void parse_symbol(void) while (true) { switch (input.c) { - DIGITS - SYMBOL_CHARS + case DIGIT_CASES: + case SYMBOL_CASES: obstack_1grow(&symbol_obstack, (char) input.c); next_char(); break; @@ -873,18 +779,18 @@ end_symbol: /* might be a wide string or character constant ( L"string"/L'c' ) */ if (input.c == '"' && string[0] == 'L' && string[1] == '\0') { obstack_free(&symbol_obstack, string); - parse_wide_string_literal(); + parse_string_literal(STRING_ENCODING_WIDE); return; } else if (input.c == '\'' && string[0] == 'L' && string[1] == '\0') { obstack_free(&symbol_obstack, string); - parse_wide_character_constant(); + parse_character_constant(STRING_ENCODING_WIDE); return; } symbol_t *symbol = symbol_table_insert(string); - pp_token.type = symbol->pp_ID; - pp_token.symbol = symbol; + pp_token.kind = symbol->ID; + pp_token.base.symbol = symbol; /* we can free the memory from symbol obstack if we already had an entry in * the symbol table */ @@ -901,8 +807,8 @@ static void parse_number(void) while (true) { switch (input.c) { case '.': - DIGITS - SYMBOL_CHARS_WITHOUT_E_P + case DIGIT_CASES: + case SYMBOL_CASES_WITHOUT_E_P: obstack_1grow(&symbol_obstack, (char) input.c); next_char(); break; @@ -925,13 +831,8 @@ static void parse_number(void) } end_number: - obstack_1grow(&symbol_obstack, '\0'); - size_t size = obstack_object_size(&symbol_obstack); - char *string = obstack_finish(&symbol_obstack); - - pp_token.type = TP_NUMBER; - pp_token.literal.begin = string; - pp_token.literal.size = size; + pp_token.kind = T_NUMBER; + pp_token.literal.string = sym_make_string(STRING_ENCODING_CHAR); } @@ -943,7 +844,7 @@ end_number: #define MAYBE(ch, set_type) \ case ch: \ next_char(); \ - pp_token.type = set_type; \ + pp_token.kind = set_type; \ return; #define ELSE_CODE(code) \ @@ -955,7 +856,7 @@ end_number: #define ELSE(set_type) \ ELSE_CODE( \ - pp_token.type = set_type; \ + pp_token.kind = set_type; \ ) static void next_preprocessing_token(void) @@ -968,7 +869,9 @@ static void next_preprocessing_token(void) info.at_line_begin = false; info.had_whitespace = false; restart: - pp_token.source_position = input.position; + pp_token.base.source_position = input.position; + pp_token.base.symbol = NULL; + switch (input.c) { case ' ': case '\t': @@ -977,26 +880,25 @@ restart: next_char(); goto restart; - MATCH_NEWLINE( + case NEWLINE: info.at_line_begin = true; info.had_whitespace = true; goto restart; - ) - SYMBOL_CHARS + case SYMBOL_CASES: parse_symbol(); return; - DIGITS + case DIGIT_CASES: parse_number(); return; case '"': - parse_string_literal(); + parse_string_literal(STRING_ENCODING_CHAR); return; case '\'': - parse_character_constant(); + parse_character_constant(STRING_ENCODING_CHAR); return; case '.': @@ -1018,40 +920,40 @@ restart: case '.': MAYBE_PROLOG - MAYBE('.', TP_DOTDOTDOT) + MAYBE('.', T_DOTDOTDOT) ELSE_CODE( put_back(input.c); input.c = '.'; - pp_token.type = '.'; + pp_token.kind = '.'; ) ELSE('.') case '&': MAYBE_PROLOG - MAYBE('&', TP_ANDAND) - MAYBE('=', TP_ANDEQUAL) + MAYBE('&', T_ANDAND) + MAYBE('=', T_ANDEQUAL) ELSE('&') case '*': MAYBE_PROLOG - MAYBE('=', TP_ASTERISKEQUAL) + MAYBE('=', T_ASTERISKEQUAL) ELSE('*') case '+': MAYBE_PROLOG - MAYBE('+', TP_PLUSPLUS) - MAYBE('=', TP_PLUSEQUAL) + MAYBE('+', T_PLUSPLUS) + MAYBE('=', T_PLUSEQUAL) ELSE('+') case '-': MAYBE_PROLOG - MAYBE('>', TP_MINUSGREATER) - MAYBE('-', TP_MINUSMINUS) - MAYBE('=', TP_MINUSEQUAL) + MAYBE('>', T_MINUSGREATER) + MAYBE('-', T_MINUSMINUS) + MAYBE('=', T_MINUSEQUAL) ELSE('-') case '!': MAYBE_PROLOG - MAYBE('=', TP_EXCLAMATIONMARKEQUAL) + MAYBE('=', T_EXCLAMATIONMARKEQUAL) ELSE('!') case '/': MAYBE_PROLOG - MAYBE('=', TP_SLASHEQUAL) + MAYBE('=', T_SLASHEQUAL) case '*': next_char(); info.had_whitespace = true; @@ -1066,16 +968,16 @@ restart: case '%': MAYBE_PROLOG MAYBE('>', '}') - MAYBE('=', TP_PERCENTEQUAL) + MAYBE('=', T_PERCENTEQUAL) case ':': MAYBE_PROLOG case '%': MAYBE_PROLOG - MAYBE(':', TP_HASHHASH) + MAYBE(':', T_HASHHASH) ELSE_CODE( put_back(input.c); input.c = '%'; - pp_token.type = '#'; + pp_token.kind = '#'; ) ELSE('#') ELSE('%') @@ -1083,28 +985,28 @@ restart: MAYBE_PROLOG MAYBE(':', '[') MAYBE('%', '{') - MAYBE('=', TP_LESSEQUAL) + MAYBE('=', T_LESSEQUAL) case '<': MAYBE_PROLOG - MAYBE('=', TP_LESSLESSEQUAL) - ELSE(TP_LESSLESS) + MAYBE('=', T_LESSLESSEQUAL) + ELSE(T_LESSLESS) ELSE('<') case '>': MAYBE_PROLOG - MAYBE('=', TP_GREATEREQUAL) + MAYBE('=', T_GREATEREQUAL) case '>': MAYBE_PROLOG - MAYBE('=', TP_GREATERGREATEREQUAL) - ELSE(TP_GREATERGREATER) + MAYBE('=', T_GREATERGREATEREQUAL) + ELSE(T_GREATERGREATER) ELSE('>') case '^': MAYBE_PROLOG - MAYBE('=', TP_CARETEQUAL) + MAYBE('=', T_CARETEQUAL) ELSE('^') case '|': MAYBE_PROLOG - MAYBE('=', TP_PIPEEQUAL) - MAYBE('|', TP_PIPEPIPE) + MAYBE('=', T_PIPEEQUAL) + MAYBE('|', T_PIPEPIPE) ELSE('|') case ':': MAYBE_PROLOG @@ -1112,13 +1014,13 @@ restart: ELSE(':') case '=': MAYBE_PROLOG - MAYBE('=', TP_EQUALEQUAL) + MAYBE('=', T_EQUALEQUAL) ELSE('=') case '#': MAYBE_PROLOG - MAYBE('#', TP_HASHHASH) + MAYBE('#', T_HASHHASH) ELSE_CODE( - pp_token.type = '#'; + pp_token.kind = '#'; ) case '?': @@ -1131,8 +1033,7 @@ restart: case '~': case ';': case ',': - case '\\': - pp_token.type = input.c; + pp_token.kind = input.c; next_char(); return; @@ -1144,22 +1045,32 @@ restart: print_line_directive(&input.position, "2"); goto restart; } else { - pp_token.source_position.lineno++; + pp_token.base.source_position.lineno++; info.at_line_begin = true; - pp_token.type = TP_EOF; + pp_token.kind = T_EOF; } return; default: - next_char(); - if (!ignore_unknown_chars) { - errorf(&pp_token.source_position, "unknown character '%c' found\n", - input.c); - pp_token.type = TP_ERROR; + if (error_on_unknown_chars) { + errorf(&pp_token.base.source_position, + "unknown character '%lc' found\n", input.c); + next_char(); + goto restart; } else { - pp_token.type = input.c; + assert(obstack_object_size(&symbol_obstack) == 0); + obstack_grow_utf8(&symbol_obstack, input.c); + obstack_1grow(&symbol_obstack, '\0'); + char *const string = obstack_finish(&symbol_obstack); + symbol_t *const symbol = symbol_table_insert(string); + if (symbol->string != string) + obstack_free(&symbol_obstack, string); + + pp_token.kind = T_UNKNOWN_CHAR; + pp_token.base.symbol = symbol; + next_char(); + return; } - return; } } @@ -1203,20 +1114,27 @@ static void print_line_directive(const source_position_t *pos, const char *add) input.output_line = pos->lineno-1; } -static void emit_newlines(void) +static bool emit_newlines(void) { - unsigned delta = pp_token.source_position.lineno - input.output_line; + unsigned delta = pp_token.base.source_position.lineno - input.output_line; + if (delta == 0) + return false; if (delta >= 9) { fputc('\n', out); - print_line_directive(&pp_token.source_position, NULL); + print_line_directive(&pp_token.base.source_position, NULL); fputc('\n', out); } else { for (unsigned i = 0; i < delta; ++i) { fputc('\n', out); } } - input.output_line = pp_token.source_position.lineno; + input.output_line = pp_token.base.source_position.lineno; + + for (unsigned i = 0; i < info.whitespace; ++i) + fputc(' ', out); + + return true; } static void emit_pp_token(void) @@ -1224,43 +1142,38 @@ static void emit_pp_token(void) if (skip_mode) return; - if (info.at_line_begin) { - emit_newlines(); - - for (unsigned i = 0; i < info.whitespace; ++i) - fputc(' ', out); - - } else if (info.had_whitespace || - tokens_would_paste(last_token, pp_token.type)) { + if (!emit_newlines() && + (info.had_whitespace || tokens_would_paste(last_token, pp_token.kind))) fputc(' ', out); - } - switch (pp_token.type) { - case TP_IDENTIFIER: - fputs(pp_token.symbol->string, out); - break; - case TP_NUMBER: - fputs(pp_token.literal.begin, out); + switch (pp_token.kind) { + case T_NUMBER: + fputs(pp_token.literal.string.begin, out); break; - case TP_WIDE_STRING_LITERAL: - fputc('L', out); - case TP_STRING_LITERAL: + + case T_STRING_LITERAL: + fputs(get_string_encoding_prefix(pp_token.literal.string.encoding), out); fputc('"', out); - fputs(pp_token.literal.begin, out); + fputs(pp_token.literal.string.begin, out); fputc('"', out); break; - case TP_WIDE_CHARACTER_CONSTANT: - fputc('L', out); - case TP_CHARACTER_CONSTANT: + + case T_CHARACTER_CONSTANT: + fputs(get_string_encoding_prefix(pp_token.literal.string.encoding), out); fputc('\'', out); - fputs(pp_token.literal.begin, out); + fputs(pp_token.literal.string.begin, out); fputc('\'', out); break; + default: - print_pp_token_type(out, pp_token.type); + if (pp_token.base.symbol) { + fputs(pp_token.base.symbol->string, out); + } else { + print_token_kind(out, pp_token.kind); + } break; } - last_token = pp_token.type; + last_token = pp_token.kind; } static void eat_pp_directive(void) @@ -1287,19 +1200,17 @@ static bool strings_equal(const string_t *string1, const string_t *string2) static bool pp_tokens_equal(const token_t *token1, const token_t *token2) { - if (token1->type != token2->type) + if (token1->kind != token2->kind) return false; - switch (token1->type) { - case TP_HEADERNAME: - /* TODO */ - return false; - case TP_IDENTIFIER: - return token1->symbol == token2->symbol; - case TP_NUMBER: - case TP_CHARACTER_CONSTANT: - case TP_STRING_LITERAL: - return strings_equal(&token1->literal, &token2->literal); + switch (token1->kind) { + case T_IDENTIFIER: + return token1->base.symbol == token2->base.symbol; + + case T_NUMBER: + case T_CHARACTER_CONSTANT: + case T_STRING_LITERAL: + return strings_equal(&token1->literal.string, &token2->literal.string); default: return true; @@ -1325,14 +1236,19 @@ static bool pp_definitions_equal(const pp_definition_t *definition1, static void parse_define_directive(void) { eat_pp(TP_define); + if (skip_mode) { + eat_pp_directive(); + return; + } + assert(obstack_object_size(&pp_obstack) == 0); - if (pp_token.type != TP_IDENTIFIER || info.at_line_begin) { - errorf(&pp_token.source_position, - "expected identifier after #define, got '%t'", &pp_token); + if (pp_token.kind != T_IDENTIFIER || info.at_line_begin) { + errorf(&pp_token.base.source_position, + "expected identifier after #define, got %K", &pp_token); goto error_out; } - symbol_t *symbol = pp_token.symbol; + symbol_t *const symbol = pp_token.base.symbol; pp_definition_t *new_definition = obstack_alloc(&pp_obstack, sizeof(new_definition[0])); @@ -1343,44 +1259,45 @@ static void parse_define_directive(void) * lexer (except for the fact that they separate tokens). #define b(x) * is something else than #define b (x) */ if (input.c == '(') { - /* eat the '(' */ - next_preprocessing_token(); - /* get next token after '(' */ - next_preprocessing_token(); + eat_token(T_IDENTIFIER); + eat_token('('); while (true) { - switch (pp_token.type) { - case TP_DOTDOTDOT: + switch (pp_token.kind) { + case T_DOTDOTDOT: new_definition->is_variadic = true; - next_preprocessing_token(); - if (pp_token.type != ')') { + eat_token(T_DOTDOTDOT); + if (pp_token.kind != ')') { errorf(&input.position, "'...' not at end of macro argument list"); goto error_out; } break; - case TP_IDENTIFIER: - obstack_ptr_grow(&pp_obstack, pp_token.symbol); - next_preprocessing_token(); - if (pp_token.type == ',') { - next_preprocessing_token(); + case T_IDENTIFIER: + obstack_ptr_grow(&pp_obstack, pp_token.base.symbol); + eat_token(T_IDENTIFIER); + + if (pp_token.kind == ',') { + eat_token(','); break; } - if (pp_token.type != ')') { - errorf(&pp_token.source_position, - "expected ',' or ')' after identifier, got '%t'", + if (pp_token.kind != ')') { + errorf(&pp_token.base.source_position, + "expected ',' or ')' after identifier, got %K", &pp_token); goto error_out; } break; + case ')': - next_preprocessing_token(); + eat_token(')'); goto finish_argument_list; + default: - errorf(&pp_token.source_position, - "expected identifier, '...' or ')' in #define argument list, got '%t'", + errorf(&pp_token.base.source_position, + "expected identifier, '...' or ')' in #define argument list, got %K", &pp_token); goto error_out; } @@ -1392,7 +1309,7 @@ static void parse_define_directive(void) = obstack_object_size(&pp_obstack) / sizeof(new_definition->parameters[0]); new_definition->parameters = obstack_finish(&pp_obstack); } else { - next_preprocessing_token(); + eat_token(T_IDENTIFIER); } /* construct a new pp_definition on the obstack */ @@ -1432,17 +1349,20 @@ error_out: static void parse_undef_directive(void) { eat_pp(TP_undef); + if (skip_mode) { + eat_pp_directive(); + return; + } - if (pp_token.type != TP_IDENTIFIER) { + if (pp_token.kind != T_IDENTIFIER) { errorf(&input.position, - "expected identifier after #undef, got '%t'", &pp_token); + "expected identifier after #undef, got %K", &pp_token); eat_pp_directive(); return; } - symbol_t *symbol = pp_token.symbol; - symbol->pp_definition = NULL; - next_preprocessing_token(); + pp_token.base.symbol->pp_definition = NULL; + eat_token(T_IDENTIFIER); if (!info.at_line_begin) { warningf(WARN_OTHER, &input.position, "extra tokens at end of #undef directive"); @@ -1450,58 +1370,49 @@ static void parse_undef_directive(void) eat_pp_directive(); } -static const char *parse_headername(void) +/** behind an #include we can have the special headername lexems. + * They're only allowed behind an #include so they're not recognized + * by the normal next_preprocessing_token. We handle them as a special + * exception here */ +static void parse_headername(void) { - /* behind an #include we can have the special headername lexems. - * They're only allowed behind an #include so they're not recognized - * by the normal next_preprocessing_token. We handle them as a special - * exception here */ + const source_position_t start_position = input.position; + string_t string = { NULL, 0, STRING_ENCODING_CHAR }; + assert(obstack_object_size(&symbol_obstack) == 0); + if (info.at_line_begin) { parse_error("expected headername after #include"); - return NULL; + goto finish_error; } - assert(obstack_object_size(&symbol_obstack) == 0); - /* check wether we have a "... or <... headername */ switch (input.c) { - case '<': + { + utf32 delimiter; + case '<': delimiter = '>'; goto parse_name; + case '"': delimiter = '"'; goto parse_name; +parse_name: next_char(); while (true) { switch (input.c) { + case NEWLINE: case EOF: - /* fallthrough */ - MATCH_NEWLINE( - parse_error("header name without closing '>'"); - return NULL; - ) - case '>': - next_char(); - goto finished_headername; - } - obstack_1grow(&symbol_obstack, (char) input.c); - next_char(); - } - /* we should never be here */ + errorf(&pp_token.base.source_position, "header name without closing '%c'", (char)delimiter); + goto finish_error; - case '"': - next_char(); - while (true) { - switch (input.c) { - case EOF: - /* fallthrough */ - MATCH_NEWLINE( - parse_error("header name without closing '>'"); - return NULL; - ) - case '"': - next_char(); - goto finished_headername; + default: + if (input.c == delimiter) { + next_char(); + goto finished_headername; + } else { + obstack_1grow(&symbol_obstack, (char)input.c); + next_char(); + } + break; } - obstack_1grow(&symbol_obstack, (char) input.c); - next_char(); } /* we should never be here */ + } default: /* TODO: do normal pp_token parsing and concatenate results */ @@ -1509,22 +1420,29 @@ static const char *parse_headername(void) } finished_headername: - obstack_1grow(&symbol_obstack, '\0'); - char *headername = obstack_finish(&symbol_obstack); - - /* TODO: iterate search-path to find the file */ + string = sym_make_string(STRING_ENCODING_CHAR); - skip_whitespace(); - - return identify_string(headername); +finish_error: + pp_token.base.source_position = start_position; + pp_token.kind = T_HEADERNAME; + pp_token.literal.string = string; } static bool do_include(bool system_include, const char *headername) { + size_t headername_len = strlen(headername); if (!system_include) { - /* for "bla" includes first try current dir - * TODO: this isn't correct, should be the directory of the source file - */ + /* put dirname of current input on obstack */ + const char *filename = input.position.input_name; + const char *last_slash = strrchr(filename, '/'); + if (last_slash != NULL) { + size_t len = last_slash - filename; + obstack_grow(&symbol_obstack, filename, len + 1); + obstack_grow0(&symbol_obstack, headername, headername_len); + char *complete_path = obstack_finish(&symbol_obstack); + headername = identify_string(complete_path); + } + FILE *file = fopen(headername, "r"); if (file != NULL) { switch_input(file, headername); @@ -1532,73 +1450,86 @@ static bool do_include(bool system_include, const char *headername) } } - size_t headername_len = strlen(headername); - assert(obstack_object_size(&pp_obstack) == 0); + assert(obstack_object_size(&symbol_obstack) == 0); /* check searchpath */ for (searchpath_entry_t *entry = searchpath; entry != NULL; entry = entry->next) { const char *path = entry->path; size_t len = strlen(path); - obstack_grow(&pp_obstack, path, len); + obstack_grow(&symbol_obstack, path, len); if (path[len-1] != '/') - obstack_1grow(&pp_obstack, '/'); - obstack_grow(&pp_obstack, headername, headername_len+1); + obstack_1grow(&symbol_obstack, '/'); + obstack_grow(&symbol_obstack, headername, headername_len+1); - char *complete_path = obstack_finish(&pp_obstack); + char *complete_path = obstack_finish(&symbol_obstack); FILE *file = fopen(complete_path, "r"); if (file != NULL) { const char *filename = identify_string(complete_path); switch_input(file, filename); return true; + } else { + obstack_free(&symbol_obstack, complete_path); } - obstack_free(&pp_obstack, complete_path); } return false; } -static bool parse_include_directive(void) +/* read till next newline character, only for parse_include_directive(), + * use eat_pp_directive() in all other cases */ +static void skip_till_newline(void) +{ + /* skip till newline */ + while (true) { + switch (input.c) { + case NEWLINE: + case EOF: + return; + } + next_char(); + } +} + +static void parse_include_directive(void) { + if (skip_mode) { + eat_pp_directive(); + return; + } + /* don't eat the TP_include here! * we need an alternative parsing for the next token */ skip_whitespace(); bool system_include = input.c == '<'; - const char *headername = parse_headername(); - if (headername == NULL) { + parse_headername(); + string_t headername = pp_token.literal.string; + if (headername.begin == NULL) { eat_pp_directive(); - return false; + return; } + skip_whitespace(); if (!info.at_line_begin) { - warningf(WARN_OTHER, &pp_token.source_position, "extra tokens at end of #include directive"); - eat_pp_directive(); + warningf(WARN_OTHER, &pp_token.base.source_position, + "extra tokens at end of #include directive"); + skip_till_newline(); } if (n_inputs > INCLUDE_LIMIT) { - errorf(&pp_token.source_position, "#include nested too deeply"); + errorf(&pp_token.base.source_position, "#include nested too deeply"); /* eat \n or EOF */ next_preprocessing_token(); - return false; + return; } - /* we have to reenable space counting and macro expansion here, - * because it is still disabled in directive parsing, - * but we will trigger a preprocessing token reading of the new file - * now and need expansions/space counting */ - in_pp_directive = false; - /* switch inputs */ emit_newlines(); push_input(); - bool res = do_include(system_include, headername); + bool res = do_include(system_include, pp_token.literal.string.begin); if (!res) { - errorf(&pp_token.source_position, - "failed including '%s': %s", headername, strerror(errno)); + errorf(&pp_token.base.source_position, "failed including '%S': %s", &pp_token.literal, strerror(errno)); pop_restore_input(); - return false; } - - return true; } static pp_conditional_t *push_conditional(void) @@ -1633,46 +1564,42 @@ static void check_unclosed_conditionals(void) } } -static void parse_ifdef_ifndef_directive(void) +static void parse_ifdef_ifndef_directive(bool const is_ifdef) { - bool is_ifndef = (pp_token.type == TP_ifndef); bool condition; - next_preprocessing_token(); + eat_pp(is_ifdef ? TP_ifdef : TP_ifndef); if (skip_mode) { eat_pp_directive(); pp_conditional_t *conditional = push_conditional(); - conditional->source_position = pp_token.source_position; + conditional->source_position = pp_token.base.source_position; conditional->skip = true; return; } - if (pp_token.type != TP_IDENTIFIER || info.at_line_begin) { - errorf(&pp_token.source_position, - "expected identifier after #%s, got '%t'", - is_ifndef ? "ifndef" : "ifdef", &pp_token); + if (pp_token.kind != T_IDENTIFIER || info.at_line_begin) { + errorf(&pp_token.base.source_position, + "expected identifier after #%s, got %K", + is_ifdef ? "ifdef" : "ifndef", &pp_token); eat_pp_directive(); /* just take the true case in the hope to avoid further errors */ condition = true; } else { - symbol_t *symbol = pp_token.symbol; - pp_definition_t *pp_definition = symbol->pp_definition; - next_preprocessing_token(); + /* evaluate wether we are in true or false case */ + condition = (bool)pp_token.base.symbol->pp_definition == is_ifdef; + eat_token(T_IDENTIFIER); if (!info.at_line_begin) { - errorf(&pp_token.source_position, + errorf(&pp_token.base.source_position, "extra tokens at end of #%s", - is_ifndef ? "ifndef" : "ifdef"); + is_ifdef ? "ifdef" : "ifndef"); eat_pp_directive(); } - - /* evaluate wether we are in true or false case */ - condition = is_ifndef ? pp_definition == NULL : pp_definition != NULL; } pp_conditional_t *conditional = push_conditional(); - conditional->source_position = pp_token.source_position; + conditional->source_position = pp_token.base.source_position; conditional->condition = condition; if (!condition) { @@ -1686,21 +1613,21 @@ static void parse_else_directive(void) if (!info.at_line_begin) { if (!skip_mode) { - warningf(WARN_OTHER, &pp_token.source_position, "extra tokens at end of #else"); + warningf(WARN_OTHER, &pp_token.base.source_position, "extra tokens at end of #else"); } eat_pp_directive(); } pp_conditional_t *conditional = conditional_stack; if (conditional == NULL) { - errorf(&pp_token.source_position, "#else without prior #if"); + errorf(&pp_token.base.source_position, "#else without prior #if"); return; } if (conditional->in_else) { - errorf(&pp_token.source_position, + errorf(&pp_token.base.source_position, "#else after #else (condition started %P)", - conditional->source_position); + &conditional->source_position); skip_mode = true; return; } @@ -1709,7 +1636,7 @@ static void parse_else_directive(void) if (!conditional->skip) { skip_mode = conditional->condition; } - conditional->source_position = pp_token.source_position; + conditional->source_position = pp_token.base.source_position; } static void parse_endif_directive(void) @@ -1718,14 +1645,14 @@ static void parse_endif_directive(void) if (!info.at_line_begin) { if (!skip_mode) { - warningf(WARN_OTHER, &pp_token.source_position, "extra tokens at end of #endif"); + warningf(WARN_OTHER, &pp_token.base.source_position, "extra tokens at end of #endif"); } eat_pp_directive(); } pp_conditional_t *conditional = conditional_stack; if (conditional == NULL) { - errorf(&pp_token.source_position, "#endif without prior #if"); + errorf(&pp_token.base.source_position, "#endif without prior #if"); return; } @@ -1737,59 +1664,32 @@ static void parse_endif_directive(void) static void parse_preprocessing_directive(void) { - in_pp_directive = true; - eat_pp('#'); + eat_token('#'); - if (skip_mode) { - switch (pp_token.type) { - case TP_ifdef: - case TP_ifndef: - parse_ifdef_ifndef_directive(); - break; - case TP_else: - parse_else_directive(); - break; - case TP_endif: - parse_endif_directive(); - break; - default: - eat_pp_directive(); - break; + if (info.at_line_begin) { + /* empty directive */ + return; + } + + if (pp_token.base.symbol) { + switch (pp_token.base.symbol->pp_ID) { + case TP_define: parse_define_directive(); break; + case TP_else: parse_else_directive(); break; + case TP_endif: parse_endif_directive(); break; + case TP_ifdef: parse_ifdef_ifndef_directive(true); break; + case TP_ifndef: parse_ifdef_ifndef_directive(false); break; + case TP_include: parse_include_directive(); break; + case TP_undef: parse_undef_directive(); break; + default: goto skip; } } else { - switch (pp_token.type) { - case TP_define: - parse_define_directive(); - break; - case TP_undef: - parse_undef_directive(); - break; - case TP_ifdef: - case TP_ifndef: - parse_ifdef_ifndef_directive(); - break; - case TP_else: - parse_else_directive(); - break; - case TP_endif: - parse_endif_directive(); - break; - case TP_include: - parse_include_directive(); - break; - default: - if (info.at_line_begin) { - /* the nop directive "#" */ - break; - } - errorf(&pp_token.source_position, - "invalid preprocessing directive #%t", &pp_token); - eat_pp_directive(); - break; +skip: + if (!skip_mode) { + errorf(&pp_token.base.source_position, "invalid preprocessing directive #%K", &pp_token); } + eat_pp_directive(); } - in_pp_directive = false; assert(info.at_line_begin); } @@ -1848,10 +1748,13 @@ int pptest_main(int argc, char **argv) obstack_init(&input_obstack); strset_init(&stringset); + error_on_unknown_chars = false; + setup_include_path(); /* simplistic commandline parser */ const char *filename = NULL; + const char *output = NULL; for (int i = 1; i < argc; ++i) { const char *opt = argv[i]; if (streq(opt, "-I")) { @@ -1859,6 +1762,9 @@ int pptest_main(int argc, char **argv) continue; } else if (streq(opt, "-E")) { /* ignore */ + } else if (streq(opt, "-o")) { + output = argv[++i]; + continue; } else if (opt[0] == '-') { fprintf(stderr, "Unknown option '%s'\n", opt); } else { @@ -1872,7 +1778,15 @@ int pptest_main(int argc, char **argv) return 1; } - out = stdout; + if (output == NULL) { + out = stdout; + } else { + out = fopen(output, "w"); + if (out == NULL) { + fprintf(stderr, "Couldn't open output '%s'\n", output); + return 1; + } + } /* just here for gcc compatibility */ fprintf(out, "# 1 \"%s\"\n", filename); @@ -1887,35 +1801,35 @@ int pptest_main(int argc, char **argv) switch_input(file, filename); while (true) { - if (pp_token.type == '#' && info.at_line_begin) { + if (pp_token.kind == '#' && info.at_line_begin) { parse_preprocessing_directive(); continue; - } else if (pp_token.type == TP_EOF) { + } else if (pp_token.kind == T_EOF) { goto end_of_main_loop; - } else if (pp_token.type == TP_IDENTIFIER && !in_pp_directive) { - symbol_t *symbol = pp_token.symbol; - pp_definition_t *pp_definition = symbol->pp_definition; + } else if (pp_token.kind == T_IDENTIFIER) { + symbol_t *const symbol = pp_token.base.symbol; + pp_definition_t *const pp_definition = symbol->pp_definition; if (pp_definition != NULL && !pp_definition->is_expanding) { - expansion_pos = pp_token.source_position; + expansion_pos = pp_token.base.source_position; if (pp_definition->has_parameters) { - source_position_t position = pp_token.source_position; + source_position_t position = pp_token.base.source_position; add_token_info_t old_info = info; - next_preprocessing_token(); + eat_token(T_IDENTIFIER); add_token_info_t new_info = info; /* no opening brace -> no expansion */ - if (pp_token.type == '(') { - eat_pp('('); + if (pp_token.kind == '(') { + eat_token('('); /* parse arguments (TODO) */ - while (pp_token.type != TP_EOF && pp_token.type != ')') + while (pp_token.kind != T_EOF && pp_token.kind != ')') next_preprocessing_token(); } else { token_t next_token = pp_token; /* restore identifier token */ - pp_token.type = TP_IDENTIFIER; - pp_token.symbol = symbol; - pp_token.source_position = position; + pp_token.kind = T_IDENTIFIER; + pp_token.base.symbol = symbol; + pp_token.base.source_position = position; info = old_info; emit_pp_token(); @@ -1941,6 +1855,8 @@ end_of_main_loop: fputc('\n', out); check_unclosed_conditionals(); close_input(); + if (out != stdout) + fclose(out); obstack_free(&input_obstack, NULL); obstack_free(&pp_obstack, NULL);