X-Git-Url: http://nsz.repo.hu/git/?a=blobdiff_plain;f=lexer.c;h=8a339aa110058836c6f208ad7eb15f9eaa34232f;hb=3ebfcac30456a053d3eba3fb1a2df7b1e73668b8;hp=ce964fa4cd11a03014d426557587f0b6bbec8b5a;hpb=3f09e64eade51d0c8a4e21d2da68d0d48f2f7f53;p=cparser diff --git a/lexer.c b/lexer.c index ce964fa..8a339aa 100644 --- a/lexer.c +++ b/lexer.c @@ -1,141 +1,187 @@ +/* + * This file is part of cparser. + * Copyright (C) 2007-2009 Matthias Braun + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA + * 02111-1307, USA. + */ #include +#include "adt/strutil.h" +#include "input.h" +#include "diagnostic.h" #include "lexer.h" +#include "symbol_t.h" #include "token_t.h" #include "symbol_table_t.h" #include "adt/error.h" #include "adt/strset.h" +#include "adt/util.h" +#include "types.h" +#include "type_t.h" +#include "target_architecture.h" +#include "parser.h" +#include "warning.h" +#include "lang_features.h" #include #include #include +#include #include -//#define DEBUG_CHARS -#define MAX_PUTBACK 3 - -static int c; -token_t lexer_token; -static FILE *input; -static char buf[1024 + MAX_PUTBACK]; -static const char *bufend; -static const char *bufpos; -static strset_t stringset; -//static FILE **input_stack; -//static char **buf_stack; - -static -void error_prefix_at(const char *input_name, unsigned linenr) -{ - fprintf(stderr, "%s:%d: Error: ", input_name, linenr); -} +#ifndef _WIN32 +#include +#endif -static -void error_prefix(void) +#define MAX_PUTBACK 16 // 3 would be enough, but 16 gives a nicer alignment +#define BUF_SIZE 1024 + +static input_t *input; +static utf32 input_buf[BUF_SIZE + MAX_PUTBACK]; +static const utf32 *bufpos; +static const utf32 *bufend; +static utf32 c; +static source_position_t lexer_pos; +token_t lexer_token; +static symbol_t *symbol_L; +static strset_t stringset; +bool allow_dollar_in_symbol = true; + +/** + * Prints a parse error message at the current token. + * + * @param msg the error message + */ +static void parse_error(const char *msg) { - error_prefix_at(lexer_token.source_position.input_name, - lexer_token.source_position.linenr); + errorf(&lexer_pos, "%s", msg); } -static -void parse_error(const char *msg) +/** + * Prints an internal error message at the current token. + * + * @param msg the error message + */ +static NORETURN internal_error(const char *msg) { - error_prefix(); - fprintf(stderr, "%s\n", msg); + internal_errorf(&lexer_pos, "%s", msg); } -static inline -void next_char(void) +static inline void next_real_char(void) { - bufpos++; - if(bufpos >= bufend) { - size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK, - input); - if(s == 0) { + assert(bufpos <= bufend); + if (bufpos >= bufend) { + size_t n = decode(input, input_buf+MAX_PUTBACK, BUF_SIZE); + if (n == 0) { c = EOF; return; } - bufpos = buf + MAX_PUTBACK; - bufend = buf + MAX_PUTBACK + s; + bufpos = input_buf + MAX_PUTBACK; + bufend = bufpos + n; } - c = *(bufpos); -#ifdef DEBUG_CHARS - printf("nchar '%c'\n", c); -#endif + c = *bufpos++; + ++lexer_pos.colno; } -static inline -void put_back(int pc) +/** + * Put a character back into the buffer. + * + * @param pc the character to put back + */ +static inline void put_back(utf32 const pc) { - char *p = (char*) bufpos - 1; - bufpos--; - assert(p >= buf); - *p = pc; - -#ifdef DEBUG_CHARS - printf("putback '%c'\n", pc); -#endif + *(--bufpos - input_buf + input_buf) = pc; + --lexer_pos.colno; } +static inline void next_char(void); + +#define MATCH_NEWLINE(code) \ + case '\r': \ + next_char(); \ + if (c == '\n') { \ + case '\n': \ + next_char(); \ + } \ + lexer_pos.lineno++; \ + lexer_pos.colno = 1; \ + code -static -int replace_trigraph(void) +#define eat(c_type) (assert(c == c_type), next_char()) + +static void maybe_concat_lines(void) { -#define MATCH_TRIGRAPH(ch,replacement) \ - case ch: \ - c = replacement; \ - return 1; + eat('\\'); + + switch (c) { + MATCH_NEWLINE(return;) - switch(c) { - MATCH_TRIGRAPH('=', '#') - MATCH_TRIGRAPH('(', '[') - MATCH_TRIGRAPH('/', '\\') - MATCH_TRIGRAPH(')', ']') - MATCH_TRIGRAPH('\'', '^') - MATCH_TRIGRAPH('<', '{') - MATCH_TRIGRAPH('!', '|') - MATCH_TRIGRAPH('>', '}') - MATCH_TRIGRAPH('-', '~') default: break; } - return 0; -} - -#define SKIP_TRIGRAPHS(custom_putback, no_trigraph_code) \ - case '?': \ - next_char(); \ - if(c != '?') { \ - custom_putback; \ - put_back(c); \ - c = '?'; \ - no_trigraph_code; \ - } \ - next_char(); \ - if(replace_trigraph()) { \ - break; \ - } \ - custom_putback; \ - put_back('?'); \ - put_back(c); \ - c = '?'; \ - no_trigraph_code; - -#define EAT_NEWLINE(newline_code) \ - if(c == '\r') { \ - next_char(); \ - if(c == '\n') \ - next_char(); \ - lexer_token.source_position.linenr++; \ - newline_code; \ - } else if(c == '\n') { \ - next_char(); \ - lexer_token.source_position.linenr++; \ - newline_code; \ + put_back(c); + c = '\\'; +} + +/** + * Set c to the next input character, ie. + * after expanding trigraphs. + */ +static inline void next_char(void) +{ + next_real_char(); + + /* filter trigraphs */ + if (UNLIKELY(c == '\\')) { + maybe_concat_lines(); + return; + } + + if (LIKELY(c != '?')) + return; + + next_real_char(); + if (LIKELY(c != '?')) { + put_back(c); + c = '?'; + return; } + next_real_char(); + switch (c) { + case '=': c = '#'; break; + case '(': c = '['; break; + case '/': c = '\\'; maybe_concat_lines(); break; + case ')': c = ']'; break; + case '\'': c = '^'; break; + case '<': c = '{'; break; + case '!': c = '|'; break; + case '>': c = '}'; break; + case '-': c = '~'; break; + default: + put_back(c); + put_back('?'); + c = '?'; + break; + } +} + #define SYMBOL_CHARS \ + case '$': if (!allow_dollar_in_symbol) goto dollar_sign; \ case 'a': \ case 'b': \ case 'c': \ @@ -202,247 +248,420 @@ int replace_trigraph(void) case '8': \ case '9': -static -void parse_symbol(void) +/** + * Read a symbol from the input and build + * the lexer_token. + */ +static void parse_symbol(void) { - symbol_t *symbol; - char *string; - - obstack_1grow(&symbol_obstack, c); + obstack_1grow(&symbol_obstack, (char) c); next_char(); - while(1) { - switch(c) { - case '\\': - next_char(); - EAT_NEWLINE(break;) - goto end_symbol; - + while (true) { + switch (c) { DIGITS SYMBOL_CHARS - obstack_1grow(&symbol_obstack, c); + obstack_1grow(&symbol_obstack, (char) c); next_char(); break; - case '?': - next_char(); - if(c != '?') { - put_back(c); - c = '?'; - goto end_symbol; - } - next_char(); - if(replace_trigraph()) - break; - put_back('?'); - put_back(c); - c = '?'; - goto end_symbol; - default: +dollar_sign: goto end_symbol; } } + end_symbol: obstack_1grow(&symbol_obstack, '\0'); - string = obstack_finish(&symbol_obstack); - symbol = symbol_table_insert(string); + char *string = obstack_finish(&symbol_obstack); + symbol_t *symbol = symbol_table_insert(string); - lexer_token.type = symbol->ID; - lexer_token.v.symbol = symbol; + lexer_token.kind = symbol->ID; + lexer_token.identifier.symbol = symbol; - if(symbol->string != string) { + if (symbol->string != string) { obstack_free(&symbol_obstack, string); } } -static -void parse_number_hex(void) +static string_t identify_string(char *string, size_t len) { - assert(c == 'x' || c == 'X'); - next_char(); - - if (!isdigit(c) && - !('A' <= c && c <= 'F') && - !('a' <= c && c <= 'f')) { - parse_error("premature end of hex number literal"); - lexer_token.type = T_ERROR; - return; + /* TODO hash */ +#if 0 + const char *result = strset_insert(&stringset, concat); + if (result != concat) { + obstack_free(&symbol_obstack, concat); } +#else + const char *result = string; +#endif + return (string_t) {result, len}; +} - int value = 0; - for(;;) { - if (isdigit(c)) { - value = 16 * value + c - '0'; - } else if ('A' <= c && c <= 'F') { - value = 16 * value + c - 'A' + 10; - } else if ('a' <= c && c <= 'f') { - value = 16 * value + c - 'a' + 10; - } else { - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = value; - return; +/** + * parse suffixes like 'LU' or 'f' after numbers + */ +static void parse_number_suffix(void) +{ + assert(obstack_object_size(&symbol_obstack) == 0); + while (true) { + switch (c) { + SYMBOL_CHARS + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + break; + default: + dollar_sign: + goto finish_suffix; } - next_char(); } +finish_suffix: + if (obstack_object_size(&symbol_obstack) == 0) { + lexer_token.number.suffix.begin = NULL; + lexer_token.number.suffix.size = 0; + return; + } + + obstack_1grow(&symbol_obstack, '\0'); + size_t size = obstack_object_size(&symbol_obstack) - 1; + char *string = obstack_finish(&symbol_obstack); + + lexer_token.number.suffix = identify_string(string, size); } -static -void parse_number_oct(void) +/** + * Parses a hex number including hex floats and set the + * lexer_token. + */ +static void parse_number_hex(void) { - assert(c == 'o' || c == 'O'); - next_char(); + bool is_float = false; + bool has_digits = false; - int value = 0; - for(;;) { - if ('0' <= c && c <= '7') { - value = 8 * value + c - '0'; - } else { - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = value; - return; + assert(obstack_object_size(&symbol_obstack) == 0); + while (isxdigit(c)) { + has_digits = true; + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + + if (c == '.') { + is_float = true; + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + + while (isxdigit(c)) { + has_digits = true; + obstack_1grow(&symbol_obstack, (char) c); + next_char(); } + } + if (c == 'p' || c == 'P') { + is_float = true; + obstack_1grow(&symbol_obstack, (char) c); next_char(); + + if (c == '-' || c == '+') { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + + while (isxdigit(c)) { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + } else if (is_float) { + errorf(&lexer_token.base.source_position, + "hexadecimal floatingpoint constant requires an exponent"); + } + obstack_1grow(&symbol_obstack, '\0'); + + size_t size = obstack_object_size(&symbol_obstack) - 1; + char *string = obstack_finish(&symbol_obstack); + lexer_token.number.number = identify_string(string, size); + + lexer_token.kind = + is_float ? T_FLOATINGPOINT_HEXADECIMAL : T_INTEGER_HEXADECIMAL; + + if (!has_digits) { + errorf(&lexer_token.base.source_position, + "invalid number literal '0x%S'", &lexer_token.number.number); + lexer_token.number.number.begin = "0"; + lexer_token.number.number.size = 1; } + + parse_number_suffix(); } -static -void parse_number_dec(int first_char) +/** + * Returns true if the given char is a octal digit. + * + * @param char the character to check + */ +static bool is_octal_digit(utf32 chr) { - int value = 0; - if(first_char > 0) { - assert(first_char >= '0' && first_char <= '9'); - value = first_char - '0'; - } + return '0' <= chr && chr <= '7'; +} - for(;;) { - if (isdigit(c)) { - value = 10 * value + c - '0'; - } else { - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = value; +/** + * Parses a number and sets the lexer_token. + */ +static void parse_number(void) +{ + bool is_float = false; + bool has_digits = false; + + assert(obstack_object_size(&symbol_obstack) == 0); + if (c == '0') { + next_char(); + if (c == 'x' || c == 'X') { + next_char(); + parse_number_hex(); return; + } else { + has_digits = true; } + obstack_1grow(&symbol_obstack, '0'); + } + + while (isdigit(c)) { + has_digits = true; + obstack_1grow(&symbol_obstack, (char) c); next_char(); } -} -static -void parse_number(void) -{ - // TODO check for overflow - // TODO check for various invalid inputs sequences + if (c == '.') { + is_float = true; + obstack_1grow(&symbol_obstack, '.'); + next_char(); - if (c == '0') { + while (isdigit(c)) { + has_digits = true; + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + } + if (c == 'e' || c == 'E') { + is_float = true; + obstack_1grow(&symbol_obstack, 'e'); next_char(); - switch (c) { - case 'X': - case 'x': parse_number_hex(); break; - case 'o': - case 'O': parse_number_oct(); break; - default: parse_number_dec('0'); + + if (c == '-' || c == '+') { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + + while (isdigit(c)) { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + } + + obstack_1grow(&symbol_obstack, '\0'); + size_t size = obstack_object_size(&symbol_obstack) - 1; + char *string = obstack_finish(&symbol_obstack); + lexer_token.number.number = identify_string(string, size); + + /* is it an octal number? */ + if (is_float) { + lexer_token.kind = T_FLOATINGPOINT; + } else if (string[0] == '0') { + lexer_token.kind = T_INTEGER_OCTAL; + + /* check for invalid octal digits */ + for (size_t i= 0; i < size; ++i) { + char t = string[i]; + if (t >= '8') + errorf(&lexer_token.base.source_position, + "invalid digit '%c' in octal number", t); } } else { - parse_number_dec(0); + lexer_token.kind = T_INTEGER; + } + + if (!has_digits) { + errorf(&lexer_token.base.source_position, "invalid number literal '%S'", + &lexer_token.number.number); } + + parse_number_suffix(); } -static -int parse_escape_sequence(void) +/** + * Returns the value of a digit. + * The only portable way to do it ... + */ +static int digit_value(utf32 const digit) { - while(1) { - int ec = c; - next_char(); + switch (digit) { + case '0': return 0; + case '1': return 1; + case '2': return 2; + case '3': return 3; + case '4': return 4; + case '5': return 5; + case '6': return 6; + case '7': return 7; + case '8': return 8; + case '9': return 9; + case 'a': + case 'A': return 10; + case 'b': + case 'B': return 11; + case 'c': + case 'C': return 12; + case 'd': + case 'D': return 13; + case 'e': + case 'E': return 14; + case 'f': + case 'F': return 15; + default: + internal_error("wrong character given"); + } +} - switch(ec) { - case '"': return '"'; - case '\'': return'\''; - case '\\': - EAT_NEWLINE(break;) - return '\\'; - case 'a': return '\a'; - case 'b': return '\b'; - case 'f': return '\f'; - case 'n': return '\n'; - case 'r': return '\r'; - case 't': return '\t'; - case 'v': return '\v'; - case 'x': /* TODO parse hex number ... */ - parse_error("hex escape sequences not implemented yet"); - return EOF; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - /* TODO parse octal number ... */ - parse_error("octal escape sequences not implemented yet"); - return EOF; - case '?': - if(c != '?') { - return '?'; - } - /* might be a trigraph */ - next_char(); - if(replace_trigraph()) { - break; - } - put_back(c); - c = '?'; - return '?'; +/** + * Parses an octal character sequence. + * + * @param first_digit the already read first digit + */ +static utf32 parse_octal_sequence(utf32 const first_digit) +{ + assert(is_octal_digit(first_digit)); + utf32 value = digit_value(first_digit); + if (!is_octal_digit(c)) return value; + value = 8 * value + digit_value(c); + next_char(); + if (!is_octal_digit(c)) return value; + value = 8 * value + digit_value(c); + next_char(); + return value; +} - case EOF: - parse_error("reached end of file while parsing escape sequence"); - return EOF; - default: - parse_error("unknown escape sequence"); - return EOF; - } +/** + * Parses a hex character sequence. + */ +static utf32 parse_hex_sequence(void) +{ + utf32 value = 0; + while (isxdigit(c)) { + value = 16 * value + digit_value(c); + next_char(); } + return value; } -static -void parse_string_literal(void) +/** + * Parse an escape sequence. + */ +static utf32 parse_escape_sequence(void) { - unsigned start_linenr = lexer_token.source_position.linenr; - char *string; - const char *result; + eat('\\'); - assert(c == '"'); + utf32 const ec = c; next_char(); - while(1) { - switch(c) { - SKIP_TRIGRAPHS(, - obstack_1grow(&symbol_obstack, '?'); - next_char(); - break; - ) + switch (ec) { + case '"': return '"'; + case '\'': return '\''; + case '\\': return '\\'; + case '?': return '\?'; + case 'a': return '\a'; + case 'b': return '\b'; + case 'f': return '\f'; + case 'n': return '\n'; + case 'r': return '\r'; + case 't': return '\t'; + case 'v': return '\v'; + case 'x': + return parse_hex_sequence(); + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + return parse_octal_sequence(ec); + case EOF: + parse_error("reached end of file while parsing escape sequence"); + return EOF; + /* \E is not documented, but handled, by GCC. It is acceptable according + * to §6.11.4, whereas \e is not. */ + case 'E': + case 'e': + if (c_mode & _GNUC) + return 27; /* hopefully 27 is ALWAYS the code for ESCAPE */ + break; + case 'u': + case 'U': + parse_error("universal character parsing not implemented yet"); + return EOF; + default: + break; + } + /* §6.4.4.4:8 footnote 64 */ + parse_error("unknown escape sequence"); + return EOF; +} - case '\\': - next_char(); - EAT_NEWLINE(break;) - int ec = parse_escape_sequence(); - obstack_1grow(&symbol_obstack, ec); +/** + * Concatenate two strings. + */ +string_t concat_strings(const string_t *const s1, const string_t *const s2) +{ + const size_t len1 = s1->size - 1; + const size_t len2 = s2->size - 1; + + char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1); + memcpy(concat, s1->begin, len1); + memcpy(concat + len1, s2->begin, len2 + 1); + + return identify_string(concat, len1 + len2 + 1); +} + +string_t make_string(const char *string) +{ + size_t len = strlen(string) + 1; + char *const space = obstack_alloc(&symbol_obstack, len); + memcpy(space, string, len); + + return identify_string(space, len); +} + +/** + * Parse a string literal and set lexer_token. + */ +static void parse_string_literal(void) +{ + eat('"'); + + while (true) { + switch (c) { + case '\\': { + utf32 const tc = parse_escape_sequence(); + if (tc >= 0x100) { + warningf(WARN_OTHER, &lexer_pos, "escape sequence out of range"); + } + obstack_1grow(&symbol_obstack, tc); break; + } - case EOF: - error_prefix_at(lexer_token.source_position.input_name, - start_linenr); - fprintf(stderr, "string has no end\n"); - lexer_token.type = T_ERROR; + case EOF: { + errorf(&lexer_token.base.source_position, "string has no end"); + lexer_token.kind = T_ERROR; return; + } case '"': next_char(); goto end_of_string; default: - obstack_1grow(&symbol_obstack, c); + obstack_grow_symbol(&symbol_obstack, c); next_char(); break; } @@ -454,50 +673,27 @@ end_of_string: /* add finishing 0 to the string */ obstack_1grow(&symbol_obstack, '\0'); - string = obstack_finish(&symbol_obstack); + const size_t size = (size_t)obstack_object_size(&symbol_obstack); + char *string = obstack_finish(&symbol_obstack); - /* check if there is already a copy of the string */ - result = strset_insert(&stringset, string); - if(result != string) { - obstack_free(&symbol_obstack, string); - } - - lexer_token.type = T_STRING_LITERAL; - lexer_token.v.string = result; + lexer_token.kind = T_STRING_LITERAL; + lexer_token.string.string = identify_string(string, size); } -#define MATCH_NEWLINE(code) \ - case '\r': \ - next_char(); \ - if(c == '\n') { \ - next_char(); \ - } \ - lexer_token.source_position.linenr++; \ - code; \ - case '\n': \ - next_char(); \ - lexer_token.source_position.linenr++; \ - code; - -static -void parse_character_constant(void) +/** + * Parse a wide character constant and set lexer_token. + */ +static void parse_wide_character_constant(void) { - assert(c == '\''); - next_char(); - - int found_char = 0; - while(1) { - switch(c) { - SKIP_TRIGRAPHS(, - found_char = '?'; - break; - ) + eat('\''); - case '\\': - next_char(); - EAT_NEWLINE(break;) - found_char = '\\'; + while (true) { + switch (c) { + case '\\': { + const utf32 tc = parse_escape_sequence(); + obstack_grow_symbol(&symbol_obstack, tc); break; + } MATCH_NEWLINE( parse_error("newline while parsing character constant"); @@ -506,119 +702,159 @@ void parse_character_constant(void) case '\'': next_char(); - goto end_of_char_constant; + goto end_of_wide_char_constant; - case EOF: - parse_error("EOF while parsing character constant"); - lexer_token.type = T_ERROR; + case EOF: { + errorf(&lexer_token.base.source_position, + "EOF while parsing character constant"); + lexer_token.kind = T_ERROR; return; + } default: - if(found_char != 0) { - parse_error("more than 1 characters in character " - "constant"); - goto end_of_char_constant; - } else { - found_char = c; - next_char(); - } + obstack_grow_symbol(&symbol_obstack, c); + next_char(); break; } } -end_of_char_constant: - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = found_char; +end_of_wide_char_constant:; + obstack_1grow(&symbol_obstack, '\0'); + size_t size = (size_t) obstack_object_size(&symbol_obstack) - 1; + char *string = obstack_finish(&symbol_obstack); + + lexer_token.kind = T_WIDE_CHARACTER_CONSTANT; + lexer_token.string.string = identify_string(string, size); + + if (size == 0) { + errorf(&lexer_token.base.source_position, "empty character constant"); + } } -static -void skip_multiline_comment(void) +/** + * Parse a wide string literal and set lexer_token. + */ +static void parse_wide_string_literal(void) { - unsigned start_linenr = lexer_token.source_position.linenr; - int had_star = 0; + parse_string_literal(); + if (lexer_token.kind == T_STRING_LITERAL) + lexer_token.kind = T_WIDE_STRING_LITERAL; +} - while(1) { - switch(c) { - case '*': - next_char(); - had_star = 1; - break; +/** + * Parse a character constant and set lexer_token. + */ +static void parse_character_constant(void) +{ + eat('\''); - case '/': - next_char(); - if(had_star) { - return; + while (true) { + switch (c) { + case '\\': { + utf32 const tc = parse_escape_sequence(); + if (tc >= 0x100) { + warningf(WARN_OTHER, &lexer_pos, "escape sequence out of range"); } - had_star = 0; + obstack_1grow(&symbol_obstack, tc); break; + } - case '\\': - next_char(); - EAT_NEWLINE(break;) - had_star = 0; + MATCH_NEWLINE( + parse_error("newline while parsing character constant"); break; + ) - case '?': - next_char(); - if(c != '?') { - had_star = 0; - break; - } + case '\'': next_char(); - if(replace_trigraph()) - break; - put_back(c); - c = '?'; - had_star = 0; - /* we don't put back the 2nd ? as the comment text is discarded - * anyway */ - break; - - MATCH_NEWLINE(had_star = 0; break;) + goto end_of_char_constant; - case EOF: - error_prefix_at(lexer_token.source_position.input_name, - start_linenr); - fprintf(stderr, "at end of file while looking for comment end\n"); + case EOF: { + errorf(&lexer_token.base.source_position, + "EOF while parsing character constant"); + lexer_token.kind = T_ERROR; return; + } + default: - had_star = 0; + obstack_grow_symbol(&symbol_obstack, c); next_char(); break; + } } + +end_of_char_constant:; + obstack_1grow(&symbol_obstack, '\0'); + const size_t size = (size_t)obstack_object_size(&symbol_obstack)-1; + char *const string = obstack_finish(&symbol_obstack); + + lexer_token.kind = T_CHARACTER_CONSTANT; + lexer_token.string.string = identify_string(string, size); + + if (size == 0) { + errorf(&lexer_token.base.source_position, "empty character constant"); + } } -static -void skip_line_comment(void) +/** + * Skip a multiline comment. + */ +static void skip_multiline_comment(void) { - while(1) { - switch(c) { - case '?': - next_char(); - if(c != '?') - break; + while (true) { + switch (c) { + case '/': next_char(); - if(replace_trigraph()) - break; - put_back('?'); - /* we don't put back the 2nd ? as the comment text is discarded - * anyway */ + if (c == '*') { + /* nested comment, warn here */ + warningf(WARN_COMMENT, &lexer_pos, "'/*' within comment"); + } break; - - case '\\': + case '*': next_char(); - if(c == '\n') { + if (c == '/') { next_char(); - lexer_token.source_position.linenr++; + return; } break; + MATCH_NEWLINE(break;) + + case EOF: { + errorf(&lexer_token.base.source_position, + "at end of file while looking for comment end"); + return; + } + + default: + next_char(); + break; + } + } +} + +/** + * Skip a single line comment. + */ +static void skip_line_comment(void) +{ + while (true) { + switch (c) { case EOF: - case '\r': + return; + case '\n': + case '\r': return; + case '\\': + next_char(); + if (c == '\n' || c == '\r') { + warningf(WARN_COMMENT, &lexer_pos, "multi-line comment"); + return; + } + break; + default: next_char(); break; @@ -626,124 +862,194 @@ void skip_line_comment(void) } } +/** The current preprocessor token. */ static token_t pp_token; -static inline -void next_pp_token(void) +/** + * Read the next preprocessor token. + */ +static inline void next_pp_token(void) { lexer_next_preprocessing_token(); pp_token = lexer_token; } -static -void eat_until_newline(void) +/** + * Eat all preprocessor tokens until newline. + */ +static void eat_until_newline(void) { - while(pp_token.type != '\n' && pp_token.type != T_EOF) { + while (pp_token.kind != '\n' && pp_token.kind != T_EOF) { next_pp_token(); } } -static -void error_directive(void) +/** + * Parse the line directive. + */ +static void parse_line_directive(void) { - error_prefix(); - fprintf(stderr, "#error directive: \n"); + if (pp_token.kind != T_INTEGER) { + parse_error("expected integer"); + } else { + /* use offset -1 as this is about the next line */ + lexer_pos.lineno = atoi(pp_token.number.number.begin) - 1; + next_pp_token(); + } + if (pp_token.kind == T_STRING_LITERAL) { + lexer_pos.input_name = pp_token.string.string.begin; + lexer_pos.is_system_header = false; + next_pp_token(); + + /* attempt to parse numeric flags as outputted by gcc preprocessor */ + while (pp_token.kind == T_INTEGER) { + /* flags: + * 1 - indicates start of a new file + * 2 - indicates return from a file + * 3 - indicates system header + * 4 - indicates implicit extern "C" in C++ mode + * + * currently we're only interested in "3" + */ + if (streq(pp_token.number.number.begin, "3")) { + lexer_pos.is_system_header = true; + } + next_pp_token(); + } + } - /* parse pp-tokens until new-line */ + eat_until_newline(); } -static -void define_directive(void) +/** + * STDC pragmas. + */ +typedef enum stdc_pragma_kind_t { + STDC_UNKNOWN, + STDC_FP_CONTRACT, + STDC_FENV_ACCESS, + STDC_CX_LIMITED_RANGE +} stdc_pragma_kind_t; + +/** + * STDC pragma values. + */ +typedef enum stdc_pragma_value_kind_t { + STDC_VALUE_UNKNOWN, + STDC_VALUE_ON, + STDC_VALUE_OFF, + STDC_VALUE_DEFAULT +} stdc_pragma_value_kind_t; + +/** + * Parse a pragma directive. + */ +static void parse_pragma(void) { - lexer_next_preprocessing_token(); - if(lexer_token.type != T_IDENTIFIER) { - parse_error("expected identifier after #define\n"); + bool unknown_pragma = true; + + next_pp_token(); + if (pp_token.kind != T_IDENTIFIER) { + warningf(WARN_UNKNOWN_PRAGMAS, &pp_token.base.source_position, + "expected identifier after #pragma"); eat_until_newline(); + return; } -} - -static -void ifdef_directive(int is_ifndef) -{ - (void) is_ifndef; - lexer_next_preprocessing_token(); - //expect_identifier(); - //extect_newline(); -} -static -void endif_directive(void) -{ - //expect_newline(); -} + symbol_t *symbol = pp_token.identifier.symbol; + if (symbol->pp_ID == TP_STDC) { + stdc_pragma_kind_t kind = STDC_UNKNOWN; + /* a STDC pragma */ + if (c_mode & _C99) { + next_pp_token(); -static -void parse_line_directive(void) -{ - if(pp_token.type != T_INTEGER) { - parse_error("expected integer"); + switch (pp_token.identifier.symbol->pp_ID) { + case TP_FP_CONTRACT: + kind = STDC_FP_CONTRACT; + break; + case TP_FENV_ACCESS: + kind = STDC_FENV_ACCESS; + break; + case TP_CX_LIMITED_RANGE: + kind = STDC_CX_LIMITED_RANGE; + break; + default: + break; + } + if (kind != STDC_UNKNOWN) { + stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN; + next_pp_token(); + switch (pp_token.identifier.symbol->pp_ID) { + case TP_ON: + value = STDC_VALUE_ON; + break; + case TP_OFF: + value = STDC_VALUE_OFF; + break; + case TP_DEFAULT: + value = STDC_VALUE_DEFAULT; + break; + default: + break; + } + if (value != STDC_VALUE_UNKNOWN) { + unknown_pragma = false; + } else { + errorf(&pp_token.base.source_position, + "bad STDC pragma argument"); + } + } + } } else { - lexer_token.source_position.linenr = pp_token.v.intvalue - 1; - next_pp_token(); + unknown_pragma = true; } - if(pp_token.type == T_STRING_LITERAL) { - lexer_token.source_position.input_name = pp_token.v.string; - next_pp_token(); - } - eat_until_newline(); + if (unknown_pragma) { + warningf(WARN_UNKNOWN_PRAGMAS, &pp_token.base.source_position, + "encountered unknown #pragma"); + } } -static -void parse_preprocessor_identifier(void) +/** + * Parse a preprocessor non-null directive. + */ +static void parse_preprocessor_identifier(void) { - assert(pp_token.type == T_IDENTIFIER); - symbol_t *symbol = pp_token.v.symbol; + assert(pp_token.kind == T_IDENTIFIER); + symbol_t *symbol = pp_token.identifier.symbol; - switch(symbol->pp_ID) { - case TP_include: - printf("include - enable header name parsing!\n"); - break; - case TP_define: - define_directive(); - break; - case TP_ifdef: - ifdef_directive(0); - break; - case TP_ifndef: - ifdef_directive(1); - break; - case TP_endif: - endif_directive(); - break; + switch (symbol->pp_ID) { case TP_line: next_pp_token(); parse_line_directive(); break; - case TP_if: - case TP_else: - case TP_elif: - case TP_undef: - case TP_error: - error_directive(); - break; case TP_pragma: + parse_pragma(); + break; + case TP_error: + /* TODO; output the rest of the line */ + parse_error("#error directive"); break; } } -static -void parse_preprocessor_directive() +/** + * Parse a preprocessor directive. + */ +static void parse_preprocessor_directive(void) { next_pp_token(); - switch(pp_token.type) { + switch (pp_token.kind) { case T_IDENTIFIER: parse_preprocessor_identifier(); break; case T_INTEGER: parse_line_directive(); break; + case '\n': + /* NULL directive, see §6.10.7 */ + break; default: parse_error("invalid preprocessor directive"); eat_until_newline(); @@ -753,52 +1059,62 @@ void parse_preprocessor_directive() #define MAYBE_PROLOG \ next_char(); \ - while(1) { \ - switch(c) { + while (true) { \ + switch (c) { #define MAYBE(ch, set_type) \ case ch: \ next_char(); \ - lexer_token.type = set_type; \ + lexer_token.kind = set_type; \ return; +/* must use this as last thing */ +#define MAYBE_MODE(ch, set_type, mode) \ + case ch: \ + if (c_mode & mode) { \ + next_char(); \ + lexer_token.kind = set_type; \ + return; \ + } \ + /* fallthrough */ + #define ELSE_CODE(code) \ - SKIP_TRIGRAPHS(, \ - code; \ - ) \ - \ - case '\\': \ - next_char(); \ - EAT_NEWLINE(break;) \ - /* fallthrough */ \ default: \ - code; \ + code \ + return; \ } \ - } /* end of while(1) */ \ - break; + } /* end of while (true) */ \ #define ELSE(set_type) \ ELSE_CODE( \ - lexer_token.type = set_type; \ - return; \ + lexer_token.kind = set_type; \ ) void lexer_next_preprocessing_token(void) { - while(1) { - switch(c) { + while (true) { + lexer_token.base.source_position = lexer_pos; + + switch (c) { case ' ': case '\t': next_char(); break; MATCH_NEWLINE( - lexer_token.type = '\n'; + lexer_token.kind = '\n'; return; ) SYMBOL_CHARS parse_symbol(); + /* might be a wide string ( L"string" ) */ + if (lexer_token.identifier.symbol == symbol_L) { + switch (c) { + case '"': parse_wide_string_literal(); break; + case '\'': parse_wide_character_constant(); break; + } + } return; DIGITS @@ -813,28 +1129,21 @@ void lexer_next_preprocessing_token(void) parse_character_constant(); return; - case '\\': - next_char(); - if(c == '\n') { - next_char(); - lexer_token.source_position.linenr++; - break; - } else { - parse_error("unexpected '\\' found"); - lexer_token.type = T_ERROR; - } - return; - case '.': MAYBE_PROLOG + DIGITS + put_back(c); + c = '.'; + parse_number(); + return; + case '.': MAYBE_PROLOG MAYBE('.', T_DOTDOTDOT) ELSE_CODE( put_back(c); c = '.'; - lexer_token.type = '.'; - return; + lexer_token.kind = '.'; ) ELSE('.') case '&': @@ -853,6 +1162,7 @@ void lexer_next_preprocessing_token(void) ELSE('+') case '-': MAYBE_PROLOG + MAYBE('>', T_MINUSGREATER) MAYBE('-', T_MINUSMINUS) MAYBE('=', T_MINUSEQUAL) ELSE('-') @@ -876,25 +1186,25 @@ void lexer_next_preprocessing_token(void) ELSE('/') case '%': MAYBE_PROLOG - MAYBE('>', T_PERCENTGREATER) + MAYBE('>', '}') MAYBE('=', T_PERCENTEQUAL) case ':': MAYBE_PROLOG case '%': MAYBE_PROLOG - MAYBE(':', T_PERCENTCOLONPERCENTCOLON) + MAYBE(':', T_HASHHASH) ELSE_CODE( put_back(c); c = '%'; - lexer_token.type = T_PERCENTCOLON; - return; + lexer_token.kind = '#'; ) - ELSE(T_PERCENTCOLON) + ELSE('#') ELSE('%') case '<': MAYBE_PROLOG - MAYBE(':', T_LESSCOLON) - MAYBE('%', T_LESSPERCENT) + MAYBE(':', '[') + MAYBE('%', '{') + MAYBE('=', T_LESSEQUAL) case '<': MAYBE_PROLOG MAYBE('=', T_LESSLESSEQUAL) @@ -902,6 +1212,7 @@ void lexer_next_preprocessing_token(void) ELSE('<') case '>': MAYBE_PROLOG + MAYBE('=', T_GREATEREQUAL) case '>': MAYBE_PROLOG MAYBE('=', T_GREATERGREATEREQUAL) @@ -918,7 +1229,8 @@ void lexer_next_preprocessing_token(void) ELSE('|') case ':': MAYBE_PROLOG - MAYBE('>', T_COLONGREATER) + MAYBE('>', ']') + MAYBE_MODE(':', T_COLONCOLON, _CXX) ELSE(':') case '=': MAYBE_PROLOG @@ -930,22 +1242,6 @@ void lexer_next_preprocessing_token(void) ELSE('#') case '?': - next_char(); - /* just a simple ? */ - if(c != '?') { - lexer_token.type = '?'; - return; - } - /* might be a trigraph */ - next_char(); - if(replace_trigraph()) { - break; - } - put_back(c); - c = '?'; - lexer_token.type = '?'; - return; - case '[': case ']': case '(': @@ -955,19 +1251,20 @@ void lexer_next_preprocessing_token(void) case '~': case ';': case ',': - lexer_token.type = c; + case '\\': + lexer_token.kind = c; next_char(); return; case EOF: - lexer_token.type = T_EOF; + lexer_token.kind = T_EOF; return; default: +dollar_sign: + errorf(&lexer_pos, "unknown character '%c' found", c); next_char(); - error_prefix(); - fprintf(stderr, "unknown character '%c' found\n", c); - lexer_token.type = T_ERROR; + lexer_token.kind = T_ERROR; return; } } @@ -976,15 +1273,13 @@ void lexer_next_preprocessing_token(void) void lexer_next_token(void) { lexer_next_preprocessing_token(); - if(lexer_token.type != '\n') - return; + while (lexer_token.kind == '\n') { newline_found: - do { lexer_next_preprocessing_token(); - } while(lexer_token.type == '\n'); + } - if(lexer_token.type == '#') { + if (lexer_token.kind == '#') { parse_preprocessor_directive(); goto newline_found; } @@ -993,16 +1288,30 @@ newline_found: void init_lexer(void) { strset_init(&stringset); + symbol_L = symbol_table_insert("L"); } -void lexer_open_stream(FILE *stream, const char *input_name) +static void input_error(unsigned delta_lines, unsigned delta_cols, + const char *message) { - input = stream; - lexer_token.source_position.linenr = 0; - lexer_token.source_position.input_name = input_name; + lexer_pos.lineno += delta_lines; + lexer_pos.colno += delta_cols; + errorf(&lexer_pos, "%s", message); +} + +void lexer_switch_input(input_t *new_input, const char *input_name) +{ + lexer_pos.lineno = 0; + lexer_pos.colno = 0; + lexer_pos.input_name = input_name; + + set_input_error_callback(input_error); + input = new_input; + bufpos = NULL; + bufend = NULL; - /* we place a virtual '\n' at the beginning so the lexer knows we're at the - * beginning of a line */ + /* place a virtual \n at the beginning so the lexer knows that we're + * at the beginning of a line */ c = '\n'; } @@ -1014,7 +1323,7 @@ void exit_lexer(void) static __attribute__((unused)) void dbg_pos(const source_position_t source_position) { - fprintf(stdout, "%s:%d\n", source_position.input_name, - source_position.linenr); + fprintf(stdout, "%s:%u:%u\n", source_position.input_name, + source_position.lineno, (unsigned)source_position.colno); fflush(stdout); }