X-Git-Url: http://nsz.repo.hu/git/?a=blobdiff_plain;ds=sidebyside;f=lexer.c;h=20993600129097ca06d55f4c1c57611c591ddec8;hb=9de06381987e3b71a9efea26cb9b596fd433191c;hp=406bc7a984c7883f39fde4446e839b840150feec;hpb=b4e35bf2387e9b22ae6a97bad28daa8a438483ed;p=cparser diff --git a/lexer.c b/lexer.c index 406bc7a..2099360 100644 --- a/lexer.c +++ b/lexer.c @@ -1,49 +1,74 @@ #include -#include "lexer_t.h" +#include "lexer.h" #include "token_t.h" #include "symbol_table_t.h" #include "adt/error.h" +#include "adt/strset.h" +#include "adt/util.h" +#include "type_t.h" #include #include #include +#include #include //#define DEBUG_CHARS #define MAX_PUTBACK 3 -static int c; -source_position_t source_position; -static FILE *input; -static char buf[1027]; -static const char *bufend; -static const char *bufpos; -static strset_t stringset; -//static FILE **input_stack; -//static char **buf_stack; +#ifdef _WIN32 +/* No strtold on windows and no replacement yet */ +#define strtold(s, e) strtod(s, e) +#endif + +#define HAS_SIGNED_CHAR +//#define HAS_UNSIGNED_CHAR + +#if defined HAS_SIGNED_CHAR +typedef signed char char_type; +#elif defined HAS_UNSIGNED_CHAR +typedef unsigned char char_type; +#else +# error signedness of char not determined +#endif -static -void error_prefix_at(const char *input_name, unsigned linenr) +static int c; +token_t lexer_token; +symbol_t *symbol_L; +static FILE *input; +static char buf[1024 + MAX_PUTBACK]; +static const char *bufend; +static const char *bufpos; +static strset_t stringset; + +static type_t *type_int = NULL; +static type_t *type_uint = NULL; +static type_t *type_ulong = NULL; +static type_t *type_longlong = NULL; +static type_t *type_ulonglong = NULL; +static type_t *type_float = NULL; +static type_t *type_double = NULL; +static type_t *type_longdouble = NULL; + +static void error_prefix_at(const char *input_name, unsigned linenr) { - fprintf(stderr, "%s:%d: Error: ", input_name, linenr); + fprintf(stderr, "%s:%u: Error: ", input_name, linenr); } -static -void error_prefix() +static void error_prefix(void) { - error_prefix_at(source_position.input_name, source_position.linenr); + error_prefix_at(lexer_token.source_position.input_name, + lexer_token.source_position.linenr); } -static -void parse_error(const char *msg) +static void parse_error(const char *msg) { error_prefix(); fprintf(stderr, "%s\n", msg); } -static inline -void next_char() +static inline void next_real_char(void) { bufpos++; if(bufpos >= bufend) { @@ -57,311 +82,609 @@ void next_char() bufend = buf + MAX_PUTBACK + s; } c = *(bufpos); -#ifdef DEBUG_CHARS - printf("nchar '%c'\n", c); -#endif } -static inline -void put_back(int pc) +static inline void put_back(int pc) { - char *p = (char*) bufpos - 1; + assert(bufpos >= buf); + //assert(bufpos < buf+MAX_PUTBACK || *bufpos == pc); + + char *p = buf + (bufpos - buf); + *p = (char) pc; + + /* going backwards in the buffer is legal as long as it's not more often + * than MAX_PUTBACK */ bufpos--; - assert(p >= buf); - *p = pc; #ifdef DEBUG_CHARS printf("putback '%c'\n", pc); #endif } +static inline void next_char(void); + +#define MATCH_NEWLINE(code) \ + case '\r': \ + next_char(); \ + if(c == '\n') { \ + next_char(); \ + } \ + lexer_token.source_position.linenr++; \ + code; \ + case '\n': \ + next_char(); \ + lexer_token.source_position.linenr++; \ + code; + +#define eat(c_type) do { assert(c == c_type); next_char(); } while(0) + +static void maybe_concat_lines(void) +{ + eat('\\'); + + switch(c) { + MATCH_NEWLINE(return;) + + default: + break; + } + + put_back(c); + c = '\\'; +} -static -int replace_trigraph(void) +static inline void next_char(void) { -#define MATCH_TRIGRAPH(ch,replacement) \ - case ch: \ - c = replacement; \ - return 1; + next_real_char(); + /* filter trigraphs */ + if(UNLIKELY(c == '\\')) { + maybe_concat_lines(); + goto end_of_next_char; + } + + if(LIKELY(c != '?')) + goto end_of_next_char; + + next_real_char(); + if(LIKELY(c != '?')) { + put_back(c); + c = '?'; + goto end_of_next_char; + } + + next_real_char(); switch(c) { - MATCH_TRIGRAPH('=', '#') - MATCH_TRIGRAPH('(', '[') - MATCH_TRIGRAPH('/', '\\') - MATCH_TRIGRAPH(')', ']') - MATCH_TRIGRAPH('\'', '^') - MATCH_TRIGRAPH('<', '{') - MATCH_TRIGRAPH('!', '|') - MATCH_TRIGRAPH('>', '}') - MATCH_TRIGRAPH('-', '~') + case '=': c = '#'; break; + case '(': c = '['; break; + case '/': c = '\\'; maybe_concat_lines(); break; + case ')': c = ']'; break; + case '\'': c = '^'; break; + case '<': c = '{'; break; + case '!': c = '|'; break; + case '>': c = '}'; break; + case '-': c = '~'; break; default: + put_back('?'); + put_back(c); + c = '?'; break; } - return 0; -} - -#define SKIP_TRIGRAPHS(custom_putback, no_trigraph_code) \ - case '?': \ - next_char(); \ - if(c != '?') { \ - custom_putback; \ - put_back(c); \ - c = '?'; \ - no_trigraph_code; \ - } \ - next_char(); \ - if(replace_trigraph()) { \ - break; \ - } \ - custom_putback; \ - put_back('?'); \ - put_back(c); \ - c = '?'; \ - no_trigraph_code; - -#define EAT_NEWLINE(newline_code) \ - if(c == '\r') { \ - next_char(); \ - if(c == '\n') \ - next_char(); \ - source_position.linenr++; \ - newline_code; \ - } else if(c == '\n') { \ - next_char(); \ - source_position.linenr++; \ - newline_code; \ - } - -static -void parse_symbol(token_t *token) +end_of_next_char:; +#ifdef DEBUG_CHARS + printf("nchar '%c'\n", c); +#endif +} + +#define SYMBOL_CHARS \ + case 'a': \ + case 'b': \ + case 'c': \ + case 'd': \ + case 'e': \ + case 'f': \ + case 'g': \ + case 'h': \ + case 'i': \ + case 'j': \ + case 'k': \ + case 'l': \ + case 'm': \ + case 'n': \ + case 'o': \ + case 'p': \ + case 'q': \ + case 'r': \ + case 's': \ + case 't': \ + case 'u': \ + case 'v': \ + case 'w': \ + case 'x': \ + case 'y': \ + case 'z': \ + case 'A': \ + case 'B': \ + case 'C': \ + case 'D': \ + case 'E': \ + case 'F': \ + case 'G': \ + case 'H': \ + case 'I': \ + case 'J': \ + case 'K': \ + case 'L': \ + case 'M': \ + case 'N': \ + case 'O': \ + case 'P': \ + case 'Q': \ + case 'R': \ + case 'S': \ + case 'T': \ + case 'U': \ + case 'V': \ + case 'W': \ + case 'X': \ + case 'Y': \ + case 'Z': \ + case '_': + +#define DIGITS \ + case '0': \ + case '1': \ + case '2': \ + case '3': \ + case '4': \ + case '5': \ + case '6': \ + case '7': \ + case '8': \ + case '9': + +static void parse_symbol(void) { symbol_t *symbol; char *string; - obstack_1grow(&symbol_obstack, c); + obstack_1grow(&symbol_obstack, (char) c); next_char(); while(1) { switch(c) { - case '\\': - next_char(); - EAT_NEWLINE(break;) - goto end_symbol; - - case 'A' ... 'Z': - case 'a' ... 'z': - case '_': - obstack_1grow(&symbol_obstack, c); + DIGITS + SYMBOL_CHARS + obstack_1grow(&symbol_obstack, (char) c); next_char(); break; - case '?': - next_char(); - if(c != '?') { - put_back(c); - c = '?'; - goto end_symbol; - } - next_char(); - if(replace_trigraph()) - break; - put_back('?'); - put_back(c); - c = '?'; - goto end_symbol; - default: goto end_symbol; } } + end_symbol: obstack_1grow(&symbol_obstack, '\0'); string = obstack_finish(&symbol_obstack); symbol = symbol_table_insert(string); - if(symbol->ID > 0) { - token->type = symbol->ID; - } else { - token->type = T_IDENTIFIER; - } - token->v.symbol = symbol; + lexer_token.type = symbol->ID; + lexer_token.v.symbol = symbol; if(symbol->string != string) { obstack_free(&symbol_obstack, string); } } -static -void parse_number_hex(token_t *token) +static void parse_integer_suffix(void) +{ + if(c == 'U' || c == 'u') { + next_char(); + if(c == 'L' || c == 'l') { + next_char(); + if(c == 'L' || c == 'l') { + next_char(); + lexer_token.datatype = type_ulonglong; + } else { + lexer_token.datatype = type_ulong; + } + } else { + lexer_token.datatype = type_uint; + } + } else if(c == 'l' || c == 'L') { + next_char(); + if(c == 'l' || c == 'L') { + next_char(); + if(c == 'u' || c == 'U') { + next_char(); + lexer_token.datatype = type_ulonglong; + } else { + lexer_token.datatype = type_longlong; + } + } else if(c == 'u' || c == 'U') { + next_char(); + lexer_token.datatype = type_ulong; + } else { + lexer_token.datatype = type_int; + } + } else { + lexer_token.datatype = type_int; + } +} + +static void parse_floating_suffix(void) +{ + switch(c) { + /* TODO: do something usefull with the suffixes... */ + case 'f': + case 'F': + next_char(); + lexer_token.datatype = type_float; + break; + case 'l': + case 'L': + next_char(); + lexer_token.datatype = type_longdouble; + break; + default: + lexer_token.datatype = type_double; + break; + } +} + +/** + * A replacement for strtoull. Only those parts needed for + * our parser are implemented. + */ +static unsigned long long parse_int_string(const char *s, const char **endptr, int base) { + unsigned long long v = 0; + + switch (base) { + case 16: + for (;; ++s) { + /* check for overrun */ + if (v >= 0x1000000000000000ULL) + break; + switch (tolower(*s)) { + case '0': v <<= 4; break; + case '1': v <<= 4; v |= 0x1; break; + case '2': v <<= 4; v |= 0x2; break; + case '3': v <<= 4; v |= 0x3; break; + case '4': v <<= 4; v |= 0x4; break; + case '5': v <<= 4; v |= 0x5; break; + case '6': v <<= 4; v |= 0x6; break; + case '7': v <<= 4; v |= 0x7; break; + case '8': v <<= 4; v |= 0x8; break; + case '9': v <<= 4; v |= 0x9; break; + case 'a': v <<= 4; v |= 0xa; break; + case 'b': v <<= 4; v |= 0xb; break; + case 'c': v <<= 4; v |= 0xc; break; + case 'd': v <<= 4; v |= 0xd; break; + case 'e': v <<= 4; v |= 0xe; break; + case 'f': v <<= 4; v |= 0xf; break; + default: + goto end; + } + } + break; + case 8: + for (;; ++s) { + /* check for overrun */ + if (v >= 0x2000000000000000ULL) + break; + switch (tolower(*s)) { + case '0': v <<= 3; break; + case '1': v <<= 3; v |= 1; break; + case '2': v <<= 3; v |= 2; break; + case '3': v <<= 3; v |= 3; break; + case '4': v <<= 3; v |= 4; break; + case '5': v <<= 3; v |= 5; break; + case '6': v <<= 3; v |= 6; break; + case '7': v <<= 3; v |= 7; break; + default: + goto end; + } + } + break; + case 10: + for (;; ++s) { + /* check for overrun */ + if (v > 0x1999999999999999ULL) + break; + switch (tolower(*s)) { + case '0': v *= 10; break; + case '1': v *= 10; v += 1; break; + case '2': v *= 10; v += 2; break; + case '3': v *= 10; v += 3; break; + case '4': v *= 10; v += 4; break; + case '5': v *= 10; v += 5; break; + case '6': v *= 10; v += 6; break; + case '7': v *= 10; v += 7; break; + case '8': v *= 10; v += 8; break; + case '9': v *= 10; v += 9; break; + default: + goto end; + } + } + break; + default: + assert(0); + break; + } +end: + *endptr = s; + return v; +} + +static void parse_number_hex(void) { assert(c == 'x' || c == 'X'); next_char(); - if (!isdigit(c) && - !('A' <= c && c <= 'F') && - !('a' <= c && c <= 'f')) { - parse_error("premature end of hex number literal"); - token->type = T_ERROR; - return; + while(isxdigit(c)) { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); } + obstack_1grow(&symbol_obstack, '\0'); + char *string = obstack_finish(&symbol_obstack); - int value = 0; - for(;;) { - if (isdigit(c)) { - value = 16 * value + c - '0'; - } else if ('A' <= c && c <= 'F') { - value = 16 * value + c - 'A' + 10; - } else if ('a' <= c && c <= 'f') { - value = 16 * value + c - 'a' + 10; - } else { - token->type = T_INTEGER; - token->v.intvalue = value; - return; - } + if(c == '.' || c == 'p' || c == 'P') { next_char(); + panic("Hex floating point numbers not implemented yet"); + } + if(*string == '\0') { + parse_error("invalid hex number"); + lexer_token.type = T_ERROR; + } + + const char *endptr; + lexer_token.type = T_INTEGER; + lexer_token.v.intvalue = parse_int_string(string, &endptr, 16); + if(*endptr != '\0') { + parse_error("hex number literal too long"); } + + obstack_free(&symbol_obstack, string); + parse_integer_suffix(); } -static -void parse_number_oct(token_t *token) +static inline bool is_octal_digit(int chr) { - assert(c == 'o' || c == 'O'); - next_char(); + return '0' <= chr && chr <= '7'; +} - int value = 0; - for(;;) { - if ('0' <= c && c <= '7') { - value = 8 * value + c - '0'; - } else { - token->type = T_INTEGER; - token->v.intvalue = value; - return; - } +static void parse_number_oct(void) +{ + while(is_octal_digit(c)) { + obstack_1grow(&symbol_obstack, (char) c); next_char(); } + obstack_1grow(&symbol_obstack, '\0'); + char *string = obstack_finish(&symbol_obstack); + + const char *endptr; + lexer_token.type = T_INTEGER; + lexer_token.v.intvalue = parse_int_string(string, &endptr, 8); + if(*endptr != '\0') { + parse_error("octal number literal too long"); + } + + obstack_free(&symbol_obstack, string); + parse_integer_suffix(); } -static -void parse_number_dec(token_t *token, int first_char) +static void parse_number_dec(void) { - int value = 0; - if(first_char > 0) { - assert(first_char >= '0' && first_char <= '9'); - value = first_char - '0'; + bool is_float = false; + while(isdigit(c)) { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); } - for(;;) { - if (isdigit(c)) { - value = 10 * value + c - '0'; - } else { - token->type = T_INTEGER; - token->v.intvalue = value; - return; + if(c == '.') { + obstack_1grow(&symbol_obstack, '.'); + next_char(); + + while(isdigit(c)) { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); } + is_float = true; + } + if(c == 'e' || c == 'E') { + obstack_1grow(&symbol_obstack, 'e'); next_char(); + + if(c == '-' || c == '+') { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + + while(isdigit(c)) { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + is_float = true; } + + obstack_1grow(&symbol_obstack, '\0'); + char *string = obstack_finish(&symbol_obstack); + + if(is_float) { + char *endptr; + lexer_token.type = T_FLOATINGPOINT; + lexer_token.v.floatvalue = strtold(string, &endptr); + + if(*endptr != '\0') { + parse_error("invalid number literal"); + } + + parse_floating_suffix(); + } else { + const char *endptr; + lexer_token.type = T_INTEGER; + lexer_token.v.intvalue = parse_int_string(string, &endptr, 10); + + if(*endptr != '\0') { + parse_error("invalid number literal"); + } + + parse_integer_suffix(); + } + obstack_free(&symbol_obstack, string); } -static -void parse_number(token_t *token) +static void parse_number(void) { - // TODO check for overflow - // TODO check for various invalid inputs sequences - if (c == '0') { next_char(); switch (c) { case 'X': - case 'x': parse_number_hex(token); break; - case 'o': - case 'O': parse_number_oct(token); break; - default: parse_number_dec(token, '0'); + case 'x': + parse_number_hex(); + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + parse_number_oct(); + break; + case '8': + case '9': + next_char(); + parse_error("invalid octal number"); + lexer_token.type = T_ERROR; + return; + case '.': + case 'e': + case 'E': + default: + obstack_1grow(&symbol_obstack, '0'); + parse_number_dec(); + return; } } else { - parse_number_dec(token, 0); + parse_number_dec(); } } -static -int parse_escape_sequence() +static int parse_octal_sequence(const int first_digit) +{ + assert(is_octal_digit(first_digit)); + int value = first_digit - '0'; + if (!is_octal_digit(c)) return value; + value = 8 * value + c - '0'; + next_char(); + if (!is_octal_digit(c)) return value; + value = 8 * value + c - '0'; + next_char(); + return (char_type)value; +} + +static int parse_hex_sequence(void) { + int value = 0; while(1) { - int ec = c; + if (c >= '0' && c <= '9') { + value = 16 * value + c - '0'; + } else if ('A' <= c && c <= 'F') { + value = 16 * value + c - 'A' + 10; + } else if ('a' <= c && c <= 'f') { + value = 16 * value + c - 'a' + 10; + } else { + break; + } next_char(); + } - switch(ec) { - case '"': return '"'; - case '\'': return'\''; - case '\\': - EAT_NEWLINE(break;) - return '\\'; - case 'a': return '\a'; - case 'b': return '\b'; - case 'f': return '\f'; - case 'n': return '\n'; - case 'r': return '\r'; - case 't': return '\t'; - case 'v': return '\v'; - case 'x': /* TODO parse hex number ... */ - parse_error("hex escape sequences not implemented yet"); - return EOF; - case 0 ... 8: /* TODO parse octal number ... */ - parse_error("octal escape sequences not implemented yet"); - return EOF; - case '?': - if(c != '?') { - return '?'; - } - /* might be a trigraph */ - next_char(); - if(replace_trigraph()) { - break; - } - put_back(c); - c = '?'; - return '?'; + return (char_type)value; +} - case EOF: - parse_error("reached end of file while parsing escape sequence"); - return EOF; - default: - parse_error("unknown escape sequence"); - return EOF; - } +static int parse_escape_sequence(void) +{ + eat('\\'); + + int ec = c; + next_char(); + + switch(ec) { + case '"': return '"'; + case '\'': return '\''; + case '\\': return '\\'; + case '?': return '\?'; + case 'a': return '\a'; + case 'b': return '\b'; + case 'f': return '\f'; + case 'n': return '\n'; + case 'r': return '\r'; + case 't': return '\t'; + case 'v': return '\v'; + case 'x': + return parse_hex_sequence(); + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + return parse_octal_sequence(ec); + case EOF: + parse_error("reached end of file while parsing escape sequence"); + return EOF; + default: + parse_error("unknown escape sequence"); + return EOF; + } +} + +const char *concat_strings(const char *s1, const char *s2) +{ + size_t len1 = strlen(s1); + size_t len2 = strlen(s2); + + char *concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1); + memcpy(concat, s1, len1); + memcpy(concat + len1, s2, len2 + 1); + + const char *result = strset_insert(&stringset, concat); + if(result != concat) { + obstack_free(&symbol_obstack, concat); } + + return result; } -static -void parse_string_literal(token_t *token) +static void parse_string_literal(void) { - unsigned start_linenr = source_position.linenr; + unsigned start_linenr = lexer_token.source_position.linenr; char *string; const char *result; assert(c == '"'); next_char(); + int tc; while(1) { switch(c) { - SKIP_TRIGRAPHS(, - obstack_1grow(&symbol_obstack, '?'); - next_char(); - break; - ) - case '\\': - next_char(); - EAT_NEWLINE(break;) - int ec = parse_escape_sequence(); - obstack_1grow(&symbol_obstack, ec); + tc = parse_escape_sequence(); + obstack_1grow(&symbol_obstack, (char) tc); break; case EOF: - error_prefix_at(source_position.input_name, start_linenr); + error_prefix_at(lexer_token.source_position.input_name, + start_linenr); fprintf(stderr, "string has no end\n"); - token->type = T_ERROR; + lexer_token.type = T_ERROR; return; case '"': @@ -369,7 +692,7 @@ void parse_string_literal(token_t *token) goto end_of_string; default: - obstack_1grow(&symbol_obstack, c); + obstack_1grow(&symbol_obstack, (char) c); next_char(); break; } @@ -389,41 +712,19 @@ end_of_string: obstack_free(&symbol_obstack, string); } - token->type = T_STRING_LITERAL; - token->v.string = result; + lexer_token.type = T_STRING_LITERAL; + lexer_token.v.string = result; } -#define MATCH_NEWLINE(code) \ - case '\r': \ - next_char(); \ - if(c == '\n') { \ - next_char(); \ - } \ - source_position.linenr++; \ - code; \ - case '\n': \ - next_char(); \ - source_position.linenr++; \ - code; - -static -void parse_character_constant(token_t *token) +static void parse_character_constant(void) { - assert(c == '\''); - next_char(); + eat('\''); int found_char = 0; while(1) { switch(c) { - SKIP_TRIGRAPHS(, - found_char = '?'; - break; - ) - case '\\': - next_char(); - EAT_NEWLINE(break;) - found_char = '\\'; + found_char = parse_escape_sequence(); break; MATCH_NEWLINE( @@ -437,7 +738,7 @@ void parse_character_constant(token_t *token) case EOF: parse_error("EOF while parsing character constant"); - token->type = T_ERROR; + lexer_token.type = T_ERROR; return; default: @@ -454,95 +755,48 @@ void parse_character_constant(token_t *token) } end_of_char_constant: - token->type = T_INTEGER; - token->v.intvalue = found_char; + lexer_token.type = T_INTEGER; + lexer_token.v.intvalue = found_char; } -static -void skip_multiline_comment(void) +static void skip_multiline_comment(void) { - unsigned start_linenr = source_position.linenr; - int had_star = 0; + unsigned start_linenr = lexer_token.source_position.linenr; while(1) { switch(c) { case '*': next_char(); - had_star = 1; - break; - - case '/': - next_char(); - if(had_star) { + if(c == '/') { + next_char(); return; } - had_star = 0; break; - case '\\': - next_char(); - EAT_NEWLINE(break;) - had_star = 0; - break; - - case '?': - next_char(); - if(c != '?') { - had_star = 0; - break; - } - next_char(); - if(replace_trigraph()) - break; - put_back(c); - c = '?'; - had_star = 0; - /* we don't put back the 2nd ? as the comment text is discarded - * anyway */ - break; - - MATCH_NEWLINE(had_star = 0; break;) + MATCH_NEWLINE(break;) case EOF: - error_prefix_at(source_position.input_name, start_linenr); + error_prefix_at(lexer_token.source_position.input_name, + start_linenr); fprintf(stderr, "at end of file while looking for comment end\n"); return; + default: - had_star = 0; next_char(); break; } } } -static -void skip_line_comment(void) +static void skip_line_comment(void) { while(1) { switch(c) { - case '?': - next_char(); - if(c != '?') - break; - next_char(); - if(replace_trigraph()) - break; - put_back('?'); - /* we don't put back the 2nd ? as the comment text is discarded - * anyway */ - break; - - case '\\': - next_char(); - if(c == '\n') { - next_char(); - source_position.linenr++; - } - break; - case EOF: - case '\r': + return; + case '\n': + case '\r': return; default: @@ -552,17 +806,22 @@ void skip_line_comment(void) } } -static -void lexer_next_preprocessing_token(token_t *token); +static token_t pp_token; -static -void eat_until_newline(void) +static inline void next_pp_token(void) { - /* TODO */ + lexer_next_preprocessing_token(); + pp_token = lexer_token; +} + +static void eat_until_newline(void) +{ + while(pp_token.type != '\n' && pp_token.type != T_EOF) { + next_pp_token(); + } } -static -void error_directive(void) +static void error_directive(void) { error_prefix(); fprintf(stderr, "#error directive: \n"); @@ -570,37 +829,49 @@ void error_directive(void) /* parse pp-tokens until new-line */ } -static -void define_directive(void) +static void define_directive(void) { - token_t temptoken; - - lexer_next_preprocessing_token(&temptoken); - if(temptoken.type != T_IDENTIFIER) { + lexer_next_preprocessing_token(); + if(lexer_token.type != T_IDENTIFIER) { parse_error("expected identifier after #define\n"); eat_until_newline(); } } -static -void ifdef_directive(int is_ifndef) +static void ifdef_directive(int is_ifndef) { (void) is_ifndef; - token_t temptoken; - lexer_next_preprocessing_token(&temptoken); + lexer_next_preprocessing_token(); //expect_identifier(); //extect_newline(); } -static -void endif_directive(void) +static void endif_directive(void) { //expect_newline(); } -static -void found_preprocessor_identifier(symbol_t *symbol) +static void parse_line_directive(void) +{ + if(pp_token.type != T_INTEGER) { + parse_error("expected integer"); + } else { + lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1); + next_pp_token(); + } + if(pp_token.type == T_STRING_LITERAL) { + lexer_token.source_position.input_name = pp_token.v.string; + next_pp_token(); + } + + eat_until_newline(); +} + +static void parse_preprocessor_identifier(void) { + assert(pp_token.type == T_IDENTIFIER); + symbol_t *symbol = pp_token.v.symbol; + switch(symbol->pp_ID) { case TP_include: printf("include - enable header name parsing!\n"); @@ -617,11 +888,14 @@ void found_preprocessor_identifier(symbol_t *symbol) case TP_endif: endif_directive(); break; + case TP_line: + next_pp_token(); + parse_line_directive(); + break; case TP_if: case TP_else: case TP_elif: case TP_undef: - case TP_line: case TP_error: error_directive(); break; @@ -630,16 +904,20 @@ void found_preprocessor_identifier(symbol_t *symbol) } } -static -void parse_preprocessor_directive(token_t *result_token) +static void parse_preprocessor_directive(void) { - token_t temptoken; + next_pp_token(); - (void) result_token; - lexer_next_preprocessing_token(&temptoken); - switch(temptoken.type) { + switch(pp_token.type) { case T_IDENTIFIER: - found_preprocessor_identifier(temptoken.v.symbol); + parse_preprocessor_identifier(); + break; + case T_INTEGER: + parse_line_directive(); + break; + default: + parse_error("invalid preprocessor directive"); + eat_until_newline(); break; } } @@ -652,18 +930,10 @@ void parse_preprocessor_directive(token_t *result_token) #define MAYBE(ch, set_type) \ case ch: \ next_char(); \ - token->type = set_type; \ + lexer_token.type = set_type; \ return; #define ELSE_CODE(code) \ - SKIP_TRIGRAPHS(, \ - code; \ - ) \ - \ - case '\\': \ - next_char(); \ - EAT_NEWLINE(break;) \ - /* fallthrough */ \ default: \ code; \ } \ @@ -672,80 +942,11 @@ void parse_preprocessor_directive(token_t *result_token) #define ELSE(set_type) \ ELSE_CODE( \ - token->type = set_type; \ + lexer_token.type = set_type; \ return; \ ) -static -void eat_whitespace() -{ - while(1) { - switch(c) { - case ' ': - case '\t': - next_char(); - break; - - case '\r': - case '\n': - return; - - case '\\': - next_char(); - if(c == '\n') { - next_char(); - source_position.linenr++; - break; - } - - put_back(c); - c = '\\'; - return; - - SKIP_TRIGRAPHS(, - return; - ) - - case '/': - next_char(); - while(1) { - switch(c) { - case '*': - next_char(); - skip_multiline_comment(); - eat_whitespace(); - return; - case '/': - next_char(); - skip_line_comment(); - eat_whitespace(); - return; - - SKIP_TRIGRAPHS( - put_back('?'); - , - c = '/'; - return; - ) - - case '\\': - next_char(); - EAT_NEWLINE(break;) - /* fallthrough */ - default: - return; - } - } - break; - - default: - return; - } - } -} - -static -void lexer_next_preprocessing_token(token_t *token) +void lexer_next_preprocessing_token(void) { while(1) { switch(c) { @@ -755,44 +956,30 @@ void lexer_next_preprocessing_token(token_t *token) break; MATCH_NEWLINE( - eat_whitespace(); - if(c == '#') { - next_char(); - parse_preprocessor_directive(token); - return; - } - token->type = '\n'; + lexer_token.type = '\n'; return; ) - case 'A' ... 'Z': - case 'a' ... 'z': - case '_': - parse_symbol(token); + SYMBOL_CHARS + parse_symbol(); + /* might be a wide string ( L"string" ) */ + if(c == '"' && (lexer_token.type == T_IDENTIFIER && + lexer_token.v.symbol == symbol_L)) { + parse_string_literal(); + return; + } return; - case '0' ... '9': - parse_number(token); + DIGITS + parse_number(); return; case '"': - parse_string_literal(token); + parse_string_literal(); return; case '\'': - parse_character_constant(token); - return; - - case '\\': - next_char(); - if(c == '\n') { - next_char(); - source_position.linenr++; - break; - } else { - parse_error("unexpected '\\' found"); - token->type = T_ERROR; - } + parse_character_constant(); return; case '.': @@ -803,7 +990,7 @@ void lexer_next_preprocessing_token(token_t *token) ELSE_CODE( put_back(c); c = '.'; - token->type = '.'; + lexer_token.type = '.'; return; ) ELSE('.') @@ -823,6 +1010,7 @@ void lexer_next_preprocessing_token(token_t *token) ELSE('+') case '-': MAYBE_PROLOG + MAYBE('>', T_MINUSGREATER) MAYBE('-', T_MINUSMINUS) MAYBE('=', T_MINUSEQUAL) ELSE('-') @@ -836,12 +1024,12 @@ void lexer_next_preprocessing_token(token_t *token) case '*': next_char(); skip_multiline_comment(); - lexer_next_preprocessing_token(token); + lexer_next_preprocessing_token(); return; case '/': next_char(); skip_line_comment(); - lexer_next_preprocessing_token(token); + lexer_next_preprocessing_token(); return; ELSE('/') case '%': @@ -856,7 +1044,7 @@ void lexer_next_preprocessing_token(token_t *token) ELSE_CODE( put_back(c); c = '%'; - token->type = T_PERCENTCOLON; + lexer_token.type = T_PERCENTCOLON; return; ) ELSE(T_PERCENTCOLON) @@ -865,6 +1053,7 @@ void lexer_next_preprocessing_token(token_t *token) MAYBE_PROLOG MAYBE(':', T_LESSCOLON) MAYBE('%', T_LESSPERCENT) + MAYBE('=', T_LESSEQUAL) case '<': MAYBE_PROLOG MAYBE('=', T_LESSLESSEQUAL) @@ -872,6 +1061,7 @@ void lexer_next_preprocessing_token(token_t *token) ELSE('<') case '>': MAYBE_PROLOG + MAYBE('=', T_GREATEREQUAL) case '>': MAYBE_PROLOG MAYBE('=', T_GREATERGREATEREQUAL) @@ -900,22 +1090,6 @@ void lexer_next_preprocessing_token(token_t *token) ELSE('#') case '?': - next_char(); - /* just a simple ? */ - if(c != '?') { - token->type = '?'; - return; - } - /* might be a trigraph */ - next_char(); - if(replace_trigraph()) { - break; - } - put_back(c); - c = '?'; - token->type = '?'; - return; - case '[': case ']': case '(': @@ -925,44 +1099,71 @@ void lexer_next_preprocessing_token(token_t *token) case '~': case ';': case ',': - token->type = c; + case '\\': + lexer_token.type = c; next_char(); return; case EOF: - token->type = T_EOF; + lexer_token.type = T_EOF; return; default: next_char(); error_prefix(); fprintf(stderr, "unknown character '%c' found\n", c); - token->type = T_ERROR; + lexer_token.type = T_ERROR; return; } } } -void lexer_next_token(token_t *token) +void lexer_next_token(void) { + lexer_next_preprocessing_token(); + if(lexer_token.type != '\n') + return; + +newline_found: do { - lexer_next_preprocessing_token(token); - } while(token->type == '\n'); + lexer_next_preprocessing_token(); + } while(lexer_token.type == '\n'); + + if(lexer_token.type == '#') { + parse_preprocessor_directive(); + goto newline_found; + } } void init_lexer(void) { strset_init(&stringset); + + type_int = make_atomic_type(ATOMIC_TYPE_INT, TYPE_QUALIFIER_CONST); + type_uint = make_atomic_type(ATOMIC_TYPE_UINT, TYPE_QUALIFIER_CONST); + type_ulong = make_atomic_type(ATOMIC_TYPE_ULONG, TYPE_QUALIFIER_CONST); + type_longlong = make_atomic_type(ATOMIC_TYPE_LONGLONG, + TYPE_QUALIFIER_CONST); + type_ulonglong = make_atomic_type(ATOMIC_TYPE_ULONGLONG, + TYPE_QUALIFIER_CONST); + + type_float = make_atomic_type(ATOMIC_TYPE_FLOAT, TYPE_QUALIFIER_CONST); + type_double = make_atomic_type(ATOMIC_TYPE_DOUBLE, + TYPE_QUALIFIER_CONST); + type_longdouble = make_atomic_type(ATOMIC_TYPE_LONG_DOUBLE, + TYPE_QUALIFIER_CONST); } void lexer_open_stream(FILE *stream, const char *input_name) { - input = stream; - source_position.linenr = 0; - source_position.input_name = input_name; + input = stream; + lexer_token.source_position.linenr = 0; + lexer_token.source_position.input_name = input_name; + + symbol_L = symbol_table_insert("L"); - /* we place a virtual '\n' at the beginning so the lexer knows we're at the - * beginning of a line */ + /* place a virtual \n at the beginning so the lexer knows that we're + * at the beginning of a line */ c = '\n'; } @@ -974,6 +1175,7 @@ void exit_lexer(void) static __attribute__((unused)) void dbg_pos(const source_position_t source_position) { - fprintf(stdout, "%s:%d\n", source_position.input_name, source_position.linenr); + fprintf(stdout, "%s:%u\n", source_position.input_name, + source_position.linenr); fflush(stdout); }