X-Git-Url: http://nsz.repo.hu/git/?a=blobdiff_plain;f=lexer.c;h=2a6cc2c6c164c7fd2e2bbcf799d50bcc368ac7f1;hb=c2d5fce07cb63445b4948ff7d66f5a590261483d;hp=7732c9dee4415c075bb216ad0f0496bc0d0239c5;hpb=bc2c469866488880921a9809d8dee3bd1333a25c;p=cparser diff --git a/lexer.c b/lexer.c index 7732c9d..2a6cc2c 100644 --- a/lexer.c +++ b/lexer.c @@ -5,6 +5,7 @@ #include "symbol_table_t.h" #include "adt/error.h" #include "adt/strset.h" +#include "adt/util.h" #include #include @@ -24,28 +25,24 @@ static strset_t stringset; //static FILE **input_stack; //static char **buf_stack; -static -void error_prefix_at(const char *input_name, unsigned linenr) +static void error_prefix_at(const char *input_name, unsigned linenr) { - fprintf(stderr, "%s:%d: Error: ", input_name, linenr); + fprintf(stderr, "%s:%u: Error: ", input_name, linenr); } -static -void error_prefix(void) +static void error_prefix(void) { error_prefix_at(lexer_token.source_position.input_name, lexer_token.source_position.linenr); } -static -void parse_error(const char *msg) +static void parse_error(const char *msg) { error_prefix(); fprintf(stderr, "%s\n", msg); } -static inline -void next_char(void) +static inline void next_real_char(void) { bufpos++; if(bufpos >= bufend) { @@ -59,13 +56,9 @@ void next_char(void) bufend = buf + MAX_PUTBACK + s; } c = *(bufpos); -#ifdef DEBUG_CHARS - printf("nchar '%c'\n", c); -#endif } -static inline -void put_back(int pc) +static inline void put_back(int pc) { char *p = (char*) bufpos - 1; bufpos--; @@ -77,64 +70,81 @@ void put_back(int pc) #endif } +static inline void next_char(void); -static -int replace_trigraph(void) -{ -#define MATCH_TRIGRAPH(ch,replacement) \ - case ch: \ - c = replacement; \ - return 1; +#define MATCH_NEWLINE(code) \ + case '\r': \ + next_char(); \ + if(c == '\n') { \ + next_char(); \ + } \ + lexer_token.source_position.linenr++; \ + code; \ + case '\n': \ + next_char(); \ + lexer_token.source_position.linenr++; \ + code; +static void maybe_concat_lines(void) +{ + next_char(); switch(c) { - MATCH_TRIGRAPH('=', '#') - MATCH_TRIGRAPH('(', '[') - MATCH_TRIGRAPH('/', '\\') - MATCH_TRIGRAPH(')', ']') - MATCH_TRIGRAPH('\'', '^') - MATCH_TRIGRAPH('<', '{') - MATCH_TRIGRAPH('!', '|') - MATCH_TRIGRAPH('>', '}') - MATCH_TRIGRAPH('-', '~') + MATCH_NEWLINE(return;) + default: break; } - return 0; + put_back(c); + c = '\\'; } -#define SKIP_TRIGRAPHS(custom_putback, no_trigraph_code) \ - case '?': \ - next_char(); \ - if(c != '?') { \ - custom_putback; \ - put_back(c); \ - c = '?'; \ - no_trigraph_code; \ - } \ - next_char(); \ - if(replace_trigraph()) { \ - break; \ - } \ - custom_putback; \ - put_back('?'); \ - put_back(c); \ - c = '?'; \ - no_trigraph_code; - -#define EAT_NEWLINE(newline_code) \ - if(c == '\r') { \ - next_char(); \ - if(c == '\n') \ - next_char(); \ - lexer_token.source_position.linenr++; \ - newline_code; \ - } else if(c == '\n') { \ - next_char(); \ - lexer_token.source_position.linenr++; \ - newline_code; \ +static inline void next_char(void) +{ + next_real_char(); + + /* filter trigraphs */ + if(UNLIKELY(c == '\\')) { + maybe_concat_lines(); + goto end_of_next_char; + } + + if(LIKELY(c != '?')) + goto end_of_next_char; + + next_real_char(); + if(LIKELY(c != '?')) { + put_back(c); + c = '?'; + goto end_of_next_char; } + next_real_char(); + switch(c) { + case '=': c = '#'; break; + case '(': c = '['; break; + case '/': c = '\\'; maybe_concat_lines(); break; + case ')': c = ']'; break; + case '\'': c = '^'; break; + case '<': c = '{'; break; + case '!': c = '|'; break; + case '>': c = '}'; break; + case '-': c = '~'; break; + default: + put_back('?'); + put_back(c); + c = '?'; + break; + } + +end_of_next_char: +#ifdef DEBUG_CHARS + printf("nchar '%c'\n", c); +#else + ; +#endif +} + #define SYMBOL_CHARS \ case 'a': \ case 'b': \ @@ -202,8 +212,7 @@ int replace_trigraph(void) case '8': \ case '9': -static -void parse_symbol(void) +static void parse_symbol(void) { symbol_t *symbol; char *string; @@ -213,36 +222,17 @@ void parse_symbol(void) while(1) { switch(c) { - case '\\': - next_char(); - EAT_NEWLINE(break;) - goto end_symbol; - DIGITS SYMBOL_CHARS obstack_1grow(&symbol_obstack, c); next_char(); break; - case '?': - next_char(); - if(c != '?') { - put_back(c); - c = '?'; - goto end_symbol; - } - next_char(); - if(replace_trigraph()) - break; - put_back('?'); - put_back(c); - c = '?'; - goto end_symbol; - default: goto end_symbol; } } + end_symbol: obstack_1grow(&symbol_obstack, '\0'); @@ -257,8 +247,31 @@ end_symbol: } } -static -void parse_number_hex(void) +static void parse_integer_suffix(void) +{ + if(c == 'U' || c == 'U') { + /* TODO do something with the suffixes... */ + next_char(); + if(c == 'L' || c == 'l') { + next_char(); + if(c == 'L' || c == 'l') { + next_char(); + } + } + } else if(c == 'l' || c == 'L') { + next_char(); + if(c == 'l' || c == 'L') { + next_char(); + if(c == 'u' || c == 'U') { + next_char(); + } + } else if(c == 'u' || c == 'U') { + next_char(); + } + } +} + +static void parse_number_hex(void) { assert(c == 'x' || c == 'X'); next_char(); @@ -272,7 +285,7 @@ void parse_number_hex(void) } int value = 0; - for(;;) { + while(1) { if (isdigit(c)) { value = 16 * value + c - '0'; } else if ('A' <= c && c <= 'F') { @@ -280,87 +293,203 @@ void parse_number_hex(void) } else if ('a' <= c && c <= 'f') { value = 16 * value + c - 'a' + 10; } else { - lexer_token.type = T_INTEGER; + parse_integer_suffix(); + + lexer_token.type = T_INTEGER; lexer_token.v.intvalue = value; return; } next_char(); } + + if(c == '.' || c == 'p' || c == 'P') { + next_char(); + panic("Hex floating point numbers not implemented yet"); + } } -static -void parse_number_oct(void) +static void parse_number_oct(void) { - assert(c == 'o' || c == 'O'); - next_char(); - int value = 0; - for(;;) { - if ('0' <= c && c <= '7') { - value = 8 * value + c - '0'; - } else { - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = value; - return; - } + while(c >= '0' && c <= '7') { + value = 8 * value + c - '0'; next_char(); } + if (c == '8' || c == '9') { + parse_error("invalid octal number"); + lexer_token.type = T_ERROR; + return; + } + + lexer_token.type = T_INTEGER; + lexer_token.v.intvalue = value; + + parse_integer_suffix(); } -static -void parse_number_dec(int first_char) +static void parse_floatingpoint_exponent(long double value) { - int value = 0; - if(first_char > 0) { - assert(first_char >= '0' && first_char <= '9'); - value = first_char - '0'; + unsigned int expo = 0; + long double factor = 10.; + + if(c == '-') { + next_char(); + factor = 0.1; + } else if(c == '+') { + next_char(); } - for(;;) { - if (isdigit(c)) { - value = 10 * value + c - '0'; - } else { - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = value; - return; - } + while(c >= '0' && c <= '9') { + expo = 10 * expo + (c - '0'); next_char(); } + + while(1) { + if(expo & 1) + value *= factor; + expo >>= 1; + if(expo == 0) + break; + factor *= factor; + } + + lexer_token.type = T_FLOATINGPOINT; + lexer_token.v.floatvalue = value; +} + +static void parse_floatingpoint_fract(int integer_part) +{ + long double value = integer_part; + long double factor = 1.; + + while(c >= '0' && c <= '9') { + factor *= 0.1; + value += (c - '0') * factor; + next_char(); + } + + if(c == 'e' || c == 'E') { + next_char(); + parse_floatingpoint_exponent(value); + return; + } + + lexer_token.type = T_FLOATINGPOINT; + lexer_token.v.floatvalue = value; } -static -void parse_number(void) +static void parse_number_dec(void) { - // TODO check for overflow - // TODO check for various invalid inputs sequences + int value = 0; + + while(isdigit(c)) { + value = 10 * value + c - '0'; + next_char(); + } + + if(c == '.') { + next_char(); + parse_floatingpoint_fract(value); + return; + } + if(c == 'e' || c == 'E') { + next_char(); + parse_floatingpoint_exponent(value); + return; + } + parse_integer_suffix(); + + lexer_token.type = T_INTEGER; + lexer_token.v.intvalue = value; +} +static void parse_number(void) +{ if (c == '0') { next_char(); switch (c) { case 'X': - case 'x': parse_number_hex(); break; - case 'o': - case 'O': parse_number_oct(); break; - default: parse_number_dec('0'); + case 'x': + parse_number_hex(); + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + parse_number_oct(); + break; + case '.': + next_char(); + parse_floatingpoint_fract(0); + break; + case 'e': + case 'E': + parse_floatingpoint_exponent(0); + break; + case '8': + case '9': + next_char(); + parse_error("invalid octal number"); + lexer_token.type = T_ERROR; + return; + default: + put_back(c); + c = '0'; + parse_number_dec(); + return; } } else { - parse_number_dec(0); + parse_number_dec(); } } -static -int parse_escape_sequence(void) +static int parse_octal_sequence(void) +{ + int value = 0; + while(1) { + if(c < '0' || c > '7') + break; + value = 8 * value + c - '0'; + next_char(); + } + + return value; +} + +static int parse_hex_sequence(void) +{ + int value = 0; + while(1) { + if (c >= '0' && c <= '9') { + value = 16 * value + c - '0'; + } else if ('A' <= c && c <= 'F') { + value = 16 * value + c - 'A' + 10; + } else if ('a' <= c && c <= 'f') { + value = 16 * value + c - 'a' + 10; + } else { + break; + } + next_char(); + } + + return value; +} + +static int parse_escape_sequence(void) { while(1) { int ec = c; next_char(); switch(ec) { - case '"': return '"'; + case '"': return '"'; case '\'': return'\''; - case '\\': - EAT_NEWLINE(break;) - return '\\'; + case '\\': return '\\'; + case '?': return '\?'; case 'a': return '\a'; case 'b': return '\b'; case 'f': return '\f'; @@ -368,9 +497,8 @@ int parse_escape_sequence(void) case 'r': return '\r'; case 't': return '\t'; case 'v': return '\v'; - case 'x': /* TODO parse hex number ... */ - parse_error("hex escape sequences not implemented yet"); - return EOF; + case 'x': + return parse_hex_sequence(); case '0': case '1': case '2': @@ -379,22 +507,7 @@ int parse_escape_sequence(void) case '5': case '6': case '7': - /* TODO parse octal number ... */ - parse_error("octal escape sequences not implemented yet"); - return EOF; - case '?': - if(c != '?') { - return '?'; - } - /* might be a trigraph */ - next_char(); - if(replace_trigraph()) { - break; - } - put_back(c); - c = '?'; - return '?'; - + return parse_octal_sequence(); case EOF: parse_error("reached end of file while parsing escape sequence"); return EOF; @@ -422,8 +535,7 @@ const char *concat_strings(const char *s1, const char *s2) return result; } -static -void parse_string_literal(void) +static void parse_string_literal(void) { unsigned start_linenr = lexer_token.source_position.linenr; char *string; @@ -434,15 +546,8 @@ void parse_string_literal(void) while(1) { switch(c) { - SKIP_TRIGRAPHS(, - obstack_1grow(&symbol_obstack, '?'); - next_char(); - break; - ) - case '\\': next_char(); - EAT_NEWLINE(break;) int ec = parse_escape_sequence(); obstack_1grow(&symbol_obstack, ec); break; @@ -483,21 +588,7 @@ end_of_string: lexer_token.v.string = result; } -#define MATCH_NEWLINE(code) \ - case '\r': \ - next_char(); \ - if(c == '\n') { \ - next_char(); \ - } \ - lexer_token.source_position.linenr++; \ - code; \ - case '\n': \ - next_char(); \ - lexer_token.source_position.linenr++; \ - code; - -static -void parse_character_constant(void) +static void parse_character_constant(void) { assert(c == '\''); next_char(); @@ -505,15 +596,9 @@ void parse_character_constant(void) int found_char = 0; while(1) { switch(c) { - SKIP_TRIGRAPHS(, - found_char = '?'; - break; - ) - case '\\': next_char(); - EAT_NEWLINE(break;) - found_char = '\\'; + found_char = parse_escape_sequence(); break; MATCH_NEWLINE( @@ -548,92 +633,44 @@ end_of_char_constant: lexer_token.v.intvalue = found_char; } -static -void skip_multiline_comment(void) +static void skip_multiline_comment(void) { unsigned start_linenr = lexer_token.source_position.linenr; - int had_star = 0; while(1) { switch(c) { case '*': next_char(); - had_star = 1; - break; - - case '/': - next_char(); - if(had_star) { + if(c == '/') { + next_char(); return; } - had_star = 0; - break; - - case '\\': - next_char(); - EAT_NEWLINE(break;) - had_star = 0; - break; - - case '?': - next_char(); - if(c != '?') { - had_star = 0; - break; - } - next_char(); - if(replace_trigraph()) - break; - put_back(c); - c = '?'; - had_star = 0; - /* we don't put back the 2nd ? as the comment text is discarded - * anyway */ break; - MATCH_NEWLINE(had_star = 0; break;) + MATCH_NEWLINE(break;) case EOF: error_prefix_at(lexer_token.source_position.input_name, start_linenr); fprintf(stderr, "at end of file while looking for comment end\n"); return; + default: - had_star = 0; next_char(); break; } } } -static -void skip_line_comment(void) +static void skip_line_comment(void) { while(1) { switch(c) { - case '?': - next_char(); - if(c != '?') - break; - next_char(); - if(replace_trigraph()) - break; - put_back('?'); - /* we don't put back the 2nd ? as the comment text is discarded - * anyway */ - break; - - case '\\': - next_char(); - if(c == '\n') { - next_char(); - lexer_token.source_position.linenr++; - } - break; - case EOF: - case '\r': + return; + case '\n': + case '\r': return; default: @@ -645,23 +682,20 @@ void skip_line_comment(void) static token_t pp_token; -static inline -void next_pp_token(void) +static inline void next_pp_token(void) { lexer_next_preprocessing_token(); pp_token = lexer_token; } -static -void eat_until_newline(void) +static void eat_until_newline(void) { while(pp_token.type != '\n' && pp_token.type != T_EOF) { next_pp_token(); } } -static -void error_directive(void) +static void error_directive(void) { error_prefix(); fprintf(stderr, "#error directive: \n"); @@ -669,8 +703,7 @@ void error_directive(void) /* parse pp-tokens until new-line */ } -static -void define_directive(void) +static void define_directive(void) { lexer_next_preprocessing_token(); if(lexer_token.type != T_IDENTIFIER) { @@ -679,8 +712,7 @@ void define_directive(void) } } -static -void ifdef_directive(int is_ifndef) +static void ifdef_directive(int is_ifndef) { (void) is_ifndef; lexer_next_preprocessing_token(); @@ -688,14 +720,12 @@ void ifdef_directive(int is_ifndef) //extect_newline(); } -static -void endif_directive(void) +static void endif_directive(void) { //expect_newline(); } -static -void parse_line_directive(void) +static void parse_line_directive(void) { if(pp_token.type != T_INTEGER) { parse_error("expected integer"); @@ -711,8 +741,7 @@ void parse_line_directive(void) eat_until_newline(); } -static -void parse_preprocessor_identifier(void) +static void parse_preprocessor_identifier(void) { assert(pp_token.type == T_IDENTIFIER); symbol_t *symbol = pp_token.v.symbol; @@ -749,8 +778,7 @@ void parse_preprocessor_identifier(void) } } -static -void parse_preprocessor_directive() +static void parse_preprocessor_directive() { next_pp_token(); @@ -780,14 +808,6 @@ void parse_preprocessor_directive() return; #define ELSE_CODE(code) \ - SKIP_TRIGRAPHS(, \ - code; \ - ) \ - \ - case '\\': \ - next_char(); \ - EAT_NEWLINE(break;) \ - /* fallthrough */ \ default: \ code; \ } \ @@ -830,18 +850,6 @@ void lexer_next_preprocessing_token(void) parse_character_constant(); return; - case '\\': - next_char(); - if(c == '\n') { - next_char(); - lexer_token.source_position.linenr++; - break; - } else { - parse_error("unexpected '\\' found"); - lexer_token.type = T_ERROR; - } - return; - case '.': MAYBE_PROLOG case '.': @@ -870,6 +878,7 @@ void lexer_next_preprocessing_token(void) ELSE('+') case '-': MAYBE_PROLOG + MAYBE('>', T_MINUSGREATER) MAYBE('-', T_MINUSMINUS) MAYBE('=', T_MINUSEQUAL) ELSE('-') @@ -912,6 +921,7 @@ void lexer_next_preprocessing_token(void) MAYBE_PROLOG MAYBE(':', T_LESSCOLON) MAYBE('%', T_LESSPERCENT) + MAYBE('=', T_LESSEQUAL) case '<': MAYBE_PROLOG MAYBE('=', T_LESSLESSEQUAL) @@ -919,6 +929,7 @@ void lexer_next_preprocessing_token(void) ELSE('<') case '>': MAYBE_PROLOG + MAYBE('=', T_GREATEREQUAL) case '>': MAYBE_PROLOG MAYBE('=', T_GREATERGREATEREQUAL) @@ -947,22 +958,6 @@ void lexer_next_preprocessing_token(void) ELSE('#') case '?': - next_char(); - /* just a simple ? */ - if(c != '?') { - lexer_token.type = '?'; - return; - } - /* might be a trigraph */ - next_char(); - if(replace_trigraph()) { - break; - } - put_back(c); - c = '?'; - lexer_token.type = '?'; - return; - case '[': case ']': case '(': @@ -972,6 +967,7 @@ void lexer_next_preprocessing_token(void) case '~': case ';': case ',': + case '\\': lexer_token.type = c; next_char(); return; @@ -1015,12 +1011,10 @@ void init_lexer(void) void lexer_open_stream(FILE *stream, const char *input_name) { input = stream; - lexer_token.source_position.linenr = 0; + lexer_token.source_position.linenr = 1; lexer_token.source_position.input_name = input_name; - /* we place a virtual '\n' at the beginning so the lexer knows we're at the - * beginning of a line */ - c = '\n'; + next_char(); } void exit_lexer(void)