X-Git-Url: http://nsz.repo.hu/git/?a=blobdiff_plain;f=lexer.c;h=699f8ee5cdd224e89970214bc27c9542ea68c4d4;hb=4b01aba61ad8d1c130517ae7bb51eea9b22e9a44;hp=ed0bab1840e76bdab04983fbed5f922e1f5ed445;hpb=1873fa8debe180e8cc4c13b6d666bd7e06886c28;p=cparser diff --git a/lexer.c b/lexer.c index ed0bab1..699f8ee 100644 --- a/lexer.c +++ b/lexer.c @@ -6,23 +6,52 @@ #include "adt/error.h" #include "adt/strset.h" #include "adt/util.h" +#include "type_t.h" #include #include #include +#include #include -#define DEBUG_CHARS +//#define DEBUG_CHARS #define MAX_PUTBACK 3 +#ifdef _WIN32 +/* No strtold on windows and no replacement yet */ +#define strtold(s, e) strtod(s, e) +#endif + +#define HAS_SIGNED_CHAR +//#define HAS_UNSIGNED_CHAR + +#if defined HAS_SIGNED_CHAR +typedef signed char char_type; +#elif defined HAS_UNSIGNED_CHAR +typedef unsigned char char_type; +#else +# error signedness of char not determined +#endif + static int c; token_t lexer_token; +symbol_t *symbol_L; static FILE *input; static char buf[1024 + MAX_PUTBACK]; static const char *bufend; static const char *bufpos; static strset_t stringset; +static type_t *type_int = NULL; +static type_t *type_uint = NULL; +static type_t *type_long = NULL; +static type_t *type_ulong = NULL; +static type_t *type_longlong = NULL; +static type_t *type_ulonglong = NULL; +static type_t *type_float = NULL; +static type_t *type_double = NULL; +static type_t *type_longdouble = NULL; + static void error_prefix_at(const char *input_name, unsigned linenr) { fprintf(stderr, "%s:%u: Error: ", input_name, linenr); @@ -58,11 +87,16 @@ static inline void next_real_char(void) static inline void put_back(int pc) { - char *p = (char*) bufpos - 1; - bufpos--; - assert(p >= buf); + assert(bufpos >= buf); + //assert(bufpos < buf+MAX_PUTBACK || *bufpos == pc); + + char *p = buf + (bufpos - buf); *p = pc; + /* going backwards in the buffer is legal as long as it's not more often + * than MAX_PUTBACK */ + bufpos--; + #ifdef DEBUG_CHARS printf("putback '%c'\n", pc); #endif @@ -83,11 +117,7 @@ static inline void next_char(void); lexer_token.source_position.linenr++; \ code; -static inline void eat(char c_type) -{ - assert(c == c_type); - next_char(); -} +#define eat(c_type) do { assert(c == c_type); next_char(); } while(0) static void maybe_concat_lines(void) { @@ -142,11 +172,9 @@ static inline void next_char(void) break; } -end_of_next_char: +end_of_next_char:; #ifdef DEBUG_CHARS printf("nchar '%c'\n", c); -#else - ; #endif } @@ -254,14 +282,18 @@ end_symbol: static void parse_integer_suffix(void) { - if(c == 'U' || c == 'U') { - /* TODO do something with the suffixes... */ + if(c == 'U' || c == 'u') { next_char(); if(c == 'L' || c == 'l') { next_char(); if(c == 'L' || c == 'l') { next_char(); + lexer_token.datatype = type_ulonglong; + } else { + lexer_token.datatype = type_ulong; } + } else { + lexer_token.datatype = type_uint; } } else if(c == 'l' || c == 'L') { next_char(); @@ -269,143 +301,241 @@ static void parse_integer_suffix(void) next_char(); if(c == 'u' || c == 'U') { next_char(); + lexer_token.datatype = type_ulonglong; + } else { + lexer_token.datatype = type_longlong; } } else if(c == 'u' || c == 'U') { next_char(); + lexer_token.datatype = type_ulong; + } else { + lexer_token.datatype = type_int; } + } else { + lexer_token.datatype = type_int; } } -static void parse_number_hex(void) +static void parse_floating_suffix(void) { - assert(c == 'x' || c == 'X'); - next_char(); + switch(c) { + /* TODO: do something usefull with the suffixes... */ + case 'f': + case 'F': + next_char(); + lexer_token.datatype = type_float; + break; + case 'l': + case 'L': + next_char(); + lexer_token.datatype = type_longdouble; + break; + default: + lexer_token.datatype = type_double; + break; + } +} - if (!isdigit(c) && - !('A' <= c && c <= 'F') && - !('a' <= c && c <= 'f')) { - parse_error("premature end of hex number literal"); - lexer_token.type = T_ERROR; - return; +/** + * A replacement for strtoull. Only those parts needed for + * our parser are implemented. + */ +static unsigned long long parse_int_string(const char *s, const char **endptr, int base) { + unsigned long long v = 0; + + switch (base) { + case 16: + for (;; ++s) { + /* check for overrun */ + if (v >= 0x1000000000000000ULL) + break; + switch (tolower(*s)) { + case '0': v <<= 4; break; + case '1': v <<= 4; v |= 0x1; break; + case '2': v <<= 4; v |= 0x2; break; + case '3': v <<= 4; v |= 0x3; break; + case '4': v <<= 4; v |= 0x4; break; + case '5': v <<= 4; v |= 0x5; break; + case '6': v <<= 4; v |= 0x6; break; + case '7': v <<= 4; v |= 0x7; break; + case '8': v <<= 4; v |= 0x8; break; + case '9': v <<= 4; v |= 0x9; break; + case 'a': v <<= 4; v |= 0xa; break; + case 'b': v <<= 4; v |= 0xb; break; + case 'c': v <<= 4; v |= 0xc; break; + case 'd': v <<= 4; v |= 0xd; break; + case 'e': v <<= 4; v |= 0xe; break; + case 'f': v <<= 4; v |= 0xf; break; + default: + goto end; + } + } + break; + case 8: + for (;; ++s) { + /* check for overrun */ + if (v >= 0x2000000000000000ULL) + break; + switch (tolower(*s)) { + case '0': v <<= 3; break; + case '1': v <<= 3; v |= 1; break; + case '2': v <<= 3; v |= 2; break; + case '3': v <<= 3; v |= 3; break; + case '4': v <<= 3; v |= 4; break; + case '5': v <<= 3; v |= 5; break; + case '6': v <<= 3; v |= 6; break; + case '7': v <<= 3; v |= 7; break; + default: + goto end; + } + } + break; + case 10: + for (;; ++s) { + /* check for overrun */ + if (v > 0x1999999999999999ULL) + break; + switch (tolower(*s)) { + case '0': v *= 10; break; + case '1': v *= 10; v += 1; break; + case '2': v *= 10; v += 2; break; + case '3': v *= 10; v += 3; break; + case '4': v *= 10; v += 4; break; + case '5': v *= 10; v += 5; break; + case '6': v *= 10; v += 6; break; + case '7': v *= 10; v += 7; break; + case '8': v *= 10; v += 8; break; + case '9': v *= 10; v += 9; break; + default: + goto end; + } + } + break; + default: + assert(0); + break; } +end: + *endptr = s; + return v; +} - int value = 0; - while(1) { - if (isdigit(c)) { - value = 16 * value + c - '0'; - } else if ('A' <= c && c <= 'F') { - value = 16 * value + c - 'A' + 10; - } else if ('a' <= c && c <= 'f') { - value = 16 * value + c - 'a' + 10; - } else { - parse_integer_suffix(); +static void parse_number_hex(void) +{ + assert(c == 'x' || c == 'X'); + next_char(); - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = value; - return; - } + while(isxdigit(c)) { + obstack_1grow(&symbol_obstack, c); next_char(); } + obstack_1grow(&symbol_obstack, '\0'); + char *string = obstack_finish(&symbol_obstack); if(c == '.' || c == 'p' || c == 'P') { next_char(); panic("Hex floating point numbers not implemented yet"); } -} - -static void parse_number_oct(void) -{ - int value = 0; - while(c >= '0' && c <= '7') { - value = 8 * value + c - '0'; - next_char(); - } - if (c == '8' || c == '9') { - parse_error("invalid octal number"); + if(*string == '\0') { + parse_error("invalid hex number"); lexer_token.type = T_ERROR; - return; } + const char *endptr; lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = value; + lexer_token.v.intvalue = parse_int_string(string, &endptr, 16); + if(*endptr != '\0') { + parse_error("hex number literal too long"); + } + obstack_free(&symbol_obstack, string); parse_integer_suffix(); } -static void parse_floatingpoint_exponent(long double value) +static inline bool is_octal_digit(int chr) { - unsigned int expo = 0; - long double factor = 10.; - - if(c == '-') { - next_char(); - factor = 0.1; - } else if(c == '+') { - next_char(); - } - - while(c >= '0' && c <= '9') { - expo = 10 * expo + (c - '0'); - next_char(); - } - - while(1) { - if(expo & 1) - value *= factor; - expo >>= 1; - if(expo == 0) - break; - factor *= factor; - } - - lexer_token.type = T_FLOATINGPOINT; - lexer_token.v.floatvalue = value; + return '0' <= chr && chr <= '7'; } -static void parse_floatingpoint_fract(int integer_part) +static void parse_number_oct(void) { - long double value = integer_part; - long double factor = 1.; - - while(c >= '0' && c <= '9') { - factor *= 0.1; - value += (c - '0') * factor; + while(is_octal_digit(c)) { + obstack_1grow(&symbol_obstack, c); next_char(); } + obstack_1grow(&symbol_obstack, '\0'); + char *string = obstack_finish(&symbol_obstack); - if(c == 'e' || c == 'E') { - next_char(); - parse_floatingpoint_exponent(value); - return; + const char *endptr; + lexer_token.type = T_INTEGER; + lexer_token.v.intvalue = parse_int_string(string, &endptr, 8); + if(*endptr != '\0') { + parse_error("octal number literal too long"); } - lexer_token.type = T_FLOATINGPOINT; - lexer_token.v.floatvalue = value; + obstack_free(&symbol_obstack, string); + parse_integer_suffix(); } static void parse_number_dec(void) { - int value = 0; - + bool is_float = false; while(isdigit(c)) { - value = 10 * value + c - '0'; + obstack_1grow(&symbol_obstack, c); next_char(); } if(c == '.') { + obstack_1grow(&symbol_obstack, '.'); next_char(); - parse_floatingpoint_fract(value); - return; + + while(isdigit(c)) { + obstack_1grow(&symbol_obstack, c); + next_char(); + } + is_float = true; } if(c == 'e' || c == 'E') { + obstack_1grow(&symbol_obstack, 'e'); next_char(); - parse_floatingpoint_exponent(value); - return; + + if(c == '-' || c == '+') { + obstack_1grow(&symbol_obstack, c); + next_char(); + } + + while(isdigit(c)) { + obstack_1grow(&symbol_obstack, c); + next_char(); + } + is_float = true; } - parse_integer_suffix(); - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = value; + obstack_1grow(&symbol_obstack, '\0'); + char *string = obstack_finish(&symbol_obstack); + + if(is_float) { + char *endptr; + lexer_token.type = T_FLOATINGPOINT; + lexer_token.v.floatvalue = strtold(string, &endptr); + + if(*endptr != '\0') { + parse_error("invalid number literal"); + } + + parse_floating_suffix(); + } else { + const char *endptr; + lexer_token.type = T_INTEGER; + lexer_token.v.intvalue = parse_int_string(string, &endptr, 10); + + if(*endptr != '\0') { + parse_error("invalid number literal"); + } + + parse_integer_suffix(); + } + obstack_free(&symbol_obstack, string); } static void parse_number(void) @@ -427,23 +557,17 @@ static void parse_number(void) case '7': parse_number_oct(); break; - case '.': - next_char(); - parse_floatingpoint_fract(0); - break; - case 'e': - case 'E': - parse_floatingpoint_exponent(0); - break; case '8': case '9': next_char(); parse_error("invalid octal number"); lexer_token.type = T_ERROR; return; + case '.': + case 'e': + case 'E': default: - put_back(c); - c = '0'; + obstack_1grow(&symbol_obstack, '0'); parse_number_dec(); return; } @@ -452,17 +576,17 @@ static void parse_number(void) } } -static int parse_octal_sequence(void) +static int parse_octal_sequence(const int first_digit) { - int value = 0; - while(1) { - if(c < '0' || c > '7') - break; - value = 8 * value + c - '0'; - next_char(); - } - - return value; + assert(is_octal_digit(first_digit)); + int value = first_digit - '0'; + if (!is_octal_digit(c)) return value; + value = 8 * value + c - '0'; + next_char(); + if (!is_octal_digit(c)) return value; + value = 8 * value + c - '0'; + next_char(); + return (char_type)value; } static int parse_hex_sequence(void) @@ -481,7 +605,7 @@ static int parse_hex_sequence(void) next_char(); } - return value; + return (char_type)value; } static int parse_escape_sequence(void) @@ -493,7 +617,7 @@ static int parse_escape_sequence(void) switch(ec) { case '"': return '"'; - case '\'': return'\''; + case '\'': return '\''; case '\\': return '\\'; case '?': return '\?'; case 'a': return '\a'; @@ -513,7 +637,7 @@ static int parse_escape_sequence(void) case '5': case '6': case '7': - return parse_octal_sequence(); + return parse_octal_sequence(ec); case EOF: parse_error("reached end of file while parsing escape sequence"); return EOF; @@ -733,7 +857,7 @@ static void parse_line_directive(void) if(pp_token.type != T_INTEGER) { parse_error("expected integer"); } else { - lexer_token.source_position.linenr = pp_token.v.intvalue - 1; + lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1); next_pp_token(); } if(pp_token.type == T_STRING_LITERAL) { @@ -781,7 +905,7 @@ static void parse_preprocessor_identifier(void) } } -static void parse_preprocessor_directive() +static void parse_preprocessor_directive(void) { next_pp_token(); @@ -839,6 +963,12 @@ void lexer_next_preprocessing_token(void) SYMBOL_CHARS parse_symbol(); + /* might be a wide string ( L"string" ) */ + if(c == '"' && (lexer_token.type == T_IDENTIFIER && + lexer_token.v.symbol == symbol_L)) { + parse_string_literal(); + return; + } return; DIGITS @@ -1009,15 +1139,34 @@ newline_found: void init_lexer(void) { strset_init(&stringset); + + type_int = make_atomic_type(ATOMIC_TYPE_INT, TYPE_QUALIFIER_CONST); + type_uint = make_atomic_type(ATOMIC_TYPE_UINT, TYPE_QUALIFIER_CONST); + type_long = make_atomic_type(ATOMIC_TYPE_LONG, TYPE_QUALIFIER_CONST); + type_ulong = make_atomic_type(ATOMIC_TYPE_ULONG, TYPE_QUALIFIER_CONST); + type_longlong = make_atomic_type(ATOMIC_TYPE_LONGLONG, + TYPE_QUALIFIER_CONST); + type_ulonglong = make_atomic_type(ATOMIC_TYPE_ULONGLONG, + TYPE_QUALIFIER_CONST); + + type_float = make_atomic_type(ATOMIC_TYPE_FLOAT, TYPE_QUALIFIER_CONST); + type_double = make_atomic_type(ATOMIC_TYPE_DOUBLE, + TYPE_QUALIFIER_CONST); + type_longdouble = make_atomic_type(ATOMIC_TYPE_LONG_DOUBLE, + TYPE_QUALIFIER_CONST); } void lexer_open_stream(FILE *stream, const char *input_name) { input = stream; - lexer_token.source_position.linenr = 1; + lexer_token.source_position.linenr = 0; lexer_token.source_position.input_name = input_name; - next_char(); + symbol_L = symbol_table_insert("L"); + + /* place a virtual \n at the beginning so the lexer knows that we're + * at the beginning of a line */ + c = '\n'; } void exit_lexer(void)