X-Git-Url: http://nsz.repo.hu/git/?a=blobdiff_plain;f=lexer.c;h=6a6a9459c1976825390f01fc1f3ae4b08cff03b3;hb=f4997f4d3a17361c4a6e7beaec96f76caecd1e51;hp=9fa306dc80ca50b7017d061e1663614941b42022;hpb=16960178ce1714e96fe33cb1ff54ce377cb05dca;p=cparser diff --git a/lexer.c b/lexer.c index 9fa306d..6a6a945 100644 --- a/lexer.c +++ b/lexer.c @@ -1,12 +1,16 @@ #include +#include "diagnostic.h" #include "lexer.h" #include "token_t.h" #include "symbol_table_t.h" #include "adt/error.h" #include "adt/strset.h" #include "adt/util.h" +#include "types.h" #include "type_t.h" +#include "target_architecture.h" +#include "parser.h" #include #include @@ -22,6 +26,14 @@ #define strtold(s, e) strtod(s, e) #endif +#if defined HAS_SIGNED_CHAR +typedef signed char char_type; +#elif defined HAS_UNSIGNED_CHAR +typedef unsigned char char_type; +#else +# error signedness of char not determined +#endif + static int c; token_t lexer_token; symbol_t *symbol_L; @@ -31,16 +43,6 @@ static const char *bufend; static const char *bufpos; static strset_t stringset; -static type_t *type_int = NULL; -static type_t *type_uint = NULL; -static type_t *type_long = NULL; -static type_t *type_ulong = NULL; -static type_t *type_longlong = NULL; -static type_t *type_ulonglong = NULL; -static type_t *type_float = NULL; -static type_t *type_double = NULL; -static type_t *type_longdouble = NULL; - static void error_prefix_at(const char *input_name, unsigned linenr) { fprintf(stderr, "%s:%u: Error: ", input_name, linenr); @@ -60,8 +62,8 @@ static void parse_error(const char *msg) static inline void next_real_char(void) { - bufpos++; - if(bufpos >= bufend) { + assert(bufpos <= bufend); + if (bufpos >= bufend) { size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK, input); if(s == 0) { @@ -71,20 +73,13 @@ static inline void next_real_char(void) bufpos = buf + MAX_PUTBACK; bufend = buf + MAX_PUTBACK + s; } - c = *(bufpos); + c = *bufpos++; } static inline void put_back(int pc) { - assert(bufpos >= buf); - //assert(bufpos < buf+MAX_PUTBACK || *bufpos == pc); - - char *p = buf + (bufpos - buf); - *p = pc; - - /* going backwards in the buffer is legal as long as it's not more often - * than MAX_PUTBACK */ - bufpos--; + assert(bufpos > buf); + *(--bufpos - buf + buf) = (char) pc; #ifdef DEBUG_CHARS printf("putback '%c'\n", pc); @@ -155,8 +150,8 @@ static inline void next_char(void) case '>': c = '}'; break; case '-': c = '~'; break; default: - put_back('?'); put_back(c); + put_back('?'); c = '?'; break; } @@ -239,14 +234,14 @@ static void parse_symbol(void) symbol_t *symbol; char *string; - obstack_1grow(&symbol_obstack, c); + obstack_1grow(&symbol_obstack, (char) c); next_char(); while(1) { switch(c) { DIGITS SYMBOL_CHARS - obstack_1grow(&symbol_obstack, c); + obstack_1grow(&symbol_obstack, (char) c); next_char(); break; @@ -269,39 +264,78 @@ end_symbol: } } -static void parse_integer_suffix(void) +static void parse_integer_suffix(bool is_oct_hex) { + bool is_unsigned = false; + bool min_long = false; + bool min_longlong = false; + if(c == 'U' || c == 'u') { + is_unsigned = true; next_char(); if(c == 'L' || c == 'l') { + min_long = true; next_char(); if(c == 'L' || c == 'l') { + min_longlong = true; next_char(); - lexer_token.datatype = type_ulonglong; - } else { - lexer_token.datatype = type_ulong; } - } else { - lexer_token.datatype = type_uint; } } else if(c == 'l' || c == 'L') { + min_long = true; next_char(); if(c == 'l' || c == 'L') { + min_longlong = true; next_char(); if(c == 'u' || c == 'U') { + is_unsigned = true; next_char(); - lexer_token.datatype = type_ulonglong; - } else { - lexer_token.datatype = type_longlong; } } else if(c == 'u' || c == 'U') { + is_unsigned = true; next_char(); - lexer_token.datatype = type_ulong; - } else { - lexer_token.datatype = type_int; + lexer_token.datatype = type_unsigned_long; } + } + + if(!is_unsigned) { + long long v = lexer_token.v.intvalue; + if(!min_long) { + if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) { + lexer_token.datatype = type_int; + return; + } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) { + lexer_token.datatype = type_unsigned_int; + return; + } + } + if(!min_longlong) { + if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) { + lexer_token.datatype = type_long; + return; + } else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) { + lexer_token.datatype = type_unsigned_long; + return; + } + } + unsigned long long uv = (unsigned long long) v; + if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) { + lexer_token.datatype = type_unsigned_long_long; + return; + } + + lexer_token.datatype = type_long_long; } else { - lexer_token.datatype = type_int; + unsigned long long v = (unsigned long long) lexer_token.v.intvalue; + if(!min_long && v <= TARGET_UINT_MAX) { + lexer_token.datatype = type_unsigned_int; + return; + } + if(!min_longlong && v <= TARGET_ULONG_MAX) { + lexer_token.datatype = type_unsigned_long; + return; + } + lexer_token.datatype = type_unsigned_long_long; } } @@ -317,7 +351,7 @@ static void parse_floating_suffix(void) case 'l': case 'L': next_char(); - lexer_token.datatype = type_longdouble; + lexer_token.datatype = type_long_double; break; default: lexer_token.datatype = type_double; @@ -336,7 +370,7 @@ static unsigned long long parse_int_string(const char *s, const char **endptr, i case 16: for (;; ++s) { /* check for overrun */ - if (v <= 0x1000000000000000ULL) + if (v >= 0x1000000000000000ULL) break; switch (tolower(*s)) { case '0': v <<= 4; break; @@ -363,7 +397,7 @@ static unsigned long long parse_int_string(const char *s, const char **endptr, i case 8: for (;; ++s) { /* check for overrun */ - if (v <= 0x2000000000000000ULL) + if (v >= 0x2000000000000000ULL) break; switch (tolower(*s)) { case '0': v <<= 3; break; @@ -415,7 +449,7 @@ static void parse_number_hex(void) next_char(); while(isxdigit(c)) { - obstack_1grow(&symbol_obstack, c); + obstack_1grow(&symbol_obstack, (char) c); next_char(); } obstack_1grow(&symbol_obstack, '\0'); @@ -438,7 +472,7 @@ static void parse_number_hex(void) } obstack_free(&symbol_obstack, string); - parse_integer_suffix(); + parse_integer_suffix(true); } static inline bool is_octal_digit(int chr) @@ -449,7 +483,7 @@ static inline bool is_octal_digit(int chr) static void parse_number_oct(void) { while(is_octal_digit(c)) { - obstack_1grow(&symbol_obstack, c); + obstack_1grow(&symbol_obstack, (char) c); next_char(); } obstack_1grow(&symbol_obstack, '\0'); @@ -463,14 +497,14 @@ static void parse_number_oct(void) } obstack_free(&symbol_obstack, string); - parse_integer_suffix(); + parse_integer_suffix(true); } static void parse_number_dec(void) { bool is_float = false; while(isdigit(c)) { - obstack_1grow(&symbol_obstack, c); + obstack_1grow(&symbol_obstack, (char) c); next_char(); } @@ -479,7 +513,7 @@ static void parse_number_dec(void) next_char(); while(isdigit(c)) { - obstack_1grow(&symbol_obstack, c); + obstack_1grow(&symbol_obstack, (char) c); next_char(); } is_float = true; @@ -489,12 +523,12 @@ static void parse_number_dec(void) next_char(); if(c == '-' || c == '+') { - obstack_1grow(&symbol_obstack, c); + obstack_1grow(&symbol_obstack, (char) c); next_char(); } while(isdigit(c)) { - obstack_1grow(&symbol_obstack, c); + obstack_1grow(&symbol_obstack, (char) c); next_char(); } is_float = true; @@ -503,8 +537,8 @@ static void parse_number_dec(void) obstack_1grow(&symbol_obstack, '\0'); char *string = obstack_finish(&symbol_obstack); - const char *endptr; if(is_float) { + char *endptr; lexer_token.type = T_FLOATINGPOINT; lexer_token.v.floatvalue = strtold(string, &endptr); @@ -514,6 +548,7 @@ static void parse_number_dec(void) parse_floating_suffix(); } else { + const char *endptr; lexer_token.type = T_INTEGER; lexer_token.v.intvalue = parse_int_string(string, &endptr, 10); @@ -521,7 +556,7 @@ static void parse_number_dec(void) parse_error("invalid number literal"); } - parse_integer_suffix(); + parse_integer_suffix(false); } obstack_free(&symbol_obstack, string); } @@ -574,7 +609,7 @@ static int parse_octal_sequence(const int first_digit) if (!is_octal_digit(c)) return value; value = 8 * value + c - '0'; next_char(); - return value; + return (char_type)value; } static int parse_hex_sequence(void) @@ -593,7 +628,7 @@ static int parse_hex_sequence(void) next_char(); } - return value; + return (char_type)value; } static int parse_escape_sequence(void) @@ -666,7 +701,7 @@ static void parse_string_literal(void) switch(c) { case '\\': tc = parse_escape_sequence(); - obstack_1grow(&symbol_obstack, tc); + obstack_1grow(&symbol_obstack, (char) tc); break; case EOF: @@ -681,7 +716,7 @@ static void parse_string_literal(void) goto end_of_string; default: - obstack_1grow(&symbol_obstack, c); + obstack_1grow(&symbol_obstack, (char) c); next_char(); break; } @@ -705,6 +740,110 @@ end_of_string: lexer_token.v.string = result; } +static void parse_wide_character_constant(void) +{ + eat('\''); + + int found_char = 0; + while(1) { + switch(c) { + case '\\': + found_char = parse_escape_sequence(); + break; + + MATCH_NEWLINE( + parse_error("newline while parsing character constant"); + break; + ) + + case '\'': + next_char(); + goto end_of_wide_char_constant; + + case EOF: + parse_error("EOF while parsing character constant"); + lexer_token.type = T_ERROR; + return; + + default: + if(found_char != 0) { + parse_error("more than 1 characters in character " + "constant"); + goto end_of_wide_char_constant; + } else { + found_char = c; + next_char(); + } + break; + } + } + +end_of_wide_char_constant: + lexer_token.type = T_INTEGER; + lexer_token.v.intvalue = found_char; + lexer_token.datatype = type_wchar_t; +} + +static void parse_wide_string_literal(void) +{ + const unsigned start_linenr = lexer_token.source_position.linenr; + + assert(c == '"'); + next_char(); + + while(1) { + switch(c) { + case '\\': { + wchar_rep_t tc = parse_escape_sequence(); + obstack_grow(&symbol_obstack, &tc, sizeof(tc)); + break; + } + + case EOF: + error_prefix_at(lexer_token.source_position.input_name, + start_linenr); + fprintf(stderr, "string has no end\n"); + lexer_token.type = T_ERROR; + return; + + case '"': + next_char(); + goto end_of_string; + + default: { + wchar_rep_t tc = c; + obstack_grow(&symbol_obstack, &tc, sizeof(tc)); + next_char(); + break; + } + } + } + +end_of_string:; + + /* TODO: concatenate multiple strings separated by whitespace... */ + + /* add finishing 0 to the string */ + wchar_rep_t nul = L'\0'; + obstack_grow(&symbol_obstack, &nul, sizeof(nul)); + const size_t size = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t); + const wchar_rep_t *const string = obstack_finish(&symbol_obstack); + +#if 0 /* TODO hash */ + /* check if there is already a copy of the string */ + const wchar_rep_t *const result = strset_insert(&stringset, string); + if(result != string) { + obstack_free(&symbol_obstack, string); + } +#else + const wchar_rep_t *const result = string; +#endif + + lexer_token.type = T_WIDE_STRING_LITERAL; + lexer_token.v.wide_string.begin = result; + lexer_token.v.wide_string.size = size; +} + static void parse_character_constant(void) { eat('\''); @@ -746,6 +885,7 @@ static void parse_character_constant(void) end_of_char_constant: lexer_token.type = T_INTEGER; lexer_token.v.intvalue = found_char; + lexer_token.datatype = type_int; } static void skip_multiline_comment(void) @@ -889,6 +1029,8 @@ static void parse_preprocessor_identifier(void) error_directive(); break; case TP_pragma: + warningf(lexer_token.source_position, "encountered unknown #pragma"); + eat_until_newline(); break; } } @@ -952,10 +1094,13 @@ void lexer_next_preprocessing_token(void) SYMBOL_CHARS parse_symbol(); /* might be a wide string ( L"string" ) */ - if(c == '"' && (lexer_token.type == T_IDENTIFIER && - lexer_token.v.symbol == symbol_L)) { - parse_string_literal(); - return; + if(lexer_token.type == T_IDENTIFIER && + lexer_token.v.symbol == symbol_L) { + if(c == '"') { + parse_wide_string_literal(); + } else if(c == '\'') { + parse_wide_character_constant(); + } } return; @@ -973,6 +1118,21 @@ void lexer_next_preprocessing_token(void) case '.': MAYBE_PROLOG + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + put_back(c); + c = '.'; + parse_number_dec(); + return; + case '.': MAYBE_PROLOG MAYBE('.', T_DOTDOTDOT) @@ -1127,21 +1287,6 @@ newline_found: void init_lexer(void) { strset_init(&stringset); - - type_int = make_atomic_type(ATOMIC_TYPE_INT, TYPE_QUALIFIER_CONST); - type_uint = make_atomic_type(ATOMIC_TYPE_UINT, TYPE_QUALIFIER_CONST); - type_long = make_atomic_type(ATOMIC_TYPE_LONG, TYPE_QUALIFIER_CONST); - type_ulong = make_atomic_type(ATOMIC_TYPE_ULONG, TYPE_QUALIFIER_CONST); - type_longlong = make_atomic_type(ATOMIC_TYPE_LONGLONG, - TYPE_QUALIFIER_CONST); - type_ulonglong = make_atomic_type(ATOMIC_TYPE_ULONGLONG, - TYPE_QUALIFIER_CONST); - - type_float = make_atomic_type(ATOMIC_TYPE_FLOAT, TYPE_QUALIFIER_CONST); - type_double = make_atomic_type(ATOMIC_TYPE_DOUBLE, - TYPE_QUALIFIER_CONST); - type_longdouble = make_atomic_type(ATOMIC_TYPE_LONG_DOUBLE, - TYPE_QUALIFIER_CONST); } void lexer_open_stream(FILE *stream, const char *input_name) @@ -1151,6 +1296,8 @@ void lexer_open_stream(FILE *stream, const char *input_name) lexer_token.source_position.input_name = input_name; symbol_L = symbol_table_insert("L"); + bufpos = NULL; + bufend = NULL; /* place a virtual \n at the beginning so the lexer knows that we're * at the beginning of a line */ @@ -1165,7 +1312,7 @@ void exit_lexer(void) static __attribute__((unused)) void dbg_pos(const source_position_t source_position) { - fprintf(stdout, "%s:%d\n", source_position.input_name, + fprintf(stdout, "%s:%u\n", source_position.input_name, source_position.linenr); fflush(stdout); }