X-Git-Url: http://nsz.repo.hu/git/?a=blobdiff_plain;f=lexer.c;h=80e49163564cd8fe88051f093d5ac364a10ba19e;hb=bb8544d38aaa4c716603a5d565c3ff3840454ffc;hp=f7d97af56daedf486a85c756301fc136f32cb60d;hpb=9f6aeae9014b1aee7fac9c93046bb033f79381e7;p=cparser diff --git a/lexer.c b/lexer.c index f7d97af..80e4916 100644 --- a/lexer.c +++ b/lexer.c @@ -1,6 +1,6 @@ /* * This file is part of cparser. - * Copyright (C) 2007-2008 Matthias Braun + * Copyright (C) 2007-2009 Matthias Braun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -37,10 +37,13 @@ #include #include #include -#include #include #include +#ifndef _WIN32 +#include +#endif + //#define DEBUG_CHARS #define MAX_PUTBACK 3 #define BUF_SIZE 1024 @@ -50,17 +53,16 @@ #define strtold(s, e) strtod(s, e) #endif -typedef unsigned int utf32; - -static utf32 c; -token_t lexer_token; -symbol_t *symbol_L; -static FILE *input; -static utf32 buf[BUF_SIZE + MAX_PUTBACK]; -static const utf32 *bufend; -static const utf32 *bufpos; -static strset_t stringset; -bool allow_dollar_in_symbol = true; +static utf32 c; +static source_position_t lexer_pos; +token_t lexer_token; +static symbol_t *symbol_L; +static FILE *input; +static utf32 buf[BUF_SIZE + MAX_PUTBACK]; +static const utf32 *bufend; +static const utf32 *bufpos; +static strset_t stringset; +bool allow_dollar_in_symbol = true; /** * Prints a parse error message at the current token. @@ -69,7 +71,7 @@ bool allow_dollar_in_symbol = true; */ static void parse_error(const char *msg) { - errorf(&lexer_token.source_position, "%s", msg); + errorf(&lexer_pos, "%s", msg); } /** @@ -79,14 +81,16 @@ static void parse_error(const char *msg) */ static NORETURN internal_error(const char *msg) { - internal_errorf(&lexer_token.source_position, "%s", msg); + internal_errorf(&lexer_pos, "%s", msg); } static size_t read_block(unsigned char *const read_buf, size_t const n) { size_t const s = fread(read_buf, 1, n, input); if (s == 0) { - if (ferror(input)) + /* on OS/X ferror appears to return true on eof as well when running + * the application in gdb... */ + if (!feof(input) && ferror(input)) parse_error("read from input failed"); buf[MAX_PUTBACK] = EOF; bufpos = buf + MAX_PUTBACK; @@ -253,6 +257,54 @@ realign: } while (bufpos == bufend); } +static void decode_windows_1252(void) +{ + unsigned char read_buf[BUF_SIZE]; + size_t const s = read_block(read_buf, sizeof(read_buf)); + if (s == 0) + return; + + unsigned char const *src = read_buf; + unsigned char const *end = read_buf + s; + utf32 *dst = buf + MAX_PUTBACK; + while (src != end) { + utf32 tc = *src++; + switch (tc) { + case 0x80: tc = 0x20AC; break; // € + case 0x82: tc = 0x201A; break; // ‚ + case 0x83: tc = 0x0192; break; // ƒ + case 0x84: tc = 0x201E; break; // „ + case 0x85: tc = 0x2026; break; // … + case 0x86: tc = 0x2020; break; // † + case 0x87: tc = 0x2021; break; // ‡ + case 0x88: tc = 0x02C6; break; // ˆ + case 0x89: tc = 0x2030; break; // ‰ + case 0x8A: tc = 0x0160; break; // Š + case 0x8B: tc = 0x2039; break; // ‹ + case 0x8C: tc = 0x0152; break; // Œ + case 0x8E: tc = 0x017D; break; // Ž + case 0x91: tc = 0x2018; break; // ‘ + case 0x92: tc = 0x2019; break; // ’ + case 0x93: tc = 0x201C; break; // “ + case 0x94: tc = 0x201D; break; // ” + case 0x95: tc = 0x2022; break; // • + case 0x96: tc = 0x2013; break; // – + case 0x97: tc = 0x2014; break; // — + case 0x98: tc = 0x02DC; break; // ˜ + case 0x99: tc = 0x2122; break; // ™ + case 0x9A: tc = 0x0161; break; // š + case 0x9B: tc = 0x203A; break; // › + case 0x9C: tc = 0x0153; break; // œ + case 0x9E: tc = 0x017E; break; // ž + case 0x9F: tc = 0x0178; break; // Ÿ + } + *dst++ = tc; + } + + bufpos = buf + MAX_PUTBACK; + bufend = dst; +} + typedef void (*decoder_t)(void); static decoder_t decoder = decode_utf8; @@ -263,29 +315,41 @@ typedef struct named_decoder_t { } named_decoder_t; static named_decoder_t const decoders[] = { - { "CP819", decode_iso_8859_1 }, // offical alias - { "IBM819", decode_iso_8859_1 }, // offical alias - { "ISO-8859-1", decode_iso_8859_1 }, // offical alias - { "ISO-8859-15", decode_iso_8859_15 }, // offical name - { "ISO8859-1", decode_iso_8859_1 }, - { "ISO8859-15", decode_iso_8859_15 }, - { "ISO_8859-1", decode_iso_8859_1 }, // offical alias - { "ISO_8859-15", decode_iso_8859_15 }, // offical alias - { "ISO_8859-1:1987", decode_iso_8859_1 }, // offical name - { "Latin-9", decode_iso_8859_15 }, // offical alias - { "UTF-8", decode_utf8 }, // offical name - { "csISOLatin1", decode_iso_8859_1 }, // offical alias - { "iso-ir-100", decode_iso_8859_1 }, // offical alias - { "l1", decode_iso_8859_1 }, // offical alias - { "latin1", decode_iso_8859_1 }, // offical alias - - { NULL, NULL } + { "CP819", decode_iso_8859_1 }, // offical alias + { "IBM819", decode_iso_8859_1 }, // offical alias + { "ISO-8859-1", decode_iso_8859_1 }, // offical alias + { "ISO-8859-15", decode_iso_8859_15 }, // offical name + { "ISO8859-1", decode_iso_8859_1 }, + { "ISO8859-15", decode_iso_8859_15 }, + { "ISO_8859-1", decode_iso_8859_1 }, // offical alias + { "ISO_8859-15", decode_iso_8859_15 }, // offical alias + { "ISO_8859-1:1987", decode_iso_8859_1 }, // offical name + { "Latin-9", decode_iso_8859_15 }, // offical alias + { "UTF-8", decode_utf8 }, // offical name + { "csISOLatin1", decode_iso_8859_1 }, // offical alias + { "cp1252", decode_windows_1252 }, + { "iso-ir-100", decode_iso_8859_1 }, // offical alias + { "l1", decode_iso_8859_1 }, // offical alias + { "latin1", decode_iso_8859_1 }, // offical alias + { "windows-1252", decode_windows_1252 }, // official name + + { NULL, NULL } }; +/** strcasecmp is not part of C99 so we need our own implementation here */ +static int my_strcasecmp(const char *s1, const char *s2) +{ + for ( ; *s1 != 0; ++s1, ++s2) { + if (tolower(*s1) != tolower(*s2)) + break; + } + return (unsigned char)*s1 - (unsigned char)*s2; +} + void select_input_encoding(char const* const encoding) { for (named_decoder_t const *i = decoders; i->name != NULL; ++i) { - if (strcasecmp(encoding, i->name) != 0) + if (my_strcasecmp(encoding, i->name) != 0) continue; decoder = i->decoder; return; @@ -304,6 +368,7 @@ static inline void next_real_char(void) decoder(); } c = *bufpos++; + ++lexer_pos.colno; } /** @@ -315,6 +380,7 @@ static inline void put_back(utf32 const pc) { assert(bufpos > buf); *(--bufpos - buf + buf) = pc; + --lexer_pos.colno; #ifdef DEBUG_CHARS printf("putback '%lc'\n", pc); @@ -323,26 +389,24 @@ static inline void put_back(utf32 const pc) static inline void next_char(void); -#define MATCH_NEWLINE(code) \ - case '\r': \ - next_char(); \ - if(c == '\n') { \ - next_char(); \ - } \ - lexer_token.source_position.linenr++; \ - code \ - case '\n': \ - next_char(); \ - lexer_token.source_position.linenr++; \ +#define MATCH_NEWLINE(code) \ + case '\r': \ + next_char(); \ + if (c == '\n') { \ + case '\n': \ + next_char(); \ + } \ + lexer_pos.lineno++; \ + lexer_pos.colno = 1; \ code -#define eat(c_type) do { assert(c == c_type); next_char(); } while(0) +#define eat(c_type) do { assert(c == c_type); next_char(); } while (0) static void maybe_concat_lines(void) { eat('\\'); - switch(c) { + switch (c) { MATCH_NEWLINE(return;) default: @@ -362,23 +426,23 @@ static inline void next_char(void) next_real_char(); /* filter trigraphs */ - if(UNLIKELY(c == '\\')) { + if (UNLIKELY(c == '\\')) { maybe_concat_lines(); goto end_of_next_char; } - if(LIKELY(c != '?')) + if (LIKELY(c != '?')) goto end_of_next_char; next_real_char(); - if(LIKELY(c != '?')) { + if (LIKELY(c != '?')) { put_back(c); c = '?'; goto end_of_next_char; } next_real_char(); - switch(c) { + switch (c) { case '=': c = '#'; break; case '(': c = '['; break; case '/': c = '\\'; maybe_concat_lines(); break; @@ -475,14 +539,11 @@ end_of_next_char:; */ static void parse_symbol(void) { - symbol_t *symbol; - char *string; - obstack_1grow(&symbol_obstack, (char) c); next_char(); - while(1) { - switch(c) { + while (true) { + switch (c) { DIGITS SYMBOL_CHARS obstack_1grow(&symbol_obstack, (char) c); @@ -498,220 +559,62 @@ dollar_sign: end_symbol: obstack_1grow(&symbol_obstack, '\0'); - string = obstack_finish(&symbol_obstack); - symbol = symbol_table_insert(string); + char *string = obstack_finish(&symbol_obstack); + symbol_t *symbol = symbol_table_insert(string); - lexer_token.type = symbol->ID; - lexer_token.v.symbol = symbol; + lexer_token.type = symbol->ID; + lexer_token.symbol = symbol; - if(symbol->string != string) { + if (symbol->string != string) { obstack_free(&symbol_obstack, string); } } -static void parse_integer_suffix(bool is_oct_hex) +/** + * parse suffixes like 'LU' or 'f' after numbers + */ +static void parse_number_suffix(void) { - bool is_unsigned = false; - bool min_long = false; - bool min_longlong = false; - bool not_traditional = false; - int pos = 0; - char suffix[4]; - - if (c == 'U' || c == 'u') { - not_traditional = true; - suffix[pos++] = toupper(c); - is_unsigned = true; - next_char(); - if (c == 'L' || c == 'l') { - suffix[pos++] = toupper(c); - min_long = true; - next_char(); - if (c == 'L' || c == 'l') { - suffix[pos++] = toupper(c); - min_longlong = true; - next_char(); - } - } - } else if (c == 'l' || c == 'L') { - suffix[pos++] = toupper(c); - min_long = true; - next_char(); - if (c == 'l' || c == 'L') { - not_traditional = true; - suffix[pos++] = toupper(c); - min_longlong = true; - next_char(); - if (c == 'u' || c == 'U') { - suffix[pos++] = toupper(c); - is_unsigned = true; - next_char(); - } - } else if (c == 'u' || c == 'U') { - not_traditional = true; - suffix[pos++] = toupper(c); - is_unsigned = true; + assert(obstack_object_size(&symbol_obstack) == 0); + while (true) { + switch (c) { + SYMBOL_CHARS + obstack_1grow(&symbol_obstack, (char) c); next_char(); - lexer_token.datatype = type_unsigned_long; + break; + default: + dollar_sign: + goto finish_suffix; } } - - if (warning.traditional && not_traditional) { - suffix[pos] = '\0'; - warningf(&lexer_token.source_position, - "traditional C rejects the '%s' suffix", suffix); +finish_suffix: + if (obstack_object_size(&symbol_obstack) == 0) { + lexer_token.symbol = NULL; + return; } - if (!is_unsigned) { - long long v = lexer_token.v.intvalue; - if (!min_long) { - if (v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) { - lexer_token.datatype = type_int; - return; - } else if (is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) { - lexer_token.datatype = type_unsigned_int; - return; - } - } - if (!min_longlong) { - if (v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) { - lexer_token.datatype = type_long; - return; - } else if (is_oct_hex && v >= 0 && (unsigned long long)v <= (unsigned long long)TARGET_ULONG_MAX) { - lexer_token.datatype = type_unsigned_long; - return; - } - } - unsigned long long uv = (unsigned long long) v; - if (is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) { - lexer_token.datatype = type_unsigned_long_long; - return; - } - lexer_token.datatype = type_long_long; - } else { - unsigned long long v = (unsigned long long) lexer_token.v.intvalue; - if (!min_long && v <= TARGET_UINT_MAX) { - lexer_token.datatype = type_unsigned_int; - return; - } - if (!min_longlong && v <= TARGET_ULONG_MAX) { - lexer_token.datatype = type_unsigned_long; - return; - } - lexer_token.datatype = type_unsigned_long_long; - } -} + obstack_1grow(&symbol_obstack, '\0'); + char *string = obstack_finish(&symbol_obstack); + symbol_t *symbol = symbol_table_insert(string); -static void parse_floating_suffix(void) -{ - switch(c) { - /* TODO: do something useful with the suffixes... */ - case 'f': - case 'F': - if (warning.traditional) { - warningf(&lexer_token.source_position, - "traditional C rejects the 'F' suffix"); - } - next_char(); - lexer_token.datatype = type_float; - break; - case 'l': - case 'L': - if (warning.traditional) { - warningf(&lexer_token.source_position, - "traditional C rejects the 'F' suffix"); - } - next_char(); - lexer_token.datatype = type_long_double; - break; - default: - lexer_token.datatype = type_double; - break; + if (symbol->string != string) { + obstack_free(&symbol_obstack, string); } + lexer_token.symbol = symbol; } -/** - * A replacement for strtoull. Only those parts needed for - * our parser are implemented. - */ -static unsigned long long parse_int_string(const char *s, const char **endptr, int base) { - unsigned long long v = 0; - - switch (base) { - case 16: - for (;; ++s) { - /* check for overrun */ - if (v >= 0x1000000000000000ULL) - break; - switch (tolower(*s)) { - case '0': v <<= 4; break; - case '1': v <<= 4; v |= 0x1; break; - case '2': v <<= 4; v |= 0x2; break; - case '3': v <<= 4; v |= 0x3; break; - case '4': v <<= 4; v |= 0x4; break; - case '5': v <<= 4; v |= 0x5; break; - case '6': v <<= 4; v |= 0x6; break; - case '7': v <<= 4; v |= 0x7; break; - case '8': v <<= 4; v |= 0x8; break; - case '9': v <<= 4; v |= 0x9; break; - case 'a': v <<= 4; v |= 0xa; break; - case 'b': v <<= 4; v |= 0xb; break; - case 'c': v <<= 4; v |= 0xc; break; - case 'd': v <<= 4; v |= 0xd; break; - case 'e': v <<= 4; v |= 0xe; break; - case 'f': v <<= 4; v |= 0xf; break; - default: - goto end; - } - } - break; - case 8: - for (;; ++s) { - /* check for overrun */ - if (v >= 0x2000000000000000ULL) - break; - switch (tolower(*s)) { - case '0': v <<= 3; break; - case '1': v <<= 3; v |= 1; break; - case '2': v <<= 3; v |= 2; break; - case '3': v <<= 3; v |= 3; break; - case '4': v <<= 3; v |= 4; break; - case '5': v <<= 3; v |= 5; break; - case '6': v <<= 3; v |= 6; break; - case '7': v <<= 3; v |= 7; break; - default: - goto end; - } - } - break; - case 10: - for (;; ++s) { - /* check for overrun */ - if (v > 0x1999999999999999ULL) - break; - switch (tolower(*s)) { - case '0': v *= 10; break; - case '1': v *= 10; v += 1; break; - case '2': v *= 10; v += 2; break; - case '3': v *= 10; v += 3; break; - case '4': v *= 10; v += 4; break; - case '5': v *= 10; v += 5; break; - case '6': v *= 10; v += 6; break; - case '7': v *= 10; v += 7; break; - case '8': v *= 10; v += 8; break; - case '9': v *= 10; v += 9; break; - default: - goto end; - } - } - break; - default: - assert(0); - break; +static string_t identify_string(char *string, size_t len) +{ + /* TODO hash */ +#if 0 + const char *result = strset_insert(&stringset, concat); + if (result != concat) { + obstack_free(&symbol_obstack, concat); } -end: - *endptr = s; - return v; +#else + const char *result = string; +#endif + return (string_t) {result, len}; } /** @@ -720,29 +623,29 @@ end: */ static void parse_number_hex(void) { - bool is_float = false; - assert(c == 'x' || c == 'X'); - next_char(); - - obstack_1grow(&symbol_obstack, '0'); - obstack_1grow(&symbol_obstack, 'x'); + bool is_float = false; + bool has_digits = false; - while(isxdigit(c)) { + assert(obstack_object_size(&symbol_obstack) == 0); + while (isxdigit(c)) { + has_digits = true; obstack_1grow(&symbol_obstack, (char) c); next_char(); } if (c == '.') { + is_float = true; obstack_1grow(&symbol_obstack, (char) c); next_char(); while (isxdigit(c)) { + has_digits = true; obstack_1grow(&symbol_obstack, (char) c); next_char(); } - is_float = true; } if (c == 'p' || c == 'P') { + is_float = true; obstack_1grow(&symbol_obstack, (char) c); next_char(); @@ -755,39 +658,27 @@ static void parse_number_hex(void) obstack_1grow(&symbol_obstack, (char) c); next_char(); } - is_float = true; + } else if (is_float) { + errorf(&lexer_token.source_position, + "hexadecimal floatingpoint constant requires an exponent"); } - obstack_1grow(&symbol_obstack, '\0'); - char *string = obstack_finish(&symbol_obstack); - if(*string == '\0') { - parse_error("invalid hex number"); - lexer_token.type = T_ERROR; - obstack_free(&symbol_obstack, string); - return; - } - if (is_float) { - char *endptr; - lexer_token.type = T_FLOATINGPOINT; - lexer_token.v.floatvalue = strtold(string, &endptr); + size_t size = obstack_object_size(&symbol_obstack) - 1; + char *string = obstack_finish(&symbol_obstack); + lexer_token.literal = identify_string(string, size); - if(*endptr != '\0') { - parse_error("invalid hex float literal"); - } + lexer_token.type = + is_float ? T_FLOATINGPOINT_HEXADECIMAL : T_INTEGER_HEXADECIMAL; - parse_floating_suffix(); - } else { - const char *endptr; - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = parse_int_string(string + 2, &endptr, 16); - if(*endptr != '\0') { - parse_error("hex number literal too long"); - } - parse_integer_suffix(true); + if (!has_digits) { + errorf(&lexer_token.source_position, "invalid number literal '0x%S'", + &lexer_token.literal); + lexer_token.literal.begin = "0"; + lexer_token.literal.size = 1; } - obstack_free(&symbol_obstack, string); + parse_number_suffix(); } /** @@ -795,9 +686,9 @@ static void parse_number_hex(void) * * @param char the character to check */ -static inline bool is_octal_digit(utf32 chr) +static bool is_octal_digit(utf32 chr) { - switch(chr) { + switch (chr) { case '0': case '1': case '2': @@ -813,132 +704,87 @@ static inline bool is_octal_digit(utf32 chr) } /** - * Parses a octal number and set the lexer_token. + * Parses a number and sets the lexer_token. */ -static void parse_number_oct(void) +static void parse_number(void) { - while(is_octal_digit(c)) { - obstack_1grow(&symbol_obstack, (char) c); - next_char(); - } - obstack_1grow(&symbol_obstack, '\0'); - char *string = obstack_finish(&symbol_obstack); + bool is_float = false; + bool has_digits = false; - const char *endptr; - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = parse_int_string(string, &endptr, 8); - if(*endptr != '\0') { - parse_error("octal number literal too long"); + assert(obstack_object_size(&symbol_obstack) == 0); + if (c == '0') { + next_char(); + if (c == 'x' || c == 'X') { + next_char(); + parse_number_hex(); + return; + } else { + has_digits = true; + } + obstack_1grow(&symbol_obstack, '0'); } - obstack_free(&symbol_obstack, string); - parse_integer_suffix(true); -} - -/** - * Parses a decimal including float number and set the - * lexer_token. - */ -static void parse_number_dec(void) -{ - bool is_float = false; while (isdigit(c)) { + has_digits = true; obstack_1grow(&symbol_obstack, (char) c); next_char(); } if (c == '.') { + is_float = true; obstack_1grow(&symbol_obstack, '.'); next_char(); while (isdigit(c)) { + has_digits = true; obstack_1grow(&symbol_obstack, (char) c); next_char(); } - is_float = true; } - if(c == 'e' || c == 'E') { - obstack_1grow(&symbol_obstack, (char) c); + if (c == 'e' || c == 'E') { + is_float = true; + obstack_1grow(&symbol_obstack, 'e'); next_char(); - if(c == '-' || c == '+') { + if (c == '-' || c == '+') { obstack_1grow(&symbol_obstack, (char) c); next_char(); } - while(isdigit(c)) { + while (isdigit(c)) { obstack_1grow(&symbol_obstack, (char) c); next_char(); } - is_float = true; } obstack_1grow(&symbol_obstack, '\0'); - char *string = obstack_finish(&symbol_obstack); + size_t size = obstack_object_size(&symbol_obstack) - 1; + char *string = obstack_finish(&symbol_obstack); + lexer_token.literal = identify_string(string, size); - if(is_float) { - char *endptr; - lexer_token.type = T_FLOATINGPOINT; - lexer_token.v.floatvalue = strtold(string, &endptr); - - if(*endptr != '\0') { - parse_error("invalid number literal"); + /* is it an octal number? */ + if (is_float) { + lexer_token.type = T_FLOATINGPOINT; + } else if (string[0] == '0') { + lexer_token.type = T_INTEGER_OCTAL; + + /* check for invalid octal digits */ + for (size_t i= 0; i < size; ++i) { + char t = string[i]; + if (t >= '8') + errorf(&lexer_token.source_position, + "invalid digit '%c' in octal number", t); } - - parse_floating_suffix(); } else { - const char *endptr; - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = parse_int_string(string, &endptr, 10); - - if(*endptr != '\0') { - parse_error("invalid number literal"); - } - - parse_integer_suffix(false); + lexer_token.type = T_INTEGER; } - obstack_free(&symbol_obstack, string); -} -/** - * Parses a number and sets the lexer_token. - */ -static void parse_number(void) -{ - if (c == '0') { - next_char(); - switch (c) { - case 'X': - case 'x': - parse_number_hex(); - break; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - parse_number_oct(); - break; - case '8': - case '9': - next_char(); - parse_error("invalid octal number"); - lexer_token.type = T_ERROR; - return; - case '.': - case 'e': - case 'E': - default: - obstack_1grow(&symbol_obstack, '0'); - parse_number_dec(); - return; - } - } else { - parse_number_dec(); + if (!has_digits) { + errorf(&lexer_token.source_position, "invalid number literal '%S'", + &lexer_token.literal); } + + parse_number_suffix(); } /** @@ -999,7 +845,7 @@ static utf32 parse_octal_sequence(utf32 const first_digit) static utf32 parse_hex_sequence(void) { utf32 value = 0; - while(isxdigit(c)) { + while (isxdigit(c)) { value = 16 * value + digit_value(c); next_char(); } @@ -1048,12 +894,17 @@ static utf32 parse_escape_sequence(void) case 'e': if (c_mode & _GNUC) return 27; /* hopefully 27 is ALWAYS the code for ESCAPE */ - /* FALLTHROUGH */ - default: - /* §6.4.4.4:8 footnote 64 */ - parse_error("unknown escape sequence"); + break; + case 'u': + case 'U': + parse_error("universal character parsing not implemented yet"); return EOF; + default: + break; } + /* §6.4.4.4:8 footnote 64 */ + parse_error("unknown escape sequence"); + return EOF; } /** @@ -1068,84 +919,16 @@ string_t concat_strings(const string_t *const s1, const string_t *const s2) memcpy(concat, s1->begin, len1); memcpy(concat + len1, s2->begin, len2 + 1); - if (warning.traditional) { - warningf(&lexer_token.source_position, - "traditional C rejects string constant concatenation"); - } -#if 0 /* TODO hash */ - const char *result = strset_insert(&stringset, concat); - if(result != concat) { - obstack_free(&symbol_obstack, concat); - } - - return result; -#else - return (string_t){ concat, len1 + len2 + 1 }; -#endif -} - -/** - * Concatenate a string and a wide string. - */ -wide_string_t concat_string_wide_string(const string_t *const s1, const wide_string_t *const s2) -{ - const size_t len1 = s1->size - 1; - const size_t len2 = s2->size - 1; - - wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat)); - const char *const src = s1->begin; - for (size_t i = 0; i != len1; ++i) { - concat[i] = src[i]; - } - memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat)); - if (warning.traditional) { - warningf(&lexer_token.source_position, - "traditional C rejects string constant concatenation"); - } - - return (wide_string_t){ concat, len1 + len2 + 1 }; + return identify_string(concat, len1 + len2 + 1); } -/** - * Concatenate two wide strings. - */ -wide_string_t concat_wide_strings(const wide_string_t *const s1, const wide_string_t *const s2) +string_t make_string(const char *string) { - const size_t len1 = s1->size - 1; - const size_t len2 = s2->size - 1; - - wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat)); - memcpy(concat, s1->begin, len1 * sizeof(*concat)); - memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat)); - if (warning.traditional) { - warningf(&lexer_token.source_position, - "traditional C rejects string constant concatenation"); - } - - return (wide_string_t){ concat, len1 + len2 + 1 }; -} - -/** - * Concatenate a wide string and a string. - */ -wide_string_t concat_wide_string_string(const wide_string_t *const s1, const string_t *const s2) -{ - const size_t len1 = s1->size - 1; - const size_t len2 = s2->size - 1; - - wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat)); - memcpy(concat, s1->begin, len1 * sizeof(*concat)); - const char *const src = s2->begin; - wchar_rep_t *const dst = concat + len1; - for (size_t i = 0; i != len2 + 1; ++i) { - dst[i] = src[i]; - } - if (warning.traditional) { - warningf(&lexer_token.source_position, - "traditional C rejects string constant concatenation"); - } + size_t len = strlen(string) + 1; + char *const space = obstack_alloc(&symbol_obstack, len); + memcpy(space, string, len); - return (wide_string_t){ concat, len1 + len2 + 1 }; + return identify_string(space, len); } static void grow_symbol(utf32 const tc) @@ -1173,27 +956,21 @@ static void grow_symbol(utf32 const tc) */ static void parse_string_literal(void) { - const unsigned start_linenr = lexer_token.source_position.linenr; - eat('"'); - while(1) { - switch(c) { + while (true) { + switch (c) { case '\\': { utf32 const tc = parse_escape_sequence(); if (tc >= 0x100) { - warningf(&lexer_token.source_position, - "escape sequence out of range"); + warningf(WARN_OTHER, &lexer_pos, "escape sequence out of range"); } obstack_1grow(&symbol_obstack, tc); break; } case EOF: { - source_position_t source_position; - source_position.input_name = lexer_token.source_position.input_name; - source_position.linenr = start_linenr; - errorf(&source_position, "string has no end"); + errorf(&lexer_token.source_position, "string has no end"); lexer_token.type = T_ERROR; return; } @@ -1215,22 +992,11 @@ end_of_string: /* add finishing 0 to the string */ obstack_1grow(&symbol_obstack, '\0'); - const size_t size = (size_t)obstack_object_size(&symbol_obstack); - const char *const string = obstack_finish(&symbol_obstack); + const size_t size = (size_t)obstack_object_size(&symbol_obstack); + char *string = obstack_finish(&symbol_obstack); -#if 0 /* TODO hash */ - /* check if there is already a copy of the string */ - result = strset_insert(&stringset, string); - if(result != string) { - obstack_free(&symbol_obstack, string); - } -#else - const char *const result = string; -#endif - - lexer_token.type = T_STRING_LITERAL; - lexer_token.v.string.begin = result; - lexer_token.v.string.size = size; + lexer_token.type = T_STRING_LITERAL; + lexer_token.literal = identify_string(string, size); } /** @@ -1238,15 +1004,13 @@ end_of_string: */ static void parse_wide_character_constant(void) { - const unsigned start_linenr = lexer_token.source_position.linenr; - eat('\''); - while(1) { - switch(c) { + while (true) { + switch (c) { case '\\': { - wchar_rep_t tc = parse_escape_sequence(); - obstack_grow(&symbol_obstack, &tc, sizeof(tc)); + const utf32 tc = parse_escape_sequence(); + grow_symbol(tc); break; } @@ -1260,33 +1024,29 @@ static void parse_wide_character_constant(void) goto end_of_wide_char_constant; case EOF: { - source_position_t source_position = lexer_token.source_position; - source_position.linenr = start_linenr; - errorf(&source_position, "EOF while parsing character constant"); + errorf(&lexer_token.source_position, "EOF while parsing character constant"); lexer_token.type = T_ERROR; return; } - default: { - wchar_rep_t tc = (wchar_rep_t) c; - obstack_grow(&symbol_obstack, &tc, sizeof(tc)); + default: + grow_symbol(c); next_char(); break; } - } } end_of_wide_char_constant:; - size_t size = (size_t) obstack_object_size(&symbol_obstack); - assert(size % sizeof(wchar_rep_t) == 0); - size /= sizeof(wchar_rep_t); + obstack_1grow(&symbol_obstack, '\0'); + size_t size = (size_t) obstack_object_size(&symbol_obstack) - 1; + char *string = obstack_finish(&symbol_obstack); - const wchar_rep_t *string = obstack_finish(&symbol_obstack); + lexer_token.type = T_WIDE_CHARACTER_CONSTANT; + lexer_token.literal = identify_string(string, size); - lexer_token.type = T_WIDE_CHARACTER_CONSTANT; - lexer_token.v.wide_string.begin = string; - lexer_token.v.wide_string.size = size; - lexer_token.datatype = type_wchar_t; + if (size == 0) { + errorf(&lexer_token.source_position, "empty character constant"); + } } /** @@ -1294,64 +1054,9 @@ end_of_wide_char_constant:; */ static void parse_wide_string_literal(void) { - const unsigned start_linenr = lexer_token.source_position.linenr; - - assert(c == '"'); - next_char(); - - while(1) { - switch(c) { - case '\\': { - wchar_rep_t tc = parse_escape_sequence(); - obstack_grow(&symbol_obstack, &tc, sizeof(tc)); - break; - } - - case EOF: { - source_position_t source_position; - source_position.input_name = lexer_token.source_position.input_name; - source_position.linenr = start_linenr; - errorf(&source_position, "string has no end"); - lexer_token.type = T_ERROR; - return; - } - - case '"': - next_char(); - goto end_of_string; - - default: { - wchar_rep_t tc = c; - obstack_grow(&symbol_obstack, &tc, sizeof(tc)); - next_char(); - break; - } - } - } - -end_of_string:; - - /* TODO: concatenate multiple strings separated by whitespace... */ - - /* add finishing 0 to the string */ - wchar_rep_t nul = L'\0'; - obstack_grow(&symbol_obstack, &nul, sizeof(nul)); - const size_t size = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t); - const wchar_rep_t *const string = obstack_finish(&symbol_obstack); - -#if 0 /* TODO hash */ - /* check if there is already a copy of the string */ - const wchar_rep_t *const result = strset_insert(&stringset, string); - if(result != string) { - obstack_free(&symbol_obstack, string); - } -#else - const wchar_rep_t *const result = string; -#endif - - lexer_token.type = T_WIDE_STRING_LITERAL; - lexer_token.v.wide_string.begin = result; - lexer_token.v.wide_string.size = size; + parse_string_literal(); + if (lexer_token.type == T_STRING_LITERAL) + lexer_token.type = T_WIDE_STRING_LITERAL; } /** @@ -1359,17 +1064,14 @@ end_of_string:; */ static void parse_character_constant(void) { - const unsigned start_linenr = lexer_token.source_position.linenr; - eat('\''); - while(1) { - switch(c) { + while (true) { + switch (c) { case '\\': { utf32 const tc = parse_escape_sequence(); if (tc >= 0x100) { - warningf(&lexer_token.source_position, - "escape sequence out of range"); + warningf(WARN_OTHER, &lexer_pos, "escape sequence out of range"); } obstack_1grow(&symbol_obstack, tc); break; @@ -1385,10 +1087,7 @@ static void parse_character_constant(void) goto end_of_char_constant; case EOF: { - source_position_t source_position; - source_position.input_name = lexer_token.source_position.input_name; - source_position.linenr = start_linenr; - errorf(&source_position, "EOF while parsing character constant"); + errorf(&lexer_token.source_position, "EOF while parsing character constant"); lexer_token.type = T_ERROR; return; } @@ -1402,13 +1101,16 @@ static void parse_character_constant(void) } end_of_char_constant:; - const size_t size = (size_t)obstack_object_size(&symbol_obstack); - const char *const string = obstack_finish(&symbol_obstack); + obstack_1grow(&symbol_obstack, '\0'); + const size_t size = (size_t)obstack_object_size(&symbol_obstack)-1; + char *const string = obstack_finish(&symbol_obstack); + + lexer_token.type = T_CHARACTER_CONSTANT; + lexer_token.literal = identify_string(string, size); - lexer_token.type = T_CHARACTER_CONSTANT; - lexer_token.v.string.begin = string; - lexer_token.v.string.size = size; - lexer_token.datatype = c_mode & _CXX && size == 1 ? type_char : type_int; + if (size == 0) { + errorf(&lexer_token.source_position, "empty character constant"); + } } /** @@ -1416,22 +1118,18 @@ end_of_char_constant:; */ static void skip_multiline_comment(void) { - unsigned start_linenr = lexer_token.source_position.linenr; - - while(1) { - switch(c) { + while (true) { + switch (c) { case '/': next_char(); if (c == '*') { /* nested comment, warn here */ - if (warning.comment) { - warningf(&lexer_token.source_position, "'/*' within comment"); - } + warningf(WARN_COMMENT, &lexer_pos, "'/*' within comment"); } break; case '*': next_char(); - if(c == '/') { + if (c == '/') { next_char(); return; } @@ -1440,10 +1138,7 @@ static void skip_multiline_comment(void) MATCH_NEWLINE(break;) case EOF: { - source_position_t source_position; - source_position.input_name = lexer_token.source_position.input_name; - source_position.linenr = start_linenr; - errorf(&source_position, "at end of file while looking for comment end"); + errorf(&lexer_token.source_position, "at end of file while looking for comment end"); return; } @@ -1459,8 +1154,8 @@ static void skip_multiline_comment(void) */ static void skip_line_comment(void) { - while(1) { - switch(c) { + while (true) { + switch (c) { case EOF: return; @@ -1471,8 +1166,7 @@ static void skip_line_comment(void) case '\\': next_char(); if (c == '\n' || c == '\r') { - if (warning.comment) - warningf(&lexer_token.source_position, "multi-line comment"); + warningf(WARN_COMMENT, &lexer_pos, "multi-line comment"); return; } break; @@ -1501,7 +1195,7 @@ static inline void next_pp_token(void) */ static void eat_until_newline(void) { - while(pp_token.type != '\n' && pp_token.type != T_EOF) { + while (pp_token.type != '\n' && pp_token.type != T_EOF) { next_pp_token(); } } @@ -1512,7 +1206,7 @@ static void eat_until_newline(void) static void define_directive(void) { lexer_next_preprocessing_token(); - if(lexer_token.type != T_IDENTIFIER) { + if (lexer_token.type != T_IDENTIFIER) { parse_error("expected identifier after #define\n"); eat_until_newline(); } @@ -1542,14 +1236,15 @@ static void endif_directive(void) */ static void parse_line_directive(void) { - if(pp_token.type != T_INTEGER) { + if (pp_token.type != T_INTEGER) { parse_error("expected integer"); } else { - lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1); + /* use offset -1 as this is about the next line */ + lexer_pos.lineno = atoi(pp_token.literal.begin) - 1; next_pp_token(); } - if(pp_token.type == T_STRING_LITERAL) { - lexer_token.source_position.input_name = pp_token.v.string.begin; + if (pp_token.type == T_STRING_LITERAL) { + lexer_pos.input_name = pp_token.literal.begin; next_pp_token(); } @@ -1579,17 +1274,18 @@ typedef enum stdc_pragma_value_kind_t { /** * Parse a pragma directive. */ -static void parse_pragma(void) { +static void parse_pragma(void) +{ bool unknown_pragma = true; next_pp_token(); - if (pp_token.v.symbol->pp_ID == TP_STDC) { + if (pp_token.symbol->pp_ID == TP_STDC) { stdc_pragma_kind_t kind = STDC_UNKNOWN; /* a STDC pragma */ if (c_mode & _C99) { next_pp_token(); - switch (pp_token.v.symbol->pp_ID) { + switch (pp_token.symbol->pp_ID) { case TP_FP_CONTRACT: kind = STDC_FP_CONTRACT; break; @@ -1605,7 +1301,7 @@ static void parse_pragma(void) { if (kind != STDC_UNKNOWN) { stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN; next_pp_token(); - switch (pp_token.v.symbol->pp_ID) { + switch (pp_token.symbol->pp_ID) { case TP_ON: value = STDC_VALUE_ON; break; @@ -1629,8 +1325,8 @@ static void parse_pragma(void) { unknown_pragma = true; } eat_until_newline(); - if (unknown_pragma && warning.unknown_pragmas) { - warningf(&pp_token.source_position, "encountered unknown #pragma"); + if (unknown_pragma) { + warningf(WARN_UNKNOWN_PRAGMAS, &pp_token.source_position, "encountered unknown #pragma"); } } @@ -1640,9 +1336,9 @@ static void parse_pragma(void) { static void parse_preprocessor_identifier(void) { assert(pp_token.type == T_IDENTIFIER); - symbol_t *symbol = pp_token.v.symbol; + symbol_t *symbol = pp_token.symbol; - switch(symbol->pp_ID) { + switch (symbol->pp_ID) { case TP_include: printf("include - enable header name parsing!\n"); break; @@ -1683,7 +1379,7 @@ static void parse_preprocessor_directive(void) { next_pp_token(); - switch(pp_token.type) { + switch (pp_token.type) { case T_IDENTIFIER: parse_preprocessor_identifier(); break; @@ -1691,7 +1387,7 @@ static void parse_preprocessor_directive(void) parse_line_directive(); break; case '\n': - /* NULL directive, see § 6.10.7 */ + /* NULL directive, see §6.10.7 */ break; default: parse_error("invalid preprocessor directive"); @@ -1702,8 +1398,8 @@ static void parse_preprocessor_directive(void) #define MAYBE_PROLOG \ next_char(); \ - while(1) { \ - switch(c) { + while (true) { \ + switch (c) { #define MAYBE(ch, set_type) \ case ch: \ @@ -1711,23 +1407,34 @@ static void parse_preprocessor_directive(void) lexer_token.type = set_type; \ return; +/* must use this as last thing */ +#define MAYBE_MODE(ch, set_type, mode) \ + case ch: \ + if (c_mode & mode) { \ + next_char(); \ + lexer_token.type = set_type; \ + return; \ + } \ + /* fallthrough */ + #define ELSE_CODE(code) \ default: \ code \ + return; \ } \ - } /* end of while(1) */ \ - break; + } /* end of while (true) */ \ #define ELSE(set_type) \ ELSE_CODE( \ lexer_token.type = set_type; \ - return; \ ) void lexer_next_preprocessing_token(void) { - while(1) { - switch(c) { + while (true) { + lexer_token.source_position = lexer_pos; + + switch (c) { case ' ': case '\t': next_char(); @@ -1741,12 +1448,10 @@ void lexer_next_preprocessing_token(void) SYMBOL_CHARS parse_symbol(); /* might be a wide string ( L"string" ) */ - if(lexer_token.type == T_IDENTIFIER && - lexer_token.v.symbol == symbol_L) { - if(c == '"') { - parse_wide_string_literal(); - } else if(c == '\'') { - parse_wide_character_constant(); + if (lexer_token.symbol == symbol_L) { + switch (c) { + case '"': parse_wide_string_literal(); break; + case '\'': parse_wide_character_constant(); break; } } return; @@ -1768,7 +1473,7 @@ void lexer_next_preprocessing_token(void) DIGITS put_back(c); c = '.'; - parse_number_dec(); + parse_number(); return; case '.': @@ -1778,7 +1483,6 @@ void lexer_next_preprocessing_token(void) put_back(c); c = '.'; lexer_token.type = '.'; - return; ) ELSE('.') case '&': @@ -1832,7 +1536,6 @@ void lexer_next_preprocessing_token(void) put_back(c); c = '%'; lexer_token.type = '#'; - return; ) ELSE('#') ELSE('%') @@ -1866,6 +1569,7 @@ void lexer_next_preprocessing_token(void) case ':': MAYBE_PROLOG MAYBE('>', ']') + MAYBE_MODE(':', T_COLONCOLON, _CXX) ELSE(':') case '=': MAYBE_PROLOG @@ -1897,7 +1601,7 @@ void lexer_next_preprocessing_token(void) default: dollar_sign: - errorf(&lexer_token.source_position, "unknown character '%c' found", c); + errorf(&lexer_pos, "unknown character '%c' found", c); next_char(); lexer_token.type = T_ERROR; return; @@ -1928,9 +1632,10 @@ void init_lexer(void) void lexer_open_stream(FILE *stream, const char *input_name) { - input = stream; - lexer_token.source_position.linenr = 0; - lexer_token.source_position.input_name = input_name; + input = stream; + lexer_pos.lineno = 0; + lexer_pos.colno = 0; + lexer_pos.input_name = input_name; bufpos = NULL; bufend = NULL; @@ -1942,22 +1647,23 @@ void lexer_open_stream(FILE *stream, const char *input_name) void lexer_open_buffer(const char *buffer, size_t len, const char *input_name) { - input = NULL; - lexer_token.source_position.linenr = 0; - lexer_token.source_position.input_name = input_name; + input = NULL; + lexer_pos.lineno = 0; + lexer_pos.colno = 0; + lexer_pos.input_name = input_name; #if 0 // TODO bufpos = buffer; bufend = buffer + len; + + /* place a virtual \n at the beginning so the lexer knows that we're + * at the beginning of a line */ + c = '\n'; #else (void)buffer; (void)len; panic("builtin lexing not done yet"); #endif - - /* place a virtual \n at the beginning so the lexer knows that we're - * at the beginning of a line */ - c = '\n'; } void exit_lexer(void) @@ -1968,7 +1674,7 @@ void exit_lexer(void) static __attribute__((unused)) void dbg_pos(const source_position_t source_position) { - fprintf(stdout, "%s:%u\n", source_position.input_name, - source_position.linenr); + fprintf(stdout, "%s:%u:%u\n", source_position.input_name, + source_position.lineno, source_position.colno); fflush(stdout); }