X-Git-Url: http://nsz.repo.hu/git/?a=blobdiff_plain;f=lexer.c;h=1f0c23d46b004cb0cea3ed84a7689f3c9c55c5df;hb=34a560b81b8899995ff67421650edcad53efb4d4;hp=aedfdd1dd715caca74df1b13c776ae9afc6f0192;hpb=c20806e328288d52cc08677c818c23ff2a604aea;p=cparser diff --git a/lexer.c b/lexer.c index aedfdd1..1f0c23d 100644 --- a/lexer.c +++ b/lexer.c @@ -1,6 +1,6 @@ /* * This file is part of cparser. - * Copyright (C) 2007-2008 Matthias Braun + * Copyright (C) 2007-2009 Matthias Braun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -19,6 +19,8 @@ */ #include +#include "adt/strutil.h" +#include "input.h" #include "diagnostic.h" #include "lexer.h" #include "symbol_t.h" @@ -29,7 +31,6 @@ #include "adt/util.h" #include "types.h" #include "type_t.h" -#include "target_architecture.h" #include "parser.h" #include "warning.h" #include "lang_features.h" @@ -44,26 +45,19 @@ #include #endif -//#define DEBUG_CHARS -#define MAX_PUTBACK 3 +#define MAX_PUTBACK 16 // 3 would be enough, but 16 gives a nicer alignment #define BUF_SIZE 1024 -#if defined(_WIN32) || defined(__CYGWIN__) -/* No strtold on windows and no replacement yet */ -#define strtold(s, e) strtod(s, e) -#endif - -typedef unsigned int utf32; - -static utf32 c; -token_t lexer_token; -symbol_t *symbol_L; -static FILE *input; -static utf32 buf[BUF_SIZE + MAX_PUTBACK]; -static const utf32 *bufend; -static const utf32 *bufpos; -static strset_t stringset; -bool allow_dollar_in_symbol = true; +static input_t *input; +static utf32 input_buf[BUF_SIZE + MAX_PUTBACK]; +static const utf32 *bufpos; +static const utf32 *bufend; +static utf32 c; +static source_position_t lexer_pos; +token_t lexer_token; +static symbol_t *symbol_L; +static strset_t stringset; +bool allow_dollar_in_symbol = true; /** * Prints a parse error message at the current token. @@ -72,7 +66,7 @@ bool allow_dollar_in_symbol = true; */ static void parse_error(const char *msg) { - errorf(&lexer_token.source_position, "%s", msg); + errorf(&lexer_pos, "%s", msg); } /** @@ -82,281 +76,23 @@ static void parse_error(const char *msg) */ static NORETURN internal_error(const char *msg) { - internal_errorf(&lexer_token.source_position, "%s", msg); -} - -static size_t read_block(unsigned char *const read_buf, size_t const n) -{ - size_t const s = fread(read_buf, 1, n, input); - if (s == 0) { - if (ferror(input)) - parse_error("read from input failed"); - buf[MAX_PUTBACK] = EOF; - bufpos = buf + MAX_PUTBACK; - bufend = buf + MAX_PUTBACK + 1; - } - return s; -} - -static void decode_iso_8859_1(void) -{ - unsigned char read_buf[BUF_SIZE]; - size_t const s = read_block(read_buf, sizeof(read_buf)); - if (s == 0) - return; - - unsigned char const *src = read_buf; - unsigned char const *end = read_buf + s; - utf32 *dst = buf + MAX_PUTBACK; - while (src != end) - *dst++ = *src++; - - bufpos = buf + MAX_PUTBACK; - bufend = dst; -} - -static void decode_iso_8859_15(void) -{ - unsigned char read_buf[BUF_SIZE]; - size_t const s = read_block(read_buf, sizeof(read_buf)); - if (s == 0) - return; - - unsigned char const *src = read_buf; - unsigned char const *end = read_buf + s; - utf32 *dst = buf + MAX_PUTBACK; - while (src != end) { - utf32 tc = *src++; - switch (tc) { - case 0xA4: tc = 0x20AC; break; // € - case 0xA6: tc = 0x0160; break; // Š - case 0xA8: tc = 0x0161; break; // š - case 0xB4: tc = 0x017D; break; // Ž - case 0xB8: tc = 0x017E; break; // ž - case 0xBC: tc = 0x0152; break; // Œ - case 0xBD: tc = 0x0153; break; // œ - case 0xBE: tc = 0x0178; break; // Ÿ - } - *dst++ = tc; - } - - bufpos = buf + MAX_PUTBACK; - bufend = dst; -} - -static void decode_utf8(void) -{ - static utf32 part_decoded_min_code; - static utf32 part_decoded_char; - static size_t part_decoded_rest_len; - - do { - unsigned char read_buf[BUF_SIZE]; - size_t const s = read_block(read_buf, sizeof(read_buf)); - if (s == 0) { - if (part_decoded_rest_len > 0) - parse_error("incomplete input char at end of input"); - return; - } - - unsigned char const *src = read_buf; - unsigned char const *end = read_buf + s; - utf32 *dst = buf + MAX_PUTBACK; - utf32 decoded; - utf32 min_code; - - if (part_decoded_rest_len != 0) { - min_code = part_decoded_min_code; - decoded = part_decoded_char; - size_t const rest_len = part_decoded_rest_len; - part_decoded_rest_len = 0; - switch (rest_len) { - case 4: goto realign; - case 3: goto three_more; - case 2: goto two_more; - default: goto one_more; - } - } - - while (src != end) { - if ((*src & 0x80) == 0) { - decoded = *src++; - } else if ((*src & 0xE0) == 0xC0) { - min_code = 0x80; - decoded = *src++ & 0x1F; -one_more: - if (src == end) { - part_decoded_min_code = min_code; - part_decoded_char = decoded; - part_decoded_rest_len = 1; - break; - } - if ((*src & 0xC0) == 0x80) { - decoded = (decoded << 6) | (*src++ & 0x3F); - } else { - goto invalid_char; - } - if (decoded < min_code || - decoded > 0x10FFFF || - (0xD800 <= decoded && decoded < 0xE000) || // high/low surrogates - (0xFDD0 <= decoded && decoded < 0xFDF0) || // noncharacters - (decoded & 0xFFFE) == 0xFFFE) { // noncharacters - parse_error("invalid byte sequence in input"); - } - } else if ((*src & 0xF0) == 0xE0) { - min_code = 0x800; - decoded = *src++ & 0x0F; -two_more: - if (src == end) { - part_decoded_min_code = min_code; - part_decoded_char = decoded; - part_decoded_rest_len = 2; - break; - } - if ((*src & 0xC0) == 0x80) { - decoded = (decoded << 6) | (*src++ & 0x3F); - } else { - goto invalid_char; - } - goto one_more; - } else if ((*src & 0xF8) == 0xF0) { - min_code = 0x10000; - decoded = *src++ & 0x07; -three_more: - if (src == end) { - part_decoded_min_code = min_code; - part_decoded_char = decoded; - part_decoded_rest_len = 3; - break; - } - if ((*src & 0xC0) == 0x80) { - decoded = (decoded << 6) | (*src++ & 0x3F); - } else { - goto invalid_char; - } - goto two_more; - } else { -invalid_char: - parse_error("invalid byte sequence in input"); -realign: - do { - ++src; - if (src == end) { - part_decoded_rest_len = 4; - break; - } - } while ((*src & 0xC0) == 0x80 || (*src & 0xF8) == 0xF8); - continue; - } - *dst++ = decoded; - } - - bufpos = buf + MAX_PUTBACK; - bufend = dst; - } while (bufpos == bufend); -} - -static void decode_windows_1252(void) -{ - unsigned char read_buf[BUF_SIZE]; - size_t const s = read_block(read_buf, sizeof(read_buf)); - if (s == 0) - return; - - unsigned char const *src = read_buf; - unsigned char const *end = read_buf + s; - utf32 *dst = buf + MAX_PUTBACK; - while (src != end) { - utf32 tc = *src++; - switch (tc) { - case 0x80: tc = 0x20AC; break; // € - case 0x82: tc = 0x201A; break; // ‚ - case 0x83: tc = 0x0192; break; // ƒ - case 0x84: tc = 0x201E; break; // „ - case 0x85: tc = 0x2026; break; // … - case 0x86: tc = 0x2020; break; // † - case 0x87: tc = 0x2021; break; // ‡ - case 0x88: tc = 0x02C6; break; // ˆ - case 0x89: tc = 0x2030; break; // ‰ - case 0x8A: tc = 0x0160; break; // Š - case 0x8B: tc = 0x2039; break; // ‹ - case 0x8C: tc = 0x0152; break; // Œ - case 0x8E: tc = 0x017D; break; // Ž - case 0x91: tc = 0x2018; break; // ‘ - case 0x92: tc = 0x2019; break; // ’ - case 0x93: tc = 0x201C; break; // “ - case 0x94: tc = 0x201D; break; // ” - case 0x95: tc = 0x2022; break; // • - case 0x96: tc = 0x2013; break; // – - case 0x97: tc = 0x2014; break; // — - case 0x98: tc = 0x02DC; break; // ˜ - case 0x99: tc = 0x2122; break; // ™ - case 0x9A: tc = 0x0161; break; // š - case 0x9B: tc = 0x203A; break; // › - case 0x9C: tc = 0x0153; break; // œ - case 0x9E: tc = 0x017E; break; // ž - case 0x9F: tc = 0x0178; break; // Ÿ - } - *dst++ = tc; - } - - bufpos = buf + MAX_PUTBACK; - bufend = dst; -} - -typedef void (*decoder_t)(void); - -static decoder_t decoder = decode_utf8; - -typedef struct named_decoder_t { - char const *name; - decoder_t decoder; -} named_decoder_t; - -static named_decoder_t const decoders[] = { - { "CP819", decode_iso_8859_1 }, // offical alias - { "IBM819", decode_iso_8859_1 }, // offical alias - { "ISO-8859-1", decode_iso_8859_1 }, // offical alias - { "ISO-8859-15", decode_iso_8859_15 }, // offical name - { "ISO8859-1", decode_iso_8859_1 }, - { "ISO8859-15", decode_iso_8859_15 }, - { "ISO_8859-1", decode_iso_8859_1 }, // offical alias - { "ISO_8859-15", decode_iso_8859_15 }, // offical alias - { "ISO_8859-1:1987", decode_iso_8859_1 }, // offical name - { "Latin-9", decode_iso_8859_15 }, // offical alias - { "UTF-8", decode_utf8 }, // offical name - { "csISOLatin1", decode_iso_8859_1 }, // offical alias - { "cp1252", decode_windows_1252 }, - { "iso-ir-100", decode_iso_8859_1 }, // offical alias - { "l1", decode_iso_8859_1 }, // offical alias - { "latin1", decode_iso_8859_1 }, // offical alias - { "windows-1252", decode_windows_1252 }, // official name - - { NULL, NULL } -}; - -void select_input_encoding(char const* const encoding) -{ - for (named_decoder_t const *i = decoders; i->name != NULL; ++i) { - if (strcasecmp(encoding, i->name) != 0) - continue; - decoder = i->decoder; - return; - } - fprintf(stderr, "error: input encoding \"%s\" not supported\n", encoding); + internal_errorf(&lexer_pos, "%s", msg); } static inline void next_real_char(void) { assert(bufpos <= bufend); if (bufpos >= bufend) { - if (input == NULL) { + size_t n = decode(input, input_buf+MAX_PUTBACK, BUF_SIZE); + if (n == 0) { c = EOF; return; } - decoder(); + bufpos = input_buf + MAX_PUTBACK; + bufend = bufpos + n; } c = *bufpos++; + ++lexer_pos.colno; } /** @@ -366,37 +102,33 @@ static inline void next_real_char(void) */ static inline void put_back(utf32 const pc) { - assert(bufpos > buf); - *(--bufpos - buf + buf) = pc; - -#ifdef DEBUG_CHARS - printf("putback '%lc'\n", pc); -#endif + *(--bufpos - input_buf + input_buf) = pc; + --lexer_pos.colno; } static inline void next_char(void); -#define MATCH_NEWLINE(code) \ - case '\r': \ - next_char(); \ - if(c == '\n') { \ - next_char(); \ - } \ - lexer_token.source_position.linenr++; \ - code \ - case '\n': \ - next_char(); \ - lexer_token.source_position.linenr++; \ - code - -#define eat(c_type) do { assert(c == c_type); next_char(); } while(0) +#define NEWLINE \ + '\r': \ + next_char(); \ + if (c == '\n') { \ + case '\n': \ + next_char(); \ + } \ + lexer_pos.lineno++; \ + lexer_pos.colno = 1; \ + goto newline; \ + newline // Let it look like an ordinary case label. + +#define eat(c_type) (assert(c == c_type), next_char()) static void maybe_concat_lines(void) { eat('\\'); - switch(c) { - MATCH_NEWLINE(return;) + switch (c) { + case NEWLINE: + return; default: break; @@ -415,23 +147,23 @@ static inline void next_char(void) next_real_char(); /* filter trigraphs */ - if(UNLIKELY(c == '\\')) { + if (UNLIKELY(c == '\\')) { maybe_concat_lines(); - goto end_of_next_char; + return; } - if(LIKELY(c != '?')) - goto end_of_next_char; + if (LIKELY(c != '?')) + return; next_real_char(); - if(LIKELY(c != '?')) { + if (LIKELY(c != '?')) { put_back(c); c = '?'; - goto end_of_next_char; + return; } next_real_char(); - switch(c) { + switch (c) { case '=': c = '#'; break; case '(': c = '['; break; case '/': c = '\\'; maybe_concat_lines(); break; @@ -447,20 +179,14 @@ static inline void next_char(void) c = '?'; break; } - -end_of_next_char:; -#ifdef DEBUG_CHARS - printf("nchar '%c'\n", c); -#endif } -#define SYMBOL_CHARS \ - case '$': if (!allow_dollar_in_symbol) goto dollar_sign; \ +#define SYMBOL_CHARS_WITHOUT_E_P \ + '$': if (!allow_dollar_in_symbol) goto dollar_sign; \ case 'a': \ case 'b': \ case 'c': \ case 'd': \ - case 'e': \ case 'f': \ case 'g': \ case 'h': \ @@ -471,7 +197,6 @@ end_of_next_char:; case 'm': \ case 'n': \ case 'o': \ - case 'p': \ case 'q': \ case 'r': \ case 's': \ @@ -486,7 +211,6 @@ end_of_next_char:; case 'B': \ case 'C': \ case 'D': \ - case 'E': \ case 'F': \ case 'G': \ case 'H': \ @@ -497,7 +221,6 @@ end_of_next_char:; case 'M': \ case 'N': \ case 'O': \ - case 'P': \ case 'Q': \ case 'R': \ case 'S': \ @@ -508,10 +231,20 @@ end_of_next_char:; case 'X': \ case 'Y': \ case 'Z': \ - case '_': + case '_' + +#define SYMBOL_CHARS_E_P \ + 'E': \ + case 'P': \ + case 'e': \ + case 'p' + +#define SYMBOL_CHARS \ + SYMBOL_CHARS_WITHOUT_E_P: \ + case SYMBOL_CHARS_E_P #define DIGITS \ - case '0': \ + '0': \ case '1': \ case '2': \ case '3': \ @@ -520,481 +253,214 @@ end_of_next_char:; case '6': \ case '7': \ case '8': \ - case '9': + case '9' -/** - * Read a symbol from the input and build - * the lexer_token. - */ -static void parse_symbol(void) +static bool is_universal_char_valid(utf32 const v) { - symbol_t *symbol; - char *string; + /* C11 §6.4.3:2 */ + if (v < 0xA0U && v != 0x24 && v != 0x40 && v != 0x60) + return false; + if (0xD800 <= v && v <= 0xDFFF) + return false; + return true; +} - obstack_1grow(&symbol_obstack, (char) c); - next_char(); +static int digit_value(utf32 digit); - while(1) { - switch(c) { - DIGITS - SYMBOL_CHARS - obstack_1grow(&symbol_obstack, (char) c); +static utf32 parse_universal_char(unsigned const n_digits) +{ + utf32 v = 0; + for (unsigned k = n_digits; k != 0; --k) { + if (isxdigit(c)) { + v = 16 * v + digit_value(c); next_char(); + } else { + errorf(&lexer_pos, "short universal character name, expected %u more digits", k); break; - - default: -dollar_sign: - goto end_symbol; } } - -end_symbol: - obstack_1grow(&symbol_obstack, '\0'); - - string = obstack_finish(&symbol_obstack); - symbol = symbol_table_insert(string); - - lexer_token.type = symbol->ID; - lexer_token.v.symbol = symbol; - - if(symbol->string != string) { - obstack_free(&symbol_obstack, string); + if (!is_universal_char_valid(v)) { + errorf(&lexer_pos, "\\%c%0*X is not a valid universal character name", n_digits == 4 ? 'u' : 'U', (int)n_digits, v); } + return v; } -static void parse_integer_suffix(bool is_oct_hex) +static bool is_universal_char_valid_identifier(utf32 const v) { - bool is_unsigned = false; - bool min_long = false; - bool min_longlong = false; - bool not_traditional = false; - int pos = 0; - char suffix[4]; - - if (c == 'U' || c == 'u') { - not_traditional = true; - suffix[pos++] = toupper(c); - is_unsigned = true; - next_char(); - if (c == 'L' || c == 'l') { - suffix[pos++] = toupper(c); - min_long = true; - next_char(); - if (c == 'L' || c == 'l') { - suffix[pos++] = toupper(c); - min_longlong = true; - next_char(); - } - } - } else if (c == 'l' || c == 'L') { - suffix[pos++] = toupper(c); - min_long = true; - next_char(); - if (c == 'l' || c == 'L') { - not_traditional = true; - suffix[pos++] = toupper(c); - min_longlong = true; - next_char(); - if (c == 'u' || c == 'U') { - suffix[pos++] = toupper(c); - is_unsigned = true; - next_char(); - } - } else if (c == 'u' || c == 'U') { - not_traditional = true; - suffix[pos++] = toupper(c); - is_unsigned = true; - next_char(); - lexer_token.datatype = type_unsigned_long; - } - } - - if (warning.traditional && not_traditional) { - suffix[pos] = '\0'; - warningf(&lexer_token.source_position, - "traditional C rejects the '%s' suffix", suffix); - } - if (!is_unsigned) { - long long v = lexer_token.v.intvalue; - if (!min_long) { - if (v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) { - lexer_token.datatype = type_int; - return; - } else if (is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) { - lexer_token.datatype = type_unsigned_int; - return; - } - } - if (!min_longlong) { - if (v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) { - lexer_token.datatype = type_long; - return; - } else if (is_oct_hex && v >= 0 && (unsigned long long)v <= (unsigned long long)TARGET_ULONG_MAX) { - lexer_token.datatype = type_unsigned_long; - return; - } - } - unsigned long long uv = (unsigned long long) v; - if (is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) { - lexer_token.datatype = type_unsigned_long_long; - return; - } - - lexer_token.datatype = type_long_long; - } else { - unsigned long long v = (unsigned long long) lexer_token.v.intvalue; - if (!min_long && v <= TARGET_UINT_MAX) { - lexer_token.datatype = type_unsigned_int; - return; - } - if (!min_longlong && v <= TARGET_ULONG_MAX) { - lexer_token.datatype = type_unsigned_long; - return; - } - lexer_token.datatype = type_unsigned_long_long; - } + /* C11 Annex D.1 */ + if ( v == 0x000A8) return true; + if ( v == 0x000AA) return true; + if ( v == 0x000AD) return true; + if ( v == 0x000AF) return true; + if (0x000B2 <= v && v <= 0x000B5) return true; + if (0x000B7 <= v && v <= 0x000BA) return true; + if (0x000BC <= v && v <= 0x000BE) return true; + if (0x000C0 <= v && v <= 0x000D6) return true; + if (0x000D8 <= v && v <= 0x000F6) return true; + if (0x000F8 <= v && v <= 0x000FF) return true; + if (0x00100 <= v && v <= 0x0167F) return true; + if (0x01681 <= v && v <= 0x0180D) return true; + if (0x0180F <= v && v <= 0x01FFF) return true; + if (0x0200B <= v && v <= 0x0200D) return true; + if (0x0202A <= v && v <= 0x0202E) return true; + if (0x0203F <= v && v <= 0x02040) return true; + if ( v == 0x02054) return true; + if (0x02060 <= v && v <= 0x0206F) return true; + if (0x02070 <= v && v <= 0x0218F) return true; + if (0x02460 <= v && v <= 0x024FF) return true; + if (0x02776 <= v && v <= 0x02793) return true; + if (0x02C00 <= v && v <= 0x02DFF) return true; + if (0x02E80 <= v && v <= 0x02FFF) return true; + if (0x03004 <= v && v <= 0x03007) return true; + if (0x03021 <= v && v <= 0x0302F) return true; + if (0x03031 <= v && v <= 0x0303F) return true; + if (0x03040 <= v && v <= 0x0D7FF) return true; + if (0x0F900 <= v && v <= 0x0FD3D) return true; + if (0x0FD40 <= v && v <= 0x0FDCF) return true; + if (0x0FDF0 <= v && v <= 0x0FE44) return true; + if (0x0FE47 <= v && v <= 0x0FFFD) return true; + if (0x10000 <= v && v <= 0x1FFFD) return true; + if (0x20000 <= v && v <= 0x2FFFD) return true; + if (0x30000 <= v && v <= 0x3FFFD) return true; + if (0x40000 <= v && v <= 0x4FFFD) return true; + if (0x50000 <= v && v <= 0x5FFFD) return true; + if (0x60000 <= v && v <= 0x6FFFD) return true; + if (0x70000 <= v && v <= 0x7FFFD) return true; + if (0x80000 <= v && v <= 0x8FFFD) return true; + if (0x90000 <= v && v <= 0x9FFFD) return true; + if (0xA0000 <= v && v <= 0xAFFFD) return true; + if (0xB0000 <= v && v <= 0xBFFFD) return true; + if (0xC0000 <= v && v <= 0xCFFFD) return true; + if (0xD0000 <= v && v <= 0xDFFFD) return true; + if (0xE0000 <= v && v <= 0xEFFFD) return true; + return false; } -static void parse_floating_suffix(void) +static bool is_universal_char_valid_identifier_start(utf32 const v) { - switch(c) { - /* TODO: do something useful with the suffixes... */ - case 'f': - case 'F': - if (warning.traditional) { - warningf(&lexer_token.source_position, - "traditional C rejects the 'F' suffix"); - } - next_char(); - lexer_token.datatype = type_float; - break; - case 'l': - case 'L': - if (warning.traditional) { - warningf(&lexer_token.source_position, - "traditional C rejects the 'F' suffix"); - } - next_char(); - lexer_token.datatype = type_long_double; - break; - default: - lexer_token.datatype = type_double; - break; - } + /* C11 Annex D.2 */ + if (0x0300 <= v && v <= 0x036F) return false; + if (0x1DC0 <= v && v <= 0x1DFF) return false; + if (0x20D0 <= v && v <= 0x20FF) return false; + if (0xFE20 <= v && v <= 0xFE2F) return false; + return true; } /** - * A replacement for strtoull. Only those parts needed for - * our parser are implemented. + * Read a symbol from the input and build + * the lexer_token. */ -static unsigned long long parse_int_string(const char *s, const char **endptr, int base) +static void parse_symbol(void) { - unsigned long long v = 0; + while (true) { + switch (c) { + case DIGITS: + case SYMBOL_CHARS: + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + break; - switch (base) { - case 16: - for (;; ++s) { - /* check for overrun */ - if (v >= 0x1000000000000000ULL) - break; - switch (tolower(*s)) { - case '0': v <<= 4; break; - case '1': v <<= 4; v |= 0x1; break; - case '2': v <<= 4; v |= 0x2; break; - case '3': v <<= 4; v |= 0x3; break; - case '4': v <<= 4; v |= 0x4; break; - case '5': v <<= 4; v |= 0x5; break; - case '6': v <<= 4; v |= 0x6; break; - case '7': v <<= 4; v |= 0x7; break; - case '8': v <<= 4; v |= 0x8; break; - case '9': v <<= 4; v |= 0x9; break; - case 'a': v <<= 4; v |= 0xa; break; - case 'b': v <<= 4; v |= 0xb; break; - case 'c': v <<= 4; v |= 0xc; break; - case 'd': v <<= 4; v |= 0xd; break; - case 'e': v <<= 4; v |= 0xe; break; - case 'f': v <<= 4; v |= 0xf; break; - default: - goto end; - } - } - break; - case 8: - for (;; ++s) { - /* check for overrun */ - if (v >= 0x2000000000000000ULL) + case '\\': + next_char(); + switch (c) { + { + unsigned n; + case 'U': n = 8; goto universal; + case 'u': n = 4; goto universal; +universal: + next_char(); + utf32 const v = parse_universal_char(n); + if (!is_universal_char_valid_identifier(v)) { + if (is_universal_char_valid(v)) { + errorf(&lexer_pos, "universal character \\%c%0*X is not valid in an identifier", n == 4 ? 'u' : 'U', (int)n, v); + } + } else if (obstack_object_size(&symbol_obstack) == 0 && !is_universal_char_valid_identifier_start(v)) { + errorf(&lexer_pos, "universal character \\%c%0*X is not valid as start of an identifier", n == 4 ? 'u' : 'U', (int)n, v); + } else { + obstack_grow_utf8(&symbol_obstack, v); + } break; - switch (tolower(*s)) { - case '0': v <<= 3; break; - case '1': v <<= 3; v |= 1; break; - case '2': v <<= 3; v |= 2; break; - case '3': v <<= 3; v |= 3; break; - case '4': v <<= 3; v |= 4; break; - case '5': v <<= 3; v |= 5; break; - case '6': v <<= 3; v |= 6; break; - case '7': v <<= 3; v |= 7; break; - default: - goto end; } - } - break; - case 10: - for (;; ++s) { - /* check for overrun */ - if (v > 0x1999999999999999ULL) - break; - switch (tolower(*s)) { - case '0': v *= 10; break; - case '1': v *= 10; v += 1; break; - case '2': v *= 10; v += 2; break; - case '3': v *= 10; v += 3; break; - case '4': v *= 10; v += 4; break; - case '5': v *= 10; v += 5; break; - case '6': v *= 10; v += 6; break; - case '7': v *= 10; v += 7; break; - case '8': v *= 10; v += 8; break; - case '9': v *= 10; v += 9; break; + default: - goto end; + put_back(c); + c = '\\'; + goto end_symbol; } - } - break; - default: - assert(0); - break; - } -end: - *endptr = s; - return v; -} -/** - * Parses a hex number including hex floats and set the - * lexer_token. - */ -static void parse_number_hex(void) -{ - bool is_float = false; - assert(c == 'x' || c == 'X'); - next_char(); - - obstack_1grow(&symbol_obstack, '0'); - obstack_1grow(&symbol_obstack, 'x'); - - while(isxdigit(c)) { - obstack_1grow(&symbol_obstack, (char) c); - next_char(); - } - - if (c == '.') { - obstack_1grow(&symbol_obstack, (char) c); - next_char(); - - while (isxdigit(c)) { - obstack_1grow(&symbol_obstack, (char) c); - next_char(); - } - is_float = true; - } - if (c == 'p' || c == 'P') { - obstack_1grow(&symbol_obstack, (char) c); - next_char(); - - if (c == '-' || c == '+') { - obstack_1grow(&symbol_obstack, (char) c); - next_char(); - } - - while (isxdigit(c)) { - obstack_1grow(&symbol_obstack, (char) c); - next_char(); + default: +dollar_sign: + goto end_symbol; } - is_float = true; } +end_symbol: obstack_1grow(&symbol_obstack, '\0'); - char *string = obstack_finish(&symbol_obstack); - if(*string == '\0') { - parse_error("invalid hex number"); - lexer_token.type = T_ERROR; - obstack_free(&symbol_obstack, string); - return; - } - if (is_float) { - char *endptr; - lexer_token.type = T_FLOATINGPOINT; - lexer_token.v.floatvalue = strtold(string, &endptr); + char *string = obstack_finish(&symbol_obstack); + symbol_t *symbol = symbol_table_insert(string); - if(*endptr != '\0') { - parse_error("invalid hex float literal"); - } + lexer_token.kind = symbol->ID; + lexer_token.base.symbol = symbol; - parse_floating_suffix(); - } else { - const char *endptr; - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = parse_int_string(string + 2, &endptr, 16); - if(*endptr != '\0') { - parse_error("hex number literal too long"); - } - parse_integer_suffix(true); - } - - obstack_free(&symbol_obstack, string); -} - -/** - * Returns true if the given char is a octal digit. - * - * @param char the character to check - */ -static inline bool is_octal_digit(utf32 chr) -{ - switch(chr) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - return true; - default: - return false; - } -} - -/** - * Parses a octal number and set the lexer_token. - */ -static void parse_number_oct(void) -{ - while(is_octal_digit(c)) { - obstack_1grow(&symbol_obstack, (char) c); - next_char(); - } - obstack_1grow(&symbol_obstack, '\0'); - char *string = obstack_finish(&symbol_obstack); - - const char *endptr; - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = parse_int_string(string, &endptr, 8); - if(*endptr != '\0') { - parse_error("octal number literal too long"); + if (symbol->string != string) { + obstack_free(&symbol_obstack, string); } - - obstack_free(&symbol_obstack, string); - parse_integer_suffix(true); } -/** - * Parses a decimal including float number and set the - * lexer_token. - */ -static void parse_number_dec(void) +static string_t sym_make_string(string_encoding_t const enc) { - bool is_float = false; - while (isdigit(c)) { - obstack_1grow(&symbol_obstack, (char) c); - next_char(); - } - - if (c == '.') { - obstack_1grow(&symbol_obstack, '.'); - next_char(); - - while (isdigit(c)) { - obstack_1grow(&symbol_obstack, (char) c); - next_char(); - } - is_float = true; - } - if(c == 'e' || c == 'E') { - obstack_1grow(&symbol_obstack, (char) c); - next_char(); - - if(c == '-' || c == '+') { - obstack_1grow(&symbol_obstack, (char) c); - next_char(); - } - - while(isdigit(c)) { - obstack_1grow(&symbol_obstack, (char) c); - next_char(); - } - is_float = true; - } - obstack_1grow(&symbol_obstack, '\0'); - char *string = obstack_finish(&symbol_obstack); - - if(is_float) { - char *endptr; - lexer_token.type = T_FLOATINGPOINT; - lexer_token.v.floatvalue = strtold(string, &endptr); - - if(*endptr != '\0') { - parse_error("invalid number literal"); - } - - parse_floating_suffix(); - } else { - const char *endptr; - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = parse_int_string(string, &endptr, 10); - - if(*endptr != '\0') { - parse_error("invalid number literal"); - } + size_t const len = obstack_object_size(&symbol_obstack) - 1; + char *const string = obstack_finish(&symbol_obstack); - parse_integer_suffix(false); + /* TODO hash */ +#if 0 + const char *result = strset_insert(&stringset, concat); + if (result != concat) { + obstack_free(&symbol_obstack, concat); } - obstack_free(&symbol_obstack, string); +#else + const char *result = string; +#endif + return (string_t){ result, len, enc }; } -/** - * Parses a number and sets the lexer_token. - */ -static void parse_number(void) +static void parse_pp_number(void) { - if (c == '0') { - next_char(); + for (;;) { switch (c) { - case 'X': - case 'x': - parse_number_hex(); - break; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - parse_number_oct(); - break; - case '8': - case '9': + case SYMBOL_CHARS_E_P: + obstack_1grow(&symbol_obstack, (char)c); + next_char(); + if (c == '+' || c == '-') { + case '.': + case DIGITS: + case SYMBOL_CHARS_WITHOUT_E_P: + obstack_1grow(&symbol_obstack, (char)c); next_char(); - parse_error("invalid octal number"); - lexer_token.type = T_ERROR; - return; - case '.': - case 'e': - case 'E': - default: - obstack_1grow(&symbol_obstack, '0'); - parse_number_dec(); - return; + } + break; + + default: +dollar_sign: + lexer_token.kind = T_NUMBER; + lexer_token.literal.string = sym_make_string(STRING_ENCODING_CHAR); + return; } - } else { - parse_number_dec(); } } +/** + * Returns true if the given char is a octal digit. + * + * @param char the character to check + */ +static bool is_octal_digit(utf32 chr) +{ + return '0' <= chr && chr <= '7'; +} + /** * Returns the value of a digit. * The only portable way to do it ... @@ -1053,7 +519,7 @@ static utf32 parse_octal_sequence(utf32 const first_digit) static utf32 parse_hex_sequence(void) { utf32 value = 0; - while(isxdigit(c)) { + while (isxdigit(c)) { value = 16 * value + digit_value(c); next_char(); } @@ -1102,367 +568,86 @@ static utf32 parse_escape_sequence(void) case 'e': if (c_mode & _GNUC) return 27; /* hopefully 27 is ALWAYS the code for ESCAPE */ - /* FALLTHROUGH */ - default: - /* §6.4.4.4:8 footnote 64 */ - parse_error("unknown escape sequence"); - return EOF; - } -} - -/** - * Concatenate two strings. - */ -string_t concat_strings(const string_t *const s1, const string_t *const s2) -{ - const size_t len1 = s1->size - 1; - const size_t len2 = s2->size - 1; - - char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1); - memcpy(concat, s1->begin, len1); - memcpy(concat + len1, s2->begin, len2 + 1); - - if (warning.traditional) { - warningf(&lexer_token.source_position, - "traditional C rejects string constant concatenation"); - } -#if 0 /* TODO hash */ - const char *result = strset_insert(&stringset, concat); - if(result != concat) { - obstack_free(&symbol_obstack, concat); - } - - return result; -#else - return (string_t){ concat, len1 + len2 + 1 }; -#endif -} - -/** - * Concatenate a string and a wide string. - */ -wide_string_t concat_string_wide_string(const string_t *const s1, const wide_string_t *const s2) -{ - const size_t len1 = s1->size - 1; - const size_t len2 = s2->size - 1; - - wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat)); - const char *const src = s1->begin; - for (size_t i = 0; i != len1; ++i) { - concat[i] = src[i]; - } - memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat)); - if (warning.traditional) { - warningf(&lexer_token.source_position, - "traditional C rejects string constant concatenation"); - } - - return (wide_string_t){ concat, len1 + len2 + 1 }; -} - -/** - * Concatenate two wide strings. - */ -wide_string_t concat_wide_strings(const wide_string_t *const s1, const wide_string_t *const s2) -{ - const size_t len1 = s1->size - 1; - const size_t len2 = s2->size - 1; - - wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat)); - memcpy(concat, s1->begin, len1 * sizeof(*concat)); - memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat)); - if (warning.traditional) { - warningf(&lexer_token.source_position, - "traditional C rejects string constant concatenation"); - } + break; - return (wide_string_t){ concat, len1 + len2 + 1 }; -} + case 'U': return parse_universal_char(8); + case 'u': return parse_universal_char(4); -/** - * Concatenate a wide string and a string. - */ -wide_string_t concat_wide_string_string(const wide_string_t *const s1, const string_t *const s2) -{ - const size_t len1 = s1->size - 1; - const size_t len2 = s2->size - 1; - - wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat)); - memcpy(concat, s1->begin, len1 * sizeof(*concat)); - const char *const src = s2->begin; - wchar_rep_t *const dst = concat + len1; - for (size_t i = 0; i != len2 + 1; ++i) { - dst[i] = src[i]; - } - if (warning.traditional) { - warningf(&lexer_token.source_position, - "traditional C rejects string constant concatenation"); + default: + break; } - - return (wide_string_t){ concat, len1 + len2 + 1 }; + /* §6.4.4.4:8 footnote 64 */ + parse_error("unknown escape sequence"); + return EOF; } -static void grow_symbol(utf32 const tc) +string_t make_string(const char *string) { - struct obstack *const o = &symbol_obstack; - if (tc < 0x80U) { - obstack_1grow(o, tc); - } else if (tc < 0x800) { - obstack_1grow(o, 0xC0 | (tc >> 6)); - obstack_1grow(o, 0x80 | (tc & 0x3F)); - } else if (tc < 0x10000) { - obstack_1grow(o, 0xE0 | ( tc >> 12)); - obstack_1grow(o, 0x80 | ((tc >> 6) & 0x3F)); - obstack_1grow(o, 0x80 | ( tc & 0x3F)); - } else { - obstack_1grow(o, 0xF0 | ( tc >> 18)); - obstack_1grow(o, 0x80 | ((tc >> 12) & 0x3F)); - obstack_1grow(o, 0x80 | ((tc >> 6) & 0x3F)); - obstack_1grow(o, 0x80 | ( tc & 0x3F)); - } + obstack_grow(&symbol_obstack, string, strlen(string)); + return sym_make_string(STRING_ENCODING_CHAR); } -/** - * Parse a string literal and set lexer_token. - */ -static void parse_string_literal(void) +static void parse_string(utf32 const delim, token_kind_t const kind, string_encoding_t const enc, char const *const context) { - const unsigned start_linenr = lexer_token.source_position.linenr; - - eat('"'); + eat(delim); - while(1) { - switch(c) { + while (true) { + switch (c) { case '\\': { utf32 const tc = parse_escape_sequence(); - if (tc >= 0x100) { - warningf(&lexer_token.source_position, - "escape sequence out of range"); + if (enc == STRING_ENCODING_CHAR) { + if (tc >= 0x100) { + warningf(WARN_OTHER, &lexer_pos, "escape sequence out of range"); + } + obstack_1grow(&symbol_obstack, tc); + } else { + obstack_grow_utf8(&symbol_obstack, tc); } - obstack_1grow(&symbol_obstack, tc); break; } - case EOF: { - source_position_t source_position; - source_position.input_name = lexer_token.source_position.input_name; - source_position.linenr = start_linenr; - errorf(&source_position, "string has no end"); - lexer_token.type = T_ERROR; - return; - } + case NEWLINE: + errorf(&lexer_pos, "newline while parsing %s", context); + break; - case '"': - next_char(); + case EOF: + errorf(&lexer_token.base.source_position, "EOF while parsing %s", context); goto end_of_string; default: - grow_symbol(c); - next_char(); - break; + if (c == delim) { + next_char(); + goto end_of_string; + } else { + obstack_grow_utf8(&symbol_obstack, c); + next_char(); + break; + } } } end_of_string: - - /* TODO: concatenate multiple strings separated by whitespace... */ - - /* add finishing 0 to the string */ - obstack_1grow(&symbol_obstack, '\0'); - const size_t size = (size_t)obstack_object_size(&symbol_obstack); - const char *const string = obstack_finish(&symbol_obstack); - -#if 0 /* TODO hash */ - /* check if there is already a copy of the string */ - result = strset_insert(&stringset, string); - if(result != string) { - obstack_free(&symbol_obstack, string); - } -#else - const char *const result = string; -#endif - - lexer_token.type = T_STRING_LITERAL; - lexer_token.v.string.begin = result; - lexer_token.v.string.size = size; -} - -/** - * Parse a wide character constant and set lexer_token. - */ -static void parse_wide_character_constant(void) -{ - const unsigned start_linenr = lexer_token.source_position.linenr; - - eat('\''); - - while(1) { - switch(c) { - case '\\': { - wchar_rep_t tc = parse_escape_sequence(); - obstack_grow(&symbol_obstack, &tc, sizeof(tc)); - break; - } - - MATCH_NEWLINE( - parse_error("newline while parsing character constant"); - break; - ) - - case '\'': - next_char(); - goto end_of_wide_char_constant; - - case EOF: { - source_position_t source_position = lexer_token.source_position; - source_position.linenr = start_linenr; - errorf(&source_position, "EOF while parsing character constant"); - lexer_token.type = T_ERROR; - return; - } - - default: { - wchar_rep_t tc = (wchar_rep_t) c; - obstack_grow(&symbol_obstack, &tc, sizeof(tc)); - next_char(); - break; - } - } - } - -end_of_wide_char_constant:; - size_t size = (size_t) obstack_object_size(&symbol_obstack); - assert(size % sizeof(wchar_rep_t) == 0); - size /= sizeof(wchar_rep_t); - - const wchar_rep_t *string = obstack_finish(&symbol_obstack); - - lexer_token.type = T_WIDE_CHARACTER_CONSTANT; - lexer_token.v.wide_string.begin = string; - lexer_token.v.wide_string.size = size; - lexer_token.datatype = type_wchar_t; + lexer_token.kind = kind; + lexer_token.literal.string = sym_make_string(enc); } /** - * Parse a wide string literal and set lexer_token. + * Parse a string literal and set lexer_token. */ -static void parse_wide_string_literal(void) +static void parse_string_literal(string_encoding_t const enc) { - const unsigned start_linenr = lexer_token.source_position.linenr; - - assert(c == '"'); - next_char(); - - while(1) { - switch(c) { - case '\\': { - wchar_rep_t tc = parse_escape_sequence(); - obstack_grow(&symbol_obstack, &tc, sizeof(tc)); - break; - } - - case EOF: { - source_position_t source_position; - source_position.input_name = lexer_token.source_position.input_name; - source_position.linenr = start_linenr; - errorf(&source_position, "string has no end"); - lexer_token.type = T_ERROR; - return; - } - - case '"': - next_char(); - goto end_of_string; - - default: { - wchar_rep_t tc = c; - obstack_grow(&symbol_obstack, &tc, sizeof(tc)); - next_char(); - break; - } - } - } - -end_of_string:; - - /* TODO: concatenate multiple strings separated by whitespace... */ - - /* add finishing 0 to the string */ - wchar_rep_t nul = L'\0'; - obstack_grow(&symbol_obstack, &nul, sizeof(nul)); - const size_t size = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t); - const wchar_rep_t *const string = obstack_finish(&symbol_obstack); - -#if 0 /* TODO hash */ - /* check if there is already a copy of the string */ - const wchar_rep_t *const result = strset_insert(&stringset, string); - if(result != string) { - obstack_free(&symbol_obstack, string); - } -#else - const wchar_rep_t *const result = string; -#endif - - lexer_token.type = T_WIDE_STRING_LITERAL; - lexer_token.v.wide_string.begin = result; - lexer_token.v.wide_string.size = size; + parse_string('"', T_STRING_LITERAL, enc, "string literal"); } /** * Parse a character constant and set lexer_token. */ -static void parse_character_constant(void) +static void parse_character_constant(string_encoding_t const enc) { - const unsigned start_linenr = lexer_token.source_position.linenr; - - eat('\''); - - while(1) { - switch(c) { - case '\\': { - utf32 const tc = parse_escape_sequence(); - if (tc >= 0x100) { - warningf(&lexer_token.source_position, - "escape sequence out of range"); - } - obstack_1grow(&symbol_obstack, tc); - break; - } - - MATCH_NEWLINE( - parse_error("newline while parsing character constant"); - break; - ) - - case '\'': - next_char(); - goto end_of_char_constant; - - case EOF: { - source_position_t source_position; - source_position.input_name = lexer_token.source_position.input_name; - source_position.linenr = start_linenr; - errorf(&source_position, "EOF while parsing character constant"); - lexer_token.type = T_ERROR; - return; - } - - default: - grow_symbol(c); - next_char(); - break; - - } + parse_string('\'', T_CHARACTER_CONSTANT, enc, "character constant"); + if (lexer_token.literal.string.size == 0) { + errorf(&lexer_token.base.source_position, "empty character constant"); } - -end_of_char_constant:; - const size_t size = (size_t)obstack_object_size(&symbol_obstack); - const char *const string = obstack_finish(&symbol_obstack); - - lexer_token.type = T_CHARACTER_CONSTANT; - lexer_token.v.string.begin = string; - lexer_token.v.string.size = size; - lexer_token.datatype = c_mode & _CXX && size == 1 ? type_char : type_int; } /** @@ -1470,34 +655,29 @@ end_of_char_constant:; */ static void skip_multiline_comment(void) { - unsigned start_linenr = lexer_token.source_position.linenr; - - while(1) { - switch(c) { + while (true) { + switch (c) { case '/': next_char(); if (c == '*') { /* nested comment, warn here */ - if (warning.comment) { - warningf(&lexer_token.source_position, "'/*' within comment"); - } + warningf(WARN_COMMENT, &lexer_pos, "'/*' within comment"); } break; case '*': next_char(); - if(c == '/') { + if (c == '/') { next_char(); return; } break; - MATCH_NEWLINE(break;) + case NEWLINE: + break; case EOF: { - source_position_t source_position; - source_position.input_name = lexer_token.source_position.input_name; - source_position.linenr = start_linenr; - errorf(&source_position, "at end of file while looking for comment end"); + errorf(&lexer_token.base.source_position, + "at end of file while looking for comment end"); return; } @@ -1513,8 +693,8 @@ static void skip_multiline_comment(void) */ static void skip_line_comment(void) { - while(1) { - switch(c) { + while (true) { + switch (c) { case EOF: return; @@ -1525,8 +705,7 @@ static void skip_line_comment(void) case '\\': next_char(); if (c == '\n' || c == '\r') { - if (warning.comment) - warningf(&lexer_token.source_position, "multi-line comment"); + warningf(WARN_COMMENT, &lexer_pos, "multi-line comment"); return; } break; @@ -1555,56 +734,49 @@ static inline void next_pp_token(void) */ static void eat_until_newline(void) { - while(pp_token.type != '\n' && pp_token.type != T_EOF) { + while (pp_token.kind != '\n' && pp_token.kind != T_EOF) { next_pp_token(); } } -/** - * Handle the define directive. - */ -static void define_directive(void) -{ - lexer_next_preprocessing_token(); - if(lexer_token.type != T_IDENTIFIER) { - parse_error("expected identifier after #define\n"); - eat_until_newline(); - } -} - -/** - * Handle the ifdef directive. - */ -static void ifdef_directive(int is_ifndef) -{ - (void) is_ifndef; - lexer_next_preprocessing_token(); - //expect_identifier(); - //extect_newline(); -} - -/** - * Handle the endif directive. - */ -static void endif_directive(void) -{ - //expect_newline(); -} - /** * Parse the line directive. */ static void parse_line_directive(void) { - if(pp_token.type != T_INTEGER) { + if (pp_token.kind != T_NUMBER) { parse_error("expected integer"); } else { - lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1); + /* use offset -1 as this is about the next line */ + char *end; + long const line = strtol(pp_token.literal.string.begin, &end, 0); + if (*end == '\0') { + lexer_pos.lineno = line - 1; + } else { + errorf(&lexer_pos, "'%S' is not a valid line number", &pp_token.literal.string); + } next_pp_token(); } - if(pp_token.type == T_STRING_LITERAL) { - lexer_token.source_position.input_name = pp_token.v.string.begin; + if (pp_token.kind == T_STRING_LITERAL && pp_token.literal.string.encoding == STRING_ENCODING_CHAR) { + lexer_pos.input_name = pp_token.literal.string.begin; + lexer_pos.is_system_header = false; next_pp_token(); + + /* attempt to parse numeric flags as outputted by gcc preprocessor */ + while (pp_token.kind == T_NUMBER) { + /* flags: + * 1 - indicates start of a new file + * 2 - indicates return from a file + * 3 - indicates system header + * 4 - indicates implicit extern "C" in C++ mode + * + * currently we're only interested in "3" + */ + if (streq(pp_token.literal.string.begin, "3")) { + lexer_pos.is_system_header = true; + } + next_pp_token(); + } } eat_until_newline(); @@ -1635,57 +807,44 @@ typedef enum stdc_pragma_value_kind_t { */ static void parse_pragma(void) { - bool unknown_pragma = true; - next_pp_token(); - if (pp_token.v.symbol->pp_ID == TP_STDC) { - stdc_pragma_kind_t kind = STDC_UNKNOWN; + if (pp_token.kind != T_IDENTIFIER) { + warningf(WARN_UNKNOWN_PRAGMAS, &pp_token.base.source_position, + "expected identifier after #pragma"); + eat_until_newline(); + return; + } + + stdc_pragma_kind_t kind = STDC_UNKNOWN; + if (pp_token.base.symbol->pp_ID == TP_STDC && c_mode & _C99) { /* a STDC pragma */ - if (c_mode & _C99) { - next_pp_token(); + next_pp_token(); - switch (pp_token.v.symbol->pp_ID) { - case TP_FP_CONTRACT: - kind = STDC_FP_CONTRACT; - break; - case TP_FENV_ACCESS: - kind = STDC_FENV_ACCESS; - break; - case TP_CX_LIMITED_RANGE: - kind = STDC_CX_LIMITED_RANGE; - break; - default: - break; + switch (pp_token.base.symbol->pp_ID) { + case TP_FP_CONTRACT: kind = STDC_FP_CONTRACT; break; + case TP_FENV_ACCESS: kind = STDC_FENV_ACCESS; break; + case TP_CX_LIMITED_RANGE: kind = STDC_CX_LIMITED_RANGE; break; + default: break; + } + if (kind != STDC_UNKNOWN) { + next_pp_token(); + stdc_pragma_value_kind_t value; + switch (pp_token.base.symbol->pp_ID) { + case TP_ON: value = STDC_VALUE_ON; break; + case TP_OFF: value = STDC_VALUE_OFF; break; + case TP_DEFAULT: value = STDC_VALUE_DEFAULT; break; + default: value = STDC_VALUE_UNKNOWN; break; } - if (kind != STDC_UNKNOWN) { - stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN; - next_pp_token(); - switch (pp_token.v.symbol->pp_ID) { - case TP_ON: - value = STDC_VALUE_ON; - break; - case TP_OFF: - value = STDC_VALUE_OFF; - break; - case TP_DEFAULT: - value = STDC_VALUE_DEFAULT; - break; - default: - break; - } - if (value != STDC_VALUE_UNKNOWN) { - unknown_pragma = false; - } else { - errorf(&pp_token.source_position, "bad STDC pragma argument"); - } + if (value == STDC_VALUE_UNKNOWN) { + kind = STDC_UNKNOWN; + errorf(&pp_token.base.source_position, "bad STDC pragma argument"); } } - } else { - unknown_pragma = true; } eat_until_newline(); - if (unknown_pragma && warning.unknown_pragmas) { - warningf(&pp_token.source_position, "encountered unknown #pragma"); + if (kind == STDC_UNKNOWN) { + warningf(WARN_UNKNOWN_PRAGMAS, &pp_token.base.source_position, + "encountered unknown #pragma"); } } @@ -1694,40 +853,19 @@ static void parse_pragma(void) */ static void parse_preprocessor_identifier(void) { - assert(pp_token.type == T_IDENTIFIER); - symbol_t *symbol = pp_token.v.symbol; - - switch(symbol->pp_ID) { - case TP_include: - printf("include - enable header name parsing!\n"); - break; - case TP_define: - define_directive(); - break; - case TP_ifdef: - ifdef_directive(0); - break; - case TP_ifndef: - ifdef_directive(1); - break; - case TP_endif: - endif_directive(); - break; + assert(pp_token.kind == T_IDENTIFIER); + switch (pp_token.base.symbol->pp_ID) { case TP_line: next_pp_token(); parse_line_directive(); break; - case TP_if: - case TP_else: - case TP_elif: - case TP_undef: - case TP_error: - /* TODO; output the rest of the line */ - parse_error("#error directive: "); - break; case TP_pragma: parse_pragma(); break; + case TP_error: + /* TODO; output the rest of the line */ + parse_error("#error directive"); + break; } } @@ -1738,11 +876,11 @@ static void parse_preprocessor_directive(void) { next_pp_token(); - switch(pp_token.type) { + switch (pp_token.kind) { case T_IDENTIFIER: parse_preprocessor_identifier(); break; - case T_INTEGER: + case T_NUMBER: parse_line_directive(); break; case '\n': @@ -1757,73 +895,84 @@ static void parse_preprocessor_directive(void) #define MAYBE_PROLOG \ next_char(); \ - while(1) { \ - switch(c) { + while (true) { \ + switch (c) { #define MAYBE(ch, set_type) \ case ch: \ next_char(); \ - lexer_token.type = set_type; \ + lexer_token.kind = set_type; \ return; +/* must use this as last thing */ +#define MAYBE_MODE(ch, set_type, mode) \ + case ch: \ + if (c_mode & mode) { \ + next_char(); \ + lexer_token.kind = set_type; \ + return; \ + } \ + /* fallthrough */ + #define ELSE_CODE(code) \ default: \ code \ + return; \ } \ - } /* end of while(1) */ \ - break; + } /* end of while (true) */ \ #define ELSE(set_type) \ ELSE_CODE( \ - lexer_token.type = set_type; \ - return; \ + lexer_token.kind = set_type; \ ) void lexer_next_preprocessing_token(void) { - while(1) { - switch(c) { + while (true) { + lexer_token.base.source_position = lexer_pos; + lexer_token.base.symbol = NULL; + + switch (c) { case ' ': case '\t': next_char(); break; - MATCH_NEWLINE( - lexer_token.type = '\n'; + case NEWLINE: + lexer_token.kind = '\n'; return; - ) - SYMBOL_CHARS + case SYMBOL_CHARS: { parse_symbol(); /* might be a wide string ( L"string" ) */ - if(lexer_token.type == T_IDENTIFIER && - lexer_token.v.symbol == symbol_L) { - if(c == '"') { - parse_wide_string_literal(); - } else if(c == '\'') { - parse_wide_character_constant(); + string_encoding_t const enc = STRING_ENCODING_WIDE; + if (lexer_token.base.symbol == symbol_L) { + switch (c) { + case '"': parse_string_literal(enc); break; + case '\'': parse_character_constant(enc); break; } } return; + } - DIGITS - parse_number(); + case DIGITS: + parse_pp_number(); return; case '"': - parse_string_literal(); + parse_string_literal(STRING_ENCODING_CHAR); return; case '\'': - parse_character_constant(); + parse_character_constant(STRING_ENCODING_CHAR); return; case '.': MAYBE_PROLOG - DIGITS + case DIGITS: put_back(c); c = '.'; - parse_number_dec(); + parse_pp_number(); return; case '.': @@ -1832,8 +981,7 @@ void lexer_next_preprocessing_token(void) ELSE_CODE( put_back(c); c = '.'; - lexer_token.type = '.'; - return; + lexer_token.kind = '.'; ) ELSE('.') case '&': @@ -1886,8 +1034,7 @@ void lexer_next_preprocessing_token(void) ELSE_CODE( put_back(c); c = '%'; - lexer_token.type = '#'; - return; + lexer_token.kind = '#'; ) ELSE('#') ELSE('%') @@ -1921,6 +1068,7 @@ void lexer_next_preprocessing_token(void) case ':': MAYBE_PROLOG MAYBE('>', ']') + MAYBE_MODE(':', T_COLONCOLON, _CXX) ELSE(':') case '=': MAYBE_PROLOG @@ -1931,6 +1079,17 @@ void lexer_next_preprocessing_token(void) MAYBE('#', T_HASHHASH) ELSE('#') + case '\\': + next_char(); + if (c == 'U' || c == 'u') { + put_back(c); + c = '\\'; + parse_symbol(); + } else { + lexer_token.kind = '\\'; + } + return; + case '?': case '[': case ']': @@ -1941,21 +1100,19 @@ void lexer_next_preprocessing_token(void) case '~': case ';': case ',': - case '\\': - lexer_token.type = c; + lexer_token.kind = c; next_char(); return; case EOF: - lexer_token.type = T_EOF; + lexer_token.kind = T_EOF; return; default: dollar_sign: - errorf(&lexer_token.source_position, "unknown character '%c' found", c); + errorf(&lexer_pos, "unknown character '%lc' found", c); next_char(); - lexer_token.type = T_ERROR; - return; + break; } } } @@ -1964,12 +1121,12 @@ void lexer_next_token(void) { lexer_next_preprocessing_token(); - while (lexer_token.type == '\n') { + while (lexer_token.kind == '\n') { newline_found: lexer_next_preprocessing_token(); } - if (lexer_token.type == '#') { + if (lexer_token.kind == '#') { parse_preprocessor_directive(); goto newline_found; } @@ -1981,34 +1138,24 @@ void init_lexer(void) symbol_L = symbol_table_insert("L"); } -void lexer_open_stream(FILE *stream, const char *input_name) +static void input_error(unsigned delta_lines, unsigned delta_cols, + const char *message) { - input = stream; - lexer_token.source_position.linenr = 0; - lexer_token.source_position.input_name = input_name; - - bufpos = NULL; - bufend = NULL; - - /* place a virtual \n at the beginning so the lexer knows that we're - * at the beginning of a line */ - c = '\n'; + lexer_pos.lineno += delta_lines; + lexer_pos.colno += delta_cols; + errorf(&lexer_pos, "%s", message); } -void lexer_open_buffer(const char *buffer, size_t len, const char *input_name) +void lexer_switch_input(input_t *new_input, const char *input_name) { - input = NULL; - lexer_token.source_position.linenr = 0; - lexer_token.source_position.input_name = input_name; + lexer_pos.lineno = 0; + lexer_pos.colno = 0; + lexer_pos.input_name = input_name; -#if 0 // TODO - bufpos = buffer; - bufend = buffer + len; -#else - (void)buffer; - (void)len; - panic("builtin lexing not done yet"); -#endif + set_input_error_callback(input_error); + input = new_input; + bufpos = NULL; + bufend = NULL; /* place a virtual \n at the beginning so the lexer knows that we're * at the beginning of a line */ @@ -2023,7 +1170,7 @@ void exit_lexer(void) static __attribute__((unused)) void dbg_pos(const source_position_t source_position) { - fprintf(stdout, "%s:%u\n", source_position.input_name, - source_position.linenr); + fprintf(stdout, "%s:%u:%u\n", source_position.input_name, + source_position.lineno, (unsigned)source_position.colno); fflush(stdout); }