X-Git-Url: http://nsz.repo.hu/git/?a=blobdiff_plain;f=lexer.c;h=a4bace030e3e049ca0c9d046a1536e57f28e30a6;hb=a8005611b36ee6057fb56a81071105cc8c43e5de;hp=990d285d531cdce71a4b4817b8991227ca1677ee;hpb=68a770de4603c696518d3dd2f278985262a1a2a2;p=cparser diff --git a/lexer.c b/lexer.c index 990d285..a4bace0 100644 --- a/lexer.c +++ b/lexer.c @@ -1,6 +1,6 @@ /* * This file is part of cparser. - * Copyright (C) 2007-2008 Matthias Braun + * Copyright (C) 2007-2009 Matthias Braun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -40,22 +40,30 @@ #include #include +#ifndef _WIN32 +#include +#endif + //#define DEBUG_CHARS #define MAX_PUTBACK 3 +#define BUF_SIZE 1024 -#ifdef _WIN32 +#if defined(_WIN32) || defined(__CYGWIN__) /* No strtold on windows and no replacement yet */ #define strtold(s, e) strtod(s, e) #endif -static int c; -token_t lexer_token; -symbol_t *symbol_L; -static FILE *input; -static char buf[1024 + MAX_PUTBACK]; -static const char *bufend; -static const char *bufpos; -static strset_t stringset; +typedef unsigned int utf32; + +static utf32 c; +token_t lexer_token; +symbol_t *symbol_L; +static FILE *input; +static utf32 buf[BUF_SIZE + MAX_PUTBACK]; +static const utf32 *bufend; +static const utf32 *bufpos; +static strset_t stringset; +bool allow_dollar_in_symbol = true; /** * Prints a parse error message at the current token. @@ -64,7 +72,7 @@ static strset_t stringset; */ static void parse_error(const char *msg) { - errorf(&lexer_token.source_position, "%s", msg); + errorf(&lexer_token.source_position, "%s", msg); } /** @@ -74,21 +82,279 @@ static void parse_error(const char *msg) */ static NORETURN internal_error(const char *msg) { - internal_errorf(&lexer_token.source_position, "%s", msg); + internal_errorf(&lexer_token.source_position, "%s", msg); +} + +static size_t read_block(unsigned char *const read_buf, size_t const n) +{ + size_t const s = fread(read_buf, 1, n, input); + if (s == 0) { + if (ferror(input)) + parse_error("read from input failed"); + buf[MAX_PUTBACK] = EOF; + bufpos = buf + MAX_PUTBACK; + bufend = buf + MAX_PUTBACK + 1; + } + return s; +} + +static void decode_iso_8859_1(void) +{ + unsigned char read_buf[BUF_SIZE]; + size_t const s = read_block(read_buf, sizeof(read_buf)); + if (s == 0) + return; + + unsigned char const *src = read_buf; + unsigned char const *end = read_buf + s; + utf32 *dst = buf + MAX_PUTBACK; + while (src != end) + *dst++ = *src++; + + bufpos = buf + MAX_PUTBACK; + bufend = dst; +} + +static void decode_iso_8859_15(void) +{ + unsigned char read_buf[BUF_SIZE]; + size_t const s = read_block(read_buf, sizeof(read_buf)); + if (s == 0) + return; + + unsigned char const *src = read_buf; + unsigned char const *end = read_buf + s; + utf32 *dst = buf + MAX_PUTBACK; + while (src != end) { + utf32 tc = *src++; + switch (tc) { + case 0xA4: tc = 0x20AC; break; // € + case 0xA6: tc = 0x0160; break; // Š + case 0xA8: tc = 0x0161; break; // š + case 0xB4: tc = 0x017D; break; // Ž + case 0xB8: tc = 0x017E; break; // ž + case 0xBC: tc = 0x0152; break; // Œ + case 0xBD: tc = 0x0153; break; // œ + case 0xBE: tc = 0x0178; break; // Ÿ + } + *dst++ = tc; + } + + bufpos = buf + MAX_PUTBACK; + bufend = dst; +} + +static void decode_utf8(void) +{ + static utf32 part_decoded_min_code; + static utf32 part_decoded_char; + static size_t part_decoded_rest_len; + + do { + unsigned char read_buf[BUF_SIZE]; + size_t const s = read_block(read_buf, sizeof(read_buf)); + if (s == 0) { + if (part_decoded_rest_len > 0) + parse_error("incomplete input char at end of input"); + return; + } + + unsigned char const *src = read_buf; + unsigned char const *end = read_buf + s; + utf32 *dst = buf + MAX_PUTBACK; + utf32 decoded; + utf32 min_code; + + if (part_decoded_rest_len != 0) { + min_code = part_decoded_min_code; + decoded = part_decoded_char; + size_t const rest_len = part_decoded_rest_len; + part_decoded_rest_len = 0; + switch (rest_len) { + case 4: goto realign; + case 3: goto three_more; + case 2: goto two_more; + default: goto one_more; + } + } + + while (src != end) { + if ((*src & 0x80) == 0) { + decoded = *src++; + } else if ((*src & 0xE0) == 0xC0) { + min_code = 0x80; + decoded = *src++ & 0x1F; +one_more: + if (src == end) { + part_decoded_min_code = min_code; + part_decoded_char = decoded; + part_decoded_rest_len = 1; + break; + } + if ((*src & 0xC0) == 0x80) { + decoded = (decoded << 6) | (*src++ & 0x3F); + } else { + goto invalid_char; + } + if (decoded < min_code || + decoded > 0x10FFFF || + (0xD800 <= decoded && decoded < 0xE000) || // high/low surrogates + (0xFDD0 <= decoded && decoded < 0xFDF0) || // noncharacters + (decoded & 0xFFFE) == 0xFFFE) { // noncharacters + parse_error("invalid byte sequence in input"); + } + } else if ((*src & 0xF0) == 0xE0) { + min_code = 0x800; + decoded = *src++ & 0x0F; +two_more: + if (src == end) { + part_decoded_min_code = min_code; + part_decoded_char = decoded; + part_decoded_rest_len = 2; + break; + } + if ((*src & 0xC0) == 0x80) { + decoded = (decoded << 6) | (*src++ & 0x3F); + } else { + goto invalid_char; + } + goto one_more; + } else if ((*src & 0xF8) == 0xF0) { + min_code = 0x10000; + decoded = *src++ & 0x07; +three_more: + if (src == end) { + part_decoded_min_code = min_code; + part_decoded_char = decoded; + part_decoded_rest_len = 3; + break; + } + if ((*src & 0xC0) == 0x80) { + decoded = (decoded << 6) | (*src++ & 0x3F); + } else { + goto invalid_char; + } + goto two_more; + } else { +invalid_char: + parse_error("invalid byte sequence in input"); +realign: + do { + ++src; + if (src == end) { + part_decoded_rest_len = 4; + break; + } + } while ((*src & 0xC0) == 0x80 || (*src & 0xF8) == 0xF8); + continue; + } + *dst++ = decoded; + } + + bufpos = buf + MAX_PUTBACK; + bufend = dst; + } while (bufpos == bufend); +} + +static void decode_windows_1252(void) +{ + unsigned char read_buf[BUF_SIZE]; + size_t const s = read_block(read_buf, sizeof(read_buf)); + if (s == 0) + return; + + unsigned char const *src = read_buf; + unsigned char const *end = read_buf + s; + utf32 *dst = buf + MAX_PUTBACK; + while (src != end) { + utf32 tc = *src++; + switch (tc) { + case 0x80: tc = 0x20AC; break; // € + case 0x82: tc = 0x201A; break; // ‚ + case 0x83: tc = 0x0192; break; // ƒ + case 0x84: tc = 0x201E; break; // „ + case 0x85: tc = 0x2026; break; // … + case 0x86: tc = 0x2020; break; // † + case 0x87: tc = 0x2021; break; // ‡ + case 0x88: tc = 0x02C6; break; // ˆ + case 0x89: tc = 0x2030; break; // ‰ + case 0x8A: tc = 0x0160; break; // Š + case 0x8B: tc = 0x2039; break; // ‹ + case 0x8C: tc = 0x0152; break; // Œ + case 0x8E: tc = 0x017D; break; // Ž + case 0x91: tc = 0x2018; break; // ‘ + case 0x92: tc = 0x2019; break; // ’ + case 0x93: tc = 0x201C; break; // “ + case 0x94: tc = 0x201D; break; // ” + case 0x95: tc = 0x2022; break; // • + case 0x96: tc = 0x2013; break; // – + case 0x97: tc = 0x2014; break; // — + case 0x98: tc = 0x02DC; break; // ˜ + case 0x99: tc = 0x2122; break; // ™ + case 0x9A: tc = 0x0161; break; // š + case 0x9B: tc = 0x203A; break; // › + case 0x9C: tc = 0x0153; break; // œ + case 0x9E: tc = 0x017E; break; // ž + case 0x9F: tc = 0x0178; break; // Ÿ + } + *dst++ = tc; + } + + bufpos = buf + MAX_PUTBACK; + bufend = dst; +} + +typedef void (*decoder_t)(void); + +static decoder_t decoder = decode_utf8; + +typedef struct named_decoder_t { + char const *name; + decoder_t decoder; +} named_decoder_t; + +static named_decoder_t const decoders[] = { + { "CP819", decode_iso_8859_1 }, // offical alias + { "IBM819", decode_iso_8859_1 }, // offical alias + { "ISO-8859-1", decode_iso_8859_1 }, // offical alias + { "ISO-8859-15", decode_iso_8859_15 }, // offical name + { "ISO8859-1", decode_iso_8859_1 }, + { "ISO8859-15", decode_iso_8859_15 }, + { "ISO_8859-1", decode_iso_8859_1 }, // offical alias + { "ISO_8859-15", decode_iso_8859_15 }, // offical alias + { "ISO_8859-1:1987", decode_iso_8859_1 }, // offical name + { "Latin-9", decode_iso_8859_15 }, // offical alias + { "UTF-8", decode_utf8 }, // offical name + { "csISOLatin1", decode_iso_8859_1 }, // offical alias + { "cp1252", decode_windows_1252 }, + { "iso-ir-100", decode_iso_8859_1 }, // offical alias + { "l1", decode_iso_8859_1 }, // offical alias + { "latin1", decode_iso_8859_1 }, // offical alias + { "windows-1252", decode_windows_1252 }, // official name + + { NULL, NULL } +}; + +void select_input_encoding(char const* const encoding) +{ + for (named_decoder_t const *i = decoders; i->name != NULL; ++i) { + if (strcasecmp(encoding, i->name) != 0) + continue; + decoder = i->decoder; + return; + } + fprintf(stderr, "error: input encoding \"%s\" not supported\n", encoding); } static inline void next_real_char(void) { assert(bufpos <= bufend); if (bufpos >= bufend) { - size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK, - input); - if(s == 0) { + if (input == NULL) { c = EOF; return; } - bufpos = buf + MAX_PUTBACK; - bufend = buf + MAX_PUTBACK + s; + decoder(); } c = *bufpos++; } @@ -98,13 +364,13 @@ static inline void next_real_char(void) * * @param pc the character to put back */ -static inline void put_back(int pc) +static inline void put_back(utf32 const pc) { assert(bufpos > buf); - *(--bufpos - buf + buf) = (char) pc; + *(--bufpos - buf + buf) = pc; #ifdef DEBUG_CHARS - printf("putback '%c'\n", pc); + printf("putback '%lc'\n", pc); #endif } @@ -189,6 +455,7 @@ end_of_next_char:; } #define SYMBOL_CHARS \ + case '$': if (!allow_dollar_in_symbol) goto dollar_sign; \ case 'a': \ case 'b': \ case 'c': \ @@ -276,6 +543,7 @@ static void parse_symbol(void) break; default: +dollar_sign: goto end_symbol; } } @@ -296,60 +564,78 @@ end_symbol: static void parse_integer_suffix(bool is_oct_hex) { - bool is_unsigned = false; - bool min_long = false; - bool min_longlong = false; - - if(c == 'U' || c == 'u') { - is_unsigned = true; + bool is_unsigned = false; + bool min_long = false; + bool min_longlong = false; + bool not_traditional = false; + int pos = 0; + char suffix[4]; + + if (c == 'U' || c == 'u') { + not_traditional = true; + suffix[pos++] = toupper(c); + is_unsigned = true; next_char(); - if(c == 'L' || c == 'l') { + if (c == 'L' || c == 'l') { + suffix[pos++] = toupper(c); min_long = true; next_char(); - if(c == 'L' || c == 'l') { + if (c == 'L' || c == 'l') { + suffix[pos++] = toupper(c); min_longlong = true; next_char(); } } - } else if(c == 'l' || c == 'L') { + } else if (c == 'l' || c == 'L') { + suffix[pos++] = toupper(c); min_long = true; next_char(); - if(c == 'l' || c == 'L') { - min_longlong = true; + if (c == 'l' || c == 'L') { + not_traditional = true; + suffix[pos++] = toupper(c); + min_longlong = true; next_char(); - if(c == 'u' || c == 'U') { - is_unsigned = true; + if (c == 'u' || c == 'U') { + suffix[pos++] = toupper(c); + is_unsigned = true; next_char(); } - } else if(c == 'u' || c == 'U') { - is_unsigned = true; + } else if (c == 'u' || c == 'U') { + not_traditional = true; + suffix[pos++] = toupper(c); + is_unsigned = true; next_char(); lexer_token.datatype = type_unsigned_long; } } - if(!is_unsigned) { + if (warning.traditional && not_traditional) { + suffix[pos] = '\0'; + warningf(&lexer_token.source_position, + "traditional C rejects the '%s' suffix", suffix); + } + if (!is_unsigned) { long long v = lexer_token.v.intvalue; - if(!min_long) { - if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) { + if (!min_long) { + if (v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) { lexer_token.datatype = type_int; return; - } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) { + } else if (is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) { lexer_token.datatype = type_unsigned_int; return; } } - if(!min_longlong) { - if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) { + if (!min_longlong) { + if (v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) { lexer_token.datatype = type_long; return; - } else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) { + } else if (is_oct_hex && v >= 0 && (unsigned long long)v <= (unsigned long long)TARGET_ULONG_MAX) { lexer_token.datatype = type_unsigned_long; return; } } unsigned long long uv = (unsigned long long) v; - if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) { + if (is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) { lexer_token.datatype = type_unsigned_long_long; return; } @@ -357,11 +643,11 @@ static void parse_integer_suffix(bool is_oct_hex) lexer_token.datatype = type_long_long; } else { unsigned long long v = (unsigned long long) lexer_token.v.intvalue; - if(!min_long && v <= TARGET_UINT_MAX) { + if (!min_long && v <= TARGET_UINT_MAX) { lexer_token.datatype = type_unsigned_int; return; } - if(!min_longlong && v <= TARGET_ULONG_MAX) { + if (!min_longlong && v <= TARGET_ULONG_MAX) { lexer_token.datatype = type_unsigned_long; return; } @@ -375,11 +661,19 @@ static void parse_floating_suffix(void) /* TODO: do something useful with the suffixes... */ case 'f': case 'F': + if (warning.traditional) { + warningf(&lexer_token.source_position, + "traditional C rejects the 'F' suffix"); + } next_char(); lexer_token.datatype = type_float; break; case 'l': case 'L': + if (warning.traditional) { + warningf(&lexer_token.source_position, + "traditional C rejects the 'F' suffix"); + } next_char(); lexer_token.datatype = type_long_double; break; @@ -393,7 +687,8 @@ static void parse_floating_suffix(void) * A replacement for strtoull. Only those parts needed for * our parser are implemented. */ -static unsigned long long parse_int_string(const char *s, const char **endptr, int base) { +static unsigned long long parse_int_string(const char *s, const char **endptr, int base) +{ unsigned long long v = 0; switch (base) { @@ -479,34 +774,74 @@ end: */ static void parse_number_hex(void) { + bool is_float = false; assert(c == 'x' || c == 'X'); next_char(); + obstack_1grow(&symbol_obstack, '0'); + obstack_1grow(&symbol_obstack, 'x'); + while(isxdigit(c)) { obstack_1grow(&symbol_obstack, (char) c); next_char(); } - obstack_1grow(&symbol_obstack, '\0'); - char *string = obstack_finish(&symbol_obstack); - if(c == '.' || c == 'p' || c == 'P') { + if (c == '.') { + obstack_1grow(&symbol_obstack, (char) c); next_char(); - internal_error("Hex floating point numbers not implemented yet"); + + while (isxdigit(c)) { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + is_float = true; } + if (c == 'p' || c == 'P') { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + + if (c == '-' || c == '+') { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + + while (isxdigit(c)) { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + is_float = true; + } + + obstack_1grow(&symbol_obstack, '\0'); + char *string = obstack_finish(&symbol_obstack); if(*string == '\0') { parse_error("invalid hex number"); lexer_token.type = T_ERROR; + obstack_free(&symbol_obstack, string); + return; } - const char *endptr; - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = parse_int_string(string, &endptr, 16); - if(*endptr != '\0') { - parse_error("hex number literal too long"); + if (is_float) { + char *endptr; + lexer_token.type = T_FLOATINGPOINT; + lexer_token.v.floatvalue = strtold(string, &endptr); + + if(*endptr != '\0') { + parse_error("invalid hex float literal"); + } + + parse_floating_suffix(); + } else { + const char *endptr; + lexer_token.type = T_INTEGER; + lexer_token.v.intvalue = parse_int_string(string + 2, &endptr, 16); + if(*endptr != '\0') { + parse_error("hex number literal too long"); + } + parse_integer_suffix(true); } obstack_free(&symbol_obstack, string); - parse_integer_suffix(true); } /** @@ -514,7 +849,7 @@ static void parse_number_hex(void) * * @param char the character to check */ -static inline bool is_octal_digit(int chr) +static inline bool is_octal_digit(utf32 chr) { switch(chr) { case '0': @@ -561,23 +896,23 @@ static void parse_number_oct(void) static void parse_number_dec(void) { bool is_float = false; - while(isdigit(c)) { + while (isdigit(c)) { obstack_1grow(&symbol_obstack, (char) c); next_char(); } - if(c == '.') { + if (c == '.') { obstack_1grow(&symbol_obstack, '.'); next_char(); - while(isdigit(c)) { + while (isdigit(c)) { obstack_1grow(&symbol_obstack, (char) c); next_char(); } is_float = true; } if(c == 'e' || c == 'E') { - obstack_1grow(&symbol_obstack, 'e'); + obstack_1grow(&symbol_obstack, (char) c); next_char(); if(c == '-' || c == '+') { @@ -664,7 +999,8 @@ static void parse_number(void) * Returns the value of a digit. * The only portable way to do it ... */ -static int digit_value(int digit) { +static int digit_value(utf32 const digit) +{ switch (digit) { case '0': return 0; case '1': return 1; @@ -698,53 +1034,43 @@ static int digit_value(int digit) { * * @param first_digit the already read first digit */ -static int parse_octal_sequence(const int first_digit) +static utf32 parse_octal_sequence(utf32 const first_digit) { assert(is_octal_digit(first_digit)); - int value = digit_value(first_digit); + utf32 value = digit_value(first_digit); if (!is_octal_digit(c)) return value; value = 8 * value + digit_value(c); next_char(); if (!is_octal_digit(c)) return value; value = 8 * value + digit_value(c); next_char(); - - if(char_is_signed) { - return (signed char) value; - } else { - return (unsigned char) value; - } + return value; } /** * Parses a hex character sequence. */ -static int parse_hex_sequence(void) +static utf32 parse_hex_sequence(void) { - int value = 0; + utf32 value = 0; while(isxdigit(c)) { value = 16 * value + digit_value(c); next_char(); } - - if(char_is_signed) { - return (signed char) value; - } else { - return (unsigned char) value; - } + return value; } /** * Parse an escape sequence. */ -static int parse_escape_sequence(void) +static utf32 parse_escape_sequence(void) { eat('\\'); - int ec = c; + utf32 const ec = c; next_char(); - switch(ec) { + switch (ec) { case '"': return '"'; case '\'': return '\''; case '\\': return '\\'; @@ -770,7 +1096,15 @@ static int parse_escape_sequence(void) case EOF: parse_error("reached end of file while parsing escape sequence"); return EOF; + /* \E is not documented, but handled, by GCC. It is acceptable according + * to §6.11.4, whereas \e is not. */ + case 'E': + case 'e': + if (c_mode & _GNUC) + return 27; /* hopefully 27 is ALWAYS the code for ESCAPE */ + /* FALLTHROUGH */ default: + /* §6.4.4.4:8 footnote 64 */ parse_error("unknown escape sequence"); return EOF; } @@ -788,6 +1122,10 @@ string_t concat_strings(const string_t *const s1, const string_t *const s2) memcpy(concat, s1->begin, len1); memcpy(concat + len1, s2->begin, len2 + 1); + if (warning.traditional) { + warningf(&lexer_token.source_position, + "traditional C rejects string constant concatenation"); + } #if 0 /* TODO hash */ const char *result = strset_insert(&stringset, concat); if(result != concat) { @@ -814,6 +1152,10 @@ wide_string_t concat_string_wide_string(const string_t *const s1, const wide_str concat[i] = src[i]; } memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat)); + if (warning.traditional) { + warningf(&lexer_token.source_position, + "traditional C rejects string constant concatenation"); + } return (wide_string_t){ concat, len1 + len2 + 1 }; } @@ -829,6 +1171,10 @@ wide_string_t concat_wide_strings(const wide_string_t *const s1, const wide_stri wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat)); memcpy(concat, s1->begin, len1 * sizeof(*concat)); memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat)); + if (warning.traditional) { + warningf(&lexer_token.source_position, + "traditional C rejects string constant concatenation"); + } return (wide_string_t){ concat, len1 + len2 + 1 }; } @@ -843,14 +1189,39 @@ wide_string_t concat_wide_string_string(const wide_string_t *const s1, const str wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat)); memcpy(concat, s1->begin, len1 * sizeof(*concat)); - const char *const src = s2->begin; + const char *const src = s2->begin; + wchar_rep_t *const dst = concat + len1; for (size_t i = 0; i != len2 + 1; ++i) { - concat[i] = src[i]; + dst[i] = src[i]; + } + if (warning.traditional) { + warningf(&lexer_token.source_position, + "traditional C rejects string constant concatenation"); } return (wide_string_t){ concat, len1 + len2 + 1 }; } +static void grow_symbol(utf32 const tc) +{ + struct obstack *const o = &symbol_obstack; + if (tc < 0x80U) { + obstack_1grow(o, tc); + } else if (tc < 0x800) { + obstack_1grow(o, 0xC0 | (tc >> 6)); + obstack_1grow(o, 0x80 | (tc & 0x3F)); + } else if (tc < 0x10000) { + obstack_1grow(o, 0xE0 | ( tc >> 12)); + obstack_1grow(o, 0x80 | ((tc >> 6) & 0x3F)); + obstack_1grow(o, 0x80 | ( tc & 0x3F)); + } else { + obstack_1grow(o, 0xF0 | ( tc >> 18)); + obstack_1grow(o, 0x80 | ((tc >> 12) & 0x3F)); + obstack_1grow(o, 0x80 | ((tc >> 6) & 0x3F)); + obstack_1grow(o, 0x80 | ( tc & 0x3F)); + } +} + /** * Parse a string literal and set lexer_token. */ @@ -860,13 +1231,17 @@ static void parse_string_literal(void) eat('"'); - int tc; while(1) { switch(c) { - case '\\': - tc = parse_escape_sequence(); - obstack_1grow(&symbol_obstack, (char) tc); + case '\\': { + utf32 const tc = parse_escape_sequence(); + if (tc >= 0x100) { + warningf(&lexer_token.source_position, + "escape sequence out of range"); + } + obstack_1grow(&symbol_obstack, tc); break; + } case EOF: { source_position_t source_position; @@ -882,7 +1257,7 @@ static void parse_string_literal(void) goto end_of_string; default: - obstack_1grow(&symbol_obstack, (char) c); + grow_symbol(c); next_char(); break; } @@ -957,6 +1332,9 @@ static void parse_wide_character_constant(void) end_of_wide_char_constant:; size_t size = (size_t) obstack_object_size(&symbol_obstack); + assert(size % sizeof(wchar_rep_t) == 0); + size /= sizeof(wchar_rep_t); + const wchar_rep_t *string = obstack_finish(&symbol_obstack); lexer_token.type = T_WIDE_CHARACTER_CONSTANT; @@ -1042,8 +1420,12 @@ static void parse_character_constant(void) while(1) { switch(c) { case '\\': { - int tc = parse_escape_sequence(); - obstack_1grow(&symbol_obstack, (char) tc); + utf32 const tc = parse_escape_sequence(); + if (tc >= 0x100) { + warningf(&lexer_token.source_position, + "escape sequence out of range"); + } + obstack_1grow(&symbol_obstack, tc); break; } @@ -1066,7 +1448,7 @@ static void parse_character_constant(void) } default: - obstack_1grow(&symbol_obstack, (char) c); + grow_symbol(c); next_char(); break; @@ -1080,7 +1462,7 @@ end_of_char_constant:; lexer_token.type = T_CHARACTER_CONSTANT; lexer_token.v.string.begin = string; lexer_token.v.string.size = size; - lexer_token.datatype = type_int; + lexer_token.datatype = c_mode & _CXX && size == 1 ? type_char : type_int; } /** @@ -1095,7 +1477,10 @@ static void skip_multiline_comment(void) case '/': next_char(); if (c == '*') { - /* TODO: nested comment, warn here */ + /* nested comment, warn here */ + if (warning.comment) { + warningf(&lexer_token.source_position, "'/*' within comment"); + } } break; case '*': @@ -1137,6 +1522,15 @@ static void skip_line_comment(void) case '\r': return; + case '\\': + next_char(); + if (c == '\n' || c == '\r') { + if (warning.comment) + warningf(&lexer_token.source_position, "multi-line comment"); + return; + } + break; + default: next_char(); break; @@ -1219,7 +1613,7 @@ static void parse_line_directive(void) /** * STDC pragmas. */ -typedef enum { +typedef enum stdc_pragma_kind_t { STDC_UNKNOWN, STDC_FP_CONTRACT, STDC_FENV_ACCESS, @@ -1229,7 +1623,7 @@ typedef enum { /** * STDC pragma values. */ -typedef enum { +typedef enum stdc_pragma_value_kind_t { STDC_VALUE_UNKNOWN, STDC_VALUE_ON, STDC_VALUE_OFF, @@ -1239,7 +1633,8 @@ typedef enum { /** * Parse a pragma directive. */ -static void parse_pragma(void) { +static void parse_pragma(void) +{ bool unknown_pragma = true; next_pp_token(); @@ -1351,7 +1746,7 @@ static void parse_preprocessor_directive(void) parse_line_directive(); break; case '\n': - /* NULL directive, see § 6.10.7 */ + /* NULL directive, see §6.10.7 */ break; default: parse_error("invalid preprocessor directive"); @@ -1371,9 +1766,19 @@ static void parse_preprocessor_directive(void) lexer_token.type = set_type; \ return; +/* must use this as last thing */ +#define MAYBE_MODE(ch, set_type, mode) \ + case ch: \ + if (c_mode & mode) { \ + next_char(); \ + lexer_token.type = set_type; \ + return; \ + } \ + /* fallthrough */ + #define ELSE_CODE(code) \ default: \ - code; \ + code \ } \ } /* end of while(1) */ \ break; @@ -1425,16 +1830,7 @@ void lexer_next_preprocessing_token(void) case '.': MAYBE_PROLOG - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': + DIGITS put_back(c); c = '.'; parse_number_dec(); @@ -1535,6 +1931,7 @@ void lexer_next_preprocessing_token(void) case ':': MAYBE_PROLOG MAYBE('>', ']') + MAYBE_MODE(':', T_COLONCOLON, _CXX) ELSE(':') case '=': MAYBE_PROLOG @@ -1565,8 +1962,9 @@ void lexer_next_preprocessing_token(void) return; default: +dollar_sign: + errorf(&lexer_token.source_position, "unknown character '%c' found", c); next_char(); - errorf(&lexer_token.source_position, "unknown character '%c' found\n", c); lexer_token.type = T_ERROR; return; } @@ -1576,15 +1974,13 @@ void lexer_next_preprocessing_token(void) void lexer_next_token(void) { lexer_next_preprocessing_token(); - if(lexer_token.type != '\n') - return; + while (lexer_token.type == '\n') { newline_found: - do { lexer_next_preprocessing_token(); - } while(lexer_token.type == '\n'); + } - if(lexer_token.type == '#') { + if (lexer_token.type == '#') { parse_preprocessor_directive(); goto newline_found; } @@ -1593,6 +1989,7 @@ newline_found: void init_lexer(void) { strset_init(&stringset); + symbol_L = symbol_table_insert("L"); } void lexer_open_stream(FILE *stream, const char *input_name) @@ -1601,7 +1998,6 @@ void lexer_open_stream(FILE *stream, const char *input_name) lexer_token.source_position.linenr = 0; lexer_token.source_position.input_name = input_name; - symbol_L = symbol_table_insert("L"); bufpos = NULL; bufend = NULL; @@ -1610,6 +2006,26 @@ void lexer_open_stream(FILE *stream, const char *input_name) c = '\n'; } +void lexer_open_buffer(const char *buffer, size_t len, const char *input_name) +{ + input = NULL; + lexer_token.source_position.linenr = 0; + lexer_token.source_position.input_name = input_name; + +#if 0 // TODO + bufpos = buffer; + bufend = buffer + len; +#else + (void)buffer; + (void)len; + panic("builtin lexing not done yet"); +#endif + + /* place a virtual \n at the beginning so the lexer knows that we're + * at the beginning of a line */ + c = '\n'; +} + void exit_lexer(void) { strset_destroy(&stringset);