X-Git-Url: http://nsz.repo.hu/git/?a=blobdiff_plain;f=lexer.c;h=aa2cc12d0523cfb900b5fbd0f85550d97bcd9407;hb=ce2e13de664ebd52daf7de055790546ecfece6ff;hp=1300b3d26f23cda1c90093658115266e03a941ab;hpb=a9c17bbc7a863b3dd3c0bfdd8884c0d8b274cb0d;p=cparser diff --git a/lexer.c b/lexer.c index 1300b3d..aa2cc12 100644 --- a/lexer.c +++ b/lexer.c @@ -1,76 +1,377 @@ +/* + * This file is part of cparser. + * Copyright (C) 2007-2008 Matthias Braun + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA + * 02111-1307, USA. + */ #include +#include "diagnostic.h" #include "lexer.h" +#include "symbol_t.h" #include "token_t.h" #include "symbol_table_t.h" #include "adt/error.h" #include "adt/strset.h" #include "adt/util.h" +#include "types.h" +#include "type_t.h" +#include "target_architecture.h" +#include "parser.h" +#include "warning.h" +#include "lang_features.h" #include #include #include +#include #include +#ifndef _WIN32 +#include +#endif + //#define DEBUG_CHARS #define MAX_PUTBACK 3 +#define BUF_SIZE 1024 -static int c; -token_t lexer_token; -symbol_t *symbol_L; -static FILE *input; -static char buf[1024 + MAX_PUTBACK]; -static const char *bufend; -static const char *bufpos; -static strset_t stringset; +#if defined(_WIN32) || defined(__CYGWIN__) +/* No strtold on windows and no replacement yet */ +#define strtold(s, e) strtod(s, e) +#define strcasecmp(a, b) stricmp(a, b) +#endif -static void error_prefix_at(const char *input_name, unsigned linenr) +typedef unsigned int utf32; + +static utf32 c; +token_t lexer_token; +symbol_t *symbol_L; +static FILE *input; +static utf32 buf[BUF_SIZE + MAX_PUTBACK]; +static const utf32 *bufend; +static const utf32 *bufpos; +static strset_t stringset; +bool allow_dollar_in_symbol = true; + +/** + * Prints a parse error message at the current token. + * + * @param msg the error message + */ +static void parse_error(const char *msg) { - fprintf(stderr, "%s:%u: Error: ", input_name, linenr); + errorf(&lexer_token.source_position, "%s", msg); } -static void error_prefix(void) +/** + * Prints an internal error message at the current token. + * + * @param msg the error message + */ +static NORETURN internal_error(const char *msg) { - error_prefix_at(lexer_token.source_position.input_name, - lexer_token.source_position.linenr); + internal_errorf(&lexer_token.source_position, "%s", msg); } -static void parse_error(const char *msg) +static size_t read_block(unsigned char *const read_buf, size_t const n) { - error_prefix(); - fprintf(stderr, "%s\n", msg); + size_t const s = fread(read_buf, 1, n, input); + if (s == 0) { + if (ferror(input)) + parse_error("read from input failed"); + buf[MAX_PUTBACK] = EOF; + bufpos = buf + MAX_PUTBACK; + bufend = buf + MAX_PUTBACK + 1; + } + return s; } -static inline void next_real_char(void) +static void decode_iso_8859_1(void) { - bufpos++; - if(bufpos >= bufend) { - size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK, - input); - if(s == 0) { - c = EOF; + unsigned char read_buf[BUF_SIZE]; + size_t const s = read_block(read_buf, sizeof(read_buf)); + if (s == 0) + return; + + unsigned char const *src = read_buf; + unsigned char const *end = read_buf + s; + utf32 *dst = buf + MAX_PUTBACK; + while (src != end) + *dst++ = *src++; + + bufpos = buf + MAX_PUTBACK; + bufend = dst; +} + +static void decode_iso_8859_15(void) +{ + unsigned char read_buf[BUF_SIZE]; + size_t const s = read_block(read_buf, sizeof(read_buf)); + if (s == 0) + return; + + unsigned char const *src = read_buf; + unsigned char const *end = read_buf + s; + utf32 *dst = buf + MAX_PUTBACK; + while (src != end) { + utf32 tc = *src++; + switch (tc) { + case 0xA4: tc = 0x20AC; break; // € + case 0xA6: tc = 0x0160; break; // Š + case 0xA8: tc = 0x0161; break; // š + case 0xB4: tc = 0x017D; break; // Ž + case 0xB8: tc = 0x017E; break; // ž + case 0xBC: tc = 0x0152; break; // Œ + case 0xBD: tc = 0x0153; break; // œ + case 0xBE: tc = 0x0178; break; // Ÿ + } + *dst++ = tc; + } + + bufpos = buf + MAX_PUTBACK; + bufend = dst; +} + +static void decode_utf8(void) +{ + static utf32 part_decoded_min_code; + static utf32 part_decoded_char; + static size_t part_decoded_rest_len; + + do { + unsigned char read_buf[BUF_SIZE]; + size_t const s = read_block(read_buf, sizeof(read_buf)); + if (s == 0) { + if (part_decoded_rest_len > 0) + parse_error("incomplete input char at end of input"); return; } + + unsigned char const *src = read_buf; + unsigned char const *end = read_buf + s; + utf32 *dst = buf + MAX_PUTBACK; + utf32 decoded; + utf32 min_code; + + if (part_decoded_rest_len != 0) { + min_code = part_decoded_min_code; + decoded = part_decoded_char; + size_t const rest_len = part_decoded_rest_len; + part_decoded_rest_len = 0; + switch (rest_len) { + case 4: goto realign; + case 3: goto three_more; + case 2: goto two_more; + default: goto one_more; + } + } + + while (src != end) { + if ((*src & 0x80) == 0) { + decoded = *src++; + } else if ((*src & 0xE0) == 0xC0) { + min_code = 0x80; + decoded = *src++ & 0x1F; +one_more: + if (src == end) { + part_decoded_min_code = min_code; + part_decoded_char = decoded; + part_decoded_rest_len = 1; + break; + } + if ((*src & 0xC0) == 0x80) { + decoded = (decoded << 6) | (*src++ & 0x3F); + } else { + goto invalid_char; + } + if (decoded < min_code || + decoded > 0x10FFFF || + (0xD800 <= decoded && decoded < 0xE000) || // high/low surrogates + (0xFDD0 <= decoded && decoded < 0xFDF0) || // noncharacters + (decoded & 0xFFFE) == 0xFFFE) { // noncharacters + parse_error("invalid byte sequence in input"); + } + } else if ((*src & 0xF0) == 0xE0) { + min_code = 0x800; + decoded = *src++ & 0x0F; +two_more: + if (src == end) { + part_decoded_min_code = min_code; + part_decoded_char = decoded; + part_decoded_rest_len = 2; + break; + } + if ((*src & 0xC0) == 0x80) { + decoded = (decoded << 6) | (*src++ & 0x3F); + } else { + goto invalid_char; + } + goto one_more; + } else if ((*src & 0xF8) == 0xF0) { + min_code = 0x10000; + decoded = *src++ & 0x07; +three_more: + if (src == end) { + part_decoded_min_code = min_code; + part_decoded_char = decoded; + part_decoded_rest_len = 3; + break; + } + if ((*src & 0xC0) == 0x80) { + decoded = (decoded << 6) | (*src++ & 0x3F); + } else { + goto invalid_char; + } + goto two_more; + } else { +invalid_char: + parse_error("invalid byte sequence in input"); +realign: + do { + ++src; + if (src == end) { + part_decoded_rest_len = 4; + break; + } + } while ((*src & 0xC0) == 0x80 || (*src & 0xF8) == 0xF8); + continue; + } + *dst++ = decoded; + } + bufpos = buf + MAX_PUTBACK; - bufend = buf + MAX_PUTBACK + s; + bufend = dst; + } while (bufpos == bufend); +} + +static void decode_windows_1252(void) +{ + unsigned char read_buf[BUF_SIZE]; + size_t const s = read_block(read_buf, sizeof(read_buf)); + if (s == 0) + return; + + unsigned char const *src = read_buf; + unsigned char const *end = read_buf + s; + utf32 *dst = buf + MAX_PUTBACK; + while (src != end) { + utf32 tc = *src++; + switch (tc) { + case 0x80: tc = 0x20AC; break; // € + case 0x82: tc = 0x201A; break; // ‚ + case 0x83: tc = 0x0192; break; // ƒ + case 0x84: tc = 0x201E; break; // „ + case 0x85: tc = 0x2026; break; // … + case 0x86: tc = 0x2020; break; // † + case 0x87: tc = 0x2021; break; // ‡ + case 0x88: tc = 0x02C6; break; // ˆ + case 0x89: tc = 0x2030; break; // ‰ + case 0x8A: tc = 0x0160; break; // Š + case 0x8B: tc = 0x2039; break; // ‹ + case 0x8C: tc = 0x0152; break; // Œ + case 0x8E: tc = 0x017D; break; // Ž + case 0x91: tc = 0x2018; break; // ‘ + case 0x92: tc = 0x2019; break; // ’ + case 0x93: tc = 0x201C; break; // “ + case 0x94: tc = 0x201D; break; // ” + case 0x95: tc = 0x2022; break; // • + case 0x96: tc = 0x2013; break; // – + case 0x97: tc = 0x2014; break; // — + case 0x98: tc = 0x02DC; break; // ˜ + case 0x99: tc = 0x2122; break; // ™ + case 0x9A: tc = 0x0161; break; // š + case 0x9B: tc = 0x203A; break; // › + case 0x9C: tc = 0x0153; break; // œ + case 0x9E: tc = 0x017E; break; // ž + case 0x9F: tc = 0x0178; break; // Ÿ + } + *dst++ = tc; } - c = *(bufpos); + + bufpos = buf + MAX_PUTBACK; + bufend = dst; } -static inline void put_back(int pc) +typedef void (*decoder_t)(void); + +static decoder_t decoder = decode_utf8; + +typedef struct named_decoder_t { + char const *name; + decoder_t decoder; +} named_decoder_t; + +static named_decoder_t const decoders[] = { + { "CP819", decode_iso_8859_1 }, // offical alias + { "IBM819", decode_iso_8859_1 }, // offical alias + { "ISO-8859-1", decode_iso_8859_1 }, // offical alias + { "ISO-8859-15", decode_iso_8859_15 }, // offical name + { "ISO8859-1", decode_iso_8859_1 }, + { "ISO8859-15", decode_iso_8859_15 }, + { "ISO_8859-1", decode_iso_8859_1 }, // offical alias + { "ISO_8859-15", decode_iso_8859_15 }, // offical alias + { "ISO_8859-1:1987", decode_iso_8859_1 }, // offical name + { "Latin-9", decode_iso_8859_15 }, // offical alias + { "UTF-8", decode_utf8 }, // offical name + { "csISOLatin1", decode_iso_8859_1 }, // offical alias + { "cp1252", decode_windows_1252 }, + { "iso-ir-100", decode_iso_8859_1 }, // offical alias + { "l1", decode_iso_8859_1 }, // offical alias + { "latin1", decode_iso_8859_1 }, // offical alias + { "windows-1252", decode_windows_1252 }, // official name + + { NULL, NULL } +}; + +void select_input_encoding(char const* const encoding) { - assert(bufpos >= buf); - assert(bufpos < buf+MAX_PUTBACK || *bufpos == pc); + for (named_decoder_t const *i = decoders; i->name != NULL; ++i) { + if (strcasecmp(encoding, i->name) != 0) + continue; + decoder = i->decoder; + return; + } + fprintf(stderr, "error: input encoding \"%s\" not supported\n", encoding); +} - char *p = buf + (bufpos - buf); - *p = pc; +static inline void next_real_char(void) +{ + assert(bufpos <= bufend); + if (bufpos >= bufend) { + if (input == NULL) { + c = EOF; + return; + } + decoder(); + } + c = *bufpos++; +} - /* going backwards in the buffer is legal as long as it's not more often - * than MAX_PUTBACK */ - bufpos--; +/** + * Put a character back into the buffer. + * + * @param pc the character to put back + */ +static inline void put_back(utf32 const pc) +{ + assert(bufpos > buf); + *(--bufpos - buf + buf) = pc; #ifdef DEBUG_CHARS - printf("putback '%c'\n", pc); + printf("putback '%lc'\n", pc); #endif } @@ -83,11 +384,11 @@ static inline void next_char(void); next_char(); \ } \ lexer_token.source_position.linenr++; \ - code; \ + code \ case '\n': \ next_char(); \ lexer_token.source_position.linenr++; \ - code; + code #define eat(c_type) do { assert(c == c_type); next_char(); } while(0) @@ -106,11 +407,14 @@ static void maybe_concat_lines(void) c = '\\'; } +/** + * Set c to the next input character, ie. + * after expanding trigraphs. + */ static inline void next_char(void) { next_real_char(); -#if 0 /* filter trigraphs */ if(UNLIKELY(c == '\\')) { maybe_concat_lines(); @@ -139,23 +443,20 @@ static inline void next_char(void) case '>': c = '}'; break; case '-': c = '~'; break; default: - put_back('?'); put_back(c); + put_back('?'); c = '?'; break; } -end_of_next_char: -#endif - (void) maybe_concat_lines; +end_of_next_char:; #ifdef DEBUG_CHARS printf("nchar '%c'\n", c); -#else - ; #endif } #define SYMBOL_CHARS \ + case '$': if (!allow_dollar_in_symbol) goto dollar_sign; \ case 'a': \ case 'b': \ case 'c': \ @@ -222,23 +523,28 @@ end_of_next_char: case '8': \ case '9': +/** + * Read a symbol from the input and build + * the lexer_token. + */ static void parse_symbol(void) { symbol_t *symbol; char *string; - obstack_1grow(&symbol_obstack, c); + obstack_1grow(&symbol_obstack, (char) c); next_char(); while(1) { switch(c) { DIGITS SYMBOL_CHARS - obstack_1grow(&symbol_obstack, c); + obstack_1grow(&symbol_obstack, (char) c); next_char(); break; default: +dollar_sign: goto end_symbol; } } @@ -257,181 +563,401 @@ end_symbol: } } -static void parse_integer_suffix(void) +static void parse_integer_suffix(bool is_oct_hex) { - if(c == 'U' || c == 'U') { - /* TODO do something with the suffixes... */ + bool is_unsigned = false; + bool min_long = false; + bool min_longlong = false; + bool not_traditional = false; + int pos = 0; + char suffix[4]; + + if (c == 'U' || c == 'u') { + not_traditional = true; + suffix[pos++] = toupper(c); + is_unsigned = true; next_char(); - if(c == 'L' || c == 'l') { + if (c == 'L' || c == 'l') { + suffix[pos++] = toupper(c); + min_long = true; next_char(); - if(c == 'L' || c == 'l') { + if (c == 'L' || c == 'l') { + suffix[pos++] = toupper(c); + min_longlong = true; next_char(); } } - } else if(c == 'l' || c == 'L') { + } else if (c == 'l' || c == 'L') { + suffix[pos++] = toupper(c); + min_long = true; next_char(); - if(c == 'l' || c == 'L') { + if (c == 'l' || c == 'L') { + not_traditional = true; + suffix[pos++] = toupper(c); + min_longlong = true; next_char(); - if(c == 'u' || c == 'U') { + if (c == 'u' || c == 'U') { + suffix[pos++] = toupper(c); + is_unsigned = true; next_char(); } - } else if(c == 'u' || c == 'U') { + } else if (c == 'u' || c == 'U') { + not_traditional = true; + suffix[pos++] = toupper(c); + is_unsigned = true; next_char(); + lexer_token.datatype = type_unsigned_long; + } + } + + if (warning.traditional && not_traditional) { + suffix[pos] = '\0'; + warningf(&lexer_token.source_position, + "traditional C rejects the '%s' suffix", suffix); + } + if (!is_unsigned) { + long long v = lexer_token.v.intvalue; + if (!min_long) { + if (v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) { + lexer_token.datatype = type_int; + return; + } else if (is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) { + lexer_token.datatype = type_unsigned_int; + return; + } + } + if (!min_longlong) { + if (v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) { + lexer_token.datatype = type_long; + return; + } else if (is_oct_hex && v >= 0 && (unsigned long long)v <= (unsigned long long)TARGET_ULONG_MAX) { + lexer_token.datatype = type_unsigned_long; + return; + } + } + unsigned long long uv = (unsigned long long) v; + if (is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) { + lexer_token.datatype = type_unsigned_long_long; + return; + } + + lexer_token.datatype = type_long_long; + } else { + unsigned long long v = (unsigned long long) lexer_token.v.intvalue; + if (!min_long && v <= TARGET_UINT_MAX) { + lexer_token.datatype = type_unsigned_int; + return; + } + if (!min_longlong && v <= TARGET_ULONG_MAX) { + lexer_token.datatype = type_unsigned_long; + return; } + lexer_token.datatype = type_unsigned_long_long; } } static void parse_floating_suffix(void) { switch(c) { - /* TODO: do something usefull with the suffixes... */ + /* TODO: do something useful with the suffixes... */ case 'f': case 'F': + if (warning.traditional) { + warningf(&lexer_token.source_position, + "traditional C rejects the 'F' suffix"); + } + next_char(); + lexer_token.datatype = type_float; + break; case 'l': case 'L': + if (warning.traditional) { + warningf(&lexer_token.source_position, + "traditional C rejects the 'F' suffix"); + } next_char(); + lexer_token.datatype = type_long_double; + break; + default: + lexer_token.datatype = type_double; + break; + } +} + +/** + * A replacement for strtoull. Only those parts needed for + * our parser are implemented. + */ +static unsigned long long parse_int_string(const char *s, const char **endptr, int base) +{ + unsigned long long v = 0; + + switch (base) { + case 16: + for (;; ++s) { + /* check for overrun */ + if (v >= 0x1000000000000000ULL) + break; + switch (tolower(*s)) { + case '0': v <<= 4; break; + case '1': v <<= 4; v |= 0x1; break; + case '2': v <<= 4; v |= 0x2; break; + case '3': v <<= 4; v |= 0x3; break; + case '4': v <<= 4; v |= 0x4; break; + case '5': v <<= 4; v |= 0x5; break; + case '6': v <<= 4; v |= 0x6; break; + case '7': v <<= 4; v |= 0x7; break; + case '8': v <<= 4; v |= 0x8; break; + case '9': v <<= 4; v |= 0x9; break; + case 'a': v <<= 4; v |= 0xa; break; + case 'b': v <<= 4; v |= 0xb; break; + case 'c': v <<= 4; v |= 0xc; break; + case 'd': v <<= 4; v |= 0xd; break; + case 'e': v <<= 4; v |= 0xe; break; + case 'f': v <<= 4; v |= 0xf; break; + default: + goto end; + } + } + break; + case 8: + for (;; ++s) { + /* check for overrun */ + if (v >= 0x2000000000000000ULL) + break; + switch (tolower(*s)) { + case '0': v <<= 3; break; + case '1': v <<= 3; v |= 1; break; + case '2': v <<= 3; v |= 2; break; + case '3': v <<= 3; v |= 3; break; + case '4': v <<= 3; v |= 4; break; + case '5': v <<= 3; v |= 5; break; + case '6': v <<= 3; v |= 6; break; + case '7': v <<= 3; v |= 7; break; + default: + goto end; + } + } + break; + case 10: + for (;; ++s) { + /* check for overrun */ + if (v > 0x1999999999999999ULL) + break; + switch (tolower(*s)) { + case '0': v *= 10; break; + case '1': v *= 10; v += 1; break; + case '2': v *= 10; v += 2; break; + case '3': v *= 10; v += 3; break; + case '4': v *= 10; v += 4; break; + case '5': v *= 10; v += 5; break; + case '6': v *= 10; v += 6; break; + case '7': v *= 10; v += 7; break; + case '8': v *= 10; v += 8; break; + case '9': v *= 10; v += 9; break; + default: + goto end; + } + } break; default: + assert(0); break; } +end: + *endptr = s; + return v; } +/** + * Parses a hex number including hex floats and set the + * lexer_token. + */ static void parse_number_hex(void) { + bool is_float = false; assert(c == 'x' || c == 'X'); next_char(); - if (!isdigit(c) && - !('A' <= c && c <= 'F') && - !('a' <= c && c <= 'f')) { - parse_error("premature end of hex number literal"); - lexer_token.type = T_ERROR; - return; - } + obstack_1grow(&symbol_obstack, '0'); + obstack_1grow(&symbol_obstack, 'x'); - int value = 0; - while(1) { - if (isdigit(c)) { - value = 16 * value + c - '0'; - } else if ('A' <= c && c <= 'F') { - value = 16 * value + c - 'A' + 10; - } else if ('a' <= c && c <= 'f') { - value = 16 * value + c - 'a' + 10; - } else { - parse_integer_suffix(); - - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = value; - return; - } + while(isxdigit(c)) { + obstack_1grow(&symbol_obstack, (char) c); next_char(); } - if(c == '.' || c == 'p' || c == 'P') { + if (c == '.') { + obstack_1grow(&symbol_obstack, (char) c); next_char(); - panic("Hex floating point numbers not implemented yet"); - } -} -static void parse_number_oct(void) -{ - int value = 0; - while(c >= '0' && c <= '7') { - value = 8 * value + c - '0'; + while (isxdigit(c)) { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + is_float = true; + } + if (c == 'p' || c == 'P') { + obstack_1grow(&symbol_obstack, (char) c); next_char(); + + if (c == '-' || c == '+') { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + + while (isxdigit(c)) { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + is_float = true; } - if (c == '8' || c == '9') { - parse_error("invalid octal number"); + + obstack_1grow(&symbol_obstack, '\0'); + char *string = obstack_finish(&symbol_obstack); + if(*string == '\0') { + parse_error("invalid hex number"); lexer_token.type = T_ERROR; + obstack_free(&symbol_obstack, string); return; } - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = value; - - parse_integer_suffix(); -} + if (is_float) { + char *endptr; + lexer_token.type = T_FLOATINGPOINT; + lexer_token.v.floatvalue = strtold(string, &endptr); -static void parse_floatingpoint_exponent(long double value) -{ - unsigned int expo = 0; - long double factor = 10.; + if(*endptr != '\0') { + parse_error("invalid hex float literal"); + } - if(c == '-') { - next_char(); - factor = 0.1; - } else if(c == '+') { - next_char(); + parse_floating_suffix(); + } else { + const char *endptr; + lexer_token.type = T_INTEGER; + lexer_token.v.intvalue = parse_int_string(string + 2, &endptr, 16); + if(*endptr != '\0') { + parse_error("hex number literal too long"); + } + parse_integer_suffix(true); } - while(c >= '0' && c <= '9') { - expo = 10 * expo + (c - '0'); - next_char(); - } + obstack_free(&symbol_obstack, string); +} - while(1) { - if(expo & 1) - value *= factor; - expo >>= 1; - if(expo == 0) - break; - factor *= factor; +/** + * Returns true if the given char is a octal digit. + * + * @param char the character to check + */ +static inline bool is_octal_digit(utf32 chr) +{ + switch(chr) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + return true; + default: + return false; } - - lexer_token.type = T_FLOATINGPOINT; - lexer_token.v.floatvalue = value; - - parse_floating_suffix(); } -static void parse_floatingpoint_fract(int integer_part) +/** + * Parses a octal number and set the lexer_token. + */ +static void parse_number_oct(void) { - long double value = integer_part; - long double factor = 1.; - - while(c >= '0' && c <= '9') { - factor *= 0.1; - value += (c - '0') * factor; + while(is_octal_digit(c)) { + obstack_1grow(&symbol_obstack, (char) c); next_char(); } + obstack_1grow(&symbol_obstack, '\0'); + char *string = obstack_finish(&symbol_obstack); - if(c == 'e' || c == 'E') { - next_char(); - parse_floatingpoint_exponent(value); - return; + const char *endptr; + lexer_token.type = T_INTEGER; + lexer_token.v.intvalue = parse_int_string(string, &endptr, 8); + if(*endptr != '\0') { + parse_error("octal number literal too long"); } - lexer_token.type = T_FLOATINGPOINT; - lexer_token.v.floatvalue = value; - - parse_floating_suffix(); + obstack_free(&symbol_obstack, string); + parse_integer_suffix(true); } +/** + * Parses a decimal including float number and set the + * lexer_token. + */ static void parse_number_dec(void) { - int value = 0; - - while(isdigit(c)) { - value = 10 * value + c - '0'; + bool is_float = false; + while (isdigit(c)) { + obstack_1grow(&symbol_obstack, (char) c); next_char(); } - if(c == '.') { + if (c == '.') { + obstack_1grow(&symbol_obstack, '.'); next_char(); - parse_floatingpoint_fract(value); - return; + + while (isdigit(c)) { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + is_float = true; } if(c == 'e' || c == 'E') { + obstack_1grow(&symbol_obstack, (char) c); next_char(); - parse_floatingpoint_exponent(value); - return; + + if(c == '-' || c == '+') { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + + while(isdigit(c)) { + obstack_1grow(&symbol_obstack, (char) c); + next_char(); + } + is_float = true; } - parse_integer_suffix(); - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = value; + obstack_1grow(&symbol_obstack, '\0'); + char *string = obstack_finish(&symbol_obstack); + + if(is_float) { + char *endptr; + lexer_token.type = T_FLOATINGPOINT; + lexer_token.v.floatvalue = strtold(string, &endptr); + + if(*endptr != '\0') { + parse_error("invalid number literal"); + } + + parse_floating_suffix(); + } else { + const char *endptr; + lexer_token.type = T_INTEGER; + lexer_token.v.intvalue = parse_int_string(string, &endptr, 10); + + if(*endptr != '\0') { + parse_error("invalid number literal"); + } + + parse_integer_suffix(false); + } + obstack_free(&symbol_obstack, string); } +/** + * Parses a number and sets the lexer_token. + */ static void parse_number(void) { if (c == '0') { @@ -451,23 +977,17 @@ static void parse_number(void) case '7': parse_number_oct(); break; - case '.': - next_char(); - parse_floatingpoint_fract(0); - break; - case 'e': - case 'E': - parse_floatingpoint_exponent(0); - break; case '8': case '9': next_char(); parse_error("invalid octal number"); lexer_token.type = T_ERROR; return; + case '.': + case 'e': + case 'E': default: - put_back(c); - c = '0'; + obstack_1grow(&symbol_obstack, '0'); parse_number_dec(); return; } @@ -476,48 +996,84 @@ static void parse_number(void) } } -static int parse_octal_sequence(void) +/** + * Returns the value of a digit. + * The only portable way to do it ... + */ +static int digit_value(utf32 const digit) { - int value = 0; - while(1) { - if(c < '0' || c > '7') - break; - value = 8 * value + c - '0'; - next_char(); + switch (digit) { + case '0': return 0; + case '1': return 1; + case '2': return 2; + case '3': return 3; + case '4': return 4; + case '5': return 5; + case '6': return 6; + case '7': return 7; + case '8': return 8; + case '9': return 9; + case 'a': + case 'A': return 10; + case 'b': + case 'B': return 11; + case 'c': + case 'C': return 12; + case 'd': + case 'D': return 13; + case 'e': + case 'E': return 14; + case 'f': + case 'F': return 15; + default: + internal_error("wrong character given"); } +} +/** + * Parses an octal character sequence. + * + * @param first_digit the already read first digit + */ +static utf32 parse_octal_sequence(utf32 const first_digit) +{ + assert(is_octal_digit(first_digit)); + utf32 value = digit_value(first_digit); + if (!is_octal_digit(c)) return value; + value = 8 * value + digit_value(c); + next_char(); + if (!is_octal_digit(c)) return value; + value = 8 * value + digit_value(c); + next_char(); return value; } -static int parse_hex_sequence(void) +/** + * Parses a hex character sequence. + */ +static utf32 parse_hex_sequence(void) { - int value = 0; - while(1) { - if (c >= '0' && c <= '9') { - value = 16 * value + c - '0'; - } else if ('A' <= c && c <= 'F') { - value = 16 * value + c - 'A' + 10; - } else if ('a' <= c && c <= 'f') { - value = 16 * value + c - 'a' + 10; - } else { - break; - } + utf32 value = 0; + while(isxdigit(c)) { + value = 16 * value + digit_value(c); next_char(); } - return value; } -static int parse_escape_sequence(void) +/** + * Parse an escape sequence. + */ +static utf32 parse_escape_sequence(void) { eat('\\'); - int ec = c; + utf32 const ec = c; next_char(); - switch(ec) { + switch (ec) { case '"': return '"'; - case '\'': return'\''; + case '\'': return '\''; case '\\': return '\\'; case '?': return '\?'; case 'a': return '\a'; @@ -537,63 +1093,172 @@ static int parse_escape_sequence(void) case '5': case '6': case '7': - return parse_octal_sequence(); + return parse_octal_sequence(ec); case EOF: parse_error("reached end of file while parsing escape sequence"); return EOF; + /* \E is not documented, but handled, by GCC. It is acceptable according + * to §6.11.4, whereas \e is not. */ + case 'E': + case 'e': + if (c_mode & _GNUC) + return 27; /* hopefully 27 is ALWAYS the code for ESCAPE */ + /* FALLTHROUGH */ default: + /* §6.4.4.4:8 footnote 64 */ parse_error("unknown escape sequence"); return EOF; } } -const char *concat_strings(const char *s1, const char *s2) +/** + * Concatenate two strings. + */ +string_t concat_strings(const string_t *const s1, const string_t *const s2) { - size_t len1 = strlen(s1); - size_t len2 = strlen(s2); + const size_t len1 = s1->size - 1; + const size_t len2 = s2->size - 1; - char *concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1); - memcpy(concat, s1, len1); - memcpy(concat + len1, s2, len2 + 1); + char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1); + memcpy(concat, s1->begin, len1); + memcpy(concat + len1, s2->begin, len2 + 1); + if (warning.traditional) { + warningf(&lexer_token.source_position, + "traditional C rejects string constant concatenation"); + } +#if 0 /* TODO hash */ const char *result = strset_insert(&stringset, concat); if(result != concat) { obstack_free(&symbol_obstack, concat); } return result; +#else + return (string_t){ concat, len1 + len2 + 1 }; +#endif +} + +/** + * Concatenate a string and a wide string. + */ +wide_string_t concat_string_wide_string(const string_t *const s1, const wide_string_t *const s2) +{ + const size_t len1 = s1->size - 1; + const size_t len2 = s2->size - 1; + + wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat)); + const char *const src = s1->begin; + for (size_t i = 0; i != len1; ++i) { + concat[i] = src[i]; + } + memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat)); + if (warning.traditional) { + warningf(&lexer_token.source_position, + "traditional C rejects string constant concatenation"); + } + + return (wide_string_t){ concat, len1 + len2 + 1 }; } +/** + * Concatenate two wide strings. + */ +wide_string_t concat_wide_strings(const wide_string_t *const s1, const wide_string_t *const s2) +{ + const size_t len1 = s1->size - 1; + const size_t len2 = s2->size - 1; + + wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat)); + memcpy(concat, s1->begin, len1 * sizeof(*concat)); + memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat)); + if (warning.traditional) { + warningf(&lexer_token.source_position, + "traditional C rejects string constant concatenation"); + } + + return (wide_string_t){ concat, len1 + len2 + 1 }; +} + +/** + * Concatenate a wide string and a string. + */ +wide_string_t concat_wide_string_string(const wide_string_t *const s1, const string_t *const s2) +{ + const size_t len1 = s1->size - 1; + const size_t len2 = s2->size - 1; + + wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat)); + memcpy(concat, s1->begin, len1 * sizeof(*concat)); + const char *const src = s2->begin; + wchar_rep_t *const dst = concat + len1; + for (size_t i = 0; i != len2 + 1; ++i) { + dst[i] = src[i]; + } + if (warning.traditional) { + warningf(&lexer_token.source_position, + "traditional C rejects string constant concatenation"); + } + + return (wide_string_t){ concat, len1 + len2 + 1 }; +} + +static void grow_symbol(utf32 const tc) +{ + struct obstack *const o = &symbol_obstack; + if (tc < 0x80U) { + obstack_1grow(o, tc); + } else if (tc < 0x800) { + obstack_1grow(o, 0xC0 | (tc >> 6)); + obstack_1grow(o, 0x80 | (tc & 0x3F)); + } else if (tc < 0x10000) { + obstack_1grow(o, 0xE0 | ( tc >> 12)); + obstack_1grow(o, 0x80 | ((tc >> 6) & 0x3F)); + obstack_1grow(o, 0x80 | ( tc & 0x3F)); + } else { + obstack_1grow(o, 0xF0 | ( tc >> 18)); + obstack_1grow(o, 0x80 | ((tc >> 12) & 0x3F)); + obstack_1grow(o, 0x80 | ((tc >> 6) & 0x3F)); + obstack_1grow(o, 0x80 | ( tc & 0x3F)); + } +} + +/** + * Parse a string literal and set lexer_token. + */ static void parse_string_literal(void) { - unsigned start_linenr = lexer_token.source_position.linenr; - char *string; - const char *result; + const unsigned start_linenr = lexer_token.source_position.linenr; - assert(c == '"'); - next_char(); + eat('"'); - int tc; while(1) { switch(c) { - case '\\': - tc = parse_escape_sequence(); + case '\\': { + utf32 const tc = parse_escape_sequence(); + if (tc >= 0x100) { + warningf(&lexer_token.source_position, + "escape sequence out of range"); + } obstack_1grow(&symbol_obstack, tc); break; + } - case EOF: - error_prefix_at(lexer_token.source_position.input_name, - start_linenr); - fprintf(stderr, "string has no end\n"); + case EOF: { + source_position_t source_position; + source_position.input_name = lexer_token.source_position.input_name; + source_position.linenr = start_linenr; + errorf(&source_position, "string has no end"); lexer_token.type = T_ERROR; return; + } case '"': next_char(); goto end_of_string; default: - obstack_1grow(&symbol_obstack, c); + grow_symbol(c); next_char(); break; } @@ -605,28 +1270,165 @@ end_of_string: /* add finishing 0 to the string */ obstack_1grow(&symbol_obstack, '\0'); - string = obstack_finish(&symbol_obstack); + const size_t size = (size_t)obstack_object_size(&symbol_obstack); + const char *const string = obstack_finish(&symbol_obstack); +#if 0 /* TODO hash */ /* check if there is already a copy of the string */ result = strset_insert(&stringset, string); if(result != string) { obstack_free(&symbol_obstack, string); } +#else + const char *const result = string; +#endif + + lexer_token.type = T_STRING_LITERAL; + lexer_token.v.string.begin = result; + lexer_token.v.string.size = size; +} + +/** + * Parse a wide character constant and set lexer_token. + */ +static void parse_wide_character_constant(void) +{ + const unsigned start_linenr = lexer_token.source_position.linenr; + + eat('\''); + + while(1) { + switch(c) { + case '\\': { + wchar_rep_t tc = parse_escape_sequence(); + obstack_grow(&symbol_obstack, &tc, sizeof(tc)); + break; + } + + MATCH_NEWLINE( + parse_error("newline while parsing character constant"); + break; + ) + + case '\'': + next_char(); + goto end_of_wide_char_constant; + + case EOF: { + source_position_t source_position = lexer_token.source_position; + source_position.linenr = start_linenr; + errorf(&source_position, "EOF while parsing character constant"); + lexer_token.type = T_ERROR; + return; + } + + default: { + wchar_rep_t tc = (wchar_rep_t) c; + obstack_grow(&symbol_obstack, &tc, sizeof(tc)); + next_char(); + break; + } + } + } + +end_of_wide_char_constant:; + size_t size = (size_t) obstack_object_size(&symbol_obstack); + assert(size % sizeof(wchar_rep_t) == 0); + size /= sizeof(wchar_rep_t); + + const wchar_rep_t *string = obstack_finish(&symbol_obstack); + + lexer_token.type = T_WIDE_CHARACTER_CONSTANT; + lexer_token.v.wide_string.begin = string; + lexer_token.v.wide_string.size = size; + lexer_token.datatype = type_wchar_t; +} + +/** + * Parse a wide string literal and set lexer_token. + */ +static void parse_wide_string_literal(void) +{ + const unsigned start_linenr = lexer_token.source_position.linenr; + + assert(c == '"'); + next_char(); + + while(1) { + switch(c) { + case '\\': { + wchar_rep_t tc = parse_escape_sequence(); + obstack_grow(&symbol_obstack, &tc, sizeof(tc)); + break; + } + + case EOF: { + source_position_t source_position; + source_position.input_name = lexer_token.source_position.input_name; + source_position.linenr = start_linenr; + errorf(&source_position, "string has no end"); + lexer_token.type = T_ERROR; + return; + } + + case '"': + next_char(); + goto end_of_string; + + default: { + wchar_rep_t tc = c; + obstack_grow(&symbol_obstack, &tc, sizeof(tc)); + next_char(); + break; + } + } + } + +end_of_string:; + + /* TODO: concatenate multiple strings separated by whitespace... */ + + /* add finishing 0 to the string */ + wchar_rep_t nul = L'\0'; + obstack_grow(&symbol_obstack, &nul, sizeof(nul)); + const size_t size = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t); + const wchar_rep_t *const string = obstack_finish(&symbol_obstack); + +#if 0 /* TODO hash */ + /* check if there is already a copy of the string */ + const wchar_rep_t *const result = strset_insert(&stringset, string); + if(result != string) { + obstack_free(&symbol_obstack, string); + } +#else + const wchar_rep_t *const result = string; +#endif - lexer_token.type = T_STRING_LITERAL; - lexer_token.v.string = result; + lexer_token.type = T_WIDE_STRING_LITERAL; + lexer_token.v.wide_string.begin = result; + lexer_token.v.wide_string.size = size; } +/** + * Parse a character constant and set lexer_token. + */ static void parse_character_constant(void) { + const unsigned start_linenr = lexer_token.source_position.linenr; + eat('\''); - int found_char = 0; while(1) { switch(c) { - case '\\': - found_char = parse_escape_sequence(); + case '\\': { + utf32 const tc = parse_escape_sequence(); + if (tc >= 0x100) { + warningf(&lexer_token.source_position, + "escape sequence out of range"); + } + obstack_1grow(&symbol_obstack, tc); break; + } MATCH_NEWLINE( parse_error("newline while parsing character constant"); @@ -637,35 +1439,51 @@ static void parse_character_constant(void) next_char(); goto end_of_char_constant; - case EOF: - parse_error("EOF while parsing character constant"); + case EOF: { + source_position_t source_position; + source_position.input_name = lexer_token.source_position.input_name; + source_position.linenr = start_linenr; + errorf(&source_position, "EOF while parsing character constant"); lexer_token.type = T_ERROR; return; + } default: - if(found_char != 0) { - parse_error("more than 1 characters in character " - "constant"); - goto end_of_char_constant; - } else { - found_char = c; - next_char(); - } + grow_symbol(c); + next_char(); break; + } } -end_of_char_constant: - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = found_char; +end_of_char_constant:; + const size_t size = (size_t)obstack_object_size(&symbol_obstack); + const char *const string = obstack_finish(&symbol_obstack); + + lexer_token.type = T_CHARACTER_CONSTANT; + lexer_token.v.string.begin = string; + lexer_token.v.string.size = size; + lexer_token.datatype = c_mode & _CXX && size == 1 ? type_char : type_int; } +/** + * Skip a multiline comment. + */ static void skip_multiline_comment(void) { unsigned start_linenr = lexer_token.source_position.linenr; while(1) { switch(c) { + case '/': + next_char(); + if (c == '*') { + /* nested comment, warn here */ + if (warning.comment) { + warningf(&lexer_token.source_position, "'/*' within comment"); + } + } + break; case '*': next_char(); if(c == '/') { @@ -676,11 +1494,13 @@ static void skip_multiline_comment(void) MATCH_NEWLINE(break;) - case EOF: - error_prefix_at(lexer_token.source_position.input_name, - start_linenr); - fprintf(stderr, "at end of file while looking for comment end\n"); + case EOF: { + source_position_t source_position; + source_position.input_name = lexer_token.source_position.input_name; + source_position.linenr = start_linenr; + errorf(&source_position, "at end of file while looking for comment end"); return; + } default: next_char(); @@ -689,6 +1509,9 @@ static void skip_multiline_comment(void) } } +/** + * Skip a single line comment. + */ static void skip_line_comment(void) { while(1) { @@ -700,6 +1523,15 @@ static void skip_line_comment(void) case '\r': return; + case '\\': + next_char(); + if (c == '\n' || c == '\r') { + if (warning.comment) + warningf(&lexer_token.source_position, "multi-line comment"); + return; + } + break; + default: next_char(); break; @@ -707,14 +1539,21 @@ static void skip_line_comment(void) } } +/** The current preprocessor token. */ static token_t pp_token; +/** + * Read the next preprocessor token. + */ static inline void next_pp_token(void) { lexer_next_preprocessing_token(); pp_token = lexer_token; } +/** + * Eat all preprocessor tokens until newline. + */ static void eat_until_newline(void) { while(pp_token.type != '\n' && pp_token.type != T_EOF) { @@ -722,14 +1561,9 @@ static void eat_until_newline(void) } } -static void error_directive(void) -{ - error_prefix(); - fprintf(stderr, "#error directive: \n"); - - /* parse pp-tokens until new-line */ -} - +/** + * Handle the define directive. + */ static void define_directive(void) { lexer_next_preprocessing_token(); @@ -739,6 +1573,9 @@ static void define_directive(void) } } +/** + * Handle the ifdef directive. + */ static void ifdef_directive(int is_ifndef) { (void) is_ifndef; @@ -747,27 +1584,115 @@ static void ifdef_directive(int is_ifndef) //extect_newline(); } +/** + * Handle the endif directive. + */ static void endif_directive(void) { //expect_newline(); } +/** + * Parse the line directive. + */ static void parse_line_directive(void) { if(pp_token.type != T_INTEGER) { parse_error("expected integer"); } else { - lexer_token.source_position.linenr = pp_token.v.intvalue - 1; + lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1); next_pp_token(); } if(pp_token.type == T_STRING_LITERAL) { - lexer_token.source_position.input_name = pp_token.v.string; + lexer_token.source_position.input_name = pp_token.v.string.begin; next_pp_token(); } eat_until_newline(); } +/** + * STDC pragmas. + */ +typedef enum stdc_pragma_kind_t { + STDC_UNKNOWN, + STDC_FP_CONTRACT, + STDC_FENV_ACCESS, + STDC_CX_LIMITED_RANGE +} stdc_pragma_kind_t; + +/** + * STDC pragma values. + */ +typedef enum stdc_pragma_value_kind_t { + STDC_VALUE_UNKNOWN, + STDC_VALUE_ON, + STDC_VALUE_OFF, + STDC_VALUE_DEFAULT +} stdc_pragma_value_kind_t; + +/** + * Parse a pragma directive. + */ +static void parse_pragma(void) +{ + bool unknown_pragma = true; + + next_pp_token(); + if (pp_token.v.symbol->pp_ID == TP_STDC) { + stdc_pragma_kind_t kind = STDC_UNKNOWN; + /* a STDC pragma */ + if (c_mode & _C99) { + next_pp_token(); + + switch (pp_token.v.symbol->pp_ID) { + case TP_FP_CONTRACT: + kind = STDC_FP_CONTRACT; + break; + case TP_FENV_ACCESS: + kind = STDC_FENV_ACCESS; + break; + case TP_CX_LIMITED_RANGE: + kind = STDC_CX_LIMITED_RANGE; + break; + default: + break; + } + if (kind != STDC_UNKNOWN) { + stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN; + next_pp_token(); + switch (pp_token.v.symbol->pp_ID) { + case TP_ON: + value = STDC_VALUE_ON; + break; + case TP_OFF: + value = STDC_VALUE_OFF; + break; + case TP_DEFAULT: + value = STDC_VALUE_DEFAULT; + break; + default: + break; + } + if (value != STDC_VALUE_UNKNOWN) { + unknown_pragma = false; + } else { + errorf(&pp_token.source_position, "bad STDC pragma argument"); + } + } + } + } else { + unknown_pragma = true; + } + eat_until_newline(); + if (unknown_pragma && warning.unknown_pragmas) { + warningf(&pp_token.source_position, "encountered unknown #pragma"); + } +} + +/** + * Parse a preprocessor non-null directive. + */ static void parse_preprocessor_identifier(void) { assert(pp_token.type == T_IDENTIFIER); @@ -798,13 +1723,18 @@ static void parse_preprocessor_identifier(void) case TP_elif: case TP_undef: case TP_error: - error_directive(); + /* TODO; output the rest of the line */ + parse_error("#error directive: "); break; case TP_pragma: + parse_pragma(); break; } } +/** + * Parse a preprocessor directive. + */ static void parse_preprocessor_directive(void) { next_pp_token(); @@ -816,6 +1746,9 @@ static void parse_preprocessor_directive(void) case T_INTEGER: parse_line_directive(); break; + case '\n': + /* NULL directive, see §6.10.7 */ + break; default: parse_error("invalid preprocessor directive"); eat_until_newline(); @@ -836,7 +1769,7 @@ static void parse_preprocessor_directive(void) #define ELSE_CODE(code) \ default: \ - code; \ + code \ } \ } /* end of while(1) */ \ break; @@ -864,10 +1797,13 @@ void lexer_next_preprocessing_token(void) SYMBOL_CHARS parse_symbol(); /* might be a wide string ( L"string" ) */ - if(c == '"' && (lexer_token.type == T_IDENTIFIER && - lexer_token.v.symbol == symbol_L)) { - parse_string_literal(); - return; + if(lexer_token.type == T_IDENTIFIER && + lexer_token.v.symbol == symbol_L) { + if(c == '"') { + parse_wide_string_literal(); + } else if(c == '\'') { + parse_wide_character_constant(); + } } return; @@ -885,6 +1821,12 @@ void lexer_next_preprocessing_token(void) case '.': MAYBE_PROLOG + DIGITS + put_back(c); + c = '.'; + parse_number_dec(); + return; + case '.': MAYBE_PROLOG MAYBE('.', T_DOTDOTDOT) @@ -935,25 +1877,25 @@ void lexer_next_preprocessing_token(void) ELSE('/') case '%': MAYBE_PROLOG - MAYBE('>', T_PERCENTGREATER) + MAYBE('>', '}') MAYBE('=', T_PERCENTEQUAL) case ':': MAYBE_PROLOG case '%': MAYBE_PROLOG - MAYBE(':', T_PERCENTCOLONPERCENTCOLON) + MAYBE(':', T_HASHHASH) ELSE_CODE( put_back(c); c = '%'; - lexer_token.type = T_PERCENTCOLON; + lexer_token.type = '#'; return; ) - ELSE(T_PERCENTCOLON) + ELSE('#') ELSE('%') case '<': MAYBE_PROLOG - MAYBE(':', T_LESSCOLON) - MAYBE('%', T_LESSPERCENT) + MAYBE(':', '[') + MAYBE('%', '{') MAYBE('=', T_LESSEQUAL) case '<': MAYBE_PROLOG @@ -979,7 +1921,7 @@ void lexer_next_preprocessing_token(void) ELSE('|') case ':': MAYBE_PROLOG - MAYBE('>', T_COLONGREATER) + MAYBE('>', ']') ELSE(':') case '=': MAYBE_PROLOG @@ -1010,9 +1952,9 @@ void lexer_next_preprocessing_token(void) return; default: +dollar_sign: + errorf(&lexer_token.source_position, "unknown character '%c' found", c); next_char(); - error_prefix(); - fprintf(stderr, "unknown character '%c' found\n", c); lexer_token.type = T_ERROR; return; } @@ -1022,15 +1964,13 @@ void lexer_next_preprocessing_token(void) void lexer_next_token(void) { lexer_next_preprocessing_token(); - if(lexer_token.type != '\n') - return; + while (lexer_token.type == '\n') { newline_found: - do { lexer_next_preprocessing_token(); - } while(lexer_token.type == '\n'); + } - if(lexer_token.type == '#') { + if (lexer_token.type == '#') { parse_preprocessor_directive(); goto newline_found; } @@ -1039,6 +1979,7 @@ newline_found: void init_lexer(void) { strset_init(&stringset); + symbol_L = symbol_table_insert("L"); } void lexer_open_stream(FILE *stream, const char *input_name) @@ -1047,7 +1988,28 @@ void lexer_open_stream(FILE *stream, const char *input_name) lexer_token.source_position.linenr = 0; lexer_token.source_position.input_name = input_name; - symbol_L = symbol_table_insert("L"); + bufpos = NULL; + bufend = NULL; + + /* place a virtual \n at the beginning so the lexer knows that we're + * at the beginning of a line */ + c = '\n'; +} + +void lexer_open_buffer(const char *buffer, size_t len, const char *input_name) +{ + input = NULL; + lexer_token.source_position.linenr = 0; + lexer_token.source_position.input_name = input_name; + +#if 0 // TODO + bufpos = buffer; + bufend = buffer + len; +#else + (void)buffer; + (void)len; + panic("builtin lexing not done yet"); +#endif /* place a virtual \n at the beginning so the lexer knows that we're * at the beginning of a line */ @@ -1062,7 +2024,7 @@ void exit_lexer(void) static __attribute__((unused)) void dbg_pos(const source_position_t source_position) { - fprintf(stdout, "%s:%d\n", source_position.input_name, + fprintf(stdout, "%s:%u\n", source_position.input_name, source_position.linenr); fflush(stdout); }