X-Git-Url: http://nsz.repo.hu/git/?a=blobdiff_plain;f=lexer.c;h=fe51c2ffa176b12abb03470be5ca50ae5a289b35;hb=e7c87251c9a4fd96a0374bd64f0a17068cdf2eaf;hp=be337acce4e19e779e53bb3892eace1a2b6f5508;hpb=db53eee277e14326fb656c9c389be4b50504c7ee;p=cparser diff --git a/lexer.c b/lexer.c index be337ac..fe51c2f 100644 --- a/lexer.c +++ b/lexer.c @@ -1,7 +1,27 @@ +/* + * This file is part of cparser. + * Copyright (C) 2007-2008 Matthias Braun + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA + * 02111-1307, USA. + */ #include #include "diagnostic.h" #include "lexer.h" +#include "symbol_t.h" #include "token_t.h" #include "symbol_table_t.h" #include "adt/error.h" @@ -11,6 +31,8 @@ #include "type_t.h" #include "target_architecture.h" #include "parser.h" +#include "warning.h" +#include "lang_features.h" #include #include @@ -26,14 +48,6 @@ #define strtold(s, e) strtod(s, e) #endif -#if defined HAS_SIGNED_CHAR -typedef signed char char_type; -#elif defined HAS_UNSIGNED_CHAR -typedef unsigned char char_type; -#else -# error signedness of char not determined -#endif - static int c; token_t lexer_token; symbol_t *symbol_L; @@ -43,27 +57,35 @@ static const char *bufend; static const char *bufpos; static strset_t stringset; -static void error_prefix_at(const char *input_name, unsigned linenr) -{ - fprintf(stderr, "%s:%u: Error: ", input_name, linenr); -} - -static void error_prefix(void) +/** + * Prints a parse error message at the current token. + * + * @param msg the error message + */ +static void parse_error(const char *msg) { - error_prefix_at(lexer_token.source_position.input_name, - lexer_token.source_position.linenr); + errorf(&lexer_token.source_position, "%s", msg); } -static void parse_error(const char *msg) +/** + * Prints an internal error message at the current token. + * + * @param msg the error message + */ +static NORETURN internal_error(const char *msg) { - error_prefix(); - fprintf(stderr, "%s\n", msg); + internal_errorf(&lexer_token.source_position, "%s", msg); } static inline void next_real_char(void) { - bufpos++; - if(bufpos >= bufend) { + assert(bufpos <= bufend); + if (bufpos >= bufend) { + if (input == NULL) { + c = EOF; + return; + } + size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK, input); if(s == 0) { @@ -73,20 +95,18 @@ static inline void next_real_char(void) bufpos = buf + MAX_PUTBACK; bufend = buf + MAX_PUTBACK + s; } - c = *(bufpos); + c = *bufpos++; } +/** + * Put a character back into the buffer. + * + * @param pc the character to put back + */ static inline void put_back(int pc) { - assert(bufpos >= buf); - //assert(bufpos < buf+MAX_PUTBACK || *bufpos == pc); - - char *p = buf + (bufpos - buf); - *p = (char) pc; - - /* going backwards in the buffer is legal as long as it's not more often - * than MAX_PUTBACK */ - bufpos--; + assert(bufpos > buf); + *(--bufpos - buf + buf) = (char) pc; #ifdef DEBUG_CHARS printf("putback '%c'\n", pc); @@ -102,11 +122,11 @@ static inline void next_char(void); next_char(); \ } \ lexer_token.source_position.linenr++; \ - code; \ + code \ case '\n': \ next_char(); \ lexer_token.source_position.linenr++; \ - code; + code #define eat(c_type) do { assert(c == c_type); next_char(); } while(0) @@ -125,6 +145,10 @@ static void maybe_concat_lines(void) c = '\\'; } +/** + * Set c to the next input character, ie. + * after expanding trigraphs. + */ static inline void next_char(void) { next_real_char(); @@ -236,6 +260,10 @@ end_of_next_char:; case '8': \ case '9': +/** + * Read a symbol from the input and build + * the lexer_token. + */ static void parse_symbol(void) { symbol_t *symbol; @@ -320,7 +348,7 @@ static void parse_integer_suffix(bool is_oct_hex) if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) { lexer_token.datatype = type_long; return; - } else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) { + } else if(is_oct_hex && v >= 0 && (unsigned long long)v <= (unsigned long long)TARGET_ULONG_MAX) { lexer_token.datatype = type_unsigned_long; return; } @@ -349,7 +377,7 @@ static void parse_integer_suffix(bool is_oct_hex) static void parse_floating_suffix(void) { switch(c) { - /* TODO: do something usefull with the suffixes... */ + /* TODO: do something useful with the suffixes... */ case 'f': case 'F': next_char(); @@ -450,6 +478,10 @@ end: return v; } +/** + * Parses a hex number including hex floats and set the + * lexer_token. + */ static void parse_number_hex(void) { assert(c == 'x' || c == 'X'); @@ -464,7 +496,7 @@ static void parse_number_hex(void) if(c == '.' || c == 'p' || c == 'P') { next_char(); - panic("Hex floating point numbers not implemented yet"); + internal_error("Hex floating point numbers not implemented yet"); } if(*string == '\0') { parse_error("invalid hex number"); @@ -482,11 +514,31 @@ static void parse_number_hex(void) parse_integer_suffix(true); } +/** + * Returns true if the given char is a octal digit. + * + * @param char the character to check + */ static inline bool is_octal_digit(int chr) { - return '0' <= chr && chr <= '7'; + switch(chr) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + return true; + default: + return false; + } } +/** + * Parses a octal number and set the lexer_token. + */ static void parse_number_oct(void) { while(is_octal_digit(c)) { @@ -507,6 +559,10 @@ static void parse_number_oct(void) parse_integer_suffix(true); } +/** + * Parses a decimal including float number and set the + * lexer_token. + */ static void parse_number_dec(void) { bool is_float = false; @@ -568,6 +624,9 @@ static void parse_number_dec(void) obstack_free(&symbol_obstack, string); } +/** + * Parses a number and sets the lexer_token. + */ static void parse_number(void) { if (c == '0') { @@ -606,38 +665,83 @@ static void parse_number(void) } } +/** + * Returns the value of a digit. + * The only portable way to do it ... + */ +static int digit_value(int digit) { + switch (digit) { + case '0': return 0; + case '1': return 1; + case '2': return 2; + case '3': return 3; + case '4': return 4; + case '5': return 5; + case '6': return 6; + case '7': return 7; + case '8': return 8; + case '9': return 9; + case 'a': + case 'A': return 10; + case 'b': + case 'B': return 11; + case 'c': + case 'C': return 12; + case 'd': + case 'D': return 13; + case 'e': + case 'E': return 14; + case 'f': + case 'F': return 15; + default: + internal_error("wrong character given"); + } +} + +/** + * Parses an octal character sequence. + * + * @param first_digit the already read first digit + */ static int parse_octal_sequence(const int first_digit) { assert(is_octal_digit(first_digit)); - int value = first_digit - '0'; + int value = digit_value(first_digit); if (!is_octal_digit(c)) return value; - value = 8 * value + c - '0'; + value = 8 * value + digit_value(c); next_char(); if (!is_octal_digit(c)) return value; - value = 8 * value + c - '0'; + value = 8 * value + digit_value(c); next_char(); - return (char_type)value; + + if(char_is_signed) { + return (signed char) value; + } else { + return (unsigned char) value; + } } +/** + * Parses a hex character sequence. + */ static int parse_hex_sequence(void) { int value = 0; - while(1) { - if (c >= '0' && c <= '9') { - value = 16 * value + c - '0'; - } else if ('A' <= c && c <= 'F') { - value = 16 * value + c - 'A' + 10; - } else if ('a' <= c && c <= 'f') { - value = 16 * value + c - 'a' + 10; - } else { - break; - } + while(isxdigit(c)) { + value = 16 * value + digit_value(c); next_char(); } - return (char_type)value; + if(char_is_signed) { + return (signed char) value; + } else { + return (unsigned char) value; + } } +/** + * Parse an escape sequence. + */ static int parse_escape_sequence(void) { eat('\\'); @@ -677,31 +781,89 @@ static int parse_escape_sequence(void) } } -const char *concat_strings(const char *s1, const char *s2) +/** + * Concatenate two strings. + */ +string_t concat_strings(const string_t *const s1, const string_t *const s2) { - size_t len1 = strlen(s1); - size_t len2 = strlen(s2); + const size_t len1 = s1->size - 1; + const size_t len2 = s2->size - 1; - char *concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1); - memcpy(concat, s1, len1); - memcpy(concat + len1, s2, len2 + 1); + char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1); + memcpy(concat, s1->begin, len1); + memcpy(concat + len1, s2->begin, len2 + 1); +#if 0 /* TODO hash */ const char *result = strset_insert(&stringset, concat); if(result != concat) { obstack_free(&symbol_obstack, concat); } return result; +#else + return (string_t){ concat, len1 + len2 + 1 }; +#endif +} + +/** + * Concatenate a string and a wide string. + */ +wide_string_t concat_string_wide_string(const string_t *const s1, const wide_string_t *const s2) +{ + const size_t len1 = s1->size - 1; + const size_t len2 = s2->size - 1; + + wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat)); + const char *const src = s1->begin; + for (size_t i = 0; i != len1; ++i) { + concat[i] = src[i]; + } + memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat)); + + return (wide_string_t){ concat, len1 + len2 + 1 }; +} + +/** + * Concatenate two wide strings. + */ +wide_string_t concat_wide_strings(const wide_string_t *const s1, const wide_string_t *const s2) +{ + const size_t len1 = s1->size - 1; + const size_t len2 = s2->size - 1; + + wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat)); + memcpy(concat, s1->begin, len1 * sizeof(*concat)); + memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat)); + + return (wide_string_t){ concat, len1 + len2 + 1 }; } +/** + * Concatenate a wide string and a string. + */ +wide_string_t concat_wide_string_string(const wide_string_t *const s1, const string_t *const s2) +{ + const size_t len1 = s1->size - 1; + const size_t len2 = s2->size - 1; + + wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat)); + memcpy(concat, s1->begin, len1 * sizeof(*concat)); + const char *const src = s2->begin; + for (size_t i = 0; i != len2 + 1; ++i) { + concat[i] = src[i]; + } + + return (wide_string_t){ concat, len1 + len2 + 1 }; +} + +/** + * Parse a string literal and set lexer_token. + */ static void parse_string_literal(void) { - unsigned start_linenr = lexer_token.source_position.linenr; - char *string; - const char *result; + const unsigned start_linenr = lexer_token.source_position.linenr; - assert(c == '"'); - next_char(); + eat('"'); int tc; while(1) { @@ -711,12 +873,14 @@ static void parse_string_literal(void) obstack_1grow(&symbol_obstack, (char) tc); break; - case EOF: - error_prefix_at(lexer_token.source_position.input_name, - start_linenr); - fprintf(stderr, "string has no end\n"); + case EOF: { + source_position_t source_position; + source_position.input_name = lexer_token.source_position.input_name; + source_position.linenr = start_linenr; + errorf(&source_position, "string has no end"); lexer_token.type = T_ERROR; return; + } case '"': next_char(); @@ -735,28 +899,40 @@ end_of_string: /* add finishing 0 to the string */ obstack_1grow(&symbol_obstack, '\0'); - string = obstack_finish(&symbol_obstack); + const size_t size = (size_t)obstack_object_size(&symbol_obstack); + const char *const string = obstack_finish(&symbol_obstack); +#if 0 /* TODO hash */ /* check if there is already a copy of the string */ result = strset_insert(&stringset, string); if(result != string) { obstack_free(&symbol_obstack, string); } +#else + const char *const result = string; +#endif - lexer_token.type = T_STRING_LITERAL; - lexer_token.v.string = result; + lexer_token.type = T_STRING_LITERAL; + lexer_token.v.string.begin = result; + lexer_token.v.string.size = size; } +/** + * Parse a wide character constant and set lexer_token. + */ static void parse_wide_character_constant(void) { + const unsigned start_linenr = lexer_token.source_position.linenr; + eat('\''); - int found_char = 0; while(1) { switch(c) { - case '\\': - found_char = parse_escape_sequence(); + case '\\': { + wchar_rep_t tc = parse_escape_sequence(); + obstack_grow(&symbol_obstack, &tc, sizeof(tc)); break; + } MATCH_NEWLINE( parse_error("newline while parsing character constant"); @@ -767,30 +943,39 @@ static void parse_wide_character_constant(void) next_char(); goto end_of_wide_char_constant; - case EOF: - parse_error("EOF while parsing character constant"); + case EOF: { + source_position_t source_position = lexer_token.source_position; + source_position.linenr = start_linenr; + errorf(&source_position, "EOF while parsing character constant"); lexer_token.type = T_ERROR; return; + } - default: - if(found_char != 0) { - parse_error("more than 1 characters in character " - "constant"); - goto end_of_wide_char_constant; - } else { - found_char = c; - next_char(); - } + default: { + wchar_rep_t tc = (wchar_rep_t) c; + obstack_grow(&symbol_obstack, &tc, sizeof(tc)); + next_char(); break; } + } } -end_of_wide_char_constant: - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = found_char; - lexer_token.datatype = type_wchar_t; +end_of_wide_char_constant:; + size_t size = (size_t) obstack_object_size(&symbol_obstack); + assert(size % sizeof(wchar_rep_t) == 0); + size /= sizeof(wchar_rep_t); + + const wchar_rep_t *string = obstack_finish(&symbol_obstack); + + lexer_token.type = T_WIDE_CHARACTER_CONSTANT; + lexer_token.v.wide_string.begin = string; + lexer_token.v.wide_string.size = size; + lexer_token.datatype = type_wchar_t; } +/** + * Parse a wide string literal and set lexer_token. + */ static void parse_wide_string_literal(void) { const unsigned start_linenr = lexer_token.source_position.linenr; @@ -800,29 +985,31 @@ static void parse_wide_string_literal(void) while(1) { switch(c) { - case '\\': { - wchar_rep_t tc = parse_escape_sequence(); - obstack_grow(&symbol_obstack, &tc, sizeof(tc)); - break; - } + case '\\': { + wchar_rep_t tc = parse_escape_sequence(); + obstack_grow(&symbol_obstack, &tc, sizeof(tc)); + break; + } - case EOF: - error_prefix_at(lexer_token.source_position.input_name, - start_linenr); - fprintf(stderr, "string has no end\n"); - lexer_token.type = T_ERROR; - return; + case EOF: { + source_position_t source_position; + source_position.input_name = lexer_token.source_position.input_name; + source_position.linenr = start_linenr; + errorf(&source_position, "string has no end"); + lexer_token.type = T_ERROR; + return; + } - case '"': - next_char(); - goto end_of_string; + case '"': + next_char(); + goto end_of_string; - default: { - wchar_rep_t tc = c; - obstack_grow(&symbol_obstack, &tc, sizeof(tc)); - next_char(); - break; - } + default: { + wchar_rep_t tc = c; + obstack_grow(&symbol_obstack, &tc, sizeof(tc)); + next_char(); + break; + } } } @@ -851,16 +1038,22 @@ end_of_string:; lexer_token.v.wide_string.size = size; } +/** + * Parse a character constant and set lexer_token. + */ static void parse_character_constant(void) { + const unsigned start_linenr = lexer_token.source_position.linenr; + eat('\''); - int found_char = 0; while(1) { switch(c) { - case '\\': - found_char = parse_escape_sequence(); + case '\\': { + int tc = parse_escape_sequence(); + obstack_1grow(&symbol_obstack, (char) tc); break; + } MATCH_NEWLINE( parse_error("newline while parsing character constant"); @@ -871,36 +1064,48 @@ static void parse_character_constant(void) next_char(); goto end_of_char_constant; - case EOF: - parse_error("EOF while parsing character constant"); + case EOF: { + source_position_t source_position; + source_position.input_name = lexer_token.source_position.input_name; + source_position.linenr = start_linenr; + errorf(&source_position, "EOF while parsing character constant"); lexer_token.type = T_ERROR; return; + } default: - if(found_char != 0) { - parse_error("more than 1 characters in character " - "constant"); - goto end_of_char_constant; - } else { - found_char = c; - next_char(); - } + obstack_1grow(&symbol_obstack, (char) c); + next_char(); break; + } } -end_of_char_constant: - lexer_token.type = T_INTEGER; - lexer_token.v.intvalue = found_char; - lexer_token.datatype = type_int; +end_of_char_constant:; + const size_t size = (size_t)obstack_object_size(&symbol_obstack); + const char *const string = obstack_finish(&symbol_obstack); + + lexer_token.type = T_CHARACTER_CONSTANT; + lexer_token.v.string.begin = string; + lexer_token.v.string.size = size; + lexer_token.datatype = type_int; } +/** + * Skip a multiline comment. + */ static void skip_multiline_comment(void) { unsigned start_linenr = lexer_token.source_position.linenr; while(1) { switch(c) { + case '/': + next_char(); + if (c == '*') { + /* TODO: nested comment, warn here */ + } + break; case '*': next_char(); if(c == '/') { @@ -911,11 +1116,13 @@ static void skip_multiline_comment(void) MATCH_NEWLINE(break;) - case EOF: - error_prefix_at(lexer_token.source_position.input_name, - start_linenr); - fprintf(stderr, "at end of file while looking for comment end\n"); + case EOF: { + source_position_t source_position; + source_position.input_name = lexer_token.source_position.input_name; + source_position.linenr = start_linenr; + errorf(&source_position, "at end of file while looking for comment end"); return; + } default: next_char(); @@ -924,6 +1131,9 @@ static void skip_multiline_comment(void) } } +/** + * Skip a single line comment. + */ static void skip_line_comment(void) { while(1) { @@ -942,14 +1152,21 @@ static void skip_line_comment(void) } } +/** The current preprocessor token. */ static token_t pp_token; +/** + * Read the next preprocessor token. + */ static inline void next_pp_token(void) { lexer_next_preprocessing_token(); pp_token = lexer_token; } +/** + * Eat all preprocessor tokens until newline. + */ static void eat_until_newline(void) { while(pp_token.type != '\n' && pp_token.type != T_EOF) { @@ -957,14 +1174,9 @@ static void eat_until_newline(void) } } -static void error_directive(void) -{ - error_prefix(); - fprintf(stderr, "#error directive: \n"); - - /* parse pp-tokens until new-line */ -} - +/** + * Handle the define directive. + */ static void define_directive(void) { lexer_next_preprocessing_token(); @@ -974,6 +1186,9 @@ static void define_directive(void) } } +/** + * Handle the ifdef directive. + */ static void ifdef_directive(int is_ifndef) { (void) is_ifndef; @@ -982,11 +1197,17 @@ static void ifdef_directive(int is_ifndef) //extect_newline(); } +/** + * Handle the endif directive. + */ static void endif_directive(void) { //expect_newline(); } +/** + * Parse the line directive. + */ static void parse_line_directive(void) { if(pp_token.type != T_INTEGER) { @@ -996,13 +1217,94 @@ static void parse_line_directive(void) next_pp_token(); } if(pp_token.type == T_STRING_LITERAL) { - lexer_token.source_position.input_name = pp_token.v.string; + lexer_token.source_position.input_name = pp_token.v.string.begin; next_pp_token(); } eat_until_newline(); } +/** + * STDC pragmas. + */ +typedef enum { + STDC_UNKNOWN, + STDC_FP_CONTRACT, + STDC_FENV_ACCESS, + STDC_CX_LIMITED_RANGE +} stdc_pragma_kind_t; + +/** + * STDC pragma values. + */ +typedef enum { + STDC_VALUE_UNKNOWN, + STDC_VALUE_ON, + STDC_VALUE_OFF, + STDC_VALUE_DEFAULT +} stdc_pragma_value_kind_t; + +/** + * Parse a pragma directive. + */ +static void parse_pragma(void) { + bool unknown_pragma = true; + + next_pp_token(); + if (pp_token.v.symbol->pp_ID == TP_STDC) { + stdc_pragma_kind_t kind = STDC_UNKNOWN; + /* a STDC pragma */ + if (c_mode & _C99) { + next_pp_token(); + + switch (pp_token.v.symbol->pp_ID) { + case TP_FP_CONTRACT: + kind = STDC_FP_CONTRACT; + break; + case TP_FENV_ACCESS: + kind = STDC_FENV_ACCESS; + break; + case TP_CX_LIMITED_RANGE: + kind = STDC_CX_LIMITED_RANGE; + break; + default: + break; + } + if (kind != STDC_UNKNOWN) { + stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN; + next_pp_token(); + switch (pp_token.v.symbol->pp_ID) { + case TP_ON: + value = STDC_VALUE_ON; + break; + case TP_OFF: + value = STDC_VALUE_OFF; + break; + case TP_DEFAULT: + value = STDC_VALUE_DEFAULT; + break; + default: + break; + } + if (value != STDC_VALUE_UNKNOWN) { + unknown_pragma = false; + } else { + errorf(&pp_token.source_position, "bad STDC pragma argument"); + } + } + } + } else { + unknown_pragma = true; + } + eat_until_newline(); + if (unknown_pragma && warning.unknown_pragmas) { + warningf(&pp_token.source_position, "encountered unknown #pragma"); + } +} + +/** + * Parse a preprocessor non-null directive. + */ static void parse_preprocessor_identifier(void) { assert(pp_token.type == T_IDENTIFIER); @@ -1033,15 +1335,18 @@ static void parse_preprocessor_identifier(void) case TP_elif: case TP_undef: case TP_error: - error_directive(); + /* TODO; output the rest of the line */ + parse_error("#error directive: "); break; case TP_pragma: - warningf(lexer_token.source_position, "encountered unknown #pragma"); - eat_until_newline(); + parse_pragma(); break; } } +/** + * Parse a preprocessor directive. + */ static void parse_preprocessor_directive(void) { next_pp_token(); @@ -1053,6 +1358,9 @@ static void parse_preprocessor_directive(void) case T_INTEGER: parse_line_directive(); break; + case '\n': + /* NULL directive, see § 6.10.7 */ + break; default: parse_error("invalid preprocessor directive"); eat_until_newline(); @@ -1073,7 +1381,7 @@ static void parse_preprocessor_directive(void) #define ELSE_CODE(code) \ default: \ - code; \ + code \ } \ } /* end of while(1) */ \ break; @@ -1190,25 +1498,25 @@ void lexer_next_preprocessing_token(void) ELSE('/') case '%': MAYBE_PROLOG - MAYBE('>', T_PERCENTGREATER) + MAYBE('>', '}') MAYBE('=', T_PERCENTEQUAL) case ':': MAYBE_PROLOG case '%': MAYBE_PROLOG - MAYBE(':', T_PERCENTCOLONPERCENTCOLON) + MAYBE(':', T_HASHHASH) ELSE_CODE( put_back(c); c = '%'; - lexer_token.type = T_PERCENTCOLON; + lexer_token.type = '#'; return; ) - ELSE(T_PERCENTCOLON) + ELSE('#') ELSE('%') case '<': MAYBE_PROLOG - MAYBE(':', T_LESSCOLON) - MAYBE('%', T_LESSPERCENT) + MAYBE(':', '[') + MAYBE('%', '{') MAYBE('=', T_LESSEQUAL) case '<': MAYBE_PROLOG @@ -1234,7 +1542,7 @@ void lexer_next_preprocessing_token(void) ELSE('|') case ':': MAYBE_PROLOG - MAYBE('>', T_COLONGREATER) + MAYBE('>', ']') ELSE(':') case '=': MAYBE_PROLOG @@ -1266,8 +1574,7 @@ void lexer_next_preprocessing_token(void) default: next_char(); - error_prefix(); - fprintf(stderr, "unknown character '%c' found\n", c); + errorf(&lexer_token.source_position, "unknown character '%c' found\n", c); lexer_token.type = T_ERROR; return; } @@ -1294,6 +1601,7 @@ newline_found: void init_lexer(void) { strset_init(&stringset); + symbol_L = symbol_table_insert("L"); } void lexer_open_stream(FILE *stream, const char *input_name) @@ -1302,7 +1610,22 @@ void lexer_open_stream(FILE *stream, const char *input_name) lexer_token.source_position.linenr = 0; lexer_token.source_position.input_name = input_name; - symbol_L = symbol_table_insert("L"); + bufpos = NULL; + bufend = NULL; + + /* place a virtual \n at the beginning so the lexer knows that we're + * at the beginning of a line */ + c = '\n'; +} + +void lexer_open_buffer(const char *buffer, size_t len, const char *input_name) +{ + input = NULL; + lexer_token.source_position.linenr = 0; + lexer_token.source_position.input_name = input_name; + + bufpos = buffer; + bufend = buffer + len; /* place a virtual \n at the beginning so the lexer knows that we're * at the beginning of a line */