3 #include "diagnostic.h"
6 #include "symbol_table_t.h"
8 #include "adt/strset.h"
12 #include "target_architecture.h"
26 /* No strtold on windows and no replacement yet */
27 #define strtold(s, e) strtod(s, e)
30 #if defined HAS_SIGNED_CHAR
31 typedef signed char char_type;
32 #elif defined HAS_UNSIGNED_CHAR
33 typedef unsigned char char_type;
35 # error signedness of char not determined
42 static char buf[1024 + MAX_PUTBACK];
43 static const char *bufend;
44 static const char *bufpos;
45 static strset_t stringset;
48 * Print an error prefix at the given coordinates.
50 * @param input_name the input file name
51 * @param linenr the line number
53 static void error_prefix_at(const char *input_name, unsigned linenr)
55 fprintf(stderr, "%s:%u: Error: ", input_name, linenr);
59 * Print an error prefix at the current token coordinates.
61 static void error_prefix(void)
63 error_prefix_at(lexer_token.source_position.input_name,
64 lexer_token.source_position.linenr);
68 * Prints a parse error message at the current token.
70 * @param msg the error message
72 static void parse_error(const char *msg)
75 fprintf(stderr, "%s\n", msg);
78 static inline void next_real_char(void)
80 assert(bufpos <= bufend);
81 if (bufpos >= bufend) {
82 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
88 bufpos = buf + MAX_PUTBACK;
89 bufend = buf + MAX_PUTBACK + s;
94 static inline void put_back(int pc)
97 *(--bufpos - buf + buf) = (char) pc;
100 printf("putback '%c'\n", pc);
104 static inline void next_char(void);
106 #define MATCH_NEWLINE(code) \
112 lexer_token.source_position.linenr++; \
116 lexer_token.source_position.linenr++; \
119 #define eat(c_type) do { assert(c == c_type); next_char(); } while(0)
121 static void maybe_concat_lines(void)
126 MATCH_NEWLINE(return;)
136 static inline void next_char(void)
140 /* filter trigraphs */
141 if(UNLIKELY(c == '\\')) {
142 maybe_concat_lines();
143 goto end_of_next_char;
147 goto end_of_next_char;
150 if(LIKELY(c != '?')) {
153 goto end_of_next_char;
158 case '=': c = '#'; break;
159 case '(': c = '['; break;
160 case '/': c = '\\'; maybe_concat_lines(); break;
161 case ')': c = ']'; break;
162 case '\'': c = '^'; break;
163 case '<': c = '{'; break;
164 case '!': c = '|'; break;
165 case '>': c = '}'; break;
166 case '-': c = '~'; break;
176 printf("nchar '%c'\n", c);
180 #define SYMBOL_CHARS \
247 static void parse_symbol(void)
252 obstack_1grow(&symbol_obstack, (char) c);
259 obstack_1grow(&symbol_obstack, (char) c);
269 obstack_1grow(&symbol_obstack, '\0');
271 string = obstack_finish(&symbol_obstack);
272 symbol = symbol_table_insert(string);
274 lexer_token.type = symbol->ID;
275 lexer_token.v.symbol = symbol;
277 if(symbol->string != string) {
278 obstack_free(&symbol_obstack, string);
282 static void parse_integer_suffix(bool is_oct_hex)
284 bool is_unsigned = false;
285 bool min_long = false;
286 bool min_longlong = false;
288 if(c == 'U' || c == 'u') {
291 if(c == 'L' || c == 'l') {
294 if(c == 'L' || c == 'l') {
299 } else if(c == 'l' || c == 'L') {
302 if(c == 'l' || c == 'L') {
305 if(c == 'u' || c == 'U') {
309 } else if(c == 'u' || c == 'U') {
312 lexer_token.datatype = type_unsigned_long;
317 long long v = lexer_token.v.intvalue;
319 if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
320 lexer_token.datatype = type_int;
322 } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
323 lexer_token.datatype = type_unsigned_int;
328 if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
329 lexer_token.datatype = type_long;
331 } else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) {
332 lexer_token.datatype = type_unsigned_long;
336 unsigned long long uv = (unsigned long long) v;
337 if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
338 lexer_token.datatype = type_unsigned_long_long;
342 lexer_token.datatype = type_long_long;
344 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
345 if(!min_long && v <= TARGET_UINT_MAX) {
346 lexer_token.datatype = type_unsigned_int;
349 if(!min_longlong && v <= TARGET_ULONG_MAX) {
350 lexer_token.datatype = type_unsigned_long;
353 lexer_token.datatype = type_unsigned_long_long;
357 static void parse_floating_suffix(void)
360 /* TODO: do something usefull with the suffixes... */
364 lexer_token.datatype = type_float;
369 lexer_token.datatype = type_long_double;
372 lexer_token.datatype = type_double;
378 * A replacement for strtoull. Only those parts needed for
379 * our parser are implemented.
381 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
382 unsigned long long v = 0;
387 /* check for overrun */
388 if (v >= 0x1000000000000000ULL)
390 switch (tolower(*s)) {
391 case '0': v <<= 4; break;
392 case '1': v <<= 4; v |= 0x1; break;
393 case '2': v <<= 4; v |= 0x2; break;
394 case '3': v <<= 4; v |= 0x3; break;
395 case '4': v <<= 4; v |= 0x4; break;
396 case '5': v <<= 4; v |= 0x5; break;
397 case '6': v <<= 4; v |= 0x6; break;
398 case '7': v <<= 4; v |= 0x7; break;
399 case '8': v <<= 4; v |= 0x8; break;
400 case '9': v <<= 4; v |= 0x9; break;
401 case 'a': v <<= 4; v |= 0xa; break;
402 case 'b': v <<= 4; v |= 0xb; break;
403 case 'c': v <<= 4; v |= 0xc; break;
404 case 'd': v <<= 4; v |= 0xd; break;
405 case 'e': v <<= 4; v |= 0xe; break;
406 case 'f': v <<= 4; v |= 0xf; break;
414 /* check for overrun */
415 if (v >= 0x2000000000000000ULL)
417 switch (tolower(*s)) {
418 case '0': v <<= 3; break;
419 case '1': v <<= 3; v |= 1; break;
420 case '2': v <<= 3; v |= 2; break;
421 case '3': v <<= 3; v |= 3; break;
422 case '4': v <<= 3; v |= 4; break;
423 case '5': v <<= 3; v |= 5; break;
424 case '6': v <<= 3; v |= 6; break;
425 case '7': v <<= 3; v |= 7; break;
433 /* check for overrun */
434 if (v > 0x1999999999999999ULL)
436 switch (tolower(*s)) {
437 case '0': v *= 10; break;
438 case '1': v *= 10; v += 1; break;
439 case '2': v *= 10; v += 2; break;
440 case '3': v *= 10; v += 3; break;
441 case '4': v *= 10; v += 4; break;
442 case '5': v *= 10; v += 5; break;
443 case '6': v *= 10; v += 6; break;
444 case '7': v *= 10; v += 7; break;
445 case '8': v *= 10; v += 8; break;
446 case '9': v *= 10; v += 9; break;
461 static void parse_number_hex(void)
463 assert(c == 'x' || c == 'X');
467 obstack_1grow(&symbol_obstack, (char) c);
470 obstack_1grow(&symbol_obstack, '\0');
471 char *string = obstack_finish(&symbol_obstack);
473 if(c == '.' || c == 'p' || c == 'P') {
475 panic("Hex floating point numbers not implemented yet");
477 if(*string == '\0') {
478 parse_error("invalid hex number");
479 lexer_token.type = T_ERROR;
483 lexer_token.type = T_INTEGER;
484 lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
485 if(*endptr != '\0') {
486 parse_error("hex number literal too long");
489 obstack_free(&symbol_obstack, string);
490 parse_integer_suffix(true);
493 static inline bool is_octal_digit(int chr)
495 return '0' <= chr && chr <= '7';
498 static void parse_number_oct(void)
500 while(is_octal_digit(c)) {
501 obstack_1grow(&symbol_obstack, (char) c);
504 obstack_1grow(&symbol_obstack, '\0');
505 char *string = obstack_finish(&symbol_obstack);
508 lexer_token.type = T_INTEGER;
509 lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
510 if(*endptr != '\0') {
511 parse_error("octal number literal too long");
514 obstack_free(&symbol_obstack, string);
515 parse_integer_suffix(true);
518 static void parse_number_dec(void)
520 bool is_float = false;
522 obstack_1grow(&symbol_obstack, (char) c);
527 obstack_1grow(&symbol_obstack, '.');
531 obstack_1grow(&symbol_obstack, (char) c);
536 if(c == 'e' || c == 'E') {
537 obstack_1grow(&symbol_obstack, 'e');
540 if(c == '-' || c == '+') {
541 obstack_1grow(&symbol_obstack, (char) c);
546 obstack_1grow(&symbol_obstack, (char) c);
552 obstack_1grow(&symbol_obstack, '\0');
553 char *string = obstack_finish(&symbol_obstack);
557 lexer_token.type = T_FLOATINGPOINT;
558 lexer_token.v.floatvalue = strtold(string, &endptr);
560 if(*endptr != '\0') {
561 parse_error("invalid number literal");
564 parse_floating_suffix();
567 lexer_token.type = T_INTEGER;
568 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
570 if(*endptr != '\0') {
571 parse_error("invalid number literal");
574 parse_integer_suffix(false);
576 obstack_free(&symbol_obstack, string);
579 static void parse_number(void)
601 parse_error("invalid octal number");
602 lexer_token.type = T_ERROR;
608 obstack_1grow(&symbol_obstack, '0');
617 static int parse_octal_sequence(const int first_digit)
619 assert(is_octal_digit(first_digit));
620 int value = first_digit - '0';
621 if (!is_octal_digit(c)) return value;
622 value = 8 * value + c - '0';
624 if (!is_octal_digit(c)) return value;
625 value = 8 * value + c - '0';
627 return (char_type)value;
630 static int parse_hex_sequence(void)
634 if (c >= '0' && c <= '9') {
635 value = 16 * value + c - '0';
636 } else if ('A' <= c && c <= 'F') {
637 value = 16 * value + c - 'A' + 10;
638 } else if ('a' <= c && c <= 'f') {
639 value = 16 * value + c - 'a' + 10;
646 return (char_type)value;
649 static int parse_escape_sequence(void)
657 case '"': return '"';
658 case '\'': return '\'';
659 case '\\': return '\\';
660 case '?': return '\?';
661 case 'a': return '\a';
662 case 'b': return '\b';
663 case 'f': return '\f';
664 case 'n': return '\n';
665 case 'r': return '\r';
666 case 't': return '\t';
667 case 'v': return '\v';
669 return parse_hex_sequence();
678 return parse_octal_sequence(ec);
680 parse_error("reached end of file while parsing escape sequence");
683 parse_error("unknown escape sequence");
688 string_t concat_strings(const string_t *const s1, const string_t *const s2)
690 const size_t len1 = s1->size - 1;
691 const size_t len2 = s2->size - 1;
693 char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
694 memcpy(concat, s1->begin, len1);
695 memcpy(concat + len1, s2->begin, len2 + 1);
697 #if 0 /* TODO hash */
698 const char *result = strset_insert(&stringset, concat);
699 if(result != concat) {
700 obstack_free(&symbol_obstack, concat);
705 return (string_t){ concat, len1 + len2 + 1 };
709 static void parse_string_literal(void)
711 const unsigned start_linenr = lexer_token.source_position.linenr;
720 tc = parse_escape_sequence();
721 obstack_1grow(&symbol_obstack, (char) tc);
725 error_prefix_at(lexer_token.source_position.input_name,
727 fprintf(stderr, "string has no end\n");
728 lexer_token.type = T_ERROR;
736 obstack_1grow(&symbol_obstack, (char) c);
744 /* TODO: concatenate multiple strings separated by whitespace... */
746 /* add finishing 0 to the string */
747 obstack_1grow(&symbol_obstack, '\0');
748 const size_t size = (size_t)obstack_object_size(&symbol_obstack);
749 const char *const string = obstack_finish(&symbol_obstack);
751 #if 0 /* TODO hash */
752 /* check if there is already a copy of the string */
753 result = strset_insert(&stringset, string);
754 if(result != string) {
755 obstack_free(&symbol_obstack, string);
758 const char *const result = string;
761 lexer_token.type = T_STRING_LITERAL;
762 lexer_token.v.string.begin = result;
763 lexer_token.v.string.size = size;
766 static void parse_wide_character_constant(void)
774 found_char = parse_escape_sequence();
778 parse_error("newline while parsing character constant");
784 goto end_of_wide_char_constant;
787 parse_error("EOF while parsing character constant");
788 lexer_token.type = T_ERROR;
792 if(found_char != 0) {
793 parse_error("more than 1 characters in character "
795 goto end_of_wide_char_constant;
804 end_of_wide_char_constant:
805 lexer_token.type = T_INTEGER;
806 lexer_token.v.intvalue = found_char;
807 lexer_token.datatype = type_wchar_t;
810 static void parse_wide_string_literal(void)
812 const unsigned start_linenr = lexer_token.source_position.linenr;
820 wchar_rep_t tc = parse_escape_sequence();
821 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
826 error_prefix_at(lexer_token.source_position.input_name,
828 fprintf(stderr, "string has no end\n");
829 lexer_token.type = T_ERROR;
838 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
847 /* TODO: concatenate multiple strings separated by whitespace... */
849 /* add finishing 0 to the string */
850 wchar_rep_t nul = L'\0';
851 obstack_grow(&symbol_obstack, &nul, sizeof(nul));
852 const size_t size = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
853 const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
855 #if 0 /* TODO hash */
856 /* check if there is already a copy of the string */
857 const wchar_rep_t *const result = strset_insert(&stringset, string);
858 if(result != string) {
859 obstack_free(&symbol_obstack, string);
862 const wchar_rep_t *const result = string;
865 lexer_token.type = T_WIDE_STRING_LITERAL;
866 lexer_token.v.wide_string.begin = result;
867 lexer_token.v.wide_string.size = size;
870 static void parse_character_constant(void)
878 found_char = parse_escape_sequence();
882 parse_error("newline while parsing character constant");
888 goto end_of_char_constant;
891 parse_error("EOF while parsing character constant");
892 lexer_token.type = T_ERROR;
896 if(found_char != 0) {
897 parse_error("more than 1 characters in character "
899 goto end_of_char_constant;
908 end_of_char_constant:
909 lexer_token.type = T_INTEGER;
910 lexer_token.v.intvalue = found_char;
911 lexer_token.datatype = type_int;
914 static void skip_multiline_comment(void)
916 unsigned start_linenr = lexer_token.source_position.linenr;
923 /* TODO: nested comment, warn here */
934 MATCH_NEWLINE(break;)
937 error_prefix_at(lexer_token.source_position.input_name,
939 fprintf(stderr, "at end of file while looking for comment end\n");
949 static void skip_line_comment(void)
967 static token_t pp_token;
969 static inline void next_pp_token(void)
971 lexer_next_preprocessing_token();
972 pp_token = lexer_token;
975 static void eat_until_newline(void)
977 while(pp_token.type != '\n' && pp_token.type != T_EOF) {
982 static void error_directive(void)
985 fprintf(stderr, "#error directive: \n");
987 /* parse pp-tokens until new-line */
990 static void define_directive(void)
992 lexer_next_preprocessing_token();
993 if(lexer_token.type != T_IDENTIFIER) {
994 parse_error("expected identifier after #define\n");
999 static void ifdef_directive(int is_ifndef)
1002 lexer_next_preprocessing_token();
1003 //expect_identifier();
1007 static void endif_directive(void)
1012 static void parse_line_directive(void)
1014 if(pp_token.type != T_INTEGER) {
1015 parse_error("expected integer");
1017 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
1020 if(pp_token.type == T_STRING_LITERAL) {
1021 lexer_token.source_position.input_name = pp_token.v.string.begin;
1025 eat_until_newline();
1028 static void parse_preprocessor_identifier(void)
1030 assert(pp_token.type == T_IDENTIFIER);
1031 symbol_t *symbol = pp_token.v.symbol;
1033 switch(symbol->pp_ID) {
1035 printf("include - enable header name parsing!\n");
1051 parse_line_directive();
1061 if (warning.unknown_pragmas) {
1062 warningf(lexer_token.source_position, "encountered unknown #pragma");
1064 eat_until_newline();
1069 static void parse_preprocessor_directive(void)
1073 switch(pp_token.type) {
1075 parse_preprocessor_identifier();
1078 parse_line_directive();
1081 parse_error("invalid preprocessor directive");
1082 eat_until_newline();
1087 #define MAYBE_PROLOG \
1092 #define MAYBE(ch, set_type) \
1095 lexer_token.type = set_type; \
1098 #define ELSE_CODE(code) \
1102 } /* end of while(1) */ \
1105 #define ELSE(set_type) \
1107 lexer_token.type = set_type; \
1111 void lexer_next_preprocessing_token(void)
1121 lexer_token.type = '\n';
1127 /* might be a wide string ( L"string" ) */
1128 if(lexer_token.type == T_IDENTIFIER &&
1129 lexer_token.v.symbol == symbol_L) {
1131 parse_wide_string_literal();
1132 } else if(c == '\'') {
1133 parse_wide_character_constant();
1143 parse_string_literal();
1147 parse_character_constant();
1169 MAYBE('.', T_DOTDOTDOT)
1173 lexer_token.type = '.';
1179 MAYBE('&', T_ANDAND)
1180 MAYBE('=', T_ANDEQUAL)
1184 MAYBE('=', T_ASTERISKEQUAL)
1188 MAYBE('+', T_PLUSPLUS)
1189 MAYBE('=', T_PLUSEQUAL)
1193 MAYBE('>', T_MINUSGREATER)
1194 MAYBE('-', T_MINUSMINUS)
1195 MAYBE('=', T_MINUSEQUAL)
1199 MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1203 MAYBE('=', T_SLASHEQUAL)
1206 skip_multiline_comment();
1207 lexer_next_preprocessing_token();
1211 skip_line_comment();
1212 lexer_next_preprocessing_token();
1217 MAYBE('>', T_PERCENTGREATER)
1218 MAYBE('=', T_PERCENTEQUAL)
1223 MAYBE(':', T_PERCENTCOLONPERCENTCOLON)
1227 lexer_token.type = T_PERCENTCOLON;
1230 ELSE(T_PERCENTCOLON)
1234 MAYBE(':', T_LESSCOLON)
1235 MAYBE('%', T_LESSPERCENT)
1236 MAYBE('=', T_LESSEQUAL)
1239 MAYBE('=', T_LESSLESSEQUAL)
1244 MAYBE('=', T_GREATEREQUAL)
1247 MAYBE('=', T_GREATERGREATEREQUAL)
1248 ELSE(T_GREATERGREATER)
1252 MAYBE('=', T_CARETEQUAL)
1256 MAYBE('=', T_PIPEEQUAL)
1257 MAYBE('|', T_PIPEPIPE)
1261 MAYBE('>', T_COLONGREATER)
1265 MAYBE('=', T_EQUALEQUAL)
1269 MAYBE('#', T_HASHHASH)
1283 lexer_token.type = c;
1288 lexer_token.type = T_EOF;
1294 fprintf(stderr, "unknown character '%c' found\n", c);
1295 lexer_token.type = T_ERROR;
1301 void lexer_next_token(void)
1303 lexer_next_preprocessing_token();
1304 if(lexer_token.type != '\n')
1309 lexer_next_preprocessing_token();
1310 } while(lexer_token.type == '\n');
1312 if(lexer_token.type == '#') {
1313 parse_preprocessor_directive();
1318 void init_lexer(void)
1320 strset_init(&stringset);
1323 void lexer_open_stream(FILE *stream, const char *input_name)
1326 lexer_token.source_position.linenr = 0;
1327 lexer_token.source_position.input_name = input_name;
1329 symbol_L = symbol_table_insert("L");
1333 /* place a virtual \n at the beginning so the lexer knows that we're
1334 * at the beginning of a line */
1338 void exit_lexer(void)
1340 strset_destroy(&stringset);
1343 static __attribute__((unused))
1344 void dbg_pos(const source_position_t source_position)
1346 fprintf(stdout, "%s:%u\n", source_position.input_name,
1347 source_position.linenr);