3 #include "diagnostic.h"
6 #include "symbol_table_t.h"
8 #include "adt/strset.h"
12 #include "target_architecture.h"
25 /* No strtold on windows and no replacement yet */
26 #define strtold(s, e) strtod(s, e)
29 #if defined HAS_SIGNED_CHAR
30 typedef signed char char_type;
31 #elif defined HAS_UNSIGNED_CHAR
32 typedef unsigned char char_type;
34 # error signedness of char not determined
41 static char buf[1024 + MAX_PUTBACK];
42 static const char *bufend;
43 static const char *bufpos;
44 static strset_t stringset;
46 static void error_prefix_at(const char *input_name, unsigned linenr)
48 fprintf(stderr, "%s:%u: Error: ", input_name, linenr);
51 static void error_prefix(void)
53 error_prefix_at(lexer_token.source_position.input_name,
54 lexer_token.source_position.linenr);
57 static void parse_error(const char *msg)
60 fprintf(stderr, "%s\n", msg);
63 static inline void next_real_char(void)
65 assert(bufpos <= bufend);
66 if (bufpos >= bufend) {
67 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
73 bufpos = buf + MAX_PUTBACK;
74 bufend = buf + MAX_PUTBACK + s;
79 static inline void put_back(int pc)
82 *(--bufpos - buf + buf) = (char) pc;
85 printf("putback '%c'\n", pc);
89 static inline void next_char(void);
91 #define MATCH_NEWLINE(code) \
97 lexer_token.source_position.linenr++; \
101 lexer_token.source_position.linenr++; \
104 #define eat(c_type) do { assert(c == c_type); next_char(); } while(0)
106 static void maybe_concat_lines(void)
111 MATCH_NEWLINE(return;)
121 static inline void next_char(void)
125 /* filter trigraphs */
126 if(UNLIKELY(c == '\\')) {
127 maybe_concat_lines();
128 goto end_of_next_char;
132 goto end_of_next_char;
135 if(LIKELY(c != '?')) {
138 goto end_of_next_char;
143 case '=': c = '#'; break;
144 case '(': c = '['; break;
145 case '/': c = '\\'; maybe_concat_lines(); break;
146 case ')': c = ']'; break;
147 case '\'': c = '^'; break;
148 case '<': c = '{'; break;
149 case '!': c = '|'; break;
150 case '>': c = '}'; break;
151 case '-': c = '~'; break;
161 printf("nchar '%c'\n", c);
165 #define SYMBOL_CHARS \
232 static void parse_symbol(void)
237 obstack_1grow(&symbol_obstack, (char) c);
244 obstack_1grow(&symbol_obstack, (char) c);
254 obstack_1grow(&symbol_obstack, '\0');
256 string = obstack_finish(&symbol_obstack);
257 symbol = symbol_table_insert(string);
259 lexer_token.type = symbol->ID;
260 lexer_token.v.symbol = symbol;
262 if(symbol->string != string) {
263 obstack_free(&symbol_obstack, string);
267 static void parse_integer_suffix(bool is_oct_hex)
269 bool is_unsigned = false;
270 bool min_long = false;
271 bool min_longlong = false;
273 if(c == 'U' || c == 'u') {
276 if(c == 'L' || c == 'l') {
279 if(c == 'L' || c == 'l') {
284 } else if(c == 'l' || c == 'L') {
287 if(c == 'l' || c == 'L') {
290 if(c == 'u' || c == 'U') {
294 } else if(c == 'u' || c == 'U') {
297 lexer_token.datatype = type_unsigned_long;
302 long long v = lexer_token.v.intvalue;
304 if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
305 lexer_token.datatype = type_int;
307 } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
308 lexer_token.datatype = type_unsigned_int;
313 if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
314 lexer_token.datatype = type_long;
316 } else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) {
317 lexer_token.datatype = type_unsigned_long;
321 unsigned long long uv = (unsigned long long) v;
322 if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
323 lexer_token.datatype = type_unsigned_long_long;
327 lexer_token.datatype = type_long_long;
329 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
330 if(!min_long && v <= TARGET_UINT_MAX) {
331 lexer_token.datatype = type_unsigned_int;
334 if(!min_longlong && v <= TARGET_ULONG_MAX) {
335 lexer_token.datatype = type_unsigned_long;
338 lexer_token.datatype = type_unsigned_long_long;
342 static void parse_floating_suffix(void)
345 /* TODO: do something usefull with the suffixes... */
349 lexer_token.datatype = type_float;
354 lexer_token.datatype = type_long_double;
357 lexer_token.datatype = type_double;
363 * A replacement for strtoull. Only those parts needed for
364 * our parser are implemented.
366 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
367 unsigned long long v = 0;
372 /* check for overrun */
373 if (v >= 0x1000000000000000ULL)
375 switch (tolower(*s)) {
376 case '0': v <<= 4; break;
377 case '1': v <<= 4; v |= 0x1; break;
378 case '2': v <<= 4; v |= 0x2; break;
379 case '3': v <<= 4; v |= 0x3; break;
380 case '4': v <<= 4; v |= 0x4; break;
381 case '5': v <<= 4; v |= 0x5; break;
382 case '6': v <<= 4; v |= 0x6; break;
383 case '7': v <<= 4; v |= 0x7; break;
384 case '8': v <<= 4; v |= 0x8; break;
385 case '9': v <<= 4; v |= 0x9; break;
386 case 'a': v <<= 4; v |= 0xa; break;
387 case 'b': v <<= 4; v |= 0xb; break;
388 case 'c': v <<= 4; v |= 0xc; break;
389 case 'd': v <<= 4; v |= 0xd; break;
390 case 'e': v <<= 4; v |= 0xe; break;
391 case 'f': v <<= 4; v |= 0xf; break;
399 /* check for overrun */
400 if (v >= 0x2000000000000000ULL)
402 switch (tolower(*s)) {
403 case '0': v <<= 3; break;
404 case '1': v <<= 3; v |= 1; break;
405 case '2': v <<= 3; v |= 2; break;
406 case '3': v <<= 3; v |= 3; break;
407 case '4': v <<= 3; v |= 4; break;
408 case '5': v <<= 3; v |= 5; break;
409 case '6': v <<= 3; v |= 6; break;
410 case '7': v <<= 3; v |= 7; break;
418 /* check for overrun */
419 if (v > 0x1999999999999999ULL)
421 switch (tolower(*s)) {
422 case '0': v *= 10; break;
423 case '1': v *= 10; v += 1; break;
424 case '2': v *= 10; v += 2; break;
425 case '3': v *= 10; v += 3; break;
426 case '4': v *= 10; v += 4; break;
427 case '5': v *= 10; v += 5; break;
428 case '6': v *= 10; v += 6; break;
429 case '7': v *= 10; v += 7; break;
430 case '8': v *= 10; v += 8; break;
431 case '9': v *= 10; v += 9; break;
446 static void parse_number_hex(void)
448 assert(c == 'x' || c == 'X');
452 obstack_1grow(&symbol_obstack, (char) c);
455 obstack_1grow(&symbol_obstack, '\0');
456 char *string = obstack_finish(&symbol_obstack);
458 if(c == '.' || c == 'p' || c == 'P') {
460 panic("Hex floating point numbers not implemented yet");
462 if(*string == '\0') {
463 parse_error("invalid hex number");
464 lexer_token.type = T_ERROR;
468 lexer_token.type = T_INTEGER;
469 lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
470 if(*endptr != '\0') {
471 parse_error("hex number literal too long");
474 obstack_free(&symbol_obstack, string);
475 parse_integer_suffix(true);
478 static inline bool is_octal_digit(int chr)
480 return '0' <= chr && chr <= '7';
483 static void parse_number_oct(void)
485 while(is_octal_digit(c)) {
486 obstack_1grow(&symbol_obstack, (char) c);
489 obstack_1grow(&symbol_obstack, '\0');
490 char *string = obstack_finish(&symbol_obstack);
493 lexer_token.type = T_INTEGER;
494 lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
495 if(*endptr != '\0') {
496 parse_error("octal number literal too long");
499 obstack_free(&symbol_obstack, string);
500 parse_integer_suffix(true);
503 static void parse_number_dec(void)
505 bool is_float = false;
507 obstack_1grow(&symbol_obstack, (char) c);
512 obstack_1grow(&symbol_obstack, '.');
516 obstack_1grow(&symbol_obstack, (char) c);
521 if(c == 'e' || c == 'E') {
522 obstack_1grow(&symbol_obstack, 'e');
525 if(c == '-' || c == '+') {
526 obstack_1grow(&symbol_obstack, (char) c);
531 obstack_1grow(&symbol_obstack, (char) c);
537 obstack_1grow(&symbol_obstack, '\0');
538 char *string = obstack_finish(&symbol_obstack);
542 lexer_token.type = T_FLOATINGPOINT;
543 lexer_token.v.floatvalue = strtold(string, &endptr);
545 if(*endptr != '\0') {
546 parse_error("invalid number literal");
549 parse_floating_suffix();
552 lexer_token.type = T_INTEGER;
553 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
555 if(*endptr != '\0') {
556 parse_error("invalid number literal");
559 parse_integer_suffix(false);
561 obstack_free(&symbol_obstack, string);
564 static void parse_number(void)
586 parse_error("invalid octal number");
587 lexer_token.type = T_ERROR;
593 obstack_1grow(&symbol_obstack, '0');
602 static int parse_octal_sequence(const int first_digit)
604 assert(is_octal_digit(first_digit));
605 int value = first_digit - '0';
606 if (!is_octal_digit(c)) return value;
607 value = 8 * value + c - '0';
609 if (!is_octal_digit(c)) return value;
610 value = 8 * value + c - '0';
612 return (char_type)value;
615 static int parse_hex_sequence(void)
619 if (c >= '0' && c <= '9') {
620 value = 16 * value + c - '0';
621 } else if ('A' <= c && c <= 'F') {
622 value = 16 * value + c - 'A' + 10;
623 } else if ('a' <= c && c <= 'f') {
624 value = 16 * value + c - 'a' + 10;
631 return (char_type)value;
634 static int parse_escape_sequence(void)
642 case '"': return '"';
643 case '\'': return '\'';
644 case '\\': return '\\';
645 case '?': return '\?';
646 case 'a': return '\a';
647 case 'b': return '\b';
648 case 'f': return '\f';
649 case 'n': return '\n';
650 case 'r': return '\r';
651 case 't': return '\t';
652 case 'v': return '\v';
654 return parse_hex_sequence();
663 return parse_octal_sequence(ec);
665 parse_error("reached end of file while parsing escape sequence");
668 parse_error("unknown escape sequence");
673 string_t concat_strings(const string_t *const s1, const string_t *const s2)
675 const size_t len1 = s1->size - 1;
676 const size_t len2 = s2->size - 1;
678 char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
679 memcpy(concat, s1->begin, len1);
680 memcpy(concat + len1, s2->begin, len2 + 1);
682 #if 0 /* TODO hash */
683 const char *result = strset_insert(&stringset, concat);
684 if(result != concat) {
685 obstack_free(&symbol_obstack, concat);
690 return (string_t){ concat, len1 + len2 + 1 };
694 static void parse_string_literal(void)
696 const unsigned start_linenr = lexer_token.source_position.linenr;
705 tc = parse_escape_sequence();
706 obstack_1grow(&symbol_obstack, (char) tc);
710 error_prefix_at(lexer_token.source_position.input_name,
712 fprintf(stderr, "string has no end\n");
713 lexer_token.type = T_ERROR;
721 obstack_1grow(&symbol_obstack, (char) c);
729 /* TODO: concatenate multiple strings separated by whitespace... */
731 /* add finishing 0 to the string */
732 obstack_1grow(&symbol_obstack, '\0');
733 const size_t size = (size_t)obstack_object_size(&symbol_obstack);
734 const char *const string = obstack_finish(&symbol_obstack);
736 #if 0 /* TODO hash */
737 /* check if there is already a copy of the string */
738 result = strset_insert(&stringset, string);
739 if(result != string) {
740 obstack_free(&symbol_obstack, string);
743 const char *const result = string;
746 lexer_token.type = T_STRING_LITERAL;
747 lexer_token.v.string.begin = result;
748 lexer_token.v.string.size = size;
751 static void parse_wide_character_constant(void)
759 found_char = parse_escape_sequence();
763 parse_error("newline while parsing character constant");
769 goto end_of_wide_char_constant;
772 parse_error("EOF while parsing character constant");
773 lexer_token.type = T_ERROR;
777 if(found_char != 0) {
778 parse_error("more than 1 characters in character "
780 goto end_of_wide_char_constant;
789 end_of_wide_char_constant:
790 lexer_token.type = T_INTEGER;
791 lexer_token.v.intvalue = found_char;
792 lexer_token.datatype = type_wchar_t;
795 static void parse_wide_string_literal(void)
797 const unsigned start_linenr = lexer_token.source_position.linenr;
805 wchar_rep_t tc = parse_escape_sequence();
806 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
811 error_prefix_at(lexer_token.source_position.input_name,
813 fprintf(stderr, "string has no end\n");
814 lexer_token.type = T_ERROR;
823 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
832 /* TODO: concatenate multiple strings separated by whitespace... */
834 /* add finishing 0 to the string */
835 wchar_rep_t nul = L'\0';
836 obstack_grow(&symbol_obstack, &nul, sizeof(nul));
837 const size_t size = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
838 const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
840 #if 0 /* TODO hash */
841 /* check if there is already a copy of the string */
842 const wchar_rep_t *const result = strset_insert(&stringset, string);
843 if(result != string) {
844 obstack_free(&symbol_obstack, string);
847 const wchar_rep_t *const result = string;
850 lexer_token.type = T_WIDE_STRING_LITERAL;
851 lexer_token.v.wide_string.begin = result;
852 lexer_token.v.wide_string.size = size;
855 static void parse_character_constant(void)
863 found_char = parse_escape_sequence();
867 parse_error("newline while parsing character constant");
873 goto end_of_char_constant;
876 parse_error("EOF while parsing character constant");
877 lexer_token.type = T_ERROR;
881 if(found_char != 0) {
882 parse_error("more than 1 characters in character "
884 goto end_of_char_constant;
893 end_of_char_constant:
894 lexer_token.type = T_INTEGER;
895 lexer_token.v.intvalue = found_char;
896 lexer_token.datatype = type_int;
899 static void skip_multiline_comment(void)
901 unsigned start_linenr = lexer_token.source_position.linenr;
913 MATCH_NEWLINE(break;)
916 error_prefix_at(lexer_token.source_position.input_name,
918 fprintf(stderr, "at end of file while looking for comment end\n");
928 static void skip_line_comment(void)
946 static token_t pp_token;
948 static inline void next_pp_token(void)
950 lexer_next_preprocessing_token();
951 pp_token = lexer_token;
954 static void eat_until_newline(void)
956 while(pp_token.type != '\n' && pp_token.type != T_EOF) {
961 static void error_directive(void)
964 fprintf(stderr, "#error directive: \n");
966 /* parse pp-tokens until new-line */
969 static void define_directive(void)
971 lexer_next_preprocessing_token();
972 if(lexer_token.type != T_IDENTIFIER) {
973 parse_error("expected identifier after #define\n");
978 static void ifdef_directive(int is_ifndef)
981 lexer_next_preprocessing_token();
982 //expect_identifier();
986 static void endif_directive(void)
991 static void parse_line_directive(void)
993 if(pp_token.type != T_INTEGER) {
994 parse_error("expected integer");
996 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
999 if(pp_token.type == T_STRING_LITERAL) {
1000 lexer_token.source_position.input_name = pp_token.v.string.begin;
1004 eat_until_newline();
1007 static void parse_preprocessor_identifier(void)
1009 assert(pp_token.type == T_IDENTIFIER);
1010 symbol_t *symbol = pp_token.v.symbol;
1012 switch(symbol->pp_ID) {
1014 printf("include - enable header name parsing!\n");
1030 parse_line_directive();
1040 warningf(lexer_token.source_position, "encountered unknown #pragma");
1041 eat_until_newline();
1046 static void parse_preprocessor_directive(void)
1050 switch(pp_token.type) {
1052 parse_preprocessor_identifier();
1055 parse_line_directive();
1058 parse_error("invalid preprocessor directive");
1059 eat_until_newline();
1064 #define MAYBE_PROLOG \
1069 #define MAYBE(ch, set_type) \
1072 lexer_token.type = set_type; \
1075 #define ELSE_CODE(code) \
1079 } /* end of while(1) */ \
1082 #define ELSE(set_type) \
1084 lexer_token.type = set_type; \
1088 void lexer_next_preprocessing_token(void)
1098 lexer_token.type = '\n';
1104 /* might be a wide string ( L"string" ) */
1105 if(lexer_token.type == T_IDENTIFIER &&
1106 lexer_token.v.symbol == symbol_L) {
1108 parse_wide_string_literal();
1109 } else if(c == '\'') {
1110 parse_wide_character_constant();
1120 parse_string_literal();
1124 parse_character_constant();
1146 MAYBE('.', T_DOTDOTDOT)
1150 lexer_token.type = '.';
1156 MAYBE('&', T_ANDAND)
1157 MAYBE('=', T_ANDEQUAL)
1161 MAYBE('=', T_ASTERISKEQUAL)
1165 MAYBE('+', T_PLUSPLUS)
1166 MAYBE('=', T_PLUSEQUAL)
1170 MAYBE('>', T_MINUSGREATER)
1171 MAYBE('-', T_MINUSMINUS)
1172 MAYBE('=', T_MINUSEQUAL)
1176 MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1180 MAYBE('=', T_SLASHEQUAL)
1183 skip_multiline_comment();
1184 lexer_next_preprocessing_token();
1188 skip_line_comment();
1189 lexer_next_preprocessing_token();
1194 MAYBE('>', T_PERCENTGREATER)
1195 MAYBE('=', T_PERCENTEQUAL)
1200 MAYBE(':', T_PERCENTCOLONPERCENTCOLON)
1204 lexer_token.type = T_PERCENTCOLON;
1207 ELSE(T_PERCENTCOLON)
1211 MAYBE(':', T_LESSCOLON)
1212 MAYBE('%', T_LESSPERCENT)
1213 MAYBE('=', T_LESSEQUAL)
1216 MAYBE('=', T_LESSLESSEQUAL)
1221 MAYBE('=', T_GREATEREQUAL)
1224 MAYBE('=', T_GREATERGREATEREQUAL)
1225 ELSE(T_GREATERGREATER)
1229 MAYBE('=', T_CARETEQUAL)
1233 MAYBE('=', T_PIPEEQUAL)
1234 MAYBE('|', T_PIPEPIPE)
1238 MAYBE('>', T_COLONGREATER)
1242 MAYBE('=', T_EQUALEQUAL)
1246 MAYBE('#', T_HASHHASH)
1260 lexer_token.type = c;
1265 lexer_token.type = T_EOF;
1271 fprintf(stderr, "unknown character '%c' found\n", c);
1272 lexer_token.type = T_ERROR;
1278 void lexer_next_token(void)
1280 lexer_next_preprocessing_token();
1281 if(lexer_token.type != '\n')
1286 lexer_next_preprocessing_token();
1287 } while(lexer_token.type == '\n');
1289 if(lexer_token.type == '#') {
1290 parse_preprocessor_directive();
1295 void init_lexer(void)
1297 strset_init(&stringset);
1300 void lexer_open_stream(FILE *stream, const char *input_name)
1303 lexer_token.source_position.linenr = 0;
1304 lexer_token.source_position.input_name = input_name;
1306 symbol_L = symbol_table_insert("L");
1310 /* place a virtual \n at the beginning so the lexer knows that we're
1311 * at the beginning of a line */
1315 void exit_lexer(void)
1317 strset_destroy(&stringset);
1320 static __attribute__((unused))
1321 void dbg_pos(const source_position_t source_position)
1323 fprintf(stdout, "%s:%u\n", source_position.input_name,
1324 source_position.linenr);