3 #include "diagnostic.h"
6 #include "symbol_table_t.h"
8 #include "adt/strset.h"
12 #include "target_architecture.h"
26 /* No strtold on windows and no replacement yet */
27 #define strtold(s, e) strtod(s, e)
30 #if defined HAS_SIGNED_CHAR
31 typedef signed char char_type;
32 #elif defined HAS_UNSIGNED_CHAR
33 typedef unsigned char char_type;
35 # error signedness of char not determined
42 static char buf[1024 + MAX_PUTBACK];
43 static const char *bufend;
44 static const char *bufpos;
45 static strset_t stringset;
47 static void error_prefix_at(const char *input_name, unsigned linenr)
49 fprintf(stderr, "%s:%u: Error: ", input_name, linenr);
52 static void error_prefix(void)
54 error_prefix_at(lexer_token.source_position.input_name,
55 lexer_token.source_position.linenr);
58 static void parse_error(const char *msg)
61 fprintf(stderr, "%s\n", msg);
64 static inline void next_real_char(void)
66 assert(bufpos <= bufend);
67 if (bufpos >= bufend) {
68 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
74 bufpos = buf + MAX_PUTBACK;
75 bufend = buf + MAX_PUTBACK + s;
80 static inline void put_back(int pc)
83 *(--bufpos - buf + buf) = (char) pc;
86 printf("putback '%c'\n", pc);
90 static inline void next_char(void);
92 #define MATCH_NEWLINE(code) \
98 lexer_token.source_position.linenr++; \
102 lexer_token.source_position.linenr++; \
105 #define eat(c_type) do { assert(c == c_type); next_char(); } while(0)
107 static void maybe_concat_lines(void)
112 MATCH_NEWLINE(return;)
122 static inline void next_char(void)
126 /* filter trigraphs */
127 if(UNLIKELY(c == '\\')) {
128 maybe_concat_lines();
129 goto end_of_next_char;
133 goto end_of_next_char;
136 if(LIKELY(c != '?')) {
139 goto end_of_next_char;
144 case '=': c = '#'; break;
145 case '(': c = '['; break;
146 case '/': c = '\\'; maybe_concat_lines(); break;
147 case ')': c = ']'; break;
148 case '\'': c = '^'; break;
149 case '<': c = '{'; break;
150 case '!': c = '|'; break;
151 case '>': c = '}'; break;
152 case '-': c = '~'; break;
162 printf("nchar '%c'\n", c);
166 #define SYMBOL_CHARS \
233 static void parse_symbol(void)
238 obstack_1grow(&symbol_obstack, (char) c);
245 obstack_1grow(&symbol_obstack, (char) c);
255 obstack_1grow(&symbol_obstack, '\0');
257 string = obstack_finish(&symbol_obstack);
258 symbol = symbol_table_insert(string);
260 lexer_token.type = symbol->ID;
261 lexer_token.v.symbol = symbol;
263 if(symbol->string != string) {
264 obstack_free(&symbol_obstack, string);
268 static void parse_integer_suffix(bool is_oct_hex)
270 bool is_unsigned = false;
271 bool min_long = false;
272 bool min_longlong = false;
274 if(c == 'U' || c == 'u') {
277 if(c == 'L' || c == 'l') {
280 if(c == 'L' || c == 'l') {
285 } else if(c == 'l' || c == 'L') {
288 if(c == 'l' || c == 'L') {
291 if(c == 'u' || c == 'U') {
295 } else if(c == 'u' || c == 'U') {
298 lexer_token.datatype = type_unsigned_long;
303 long long v = lexer_token.v.intvalue;
305 if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
306 lexer_token.datatype = type_int;
308 } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
309 lexer_token.datatype = type_unsigned_int;
314 if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
315 lexer_token.datatype = type_long;
317 } else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) {
318 lexer_token.datatype = type_unsigned_long;
322 unsigned long long uv = (unsigned long long) v;
323 if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
324 lexer_token.datatype = type_unsigned_long_long;
328 lexer_token.datatype = type_long_long;
330 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
331 if(!min_long && v <= TARGET_UINT_MAX) {
332 lexer_token.datatype = type_unsigned_int;
335 if(!min_longlong && v <= TARGET_ULONG_MAX) {
336 lexer_token.datatype = type_unsigned_long;
339 lexer_token.datatype = type_unsigned_long_long;
343 static void parse_floating_suffix(void)
346 /* TODO: do something usefull with the suffixes... */
350 lexer_token.datatype = type_float;
355 lexer_token.datatype = type_long_double;
358 lexer_token.datatype = type_double;
364 * A replacement for strtoull. Only those parts needed for
365 * our parser are implemented.
367 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
368 unsigned long long v = 0;
373 /* check for overrun */
374 if (v >= 0x1000000000000000ULL)
376 switch (tolower(*s)) {
377 case '0': v <<= 4; break;
378 case '1': v <<= 4; v |= 0x1; break;
379 case '2': v <<= 4; v |= 0x2; break;
380 case '3': v <<= 4; v |= 0x3; break;
381 case '4': v <<= 4; v |= 0x4; break;
382 case '5': v <<= 4; v |= 0x5; break;
383 case '6': v <<= 4; v |= 0x6; break;
384 case '7': v <<= 4; v |= 0x7; break;
385 case '8': v <<= 4; v |= 0x8; break;
386 case '9': v <<= 4; v |= 0x9; break;
387 case 'a': v <<= 4; v |= 0xa; break;
388 case 'b': v <<= 4; v |= 0xb; break;
389 case 'c': v <<= 4; v |= 0xc; break;
390 case 'd': v <<= 4; v |= 0xd; break;
391 case 'e': v <<= 4; v |= 0xe; break;
392 case 'f': v <<= 4; v |= 0xf; break;
400 /* check for overrun */
401 if (v >= 0x2000000000000000ULL)
403 switch (tolower(*s)) {
404 case '0': v <<= 3; break;
405 case '1': v <<= 3; v |= 1; break;
406 case '2': v <<= 3; v |= 2; break;
407 case '3': v <<= 3; v |= 3; break;
408 case '4': v <<= 3; v |= 4; break;
409 case '5': v <<= 3; v |= 5; break;
410 case '6': v <<= 3; v |= 6; break;
411 case '7': v <<= 3; v |= 7; break;
419 /* check for overrun */
420 if (v > 0x1999999999999999ULL)
422 switch (tolower(*s)) {
423 case '0': v *= 10; break;
424 case '1': v *= 10; v += 1; break;
425 case '2': v *= 10; v += 2; break;
426 case '3': v *= 10; v += 3; break;
427 case '4': v *= 10; v += 4; break;
428 case '5': v *= 10; v += 5; break;
429 case '6': v *= 10; v += 6; break;
430 case '7': v *= 10; v += 7; break;
431 case '8': v *= 10; v += 8; break;
432 case '9': v *= 10; v += 9; break;
447 static void parse_number_hex(void)
449 assert(c == 'x' || c == 'X');
453 obstack_1grow(&symbol_obstack, (char) c);
456 obstack_1grow(&symbol_obstack, '\0');
457 char *string = obstack_finish(&symbol_obstack);
459 if(c == '.' || c == 'p' || c == 'P') {
461 panic("Hex floating point numbers not implemented yet");
463 if(*string == '\0') {
464 parse_error("invalid hex number");
465 lexer_token.type = T_ERROR;
469 lexer_token.type = T_INTEGER;
470 lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
471 if(*endptr != '\0') {
472 parse_error("hex number literal too long");
475 obstack_free(&symbol_obstack, string);
476 parse_integer_suffix(true);
479 static inline bool is_octal_digit(int chr)
481 return '0' <= chr && chr <= '7';
484 static void parse_number_oct(void)
486 while(is_octal_digit(c)) {
487 obstack_1grow(&symbol_obstack, (char) c);
490 obstack_1grow(&symbol_obstack, '\0');
491 char *string = obstack_finish(&symbol_obstack);
494 lexer_token.type = T_INTEGER;
495 lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
496 if(*endptr != '\0') {
497 parse_error("octal number literal too long");
500 obstack_free(&symbol_obstack, string);
501 parse_integer_suffix(true);
504 static void parse_number_dec(void)
506 bool is_float = false;
508 obstack_1grow(&symbol_obstack, (char) c);
513 obstack_1grow(&symbol_obstack, '.');
517 obstack_1grow(&symbol_obstack, (char) c);
522 if(c == 'e' || c == 'E') {
523 obstack_1grow(&symbol_obstack, 'e');
526 if(c == '-' || c == '+') {
527 obstack_1grow(&symbol_obstack, (char) c);
532 obstack_1grow(&symbol_obstack, (char) c);
538 obstack_1grow(&symbol_obstack, '\0');
539 char *string = obstack_finish(&symbol_obstack);
543 lexer_token.type = T_FLOATINGPOINT;
544 lexer_token.v.floatvalue = strtold(string, &endptr);
546 if(*endptr != '\0') {
547 parse_error("invalid number literal");
550 parse_floating_suffix();
553 lexer_token.type = T_INTEGER;
554 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
556 if(*endptr != '\0') {
557 parse_error("invalid number literal");
560 parse_integer_suffix(false);
562 obstack_free(&symbol_obstack, string);
565 static void parse_number(void)
587 parse_error("invalid octal number");
588 lexer_token.type = T_ERROR;
594 obstack_1grow(&symbol_obstack, '0');
603 static int parse_octal_sequence(const int first_digit)
605 assert(is_octal_digit(first_digit));
606 int value = first_digit - '0';
607 if (!is_octal_digit(c)) return value;
608 value = 8 * value + c - '0';
610 if (!is_octal_digit(c)) return value;
611 value = 8 * value + c - '0';
613 return (char_type)value;
616 static int parse_hex_sequence(void)
620 if (c >= '0' && c <= '9') {
621 value = 16 * value + c - '0';
622 } else if ('A' <= c && c <= 'F') {
623 value = 16 * value + c - 'A' + 10;
624 } else if ('a' <= c && c <= 'f') {
625 value = 16 * value + c - 'a' + 10;
632 return (char_type)value;
635 static int parse_escape_sequence(void)
643 case '"': return '"';
644 case '\'': return '\'';
645 case '\\': return '\\';
646 case '?': return '\?';
647 case 'a': return '\a';
648 case 'b': return '\b';
649 case 'f': return '\f';
650 case 'n': return '\n';
651 case 'r': return '\r';
652 case 't': return '\t';
653 case 'v': return '\v';
655 return parse_hex_sequence();
664 return parse_octal_sequence(ec);
666 parse_error("reached end of file while parsing escape sequence");
669 parse_error("unknown escape sequence");
674 string_t concat_strings(const string_t *const s1, const string_t *const s2)
676 const size_t len1 = s1->size - 1;
677 const size_t len2 = s2->size - 1;
679 char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
680 memcpy(concat, s1->begin, len1);
681 memcpy(concat + len1, s2->begin, len2 + 1);
683 #if 0 /* TODO hash */
684 const char *result = strset_insert(&stringset, concat);
685 if(result != concat) {
686 obstack_free(&symbol_obstack, concat);
691 return (string_t){ concat, len1 + len2 + 1 };
695 static void parse_string_literal(void)
697 const unsigned start_linenr = lexer_token.source_position.linenr;
706 tc = parse_escape_sequence();
707 obstack_1grow(&symbol_obstack, (char) tc);
711 error_prefix_at(lexer_token.source_position.input_name,
713 fprintf(stderr, "string has no end\n");
714 lexer_token.type = T_ERROR;
722 obstack_1grow(&symbol_obstack, (char) c);
730 /* TODO: concatenate multiple strings separated by whitespace... */
732 /* add finishing 0 to the string */
733 obstack_1grow(&symbol_obstack, '\0');
734 const size_t size = (size_t)obstack_object_size(&symbol_obstack);
735 const char *const string = obstack_finish(&symbol_obstack);
737 #if 0 /* TODO hash */
738 /* check if there is already a copy of the string */
739 result = strset_insert(&stringset, string);
740 if(result != string) {
741 obstack_free(&symbol_obstack, string);
744 const char *const result = string;
747 lexer_token.type = T_STRING_LITERAL;
748 lexer_token.v.string.begin = result;
749 lexer_token.v.string.size = size;
752 static void parse_wide_character_constant(void)
760 found_char = parse_escape_sequence();
764 parse_error("newline while parsing character constant");
770 goto end_of_wide_char_constant;
773 parse_error("EOF while parsing character constant");
774 lexer_token.type = T_ERROR;
778 if(found_char != 0) {
779 parse_error("more than 1 characters in character "
781 goto end_of_wide_char_constant;
790 end_of_wide_char_constant:
791 lexer_token.type = T_INTEGER;
792 lexer_token.v.intvalue = found_char;
793 lexer_token.datatype = type_wchar_t;
796 static void parse_wide_string_literal(void)
798 const unsigned start_linenr = lexer_token.source_position.linenr;
806 wchar_rep_t tc = parse_escape_sequence();
807 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
812 error_prefix_at(lexer_token.source_position.input_name,
814 fprintf(stderr, "string has no end\n");
815 lexer_token.type = T_ERROR;
824 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
833 /* TODO: concatenate multiple strings separated by whitespace... */
835 /* add finishing 0 to the string */
836 wchar_rep_t nul = L'\0';
837 obstack_grow(&symbol_obstack, &nul, sizeof(nul));
838 const size_t size = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
839 const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
841 #if 0 /* TODO hash */
842 /* check if there is already a copy of the string */
843 const wchar_rep_t *const result = strset_insert(&stringset, string);
844 if(result != string) {
845 obstack_free(&symbol_obstack, string);
848 const wchar_rep_t *const result = string;
851 lexer_token.type = T_WIDE_STRING_LITERAL;
852 lexer_token.v.wide_string.begin = result;
853 lexer_token.v.wide_string.size = size;
856 static void parse_character_constant(void)
864 found_char = parse_escape_sequence();
868 parse_error("newline while parsing character constant");
874 goto end_of_char_constant;
877 parse_error("EOF while parsing character constant");
878 lexer_token.type = T_ERROR;
882 if(found_char != 0) {
883 parse_error("more than 1 characters in character "
885 goto end_of_char_constant;
894 end_of_char_constant:
895 lexer_token.type = T_INTEGER;
896 lexer_token.v.intvalue = found_char;
897 lexer_token.datatype = type_int;
900 static void skip_multiline_comment(void)
902 unsigned start_linenr = lexer_token.source_position.linenr;
914 MATCH_NEWLINE(break;)
917 error_prefix_at(lexer_token.source_position.input_name,
919 fprintf(stderr, "at end of file while looking for comment end\n");
929 static void skip_line_comment(void)
947 static token_t pp_token;
949 static inline void next_pp_token(void)
951 lexer_next_preprocessing_token();
952 pp_token = lexer_token;
955 static void eat_until_newline(void)
957 while(pp_token.type != '\n' && pp_token.type != T_EOF) {
962 static void error_directive(void)
965 fprintf(stderr, "#error directive: \n");
967 /* parse pp-tokens until new-line */
970 static void define_directive(void)
972 lexer_next_preprocessing_token();
973 if(lexer_token.type != T_IDENTIFIER) {
974 parse_error("expected identifier after #define\n");
979 static void ifdef_directive(int is_ifndef)
982 lexer_next_preprocessing_token();
983 //expect_identifier();
987 static void endif_directive(void)
992 static void parse_line_directive(void)
994 if(pp_token.type != T_INTEGER) {
995 parse_error("expected integer");
997 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
1000 if(pp_token.type == T_STRING_LITERAL) {
1001 lexer_token.source_position.input_name = pp_token.v.string.begin;
1005 eat_until_newline();
1008 static void parse_preprocessor_identifier(void)
1010 assert(pp_token.type == T_IDENTIFIER);
1011 symbol_t *symbol = pp_token.v.symbol;
1013 switch(symbol->pp_ID) {
1015 printf("include - enable header name parsing!\n");
1031 parse_line_directive();
1041 if (warning.unknown_pragmas) {
1042 warningf(lexer_token.source_position, "encountered unknown #pragma");
1044 eat_until_newline();
1049 static void parse_preprocessor_directive(void)
1053 switch(pp_token.type) {
1055 parse_preprocessor_identifier();
1058 parse_line_directive();
1061 parse_error("invalid preprocessor directive");
1062 eat_until_newline();
1067 #define MAYBE_PROLOG \
1072 #define MAYBE(ch, set_type) \
1075 lexer_token.type = set_type; \
1078 #define ELSE_CODE(code) \
1082 } /* end of while(1) */ \
1085 #define ELSE(set_type) \
1087 lexer_token.type = set_type; \
1091 void lexer_next_preprocessing_token(void)
1101 lexer_token.type = '\n';
1107 /* might be a wide string ( L"string" ) */
1108 if(lexer_token.type == T_IDENTIFIER &&
1109 lexer_token.v.symbol == symbol_L) {
1111 parse_wide_string_literal();
1112 } else if(c == '\'') {
1113 parse_wide_character_constant();
1123 parse_string_literal();
1127 parse_character_constant();
1149 MAYBE('.', T_DOTDOTDOT)
1153 lexer_token.type = '.';
1159 MAYBE('&', T_ANDAND)
1160 MAYBE('=', T_ANDEQUAL)
1164 MAYBE('=', T_ASTERISKEQUAL)
1168 MAYBE('+', T_PLUSPLUS)
1169 MAYBE('=', T_PLUSEQUAL)
1173 MAYBE('>', T_MINUSGREATER)
1174 MAYBE('-', T_MINUSMINUS)
1175 MAYBE('=', T_MINUSEQUAL)
1179 MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1183 MAYBE('=', T_SLASHEQUAL)
1186 skip_multiline_comment();
1187 lexer_next_preprocessing_token();
1191 skip_line_comment();
1192 lexer_next_preprocessing_token();
1197 MAYBE('>', T_PERCENTGREATER)
1198 MAYBE('=', T_PERCENTEQUAL)
1203 MAYBE(':', T_PERCENTCOLONPERCENTCOLON)
1207 lexer_token.type = T_PERCENTCOLON;
1210 ELSE(T_PERCENTCOLON)
1214 MAYBE(':', T_LESSCOLON)
1215 MAYBE('%', T_LESSPERCENT)
1216 MAYBE('=', T_LESSEQUAL)
1219 MAYBE('=', T_LESSLESSEQUAL)
1224 MAYBE('=', T_GREATEREQUAL)
1227 MAYBE('=', T_GREATERGREATEREQUAL)
1228 ELSE(T_GREATERGREATER)
1232 MAYBE('=', T_CARETEQUAL)
1236 MAYBE('=', T_PIPEEQUAL)
1237 MAYBE('|', T_PIPEPIPE)
1241 MAYBE('>', T_COLONGREATER)
1245 MAYBE('=', T_EQUALEQUAL)
1249 MAYBE('#', T_HASHHASH)
1263 lexer_token.type = c;
1268 lexer_token.type = T_EOF;
1274 fprintf(stderr, "unknown character '%c' found\n", c);
1275 lexer_token.type = T_ERROR;
1281 void lexer_next_token(void)
1283 lexer_next_preprocessing_token();
1284 if(lexer_token.type != '\n')
1289 lexer_next_preprocessing_token();
1290 } while(lexer_token.type == '\n');
1292 if(lexer_token.type == '#') {
1293 parse_preprocessor_directive();
1298 void init_lexer(void)
1300 strset_init(&stringset);
1303 void lexer_open_stream(FILE *stream, const char *input_name)
1306 lexer_token.source_position.linenr = 0;
1307 lexer_token.source_position.input_name = input_name;
1309 symbol_L = symbol_table_insert("L");
1313 /* place a virtual \n at the beginning so the lexer knows that we're
1314 * at the beginning of a line */
1318 void exit_lexer(void)
1320 strset_destroy(&stringset);
1323 static __attribute__((unused))
1324 void dbg_pos(const source_position_t source_position)
1326 fprintf(stdout, "%s:%u\n", source_position.input_name,
1327 source_position.linenr);