5 #include "symbol_table_t.h"
7 #include "adt/strset.h"
11 #include "target_architecture.h"
24 /* No strtold on windows and no replacement yet */
25 #define strtold(s, e) strtod(s, e)
28 #if defined HAS_SIGNED_CHAR
29 typedef signed char char_type;
30 #elif defined HAS_UNSIGNED_CHAR
31 typedef unsigned char char_type;
33 # error signedness of char not determined
40 static char buf[1024 + MAX_PUTBACK];
41 static const char *bufend;
42 static const char *bufpos;
43 static strset_t stringset;
45 static void error_prefix_at(const char *input_name, unsigned linenr)
47 fprintf(stderr, "%s:%u: Error: ", input_name, linenr);
50 static void error_prefix(void)
52 error_prefix_at(lexer_token.source_position.input_name,
53 lexer_token.source_position.linenr);
56 static void parse_error(const char *msg)
59 fprintf(stderr, "%s\n", msg);
62 static inline void next_real_char(void)
65 if(bufpos >= bufend) {
66 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
72 bufpos = buf + MAX_PUTBACK;
73 bufend = buf + MAX_PUTBACK + s;
78 static inline void put_back(int pc)
80 assert(bufpos >= buf);
81 //assert(bufpos < buf+MAX_PUTBACK || *bufpos == pc);
83 char *p = buf + (bufpos - buf);
86 /* going backwards in the buffer is legal as long as it's not more often
91 printf("putback '%c'\n", pc);
95 static inline void next_char(void);
97 #define MATCH_NEWLINE(code) \
103 lexer_token.source_position.linenr++; \
107 lexer_token.source_position.linenr++; \
110 #define eat(c_type) do { assert(c == c_type); next_char(); } while(0)
112 static void maybe_concat_lines(void)
117 MATCH_NEWLINE(return;)
127 static inline void next_char(void)
131 /* filter trigraphs */
132 if(UNLIKELY(c == '\\')) {
133 maybe_concat_lines();
134 goto end_of_next_char;
138 goto end_of_next_char;
141 if(LIKELY(c != '?')) {
144 goto end_of_next_char;
149 case '=': c = '#'; break;
150 case '(': c = '['; break;
151 case '/': c = '\\'; maybe_concat_lines(); break;
152 case ')': c = ']'; break;
153 case '\'': c = '^'; break;
154 case '<': c = '{'; break;
155 case '!': c = '|'; break;
156 case '>': c = '}'; break;
157 case '-': c = '~'; break;
167 printf("nchar '%c'\n", c);
171 #define SYMBOL_CHARS \
238 static void parse_symbol(void)
243 obstack_1grow(&symbol_obstack, (char) c);
250 obstack_1grow(&symbol_obstack, (char) c);
260 obstack_1grow(&symbol_obstack, '\0');
262 string = obstack_finish(&symbol_obstack);
263 symbol = symbol_table_insert(string);
265 lexer_token.type = symbol->ID;
266 lexer_token.v.symbol = symbol;
268 if(symbol->string != string) {
269 obstack_free(&symbol_obstack, string);
273 static void parse_integer_suffix(bool is_oct_hex)
275 bool is_unsigned = false;
276 bool min_long = false;
277 bool min_longlong = false;
279 if(c == 'U' || c == 'u') {
282 if(c == 'L' || c == 'l') {
285 if(c == 'L' || c == 'l') {
290 } else if(c == 'l' || c == 'L') {
293 if(c == 'l' || c == 'L') {
296 if(c == 'u' || c == 'U') {
300 } else if(c == 'u' || c == 'U') {
303 lexer_token.datatype = type_unsigned_long;
308 long long v = lexer_token.v.intvalue;
310 if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
311 lexer_token.datatype = type_int;
313 } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
314 lexer_token.datatype = type_unsigned_int;
319 if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
320 lexer_token.datatype = type_long;
322 } else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) {
323 lexer_token.datatype = type_unsigned_long;
327 unsigned long long uv = (unsigned long long) v;
328 if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
329 lexer_token.datatype = type_unsigned_long_long;
333 lexer_token.datatype = type_long_long;
335 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
336 if(!min_long && v <= TARGET_UINT_MAX) {
337 lexer_token.datatype = type_unsigned_int;
340 if(!min_longlong && v <= TARGET_ULONG_MAX) {
341 lexer_token.datatype = type_unsigned_long;
344 lexer_token.datatype = type_unsigned_long_long;
348 static void parse_floating_suffix(void)
351 /* TODO: do something usefull with the suffixes... */
355 lexer_token.datatype = type_float;
360 lexer_token.datatype = type_long_double;
363 lexer_token.datatype = type_double;
369 * A replacement for strtoull. Only those parts needed for
370 * our parser are implemented.
372 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
373 unsigned long long v = 0;
378 /* check for overrun */
379 if (v >= 0x1000000000000000ULL)
381 switch (tolower(*s)) {
382 case '0': v <<= 4; break;
383 case '1': v <<= 4; v |= 0x1; break;
384 case '2': v <<= 4; v |= 0x2; break;
385 case '3': v <<= 4; v |= 0x3; break;
386 case '4': v <<= 4; v |= 0x4; break;
387 case '5': v <<= 4; v |= 0x5; break;
388 case '6': v <<= 4; v |= 0x6; break;
389 case '7': v <<= 4; v |= 0x7; break;
390 case '8': v <<= 4; v |= 0x8; break;
391 case '9': v <<= 4; v |= 0x9; break;
392 case 'a': v <<= 4; v |= 0xa; break;
393 case 'b': v <<= 4; v |= 0xb; break;
394 case 'c': v <<= 4; v |= 0xc; break;
395 case 'd': v <<= 4; v |= 0xd; break;
396 case 'e': v <<= 4; v |= 0xe; break;
397 case 'f': v <<= 4; v |= 0xf; break;
405 /* check for overrun */
406 if (v >= 0x2000000000000000ULL)
408 switch (tolower(*s)) {
409 case '0': v <<= 3; break;
410 case '1': v <<= 3; v |= 1; break;
411 case '2': v <<= 3; v |= 2; break;
412 case '3': v <<= 3; v |= 3; break;
413 case '4': v <<= 3; v |= 4; break;
414 case '5': v <<= 3; v |= 5; break;
415 case '6': v <<= 3; v |= 6; break;
416 case '7': v <<= 3; v |= 7; break;
424 /* check for overrun */
425 if (v > 0x1999999999999999ULL)
427 switch (tolower(*s)) {
428 case '0': v *= 10; break;
429 case '1': v *= 10; v += 1; break;
430 case '2': v *= 10; v += 2; break;
431 case '3': v *= 10; v += 3; break;
432 case '4': v *= 10; v += 4; break;
433 case '5': v *= 10; v += 5; break;
434 case '6': v *= 10; v += 6; break;
435 case '7': v *= 10; v += 7; break;
436 case '8': v *= 10; v += 8; break;
437 case '9': v *= 10; v += 9; break;
452 static void parse_number_hex(void)
454 assert(c == 'x' || c == 'X');
458 obstack_1grow(&symbol_obstack, (char) c);
461 obstack_1grow(&symbol_obstack, '\0');
462 char *string = obstack_finish(&symbol_obstack);
464 if(c == '.' || c == 'p' || c == 'P') {
466 panic("Hex floating point numbers not implemented yet");
468 if(*string == '\0') {
469 parse_error("invalid hex number");
470 lexer_token.type = T_ERROR;
474 lexer_token.type = T_INTEGER;
475 lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
476 if(*endptr != '\0') {
477 parse_error("hex number literal too long");
480 obstack_free(&symbol_obstack, string);
481 parse_integer_suffix(true);
484 static inline bool is_octal_digit(int chr)
486 return '0' <= chr && chr <= '7';
489 static void parse_number_oct(void)
491 while(is_octal_digit(c)) {
492 obstack_1grow(&symbol_obstack, (char) c);
495 obstack_1grow(&symbol_obstack, '\0');
496 char *string = obstack_finish(&symbol_obstack);
499 lexer_token.type = T_INTEGER;
500 lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
501 if(*endptr != '\0') {
502 parse_error("octal number literal too long");
505 obstack_free(&symbol_obstack, string);
506 parse_integer_suffix(true);
509 static void parse_number_dec(void)
511 bool is_float = false;
513 obstack_1grow(&symbol_obstack, (char) c);
518 obstack_1grow(&symbol_obstack, '.');
522 obstack_1grow(&symbol_obstack, (char) c);
527 if(c == 'e' || c == 'E') {
528 obstack_1grow(&symbol_obstack, 'e');
531 if(c == '-' || c == '+') {
532 obstack_1grow(&symbol_obstack, (char) c);
537 obstack_1grow(&symbol_obstack, (char) c);
543 obstack_1grow(&symbol_obstack, '\0');
544 char *string = obstack_finish(&symbol_obstack);
548 lexer_token.type = T_FLOATINGPOINT;
549 lexer_token.v.floatvalue = strtold(string, &endptr);
551 if(*endptr != '\0') {
552 parse_error("invalid number literal");
555 parse_floating_suffix();
558 lexer_token.type = T_INTEGER;
559 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
561 if(*endptr != '\0') {
562 parse_error("invalid number literal");
565 parse_integer_suffix(false);
567 obstack_free(&symbol_obstack, string);
570 static void parse_number(void)
592 parse_error("invalid octal number");
593 lexer_token.type = T_ERROR;
599 obstack_1grow(&symbol_obstack, '0');
608 static int parse_octal_sequence(const int first_digit)
610 assert(is_octal_digit(first_digit));
611 int value = first_digit - '0';
612 if (!is_octal_digit(c)) return value;
613 value = 8 * value + c - '0';
615 if (!is_octal_digit(c)) return value;
616 value = 8 * value + c - '0';
618 return (char_type)value;
621 static int parse_hex_sequence(void)
625 if (c >= '0' && c <= '9') {
626 value = 16 * value + c - '0';
627 } else if ('A' <= c && c <= 'F') {
628 value = 16 * value + c - 'A' + 10;
629 } else if ('a' <= c && c <= 'f') {
630 value = 16 * value + c - 'a' + 10;
637 return (char_type)value;
640 static int parse_escape_sequence(void)
648 case '"': return '"';
649 case '\'': return '\'';
650 case '\\': return '\\';
651 case '?': return '\?';
652 case 'a': return '\a';
653 case 'b': return '\b';
654 case 'f': return '\f';
655 case 'n': return '\n';
656 case 'r': return '\r';
657 case 't': return '\t';
658 case 'v': return '\v';
660 return parse_hex_sequence();
669 return parse_octal_sequence(ec);
671 parse_error("reached end of file while parsing escape sequence");
674 parse_error("unknown escape sequence");
679 const char *concat_strings(const char *s1, const char *s2)
681 size_t len1 = strlen(s1);
682 size_t len2 = strlen(s2);
684 char *concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
685 memcpy(concat, s1, len1);
686 memcpy(concat + len1, s2, len2 + 1);
688 const char *result = strset_insert(&stringset, concat);
689 if(result != concat) {
690 obstack_free(&symbol_obstack, concat);
696 static void parse_string_literal(void)
698 unsigned start_linenr = lexer_token.source_position.linenr;
709 tc = parse_escape_sequence();
710 obstack_1grow(&symbol_obstack, (char) tc);
714 error_prefix_at(lexer_token.source_position.input_name,
716 fprintf(stderr, "string has no end\n");
717 lexer_token.type = T_ERROR;
725 obstack_1grow(&symbol_obstack, (char) c);
733 /* TODO: concatenate multiple strings separated by whitespace... */
735 /* add finishing 0 to the string */
736 obstack_1grow(&symbol_obstack, '\0');
737 string = obstack_finish(&symbol_obstack);
739 /* check if there is already a copy of the string */
740 result = strset_insert(&stringset, string);
741 if(result != string) {
742 obstack_free(&symbol_obstack, string);
745 lexer_token.type = T_STRING_LITERAL;
746 lexer_token.v.string = result;
749 static void parse_wide_character_constant(void)
757 found_char = parse_escape_sequence();
761 parse_error("newline while parsing character constant");
767 goto end_of_wide_char_constant;
770 parse_error("EOF while parsing character constant");
771 lexer_token.type = T_ERROR;
775 if(found_char != 0) {
776 parse_error("more than 1 characters in character "
778 goto end_of_wide_char_constant;
787 end_of_wide_char_constant:
788 lexer_token.type = T_INTEGER;
789 lexer_token.v.intvalue = found_char;
790 lexer_token.datatype = type_wchar_t;
793 static void parse_wide_string_literal(void)
795 const unsigned start_linenr = lexer_token.source_position.linenr;
803 wchar_rep_t tc = parse_escape_sequence();
804 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
809 error_prefix_at(lexer_token.source_position.input_name,
811 fprintf(stderr, "string has no end\n");
812 lexer_token.type = T_ERROR;
821 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
830 /* TODO: concatenate multiple strings separated by whitespace... */
832 /* add finishing 0 to the string */
833 wchar_rep_t nul = L'\0';
834 obstack_grow(&symbol_obstack, &nul, sizeof(nul));
835 const size_t size = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
836 const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
838 #if 0 /* TODO hash */
839 /* check if there is already a copy of the string */
840 const wchar_rep_t *const result = strset_insert(&stringset, string);
841 if(result != string) {
842 obstack_free(&symbol_obstack, string);
845 const wchar_rep_t *const result = string;
848 lexer_token.type = T_WIDE_STRING_LITERAL;
849 lexer_token.v.wide_string.begin = result;
850 lexer_token.v.wide_string.size = size;
853 static void parse_character_constant(void)
861 found_char = parse_escape_sequence();
865 parse_error("newline while parsing character constant");
871 goto end_of_char_constant;
874 parse_error("EOF while parsing character constant");
875 lexer_token.type = T_ERROR;
879 if(found_char != 0) {
880 parse_error("more than 1 characters in character "
882 goto end_of_char_constant;
891 end_of_char_constant:
892 lexer_token.type = T_INTEGER;
893 lexer_token.v.intvalue = found_char;
894 lexer_token.datatype = type_int;
897 static void skip_multiline_comment(void)
899 unsigned start_linenr = lexer_token.source_position.linenr;
911 MATCH_NEWLINE(break;)
914 error_prefix_at(lexer_token.source_position.input_name,
916 fprintf(stderr, "at end of file while looking for comment end\n");
926 static void skip_line_comment(void)
944 static token_t pp_token;
946 static inline void next_pp_token(void)
948 lexer_next_preprocessing_token();
949 pp_token = lexer_token;
952 static void eat_until_newline(void)
954 while(pp_token.type != '\n' && pp_token.type != T_EOF) {
959 static void error_directive(void)
962 fprintf(stderr, "#error directive: \n");
964 /* parse pp-tokens until new-line */
967 static void define_directive(void)
969 lexer_next_preprocessing_token();
970 if(lexer_token.type != T_IDENTIFIER) {
971 parse_error("expected identifier after #define\n");
976 static void ifdef_directive(int is_ifndef)
979 lexer_next_preprocessing_token();
980 //expect_identifier();
984 static void endif_directive(void)
989 static void parse_line_directive(void)
991 if(pp_token.type != T_INTEGER) {
992 parse_error("expected integer");
994 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
997 if(pp_token.type == T_STRING_LITERAL) {
998 lexer_token.source_position.input_name = pp_token.v.string;
1002 eat_until_newline();
1005 static void parse_preprocessor_identifier(void)
1007 assert(pp_token.type == T_IDENTIFIER);
1008 symbol_t *symbol = pp_token.v.symbol;
1010 switch(symbol->pp_ID) {
1012 printf("include - enable header name parsing!\n");
1028 parse_line_directive();
1042 static void parse_preprocessor_directive(void)
1046 switch(pp_token.type) {
1048 parse_preprocessor_identifier();
1051 parse_line_directive();
1054 parse_error("invalid preprocessor directive");
1055 eat_until_newline();
1060 #define MAYBE_PROLOG \
1065 #define MAYBE(ch, set_type) \
1068 lexer_token.type = set_type; \
1071 #define ELSE_CODE(code) \
1075 } /* end of while(1) */ \
1078 #define ELSE(set_type) \
1080 lexer_token.type = set_type; \
1084 void lexer_next_preprocessing_token(void)
1094 lexer_token.type = '\n';
1100 /* might be a wide string ( L"string" ) */
1101 if(lexer_token.type == T_IDENTIFIER &&
1102 lexer_token.v.symbol == symbol_L) {
1104 parse_wide_string_literal();
1105 } else if(c == '\'') {
1106 parse_wide_character_constant();
1116 parse_string_literal();
1120 parse_character_constant();
1142 MAYBE('.', T_DOTDOTDOT)
1146 lexer_token.type = '.';
1152 MAYBE('&', T_ANDAND)
1153 MAYBE('=', T_ANDEQUAL)
1157 MAYBE('=', T_ASTERISKEQUAL)
1161 MAYBE('+', T_PLUSPLUS)
1162 MAYBE('=', T_PLUSEQUAL)
1166 MAYBE('>', T_MINUSGREATER)
1167 MAYBE('-', T_MINUSMINUS)
1168 MAYBE('=', T_MINUSEQUAL)
1172 MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1176 MAYBE('=', T_SLASHEQUAL)
1179 skip_multiline_comment();
1180 lexer_next_preprocessing_token();
1184 skip_line_comment();
1185 lexer_next_preprocessing_token();
1190 MAYBE('>', T_PERCENTGREATER)
1191 MAYBE('=', T_PERCENTEQUAL)
1196 MAYBE(':', T_PERCENTCOLONPERCENTCOLON)
1200 lexer_token.type = T_PERCENTCOLON;
1203 ELSE(T_PERCENTCOLON)
1207 MAYBE(':', T_LESSCOLON)
1208 MAYBE('%', T_LESSPERCENT)
1209 MAYBE('=', T_LESSEQUAL)
1212 MAYBE('=', T_LESSLESSEQUAL)
1217 MAYBE('=', T_GREATEREQUAL)
1220 MAYBE('=', T_GREATERGREATEREQUAL)
1221 ELSE(T_GREATERGREATER)
1225 MAYBE('=', T_CARETEQUAL)
1229 MAYBE('=', T_PIPEEQUAL)
1230 MAYBE('|', T_PIPEPIPE)
1234 MAYBE('>', T_COLONGREATER)
1238 MAYBE('=', T_EQUALEQUAL)
1242 MAYBE('#', T_HASHHASH)
1256 lexer_token.type = c;
1261 lexer_token.type = T_EOF;
1267 fprintf(stderr, "unknown character '%c' found\n", c);
1268 lexer_token.type = T_ERROR;
1274 void lexer_next_token(void)
1276 lexer_next_preprocessing_token();
1277 if(lexer_token.type != '\n')
1282 lexer_next_preprocessing_token();
1283 } while(lexer_token.type == '\n');
1285 if(lexer_token.type == '#') {
1286 parse_preprocessor_directive();
1291 void init_lexer(void)
1293 strset_init(&stringset);
1296 void lexer_open_stream(FILE *stream, const char *input_name)
1299 lexer_token.source_position.linenr = 0;
1300 lexer_token.source_position.input_name = input_name;
1302 symbol_L = symbol_table_insert("L");
1304 /* place a virtual \n at the beginning so the lexer knows that we're
1305 * at the beginning of a line */
1309 void exit_lexer(void)
1311 strset_destroy(&stringset);
1314 static __attribute__((unused))
1315 void dbg_pos(const source_position_t source_position)
1317 fprintf(stdout, "%s:%u\n", source_position.input_name,
1318 source_position.linenr);