5 #include "symbol_table_t.h"
7 #include "adt/strset.h"
21 /* No strtold on windows and no replacement yet */
22 #define strtold(s, e) strtod(s, e)
29 static char buf[1024 + MAX_PUTBACK];
30 static const char *bufend;
31 static const char *bufpos;
32 static strset_t stringset;
34 static type_t *type_int = NULL;
35 static type_t *type_uint = NULL;
36 static type_t *type_long = NULL;
37 static type_t *type_ulong = NULL;
38 static type_t *type_longlong = NULL;
39 static type_t *type_ulonglong = NULL;
40 static type_t *type_float = NULL;
41 static type_t *type_double = NULL;
42 static type_t *type_longdouble = NULL;
44 static void error_prefix_at(const char *input_name, unsigned linenr)
46 fprintf(stderr, "%s:%u: Error: ", input_name, linenr);
49 static void error_prefix(void)
51 error_prefix_at(lexer_token.source_position.input_name,
52 lexer_token.source_position.linenr);
55 static void parse_error(const char *msg)
58 fprintf(stderr, "%s\n", msg);
61 static inline void next_real_char(void)
64 if(bufpos >= bufend) {
65 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
71 bufpos = buf + MAX_PUTBACK;
72 bufend = buf + MAX_PUTBACK + s;
77 static inline void put_back(int pc)
79 assert(bufpos >= buf);
80 //assert(bufpos < buf+MAX_PUTBACK || *bufpos == pc);
82 char *p = buf + (bufpos - buf);
85 /* going backwards in the buffer is legal as long as it's not more often
90 printf("putback '%c'\n", pc);
94 static inline void next_char(void);
96 #define MATCH_NEWLINE(code) \
102 lexer_token.source_position.linenr++; \
106 lexer_token.source_position.linenr++; \
109 #define eat(c_type) do { assert(c == c_type); next_char(); } while(0)
111 static void maybe_concat_lines(void)
116 MATCH_NEWLINE(return;)
126 static inline void next_char(void)
130 /* filter trigraphs */
131 if(UNLIKELY(c == '\\')) {
132 maybe_concat_lines();
133 goto end_of_next_char;
137 goto end_of_next_char;
140 if(LIKELY(c != '?')) {
143 goto end_of_next_char;
148 case '=': c = '#'; break;
149 case '(': c = '['; break;
150 case '/': c = '\\'; maybe_concat_lines(); break;
151 case ')': c = ']'; break;
152 case '\'': c = '^'; break;
153 case '<': c = '{'; break;
154 case '!': c = '|'; break;
155 case '>': c = '}'; break;
156 case '-': c = '~'; break;
166 printf("nchar '%c'\n", c);
170 #define SYMBOL_CHARS \
237 static void parse_symbol(void)
242 obstack_1grow(&symbol_obstack, c);
249 obstack_1grow(&symbol_obstack, c);
259 obstack_1grow(&symbol_obstack, '\0');
261 string = obstack_finish(&symbol_obstack);
262 symbol = symbol_table_insert(string);
264 lexer_token.type = symbol->ID;
265 lexer_token.v.symbol = symbol;
267 if(symbol->string != string) {
268 obstack_free(&symbol_obstack, string);
272 static void parse_integer_suffix(void)
274 if(c == 'U' || c == 'u') {
276 if(c == 'L' || c == 'l') {
278 if(c == 'L' || c == 'l') {
280 lexer_token.datatype = type_ulonglong;
282 lexer_token.datatype = type_ulong;
285 lexer_token.datatype = type_uint;
287 } else if(c == 'l' || c == 'L') {
289 if(c == 'l' || c == 'L') {
291 if(c == 'u' || c == 'U') {
293 lexer_token.datatype = type_ulonglong;
295 lexer_token.datatype = type_longlong;
297 } else if(c == 'u' || c == 'U') {
299 lexer_token.datatype = type_ulong;
301 lexer_token.datatype = type_int;
304 lexer_token.datatype = type_int;
308 static void parse_floating_suffix(void)
311 /* TODO: do something usefull with the suffixes... */
315 lexer_token.datatype = type_float;
320 lexer_token.datatype = type_longdouble;
323 lexer_token.datatype = type_double;
329 * A replacement for strtoull. Only those parts needed for
330 * our parser are implemented.
332 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
333 unsigned long long v = 0;
338 /* check for overrun */
339 if (v <= 0x1000000000000000ULL)
341 switch (tolower(*s)) {
342 case '0': v <<= 4; break;
343 case '1': v <<= 4; v |= 0x1; break;
344 case '2': v <<= 4; v |= 0x2; break;
345 case '3': v <<= 4; v |= 0x3; break;
346 case '4': v <<= 4; v |= 0x4; break;
347 case '5': v <<= 4; v |= 0x5; break;
348 case '6': v <<= 4; v |= 0x6; break;
349 case '7': v <<= 4; v |= 0x7; break;
350 case '8': v <<= 4; v |= 0x8; break;
351 case '9': v <<= 4; v |= 0x9; break;
352 case 'a': v <<= 4; v |= 0xa; break;
353 case 'b': v <<= 4; v |= 0xb; break;
354 case 'c': v <<= 4; v |= 0xc; break;
355 case 'd': v <<= 4; v |= 0xd; break;
356 case 'e': v <<= 4; v |= 0xe; break;
357 case 'f': v <<= 4; v |= 0xf; break;
365 /* check for overrun */
366 if (v <= 0x2000000000000000ULL)
368 switch (tolower(*s)) {
369 case '0': v <<= 3; break;
370 case '1': v <<= 3; v |= 1; break;
371 case '2': v <<= 3; v |= 2; break;
372 case '3': v <<= 3; v |= 3; break;
373 case '4': v <<= 3; v |= 4; break;
374 case '5': v <<= 3; v |= 5; break;
375 case '6': v <<= 3; v |= 6; break;
376 case '7': v <<= 3; v |= 7; break;
384 /* check for overrun */
385 if (v > 0x1999999999999999ULL)
387 switch (tolower(*s)) {
388 case '0': v *= 10; break;
389 case '1': v *= 10; v += 1; break;
390 case '2': v *= 10; v += 2; break;
391 case '3': v *= 10; v += 3; break;
392 case '4': v *= 10; v += 4; break;
393 case '5': v *= 10; v += 5; break;
394 case '6': v *= 10; v += 6; break;
395 case '7': v *= 10; v += 7; break;
396 case '8': v *= 10; v += 8; break;
397 case '9': v *= 10; v += 9; break;
411 static void parse_number_hex(void)
413 assert(c == 'x' || c == 'X');
417 obstack_1grow(&symbol_obstack, c);
420 obstack_1grow(&symbol_obstack, '\0');
421 char *string = obstack_finish(&symbol_obstack);
423 if(c == '.' || c == 'p' || c == 'P') {
425 panic("Hex floating point numbers not implemented yet");
427 if(*string == '\0') {
428 parse_error("invalid hex number");
429 lexer_token.type = T_ERROR;
433 lexer_token.type = T_INTEGER;
434 lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
435 if(*endptr != '\0') {
436 parse_error("hex number literal too long");
439 obstack_free(&symbol_obstack, string);
440 parse_integer_suffix();
443 static inline bool is_octal_digit(int chr)
445 return '0' <= chr && chr <= '7';
448 static void parse_number_oct(void)
450 while(is_octal_digit(c)) {
451 obstack_1grow(&symbol_obstack, c);
454 obstack_1grow(&symbol_obstack, '\0');
455 char *string = obstack_finish(&symbol_obstack);
458 lexer_token.type = T_INTEGER;
459 lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
460 if(*endptr != '\0') {
461 parse_error("octal number literal too long");
464 obstack_free(&symbol_obstack, string);
465 parse_integer_suffix();
468 static void parse_number_dec(void)
470 bool is_float = false;
472 obstack_1grow(&symbol_obstack, c);
477 obstack_1grow(&symbol_obstack, '.');
481 obstack_1grow(&symbol_obstack, c);
486 if(c == 'e' || c == 'E') {
487 obstack_1grow(&symbol_obstack, 'e');
490 if(c == '-' || c == '+') {
491 obstack_1grow(&symbol_obstack, c);
496 obstack_1grow(&symbol_obstack, c);
502 obstack_1grow(&symbol_obstack, '\0');
503 char *string = obstack_finish(&symbol_obstack);
507 lexer_token.type = T_FLOATINGPOINT;
508 lexer_token.v.floatvalue = strtold(string, &endptr);
510 if(*endptr != '\0') {
511 parse_error("invalid number literal");
514 parse_floating_suffix();
516 lexer_token.type = T_INTEGER;
517 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
519 if(*endptr != '\0') {
520 parse_error("invalid number literal");
523 parse_integer_suffix();
525 obstack_free(&symbol_obstack, string);
528 static void parse_number(void)
550 parse_error("invalid octal number");
551 lexer_token.type = T_ERROR;
557 obstack_1grow(&symbol_obstack, '0');
566 static int parse_octal_sequence(const int first_digit)
568 assert(is_octal_digit(first_digit));
569 int value = first_digit - '0';
570 if (!is_octal_digit(c)) return value;
571 value = 8 * value + c - '0';
573 if (!is_octal_digit(c)) return value;
574 value = 8 * value + c - '0';
579 static int parse_hex_sequence(void)
583 if (c >= '0' && c <= '9') {
584 value = 16 * value + c - '0';
585 } else if ('A' <= c && c <= 'F') {
586 value = 16 * value + c - 'A' + 10;
587 } else if ('a' <= c && c <= 'f') {
588 value = 16 * value + c - 'a' + 10;
598 static int parse_escape_sequence(void)
606 case '"': return '"';
607 case '\'': return '\'';
608 case '\\': return '\\';
609 case '?': return '\?';
610 case 'a': return '\a';
611 case 'b': return '\b';
612 case 'f': return '\f';
613 case 'n': return '\n';
614 case 'r': return '\r';
615 case 't': return '\t';
616 case 'v': return '\v';
618 return parse_hex_sequence();
627 return parse_octal_sequence(ec);
629 parse_error("reached end of file while parsing escape sequence");
632 parse_error("unknown escape sequence");
637 const char *concat_strings(const char *s1, const char *s2)
639 size_t len1 = strlen(s1);
640 size_t len2 = strlen(s2);
642 char *concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
643 memcpy(concat, s1, len1);
644 memcpy(concat + len1, s2, len2 + 1);
646 const char *result = strset_insert(&stringset, concat);
647 if(result != concat) {
648 obstack_free(&symbol_obstack, concat);
654 static void parse_string_literal(void)
656 unsigned start_linenr = lexer_token.source_position.linenr;
667 tc = parse_escape_sequence();
668 obstack_1grow(&symbol_obstack, tc);
672 error_prefix_at(lexer_token.source_position.input_name,
674 fprintf(stderr, "string has no end\n");
675 lexer_token.type = T_ERROR;
683 obstack_1grow(&symbol_obstack, c);
691 /* TODO: concatenate multiple strings separated by whitespace... */
693 /* add finishing 0 to the string */
694 obstack_1grow(&symbol_obstack, '\0');
695 string = obstack_finish(&symbol_obstack);
697 /* check if there is already a copy of the string */
698 result = strset_insert(&stringset, string);
699 if(result != string) {
700 obstack_free(&symbol_obstack, string);
703 lexer_token.type = T_STRING_LITERAL;
704 lexer_token.v.string = result;
707 static void parse_character_constant(void)
715 found_char = parse_escape_sequence();
719 parse_error("newline while parsing character constant");
725 goto end_of_char_constant;
728 parse_error("EOF while parsing character constant");
729 lexer_token.type = T_ERROR;
733 if(found_char != 0) {
734 parse_error("more than 1 characters in character "
736 goto end_of_char_constant;
745 end_of_char_constant:
746 lexer_token.type = T_INTEGER;
747 lexer_token.v.intvalue = found_char;
750 static void skip_multiline_comment(void)
752 unsigned start_linenr = lexer_token.source_position.linenr;
764 MATCH_NEWLINE(break;)
767 error_prefix_at(lexer_token.source_position.input_name,
769 fprintf(stderr, "at end of file while looking for comment end\n");
779 static void skip_line_comment(void)
797 static token_t pp_token;
799 static inline void next_pp_token(void)
801 lexer_next_preprocessing_token();
802 pp_token = lexer_token;
805 static void eat_until_newline(void)
807 while(pp_token.type != '\n' && pp_token.type != T_EOF) {
812 static void error_directive(void)
815 fprintf(stderr, "#error directive: \n");
817 /* parse pp-tokens until new-line */
820 static void define_directive(void)
822 lexer_next_preprocessing_token();
823 if(lexer_token.type != T_IDENTIFIER) {
824 parse_error("expected identifier after #define\n");
829 static void ifdef_directive(int is_ifndef)
832 lexer_next_preprocessing_token();
833 //expect_identifier();
837 static void endif_directive(void)
842 static void parse_line_directive(void)
844 if(pp_token.type != T_INTEGER) {
845 parse_error("expected integer");
847 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
850 if(pp_token.type == T_STRING_LITERAL) {
851 lexer_token.source_position.input_name = pp_token.v.string;
858 static void parse_preprocessor_identifier(void)
860 assert(pp_token.type == T_IDENTIFIER);
861 symbol_t *symbol = pp_token.v.symbol;
863 switch(symbol->pp_ID) {
865 printf("include - enable header name parsing!\n");
881 parse_line_directive();
895 static void parse_preprocessor_directive(void)
899 switch(pp_token.type) {
901 parse_preprocessor_identifier();
904 parse_line_directive();
907 parse_error("invalid preprocessor directive");
913 #define MAYBE_PROLOG \
918 #define MAYBE(ch, set_type) \
921 lexer_token.type = set_type; \
924 #define ELSE_CODE(code) \
928 } /* end of while(1) */ \
931 #define ELSE(set_type) \
933 lexer_token.type = set_type; \
937 void lexer_next_preprocessing_token(void)
947 lexer_token.type = '\n';
953 /* might be a wide string ( L"string" ) */
954 if(c == '"' && (lexer_token.type == T_IDENTIFIER &&
955 lexer_token.v.symbol == symbol_L)) {
956 parse_string_literal();
966 parse_string_literal();
970 parse_character_constant();
977 MAYBE('.', T_DOTDOTDOT)
981 lexer_token.type = '.';
988 MAYBE('=', T_ANDEQUAL)
992 MAYBE('=', T_ASTERISKEQUAL)
996 MAYBE('+', T_PLUSPLUS)
997 MAYBE('=', T_PLUSEQUAL)
1001 MAYBE('>', T_MINUSGREATER)
1002 MAYBE('-', T_MINUSMINUS)
1003 MAYBE('=', T_MINUSEQUAL)
1007 MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1011 MAYBE('=', T_SLASHEQUAL)
1014 skip_multiline_comment();
1015 lexer_next_preprocessing_token();
1019 skip_line_comment();
1020 lexer_next_preprocessing_token();
1025 MAYBE('>', T_PERCENTGREATER)
1026 MAYBE('=', T_PERCENTEQUAL)
1031 MAYBE(':', T_PERCENTCOLONPERCENTCOLON)
1035 lexer_token.type = T_PERCENTCOLON;
1038 ELSE(T_PERCENTCOLON)
1042 MAYBE(':', T_LESSCOLON)
1043 MAYBE('%', T_LESSPERCENT)
1044 MAYBE('=', T_LESSEQUAL)
1047 MAYBE('=', T_LESSLESSEQUAL)
1052 MAYBE('=', T_GREATEREQUAL)
1055 MAYBE('=', T_GREATERGREATEREQUAL)
1056 ELSE(T_GREATERGREATER)
1060 MAYBE('=', T_CARETEQUAL)
1064 MAYBE('=', T_PIPEEQUAL)
1065 MAYBE('|', T_PIPEPIPE)
1069 MAYBE('>', T_COLONGREATER)
1073 MAYBE('=', T_EQUALEQUAL)
1077 MAYBE('#', T_HASHHASH)
1091 lexer_token.type = c;
1096 lexer_token.type = T_EOF;
1102 fprintf(stderr, "unknown character '%c' found\n", c);
1103 lexer_token.type = T_ERROR;
1109 void lexer_next_token(void)
1111 lexer_next_preprocessing_token();
1112 if(lexer_token.type != '\n')
1117 lexer_next_preprocessing_token();
1118 } while(lexer_token.type == '\n');
1120 if(lexer_token.type == '#') {
1121 parse_preprocessor_directive();
1126 void init_lexer(void)
1128 strset_init(&stringset);
1130 type_int = make_atomic_type(ATOMIC_TYPE_INT, TYPE_QUALIFIER_CONST);
1131 type_uint = make_atomic_type(ATOMIC_TYPE_UINT, TYPE_QUALIFIER_CONST);
1132 type_long = make_atomic_type(ATOMIC_TYPE_LONG, TYPE_QUALIFIER_CONST);
1133 type_ulong = make_atomic_type(ATOMIC_TYPE_ULONG, TYPE_QUALIFIER_CONST);
1134 type_longlong = make_atomic_type(ATOMIC_TYPE_LONGLONG,
1135 TYPE_QUALIFIER_CONST);
1136 type_ulonglong = make_atomic_type(ATOMIC_TYPE_ULONGLONG,
1137 TYPE_QUALIFIER_CONST);
1139 type_float = make_atomic_type(ATOMIC_TYPE_FLOAT, TYPE_QUALIFIER_CONST);
1140 type_double = make_atomic_type(ATOMIC_TYPE_DOUBLE,
1141 TYPE_QUALIFIER_CONST);
1142 type_longdouble = make_atomic_type(ATOMIC_TYPE_LONG_DOUBLE,
1143 TYPE_QUALIFIER_CONST);
1146 void lexer_open_stream(FILE *stream, const char *input_name)
1149 lexer_token.source_position.linenr = 0;
1150 lexer_token.source_position.input_name = input_name;
1152 symbol_L = symbol_table_insert("L");
1154 /* place a virtual \n at the beginning so the lexer knows that we're
1155 * at the beginning of a line */
1159 void exit_lexer(void)
1161 strset_destroy(&stringset);
1164 static __attribute__((unused))
1165 void dbg_pos(const source_position_t source_position)
1167 fprintf(stdout, "%s:%d\n", source_position.input_name,
1168 source_position.linenr);