5 #include "symbol_table_t.h"
7 #include "adt/strset.h"
21 /* No strtold on windows and no replacement yet */
22 #define strtold(s, e) strtod(s, e)
25 #define HAS_SIGNED_CHAR
26 //#define HAS_UNSIGNED_CHAR
28 #if defined HAS_SIGNED_CHAR
29 typedef signed char char_type;
30 #elif defined HAS_UNSIGNED_CHAR
31 typedef unsigned char char_type;
33 # error signedness of char not determined
40 static char buf[1024 + MAX_PUTBACK];
41 static const char *bufend;
42 static const char *bufpos;
43 static strset_t stringset;
45 static type_t *type_int = NULL;
46 static type_t *type_uint = NULL;
47 static type_t *type_long = NULL;
48 static type_t *type_ulong = NULL;
49 static type_t *type_longlong = NULL;
50 static type_t *type_ulonglong = NULL;
51 static type_t *type_float = NULL;
52 static type_t *type_double = NULL;
53 static type_t *type_longdouble = NULL;
55 static void error_prefix_at(const char *input_name, unsigned linenr)
57 fprintf(stderr, "%s:%u: Error: ", input_name, linenr);
60 static void error_prefix(void)
62 error_prefix_at(lexer_token.source_position.input_name,
63 lexer_token.source_position.linenr);
66 static void parse_error(const char *msg)
69 fprintf(stderr, "%s\n", msg);
72 static inline void next_real_char(void)
75 if(bufpos >= bufend) {
76 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
82 bufpos = buf + MAX_PUTBACK;
83 bufend = buf + MAX_PUTBACK + s;
88 static inline void put_back(int pc)
90 assert(bufpos >= buf);
91 //assert(bufpos < buf+MAX_PUTBACK || *bufpos == pc);
93 char *p = buf + (bufpos - buf);
96 /* going backwards in the buffer is legal as long as it's not more often
101 printf("putback '%c'\n", pc);
105 static inline void next_char(void);
107 #define MATCH_NEWLINE(code) \
113 lexer_token.source_position.linenr++; \
117 lexer_token.source_position.linenr++; \
120 #define eat(c_type) do { assert(c == c_type); next_char(); } while(0)
122 static void maybe_concat_lines(void)
127 MATCH_NEWLINE(return;)
137 static inline void next_char(void)
141 /* filter trigraphs */
142 if(UNLIKELY(c == '\\')) {
143 maybe_concat_lines();
144 goto end_of_next_char;
148 goto end_of_next_char;
151 if(LIKELY(c != '?')) {
154 goto end_of_next_char;
159 case '=': c = '#'; break;
160 case '(': c = '['; break;
161 case '/': c = '\\'; maybe_concat_lines(); break;
162 case ')': c = ']'; break;
163 case '\'': c = '^'; break;
164 case '<': c = '{'; break;
165 case '!': c = '|'; break;
166 case '>': c = '}'; break;
167 case '-': c = '~'; break;
177 printf("nchar '%c'\n", c);
181 #define SYMBOL_CHARS \
248 static void parse_symbol(void)
253 obstack_1grow(&symbol_obstack, c);
260 obstack_1grow(&symbol_obstack, c);
270 obstack_1grow(&symbol_obstack, '\0');
272 string = obstack_finish(&symbol_obstack);
273 symbol = symbol_table_insert(string);
275 lexer_token.type = symbol->ID;
276 lexer_token.v.symbol = symbol;
278 if(symbol->string != string) {
279 obstack_free(&symbol_obstack, string);
283 static void parse_integer_suffix(void)
285 if(c == 'U' || c == 'u') {
287 if(c == 'L' || c == 'l') {
289 if(c == 'L' || c == 'l') {
291 lexer_token.datatype = type_ulonglong;
293 lexer_token.datatype = type_ulong;
296 lexer_token.datatype = type_uint;
298 } else if(c == 'l' || c == 'L') {
300 if(c == 'l' || c == 'L') {
302 if(c == 'u' || c == 'U') {
304 lexer_token.datatype = type_ulonglong;
306 lexer_token.datatype = type_longlong;
308 } else if(c == 'u' || c == 'U') {
310 lexer_token.datatype = type_ulong;
312 lexer_token.datatype = type_int;
315 lexer_token.datatype = type_int;
319 static void parse_floating_suffix(void)
322 /* TODO: do something usefull with the suffixes... */
326 lexer_token.datatype = type_float;
331 lexer_token.datatype = type_longdouble;
334 lexer_token.datatype = type_double;
340 * A replacement for strtoull. Only those parts needed for
341 * our parser are implemented.
343 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
344 unsigned long long v = 0;
349 /* check for overrun */
350 if (v >= 0x1000000000000000ULL)
352 switch (tolower(*s)) {
353 case '0': v <<= 4; break;
354 case '1': v <<= 4; v |= 0x1; break;
355 case '2': v <<= 4; v |= 0x2; break;
356 case '3': v <<= 4; v |= 0x3; break;
357 case '4': v <<= 4; v |= 0x4; break;
358 case '5': v <<= 4; v |= 0x5; break;
359 case '6': v <<= 4; v |= 0x6; break;
360 case '7': v <<= 4; v |= 0x7; break;
361 case '8': v <<= 4; v |= 0x8; break;
362 case '9': v <<= 4; v |= 0x9; break;
363 case 'a': v <<= 4; v |= 0xa; break;
364 case 'b': v <<= 4; v |= 0xb; break;
365 case 'c': v <<= 4; v |= 0xc; break;
366 case 'd': v <<= 4; v |= 0xd; break;
367 case 'e': v <<= 4; v |= 0xe; break;
368 case 'f': v <<= 4; v |= 0xf; break;
376 /* check for overrun */
377 if (v >= 0x2000000000000000ULL)
379 switch (tolower(*s)) {
380 case '0': v <<= 3; break;
381 case '1': v <<= 3; v |= 1; break;
382 case '2': v <<= 3; v |= 2; break;
383 case '3': v <<= 3; v |= 3; break;
384 case '4': v <<= 3; v |= 4; break;
385 case '5': v <<= 3; v |= 5; break;
386 case '6': v <<= 3; v |= 6; break;
387 case '7': v <<= 3; v |= 7; break;
395 /* check for overrun */
396 if (v > 0x1999999999999999ULL)
398 switch (tolower(*s)) {
399 case '0': v *= 10; break;
400 case '1': v *= 10; v += 1; break;
401 case '2': v *= 10; v += 2; break;
402 case '3': v *= 10; v += 3; break;
403 case '4': v *= 10; v += 4; break;
404 case '5': v *= 10; v += 5; break;
405 case '6': v *= 10; v += 6; break;
406 case '7': v *= 10; v += 7; break;
407 case '8': v *= 10; v += 8; break;
408 case '9': v *= 10; v += 9; break;
423 static void parse_number_hex(void)
425 assert(c == 'x' || c == 'X');
429 obstack_1grow(&symbol_obstack, c);
432 obstack_1grow(&symbol_obstack, '\0');
433 char *string = obstack_finish(&symbol_obstack);
435 if(c == '.' || c == 'p' || c == 'P') {
437 panic("Hex floating point numbers not implemented yet");
439 if(*string == '\0') {
440 parse_error("invalid hex number");
441 lexer_token.type = T_ERROR;
445 lexer_token.type = T_INTEGER;
446 lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
447 if(*endptr != '\0') {
448 parse_error("hex number literal too long");
451 obstack_free(&symbol_obstack, string);
452 parse_integer_suffix();
455 static inline bool is_octal_digit(int chr)
457 return '0' <= chr && chr <= '7';
460 static void parse_number_oct(void)
462 while(is_octal_digit(c)) {
463 obstack_1grow(&symbol_obstack, c);
466 obstack_1grow(&symbol_obstack, '\0');
467 char *string = obstack_finish(&symbol_obstack);
470 lexer_token.type = T_INTEGER;
471 lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
472 if(*endptr != '\0') {
473 parse_error("octal number literal too long");
476 obstack_free(&symbol_obstack, string);
477 parse_integer_suffix();
480 static void parse_number_dec(void)
482 bool is_float = false;
484 obstack_1grow(&symbol_obstack, c);
489 obstack_1grow(&symbol_obstack, '.');
493 obstack_1grow(&symbol_obstack, c);
498 if(c == 'e' || c == 'E') {
499 obstack_1grow(&symbol_obstack, 'e');
502 if(c == '-' || c == '+') {
503 obstack_1grow(&symbol_obstack, c);
508 obstack_1grow(&symbol_obstack, c);
514 obstack_1grow(&symbol_obstack, '\0');
515 char *string = obstack_finish(&symbol_obstack);
519 lexer_token.type = T_FLOATINGPOINT;
520 lexer_token.v.floatvalue = strtold(string, &endptr);
522 if(*endptr != '\0') {
523 parse_error("invalid number literal");
526 parse_floating_suffix();
529 lexer_token.type = T_INTEGER;
530 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
532 if(*endptr != '\0') {
533 parse_error("invalid number literal");
536 parse_integer_suffix();
538 obstack_free(&symbol_obstack, string);
541 static void parse_number(void)
563 parse_error("invalid octal number");
564 lexer_token.type = T_ERROR;
570 obstack_1grow(&symbol_obstack, '0');
579 static int parse_octal_sequence(const int first_digit)
581 assert(is_octal_digit(first_digit));
582 int value = first_digit - '0';
583 if (!is_octal_digit(c)) return value;
584 value = 8 * value + c - '0';
586 if (!is_octal_digit(c)) return value;
587 value = 8 * value + c - '0';
589 return (char_type)value;
592 static int parse_hex_sequence(void)
596 if (c >= '0' && c <= '9') {
597 value = 16 * value + c - '0';
598 } else if ('A' <= c && c <= 'F') {
599 value = 16 * value + c - 'A' + 10;
600 } else if ('a' <= c && c <= 'f') {
601 value = 16 * value + c - 'a' + 10;
608 return (char_type)value;
611 static int parse_escape_sequence(void)
619 case '"': return '"';
620 case '\'': return '\'';
621 case '\\': return '\\';
622 case '?': return '\?';
623 case 'a': return '\a';
624 case 'b': return '\b';
625 case 'f': return '\f';
626 case 'n': return '\n';
627 case 'r': return '\r';
628 case 't': return '\t';
629 case 'v': return '\v';
631 return parse_hex_sequence();
640 return parse_octal_sequence(ec);
642 parse_error("reached end of file while parsing escape sequence");
645 parse_error("unknown escape sequence");
650 const char *concat_strings(const char *s1, const char *s2)
652 size_t len1 = strlen(s1);
653 size_t len2 = strlen(s2);
655 char *concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
656 memcpy(concat, s1, len1);
657 memcpy(concat + len1, s2, len2 + 1);
659 const char *result = strset_insert(&stringset, concat);
660 if(result != concat) {
661 obstack_free(&symbol_obstack, concat);
667 static void parse_string_literal(void)
669 unsigned start_linenr = lexer_token.source_position.linenr;
680 tc = parse_escape_sequence();
681 obstack_1grow(&symbol_obstack, tc);
685 error_prefix_at(lexer_token.source_position.input_name,
687 fprintf(stderr, "string has no end\n");
688 lexer_token.type = T_ERROR;
696 obstack_1grow(&symbol_obstack, c);
704 /* TODO: concatenate multiple strings separated by whitespace... */
706 /* add finishing 0 to the string */
707 obstack_1grow(&symbol_obstack, '\0');
708 string = obstack_finish(&symbol_obstack);
710 /* check if there is already a copy of the string */
711 result = strset_insert(&stringset, string);
712 if(result != string) {
713 obstack_free(&symbol_obstack, string);
716 lexer_token.type = T_STRING_LITERAL;
717 lexer_token.v.string = result;
720 static void parse_character_constant(void)
728 found_char = parse_escape_sequence();
732 parse_error("newline while parsing character constant");
738 goto end_of_char_constant;
741 parse_error("EOF while parsing character constant");
742 lexer_token.type = T_ERROR;
746 if(found_char != 0) {
747 parse_error("more than 1 characters in character "
749 goto end_of_char_constant;
758 end_of_char_constant:
759 lexer_token.type = T_INTEGER;
760 lexer_token.v.intvalue = found_char;
763 static void skip_multiline_comment(void)
765 unsigned start_linenr = lexer_token.source_position.linenr;
777 MATCH_NEWLINE(break;)
780 error_prefix_at(lexer_token.source_position.input_name,
782 fprintf(stderr, "at end of file while looking for comment end\n");
792 static void skip_line_comment(void)
810 static token_t pp_token;
812 static inline void next_pp_token(void)
814 lexer_next_preprocessing_token();
815 pp_token = lexer_token;
818 static void eat_until_newline(void)
820 while(pp_token.type != '\n' && pp_token.type != T_EOF) {
825 static void error_directive(void)
828 fprintf(stderr, "#error directive: \n");
830 /* parse pp-tokens until new-line */
833 static void define_directive(void)
835 lexer_next_preprocessing_token();
836 if(lexer_token.type != T_IDENTIFIER) {
837 parse_error("expected identifier after #define\n");
842 static void ifdef_directive(int is_ifndef)
845 lexer_next_preprocessing_token();
846 //expect_identifier();
850 static void endif_directive(void)
855 static void parse_line_directive(void)
857 if(pp_token.type != T_INTEGER) {
858 parse_error("expected integer");
860 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
863 if(pp_token.type == T_STRING_LITERAL) {
864 lexer_token.source_position.input_name = pp_token.v.string;
871 static void parse_preprocessor_identifier(void)
873 assert(pp_token.type == T_IDENTIFIER);
874 symbol_t *symbol = pp_token.v.symbol;
876 switch(symbol->pp_ID) {
878 printf("include - enable header name parsing!\n");
894 parse_line_directive();
908 static void parse_preprocessor_directive(void)
912 switch(pp_token.type) {
914 parse_preprocessor_identifier();
917 parse_line_directive();
920 parse_error("invalid preprocessor directive");
926 #define MAYBE_PROLOG \
931 #define MAYBE(ch, set_type) \
934 lexer_token.type = set_type; \
937 #define ELSE_CODE(code) \
941 } /* end of while(1) */ \
944 #define ELSE(set_type) \
946 lexer_token.type = set_type; \
950 void lexer_next_preprocessing_token(void)
960 lexer_token.type = '\n';
966 /* might be a wide string ( L"string" ) */
967 if(c == '"' && (lexer_token.type == T_IDENTIFIER &&
968 lexer_token.v.symbol == symbol_L)) {
969 parse_string_literal();
979 parse_string_literal();
983 parse_character_constant();
990 MAYBE('.', T_DOTDOTDOT)
994 lexer_token.type = '.';
1000 MAYBE('&', T_ANDAND)
1001 MAYBE('=', T_ANDEQUAL)
1005 MAYBE('=', T_ASTERISKEQUAL)
1009 MAYBE('+', T_PLUSPLUS)
1010 MAYBE('=', T_PLUSEQUAL)
1014 MAYBE('>', T_MINUSGREATER)
1015 MAYBE('-', T_MINUSMINUS)
1016 MAYBE('=', T_MINUSEQUAL)
1020 MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1024 MAYBE('=', T_SLASHEQUAL)
1027 skip_multiline_comment();
1028 lexer_next_preprocessing_token();
1032 skip_line_comment();
1033 lexer_next_preprocessing_token();
1038 MAYBE('>', T_PERCENTGREATER)
1039 MAYBE('=', T_PERCENTEQUAL)
1044 MAYBE(':', T_PERCENTCOLONPERCENTCOLON)
1048 lexer_token.type = T_PERCENTCOLON;
1051 ELSE(T_PERCENTCOLON)
1055 MAYBE(':', T_LESSCOLON)
1056 MAYBE('%', T_LESSPERCENT)
1057 MAYBE('=', T_LESSEQUAL)
1060 MAYBE('=', T_LESSLESSEQUAL)
1065 MAYBE('=', T_GREATEREQUAL)
1068 MAYBE('=', T_GREATERGREATEREQUAL)
1069 ELSE(T_GREATERGREATER)
1073 MAYBE('=', T_CARETEQUAL)
1077 MAYBE('=', T_PIPEEQUAL)
1078 MAYBE('|', T_PIPEPIPE)
1082 MAYBE('>', T_COLONGREATER)
1086 MAYBE('=', T_EQUALEQUAL)
1090 MAYBE('#', T_HASHHASH)
1104 lexer_token.type = c;
1109 lexer_token.type = T_EOF;
1115 fprintf(stderr, "unknown character '%c' found\n", c);
1116 lexer_token.type = T_ERROR;
1122 void lexer_next_token(void)
1124 lexer_next_preprocessing_token();
1125 if(lexer_token.type != '\n')
1130 lexer_next_preprocessing_token();
1131 } while(lexer_token.type == '\n');
1133 if(lexer_token.type == '#') {
1134 parse_preprocessor_directive();
1139 void init_lexer(void)
1141 strset_init(&stringset);
1143 type_int = make_atomic_type(ATOMIC_TYPE_INT, TYPE_QUALIFIER_CONST);
1144 type_uint = make_atomic_type(ATOMIC_TYPE_UINT, TYPE_QUALIFIER_CONST);
1145 type_long = make_atomic_type(ATOMIC_TYPE_LONG, TYPE_QUALIFIER_CONST);
1146 type_ulong = make_atomic_type(ATOMIC_TYPE_ULONG, TYPE_QUALIFIER_CONST);
1147 type_longlong = make_atomic_type(ATOMIC_TYPE_LONGLONG,
1148 TYPE_QUALIFIER_CONST);
1149 type_ulonglong = make_atomic_type(ATOMIC_TYPE_ULONGLONG,
1150 TYPE_QUALIFIER_CONST);
1152 type_float = make_atomic_type(ATOMIC_TYPE_FLOAT, TYPE_QUALIFIER_CONST);
1153 type_double = make_atomic_type(ATOMIC_TYPE_DOUBLE,
1154 TYPE_QUALIFIER_CONST);
1155 type_longdouble = make_atomic_type(ATOMIC_TYPE_LONG_DOUBLE,
1156 TYPE_QUALIFIER_CONST);
1159 void lexer_open_stream(FILE *stream, const char *input_name)
1162 lexer_token.source_position.linenr = 0;
1163 lexer_token.source_position.input_name = input_name;
1165 symbol_L = symbol_table_insert("L");
1167 /* place a virtual \n at the beginning so the lexer knows that we're
1168 * at the beginning of a line */
1172 void exit_lexer(void)
1174 strset_destroy(&stringset);
1177 static __attribute__((unused))
1178 void dbg_pos(const source_position_t source_position)
1180 fprintf(stdout, "%s:%d\n", source_position.input_name,
1181 source_position.linenr);