5 #include "symbol_table_t.h"
7 #include "adt/strset.h"
10 #include "target_architecture.h"
22 /* No strtold on windows and no replacement yet */
23 #define strtold(s, e) strtod(s, e)
26 #if defined HAS_SIGNED_CHAR
27 typedef signed char char_type;
28 #elif defined HAS_UNSIGNED_CHAR
29 typedef unsigned char char_type;
31 # error signedness of char not determined
38 static char buf[1024 + MAX_PUTBACK];
39 static const char *bufend;
40 static const char *bufpos;
41 static strset_t stringset;
43 static type_t *type_int = NULL;
44 static type_t *type_uint = NULL;
45 static type_t *type_long = NULL;
46 static type_t *type_ulong = NULL;
47 static type_t *type_longlong = NULL;
48 static type_t *type_ulonglong = NULL;
49 static type_t *type_float = NULL;
50 static type_t *type_double = NULL;
51 static type_t *type_longdouble = NULL;
53 static void error_prefix_at(const char *input_name, unsigned linenr)
55 fprintf(stderr, "%s:%u: Error: ", input_name, linenr);
58 static void error_prefix(void)
60 error_prefix_at(lexer_token.source_position.input_name,
61 lexer_token.source_position.linenr);
64 static void parse_error(const char *msg)
67 fprintf(stderr, "%s\n", msg);
70 static inline void next_real_char(void)
73 if(bufpos >= bufend) {
74 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
80 bufpos = buf + MAX_PUTBACK;
81 bufend = buf + MAX_PUTBACK + s;
86 static inline void put_back(int pc)
88 assert(bufpos >= buf);
89 //assert(bufpos < buf+MAX_PUTBACK || *bufpos == pc);
91 char *p = buf + (bufpos - buf);
94 /* going backwards in the buffer is legal as long as it's not more often
99 printf("putback '%c'\n", pc);
103 static inline void next_char(void);
105 #define MATCH_NEWLINE(code) \
111 lexer_token.source_position.linenr++; \
115 lexer_token.source_position.linenr++; \
118 #define eat(c_type) do { assert(c == c_type); next_char(); } while(0)
120 static void maybe_concat_lines(void)
125 MATCH_NEWLINE(return;)
135 static inline void next_char(void)
139 /* filter trigraphs */
140 if(UNLIKELY(c == '\\')) {
141 maybe_concat_lines();
142 goto end_of_next_char;
146 goto end_of_next_char;
149 if(LIKELY(c != '?')) {
152 goto end_of_next_char;
157 case '=': c = '#'; break;
158 case '(': c = '['; break;
159 case '/': c = '\\'; maybe_concat_lines(); break;
160 case ')': c = ']'; break;
161 case '\'': c = '^'; break;
162 case '<': c = '{'; break;
163 case '!': c = '|'; break;
164 case '>': c = '}'; break;
165 case '-': c = '~'; break;
175 printf("nchar '%c'\n", c);
179 #define SYMBOL_CHARS \
246 static void parse_symbol(void)
251 obstack_1grow(&symbol_obstack, (char) c);
258 obstack_1grow(&symbol_obstack, (char) c);
268 obstack_1grow(&symbol_obstack, '\0');
270 string = obstack_finish(&symbol_obstack);
271 symbol = symbol_table_insert(string);
273 lexer_token.type = symbol->ID;
274 lexer_token.v.symbol = symbol;
276 if(symbol->string != string) {
277 obstack_free(&symbol_obstack, string);
281 static void parse_integer_suffix(bool is_oct_hex)
283 bool is_unsigned = false;
284 bool min_long = false;
285 bool min_longlong = false;
287 if(c == 'U' || c == 'u') {
290 if(c == 'L' || c == 'l') {
293 if(c == 'L' || c == 'l') {
298 } else if(c == 'l' || c == 'L') {
301 if(c == 'l' || c == 'L') {
304 if(c == 'u' || c == 'U') {
308 } else if(c == 'u' || c == 'U') {
311 lexer_token.datatype = type_ulong;
316 long long v = lexer_token.v.intvalue;
318 if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
319 lexer_token.datatype = type_int;
321 } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
322 lexer_token.datatype = type_uint;
327 if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
328 lexer_token.datatype = type_long;
330 } else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) {
331 lexer_token.datatype = type_ulong;
335 unsigned long long uv = (unsigned long long) v;
336 if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
337 lexer_token.datatype = type_ulonglong;
341 lexer_token.datatype = type_longlong;
343 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
344 if(!min_long && v <= TARGET_UINT_MAX) {
345 lexer_token.datatype = type_uint;
348 if(!min_longlong && v <= TARGET_ULONG_MAX) {
349 lexer_token.datatype = type_ulong;
352 lexer_token.datatype = type_ulonglong;
356 static void parse_floating_suffix(void)
359 /* TODO: do something usefull with the suffixes... */
363 lexer_token.datatype = type_float;
368 lexer_token.datatype = type_longdouble;
371 lexer_token.datatype = type_double;
377 * A replacement for strtoull. Only those parts needed for
378 * our parser are implemented.
380 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
381 unsigned long long v = 0;
386 /* check for overrun */
387 if (v >= 0x1000000000000000ULL)
389 switch (tolower(*s)) {
390 case '0': v <<= 4; break;
391 case '1': v <<= 4; v |= 0x1; break;
392 case '2': v <<= 4; v |= 0x2; break;
393 case '3': v <<= 4; v |= 0x3; break;
394 case '4': v <<= 4; v |= 0x4; break;
395 case '5': v <<= 4; v |= 0x5; break;
396 case '6': v <<= 4; v |= 0x6; break;
397 case '7': v <<= 4; v |= 0x7; break;
398 case '8': v <<= 4; v |= 0x8; break;
399 case '9': v <<= 4; v |= 0x9; break;
400 case 'a': v <<= 4; v |= 0xa; break;
401 case 'b': v <<= 4; v |= 0xb; break;
402 case 'c': v <<= 4; v |= 0xc; break;
403 case 'd': v <<= 4; v |= 0xd; break;
404 case 'e': v <<= 4; v |= 0xe; break;
405 case 'f': v <<= 4; v |= 0xf; break;
413 /* check for overrun */
414 if (v >= 0x2000000000000000ULL)
416 switch (tolower(*s)) {
417 case '0': v <<= 3; break;
418 case '1': v <<= 3; v |= 1; break;
419 case '2': v <<= 3; v |= 2; break;
420 case '3': v <<= 3; v |= 3; break;
421 case '4': v <<= 3; v |= 4; break;
422 case '5': v <<= 3; v |= 5; break;
423 case '6': v <<= 3; v |= 6; break;
424 case '7': v <<= 3; v |= 7; break;
432 /* check for overrun */
433 if (v > 0x1999999999999999ULL)
435 switch (tolower(*s)) {
436 case '0': v *= 10; break;
437 case '1': v *= 10; v += 1; break;
438 case '2': v *= 10; v += 2; break;
439 case '3': v *= 10; v += 3; break;
440 case '4': v *= 10; v += 4; break;
441 case '5': v *= 10; v += 5; break;
442 case '6': v *= 10; v += 6; break;
443 case '7': v *= 10; v += 7; break;
444 case '8': v *= 10; v += 8; break;
445 case '9': v *= 10; v += 9; break;
460 static void parse_number_hex(void)
462 assert(c == 'x' || c == 'X');
466 obstack_1grow(&symbol_obstack, (char) c);
469 obstack_1grow(&symbol_obstack, '\0');
470 char *string = obstack_finish(&symbol_obstack);
472 if(c == '.' || c == 'p' || c == 'P') {
474 panic("Hex floating point numbers not implemented yet");
476 if(*string == '\0') {
477 parse_error("invalid hex number");
478 lexer_token.type = T_ERROR;
482 lexer_token.type = T_INTEGER;
483 lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
484 if(*endptr != '\0') {
485 parse_error("hex number literal too long");
488 obstack_free(&symbol_obstack, string);
489 parse_integer_suffix(true);
492 static inline bool is_octal_digit(int chr)
494 return '0' <= chr && chr <= '7';
497 static void parse_number_oct(void)
499 while(is_octal_digit(c)) {
500 obstack_1grow(&symbol_obstack, (char) c);
503 obstack_1grow(&symbol_obstack, '\0');
504 char *string = obstack_finish(&symbol_obstack);
507 lexer_token.type = T_INTEGER;
508 lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
509 if(*endptr != '\0') {
510 parse_error("octal number literal too long");
513 obstack_free(&symbol_obstack, string);
514 parse_integer_suffix(true);
517 static void parse_number_dec(void)
519 bool is_float = false;
521 obstack_1grow(&symbol_obstack, (char) c);
526 obstack_1grow(&symbol_obstack, '.');
530 obstack_1grow(&symbol_obstack, (char) c);
535 if(c == 'e' || c == 'E') {
536 obstack_1grow(&symbol_obstack, 'e');
539 if(c == '-' || c == '+') {
540 obstack_1grow(&symbol_obstack, (char) c);
545 obstack_1grow(&symbol_obstack, (char) c);
551 obstack_1grow(&symbol_obstack, '\0');
552 char *string = obstack_finish(&symbol_obstack);
556 lexer_token.type = T_FLOATINGPOINT;
557 lexer_token.v.floatvalue = strtold(string, &endptr);
559 if(*endptr != '\0') {
560 parse_error("invalid number literal");
563 parse_floating_suffix();
566 lexer_token.type = T_INTEGER;
567 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
569 if(*endptr != '\0') {
570 parse_error("invalid number literal");
573 parse_integer_suffix(false);
575 obstack_free(&symbol_obstack, string);
578 static void parse_number(void)
600 parse_error("invalid octal number");
601 lexer_token.type = T_ERROR;
607 obstack_1grow(&symbol_obstack, '0');
616 static int parse_octal_sequence(const int first_digit)
618 assert(is_octal_digit(first_digit));
619 int value = first_digit - '0';
620 if (!is_octal_digit(c)) return value;
621 value = 8 * value + c - '0';
623 if (!is_octal_digit(c)) return value;
624 value = 8 * value + c - '0';
626 return (char_type)value;
629 static int parse_hex_sequence(void)
633 if (c >= '0' && c <= '9') {
634 value = 16 * value + c - '0';
635 } else if ('A' <= c && c <= 'F') {
636 value = 16 * value + c - 'A' + 10;
637 } else if ('a' <= c && c <= 'f') {
638 value = 16 * value + c - 'a' + 10;
645 return (char_type)value;
648 static int parse_escape_sequence(void)
656 case '"': return '"';
657 case '\'': return '\'';
658 case '\\': return '\\';
659 case '?': return '\?';
660 case 'a': return '\a';
661 case 'b': return '\b';
662 case 'f': return '\f';
663 case 'n': return '\n';
664 case 'r': return '\r';
665 case 't': return '\t';
666 case 'v': return '\v';
668 return parse_hex_sequence();
677 return parse_octal_sequence(ec);
679 parse_error("reached end of file while parsing escape sequence");
682 parse_error("unknown escape sequence");
687 const char *concat_strings(const char *s1, const char *s2)
689 size_t len1 = strlen(s1);
690 size_t len2 = strlen(s2);
692 char *concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
693 memcpy(concat, s1, len1);
694 memcpy(concat + len1, s2, len2 + 1);
696 const char *result = strset_insert(&stringset, concat);
697 if(result != concat) {
698 obstack_free(&symbol_obstack, concat);
704 static void parse_string_literal(void)
706 unsigned start_linenr = lexer_token.source_position.linenr;
717 tc = parse_escape_sequence();
718 obstack_1grow(&symbol_obstack, (char) tc);
722 error_prefix_at(lexer_token.source_position.input_name,
724 fprintf(stderr, "string has no end\n");
725 lexer_token.type = T_ERROR;
733 obstack_1grow(&symbol_obstack, (char) c);
741 /* TODO: concatenate multiple strings separated by whitespace... */
743 /* add finishing 0 to the string */
744 obstack_1grow(&symbol_obstack, '\0');
745 string = obstack_finish(&symbol_obstack);
747 /* check if there is already a copy of the string */
748 result = strset_insert(&stringset, string);
749 if(result != string) {
750 obstack_free(&symbol_obstack, string);
753 lexer_token.type = T_STRING_LITERAL;
754 lexer_token.v.string = result;
757 static void parse_character_constant(void)
765 found_char = parse_escape_sequence();
769 parse_error("newline while parsing character constant");
775 goto end_of_char_constant;
778 parse_error("EOF while parsing character constant");
779 lexer_token.type = T_ERROR;
783 if(found_char != 0) {
784 parse_error("more than 1 characters in character "
786 goto end_of_char_constant;
795 end_of_char_constant:
796 lexer_token.type = T_INTEGER;
797 lexer_token.v.intvalue = found_char;
800 static void skip_multiline_comment(void)
802 unsigned start_linenr = lexer_token.source_position.linenr;
814 MATCH_NEWLINE(break;)
817 error_prefix_at(lexer_token.source_position.input_name,
819 fprintf(stderr, "at end of file while looking for comment end\n");
829 static void skip_line_comment(void)
847 static token_t pp_token;
849 static inline void next_pp_token(void)
851 lexer_next_preprocessing_token();
852 pp_token = lexer_token;
855 static void eat_until_newline(void)
857 while(pp_token.type != '\n' && pp_token.type != T_EOF) {
862 static void error_directive(void)
865 fprintf(stderr, "#error directive: \n");
867 /* parse pp-tokens until new-line */
870 static void define_directive(void)
872 lexer_next_preprocessing_token();
873 if(lexer_token.type != T_IDENTIFIER) {
874 parse_error("expected identifier after #define\n");
879 static void ifdef_directive(int is_ifndef)
882 lexer_next_preprocessing_token();
883 //expect_identifier();
887 static void endif_directive(void)
892 static void parse_line_directive(void)
894 if(pp_token.type != T_INTEGER) {
895 parse_error("expected integer");
897 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
900 if(pp_token.type == T_STRING_LITERAL) {
901 lexer_token.source_position.input_name = pp_token.v.string;
908 static void parse_preprocessor_identifier(void)
910 assert(pp_token.type == T_IDENTIFIER);
911 symbol_t *symbol = pp_token.v.symbol;
913 switch(symbol->pp_ID) {
915 printf("include - enable header name parsing!\n");
931 parse_line_directive();
945 static void parse_preprocessor_directive(void)
949 switch(pp_token.type) {
951 parse_preprocessor_identifier();
954 parse_line_directive();
957 parse_error("invalid preprocessor directive");
963 #define MAYBE_PROLOG \
968 #define MAYBE(ch, set_type) \
971 lexer_token.type = set_type; \
974 #define ELSE_CODE(code) \
978 } /* end of while(1) */ \
981 #define ELSE(set_type) \
983 lexer_token.type = set_type; \
987 void lexer_next_preprocessing_token(void)
997 lexer_token.type = '\n';
1003 /* might be a wide string ( L"string" ) */
1004 if(c == '"' && (lexer_token.type == T_IDENTIFIER &&
1005 lexer_token.v.symbol == symbol_L)) {
1006 parse_string_literal();
1016 parse_string_literal();
1020 parse_character_constant();
1027 MAYBE('.', T_DOTDOTDOT)
1031 lexer_token.type = '.';
1037 MAYBE('&', T_ANDAND)
1038 MAYBE('=', T_ANDEQUAL)
1042 MAYBE('=', T_ASTERISKEQUAL)
1046 MAYBE('+', T_PLUSPLUS)
1047 MAYBE('=', T_PLUSEQUAL)
1051 MAYBE('>', T_MINUSGREATER)
1052 MAYBE('-', T_MINUSMINUS)
1053 MAYBE('=', T_MINUSEQUAL)
1057 MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1061 MAYBE('=', T_SLASHEQUAL)
1064 skip_multiline_comment();
1065 lexer_next_preprocessing_token();
1069 skip_line_comment();
1070 lexer_next_preprocessing_token();
1075 MAYBE('>', T_PERCENTGREATER)
1076 MAYBE('=', T_PERCENTEQUAL)
1081 MAYBE(':', T_PERCENTCOLONPERCENTCOLON)
1085 lexer_token.type = T_PERCENTCOLON;
1088 ELSE(T_PERCENTCOLON)
1092 MAYBE(':', T_LESSCOLON)
1093 MAYBE('%', T_LESSPERCENT)
1094 MAYBE('=', T_LESSEQUAL)
1097 MAYBE('=', T_LESSLESSEQUAL)
1102 MAYBE('=', T_GREATEREQUAL)
1105 MAYBE('=', T_GREATERGREATEREQUAL)
1106 ELSE(T_GREATERGREATER)
1110 MAYBE('=', T_CARETEQUAL)
1114 MAYBE('=', T_PIPEEQUAL)
1115 MAYBE('|', T_PIPEPIPE)
1119 MAYBE('>', T_COLONGREATER)
1123 MAYBE('=', T_EQUALEQUAL)
1127 MAYBE('#', T_HASHHASH)
1141 lexer_token.type = c;
1146 lexer_token.type = T_EOF;
1152 fprintf(stderr, "unknown character '%c' found\n", c);
1153 lexer_token.type = T_ERROR;
1159 void lexer_next_token(void)
1161 lexer_next_preprocessing_token();
1162 if(lexer_token.type != '\n')
1167 lexer_next_preprocessing_token();
1168 } while(lexer_token.type == '\n');
1170 if(lexer_token.type == '#') {
1171 parse_preprocessor_directive();
1176 void init_lexer(void)
1178 strset_init(&stringset);
1180 type_int = make_atomic_type(ATOMIC_TYPE_INT, TYPE_QUALIFIER_NONE);
1181 type_uint = make_atomic_type(ATOMIC_TYPE_UINT, TYPE_QUALIFIER_NONE);
1182 type_long = make_atomic_type(ATOMIC_TYPE_LONG, TYPE_QUALIFIER_NONE);
1183 type_ulong = make_atomic_type(ATOMIC_TYPE_ULONG, TYPE_QUALIFIER_NONE);
1184 type_longlong = make_atomic_type(ATOMIC_TYPE_LONGLONG,
1185 TYPE_QUALIFIER_NONE);
1186 type_ulonglong = make_atomic_type(ATOMIC_TYPE_ULONGLONG,
1187 TYPE_QUALIFIER_NONE);
1189 type_float = make_atomic_type(ATOMIC_TYPE_FLOAT, TYPE_QUALIFIER_CONST);
1190 type_double = make_atomic_type(ATOMIC_TYPE_DOUBLE,
1191 TYPE_QUALIFIER_CONST);
1192 type_longdouble = make_atomic_type(ATOMIC_TYPE_LONG_DOUBLE,
1193 TYPE_QUALIFIER_CONST);
1196 void lexer_open_stream(FILE *stream, const char *input_name)
1199 lexer_token.source_position.linenr = 0;
1200 lexer_token.source_position.input_name = input_name;
1202 symbol_L = symbol_table_insert("L");
1204 /* place a virtual \n at the beginning so the lexer knows that we're
1205 * at the beginning of a line */
1209 void exit_lexer(void)
1211 strset_destroy(&stringset);
1214 static __attribute__((unused))
1215 void dbg_pos(const source_position_t source_position)
1217 fprintf(stdout, "%s:%u\n", source_position.input_name,
1218 source_position.linenr);