2 * This file is part of cparser.
3 * Copyright (C) 2007-2008 Matthias Braun <matze@braunis.de>
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version 2
8 * of the License, or (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
22 #include "diagnostic.h"
26 #include "symbol_table_t.h"
27 #include "adt/error.h"
28 #include "adt/strset.h"
32 #include "target_architecture.h"
35 #include "lang_features.h"
48 #if defined(_WIN32) || defined(__CYGWIN__)
49 /* No strtold on windows and no replacement yet */
50 #define strtold(s, e) strtod(s, e)
53 typedef unsigned int utf32;
59 static utf32 buf[BUF_SIZE + MAX_PUTBACK];
60 static const utf32 *bufend;
61 static const utf32 *bufpos;
62 static strset_t stringset;
63 bool allow_dollar_in_symbol = true;
66 * Prints a parse error message at the current token.
68 * @param msg the error message
70 static void parse_error(const char *msg)
72 errorf(&lexer_token.source_position, "%s", msg);
76 * Prints an internal error message at the current token.
78 * @param msg the error message
80 static NORETURN internal_error(const char *msg)
82 internal_errorf(&lexer_token.source_position, "%s", msg);
85 static size_t read_block(unsigned char *const read_buf, size_t const n)
87 size_t const s = fread(read_buf, 1, n, input);
90 parse_error("read from input failed");
91 buf[MAX_PUTBACK] = EOF;
92 bufpos = buf + MAX_PUTBACK;
93 bufend = buf + MAX_PUTBACK + 1;
98 static void decode_iso_8859_1(void)
100 unsigned char read_buf[BUF_SIZE];
101 size_t const s = read_block(read_buf, sizeof(read_buf));
105 unsigned char const *src = read_buf;
106 unsigned char const *end = read_buf + s;
107 utf32 *dst = buf + MAX_PUTBACK;
111 bufpos = buf + MAX_PUTBACK;
115 static void decode_iso_8859_15(void)
117 unsigned char read_buf[BUF_SIZE];
118 size_t const s = read_block(read_buf, sizeof(read_buf));
122 unsigned char const *src = read_buf;
123 unsigned char const *end = read_buf + s;
124 utf32 *dst = buf + MAX_PUTBACK;
128 case 0xA4: tc = 0x20AC; break; // €
129 case 0xA6: tc = 0x0160; break; // Š
130 case 0xA8: tc = 0x0161; break; // š
131 case 0xB4: tc = 0x017D; break; // Ž
132 case 0xB8: tc = 0x017E; break; // ž
133 case 0xBC: tc = 0x0152; break; // Œ
134 case 0xBD: tc = 0x0153; break; // œ
135 case 0xBE: tc = 0x0178; break; // Ÿ
140 bufpos = buf + MAX_PUTBACK;
144 static void decode_utf8(void)
146 static utf32 part_decoded_min_code;
147 static utf32 part_decoded_char;
148 static size_t part_decoded_rest_len;
151 unsigned char read_buf[BUF_SIZE];
152 size_t const s = read_block(read_buf, sizeof(read_buf));
154 if (part_decoded_rest_len > 0)
155 parse_error("incomplete input char at end of input");
159 unsigned char const *src = read_buf;
160 unsigned char const *end = read_buf + s;
161 utf32 *dst = buf + MAX_PUTBACK;
165 if (part_decoded_rest_len != 0) {
166 min_code = part_decoded_min_code;
167 decoded = part_decoded_char;
168 size_t const rest_len = part_decoded_rest_len;
169 part_decoded_rest_len = 0;
171 case 4: goto realign;
172 case 3: goto three_more;
173 case 2: goto two_more;
174 default: goto one_more;
179 if ((*src & 0x80) == 0) {
181 } else if ((*src & 0xE0) == 0xC0) {
183 decoded = *src++ & 0x1F;
186 part_decoded_min_code = min_code;
187 part_decoded_char = decoded;
188 part_decoded_rest_len = 1;
191 if ((*src & 0xC0) == 0x80) {
192 decoded = (decoded << 6) | (*src++ & 0x3F);
196 if (decoded < min_code ||
197 decoded > 0x10FFFF ||
198 (0xD800 <= decoded && decoded < 0xE000) || // high/low surrogates
199 (0xFDD0 <= decoded && decoded < 0xFDF0) || // noncharacters
200 (decoded & 0xFFFE) == 0xFFFE) { // noncharacters
201 parse_error("invalid byte sequence in input");
203 } else if ((*src & 0xF0) == 0xE0) {
205 decoded = *src++ & 0x0F;
208 part_decoded_min_code = min_code;
209 part_decoded_char = decoded;
210 part_decoded_rest_len = 2;
213 if ((*src & 0xC0) == 0x80) {
214 decoded = (decoded << 6) | (*src++ & 0x3F);
219 } else if ((*src & 0xF8) == 0xF0) {
221 decoded = *src++ & 0x07;
224 part_decoded_min_code = min_code;
225 part_decoded_char = decoded;
226 part_decoded_rest_len = 3;
229 if ((*src & 0xC0) == 0x80) {
230 decoded = (decoded << 6) | (*src++ & 0x3F);
237 parse_error("invalid byte sequence in input");
242 part_decoded_rest_len = 4;
245 } while ((*src & 0xC0) == 0x80 || (*src & 0xF8) == 0xF8);
251 bufpos = buf + MAX_PUTBACK;
253 } while (bufpos == bufend);
256 typedef void (*decoder_t)(void);
258 static decoder_t decoder = decode_utf8;
260 typedef struct named_decoder_t {
265 static named_decoder_t const decoders[] = {
266 { "CP819", decode_iso_8859_1 }, // offical alias
267 { "IBM819", decode_iso_8859_1 }, // offical alias
268 { "ISO-8859-1", decode_iso_8859_1 }, // offical alias
269 { "ISO-8859-15", decode_iso_8859_15 }, // offical name
270 { "ISO8859-1", decode_iso_8859_1 },
271 { "ISO8859-15", decode_iso_8859_15 },
272 { "ISO_8859-1", decode_iso_8859_1 }, // offical alias
273 { "ISO_8859-15", decode_iso_8859_15 }, // offical alias
274 { "ISO_8859-1:1987", decode_iso_8859_1 }, // offical name
275 { "Latin-9", decode_iso_8859_15 }, // offical alias
276 { "UTF-8", decode_utf8 }, // offical name
277 { "csISOLatin1", decode_iso_8859_1 }, // offical alias
278 { "iso-ir-100", decode_iso_8859_1 }, // offical alias
279 { "l1", decode_iso_8859_1 }, // offical alias
280 { "latin1", decode_iso_8859_1 }, // offical alias
285 void select_input_encoding(char const* const encoding)
287 for (named_decoder_t const *i = decoders; i->name != NULL; ++i) {
288 if (strcasecmp(encoding, i->name) != 0)
290 decoder = i->decoder;
293 fprintf(stderr, "error: input encoding \"%s\" not supported\n", encoding);
296 static inline void next_real_char(void)
298 assert(bufpos <= bufend);
299 if (bufpos >= bufend) {
310 * Put a character back into the buffer.
312 * @param pc the character to put back
314 static inline void put_back(utf32 const pc)
316 assert(bufpos > buf);
317 *(--bufpos - buf + buf) = pc;
320 printf("putback '%lc'\n", pc);
324 static inline void next_char(void);
326 #define MATCH_NEWLINE(code) \
332 lexer_token.source_position.linenr++; \
336 lexer_token.source_position.linenr++; \
339 #define eat(c_type) do { assert(c == c_type); next_char(); } while(0)
341 static void maybe_concat_lines(void)
346 MATCH_NEWLINE(return;)
357 * Set c to the next input character, ie.
358 * after expanding trigraphs.
360 static inline void next_char(void)
364 /* filter trigraphs */
365 if(UNLIKELY(c == '\\')) {
366 maybe_concat_lines();
367 goto end_of_next_char;
371 goto end_of_next_char;
374 if(LIKELY(c != '?')) {
377 goto end_of_next_char;
382 case '=': c = '#'; break;
383 case '(': c = '['; break;
384 case '/': c = '\\'; maybe_concat_lines(); break;
385 case ')': c = ']'; break;
386 case '\'': c = '^'; break;
387 case '<': c = '{'; break;
388 case '!': c = '|'; break;
389 case '>': c = '}'; break;
390 case '-': c = '~'; break;
400 printf("nchar '%c'\n", c);
404 #define SYMBOL_CHARS \
405 case '$': if (!allow_dollar_in_symbol) goto dollar_sign; \
473 * Read a symbol from the input and build
476 static void parse_symbol(void)
481 obstack_1grow(&symbol_obstack, (char) c);
488 obstack_1grow(&symbol_obstack, (char) c);
499 obstack_1grow(&symbol_obstack, '\0');
501 string = obstack_finish(&symbol_obstack);
502 symbol = symbol_table_insert(string);
504 lexer_token.type = symbol->ID;
505 lexer_token.v.symbol = symbol;
507 if(symbol->string != string) {
508 obstack_free(&symbol_obstack, string);
512 static void parse_integer_suffix(bool is_oct_hex)
514 bool is_unsigned = false;
515 bool min_long = false;
516 bool min_longlong = false;
517 bool not_traditional = false;
521 if (c == 'U' || c == 'u') {
522 not_traditional = true;
523 suffix[pos++] = toupper(c);
526 if (c == 'L' || c == 'l') {
527 suffix[pos++] = toupper(c);
530 if (c == 'L' || c == 'l') {
531 suffix[pos++] = toupper(c);
536 } else if (c == 'l' || c == 'L') {
537 suffix[pos++] = toupper(c);
540 if (c == 'l' || c == 'L') {
541 not_traditional = true;
542 suffix[pos++] = toupper(c);
545 if (c == 'u' || c == 'U') {
546 suffix[pos++] = toupper(c);
550 } else if (c == 'u' || c == 'U') {
551 not_traditional = true;
552 suffix[pos++] = toupper(c);
555 lexer_token.datatype = type_unsigned_long;
559 if (warning.traditional && not_traditional) {
561 warningf(&lexer_token.source_position,
562 "traditional C rejects the '%s' suffix", suffix);
565 long long v = lexer_token.v.intvalue;
567 if (v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
568 lexer_token.datatype = type_int;
570 } else if (is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
571 lexer_token.datatype = type_unsigned_int;
576 if (v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
577 lexer_token.datatype = type_long;
579 } else if (is_oct_hex && v >= 0 && (unsigned long long)v <= (unsigned long long)TARGET_ULONG_MAX) {
580 lexer_token.datatype = type_unsigned_long;
584 unsigned long long uv = (unsigned long long) v;
585 if (is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
586 lexer_token.datatype = type_unsigned_long_long;
590 lexer_token.datatype = type_long_long;
592 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
593 if (!min_long && v <= TARGET_UINT_MAX) {
594 lexer_token.datatype = type_unsigned_int;
597 if (!min_longlong && v <= TARGET_ULONG_MAX) {
598 lexer_token.datatype = type_unsigned_long;
601 lexer_token.datatype = type_unsigned_long_long;
605 static void parse_floating_suffix(void)
608 /* TODO: do something useful with the suffixes... */
611 if (warning.traditional) {
612 warningf(&lexer_token.source_position,
613 "traditional C rejects the 'F' suffix");
616 lexer_token.datatype = type_float;
620 if (warning.traditional) {
621 warningf(&lexer_token.source_position,
622 "traditional C rejects the 'F' suffix");
625 lexer_token.datatype = type_long_double;
628 lexer_token.datatype = type_double;
634 * A replacement for strtoull. Only those parts needed for
635 * our parser are implemented.
637 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
638 unsigned long long v = 0;
643 /* check for overrun */
644 if (v >= 0x1000000000000000ULL)
646 switch (tolower(*s)) {
647 case '0': v <<= 4; break;
648 case '1': v <<= 4; v |= 0x1; break;
649 case '2': v <<= 4; v |= 0x2; break;
650 case '3': v <<= 4; v |= 0x3; break;
651 case '4': v <<= 4; v |= 0x4; break;
652 case '5': v <<= 4; v |= 0x5; break;
653 case '6': v <<= 4; v |= 0x6; break;
654 case '7': v <<= 4; v |= 0x7; break;
655 case '8': v <<= 4; v |= 0x8; break;
656 case '9': v <<= 4; v |= 0x9; break;
657 case 'a': v <<= 4; v |= 0xa; break;
658 case 'b': v <<= 4; v |= 0xb; break;
659 case 'c': v <<= 4; v |= 0xc; break;
660 case 'd': v <<= 4; v |= 0xd; break;
661 case 'e': v <<= 4; v |= 0xe; break;
662 case 'f': v <<= 4; v |= 0xf; break;
670 /* check for overrun */
671 if (v >= 0x2000000000000000ULL)
673 switch (tolower(*s)) {
674 case '0': v <<= 3; break;
675 case '1': v <<= 3; v |= 1; break;
676 case '2': v <<= 3; v |= 2; break;
677 case '3': v <<= 3; v |= 3; break;
678 case '4': v <<= 3; v |= 4; break;
679 case '5': v <<= 3; v |= 5; break;
680 case '6': v <<= 3; v |= 6; break;
681 case '7': v <<= 3; v |= 7; break;
689 /* check for overrun */
690 if (v > 0x1999999999999999ULL)
692 switch (tolower(*s)) {
693 case '0': v *= 10; break;
694 case '1': v *= 10; v += 1; break;
695 case '2': v *= 10; v += 2; break;
696 case '3': v *= 10; v += 3; break;
697 case '4': v *= 10; v += 4; break;
698 case '5': v *= 10; v += 5; break;
699 case '6': v *= 10; v += 6; break;
700 case '7': v *= 10; v += 7; break;
701 case '8': v *= 10; v += 8; break;
702 case '9': v *= 10; v += 9; break;
718 * Parses a hex number including hex floats and set the
721 static void parse_number_hex(void)
723 bool is_float = false;
724 assert(c == 'x' || c == 'X');
727 obstack_1grow(&symbol_obstack, '0');
728 obstack_1grow(&symbol_obstack, 'x');
731 obstack_1grow(&symbol_obstack, (char) c);
736 obstack_1grow(&symbol_obstack, (char) c);
739 while (isxdigit(c)) {
740 obstack_1grow(&symbol_obstack, (char) c);
745 if (c == 'p' || c == 'P') {
746 obstack_1grow(&symbol_obstack, (char) c);
749 if (c == '-' || c == '+') {
750 obstack_1grow(&symbol_obstack, (char) c);
754 while (isxdigit(c)) {
755 obstack_1grow(&symbol_obstack, (char) c);
761 obstack_1grow(&symbol_obstack, '\0');
762 char *string = obstack_finish(&symbol_obstack);
763 if(*string == '\0') {
764 parse_error("invalid hex number");
765 lexer_token.type = T_ERROR;
766 obstack_free(&symbol_obstack, string);
772 lexer_token.type = T_FLOATINGPOINT;
773 lexer_token.v.floatvalue = strtold(string, &endptr);
775 if(*endptr != '\0') {
776 parse_error("invalid hex float literal");
779 parse_floating_suffix();
782 lexer_token.type = T_INTEGER;
783 lexer_token.v.intvalue = parse_int_string(string + 2, &endptr, 16);
784 if(*endptr != '\0') {
785 parse_error("hex number literal too long");
787 parse_integer_suffix(true);
790 obstack_free(&symbol_obstack, string);
794 * Returns true if the given char is a octal digit.
796 * @param char the character to check
798 static inline bool is_octal_digit(utf32 chr)
816 * Parses a octal number and set the lexer_token.
818 static void parse_number_oct(void)
820 while(is_octal_digit(c)) {
821 obstack_1grow(&symbol_obstack, (char) c);
824 obstack_1grow(&symbol_obstack, '\0');
825 char *string = obstack_finish(&symbol_obstack);
828 lexer_token.type = T_INTEGER;
829 lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
830 if(*endptr != '\0') {
831 parse_error("octal number literal too long");
834 obstack_free(&symbol_obstack, string);
835 parse_integer_suffix(true);
839 * Parses a decimal including float number and set the
842 static void parse_number_dec(void)
844 bool is_float = false;
846 obstack_1grow(&symbol_obstack, (char) c);
851 obstack_1grow(&symbol_obstack, '.');
855 obstack_1grow(&symbol_obstack, (char) c);
860 if(c == 'e' || c == 'E') {
861 obstack_1grow(&symbol_obstack, (char) c);
864 if(c == '-' || c == '+') {
865 obstack_1grow(&symbol_obstack, (char) c);
870 obstack_1grow(&symbol_obstack, (char) c);
876 obstack_1grow(&symbol_obstack, '\0');
877 char *string = obstack_finish(&symbol_obstack);
881 lexer_token.type = T_FLOATINGPOINT;
882 lexer_token.v.floatvalue = strtold(string, &endptr);
884 if(*endptr != '\0') {
885 parse_error("invalid number literal");
888 parse_floating_suffix();
891 lexer_token.type = T_INTEGER;
892 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
894 if(*endptr != '\0') {
895 parse_error("invalid number literal");
898 parse_integer_suffix(false);
900 obstack_free(&symbol_obstack, string);
904 * Parses a number and sets the lexer_token.
906 static void parse_number(void)
928 parse_error("invalid octal number");
929 lexer_token.type = T_ERROR;
935 obstack_1grow(&symbol_obstack, '0');
945 * Returns the value of a digit.
946 * The only portable way to do it ...
948 static int digit_value(utf32 const digit)
974 internal_error("wrong character given");
979 * Parses an octal character sequence.
981 * @param first_digit the already read first digit
983 static utf32 parse_octal_sequence(utf32 const first_digit)
985 assert(is_octal_digit(first_digit));
986 utf32 value = digit_value(first_digit);
987 if (!is_octal_digit(c)) return value;
988 value = 8 * value + digit_value(c);
990 if (!is_octal_digit(c)) return value;
991 value = 8 * value + digit_value(c);
997 * Parses a hex character sequence.
999 static utf32 parse_hex_sequence(void)
1002 while(isxdigit(c)) {
1003 value = 16 * value + digit_value(c);
1010 * Parse an escape sequence.
1012 static utf32 parse_escape_sequence(void)
1020 case '"': return '"';
1021 case '\'': return '\'';
1022 case '\\': return '\\';
1023 case '?': return '\?';
1024 case 'a': return '\a';
1025 case 'b': return '\b';
1026 case 'f': return '\f';
1027 case 'n': return '\n';
1028 case 'r': return '\r';
1029 case 't': return '\t';
1030 case 'v': return '\v';
1032 return parse_hex_sequence();
1041 return parse_octal_sequence(ec);
1043 parse_error("reached end of file while parsing escape sequence");
1045 /* \E is not documented, but handled, by GCC. It is acceptable according
1046 * to §6.11.4, whereas \e is not. */
1050 return 27; /* hopefully 27 is ALWAYS the code for ESCAPE */
1053 /* §6.4.4.4:8 footnote 64 */
1054 parse_error("unknown escape sequence");
1060 * Concatenate two strings.
1062 string_t concat_strings(const string_t *const s1, const string_t *const s2)
1064 const size_t len1 = s1->size - 1;
1065 const size_t len2 = s2->size - 1;
1067 char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
1068 memcpy(concat, s1->begin, len1);
1069 memcpy(concat + len1, s2->begin, len2 + 1);
1071 if (warning.traditional) {
1072 warningf(&lexer_token.source_position,
1073 "traditional C rejects string constant concatenation");
1075 #if 0 /* TODO hash */
1076 const char *result = strset_insert(&stringset, concat);
1077 if(result != concat) {
1078 obstack_free(&symbol_obstack, concat);
1083 return (string_t){ concat, len1 + len2 + 1 };
1088 * Concatenate a string and a wide string.
1090 wide_string_t concat_string_wide_string(const string_t *const s1, const wide_string_t *const s2)
1092 const size_t len1 = s1->size - 1;
1093 const size_t len2 = s2->size - 1;
1095 wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
1096 const char *const src = s1->begin;
1097 for (size_t i = 0; i != len1; ++i) {
1100 memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
1101 if (warning.traditional) {
1102 warningf(&lexer_token.source_position,
1103 "traditional C rejects string constant concatenation");
1106 return (wide_string_t){ concat, len1 + len2 + 1 };
1110 * Concatenate two wide strings.
1112 wide_string_t concat_wide_strings(const wide_string_t *const s1, const wide_string_t *const s2)
1114 const size_t len1 = s1->size - 1;
1115 const size_t len2 = s2->size - 1;
1117 wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
1118 memcpy(concat, s1->begin, len1 * sizeof(*concat));
1119 memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
1120 if (warning.traditional) {
1121 warningf(&lexer_token.source_position,
1122 "traditional C rejects string constant concatenation");
1125 return (wide_string_t){ concat, len1 + len2 + 1 };
1129 * Concatenate a wide string and a string.
1131 wide_string_t concat_wide_string_string(const wide_string_t *const s1, const string_t *const s2)
1133 const size_t len1 = s1->size - 1;
1134 const size_t len2 = s2->size - 1;
1136 wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
1137 memcpy(concat, s1->begin, len1 * sizeof(*concat));
1138 const char *const src = s2->begin;
1139 wchar_rep_t *const dst = concat + len1;
1140 for (size_t i = 0; i != len2 + 1; ++i) {
1143 if (warning.traditional) {
1144 warningf(&lexer_token.source_position,
1145 "traditional C rejects string constant concatenation");
1148 return (wide_string_t){ concat, len1 + len2 + 1 };
1151 static void grow_symbol(utf32 const tc)
1153 struct obstack *const o = &symbol_obstack;
1155 obstack_1grow(o, tc);
1156 } else if (tc < 0x800) {
1157 obstack_1grow(o, 0xC0 | (tc >> 6));
1158 obstack_1grow(o, 0x80 | (tc & 0x3F));
1159 } else if (tc < 0x10000) {
1160 obstack_1grow(o, 0xE0 | ( tc >> 12));
1161 obstack_1grow(o, 0x80 | ((tc >> 6) & 0x3F));
1162 obstack_1grow(o, 0x80 | ( tc & 0x3F));
1164 obstack_1grow(o, 0xF0 | ( tc >> 18));
1165 obstack_1grow(o, 0x80 | ((tc >> 12) & 0x3F));
1166 obstack_1grow(o, 0x80 | ((tc >> 6) & 0x3F));
1167 obstack_1grow(o, 0x80 | ( tc & 0x3F));
1172 * Parse a string literal and set lexer_token.
1174 static void parse_string_literal(void)
1176 const unsigned start_linenr = lexer_token.source_position.linenr;
1183 utf32 const tc = parse_escape_sequence();
1185 warningf(&lexer_token.source_position,
1186 "escape sequence out of range");
1188 obstack_1grow(&symbol_obstack, tc);
1193 source_position_t source_position;
1194 source_position.input_name = lexer_token.source_position.input_name;
1195 source_position.linenr = start_linenr;
1196 errorf(&source_position, "string has no end");
1197 lexer_token.type = T_ERROR;
1214 /* TODO: concatenate multiple strings separated by whitespace... */
1216 /* add finishing 0 to the string */
1217 obstack_1grow(&symbol_obstack, '\0');
1218 const size_t size = (size_t)obstack_object_size(&symbol_obstack);
1219 const char *const string = obstack_finish(&symbol_obstack);
1221 #if 0 /* TODO hash */
1222 /* check if there is already a copy of the string */
1223 result = strset_insert(&stringset, string);
1224 if(result != string) {
1225 obstack_free(&symbol_obstack, string);
1228 const char *const result = string;
1231 lexer_token.type = T_STRING_LITERAL;
1232 lexer_token.v.string.begin = result;
1233 lexer_token.v.string.size = size;
1237 * Parse a wide character constant and set lexer_token.
1239 static void parse_wide_character_constant(void)
1241 const unsigned start_linenr = lexer_token.source_position.linenr;
1248 wchar_rep_t tc = parse_escape_sequence();
1249 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1254 parse_error("newline while parsing character constant");
1260 goto end_of_wide_char_constant;
1263 source_position_t source_position = lexer_token.source_position;
1264 source_position.linenr = start_linenr;
1265 errorf(&source_position, "EOF while parsing character constant");
1266 lexer_token.type = T_ERROR;
1271 wchar_rep_t tc = (wchar_rep_t) c;
1272 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1279 end_of_wide_char_constant:;
1280 size_t size = (size_t) obstack_object_size(&symbol_obstack);
1281 assert(size % sizeof(wchar_rep_t) == 0);
1282 size /= sizeof(wchar_rep_t);
1284 const wchar_rep_t *string = obstack_finish(&symbol_obstack);
1286 lexer_token.type = T_WIDE_CHARACTER_CONSTANT;
1287 lexer_token.v.wide_string.begin = string;
1288 lexer_token.v.wide_string.size = size;
1289 lexer_token.datatype = type_wchar_t;
1293 * Parse a wide string literal and set lexer_token.
1295 static void parse_wide_string_literal(void)
1297 const unsigned start_linenr = lexer_token.source_position.linenr;
1305 wchar_rep_t tc = parse_escape_sequence();
1306 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1311 source_position_t source_position;
1312 source_position.input_name = lexer_token.source_position.input_name;
1313 source_position.linenr = start_linenr;
1314 errorf(&source_position, "string has no end");
1315 lexer_token.type = T_ERROR;
1325 obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1334 /* TODO: concatenate multiple strings separated by whitespace... */
1336 /* add finishing 0 to the string */
1337 wchar_rep_t nul = L'\0';
1338 obstack_grow(&symbol_obstack, &nul, sizeof(nul));
1339 const size_t size = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
1340 const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
1342 #if 0 /* TODO hash */
1343 /* check if there is already a copy of the string */
1344 const wchar_rep_t *const result = strset_insert(&stringset, string);
1345 if(result != string) {
1346 obstack_free(&symbol_obstack, string);
1349 const wchar_rep_t *const result = string;
1352 lexer_token.type = T_WIDE_STRING_LITERAL;
1353 lexer_token.v.wide_string.begin = result;
1354 lexer_token.v.wide_string.size = size;
1358 * Parse a character constant and set lexer_token.
1360 static void parse_character_constant(void)
1362 const unsigned start_linenr = lexer_token.source_position.linenr;
1369 utf32 const tc = parse_escape_sequence();
1371 warningf(&lexer_token.source_position,
1372 "escape sequence out of range");
1374 obstack_1grow(&symbol_obstack, tc);
1379 parse_error("newline while parsing character constant");
1385 goto end_of_char_constant;
1388 source_position_t source_position;
1389 source_position.input_name = lexer_token.source_position.input_name;
1390 source_position.linenr = start_linenr;
1391 errorf(&source_position, "EOF while parsing character constant");
1392 lexer_token.type = T_ERROR;
1404 end_of_char_constant:;
1405 const size_t size = (size_t)obstack_object_size(&symbol_obstack);
1406 const char *const string = obstack_finish(&symbol_obstack);
1408 lexer_token.type = T_CHARACTER_CONSTANT;
1409 lexer_token.v.string.begin = string;
1410 lexer_token.v.string.size = size;
1411 lexer_token.datatype = c_mode & _CXX && size == 1 ? type_char : type_int;
1415 * Skip a multiline comment.
1417 static void skip_multiline_comment(void)
1419 unsigned start_linenr = lexer_token.source_position.linenr;
1426 /* nested comment, warn here */
1427 if (warning.comment) {
1428 warningf(&lexer_token.source_position, "'/*' within comment");
1440 MATCH_NEWLINE(break;)
1443 source_position_t source_position;
1444 source_position.input_name = lexer_token.source_position.input_name;
1445 source_position.linenr = start_linenr;
1446 errorf(&source_position, "at end of file while looking for comment end");
1458 * Skip a single line comment.
1460 static void skip_line_comment(void)
1473 if (c == '\n' || c == '\r') {
1474 if (warning.comment)
1475 warningf(&lexer_token.source_position, "multi-line comment");
1487 /** The current preprocessor token. */
1488 static token_t pp_token;
1491 * Read the next preprocessor token.
1493 static inline void next_pp_token(void)
1495 lexer_next_preprocessing_token();
1496 pp_token = lexer_token;
1500 * Eat all preprocessor tokens until newline.
1502 static void eat_until_newline(void)
1504 while(pp_token.type != '\n' && pp_token.type != T_EOF) {
1510 * Handle the define directive.
1512 static void define_directive(void)
1514 lexer_next_preprocessing_token();
1515 if(lexer_token.type != T_IDENTIFIER) {
1516 parse_error("expected identifier after #define\n");
1517 eat_until_newline();
1522 * Handle the ifdef directive.
1524 static void ifdef_directive(int is_ifndef)
1527 lexer_next_preprocessing_token();
1528 //expect_identifier();
1533 * Handle the endif directive.
1535 static void endif_directive(void)
1541 * Parse the line directive.
1543 static void parse_line_directive(void)
1545 if(pp_token.type != T_INTEGER) {
1546 parse_error("expected integer");
1548 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
1551 if(pp_token.type == T_STRING_LITERAL) {
1552 lexer_token.source_position.input_name = pp_token.v.string.begin;
1556 eat_until_newline();
1562 typedef enum stdc_pragma_kind_t {
1566 STDC_CX_LIMITED_RANGE
1567 } stdc_pragma_kind_t;
1570 * STDC pragma values.
1572 typedef enum stdc_pragma_value_kind_t {
1577 } stdc_pragma_value_kind_t;
1580 * Parse a pragma directive.
1582 static void parse_pragma(void) {
1583 bool unknown_pragma = true;
1586 if (pp_token.v.symbol->pp_ID == TP_STDC) {
1587 stdc_pragma_kind_t kind = STDC_UNKNOWN;
1589 if (c_mode & _C99) {
1592 switch (pp_token.v.symbol->pp_ID) {
1593 case TP_FP_CONTRACT:
1594 kind = STDC_FP_CONTRACT;
1596 case TP_FENV_ACCESS:
1597 kind = STDC_FENV_ACCESS;
1599 case TP_CX_LIMITED_RANGE:
1600 kind = STDC_CX_LIMITED_RANGE;
1605 if (kind != STDC_UNKNOWN) {
1606 stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN;
1608 switch (pp_token.v.symbol->pp_ID) {
1610 value = STDC_VALUE_ON;
1613 value = STDC_VALUE_OFF;
1616 value = STDC_VALUE_DEFAULT;
1621 if (value != STDC_VALUE_UNKNOWN) {
1622 unknown_pragma = false;
1624 errorf(&pp_token.source_position, "bad STDC pragma argument");
1629 unknown_pragma = true;
1631 eat_until_newline();
1632 if (unknown_pragma && warning.unknown_pragmas) {
1633 warningf(&pp_token.source_position, "encountered unknown #pragma");
1638 * Parse a preprocessor non-null directive.
1640 static void parse_preprocessor_identifier(void)
1642 assert(pp_token.type == T_IDENTIFIER);
1643 symbol_t *symbol = pp_token.v.symbol;
1645 switch(symbol->pp_ID) {
1647 printf("include - enable header name parsing!\n");
1663 parse_line_directive();
1670 /* TODO; output the rest of the line */
1671 parse_error("#error directive: ");
1680 * Parse a preprocessor directive.
1682 static void parse_preprocessor_directive(void)
1686 switch(pp_token.type) {
1688 parse_preprocessor_identifier();
1691 parse_line_directive();
1694 /* NULL directive, see § 6.10.7 */
1697 parse_error("invalid preprocessor directive");
1698 eat_until_newline();
1703 #define MAYBE_PROLOG \
1708 #define MAYBE(ch, set_type) \
1711 lexer_token.type = set_type; \
1714 #define ELSE_CODE(code) \
1718 } /* end of while(1) */ \
1721 #define ELSE(set_type) \
1723 lexer_token.type = set_type; \
1727 void lexer_next_preprocessing_token(void)
1737 lexer_token.type = '\n';
1743 /* might be a wide string ( L"string" ) */
1744 if(lexer_token.type == T_IDENTIFIER &&
1745 lexer_token.v.symbol == symbol_L) {
1747 parse_wide_string_literal();
1748 } else if(c == '\'') {
1749 parse_wide_character_constant();
1759 parse_string_literal();
1763 parse_character_constant();
1776 MAYBE('.', T_DOTDOTDOT)
1780 lexer_token.type = '.';
1786 MAYBE('&', T_ANDAND)
1787 MAYBE('=', T_ANDEQUAL)
1791 MAYBE('=', T_ASTERISKEQUAL)
1795 MAYBE('+', T_PLUSPLUS)
1796 MAYBE('=', T_PLUSEQUAL)
1800 MAYBE('>', T_MINUSGREATER)
1801 MAYBE('-', T_MINUSMINUS)
1802 MAYBE('=', T_MINUSEQUAL)
1806 MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1810 MAYBE('=', T_SLASHEQUAL)
1813 skip_multiline_comment();
1814 lexer_next_preprocessing_token();
1818 skip_line_comment();
1819 lexer_next_preprocessing_token();
1825 MAYBE('=', T_PERCENTEQUAL)
1830 MAYBE(':', T_HASHHASH)
1834 lexer_token.type = '#';
1843 MAYBE('=', T_LESSEQUAL)
1846 MAYBE('=', T_LESSLESSEQUAL)
1851 MAYBE('=', T_GREATEREQUAL)
1854 MAYBE('=', T_GREATERGREATEREQUAL)
1855 ELSE(T_GREATERGREATER)
1859 MAYBE('=', T_CARETEQUAL)
1863 MAYBE('=', T_PIPEEQUAL)
1864 MAYBE('|', T_PIPEPIPE)
1872 MAYBE('=', T_EQUALEQUAL)
1876 MAYBE('#', T_HASHHASH)
1890 lexer_token.type = c;
1895 lexer_token.type = T_EOF;
1900 errorf(&lexer_token.source_position, "unknown character '%c' found", c);
1902 lexer_token.type = T_ERROR;
1908 void lexer_next_token(void)
1910 lexer_next_preprocessing_token();
1912 while (lexer_token.type == '\n') {
1914 lexer_next_preprocessing_token();
1917 if (lexer_token.type == '#') {
1918 parse_preprocessor_directive();
1923 void init_lexer(void)
1925 strset_init(&stringset);
1926 symbol_L = symbol_table_insert("L");
1929 void lexer_open_stream(FILE *stream, const char *input_name)
1932 lexer_token.source_position.linenr = 0;
1933 lexer_token.source_position.input_name = input_name;
1938 /* place a virtual \n at the beginning so the lexer knows that we're
1939 * at the beginning of a line */
1943 void lexer_open_buffer(const char *buffer, size_t len, const char *input_name)
1946 lexer_token.source_position.linenr = 0;
1947 lexer_token.source_position.input_name = input_name;
1951 bufend = buffer + len;
1955 panic("builtin lexing not done yet");
1958 /* place a virtual \n at the beginning so the lexer knows that we're
1959 * at the beginning of a line */
1963 void exit_lexer(void)
1965 strset_destroy(&stringset);
1968 static __attribute__((unused))
1969 void dbg_pos(const source_position_t source_position)
1971 fprintf(stdout, "%s:%u\n", source_position.input_name,
1972 source_position.linenr);