fix test
[cparser] / lexer.c
1 /*
2  * This file is part of cparser.
3  * Copyright (C) 2007-2008 Matthias Braun <matze@braunis.de>
4  *
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU General Public License
7  * as published by the Free Software Foundation; either version 2
8  * of the License, or (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
18  * 02111-1307, USA.
19  */
20 #include <config.h>
21
22 #include "diagnostic.h"
23 #include "lexer.h"
24 #include "symbol_t.h"
25 #include "token_t.h"
26 #include "symbol_table_t.h"
27 #include "adt/error.h"
28 #include "adt/strset.h"
29 #include "adt/util.h"
30 #include "types.h"
31 #include "type_t.h"
32 #include "target_architecture.h"
33 #include "parser.h"
34 #include "warning.h"
35 #include "lang_features.h"
36
37 #include <assert.h>
38 #include <errno.h>
39 #include <string.h>
40 #include <stdbool.h>
41 #include <ctype.h>
42
43 //#define DEBUG_CHARS
44 #define MAX_PUTBACK 3
45
46 #ifdef _WIN32
47 /* No strtold on windows and no replacement yet */
48 #define strtold(s, e) strtod(s, e)
49 #endif
50
51 static int         c;
52 token_t            lexer_token;
53 symbol_t          *symbol_L;
54 static FILE       *input;
55 static char        buf[1024 + MAX_PUTBACK];
56 static const char *bufend;
57 static const char *bufpos;
58 static strset_t    stringset;
59
60 /**
61  * Prints a parse error message at the current token.
62  *
63  * @param msg   the error message
64  */
65 static void parse_error(const char *msg)
66 {
67         errorf(lexer_token.source_position,  "%s", msg);
68 }
69
70 /**
71  * Prints an internal error message at the current token.
72  *
73  * @param msg   the error message
74  */
75 static NORETURN internal_error(const char *msg)
76 {
77         internal_errorf(lexer_token.source_position,  "%s", msg);
78 }
79
80 static inline void next_real_char(void)
81 {
82         assert(bufpos <= bufend);
83         if (bufpos >= bufend) {
84                 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
85                                  input);
86                 if(s == 0) {
87                         c = EOF;
88                         return;
89                 }
90                 bufpos = buf + MAX_PUTBACK;
91                 bufend = buf + MAX_PUTBACK + s;
92         }
93         c = *bufpos++;
94 }
95
96 /**
97  * Put a character back into the buffer.
98  *
99  * @param pc  the character to put back
100  */
101 static inline void put_back(int pc)
102 {
103         assert(bufpos > buf);
104         *(--bufpos - buf + buf) = (char) pc;
105
106 #ifdef DEBUG_CHARS
107         printf("putback '%c'\n", pc);
108 #endif
109 }
110
111 static inline void next_char(void);
112
113 #define MATCH_NEWLINE(code)                   \
114         case '\r':                                \
115                 next_char();                          \
116                 if(c == '\n') {                       \
117                         next_char();                      \
118                 }                                     \
119                 lexer_token.source_position.linenr++; \
120                 code                                  \
121         case '\n':                                \
122                 next_char();                          \
123                 lexer_token.source_position.linenr++; \
124                 code
125
126 #define eat(c_type)  do { assert(c == c_type); next_char(); } while(0)
127
128 static void maybe_concat_lines(void)
129 {
130         eat('\\');
131
132         switch(c) {
133         MATCH_NEWLINE(return;)
134
135         default:
136                 break;
137         }
138
139         put_back(c);
140         c = '\\';
141 }
142
143 /**
144  * Set c to the next input character, ie.
145  * after expanding trigraphs.
146  */
147 static inline void next_char(void)
148 {
149         next_real_char();
150
151         /* filter trigraphs */
152         if(UNLIKELY(c == '\\')) {
153                 maybe_concat_lines();
154                 goto end_of_next_char;
155         }
156
157         if(LIKELY(c != '?'))
158                 goto end_of_next_char;
159
160         next_real_char();
161         if(LIKELY(c != '?')) {
162                 put_back(c);
163                 c = '?';
164                 goto end_of_next_char;
165         }
166
167         next_real_char();
168         switch(c) {
169         case '=': c = '#'; break;
170         case '(': c = '['; break;
171         case '/': c = '\\'; maybe_concat_lines(); break;
172         case ')': c = ']'; break;
173         case '\'': c = '^'; break;
174         case '<': c = '{'; break;
175         case '!': c = '|'; break;
176         case '>': c = '}'; break;
177         case '-': c = '~'; break;
178         default:
179                 put_back(c);
180                 put_back('?');
181                 c = '?';
182                 break;
183         }
184
185 end_of_next_char:;
186 #ifdef DEBUG_CHARS
187         printf("nchar '%c'\n", c);
188 #endif
189 }
190
191 #define SYMBOL_CHARS  \
192         case 'a':         \
193         case 'b':         \
194         case 'c':         \
195         case 'd':         \
196         case 'e':         \
197         case 'f':         \
198         case 'g':         \
199         case 'h':         \
200         case 'i':         \
201         case 'j':         \
202         case 'k':         \
203         case 'l':         \
204         case 'm':         \
205         case 'n':         \
206         case 'o':         \
207         case 'p':         \
208         case 'q':         \
209         case 'r':         \
210         case 's':         \
211         case 't':         \
212         case 'u':         \
213         case 'v':         \
214         case 'w':         \
215         case 'x':         \
216         case 'y':         \
217         case 'z':         \
218         case 'A':         \
219         case 'B':         \
220         case 'C':         \
221         case 'D':         \
222         case 'E':         \
223         case 'F':         \
224         case 'G':         \
225         case 'H':         \
226         case 'I':         \
227         case 'J':         \
228         case 'K':         \
229         case 'L':         \
230         case 'M':         \
231         case 'N':         \
232         case 'O':         \
233         case 'P':         \
234         case 'Q':         \
235         case 'R':         \
236         case 'S':         \
237         case 'T':         \
238         case 'U':         \
239         case 'V':         \
240         case 'W':         \
241         case 'X':         \
242         case 'Y':         \
243         case 'Z':         \
244         case '_':
245
246 #define DIGITS        \
247         case '0':         \
248         case '1':         \
249         case '2':         \
250         case '3':         \
251         case '4':         \
252         case '5':         \
253         case '6':         \
254         case '7':         \
255         case '8':         \
256         case '9':
257
258 /**
259  * Read a symbol from the input and build
260  * the lexer_token.
261  */
262 static void parse_symbol(void)
263 {
264         symbol_t *symbol;
265         char     *string;
266
267         obstack_1grow(&symbol_obstack, (char) c);
268         next_char();
269
270         while(1) {
271                 switch(c) {
272                 DIGITS
273                 SYMBOL_CHARS
274                         obstack_1grow(&symbol_obstack, (char) c);
275                         next_char();
276                         break;
277
278                 default:
279                         goto end_symbol;
280                 }
281         }
282
283 end_symbol:
284         obstack_1grow(&symbol_obstack, '\0');
285
286         string = obstack_finish(&symbol_obstack);
287         symbol = symbol_table_insert(string);
288
289         lexer_token.type     = symbol->ID;
290         lexer_token.v.symbol = symbol;
291
292         if(symbol->string != string) {
293                 obstack_free(&symbol_obstack, string);
294         }
295 }
296
297 static void parse_integer_suffix(bool is_oct_hex)
298 {
299         bool is_unsigned  = false;
300         bool min_long     = false;
301         bool min_longlong = false;
302
303         if(c == 'U' || c == 'u') {
304                 is_unsigned = true;
305                 next_char();
306                 if(c == 'L' || c == 'l') {
307                         min_long = true;
308                         next_char();
309                         if(c == 'L' || c == 'l') {
310                                 min_longlong = true;
311                                 next_char();
312                         }
313                 }
314         } else if(c == 'l' || c == 'L') {
315                 min_long = true;
316                 next_char();
317                 if(c == 'l' || c == 'L') {
318                         min_longlong = true;
319                         next_char();
320                         if(c == 'u' || c == 'U') {
321                                 is_unsigned = true;
322                                 next_char();
323                         }
324                 } else if(c == 'u' || c == 'U') {
325                         is_unsigned = true;
326                         next_char();
327                         lexer_token.datatype = type_unsigned_long;
328                 }
329         }
330
331         if(!is_unsigned) {
332                 long long v = lexer_token.v.intvalue;
333                 if(!min_long) {
334                         if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
335                                 lexer_token.datatype = type_int;
336                                 return;
337                         } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
338                                 lexer_token.datatype = type_unsigned_int;
339                                 return;
340                         }
341                 }
342                 if(!min_longlong) {
343                         if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
344                                 lexer_token.datatype = type_long;
345                                 return;
346                         } else if(is_oct_hex && v >= 0 && v <= TARGET_ULONG_MAX) {
347                                 lexer_token.datatype = type_unsigned_long;
348                                 return;
349                         }
350                 }
351                 unsigned long long uv = (unsigned long long) v;
352                 if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
353                         lexer_token.datatype = type_unsigned_long_long;
354                         return;
355                 }
356
357                 lexer_token.datatype = type_long_long;
358         } else {
359                 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
360                 if(!min_long && v <= TARGET_UINT_MAX) {
361                         lexer_token.datatype = type_unsigned_int;
362                         return;
363                 }
364                 if(!min_longlong && v <= TARGET_ULONG_MAX) {
365                         lexer_token.datatype = type_unsigned_long;
366                         return;
367                 }
368                 lexer_token.datatype = type_unsigned_long_long;
369         }
370 }
371
372 static void parse_floating_suffix(void)
373 {
374         switch(c) {
375         /* TODO: do something useful with the suffixes... */
376         case 'f':
377         case 'F':
378                 next_char();
379                 lexer_token.datatype = type_float;
380                 break;
381         case 'l':
382         case 'L':
383                 next_char();
384                 lexer_token.datatype = type_long_double;
385                 break;
386         default:
387                 lexer_token.datatype = type_double;
388                 break;
389         }
390 }
391
392 /**
393  * A replacement for strtoull. Only those parts needed for
394  * our parser are implemented.
395  */
396 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
397         unsigned long long v = 0;
398
399         switch (base) {
400         case 16:
401                 for (;; ++s) {
402                         /* check for overrun */
403                         if (v >= 0x1000000000000000ULL)
404                                 break;
405                         switch (tolower(*s)) {
406                         case '0': v <<= 4; break;
407                         case '1': v <<= 4; v |= 0x1; break;
408                         case '2': v <<= 4; v |= 0x2; break;
409                         case '3': v <<= 4; v |= 0x3; break;
410                         case '4': v <<= 4; v |= 0x4; break;
411                         case '5': v <<= 4; v |= 0x5; break;
412                         case '6': v <<= 4; v |= 0x6; break;
413                         case '7': v <<= 4; v |= 0x7; break;
414                         case '8': v <<= 4; v |= 0x8; break;
415                         case '9': v <<= 4; v |= 0x9; break;
416                         case 'a': v <<= 4; v |= 0xa; break;
417                         case 'b': v <<= 4; v |= 0xb; break;
418                         case 'c': v <<= 4; v |= 0xc; break;
419                         case 'd': v <<= 4; v |= 0xd; break;
420                         case 'e': v <<= 4; v |= 0xe; break;
421                         case 'f': v <<= 4; v |= 0xf; break;
422                         default:
423                                 goto end;
424                         }
425                 }
426                 break;
427         case 8:
428                 for (;; ++s) {
429                         /* check for overrun */
430                         if (v >= 0x2000000000000000ULL)
431                                 break;
432                         switch (tolower(*s)) {
433                         case '0': v <<= 3; break;
434                         case '1': v <<= 3; v |= 1; break;
435                         case '2': v <<= 3; v |= 2; break;
436                         case '3': v <<= 3; v |= 3; break;
437                         case '4': v <<= 3; v |= 4; break;
438                         case '5': v <<= 3; v |= 5; break;
439                         case '6': v <<= 3; v |= 6; break;
440                         case '7': v <<= 3; v |= 7; break;
441                         default:
442                                 goto end;
443                         }
444                 }
445                 break;
446         case 10:
447                 for (;; ++s) {
448                         /* check for overrun */
449                         if (v > 0x1999999999999999ULL)
450                                 break;
451                         switch (tolower(*s)) {
452                         case '0': v *= 10; break;
453                         case '1': v *= 10; v += 1; break;
454                         case '2': v *= 10; v += 2; break;
455                         case '3': v *= 10; v += 3; break;
456                         case '4': v *= 10; v += 4; break;
457                         case '5': v *= 10; v += 5; break;
458                         case '6': v *= 10; v += 6; break;
459                         case '7': v *= 10; v += 7; break;
460                         case '8': v *= 10; v += 8; break;
461                         case '9': v *= 10; v += 9; break;
462                         default:
463                                 goto end;
464                         }
465                 }
466                 break;
467         default:
468                 assert(0);
469                 break;
470         }
471 end:
472         *endptr = s;
473         return v;
474 }
475
476 /**
477  * Parses a hex number including hex floats and set the
478  * lexer_token.
479  */
480 static void parse_number_hex(void)
481 {
482         assert(c == 'x' || c == 'X');
483         next_char();
484
485         while(isxdigit(c)) {
486                 obstack_1grow(&symbol_obstack, (char) c);
487                 next_char();
488         }
489         obstack_1grow(&symbol_obstack, '\0');
490         char *string = obstack_finish(&symbol_obstack);
491
492         if(c == '.' || c == 'p' || c == 'P') {
493                 next_char();
494                 internal_error("Hex floating point numbers not implemented yet");
495         }
496         if(*string == '\0') {
497                 parse_error("invalid hex number");
498                 lexer_token.type = T_ERROR;
499         }
500
501         const char *endptr;
502         lexer_token.type       = T_INTEGER;
503         lexer_token.v.intvalue = parse_int_string(string, &endptr, 16);
504         if(*endptr != '\0') {
505                 parse_error("hex number literal too long");
506         }
507
508         obstack_free(&symbol_obstack, string);
509         parse_integer_suffix(true);
510 }
511
512 /**
513  * Returns true if the given char is a octal digit.
514  *
515  * @param char  the character to check
516  */
517 static inline bool is_octal_digit(int chr)
518 {
519         switch(chr) {
520         case '0':
521         case '1':
522         case '2':
523         case '3':
524         case '4':
525         case '5':
526         case '6':
527         case '7':
528                 return true;
529         default:
530                 return false;
531         }
532 }
533
534 /**
535  * Parses a octal number and set the lexer_token.
536  */
537 static void parse_number_oct(void)
538 {
539         while(is_octal_digit(c)) {
540                 obstack_1grow(&symbol_obstack, (char) c);
541                 next_char();
542         }
543         obstack_1grow(&symbol_obstack, '\0');
544         char *string = obstack_finish(&symbol_obstack);
545
546         const char *endptr;
547         lexer_token.type       = T_INTEGER;
548         lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
549         if(*endptr != '\0') {
550                 parse_error("octal number literal too long");
551         }
552
553         obstack_free(&symbol_obstack, string);
554         parse_integer_suffix(true);
555 }
556
557 /**
558  * Parses a decimal including float number and set the
559  * lexer_token.
560  */
561 static void parse_number_dec(void)
562 {
563         bool is_float = false;
564         while(isdigit(c)) {
565                 obstack_1grow(&symbol_obstack, (char) c);
566                 next_char();
567         }
568
569         if(c == '.') {
570                 obstack_1grow(&symbol_obstack, '.');
571                 next_char();
572
573                 while(isdigit(c)) {
574                         obstack_1grow(&symbol_obstack, (char) c);
575                         next_char();
576                 }
577                 is_float = true;
578         }
579         if(c == 'e' || c == 'E') {
580                 obstack_1grow(&symbol_obstack, 'e');
581                 next_char();
582
583                 if(c == '-' || c == '+') {
584                         obstack_1grow(&symbol_obstack, (char) c);
585                         next_char();
586                 }
587
588                 while(isdigit(c)) {
589                         obstack_1grow(&symbol_obstack, (char) c);
590                         next_char();
591                 }
592                 is_float = true;
593         }
594
595         obstack_1grow(&symbol_obstack, '\0');
596         char *string = obstack_finish(&symbol_obstack);
597
598         if(is_float) {
599                 char *endptr;
600                 lexer_token.type         = T_FLOATINGPOINT;
601                 lexer_token.v.floatvalue = strtold(string, &endptr);
602
603                 if(*endptr != '\0') {
604                         parse_error("invalid number literal");
605                 }
606
607                 parse_floating_suffix();
608         } else {
609                 const char *endptr;
610                 lexer_token.type       = T_INTEGER;
611                 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
612
613                 if(*endptr != '\0') {
614                         parse_error("invalid number literal");
615                 }
616
617                 parse_integer_suffix(false);
618         }
619         obstack_free(&symbol_obstack, string);
620 }
621
622 /**
623  * Parses a number and sets the lexer_token.
624  */
625 static void parse_number(void)
626 {
627         if (c == '0') {
628                 next_char();
629                 switch (c) {
630                         case 'X':
631                         case 'x':
632                                 parse_number_hex();
633                                 break;
634                         case '0':
635                         case '1':
636                         case '2':
637                         case '3':
638                         case '4':
639                         case '5':
640                         case '6':
641                         case '7':
642                                 parse_number_oct();
643                                 break;
644                         case '8':
645                         case '9':
646                                 next_char();
647                                 parse_error("invalid octal number");
648                                 lexer_token.type = T_ERROR;
649                                 return;
650                         case '.':
651                         case 'e':
652                         case 'E':
653                         default:
654                                 obstack_1grow(&symbol_obstack, '0');
655                                 parse_number_dec();
656                                 return;
657                 }
658         } else {
659                 parse_number_dec();
660         }
661 }
662
663 /**
664  * Returns the value of a digit.
665  * The only portable way to do it ...
666  */
667 static int digit_value(int digit) {
668         switch (digit) {
669         case '0': return 0;
670         case '1': return 1;
671         case '2': return 2;
672         case '3': return 3;
673         case '4': return 4;
674         case '5': return 5;
675         case '6': return 6;
676         case '7': return 7;
677         case '8': return 8;
678         case '9': return 9;
679         case 'a':
680         case 'A': return 10;
681         case 'b':
682         case 'B': return 11;
683         case 'c':
684         case 'C': return 12;
685         case 'd':
686         case 'D': return 13;
687         case 'e':
688         case 'E': return 14;
689         case 'f':
690         case 'F': return 15;
691         default:
692                 internal_error("wrong character given");
693         }
694 }
695
696 /**
697  * Parses an octal character sequence.
698  *
699  * @param first_digit  the already read first digit
700  */
701 static int parse_octal_sequence(const int first_digit)
702 {
703         assert(is_octal_digit(first_digit));
704         int value = digit_value(first_digit);
705         if (!is_octal_digit(c)) return value;
706         value = 8 * value + digit_value(c);
707         next_char();
708         if (!is_octal_digit(c)) return value;
709         value = 8 * value + digit_value(c);
710         next_char();
711
712         if(char_is_signed) {
713                 return (signed char) value;
714         } else {
715                 return (unsigned char) value;
716         }
717 }
718
719 /**
720  * Parses a hex character sequence.
721  */
722 static int parse_hex_sequence(void)
723 {
724         int value = 0;
725         while(isxdigit(c)) {
726                 value = 16 * value + digit_value(c);
727                 next_char();
728         }
729
730         if(char_is_signed) {
731                 return (signed char) value;
732         } else {
733                 return (unsigned char) value;
734         }
735 }
736
737 /**
738  * Parse an escape sequence.
739  */
740 static int parse_escape_sequence(void)
741 {
742         eat('\\');
743
744         int ec = c;
745         next_char();
746
747         switch(ec) {
748         case '"':  return '"';
749         case '\'': return '\'';
750         case '\\': return '\\';
751         case '?': return '\?';
752         case 'a': return '\a';
753         case 'b': return '\b';
754         case 'f': return '\f';
755         case 'n': return '\n';
756         case 'r': return '\r';
757         case 't': return '\t';
758         case 'v': return '\v';
759         case 'x':
760                 return parse_hex_sequence();
761         case '0':
762         case '1':
763         case '2':
764         case '3':
765         case '4':
766         case '5':
767         case '6':
768         case '7':
769                 return parse_octal_sequence(ec);
770         case EOF:
771                 parse_error("reached end of file while parsing escape sequence");
772                 return EOF;
773         default:
774                 parse_error("unknown escape sequence");
775                 return EOF;
776         }
777 }
778
779 /**
780  * Concatenate two strings.
781  */
782 string_t concat_strings(const string_t *const s1, const string_t *const s2)
783 {
784         const size_t len1 = s1->size - 1;
785         const size_t len2 = s2->size - 1;
786
787         char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
788         memcpy(concat, s1->begin, len1);
789         memcpy(concat + len1, s2->begin, len2 + 1);
790
791 #if 0 /* TODO hash */
792         const char *result = strset_insert(&stringset, concat);
793         if(result != concat) {
794                 obstack_free(&symbol_obstack, concat);
795         }
796
797         return result;
798 #else
799         return (string_t){ concat, len1 + len2 + 1 };
800 #endif
801 }
802
803 /**
804  * Concatenate a string and a wide string.
805  */
806 wide_string_t concat_string_wide_string(const string_t *const s1, const wide_string_t *const s2)
807 {
808         const size_t len1 = s1->size - 1;
809         const size_t len2 = s2->size - 1;
810
811         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
812         const char *const src = s1->begin;
813         for (size_t i = 0; i != len1; ++i) {
814                 concat[i] = src[i];
815         }
816         memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
817
818         return (wide_string_t){ concat, len1 + len2 + 1 };
819 }
820
821 /**
822  * Concatenate two wide strings.
823  */
824 wide_string_t concat_wide_strings(const wide_string_t *const s1, const wide_string_t *const s2)
825 {
826         const size_t len1 = s1->size - 1;
827         const size_t len2 = s2->size - 1;
828
829         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
830         memcpy(concat,        s1->begin, len1       * sizeof(*concat));
831         memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
832
833         return (wide_string_t){ concat, len1 + len2 + 1 };
834 }
835
836 /**
837  * Concatenate a wide string and a string.
838  */
839 wide_string_t concat_wide_string_string(const wide_string_t *const s1, const string_t *const s2)
840 {
841         const size_t len1 = s1->size - 1;
842         const size_t len2 = s2->size - 1;
843
844         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
845         memcpy(concat, s1->begin, len1 * sizeof(*concat));
846         const char *const src = s2->begin;
847         for (size_t i = 0; i != len2 + 1; ++i) {
848                 concat[i] = src[i];
849         }
850
851         return (wide_string_t){ concat, len1 + len2 + 1 };
852 }
853
854 /**
855  * Parse a string literal and set lexer_token.
856  */
857 static void parse_string_literal(void)
858 {
859         const unsigned start_linenr = lexer_token.source_position.linenr;
860
861         eat('"');
862
863         int tc;
864         while(1) {
865                 switch(c) {
866                 case '\\':
867                         tc = parse_escape_sequence();
868                         obstack_1grow(&symbol_obstack, (char) tc);
869                         break;
870
871                 case EOF: {
872                         source_position_t source_position;
873                         source_position.input_name = lexer_token.source_position.input_name;
874                         source_position.linenr     = start_linenr;
875                         errorf(source_position, "string has no end");
876                         lexer_token.type = T_ERROR;
877                         return;
878                 }
879
880                 case '"':
881                         next_char();
882                         goto end_of_string;
883
884                 default:
885                         obstack_1grow(&symbol_obstack, (char) c);
886                         next_char();
887                         break;
888                 }
889         }
890
891 end_of_string:
892
893         /* TODO: concatenate multiple strings separated by whitespace... */
894
895         /* add finishing 0 to the string */
896         obstack_1grow(&symbol_obstack, '\0');
897         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
898         const char *const string = obstack_finish(&symbol_obstack);
899
900 #if 0 /* TODO hash */
901         /* check if there is already a copy of the string */
902         result = strset_insert(&stringset, string);
903         if(result != string) {
904                 obstack_free(&symbol_obstack, string);
905         }
906 #else
907         const char *const result = string;
908 #endif
909
910         lexer_token.type           = T_STRING_LITERAL;
911         lexer_token.v.string.begin = result;
912         lexer_token.v.string.size  = size;
913 }
914
915 /**
916  * Parse a wide character constant and set lexer_token.
917  */
918 static void parse_wide_character_constant(void)
919 {
920         const unsigned start_linenr = lexer_token.source_position.linenr;
921
922         eat('\'');
923
924         while(1) {
925                 switch(c) {
926                 case '\\': {
927                         wchar_rep_t tc = parse_escape_sequence();
928                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
929                         break;
930                 }
931
932                 MATCH_NEWLINE(
933                         parse_error("newline while parsing character constant");
934                         break;
935                 )
936
937                 case '\'':
938                         next_char();
939                         goto end_of_wide_char_constant;
940
941                 case EOF: {
942                         source_position_t source_position = lexer_token.source_position;
943                         source_position.linenr = start_linenr;
944                         errorf(source_position, "EOF while parsing character constant");
945                         lexer_token.type = T_ERROR;
946                         return;
947                 }
948
949                 default: {
950                         wchar_rep_t tc = (wchar_rep_t) c;
951                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
952                         next_char();
953                         break;
954                 }
955                 }
956         }
957
958 end_of_wide_char_constant:;
959         size_t             size   = (size_t) obstack_object_size(&symbol_obstack);
960         const wchar_rep_t *string = obstack_finish(&symbol_obstack);
961
962         lexer_token.type                = T_WIDE_CHARACTER_CONSTANT;
963         lexer_token.v.wide_string.begin = string;
964         lexer_token.v.wide_string.size  = size;
965         lexer_token.datatype            = type_wchar_t;
966 }
967
968 /**
969  * Parse a wide string literal and set lexer_token.
970  */
971 static void parse_wide_string_literal(void)
972 {
973         const unsigned start_linenr = lexer_token.source_position.linenr;
974
975         assert(c == '"');
976         next_char();
977
978         while(1) {
979                 switch(c) {
980                 case '\\': {
981                         wchar_rep_t tc = parse_escape_sequence();
982                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
983                         break;
984                 }
985
986                 case EOF: {
987                         source_position_t source_position;
988                         source_position.input_name = lexer_token.source_position.input_name;
989                         source_position.linenr     = start_linenr;
990                         errorf(source_position, "string has no end");
991                         lexer_token.type = T_ERROR;
992                         return;
993                 }
994
995                 case '"':
996                         next_char();
997                         goto end_of_string;
998
999                 default: {
1000                         wchar_rep_t tc = c;
1001                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1002                         next_char();
1003                         break;
1004                 }
1005                 }
1006         }
1007
1008 end_of_string:;
1009
1010         /* TODO: concatenate multiple strings separated by whitespace... */
1011
1012         /* add finishing 0 to the string */
1013         wchar_rep_t nul = L'\0';
1014         obstack_grow(&symbol_obstack, &nul, sizeof(nul));
1015         const size_t             size   = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
1016         const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
1017
1018 #if 0 /* TODO hash */
1019         /* check if there is already a copy of the string */
1020         const wchar_rep_t *const result = strset_insert(&stringset, string);
1021         if(result != string) {
1022                 obstack_free(&symbol_obstack, string);
1023         }
1024 #else
1025         const wchar_rep_t *const result = string;
1026 #endif
1027
1028         lexer_token.type                = T_WIDE_STRING_LITERAL;
1029         lexer_token.v.wide_string.begin = result;
1030         lexer_token.v.wide_string.size  = size;
1031 }
1032
1033 /**
1034  * Parse a character constant and set lexer_token.
1035  */
1036 static void parse_character_constant(void)
1037 {
1038         const unsigned start_linenr = lexer_token.source_position.linenr;
1039
1040         eat('\'');
1041
1042         while(1) {
1043                 switch(c) {
1044                 case '\\': {
1045                         int tc = parse_escape_sequence();
1046                         obstack_1grow(&symbol_obstack, (char) tc);
1047                         break;
1048                 }
1049
1050                 MATCH_NEWLINE(
1051                         parse_error("newline while parsing character constant");
1052                         break;
1053                 )
1054
1055                 case '\'':
1056                         next_char();
1057                         goto end_of_char_constant;
1058
1059                 case EOF: {
1060                         source_position_t source_position;
1061                         source_position.input_name = lexer_token.source_position.input_name;
1062                         source_position.linenr     = start_linenr;
1063                         errorf(source_position, "EOF while parsing character constant");
1064                         lexer_token.type = T_ERROR;
1065                         return;
1066                 }
1067
1068                 default:
1069                         obstack_1grow(&symbol_obstack, (char) c);
1070                         next_char();
1071                         break;
1072
1073                 }
1074         }
1075
1076 end_of_char_constant:;
1077         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
1078         const char *const string = obstack_finish(&symbol_obstack);
1079
1080         lexer_token.type           = T_CHARACTER_CONSTANT;
1081         lexer_token.v.string.begin = string;
1082         lexer_token.v.string.size  = size;
1083         lexer_token.datatype       = type_int;
1084 }
1085
1086 /**
1087  * Skip a multiline comment.
1088  */
1089 static void skip_multiline_comment(void)
1090 {
1091         unsigned start_linenr = lexer_token.source_position.linenr;
1092
1093         while(1) {
1094                 switch(c) {
1095                 case '/':
1096                         next_char();
1097                         if (c == '*') {
1098                                 /* TODO: nested comment, warn here */
1099                         }
1100                         break;
1101                 case '*':
1102                         next_char();
1103                         if(c == '/') {
1104                                 next_char();
1105                                 return;
1106                         }
1107                         break;
1108
1109                 MATCH_NEWLINE(break;)
1110
1111                 case EOF: {
1112                         source_position_t source_position;
1113                         source_position.input_name = lexer_token.source_position.input_name;
1114                         source_position.linenr     = start_linenr;
1115                         errorf(source_position, "at end of file while looking for comment end");
1116                         return;
1117                 }
1118
1119                 default:
1120                         next_char();
1121                         break;
1122                 }
1123         }
1124 }
1125
1126 /**
1127  * Skip a single line comment.
1128  */
1129 static void skip_line_comment(void)
1130 {
1131         while(1) {
1132                 switch(c) {
1133                 case EOF:
1134                         return;
1135
1136                 case '\n':
1137                 case '\r':
1138                         return;
1139
1140                 default:
1141                         next_char();
1142                         break;
1143                 }
1144         }
1145 }
1146
1147 /** The current preprocessor token. */
1148 static token_t pp_token;
1149
1150 /**
1151  * Read the next preprocessor token.
1152  */
1153 static inline void next_pp_token(void)
1154 {
1155         lexer_next_preprocessing_token();
1156         pp_token = lexer_token;
1157 }
1158
1159 /**
1160  * Eat all preprocessor tokens until newline.
1161  */
1162 static void eat_until_newline(void)
1163 {
1164         while(pp_token.type != '\n' && pp_token.type != T_EOF) {
1165                 next_pp_token();
1166         }
1167 }
1168
1169 /**
1170  * Handle the define directive.
1171  */
1172 static void define_directive(void)
1173 {
1174         lexer_next_preprocessing_token();
1175         if(lexer_token.type != T_IDENTIFIER) {
1176                 parse_error("expected identifier after #define\n");
1177                 eat_until_newline();
1178         }
1179 }
1180
1181 /**
1182  * Handle the ifdef directive.
1183  */
1184 static void ifdef_directive(int is_ifndef)
1185 {
1186         (void) is_ifndef;
1187         lexer_next_preprocessing_token();
1188         //expect_identifier();
1189         //extect_newline();
1190 }
1191
1192 /**
1193  * Handle the endif directive.
1194  */
1195 static void endif_directive(void)
1196 {
1197         //expect_newline();
1198 }
1199
1200 /**
1201  * Parse the line directive.
1202  */
1203 static void parse_line_directive(void)
1204 {
1205         if(pp_token.type != T_INTEGER) {
1206                 parse_error("expected integer");
1207         } else {
1208                 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
1209                 next_pp_token();
1210         }
1211         if(pp_token.type == T_STRING_LITERAL) {
1212                 lexer_token.source_position.input_name = pp_token.v.string.begin;
1213                 next_pp_token();
1214         }
1215
1216         eat_until_newline();
1217 }
1218
1219 /**
1220  * STDC pragmas.
1221  */
1222 typedef enum {
1223         STDC_UNKNOWN,
1224         STDC_FP_CONTRACT,
1225         STDC_FENV_ACCESS,
1226         STDC_CX_LIMITED_RANGE
1227 } stdc_pragma_kind_t;
1228
1229 /**
1230  * STDC pragma values.
1231  */
1232 typedef enum {
1233         STDC_VALUE_UNKNOWN,
1234         STDC_VALUE_ON,
1235         STDC_VALUE_OFF,
1236         STDC_VALUE_DEFAULT
1237 } stdc_pragma_value_kind_t;
1238
1239 /**
1240  * Parse a pragma directive.
1241  */
1242 static void parse_pragma(void) {
1243         bool unknown_pragma = true;
1244
1245         next_pp_token();
1246         if (pp_token.v.symbol->pp_ID == TP_STDC) {
1247                 stdc_pragma_kind_t kind = STDC_UNKNOWN;
1248                 /* a STDC pragma */
1249                 if (c_mode & _C99) {
1250                         next_pp_token();
1251
1252                         switch (pp_token.v.symbol->pp_ID) {
1253                         case TP_FP_CONTRACT:
1254                                 kind = STDC_FP_CONTRACT;
1255                                 break;
1256                         case TP_FENV_ACCESS:
1257                                 kind = STDC_FENV_ACCESS;
1258                                 break;
1259                         case TP_CX_LIMITED_RANGE:
1260                                 kind = STDC_CX_LIMITED_RANGE;
1261                                 break;
1262                         default:
1263                                 break;
1264                         }
1265                         if (kind != STDC_UNKNOWN) {
1266                                 stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN;
1267                                 next_pp_token();
1268                                 switch (pp_token.v.symbol->pp_ID) {
1269                                 case TP_ON:
1270                                         value = STDC_VALUE_ON;
1271                                         break;
1272                                 case TP_OFF:
1273                                         value = STDC_VALUE_OFF;
1274                                         break;
1275                                 case TP_DEFAULT:
1276                                         value = STDC_VALUE_DEFAULT;
1277                                         break;
1278                                 default:
1279                                         break;
1280                                 }
1281                                 if (value != STDC_VALUE_UNKNOWN) {
1282                                         unknown_pragma = false;
1283                                 } else {
1284                                         errorf(pp_token.source_position, "bad STDC pragma argument");
1285                                 }
1286                         }
1287                 }
1288         } else {
1289                 unknown_pragma = true;
1290         }
1291         eat_until_newline();
1292         if (unknown_pragma && warning.unknown_pragmas) {
1293                 warningf(pp_token.source_position, "encountered unknown #pragma");
1294         }
1295 }
1296
1297 /**
1298  * Parse a preprocessor non-null directive.
1299  */
1300 static void parse_preprocessor_identifier(void)
1301 {
1302         assert(pp_token.type == T_IDENTIFIER);
1303         symbol_t *symbol = pp_token.v.symbol;
1304
1305         switch(symbol->pp_ID) {
1306         case TP_include:
1307                 printf("include - enable header name parsing!\n");
1308                 break;
1309         case TP_define:
1310                 define_directive();
1311                 break;
1312         case TP_ifdef:
1313                 ifdef_directive(0);
1314                 break;
1315         case TP_ifndef:
1316                 ifdef_directive(1);
1317                 break;
1318         case TP_endif:
1319                 endif_directive();
1320                 break;
1321         case TP_line:
1322                 next_pp_token();
1323                 parse_line_directive();
1324                 break;
1325         case TP_if:
1326         case TP_else:
1327         case TP_elif:
1328         case TP_undef:
1329         case TP_error:
1330                 /* TODO; output the rest of the line */
1331                 parse_error("#error directive: ");
1332                 break;
1333         case TP_pragma:
1334                 parse_pragma();
1335                 break;
1336         }
1337 }
1338
1339 /**
1340  * Parse a preprocessor directive.
1341  */
1342 static void parse_preprocessor_directive(void)
1343 {
1344         next_pp_token();
1345
1346         switch(pp_token.type) {
1347         case T_IDENTIFIER:
1348                 parse_preprocessor_identifier();
1349                 break;
1350         case T_INTEGER:
1351                 parse_line_directive();
1352                 break;
1353         case '\n':
1354                 /* NULL directive, see Â§ 6.10.7 */
1355                 break;
1356         default:
1357                 parse_error("invalid preprocessor directive");
1358                 eat_until_newline();
1359                 break;
1360         }
1361 }
1362
1363 #define MAYBE_PROLOG                                       \
1364                         next_char();                                   \
1365                         while(1) {                                     \
1366                                 switch(c) {
1367
1368 #define MAYBE(ch, set_type)                                \
1369                                 case ch:                                   \
1370                                         next_char();                           \
1371                                         lexer_token.type = set_type;           \
1372                                         return;
1373
1374 #define ELSE_CODE(code)                                    \
1375                                 default:                                   \
1376                                         code;                                  \
1377                                 }                                          \
1378                         } /* end of while(1) */                        \
1379                         break;
1380
1381 #define ELSE(set_type)                                     \
1382                 ELSE_CODE(                                         \
1383                         lexer_token.type = set_type;                   \
1384                         return;                                        \
1385                 )
1386
1387 void lexer_next_preprocessing_token(void)
1388 {
1389         while(1) {
1390                 switch(c) {
1391                 case ' ':
1392                 case '\t':
1393                         next_char();
1394                         break;
1395
1396                 MATCH_NEWLINE(
1397                         lexer_token.type = '\n';
1398                         return;
1399                 )
1400
1401                 SYMBOL_CHARS
1402                         parse_symbol();
1403                         /* might be a wide string ( L"string" ) */
1404                         if(lexer_token.type == T_IDENTIFIER &&
1405                             lexer_token.v.symbol == symbol_L) {
1406                             if(c == '"') {
1407                                         parse_wide_string_literal();
1408                                 } else if(c == '\'') {
1409                                         parse_wide_character_constant();
1410                                 }
1411                         }
1412                         return;
1413
1414                 DIGITS
1415                         parse_number();
1416                         return;
1417
1418                 case '"':
1419                         parse_string_literal();
1420                         return;
1421
1422                 case '\'':
1423                         parse_character_constant();
1424                         return;
1425
1426                 case '.':
1427                         MAYBE_PROLOG
1428                                 case '0':
1429                                 case '1':
1430                                 case '2':
1431                                 case '3':
1432                                 case '4':
1433                                 case '5':
1434                                 case '6':
1435                                 case '7':
1436                                 case '8':
1437                                 case '9':
1438                                         put_back(c);
1439                                         c = '.';
1440                                         parse_number_dec();
1441                                         return;
1442
1443                                 case '.':
1444                                         MAYBE_PROLOG
1445                                         MAYBE('.', T_DOTDOTDOT)
1446                                         ELSE_CODE(
1447                                                 put_back(c);
1448                                                 c = '.';
1449                                                 lexer_token.type = '.';
1450                                                 return;
1451                                         )
1452                         ELSE('.')
1453                 case '&':
1454                         MAYBE_PROLOG
1455                         MAYBE('&', T_ANDAND)
1456                         MAYBE('=', T_ANDEQUAL)
1457                         ELSE('&')
1458                 case '*':
1459                         MAYBE_PROLOG
1460                         MAYBE('=', T_ASTERISKEQUAL)
1461                         ELSE('*')
1462                 case '+':
1463                         MAYBE_PROLOG
1464                         MAYBE('+', T_PLUSPLUS)
1465                         MAYBE('=', T_PLUSEQUAL)
1466                         ELSE('+')
1467                 case '-':
1468                         MAYBE_PROLOG
1469                         MAYBE('>', T_MINUSGREATER)
1470                         MAYBE('-', T_MINUSMINUS)
1471                         MAYBE('=', T_MINUSEQUAL)
1472                         ELSE('-')
1473                 case '!':
1474                         MAYBE_PROLOG
1475                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1476                         ELSE('!')
1477                 case '/':
1478                         MAYBE_PROLOG
1479                         MAYBE('=', T_SLASHEQUAL)
1480                                 case '*':
1481                                         next_char();
1482                                         skip_multiline_comment();
1483                                         lexer_next_preprocessing_token();
1484                                         return;
1485                                 case '/':
1486                                         next_char();
1487                                         skip_line_comment();
1488                                         lexer_next_preprocessing_token();
1489                                         return;
1490                         ELSE('/')
1491                 case '%':
1492                         MAYBE_PROLOG
1493                         MAYBE('>', '}')
1494                         MAYBE('=', T_PERCENTEQUAL)
1495                                 case ':':
1496                                         MAYBE_PROLOG
1497                                                 case '%':
1498                                                         MAYBE_PROLOG
1499                                                         MAYBE(':', T_HASHHASH)
1500                                                         ELSE_CODE(
1501                                                                 put_back(c);
1502                                                                 c = '%';
1503                                                                 lexer_token.type = '#';
1504                                                                 return;
1505                                                         )
1506                                         ELSE('#')
1507                         ELSE('%')
1508                 case '<':
1509                         MAYBE_PROLOG
1510                         MAYBE(':', '[')
1511                         MAYBE('%', '{')
1512                         MAYBE('=', T_LESSEQUAL)
1513                                 case '<':
1514                                         MAYBE_PROLOG
1515                                         MAYBE('=', T_LESSLESSEQUAL)
1516                                         ELSE(T_LESSLESS)
1517                         ELSE('<')
1518                 case '>':
1519                         MAYBE_PROLOG
1520                         MAYBE('=', T_GREATEREQUAL)
1521                                 case '>':
1522                                         MAYBE_PROLOG
1523                                         MAYBE('=', T_GREATERGREATEREQUAL)
1524                                         ELSE(T_GREATERGREATER)
1525                         ELSE('>')
1526                 case '^':
1527                         MAYBE_PROLOG
1528                         MAYBE('=', T_CARETEQUAL)
1529                         ELSE('^')
1530                 case '|':
1531                         MAYBE_PROLOG
1532                         MAYBE('=', T_PIPEEQUAL)
1533                         MAYBE('|', T_PIPEPIPE)
1534                         ELSE('|')
1535                 case ':':
1536                         MAYBE_PROLOG
1537                         MAYBE('>', ']')
1538                         ELSE(':')
1539                 case '=':
1540                         MAYBE_PROLOG
1541                         MAYBE('=', T_EQUALEQUAL)
1542                         ELSE('=')
1543                 case '#':
1544                         MAYBE_PROLOG
1545                         MAYBE('#', T_HASHHASH)
1546                         ELSE('#')
1547
1548                 case '?':
1549                 case '[':
1550                 case ']':
1551                 case '(':
1552                 case ')':
1553                 case '{':
1554                 case '}':
1555                 case '~':
1556                 case ';':
1557                 case ',':
1558                 case '\\':
1559                         lexer_token.type = c;
1560                         next_char();
1561                         return;
1562
1563                 case EOF:
1564                         lexer_token.type = T_EOF;
1565                         return;
1566
1567                 default:
1568                         next_char();
1569                         errorf(lexer_token.source_position, "unknown character '%c' found\n", c);
1570                         lexer_token.type = T_ERROR;
1571                         return;
1572                 }
1573         }
1574 }
1575
1576 void lexer_next_token(void)
1577 {
1578         lexer_next_preprocessing_token();
1579         if(lexer_token.type != '\n')
1580                 return;
1581
1582 newline_found:
1583         do {
1584                 lexer_next_preprocessing_token();
1585         } while(lexer_token.type == '\n');
1586
1587         if(lexer_token.type == '#') {
1588                 parse_preprocessor_directive();
1589                 goto newline_found;
1590         }
1591 }
1592
1593 void init_lexer(void)
1594 {
1595         strset_init(&stringset);
1596 }
1597
1598 void lexer_open_stream(FILE *stream, const char *input_name)
1599 {
1600         input                                  = stream;
1601         lexer_token.source_position.linenr     = 0;
1602         lexer_token.source_position.input_name = input_name;
1603
1604         symbol_L = symbol_table_insert("L");
1605         bufpos = NULL;
1606         bufend = NULL;
1607
1608         /* place a virtual \n at the beginning so the lexer knows that we're
1609          * at the beginning of a line */
1610         c = '\n';
1611 }
1612
1613 void exit_lexer(void)
1614 {
1615         strset_destroy(&stringset);
1616 }
1617
1618 static __attribute__((unused))
1619 void dbg_pos(const source_position_t source_position)
1620 {
1621         fprintf(stdout, "%s:%u\n", source_position.input_name,
1622                 source_position.linenr);
1623         fflush(stdout);
1624 }