nsz Git - cparser/blob - lexer.c

   1 /*
   2  * This file is part of cparser.
   3  * Copyright (C) 2007-2009 Matthias Braun <matze@braunis.de>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; either version 2
   8  * of the License, or (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  18  * 02111-1307, USA.
  19  */
  20 #include <config.h>
  21
  22 #include "diagnostic.h"
  23 #include "lexer.h"
  24 #include "symbol_t.h"
  25 #include "token_t.h"
  26 #include "symbol_table_t.h"
  27 #include "adt/error.h"
  28 #include "adt/strset.h"
  29 #include "adt/util.h"
  30 #include "types.h"
  31 #include "type_t.h"
  32 #include "target_architecture.h"
  33 #include "parser.h"
  34 #include "warning.h"
  35 #include "lang_features.h"
  36
  37 #include <assert.h>
  38 #include <errno.h>
  39 #include <string.h>
  40 #include <stdbool.h>
  41 #include <ctype.h>
  42
  43 #ifndef _WIN32
  44 #include <strings.h>
  45 #endif
  46
  47 //#define DEBUG_CHARS
  48 #define MAX_PUTBACK 3
  49 #define BUF_SIZE    1024
  50
  51 #if defined(_WIN32) || defined(__CYGWIN__)
  52 /* No strtold on windows and no replacement yet */
  53 #define strtold(s, e) strtod(s, e)
  54 #endif
  55
  56 static utf32             c;
  57 static source_position_t lexer_pos;
  58 token_t                  lexer_token;
  59 static symbol_t         *symbol_L;
  60 static FILE             *input;
  61 static utf32             buf[BUF_SIZE + MAX_PUTBACK];
  62 static const utf32      *bufend;
  63 static const utf32      *bufpos;
  64 static strset_t          stringset;
  65 bool                     allow_dollar_in_symbol = true;
  66
  67 /**
  68  * Prints a parse error message at the current token.
  69  *
  70  * @param msg   the error message
  71  */
  72 static void parse_error(const char *msg)
  73 {
  74         errorf(&lexer_pos, "%s", msg);
  75 }
  76
  77 /**
  78  * Prints an internal error message at the current token.
  79  *
  80  * @param msg   the error message
  81  */
  82 static NORETURN internal_error(const char *msg)
  83 {
  84         internal_errorf(&lexer_pos, "%s", msg);
  85 }
  86
  87 static size_t read_block(unsigned char *const read_buf, size_t const n)
  88 {
  89         size_t const s = fread(read_buf, 1, n, input);
  90         if (s == 0) {
  91                 /* on OS/X ferror appears to return true on eof as well when running
  92                  * the application in gdb... */
  93                 if (!feof(input) && ferror(input))
  94                         parse_error("read from input failed");
  95                 buf[MAX_PUTBACK] = EOF;
  96                 bufpos           = buf + MAX_PUTBACK;
  97                 bufend           = buf + MAX_PUTBACK + 1;
  98         }
  99         return s;
 100 }
 101
 102 static void decode_iso_8859_1(void)
 103 {
 104         unsigned char read_buf[BUF_SIZE];
 105         size_t const s = read_block(read_buf, sizeof(read_buf));
 106         if (s == 0)
 107                 return;
 108
 109         unsigned char const *src = read_buf;
 110         unsigned char const *end = read_buf + s;
 111         utf32               *dst = buf + MAX_PUTBACK;
 112         while (src != end)
 113                 *dst++ = *src++;
 114
 115         bufpos = buf + MAX_PUTBACK;
 116         bufend = dst;
 117 }
 118
 119 static void decode_iso_8859_15(void)
 120 {
 121         unsigned char read_buf[BUF_SIZE];
 122         size_t const s = read_block(read_buf, sizeof(read_buf));
 123         if (s == 0)
 124                 return;
 125
 126         unsigned char const *src = read_buf;
 127         unsigned char const *end = read_buf + s;
 128         utf32               *dst = buf + MAX_PUTBACK;
 129         while (src != end) {
 130                 utf32 tc = *src++;
 131                 switch (tc) {
 132                         case 0xA4: tc = 0x20AC; break; // €
 133                         case 0xA6: tc = 0x0160; break; // Š
 134                         case 0xA8: tc = 0x0161; break; // š
 135                         case 0xB4: tc = 0x017D; break; // Ž
 136                         case 0xB8: tc = 0x017E; break; // ž
 137                         case 0xBC: tc = 0x0152; break; // Œ
 138                         case 0xBD: tc = 0x0153; break; // œ
 139                         case 0xBE: tc = 0x0178; break; // Ÿ
 140                 }
 141                 *dst++ = tc;
 142         }
 143
 144         bufpos = buf + MAX_PUTBACK;
 145         bufend = dst;
 146 }
 147
 148 static void decode_utf8(void)
 149 {
 150         static utf32  part_decoded_min_code;
 151         static utf32  part_decoded_char;
 152         static size_t part_decoded_rest_len;
 153
 154         do {
 155                 unsigned char read_buf[BUF_SIZE];
 156                 size_t const s = read_block(read_buf, sizeof(read_buf));
 157                 if (s == 0) {
 158                         if (part_decoded_rest_len > 0)
 159                                 parse_error("incomplete input char at end of input");
 160                         return;
 161                 }
 162
 163                 unsigned char const *src = read_buf;
 164                 unsigned char const *end = read_buf + s;
 165                 utf32               *dst = buf + MAX_PUTBACK;
 166                 utf32                decoded;
 167                 utf32                min_code;
 168
 169                 if (part_decoded_rest_len != 0) {
 170                         min_code              = part_decoded_min_code;
 171                         decoded               = part_decoded_char;
 172                         size_t const rest_len = part_decoded_rest_len;
 173                         part_decoded_rest_len = 0;
 174                         switch (rest_len) {
 175                                 case 4:  goto realign;
 176                                 case 3:  goto three_more;
 177                                 case 2:  goto two_more;
 178                                 default: goto one_more;
 179                         }
 180                 }
 181
 182                 while (src != end) {
 183                         if ((*src & 0x80) == 0) {
 184                                 decoded = *src++;
 185                         } else if ((*src & 0xE0) == 0xC0) {
 186                                 min_code = 0x80;
 187                                 decoded  = *src++ & 0x1F;
 188 one_more:
 189                                 if (src == end) {
 190                                         part_decoded_min_code = min_code;
 191                                         part_decoded_char     = decoded;
 192                                         part_decoded_rest_len = 1;
 193                                         break;
 194                                 }
 195                                 if ((*src & 0xC0) == 0x80) {
 196                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 197                                 } else {
 198                                         goto invalid_char;
 199                                 }
 200                                 if (decoded < min_code                      ||
 201                                                 decoded > 0x10FFFF                      ||
 202                                                 (0xD800 <= decoded && decoded < 0xE000) || // high/low surrogates
 203                                                 (0xFDD0 <= decoded && decoded < 0xFDF0) || // noncharacters
 204                                                 (decoded & 0xFFFE) == 0xFFFE) {            // noncharacters
 205                                         parse_error("invalid byte sequence in input");
 206                                 }
 207                         } else if ((*src & 0xF0) == 0xE0) {
 208                                 min_code = 0x800;
 209                                 decoded  = *src++ & 0x0F;
 210 two_more:
 211                                 if (src == end) {
 212                                         part_decoded_min_code = min_code;
 213                                         part_decoded_char     = decoded;
 214                                         part_decoded_rest_len = 2;
 215                                         break;
 216                                 }
 217                                 if ((*src & 0xC0) == 0x80) {
 218                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 219                                 } else {
 220                                         goto invalid_char;
 221                                 }
 222                                 goto one_more;
 223                         } else if ((*src & 0xF8) == 0xF0) {
 224                                 min_code = 0x10000;
 225                                 decoded  = *src++ & 0x07;
 226 three_more:
 227                                 if (src == end) {
 228                                         part_decoded_min_code = min_code;
 229                                         part_decoded_char     = decoded;
 230                                         part_decoded_rest_len = 3;
 231                                         break;
 232                                 }
 233                                 if ((*src & 0xC0) == 0x80) {
 234                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 235                                 } else {
 236                                         goto invalid_char;
 237                                 }
 238                                 goto two_more;
 239                         } else {
 240 invalid_char:
 241                                 parse_error("invalid byte sequence in input");
 242 realign:
 243                                 do {
 244                                         ++src;
 245                                         if (src == end) {
 246                                                 part_decoded_rest_len = 4;
 247                                                 break;
 248                                         }
 249                                 } while ((*src & 0xC0) == 0x80 || (*src & 0xF8) == 0xF8);
 250                                 continue;
 251                         }
 252                         *dst++ = decoded;
 253                 }
 254
 255                 bufpos = buf + MAX_PUTBACK;
 256                 bufend = dst;
 257         } while (bufpos == bufend);
 258 }
 259
 260 static void decode_windows_1252(void)
 261 {
 262         unsigned char read_buf[BUF_SIZE];
 263         size_t const s = read_block(read_buf, sizeof(read_buf));
 264         if (s == 0)
 265                 return;
 266
 267         unsigned char const *src = read_buf;
 268         unsigned char const *end = read_buf + s;
 269         utf32               *dst = buf + MAX_PUTBACK;
 270         while (src != end) {
 271                 utf32 tc = *src++;
 272                 switch (tc) {
 273                         case 0x80: tc = 0x20AC; break; // €
 274                         case 0x82: tc = 0x201A; break; // ‚
 275                         case 0x83: tc = 0x0192; break; // ƒ
 276                         case 0x84: tc = 0x201E; break; // „
 277                         case 0x85: tc = 0x2026; break; // …
 278                         case 0x86: tc = 0x2020; break; // †
 279                         case 0x87: tc = 0x2021; break; // ‡
 280                         case 0x88: tc = 0x02C6; break; // ˆ
 281                         case 0x89: tc = 0x2030; break; // ‰
 282                         case 0x8A: tc = 0x0160; break; // Š
 283                         case 0x8B: tc = 0x2039; break; // ‹
 284                         case 0x8C: tc = 0x0152; break; // Œ
 285                         case 0x8E: tc = 0x017D; break; // Ž
 286                         case 0x91: tc = 0x2018; break; // ‘
 287                         case 0x92: tc = 0x2019; break; // ’
 288                         case 0x93: tc = 0x201C; break; // “
 289                         case 0x94: tc = 0x201D; break; // ”
 290                         case 0x95: tc = 0x2022; break; // •
 291                         case 0x96: tc = 0x2013; break; // –
 292                         case 0x97: tc = 0x2014; break; // —
 293                         case 0x98: tc = 0x02DC; break; // ˜
 294                         case 0x99: tc = 0x2122; break; // ™
 295                         case 0x9A: tc = 0x0161; break; // š
 296                         case 0x9B: tc = 0x203A; break; // ›
 297                         case 0x9C: tc = 0x0153; break; // œ
 298                         case 0x9E: tc = 0x017E; break; // ž
 299                         case 0x9F: tc = 0x0178; break; // Ÿ
 300                 }
 301                 *dst++ = tc;
 302         }
 303
 304         bufpos = buf + MAX_PUTBACK;
 305         bufend = dst;
 306 }
 307
 308 typedef void (*decoder_t)(void);
 309
 310 static decoder_t decoder = decode_utf8;
 311
 312 typedef struct named_decoder_t {
 313         char const *name;
 314         decoder_t   decoder;
 315 } named_decoder_t;
 316
 317 static named_decoder_t const decoders[] = {
 318         { "CP819",           decode_iso_8859_1   }, // offical alias
 319         { "IBM819",          decode_iso_8859_1   }, // offical alias
 320         { "ISO-8859-1",      decode_iso_8859_1   }, // offical alias
 321         { "ISO-8859-15",     decode_iso_8859_15  }, // offical name
 322         { "ISO8859-1",       decode_iso_8859_1   },
 323         { "ISO8859-15",      decode_iso_8859_15  },
 324         { "ISO_8859-1",      decode_iso_8859_1   }, // offical alias
 325         { "ISO_8859-15",     decode_iso_8859_15  }, // offical alias
 326         { "ISO_8859-1:1987", decode_iso_8859_1   }, // offical name
 327         { "Latin-9",         decode_iso_8859_15  }, // offical alias
 328         { "UTF-8",           decode_utf8         }, // offical name
 329         { "csISOLatin1",     decode_iso_8859_1   }, // offical alias
 330         { "cp1252",          decode_windows_1252 },
 331         { "iso-ir-100",      decode_iso_8859_1   }, // offical alias
 332         { "l1",              decode_iso_8859_1   }, // offical alias
 333         { "latin1",          decode_iso_8859_1   }, // offical alias
 334         { "windows-1252",    decode_windows_1252 }, // official name
 335
 336         { NULL,              NULL                }
 337 };
 338
 339 /** strcasecmp is not part of C99 so we need our own implementation here */
 340 static int my_strcasecmp(const char *s1, const char *s2)
 341 {
 342         for ( ; *s1 != 0; ++s1, ++s2) {
 343                 if (tolower(*s1) != tolower(*s2))
 344                         break;
 345         }
 346         return (unsigned char)*s1 - (unsigned char)*s2;
 347 }
 348
 349 void select_input_encoding(char const* const encoding)
 350 {
 351         for (named_decoder_t const *i = decoders; i->name != NULL; ++i) {
 352                 if (my_strcasecmp(encoding, i->name) != 0)
 353                         continue;
 354                 decoder = i->decoder;
 355                 return;
 356         }
 357         fprintf(stderr, "error: input encoding \"%s\" not supported\n", encoding);
 358 }
 359
 360 static inline void next_real_char(void)
 361 {
 362         assert(bufpos <= bufend);
 363         if (bufpos >= bufend) {
 364                 if (input == NULL) {
 365                         c = EOF;
 366                         return;
 367                 }
 368                 decoder();
 369         }
 370         c = *bufpos++;
 371         ++lexer_pos.colno;
 372 }
 373
 374 /**
 375  * Put a character back into the buffer.
 376  *
 377  * @param pc  the character to put back
 378  */
 379 static inline void put_back(utf32 const pc)
 380 {
 381         assert(bufpos > buf);
 382         *(--bufpos - buf + buf) = pc;
 383         --lexer_pos.colno;
 384
 385 #ifdef DEBUG_CHARS
 386         printf("putback '%lc'\n", pc);
 387 #endif
 388 }
 389
 390 static inline void next_char(void);
 391
 392 #define MATCH_NEWLINE(code)  \
 393         case '\r':               \
 394                 next_char();         \
 395                 if (c == '\n') {     \
 396         case '\n':               \
 397                         next_char();     \
 398                 }                    \
 399                 lexer_pos.lineno++;  \
 400                 lexer_pos.colno = 1; \
 401                 code
 402
 403 #define eat(c_type) (assert(c == c_type), next_char())
 404
 405 static void maybe_concat_lines(void)
 406 {
 407         eat('\\');
 408
 409         switch (c) {
 410         MATCH_NEWLINE(return;)
 411
 412         default:
 413                 break;
 414         }
 415
 416         put_back(c);
 417         c = '\\';
 418 }
 419
 420 /**
 421  * Set c to the next input character, ie.
 422  * after expanding trigraphs.
 423  */
 424 static inline void next_char(void)
 425 {
 426         next_real_char();
 427
 428         /* filter trigraphs */
 429         if (UNLIKELY(c == '\\')) {
 430                 maybe_concat_lines();
 431                 goto end_of_next_char;
 432         }
 433
 434         if (LIKELY(c != '?'))
 435                 goto end_of_next_char;
 436
 437         next_real_char();
 438         if (LIKELY(c != '?')) {
 439                 put_back(c);
 440                 c = '?';
 441                 goto end_of_next_char;
 442         }
 443
 444         next_real_char();
 445         switch (c) {
 446         case '=': c = '#'; break;
 447         case '(': c = '['; break;
 448         case '/': c = '\\'; maybe_concat_lines(); break;
 449         case ')': c = ']'; break;
 450         case '\'': c = '^'; break;
 451         case '<': c = '{'; break;
 452         case '!': c = '|'; break;
 453         case '>': c = '}'; break;
 454         case '-': c = '~'; break;
 455         default:
 456                 put_back(c);
 457                 put_back('?');
 458                 c = '?';
 459                 break;
 460         }
 461
 462 end_of_next_char:;
 463 #ifdef DEBUG_CHARS
 464         printf("nchar '%c'\n", c);
 465 #endif
 466 }
 467
 468 #define SYMBOL_CHARS  \
 469         case '$': if (!allow_dollar_in_symbol) goto dollar_sign; \
 470         case 'a':         \
 471         case 'b':         \
 472         case 'c':         \
 473         case 'd':         \
 474         case 'e':         \
 475         case 'f':         \
 476         case 'g':         \
 477         case 'h':         \
 478         case 'i':         \
 479         case 'j':         \
 480         case 'k':         \
 481         case 'l':         \
 482         case 'm':         \
 483         case 'n':         \
 484         case 'o':         \
 485         case 'p':         \
 486         case 'q':         \
 487         case 'r':         \
 488         case 's':         \
 489         case 't':         \
 490         case 'u':         \
 491         case 'v':         \
 492         case 'w':         \
 493         case 'x':         \
 494         case 'y':         \
 495         case 'z':         \
 496         case 'A':         \
 497         case 'B':         \
 498         case 'C':         \
 499         case 'D':         \
 500         case 'E':         \
 501         case 'F':         \
 502         case 'G':         \
 503         case 'H':         \
 504         case 'I':         \
 505         case 'J':         \
 506         case 'K':         \
 507         case 'L':         \
 508         case 'M':         \
 509         case 'N':         \
 510         case 'O':         \
 511         case 'P':         \
 512         case 'Q':         \
 513         case 'R':         \
 514         case 'S':         \
 515         case 'T':         \
 516         case 'U':         \
 517         case 'V':         \
 518         case 'W':         \
 519         case 'X':         \
 520         case 'Y':         \
 521         case 'Z':         \
 522         case '_':
 523
 524 #define DIGITS        \
 525         case '0':         \
 526         case '1':         \
 527         case '2':         \
 528         case '3':         \
 529         case '4':         \
 530         case '5':         \
 531         case '6':         \
 532         case '7':         \
 533         case '8':         \
 534         case '9':
 535
 536 /**
 537  * Read a symbol from the input and build
 538  * the lexer_token.
 539  */
 540 static void parse_symbol(void)
 541 {
 542         obstack_1grow(&symbol_obstack, (char) c);
 543         next_char();
 544
 545         while (true) {
 546                 switch (c) {
 547                 DIGITS
 548                 SYMBOL_CHARS
 549                         obstack_1grow(&symbol_obstack, (char) c);
 550                         next_char();
 551                         break;
 552
 553                 default:
 554 dollar_sign:
 555                         goto end_symbol;
 556                 }
 557         }
 558
 559 end_symbol:
 560         obstack_1grow(&symbol_obstack, '\0');
 561
 562         char     *string = obstack_finish(&symbol_obstack);
 563         symbol_t *symbol = symbol_table_insert(string);
 564
 565         lexer_token.type   = symbol->ID;
 566         lexer_token.symbol = symbol;
 567
 568         if (symbol->string != string) {
 569                 obstack_free(&symbol_obstack, string);
 570         }
 571 }
 572
 573 /**
 574  * parse suffixes like 'LU' or 'f' after numbers
 575  */
 576 static void parse_number_suffix(void)
 577 {
 578         assert(obstack_object_size(&symbol_obstack) == 0);
 579         while (true) {
 580                 switch (c) {
 581                 SYMBOL_CHARS
 582                         obstack_1grow(&symbol_obstack, (char) c);
 583                         next_char();
 584                         break;
 585                 default:
 586                 dollar_sign:
 587                         goto finish_suffix;
 588                 }
 589         }
 590 finish_suffix:
 591         if (obstack_object_size(&symbol_obstack) == 0) {
 592                 lexer_token.symbol = NULL;
 593                 return;
 594         }
 595
 596         obstack_1grow(&symbol_obstack, '\0');
 597         char     *string = obstack_finish(&symbol_obstack);
 598         symbol_t *symbol = symbol_table_insert(string);
 599
 600         if (symbol->string != string) {
 601                 obstack_free(&symbol_obstack, string);
 602         }
 603         lexer_token.symbol = symbol;
 604 }
 605
 606 static string_t identify_string(char *string, size_t len)
 607 {
 608         /* TODO hash */
 609 #if 0
 610         const char *result = strset_insert(&stringset, concat);
 611         if (result != concat) {
 612                 obstack_free(&symbol_obstack, concat);
 613         }
 614 #else
 615         const char *result = string;
 616 #endif
 617         return (string_t) {result, len};
 618 }
 619
 620 /**
 621  * Parses a hex number including hex floats and set the
 622  * lexer_token.
 623  */
 624 static void parse_number_hex(void)
 625 {
 626         bool is_float   = false;
 627         bool has_digits = false;
 628
 629         assert(obstack_object_size(&symbol_obstack) == 0);
 630         while (isxdigit(c)) {
 631                 has_digits = true;
 632                 obstack_1grow(&symbol_obstack, (char) c);
 633                 next_char();
 634         }
 635
 636         if (c == '.') {
 637                 is_float = true;
 638                 obstack_1grow(&symbol_obstack, (char) c);
 639                 next_char();
 640
 641                 while (isxdigit(c)) {
 642                         has_digits = true;
 643                         obstack_1grow(&symbol_obstack, (char) c);
 644                         next_char();
 645                 }
 646         }
 647         if (c == 'p' || c == 'P') {
 648                 is_float = true;
 649                 obstack_1grow(&symbol_obstack, (char) c);
 650                 next_char();
 651
 652                 if (c == '-' || c == '+') {
 653                         obstack_1grow(&symbol_obstack, (char) c);
 654                         next_char();
 655                 }
 656
 657                 while (isxdigit(c)) {
 658                         obstack_1grow(&symbol_obstack, (char) c);
 659                         next_char();
 660                 }
 661         } else if (is_float) {
 662                 errorf(&lexer_token.source_position,
 663                        "hexadecimal floatingpoint constant requires an exponent");
 664         }
 665         obstack_1grow(&symbol_obstack, '\0');
 666
 667         size_t  size   = obstack_object_size(&symbol_obstack) - 1;
 668         char   *string = obstack_finish(&symbol_obstack);
 669         lexer_token.literal = identify_string(string, size);
 670
 671         lexer_token.type    =
 672                 is_float ? T_FLOATINGPOINT_HEXADECIMAL : T_INTEGER_HEXADECIMAL;
 673
 674         if (!has_digits) {
 675                 errorf(&lexer_token.source_position, "invalid number literal '0x%S'",
 676                        &lexer_token.literal);
 677                 lexer_token.literal.begin = "0";
 678                 lexer_token.literal.size  = 1;
 679         }
 680
 681         parse_number_suffix();
 682 }
 683
 684 /**
 685  * Returns true if the given char is a octal digit.
 686  *
 687  * @param char  the character to check
 688  */
 689 static bool is_octal_digit(utf32 chr)
 690 {
 691         return '0' <= chr && chr <= '7';
 692 }
 693
 694 /**
 695  * Parses a number and sets the lexer_token.
 696  */
 697 static void parse_number(void)
 698 {
 699         bool is_float   = false;
 700         bool has_digits = false;
 701
 702         assert(obstack_object_size(&symbol_obstack) == 0);
 703         if (c == '0') {
 704                 next_char();
 705                 if (c == 'x' || c == 'X') {
 706                         next_char();
 707                         parse_number_hex();
 708                         return;
 709                 } else {
 710                         has_digits = true;
 711                 }
 712                 obstack_1grow(&symbol_obstack, '0');
 713         }
 714
 715         while (isdigit(c)) {
 716                 has_digits = true;
 717                 obstack_1grow(&symbol_obstack, (char) c);
 718                 next_char();
 719         }
 720
 721         if (c == '.') {
 722                 is_float = true;
 723                 obstack_1grow(&symbol_obstack, '.');
 724                 next_char();
 725
 726                 while (isdigit(c)) {
 727                         has_digits = true;
 728                         obstack_1grow(&symbol_obstack, (char) c);
 729                         next_char();
 730                 }
 731         }
 732         if (c == 'e' || c == 'E') {
 733                 is_float = true;
 734                 obstack_1grow(&symbol_obstack, 'e');
 735                 next_char();
 736
 737                 if (c == '-' || c == '+') {
 738                         obstack_1grow(&symbol_obstack, (char) c);
 739                         next_char();
 740                 }
 741
 742                 while (isdigit(c)) {
 743                         obstack_1grow(&symbol_obstack, (char) c);
 744                         next_char();
 745                 }
 746         }
 747
 748         obstack_1grow(&symbol_obstack, '\0');
 749         size_t  size   = obstack_object_size(&symbol_obstack) - 1;
 750         char   *string = obstack_finish(&symbol_obstack);
 751         lexer_token.literal = identify_string(string, size);
 752
 753         /* is it an octal number? */
 754         if (is_float) {
 755                 lexer_token.type = T_FLOATINGPOINT;
 756         } else if (string[0] == '0') {
 757                 lexer_token.type = T_INTEGER_OCTAL;
 758
 759                 /* check for invalid octal digits */
 760                 for (size_t i= 0; i < size; ++i) {
 761                         char t = string[i];
 762                         if (t >= '8')
 763                                 errorf(&lexer_token.source_position,
 764                                        "invalid digit '%c' in octal number", t);
 765                 }
 766         } else {
 767                 lexer_token.type = T_INTEGER;
 768         }
 769
 770         if (!has_digits) {
 771                 errorf(&lexer_token.source_position, "invalid number literal '%S'",
 772                        &lexer_token.literal);
 773         }
 774
 775         parse_number_suffix();
 776 }
 777
 778 /**
 779  * Returns the value of a digit.
 780  * The only portable way to do it ...
 781  */
 782 static int digit_value(utf32 const digit)
 783 {
 784         switch (digit) {
 785         case '0': return 0;
 786         case '1': return 1;
 787         case '2': return 2;
 788         case '3': return 3;
 789         case '4': return 4;
 790         case '5': return 5;
 791         case '6': return 6;
 792         case '7': return 7;
 793         case '8': return 8;
 794         case '9': return 9;
 795         case 'a':
 796         case 'A': return 10;
 797         case 'b':
 798         case 'B': return 11;
 799         case 'c':
 800         case 'C': return 12;
 801         case 'd':
 802         case 'D': return 13;
 803         case 'e':
 804         case 'E': return 14;
 805         case 'f':
 806         case 'F': return 15;
 807         default:
 808                 internal_error("wrong character given");
 809         }
 810 }
 811
 812 /**
 813  * Parses an octal character sequence.
 814  *
 815  * @param first_digit  the already read first digit
 816  */
 817 static utf32 parse_octal_sequence(utf32 const first_digit)
 818 {
 819         assert(is_octal_digit(first_digit));
 820         utf32 value = digit_value(first_digit);
 821         if (!is_octal_digit(c)) return value;
 822         value = 8 * value + digit_value(c);
 823         next_char();
 824         if (!is_octal_digit(c)) return value;
 825         value = 8 * value + digit_value(c);
 826         next_char();
 827         return value;
 828 }
 829
 830 /**
 831  * Parses a hex character sequence.
 832  */
 833 static utf32 parse_hex_sequence(void)
 834 {
 835         utf32 value = 0;
 836         while (isxdigit(c)) {
 837                 value = 16 * value + digit_value(c);
 838                 next_char();
 839         }
 840         return value;
 841 }
 842
 843 /**
 844  * Parse an escape sequence.
 845  */
 846 static utf32 parse_escape_sequence(void)
 847 {
 848         eat('\\');
 849
 850         utf32 const ec = c;
 851         next_char();
 852
 853         switch (ec) {
 854         case '"':  return '"';
 855         case '\'': return '\'';
 856         case '\\': return '\\';
 857         case '?': return '\?';
 858         case 'a': return '\a';
 859         case 'b': return '\b';
 860         case 'f': return '\f';
 861         case 'n': return '\n';
 862         case 'r': return '\r';
 863         case 't': return '\t';
 864         case 'v': return '\v';
 865         case 'x':
 866                 return parse_hex_sequence();
 867         case '0':
 868         case '1':
 869         case '2':
 870         case '3':
 871         case '4':
 872         case '5':
 873         case '6':
 874         case '7':
 875                 return parse_octal_sequence(ec);
 876         case EOF:
 877                 parse_error("reached end of file while parsing escape sequence");
 878                 return EOF;
 879         /* \E is not documented, but handled, by GCC.  It is acceptable according
 880          * to §6.11.4, whereas \e is not. */
 881         case 'E':
 882         case 'e':
 883                 if (c_mode & _GNUC)
 884                         return 27;   /* hopefully 27 is ALWAYS the code for ESCAPE */
 885                 break;
 886         case 'u':
 887         case 'U':
 888                 parse_error("universal character parsing not implemented yet");
 889                 return EOF;
 890         default:
 891                 break;
 892         }
 893         /* §6.4.4.4:8 footnote 64 */
 894         parse_error("unknown escape sequence");
 895         return EOF;
 896 }
 897
 898 /**
 899  * Concatenate two strings.
 900  */
 901 string_t concat_strings(const string_t *const s1, const string_t *const s2)
 902 {
 903         const size_t len1 = s1->size - 1;
 904         const size_t len2 = s2->size - 1;
 905
 906         char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
 907         memcpy(concat, s1->begin, len1);
 908         memcpy(concat + len1, s2->begin, len2 + 1);
 909
 910         return identify_string(concat, len1 + len2 + 1);
 911 }
 912
 913 string_t make_string(const char *string)
 914 {
 915         size_t      len   = strlen(string) + 1;
 916         char *const space = obstack_alloc(&symbol_obstack, len);
 917         memcpy(space, string, len);
 918
 919         return identify_string(space, len);
 920 }
 921
 922 static void grow_symbol(utf32 const tc)
 923 {
 924         struct obstack *const o  = &symbol_obstack;
 925         if (tc < 0x80U) {
 926                 obstack_1grow(o, tc);
 927         } else if (tc < 0x800) {
 928                 obstack_1grow(o, 0xC0 | (tc >> 6));
 929                 obstack_1grow(o, 0x80 | (tc & 0x3F));
 930         } else if (tc < 0x10000) {
 931                 obstack_1grow(o, 0xE0 | ( tc >> 12));
 932                 obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
 933                 obstack_1grow(o, 0x80 | ( tc        & 0x3F));
 934         } else {
 935                 obstack_1grow(o, 0xF0 | ( tc >> 18));
 936                 obstack_1grow(o, 0x80 | ((tc >> 12) & 0x3F));
 937                 obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
 938                 obstack_1grow(o, 0x80 | ( tc        & 0x3F));
 939         }
 940 }
 941
 942 /**
 943  * Parse a string literal and set lexer_token.
 944  */
 945 static void parse_string_literal(void)
 946 {
 947         eat('"');
 948
 949         while (true) {
 950                 switch (c) {
 951                 case '\\': {
 952                         utf32 const tc = parse_escape_sequence();
 953                         if (tc >= 0x100) {
 954                                 warningf(WARN_OTHER, &lexer_pos, "escape sequence out of range");
 955                         }
 956                         obstack_1grow(&symbol_obstack, tc);
 957                         break;
 958                 }
 959
 960                 case EOF: {
 961                         errorf(&lexer_token.source_position, "string has no end");
 962                         lexer_token.type = T_ERROR;
 963                         return;
 964                 }
 965
 966                 case '"':
 967                         next_char();
 968                         goto end_of_string;
 969
 970                 default:
 971                         grow_symbol(c);
 972                         next_char();
 973                         break;
 974                 }
 975         }
 976
 977 end_of_string:
 978
 979         /* TODO: concatenate multiple strings separated by whitespace... */
 980
 981         /* add finishing 0 to the string */
 982         obstack_1grow(&symbol_obstack, '\0');
 983         const size_t  size   = (size_t)obstack_object_size(&symbol_obstack);
 984         char         *string = obstack_finish(&symbol_obstack);
 985
 986         lexer_token.type    = T_STRING_LITERAL;
 987         lexer_token.literal = identify_string(string, size);
 988 }
 989
 990 /**
 991  * Parse a wide character constant and set lexer_token.
 992  */
 993 static void parse_wide_character_constant(void)
 994 {
 995         eat('\'');
 996
 997         while (true) {
 998                 switch (c) {
 999                 case '\\': {
1000                         const utf32 tc = parse_escape_sequence();
1001                         grow_symbol(tc);
1002                         break;
1003                 }
1004
1005                 MATCH_NEWLINE(
1006                         parse_error("newline while parsing character constant");
1007                         break;
1008                 )
1009
1010                 case '\'':
1011                         next_char();
1012                         goto end_of_wide_char_constant;
1013
1014                 case EOF: {
1015                         errorf(&lexer_token.source_position, "EOF while parsing character constant");
1016                         lexer_token.type = T_ERROR;
1017                         return;
1018                 }
1019
1020                 default:
1021                         grow_symbol(c);
1022                         next_char();
1023                         break;
1024                 }
1025         }
1026
1027 end_of_wide_char_constant:;
1028         obstack_1grow(&symbol_obstack, '\0');
1029         size_t  size   = (size_t) obstack_object_size(&symbol_obstack) - 1;
1030         char   *string = obstack_finish(&symbol_obstack);
1031
1032         lexer_token.type     = T_WIDE_CHARACTER_CONSTANT;
1033         lexer_token.literal  = identify_string(string, size);
1034
1035         if (size == 0) {
1036                 errorf(&lexer_token.source_position, "empty character constant");
1037         }
1038 }
1039
1040 /**
1041  * Parse a wide string literal and set lexer_token.
1042  */
1043 static void parse_wide_string_literal(void)
1044 {
1045         parse_string_literal();
1046         if (lexer_token.type == T_STRING_LITERAL)
1047                 lexer_token.type = T_WIDE_STRING_LITERAL;
1048 }
1049
1050 /**
1051  * Parse a character constant and set lexer_token.
1052  */
1053 static void parse_character_constant(void)
1054 {
1055         eat('\'');
1056
1057         while (true) {
1058                 switch (c) {
1059                 case '\\': {
1060                         utf32 const tc = parse_escape_sequence();
1061                         if (tc >= 0x100) {
1062                                 warningf(WARN_OTHER, &lexer_pos, "escape sequence out of range");
1063                         }
1064                         obstack_1grow(&symbol_obstack, tc);
1065                         break;
1066                 }
1067
1068                 MATCH_NEWLINE(
1069                         parse_error("newline while parsing character constant");
1070                         break;
1071                 )
1072
1073                 case '\'':
1074                         next_char();
1075                         goto end_of_char_constant;
1076
1077                 case EOF: {
1078                         errorf(&lexer_token.source_position, "EOF while parsing character constant");
1079                         lexer_token.type = T_ERROR;
1080                         return;
1081                 }
1082
1083                 default:
1084                         grow_symbol(c);
1085                         next_char();
1086                         break;
1087
1088                 }
1089         }
1090
1091 end_of_char_constant:;
1092         obstack_1grow(&symbol_obstack, '\0');
1093         const size_t        size   = (size_t)obstack_object_size(&symbol_obstack)-1;
1094         char         *const string = obstack_finish(&symbol_obstack);
1095
1096         lexer_token.type    = T_CHARACTER_CONSTANT;
1097         lexer_token.literal = identify_string(string, size);
1098
1099         if (size == 0) {
1100                 errorf(&lexer_token.source_position, "empty character constant");
1101         }
1102 }
1103
1104 /**
1105  * Skip a multiline comment.
1106  */
1107 static void skip_multiline_comment(void)
1108 {
1109         while (true) {
1110                 switch (c) {
1111                 case '/':
1112                         next_char();
1113                         if (c == '*') {
1114                                 /* nested comment, warn here */
1115                                 warningf(WARN_COMMENT, &lexer_pos, "'/*' within comment");
1116                         }
1117                         break;
1118                 case '*':
1119                         next_char();
1120                         if (c == '/') {
1121                                 next_char();
1122                                 return;
1123                         }
1124                         break;
1125
1126                 MATCH_NEWLINE(break;)
1127
1128                 case EOF: {
1129                         errorf(&lexer_token.source_position, "at end of file while looking for comment end");
1130                         return;
1131                 }
1132
1133                 default:
1134                         next_char();
1135                         break;
1136                 }
1137         }
1138 }
1139
1140 /**
1141  * Skip a single line comment.
1142  */
1143 static void skip_line_comment(void)
1144 {
1145         while (true) {
1146                 switch (c) {
1147                 case EOF:
1148                         return;
1149
1150                 case '\n':
1151                 case '\r':
1152                         return;
1153
1154                 case '\\':
1155                         next_char();
1156                         if (c == '\n' || c == '\r') {
1157                                 warningf(WARN_COMMENT, &lexer_pos, "multi-line comment");
1158                                 return;
1159                         }
1160                         break;
1161
1162                 default:
1163                         next_char();
1164                         break;
1165                 }
1166         }
1167 }
1168
1169 /** The current preprocessor token. */
1170 static token_t pp_token;
1171
1172 /**
1173  * Read the next preprocessor token.
1174  */
1175 static inline void next_pp_token(void)
1176 {
1177         lexer_next_preprocessing_token();
1178         pp_token = lexer_token;
1179 }
1180
1181 /**
1182  * Eat all preprocessor tokens until newline.
1183  */
1184 static void eat_until_newline(void)
1185 {
1186         while (pp_token.type != '\n' && pp_token.type != T_EOF) {
1187                 next_pp_token();
1188         }
1189 }
1190
1191 /**
1192  * Handle the define directive.
1193  */
1194 static void define_directive(void)
1195 {
1196         lexer_next_preprocessing_token();
1197         if (lexer_token.type != T_IDENTIFIER) {
1198                 parse_error("expected identifier after #define\n");
1199                 eat_until_newline();
1200         }
1201 }
1202
1203 /**
1204  * Handle the ifdef directive.
1205  */
1206 static void ifdef_directive(int is_ifndef)
1207 {
1208         (void) is_ifndef;
1209         lexer_next_preprocessing_token();
1210         //expect_identifier();
1211         //extect_newline();
1212 }
1213
1214 /**
1215  * Handle the endif directive.
1216  */
1217 static void endif_directive(void)
1218 {
1219         //expect_newline();
1220 }
1221
1222 /**
1223  * Parse the line directive.
1224  */
1225 static void parse_line_directive(void)
1226 {
1227         if (pp_token.type != T_INTEGER) {
1228                 parse_error("expected integer");
1229         } else {
1230                 /* use offset -1 as this is about the next line */
1231                 lexer_pos.lineno = atoi(pp_token.literal.begin) - 1;
1232                 next_pp_token();
1233         }
1234         if (pp_token.type == T_STRING_LITERAL) {
1235                 lexer_pos.input_name = pp_token.literal.begin;
1236                 next_pp_token();
1237         }
1238
1239         eat_until_newline();
1240 }
1241
1242 /**
1243  * STDC pragmas.
1244  */
1245 typedef enum stdc_pragma_kind_t {
1246         STDC_UNKNOWN,
1247         STDC_FP_CONTRACT,
1248         STDC_FENV_ACCESS,
1249         STDC_CX_LIMITED_RANGE
1250 } stdc_pragma_kind_t;
1251
1252 /**
1253  * STDC pragma values.
1254  */
1255 typedef enum stdc_pragma_value_kind_t {
1256         STDC_VALUE_UNKNOWN,
1257         STDC_VALUE_ON,
1258         STDC_VALUE_OFF,
1259         STDC_VALUE_DEFAULT
1260 } stdc_pragma_value_kind_t;
1261
1262 /**
1263  * Parse a pragma directive.
1264  */
1265 static void parse_pragma(void)
1266 {
1267         bool unknown_pragma = true;
1268
1269         next_pp_token();
1270         if (pp_token.symbol->pp_ID == TP_STDC) {
1271                 stdc_pragma_kind_t kind = STDC_UNKNOWN;
1272                 /* a STDC pragma */
1273                 if (c_mode & _C99) {
1274                         next_pp_token();
1275
1276                         switch (pp_token.symbol->pp_ID) {
1277                         case TP_FP_CONTRACT:
1278                                 kind = STDC_FP_CONTRACT;
1279                                 break;
1280                         case TP_FENV_ACCESS:
1281                                 kind = STDC_FENV_ACCESS;
1282                                 break;
1283                         case TP_CX_LIMITED_RANGE:
1284                                 kind = STDC_CX_LIMITED_RANGE;
1285                                 break;
1286                         default:
1287                                 break;
1288                         }
1289                         if (kind != STDC_UNKNOWN) {
1290                                 stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN;
1291                                 next_pp_token();
1292                                 switch (pp_token.symbol->pp_ID) {
1293                                 case TP_ON:
1294                                         value = STDC_VALUE_ON;
1295                                         break;
1296                                 case TP_OFF:
1297                                         value = STDC_VALUE_OFF;
1298                                         break;
1299                                 case TP_DEFAULT:
1300                                         value = STDC_VALUE_DEFAULT;
1301                                         break;
1302                                 default:
1303                                         break;
1304                                 }
1305                                 if (value != STDC_VALUE_UNKNOWN) {
1306                                         unknown_pragma = false;
1307                                 } else {
1308                                         errorf(&pp_token.source_position, "bad STDC pragma argument");
1309                                 }
1310                         }
1311                 }
1312         } else {
1313                 unknown_pragma = true;
1314         }
1315         eat_until_newline();
1316         if (unknown_pragma) {
1317                 warningf(WARN_UNKNOWN_PRAGMAS, &pp_token.source_position, "encountered unknown #pragma");
1318         }
1319 }
1320
1321 /**
1322  * Parse a preprocessor non-null directive.
1323  */
1324 static void parse_preprocessor_identifier(void)
1325 {
1326         assert(pp_token.type == T_IDENTIFIER);
1327         symbol_t *symbol = pp_token.symbol;
1328
1329         switch (symbol->pp_ID) {
1330         case TP_include:
1331                 printf("include - enable header name parsing!\n");
1332                 break;
1333         case TP_define:
1334                 define_directive();
1335                 break;
1336         case TP_ifdef:
1337                 ifdef_directive(0);
1338                 break;
1339         case TP_ifndef:
1340                 ifdef_directive(1);
1341                 break;
1342         case TP_endif:
1343                 endif_directive();
1344                 break;
1345         case TP_line:
1346                 next_pp_token();
1347                 parse_line_directive();
1348                 break;
1349         case TP_if:
1350         case TP_else:
1351         case TP_elif:
1352         case TP_undef:
1353         case TP_error:
1354                 /* TODO; output the rest of the line */
1355                 parse_error("#error directive: ");
1356                 break;
1357         case TP_pragma:
1358                 parse_pragma();
1359                 break;
1360         }
1361 }
1362
1363 /**
1364  * Parse a preprocessor directive.
1365  */
1366 static void parse_preprocessor_directive(void)
1367 {
1368         next_pp_token();
1369
1370         switch (pp_token.type) {
1371         case T_IDENTIFIER:
1372                 parse_preprocessor_identifier();
1373                 break;
1374         case T_INTEGER:
1375                 parse_line_directive();
1376                 break;
1377         case '\n':
1378                 /* NULL directive, see §6.10.7 */
1379                 break;
1380         default:
1381                 parse_error("invalid preprocessor directive");
1382                 eat_until_newline();
1383                 break;
1384         }
1385 }
1386
1387 #define MAYBE_PROLOG                                       \
1388                         next_char();                                   \
1389                         while (true) {                                 \
1390                                 switch (c) {
1391
1392 #define MAYBE(ch, set_type)                                \
1393                                 case ch:                                   \
1394                                         next_char();                           \
1395                                         lexer_token.type = set_type;           \
1396                                         return;
1397
1398 /* must use this as last thing */
1399 #define MAYBE_MODE(ch, set_type, mode)                     \
1400                                 case ch:                                   \
1401                                         if (c_mode & mode) {                   \
1402                                                 next_char();                       \
1403                                                 lexer_token.type = set_type;       \
1404                                                 return;                            \
1405                                         }                                      \
1406                                         /* fallthrough */
1407
1408 #define ELSE_CODE(code)                                    \
1409                                 default:                                   \
1410                                         code                                   \
1411                                         return;                                \
1412                                 }                                          \
1413                         } /* end of while (true) */                    \
1414
1415 #define ELSE(set_type)                                     \
1416                 ELSE_CODE(                                         \
1417                         lexer_token.type = set_type;                   \
1418                 )
1419
1420 void lexer_next_preprocessing_token(void)
1421 {
1422         while (true) {
1423                 lexer_token.source_position = lexer_pos;
1424
1425                 switch (c) {
1426                 case ' ':
1427                 case '\t':
1428                         next_char();
1429                         break;
1430
1431                 MATCH_NEWLINE(
1432                         lexer_token.type = '\n';
1433                         return;
1434                 )
1435
1436                 SYMBOL_CHARS
1437                         parse_symbol();
1438                         /* might be a wide string ( L"string" ) */
1439                         if (lexer_token.symbol == symbol_L) {
1440                                 switch (c) {
1441                                         case '"':  parse_wide_string_literal();     break;
1442                                         case '\'': parse_wide_character_constant(); break;
1443                                 }
1444                         }
1445                         return;
1446
1447                 DIGITS
1448                         parse_number();
1449                         return;
1450
1451                 case '"':
1452                         parse_string_literal();
1453                         return;
1454
1455                 case '\'':
1456                         parse_character_constant();
1457                         return;
1458
1459                 case '.':
1460                         MAYBE_PROLOG
1461                                 DIGITS
1462                                         put_back(c);
1463                                         c = '.';
1464                                         parse_number();
1465                                         return;
1466
1467                                 case '.':
1468                                         MAYBE_PROLOG
1469                                         MAYBE('.', T_DOTDOTDOT)
1470                                         ELSE_CODE(
1471                                                 put_back(c);
1472                                                 c = '.';
1473                                                 lexer_token.type = '.';
1474                                         )
1475                         ELSE('.')
1476                 case '&':
1477                         MAYBE_PROLOG
1478                         MAYBE('&', T_ANDAND)
1479                         MAYBE('=', T_ANDEQUAL)
1480                         ELSE('&')
1481                 case '*':
1482                         MAYBE_PROLOG
1483                         MAYBE('=', T_ASTERISKEQUAL)
1484                         ELSE('*')
1485                 case '+':
1486                         MAYBE_PROLOG
1487                         MAYBE('+', T_PLUSPLUS)
1488                         MAYBE('=', T_PLUSEQUAL)
1489                         ELSE('+')
1490                 case '-':
1491                         MAYBE_PROLOG
1492                         MAYBE('>', T_MINUSGREATER)
1493                         MAYBE('-', T_MINUSMINUS)
1494                         MAYBE('=', T_MINUSEQUAL)
1495                         ELSE('-')
1496                 case '!':
1497                         MAYBE_PROLOG
1498                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1499                         ELSE('!')
1500                 case '/':
1501                         MAYBE_PROLOG
1502                         MAYBE('=', T_SLASHEQUAL)
1503                                 case '*':
1504                                         next_char();
1505                                         skip_multiline_comment();
1506                                         lexer_next_preprocessing_token();
1507                                         return;
1508                                 case '/':
1509                                         next_char();
1510                                         skip_line_comment();
1511                                         lexer_next_preprocessing_token();
1512                                         return;
1513                         ELSE('/')
1514                 case '%':
1515                         MAYBE_PROLOG
1516                         MAYBE('>', '}')
1517                         MAYBE('=', T_PERCENTEQUAL)
1518                                 case ':':
1519                                         MAYBE_PROLOG
1520                                                 case '%':
1521                                                         MAYBE_PROLOG
1522                                                         MAYBE(':', T_HASHHASH)
1523                                                         ELSE_CODE(
1524                                                                 put_back(c);
1525                                                                 c = '%';
1526                                                                 lexer_token.type = '#';
1527                                                         )
1528                                         ELSE('#')
1529                         ELSE('%')
1530                 case '<':
1531                         MAYBE_PROLOG
1532                         MAYBE(':', '[')
1533                         MAYBE('%', '{')
1534                         MAYBE('=', T_LESSEQUAL)
1535                                 case '<':
1536                                         MAYBE_PROLOG
1537                                         MAYBE('=', T_LESSLESSEQUAL)
1538                                         ELSE(T_LESSLESS)
1539                         ELSE('<')
1540                 case '>':
1541                         MAYBE_PROLOG
1542                         MAYBE('=', T_GREATEREQUAL)
1543                                 case '>':
1544                                         MAYBE_PROLOG
1545                                         MAYBE('=', T_GREATERGREATEREQUAL)
1546                                         ELSE(T_GREATERGREATER)
1547                         ELSE('>')
1548                 case '^':
1549                         MAYBE_PROLOG
1550                         MAYBE('=', T_CARETEQUAL)
1551                         ELSE('^')
1552                 case '|':
1553                         MAYBE_PROLOG
1554                         MAYBE('=', T_PIPEEQUAL)
1555                         MAYBE('|', T_PIPEPIPE)
1556                         ELSE('|')
1557                 case ':':
1558                         MAYBE_PROLOG
1559                         MAYBE('>', ']')
1560                         MAYBE_MODE(':', T_COLONCOLON, _CXX)
1561                         ELSE(':')
1562                 case '=':
1563                         MAYBE_PROLOG
1564                         MAYBE('=', T_EQUALEQUAL)
1565                         ELSE('=')
1566                 case '#':
1567                         MAYBE_PROLOG
1568                         MAYBE('#', T_HASHHASH)
1569                         ELSE('#')
1570
1571                 case '?':
1572                 case '[':
1573                 case ']':
1574                 case '(':
1575                 case ')':
1576                 case '{':
1577                 case '}':
1578                 case '~':
1579                 case ';':
1580                 case ',':
1581                 case '\\':
1582                         lexer_token.type = c;
1583                         next_char();
1584                         return;
1585
1586                 case EOF:
1587                         lexer_token.type = T_EOF;
1588                         return;
1589
1590                 default:
1591 dollar_sign:
1592                         errorf(&lexer_pos, "unknown character '%c' found", c);
1593                         next_char();
1594                         lexer_token.type = T_ERROR;
1595                         return;
1596                 }
1597         }
1598 }
1599
1600 void lexer_next_token(void)
1601 {
1602         lexer_next_preprocessing_token();
1603
1604         while (lexer_token.type == '\n') {
1605 newline_found:
1606                 lexer_next_preprocessing_token();
1607         }
1608
1609         if (lexer_token.type == '#') {
1610                 parse_preprocessor_directive();
1611                 goto newline_found;
1612         }
1613 }
1614
1615 void init_lexer(void)
1616 {
1617         strset_init(&stringset);
1618         symbol_L = symbol_table_insert("L");
1619 }
1620
1621 void lexer_open_stream(FILE *stream, const char *input_name)
1622 {
1623         input                = stream;
1624         lexer_pos.lineno     = 0;
1625         lexer_pos.colno      = 0;
1626         lexer_pos.input_name = input_name;
1627
1628         bufpos = NULL;
1629         bufend = NULL;
1630
1631         /* place a virtual \n at the beginning so the lexer knows that we're
1632          * at the beginning of a line */
1633         c = '\n';
1634 }
1635
1636 void lexer_open_buffer(const char *buffer, size_t len, const char *input_name)
1637 {
1638         input                = NULL;
1639         lexer_pos.lineno     = 0;
1640         lexer_pos.colno      = 0;
1641         lexer_pos.input_name = input_name;
1642
1643 #if 0 // TODO
1644         bufpos = buffer;
1645         bufend = buffer + len;
1646
1647         /* place a virtual \n at the beginning so the lexer knows that we're
1648          * at the beginning of a line */
1649         c = '\n';
1650 #else
1651         (void)buffer;
1652         (void)len;
1653         panic("builtin lexing not done yet");
1654 #endif
1655 }
1656
1657 void exit_lexer(void)
1658 {
1659         strset_destroy(&stringset);
1660 }
1661
1662 static __attribute__((unused))
1663 void dbg_pos(const source_position_t source_position)
1664 {
1665         fprintf(stdout, "%s:%u:%u\n", source_position.input_name,
1666                 source_position.lineno, source_position.colno);
1667         fflush(stdout);
1668 }