nsz Git - cparser/blob - lexer.c

   1 /*
   2  * This file is part of cparser.
   3  * Copyright (C) 2007-2009 Matthias Braun <matze@braunis.de>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; either version 2
   8  * of the License, or (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  18  * 02111-1307, USA.
  19  */
  20 #include <config.h>
  21
  22 #include "diagnostic.h"
  23 #include "lexer.h"
  24 #include "symbol_t.h"
  25 #include "token_t.h"
  26 #include "symbol_table_t.h"
  27 #include "adt/error.h"
  28 #include "adt/strset.h"
  29 #include "adt/util.h"
  30 #include "types.h"
  31 #include "type_t.h"
  32 #include "target_architecture.h"
  33 #include "parser.h"
  34 #include "warning.h"
  35 #include "lang_features.h"
  36
  37 #include <assert.h>
  38 #include <errno.h>
  39 #include <string.h>
  40 #include <stdbool.h>
  41 #include <ctype.h>
  42
  43 #ifndef _WIN32
  44 #include <strings.h>
  45 #endif
  46
  47 //#define DEBUG_CHARS
  48 #define MAX_PUTBACK 3
  49 #define BUF_SIZE    1024
  50
  51 #if defined(_WIN32) || defined(__CYGWIN__)
  52 /* No strtold on windows and no replacement yet */
  53 #define strtold(s, e) strtod(s, e)
  54 #endif
  55
  56 static utf32             c;
  57 static source_position_t lexer_pos;
  58 token_t                  lexer_token;
  59 symbol_t                *symbol_L;
  60 static FILE             *input;
  61 static utf32             buf[BUF_SIZE + MAX_PUTBACK];
  62 static const utf32      *bufend;
  63 static const utf32      *bufpos;
  64 static strset_t          stringset;
  65 bool                     allow_dollar_in_symbol = true;
  66
  67 /**
  68  * Prints a parse error message at the current token.
  69  *
  70  * @param msg   the error message
  71  */
  72 static void parse_error(const char *msg)
  73 {
  74         errorf(&lexer_pos, "%s", msg);
  75 }
  76
  77 /**
  78  * Prints an internal error message at the current token.
  79  *
  80  * @param msg   the error message
  81  */
  82 static NORETURN internal_error(const char *msg)
  83 {
  84         internal_errorf(&lexer_pos, "%s", msg);
  85 }
  86
  87 static size_t read_block(unsigned char *const read_buf, size_t const n)
  88 {
  89         size_t const s = fread(read_buf, 1, n, input);
  90         if (s == 0) {
  91                 /* on OS/X ferror appears to return true on eof as well when running
  92                  * the application in gdb... */
  93                 if (!feof(input) && ferror(input))
  94                         parse_error("read from input failed");
  95                 buf[MAX_PUTBACK] = EOF;
  96                 bufpos           = buf + MAX_PUTBACK;
  97                 bufend           = buf + MAX_PUTBACK + 1;
  98         }
  99         return s;
 100 }
 101
 102 static void decode_iso_8859_1(void)
 103 {
 104         unsigned char read_buf[BUF_SIZE];
 105         size_t const s = read_block(read_buf, sizeof(read_buf));
 106         if (s == 0)
 107                 return;
 108
 109         unsigned char const *src = read_buf;
 110         unsigned char const *end = read_buf + s;
 111         utf32               *dst = buf + MAX_PUTBACK;
 112         while (src != end)
 113                 *dst++ = *src++;
 114
 115         bufpos = buf + MAX_PUTBACK;
 116         bufend = dst;
 117 }
 118
 119 static void decode_iso_8859_15(void)
 120 {
 121         unsigned char read_buf[BUF_SIZE];
 122         size_t const s = read_block(read_buf, sizeof(read_buf));
 123         if (s == 0)
 124                 return;
 125
 126         unsigned char const *src = read_buf;
 127         unsigned char const *end = read_buf + s;
 128         utf32               *dst = buf + MAX_PUTBACK;
 129         while (src != end) {
 130                 utf32 tc = *src++;
 131                 switch (tc) {
 132                         case 0xA4: tc = 0x20AC; break; // €
 133                         case 0xA6: tc = 0x0160; break; // Š
 134                         case 0xA8: tc = 0x0161; break; // š
 135                         case 0xB4: tc = 0x017D; break; // Ž
 136                         case 0xB8: tc = 0x017E; break; // ž
 137                         case 0xBC: tc = 0x0152; break; // Œ
 138                         case 0xBD: tc = 0x0153; break; // œ
 139                         case 0xBE: tc = 0x0178; break; // Ÿ
 140                 }
 141                 *dst++ = tc;
 142         }
 143
 144         bufpos = buf + MAX_PUTBACK;
 145         bufend = dst;
 146 }
 147
 148 static void decode_utf8(void)
 149 {
 150         static utf32  part_decoded_min_code;
 151         static utf32  part_decoded_char;
 152         static size_t part_decoded_rest_len;
 153
 154         do {
 155                 unsigned char read_buf[BUF_SIZE];
 156                 size_t const s = read_block(read_buf, sizeof(read_buf));
 157                 if (s == 0) {
 158                         if (part_decoded_rest_len > 0)
 159                                 parse_error("incomplete input char at end of input");
 160                         return;
 161                 }
 162
 163                 unsigned char const *src = read_buf;
 164                 unsigned char const *end = read_buf + s;
 165                 utf32               *dst = buf + MAX_PUTBACK;
 166                 utf32                decoded;
 167                 utf32                min_code;
 168
 169                 if (part_decoded_rest_len != 0) {
 170                         min_code              = part_decoded_min_code;
 171                         decoded               = part_decoded_char;
 172                         size_t const rest_len = part_decoded_rest_len;
 173                         part_decoded_rest_len = 0;
 174                         switch (rest_len) {
 175                                 case 4:  goto realign;
 176                                 case 3:  goto three_more;
 177                                 case 2:  goto two_more;
 178                                 default: goto one_more;
 179                         }
 180                 }
 181
 182                 while (src != end) {
 183                         if ((*src & 0x80) == 0) {
 184                                 decoded = *src++;
 185                         } else if ((*src & 0xE0) == 0xC0) {
 186                                 min_code = 0x80;
 187                                 decoded  = *src++ & 0x1F;
 188 one_more:
 189                                 if (src == end) {
 190                                         part_decoded_min_code = min_code;
 191                                         part_decoded_char     = decoded;
 192                                         part_decoded_rest_len = 1;
 193                                         break;
 194                                 }
 195                                 if ((*src & 0xC0) == 0x80) {
 196                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 197                                 } else {
 198                                         goto invalid_char;
 199                                 }
 200                                 if (decoded < min_code                      ||
 201                                                 decoded > 0x10FFFF                      ||
 202                                                 (0xD800 <= decoded && decoded < 0xE000) || // high/low surrogates
 203                                                 (0xFDD0 <= decoded && decoded < 0xFDF0) || // noncharacters
 204                                                 (decoded & 0xFFFE) == 0xFFFE) {            // noncharacters
 205                                         parse_error("invalid byte sequence in input");
 206                                 }
 207                         } else if ((*src & 0xF0) == 0xE0) {
 208                                 min_code = 0x800;
 209                                 decoded  = *src++ & 0x0F;
 210 two_more:
 211                                 if (src == end) {
 212                                         part_decoded_min_code = min_code;
 213                                         part_decoded_char     = decoded;
 214                                         part_decoded_rest_len = 2;
 215                                         break;
 216                                 }
 217                                 if ((*src & 0xC0) == 0x80) {
 218                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 219                                 } else {
 220                                         goto invalid_char;
 221                                 }
 222                                 goto one_more;
 223                         } else if ((*src & 0xF8) == 0xF0) {
 224                                 min_code = 0x10000;
 225                                 decoded  = *src++ & 0x07;
 226 three_more:
 227                                 if (src == end) {
 228                                         part_decoded_min_code = min_code;
 229                                         part_decoded_char     = decoded;
 230                                         part_decoded_rest_len = 3;
 231                                         break;
 232                                 }
 233                                 if ((*src & 0xC0) == 0x80) {
 234                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 235                                 } else {
 236                                         goto invalid_char;
 237                                 }
 238                                 goto two_more;
 239                         } else {
 240 invalid_char:
 241                                 parse_error("invalid byte sequence in input");
 242 realign:
 243                                 do {
 244                                         ++src;
 245                                         if (src == end) {
 246                                                 part_decoded_rest_len = 4;
 247                                                 break;
 248                                         }
 249                                 } while ((*src & 0xC0) == 0x80 || (*src & 0xF8) == 0xF8);
 250                                 continue;
 251                         }
 252                         *dst++ = decoded;
 253                 }
 254
 255                 bufpos = buf + MAX_PUTBACK;
 256                 bufend = dst;
 257         } while (bufpos == bufend);
 258 }
 259
 260 static void decode_windows_1252(void)
 261 {
 262         unsigned char read_buf[BUF_SIZE];
 263         size_t const s = read_block(read_buf, sizeof(read_buf));
 264         if (s == 0)
 265                 return;
 266
 267         unsigned char const *src = read_buf;
 268         unsigned char const *end = read_buf + s;
 269         utf32               *dst = buf + MAX_PUTBACK;
 270         while (src != end) {
 271                 utf32 tc = *src++;
 272                 switch (tc) {
 273                         case 0x80: tc = 0x20AC; break; // €
 274                         case 0x82: tc = 0x201A; break; // ‚
 275                         case 0x83: tc = 0x0192; break; // ƒ
 276                         case 0x84: tc = 0x201E; break; // „
 277                         case 0x85: tc = 0x2026; break; // …
 278                         case 0x86: tc = 0x2020; break; // †
 279                         case 0x87: tc = 0x2021; break; // ‡
 280                         case 0x88: tc = 0x02C6; break; // ˆ
 281                         case 0x89: tc = 0x2030; break; // ‰
 282                         case 0x8A: tc = 0x0160; break; // Š
 283                         case 0x8B: tc = 0x2039; break; // ‹
 284                         case 0x8C: tc = 0x0152; break; // Œ
 285                         case 0x8E: tc = 0x017D; break; // Ž
 286                         case 0x91: tc = 0x2018; break; // ‘
 287                         case 0x92: tc = 0x2019; break; // ’
 288                         case 0x93: tc = 0x201C; break; // “
 289                         case 0x94: tc = 0x201D; break; // ”
 290                         case 0x95: tc = 0x2022; break; // •
 291                         case 0x96: tc = 0x2013; break; // –
 292                         case 0x97: tc = 0x2014; break; // —
 293                         case 0x98: tc = 0x02DC; break; // ˜
 294                         case 0x99: tc = 0x2122; break; // ™
 295                         case 0x9A: tc = 0x0161; break; // š
 296                         case 0x9B: tc = 0x203A; break; // ›
 297                         case 0x9C: tc = 0x0153; break; // œ
 298                         case 0x9E: tc = 0x017E; break; // ž
 299                         case 0x9F: tc = 0x0178; break; // Ÿ
 300                 }
 301                 *dst++ = tc;
 302         }
 303
 304         bufpos = buf + MAX_PUTBACK;
 305         bufend = dst;
 306 }
 307
 308 typedef void (*decoder_t)(void);
 309
 310 static decoder_t decoder = decode_utf8;
 311
 312 typedef struct named_decoder_t {
 313         char const *name;
 314         decoder_t   decoder;
 315 } named_decoder_t;
 316
 317 static named_decoder_t const decoders[] = {
 318         { "CP819",           decode_iso_8859_1   }, // offical alias
 319         { "IBM819",          decode_iso_8859_1   }, // offical alias
 320         { "ISO-8859-1",      decode_iso_8859_1   }, // offical alias
 321         { "ISO-8859-15",     decode_iso_8859_15  }, // offical name
 322         { "ISO8859-1",       decode_iso_8859_1   },
 323         { "ISO8859-15",      decode_iso_8859_15  },
 324         { "ISO_8859-1",      decode_iso_8859_1   }, // offical alias
 325         { "ISO_8859-15",     decode_iso_8859_15  }, // offical alias
 326         { "ISO_8859-1:1987", decode_iso_8859_1   }, // offical name
 327         { "Latin-9",         decode_iso_8859_15  }, // offical alias
 328         { "UTF-8",           decode_utf8         }, // offical name
 329         { "csISOLatin1",     decode_iso_8859_1   }, // offical alias
 330         { "cp1252",          decode_windows_1252 },
 331         { "iso-ir-100",      decode_iso_8859_1   }, // offical alias
 332         { "l1",              decode_iso_8859_1   }, // offical alias
 333         { "latin1",          decode_iso_8859_1   }, // offical alias
 334         { "windows-1252",    decode_windows_1252 }, // official name
 335
 336         { NULL,              NULL                }
 337 };
 338
 339 /** strcasecmp is not part of C99 so we need our own implementation here */
 340 static int my_strcasecmp(const char *s1, const char *s2)
 341 {
 342         for ( ; *s1 != 0; ++s1, ++s2) {
 343                 if (tolower(*s1) != tolower(*s2))
 344                         break;
 345         }
 346         return (unsigned char)*s1 - (unsigned char)*s2;
 347 }
 348
 349 void select_input_encoding(char const* const encoding)
 350 {
 351         for (named_decoder_t const *i = decoders; i->name != NULL; ++i) {
 352                 if (my_strcasecmp(encoding, i->name) != 0)
 353                         continue;
 354                 decoder = i->decoder;
 355                 return;
 356         }
 357         fprintf(stderr, "error: input encoding \"%s\" not supported\n", encoding);
 358 }
 359
 360 static inline void next_real_char(void)
 361 {
 362         assert(bufpos <= bufend);
 363         if (bufpos >= bufend) {
 364                 if (input == NULL) {
 365                         c = EOF;
 366                         return;
 367                 }
 368                 decoder();
 369         }
 370         c = *bufpos++;
 371         ++lexer_pos.colno;
 372 }
 373
 374 /**
 375  * Put a character back into the buffer.
 376  *
 377  * @param pc  the character to put back
 378  */
 379 static inline void put_back(utf32 const pc)
 380 {
 381         assert(bufpos > buf);
 382         *(--bufpos - buf + buf) = pc;
 383         --lexer_pos.colno;
 384
 385 #ifdef DEBUG_CHARS
 386         printf("putback '%lc'\n", pc);
 387 #endif
 388 }
 389
 390 static inline void next_char(void);
 391
 392 #define MATCH_NEWLINE(code)  \
 393         case '\r':               \
 394                 next_char();         \
 395                 if (c == '\n') {     \
 396         case '\n':               \
 397                         next_char();     \
 398                 }                    \
 399                 lexer_pos.lineno++;  \
 400                 lexer_pos.colno = 1; \
 401                 code
 402
 403 #define eat(c_type)  do { assert(c == c_type); next_char(); } while (0)
 404
 405 static void maybe_concat_lines(void)
 406 {
 407         eat('\\');
 408
 409         switch (c) {
 410         MATCH_NEWLINE(return;)
 411
 412         default:
 413                 break;
 414         }
 415
 416         put_back(c);
 417         c = '\\';
 418 }
 419
 420 /**
 421  * Set c to the next input character, ie.
 422  * after expanding trigraphs.
 423  */
 424 static inline void next_char(void)
 425 {
 426         next_real_char();
 427
 428         /* filter trigraphs */
 429         if (UNLIKELY(c == '\\')) {
 430                 maybe_concat_lines();
 431                 goto end_of_next_char;
 432         }
 433
 434         if (LIKELY(c != '?'))
 435                 goto end_of_next_char;
 436
 437         next_real_char();
 438         if (LIKELY(c != '?')) {
 439                 put_back(c);
 440                 c = '?';
 441                 goto end_of_next_char;
 442         }
 443
 444         next_real_char();
 445         switch (c) {
 446         case '=': c = '#'; break;
 447         case '(': c = '['; break;
 448         case '/': c = '\\'; maybe_concat_lines(); break;
 449         case ')': c = ']'; break;
 450         case '\'': c = '^'; break;
 451         case '<': c = '{'; break;
 452         case '!': c = '|'; break;
 453         case '>': c = '}'; break;
 454         case '-': c = '~'; break;
 455         default:
 456                 put_back(c);
 457                 put_back('?');
 458                 c = '?';
 459                 break;
 460         }
 461
 462 end_of_next_char:;
 463 #ifdef DEBUG_CHARS
 464         printf("nchar '%c'\n", c);
 465 #endif
 466 }
 467
 468 #define SYMBOL_CHARS  \
 469         case '$': if (!allow_dollar_in_symbol) goto dollar_sign; \
 470         case 'a':         \
 471         case 'b':         \
 472         case 'c':         \
 473         case 'd':         \
 474         case 'e':         \
 475         case 'f':         \
 476         case 'g':         \
 477         case 'h':         \
 478         case 'i':         \
 479         case 'j':         \
 480         case 'k':         \
 481         case 'l':         \
 482         case 'm':         \
 483         case 'n':         \
 484         case 'o':         \
 485         case 'p':         \
 486         case 'q':         \
 487         case 'r':         \
 488         case 's':         \
 489         case 't':         \
 490         case 'u':         \
 491         case 'v':         \
 492         case 'w':         \
 493         case 'x':         \
 494         case 'y':         \
 495         case 'z':         \
 496         case 'A':         \
 497         case 'B':         \
 498         case 'C':         \
 499         case 'D':         \
 500         case 'E':         \
 501         case 'F':         \
 502         case 'G':         \
 503         case 'H':         \
 504         case 'I':         \
 505         case 'J':         \
 506         case 'K':         \
 507         case 'L':         \
 508         case 'M':         \
 509         case 'N':         \
 510         case 'O':         \
 511         case 'P':         \
 512         case 'Q':         \
 513         case 'R':         \
 514         case 'S':         \
 515         case 'T':         \
 516         case 'U':         \
 517         case 'V':         \
 518         case 'W':         \
 519         case 'X':         \
 520         case 'Y':         \
 521         case 'Z':         \
 522         case '_':
 523
 524 #define DIGITS        \
 525         case '0':         \
 526         case '1':         \
 527         case '2':         \
 528         case '3':         \
 529         case '4':         \
 530         case '5':         \
 531         case '6':         \
 532         case '7':         \
 533         case '8':         \
 534         case '9':
 535
 536 /**
 537  * Read a symbol from the input and build
 538  * the lexer_token.
 539  */
 540 static void parse_symbol(void)
 541 {
 542         obstack_1grow(&symbol_obstack, (char) c);
 543         next_char();
 544
 545         while (true) {
 546                 switch (c) {
 547                 DIGITS
 548                 SYMBOL_CHARS
 549                         obstack_1grow(&symbol_obstack, (char) c);
 550                         next_char();
 551                         break;
 552
 553                 default:
 554 dollar_sign:
 555                         goto end_symbol;
 556                 }
 557         }
 558
 559 end_symbol:
 560         obstack_1grow(&symbol_obstack, '\0');
 561
 562         char     *string = obstack_finish(&symbol_obstack);
 563         symbol_t *symbol = symbol_table_insert(string);
 564
 565         lexer_token.type   = symbol->ID;
 566         lexer_token.symbol = symbol;
 567
 568         if (symbol->string != string) {
 569                 obstack_free(&symbol_obstack, string);
 570         }
 571 }
 572
 573 /**
 574  * parse suffixes like 'LU' or 'f' after numbers
 575  */
 576 static void parse_number_suffix(void)
 577 {
 578         assert(obstack_object_size(&symbol_obstack) == 0);
 579         while (true) {
 580                 switch (c) {
 581                 SYMBOL_CHARS
 582                         obstack_1grow(&symbol_obstack, (char) c);
 583                         next_char();
 584                         break;
 585                 default:
 586                 dollar_sign:
 587                         goto finish_suffix;
 588                 }
 589         }
 590 finish_suffix:
 591         if (obstack_object_size(&symbol_obstack) == 0) {
 592                 lexer_token.symbol = NULL;
 593                 return;
 594         }
 595
 596         obstack_1grow(&symbol_obstack, '\0');
 597         char     *string = obstack_finish(&symbol_obstack);
 598         symbol_t *symbol = symbol_table_insert(string);
 599
 600         if (symbol->string != string) {
 601                 obstack_free(&symbol_obstack, string);
 602         }
 603         lexer_token.symbol = symbol;
 604 }
 605
 606 static string_t identify_string(char *string, size_t len)
 607 {
 608         /* TODO hash */
 609 #if 0
 610         const char *result = strset_insert(&stringset, concat);
 611         if (result != concat) {
 612                 obstack_free(&symbol_obstack, concat);
 613         }
 614 #else
 615         const char *result = string;
 616 #endif
 617         return (string_t) {result, len};
 618 }
 619
 620 /**
 621  * Parses a hex number including hex floats and set the
 622  * lexer_token.
 623  */
 624 static void parse_number_hex(void)
 625 {
 626         bool is_float   = false;
 627         bool has_digits = false;
 628
 629         assert(obstack_object_size(&symbol_obstack) == 0);
 630         while (isxdigit(c)) {
 631                 has_digits = true;
 632                 obstack_1grow(&symbol_obstack, (char) c);
 633                 next_char();
 634         }
 635
 636         if (c == '.') {
 637                 is_float = true;
 638                 obstack_1grow(&symbol_obstack, (char) c);
 639                 next_char();
 640
 641                 while (isxdigit(c)) {
 642                         has_digits = true;
 643                         obstack_1grow(&symbol_obstack, (char) c);
 644                         next_char();
 645                 }
 646         }
 647         if (c == 'p' || c == 'P') {
 648                 is_float = true;
 649                 obstack_1grow(&symbol_obstack, (char) c);
 650                 next_char();
 651
 652                 if (c == '-' || c == '+') {
 653                         obstack_1grow(&symbol_obstack, (char) c);
 654                         next_char();
 655                 }
 656
 657                 while (isxdigit(c)) {
 658                         obstack_1grow(&symbol_obstack, (char) c);
 659                         next_char();
 660                 }
 661         } else if (is_float) {
 662                 errorf(&lexer_token.source_position,
 663                        "hexadecimal floatingpoint constant requires an exponent");
 664         }
 665         obstack_1grow(&symbol_obstack, '\0');
 666
 667         size_t  size   = obstack_object_size(&symbol_obstack) - 1;
 668         char   *string = obstack_finish(&symbol_obstack);
 669         lexer_token.literal = identify_string(string, size);
 670
 671         lexer_token.type    =
 672                 is_float ? T_FLOATINGPOINT_HEXADECIMAL : T_INTEGER_HEXADECIMAL;
 673
 674         if (!has_digits) {
 675                 errorf(&lexer_token.source_position, "invalid number literal '0x%S'",
 676                        &lexer_token.literal);
 677                 lexer_token.literal.begin = "0";
 678                 lexer_token.literal.size  = 1;
 679         }
 680
 681         parse_number_suffix();
 682 }
 683
 684 /**
 685  * Returns true if the given char is a octal digit.
 686  *
 687  * @param char  the character to check
 688  */
 689 static bool is_octal_digit(utf32 chr)
 690 {
 691         switch (chr) {
 692         case '0':
 693         case '1':
 694         case '2':
 695         case '3':
 696         case '4':
 697         case '5':
 698         case '6':
 699         case '7':
 700                 return true;
 701         default:
 702                 return false;
 703         }
 704 }
 705
 706 /**
 707  * Parses a number and sets the lexer_token.
 708  */
 709 static void parse_number(void)
 710 {
 711         bool is_float   = false;
 712         bool has_digits = false;
 713
 714         assert(obstack_object_size(&symbol_obstack) == 0);
 715         if (c == '0') {
 716                 next_char();
 717                 if (c == 'x' || c == 'X') {
 718                         next_char();
 719                         parse_number_hex();
 720                         return;
 721                 } else {
 722                         has_digits = true;
 723                 }
 724                 obstack_1grow(&symbol_obstack, '0');
 725         }
 726
 727         while (isdigit(c)) {
 728                 has_digits = true;
 729                 obstack_1grow(&symbol_obstack, (char) c);
 730                 next_char();
 731         }
 732
 733         if (c == '.') {
 734                 is_float = true;
 735                 obstack_1grow(&symbol_obstack, '.');
 736                 next_char();
 737
 738                 while (isdigit(c)) {
 739                         has_digits = true;
 740                         obstack_1grow(&symbol_obstack, (char) c);
 741                         next_char();
 742                 }
 743         }
 744         if (c == 'e' || c == 'E') {
 745                 is_float = true;
 746                 obstack_1grow(&symbol_obstack, 'e');
 747                 next_char();
 748
 749                 if (c == '-' || c == '+') {
 750                         obstack_1grow(&symbol_obstack, (char) c);
 751                         next_char();
 752                 }
 753
 754                 while (isdigit(c)) {
 755                         obstack_1grow(&symbol_obstack, (char) c);
 756                         next_char();
 757                 }
 758         }
 759
 760         obstack_1grow(&symbol_obstack, '\0');
 761         size_t  size   = obstack_object_size(&symbol_obstack) - 1;
 762         char   *string = obstack_finish(&symbol_obstack);
 763         lexer_token.literal = identify_string(string, size);
 764
 765         /* is it an octal number? */
 766         if (is_float) {
 767                 lexer_token.type = T_FLOATINGPOINT;
 768         } else if (string[0] == '0') {
 769                 lexer_token.type = T_INTEGER_OCTAL;
 770
 771                 /* check for invalid octal digits */
 772                 for (size_t i= 0; i < size; ++i) {
 773                         char t = string[i];
 774                         if (t >= '8')
 775                                 errorf(&lexer_token.source_position,
 776                                        "invalid digit '%c' in octal number", t);
 777                 }
 778         } else {
 779                 lexer_token.type = T_INTEGER;
 780         }
 781
 782         if (!has_digits) {
 783                 errorf(&lexer_token.source_position, "invalid number literal '%S'",
 784                        &lexer_token.literal);
 785         }
 786
 787         parse_number_suffix();
 788 }
 789
 790 /**
 791  * Returns the value of a digit.
 792  * The only portable way to do it ...
 793  */
 794 static int digit_value(utf32 const digit)
 795 {
 796         switch (digit) {
 797         case '0': return 0;
 798         case '1': return 1;
 799         case '2': return 2;
 800         case '3': return 3;
 801         case '4': return 4;
 802         case '5': return 5;
 803         case '6': return 6;
 804         case '7': return 7;
 805         case '8': return 8;
 806         case '9': return 9;
 807         case 'a':
 808         case 'A': return 10;
 809         case 'b':
 810         case 'B': return 11;
 811         case 'c':
 812         case 'C': return 12;
 813         case 'd':
 814         case 'D': return 13;
 815         case 'e':
 816         case 'E': return 14;
 817         case 'f':
 818         case 'F': return 15;
 819         default:
 820                 internal_error("wrong character given");
 821         }
 822 }
 823
 824 /**
 825  * Parses an octal character sequence.
 826  *
 827  * @param first_digit  the already read first digit
 828  */
 829 static utf32 parse_octal_sequence(utf32 const first_digit)
 830 {
 831         assert(is_octal_digit(first_digit));
 832         utf32 value = digit_value(first_digit);
 833         if (!is_octal_digit(c)) return value;
 834         value = 8 * value + digit_value(c);
 835         next_char();
 836         if (!is_octal_digit(c)) return value;
 837         value = 8 * value + digit_value(c);
 838         next_char();
 839         return value;
 840 }
 841
 842 /**
 843  * Parses a hex character sequence.
 844  */
 845 static utf32 parse_hex_sequence(void)
 846 {
 847         utf32 value = 0;
 848         while (isxdigit(c)) {
 849                 value = 16 * value + digit_value(c);
 850                 next_char();
 851         }
 852         return value;
 853 }
 854
 855 /**
 856  * Parse an escape sequence.
 857  */
 858 static utf32 parse_escape_sequence(void)
 859 {
 860         eat('\\');
 861
 862         utf32 const ec = c;
 863         next_char();
 864
 865         switch (ec) {
 866         case '"':  return '"';
 867         case '\'': return '\'';
 868         case '\\': return '\\';
 869         case '?': return '\?';
 870         case 'a': return '\a';
 871         case 'b': return '\b';
 872         case 'f': return '\f';
 873         case 'n': return '\n';
 874         case 'r': return '\r';
 875         case 't': return '\t';
 876         case 'v': return '\v';
 877         case 'x':
 878                 return parse_hex_sequence();
 879         case '0':
 880         case '1':
 881         case '2':
 882         case '3':
 883         case '4':
 884         case '5':
 885         case '6':
 886         case '7':
 887                 return parse_octal_sequence(ec);
 888         case EOF:
 889                 parse_error("reached end of file while parsing escape sequence");
 890                 return EOF;
 891         /* \E is not documented, but handled, by GCC.  It is acceptable according
 892          * to §6.11.4, whereas \e is not. */
 893         case 'E':
 894         case 'e':
 895                 if (c_mode & _GNUC)
 896                         return 27;   /* hopefully 27 is ALWAYS the code for ESCAPE */
 897                 break;
 898         case 'u':
 899         case 'U':
 900                 parse_error("universal character parsing not implemented yet");
 901                 return EOF;
 902         default:
 903                 break;
 904         }
 905         /* §6.4.4.4:8 footnote 64 */
 906         parse_error("unknown escape sequence");
 907         return EOF;
 908 }
 909
 910 /**
 911  * Concatenate two strings.
 912  */
 913 string_t concat_strings(const string_t *const s1, const string_t *const s2)
 914 {
 915         const size_t len1 = s1->size - 1;
 916         const size_t len2 = s2->size - 1;
 917
 918         char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
 919         memcpy(concat, s1->begin, len1);
 920         memcpy(concat + len1, s2->begin, len2 + 1);
 921
 922         return identify_string(concat, len1 + len2 + 1);
 923 }
 924
 925 string_t make_string(const char *string)
 926 {
 927         size_t      len   = strlen(string) + 1;
 928         char *const space = obstack_alloc(&symbol_obstack, len);
 929         memcpy(space, string, len);
 930
 931         return identify_string(space, len);
 932 }
 933
 934 static void grow_symbol(utf32 const tc)
 935 {
 936         struct obstack *const o  = &symbol_obstack;
 937         if (tc < 0x80U) {
 938                 obstack_1grow(o, tc);
 939         } else if (tc < 0x800) {
 940                 obstack_1grow(o, 0xC0 | (tc >> 6));
 941                 obstack_1grow(o, 0x80 | (tc & 0x3F));
 942         } else if (tc < 0x10000) {
 943                 obstack_1grow(o, 0xE0 | ( tc >> 12));
 944                 obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
 945                 obstack_1grow(o, 0x80 | ( tc        & 0x3F));
 946         } else {
 947                 obstack_1grow(o, 0xF0 | ( tc >> 18));
 948                 obstack_1grow(o, 0x80 | ((tc >> 12) & 0x3F));
 949                 obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
 950                 obstack_1grow(o, 0x80 | ( tc        & 0x3F));
 951         }
 952 }
 953
 954 /**
 955  * Parse a string literal and set lexer_token.
 956  */
 957 static void parse_string_literal(void)
 958 {
 959         eat('"');
 960
 961         while (true) {
 962                 switch (c) {
 963                 case '\\': {
 964                         utf32 const tc = parse_escape_sequence();
 965                         if (tc >= 0x100) {
 966                                 warningf(&lexer_pos, "escape sequence out of range");
 967                         }
 968                         obstack_1grow(&symbol_obstack, tc);
 969                         break;
 970                 }
 971
 972                 case EOF: {
 973                         errorf(&lexer_token.source_position, "string has no end");
 974                         lexer_token.type = T_ERROR;
 975                         return;
 976                 }
 977
 978                 case '"':
 979                         next_char();
 980                         goto end_of_string;
 981
 982                 default:
 983                         grow_symbol(c);
 984                         next_char();
 985                         break;
 986                 }
 987         }
 988
 989 end_of_string:
 990
 991         /* TODO: concatenate multiple strings separated by whitespace... */
 992
 993         /* add finishing 0 to the string */
 994         obstack_1grow(&symbol_obstack, '\0');
 995         const size_t  size   = (size_t)obstack_object_size(&symbol_obstack);
 996         char         *string = obstack_finish(&symbol_obstack);
 997
 998         lexer_token.type    = T_STRING_LITERAL;
 999         lexer_token.literal = identify_string(string, size);
1000 }
1001
1002 /**
1003  * Parse a wide character constant and set lexer_token.
1004  */
1005 static void parse_wide_character_constant(void)
1006 {
1007         eat('\'');
1008
1009         while (true) {
1010                 switch (c) {
1011                 case '\\': {
1012                         const utf32 tc = parse_escape_sequence();
1013                         grow_symbol(tc);
1014                         break;
1015                 }
1016
1017                 MATCH_NEWLINE(
1018                         parse_error("newline while parsing character constant");
1019                         break;
1020                 )
1021
1022                 case '\'':
1023                         next_char();
1024                         goto end_of_wide_char_constant;
1025
1026                 case EOF: {
1027                         errorf(&lexer_token.source_position, "EOF while parsing character constant");
1028                         lexer_token.type = T_ERROR;
1029                         return;
1030                 }
1031
1032                 default:
1033                         grow_symbol(c);
1034                         next_char();
1035                         break;
1036                 }
1037         }
1038
1039 end_of_wide_char_constant:;
1040         obstack_1grow(&symbol_obstack, '\0');
1041         size_t  size   = (size_t) obstack_object_size(&symbol_obstack) - 1;
1042         char   *string = obstack_finish(&symbol_obstack);
1043
1044         lexer_token.type     = T_WIDE_CHARACTER_CONSTANT;
1045         lexer_token.literal  = identify_string(string, size);
1046
1047         if (size == 0) {
1048                 parse_error("empty character constant");
1049         }
1050 }
1051
1052 /**
1053  * Parse a wide string literal and set lexer_token.
1054  */
1055 static void parse_wide_string_literal(void)
1056 {
1057         parse_string_literal();
1058         if (lexer_token.type == T_STRING_LITERAL)
1059                 lexer_token.type = T_WIDE_STRING_LITERAL;
1060 }
1061
1062 /**
1063  * Parse a character constant and set lexer_token.
1064  */
1065 static void parse_character_constant(void)
1066 {
1067         eat('\'');
1068
1069         while (true) {
1070                 switch (c) {
1071                 case '\\': {
1072                         utf32 const tc = parse_escape_sequence();
1073                         if (tc >= 0x100) {
1074                                 warningf(&lexer_pos, "escape sequence out of range");
1075                         }
1076                         obstack_1grow(&symbol_obstack, tc);
1077                         break;
1078                 }
1079
1080                 MATCH_NEWLINE(
1081                         parse_error("newline while parsing character constant");
1082                         break;
1083                 )
1084
1085                 case '\'':
1086                         next_char();
1087                         goto end_of_char_constant;
1088
1089                 case EOF: {
1090                         errorf(&lexer_token.source_position, "EOF while parsing character constant");
1091                         lexer_token.type = T_ERROR;
1092                         return;
1093                 }
1094
1095                 default:
1096                         grow_symbol(c);
1097                         next_char();
1098                         break;
1099
1100                 }
1101         }
1102
1103 end_of_char_constant:;
1104         obstack_1grow(&symbol_obstack, '\0');
1105         const size_t        size   = (size_t)obstack_object_size(&symbol_obstack)-1;
1106         char         *const string = obstack_finish(&symbol_obstack);
1107
1108         lexer_token.type    = T_CHARACTER_CONSTANT;
1109         lexer_token.literal = identify_string(string, size);
1110
1111         if (size == 0) {
1112                 parse_error("empty character constant");
1113         }
1114 }
1115
1116 /**
1117  * Skip a multiline comment.
1118  */
1119 static void skip_multiline_comment(void)
1120 {
1121         while (true) {
1122                 switch (c) {
1123                 case '/':
1124                         next_char();
1125                         if (c == '*') {
1126                                 /* nested comment, warn here */
1127                                 if (warning.comment) {
1128                                         warningf(&lexer_pos, "'/*' within comment");
1129                                 }
1130                         }
1131                         break;
1132                 case '*':
1133                         next_char();
1134                         if (c == '/') {
1135                                 next_char();
1136                                 return;
1137                         }
1138                         break;
1139
1140                 MATCH_NEWLINE(break;)
1141
1142                 case EOF: {
1143                         errorf(&lexer_token.source_position, "at end of file while looking for comment end");
1144                         return;
1145                 }
1146
1147                 default:
1148                         next_char();
1149                         break;
1150                 }
1151         }
1152 }
1153
1154 /**
1155  * Skip a single line comment.
1156  */
1157 static void skip_line_comment(void)
1158 {
1159         while (true) {
1160                 switch (c) {
1161                 case EOF:
1162                         return;
1163
1164                 case '\n':
1165                 case '\r':
1166                         return;
1167
1168                 case '\\':
1169                         next_char();
1170                         if (c == '\n' || c == '\r') {
1171                                 if (warning.comment)
1172                                         warningf(&lexer_pos, "multi-line comment");
1173                                 return;
1174                         }
1175                         break;
1176
1177                 default:
1178                         next_char();
1179                         break;
1180                 }
1181         }
1182 }
1183
1184 /** The current preprocessor token. */
1185 static token_t pp_token;
1186
1187 /**
1188  * Read the next preprocessor token.
1189  */
1190 static inline void next_pp_token(void)
1191 {
1192         lexer_next_preprocessing_token();
1193         pp_token = lexer_token;
1194 }
1195
1196 /**
1197  * Eat all preprocessor tokens until newline.
1198  */
1199 static void eat_until_newline(void)
1200 {
1201         while (pp_token.type != '\n' && pp_token.type != T_EOF) {
1202                 next_pp_token();
1203         }
1204 }
1205
1206 /**
1207  * Handle the define directive.
1208  */
1209 static void define_directive(void)
1210 {
1211         lexer_next_preprocessing_token();
1212         if (lexer_token.type != T_IDENTIFIER) {
1213                 parse_error("expected identifier after #define\n");
1214                 eat_until_newline();
1215         }
1216 }
1217
1218 /**
1219  * Handle the ifdef directive.
1220  */
1221 static void ifdef_directive(int is_ifndef)
1222 {
1223         (void) is_ifndef;
1224         lexer_next_preprocessing_token();
1225         //expect_identifier();
1226         //extect_newline();
1227 }
1228
1229 /**
1230  * Handle the endif directive.
1231  */
1232 static void endif_directive(void)
1233 {
1234         //expect_newline();
1235 }
1236
1237 /**
1238  * Parse the line directive.
1239  */
1240 static void parse_line_directive(void)
1241 {
1242         if (pp_token.type != T_INTEGER) {
1243                 parse_error("expected integer");
1244         } else {
1245                 /* use offset -1 as this is about the next line */
1246                 lexer_pos.lineno = atoi(pp_token.literal.begin) - 1;
1247                 next_pp_token();
1248         }
1249         if (pp_token.type == T_STRING_LITERAL) {
1250                 lexer_pos.input_name = pp_token.literal.begin;
1251                 next_pp_token();
1252         }
1253
1254         eat_until_newline();
1255 }
1256
1257 /**
1258  * STDC pragmas.
1259  */
1260 typedef enum stdc_pragma_kind_t {
1261         STDC_UNKNOWN,
1262         STDC_FP_CONTRACT,
1263         STDC_FENV_ACCESS,
1264         STDC_CX_LIMITED_RANGE
1265 } stdc_pragma_kind_t;
1266
1267 /**
1268  * STDC pragma values.
1269  */
1270 typedef enum stdc_pragma_value_kind_t {
1271         STDC_VALUE_UNKNOWN,
1272         STDC_VALUE_ON,
1273         STDC_VALUE_OFF,
1274         STDC_VALUE_DEFAULT
1275 } stdc_pragma_value_kind_t;
1276
1277 /**
1278  * Parse a pragma directive.
1279  */
1280 static void parse_pragma(void)
1281 {
1282         bool unknown_pragma = true;
1283
1284         next_pp_token();
1285         if (pp_token.symbol->pp_ID == TP_STDC) {
1286                 stdc_pragma_kind_t kind = STDC_UNKNOWN;
1287                 /* a STDC pragma */
1288                 if (c_mode & _C99) {
1289                         next_pp_token();
1290
1291                         switch (pp_token.symbol->pp_ID) {
1292                         case TP_FP_CONTRACT:
1293                                 kind = STDC_FP_CONTRACT;
1294                                 break;
1295                         case TP_FENV_ACCESS:
1296                                 kind = STDC_FENV_ACCESS;
1297                                 break;
1298                         case TP_CX_LIMITED_RANGE:
1299                                 kind = STDC_CX_LIMITED_RANGE;
1300                                 break;
1301                         default:
1302                                 break;
1303                         }
1304                         if (kind != STDC_UNKNOWN) {
1305                                 stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN;
1306                                 next_pp_token();
1307                                 switch (pp_token.symbol->pp_ID) {
1308                                 case TP_ON:
1309                                         value = STDC_VALUE_ON;
1310                                         break;
1311                                 case TP_OFF:
1312                                         value = STDC_VALUE_OFF;
1313                                         break;
1314                                 case TP_DEFAULT:
1315                                         value = STDC_VALUE_DEFAULT;
1316                                         break;
1317                                 default:
1318                                         break;
1319                                 }
1320                                 if (value != STDC_VALUE_UNKNOWN) {
1321                                         unknown_pragma = false;
1322                                 } else {
1323                                         errorf(&pp_token.source_position, "bad STDC pragma argument");
1324                                 }
1325                         }
1326                 }
1327         } else {
1328                 unknown_pragma = true;
1329         }
1330         eat_until_newline();
1331         if (unknown_pragma && warning.unknown_pragmas) {
1332                 warningf(&pp_token.source_position, "encountered unknown #pragma");
1333         }
1334 }
1335
1336 /**
1337  * Parse a preprocessor non-null directive.
1338  */
1339 static void parse_preprocessor_identifier(void)
1340 {
1341         assert(pp_token.type == T_IDENTIFIER);
1342         symbol_t *symbol = pp_token.symbol;
1343
1344         switch (symbol->pp_ID) {
1345         case TP_include:
1346                 printf("include - enable header name parsing!\n");
1347                 break;
1348         case TP_define:
1349                 define_directive();
1350                 break;
1351         case TP_ifdef:
1352                 ifdef_directive(0);
1353                 break;
1354         case TP_ifndef:
1355                 ifdef_directive(1);
1356                 break;
1357         case TP_endif:
1358                 endif_directive();
1359                 break;
1360         case TP_line:
1361                 next_pp_token();
1362                 parse_line_directive();
1363                 break;
1364         case TP_if:
1365         case TP_else:
1366         case TP_elif:
1367         case TP_undef:
1368         case TP_error:
1369                 /* TODO; output the rest of the line */
1370                 parse_error("#error directive: ");
1371                 break;
1372         case TP_pragma:
1373                 parse_pragma();
1374                 break;
1375         }
1376 }
1377
1378 /**
1379  * Parse a preprocessor directive.
1380  */
1381 static void parse_preprocessor_directive(void)
1382 {
1383         next_pp_token();
1384
1385         switch (pp_token.type) {
1386         case T_IDENTIFIER:
1387                 parse_preprocessor_identifier();
1388                 break;
1389         case T_INTEGER:
1390                 parse_line_directive();
1391                 break;
1392         case '\n':
1393                 /* NULL directive, see §6.10.7 */
1394                 break;
1395         default:
1396                 parse_error("invalid preprocessor directive");
1397                 eat_until_newline();
1398                 break;
1399         }
1400 }
1401
1402 #define MAYBE_PROLOG                                       \
1403                         next_char();                                   \
1404                         while (true) {                                 \
1405                                 switch (c) {
1406
1407 #define MAYBE(ch, set_type)                                \
1408                                 case ch:                                   \
1409                                         next_char();                           \
1410                                         lexer_token.type = set_type;           \
1411                                         return;
1412
1413 /* must use this as last thing */
1414 #define MAYBE_MODE(ch, set_type, mode)                     \
1415                                 case ch:                                   \
1416                                         if (c_mode & mode) {                   \
1417                                                 next_char();                       \
1418                                                 lexer_token.type = set_type;       \
1419                                                 return;                            \
1420                                         }                                      \
1421                                         /* fallthrough */
1422
1423 #define ELSE_CODE(code)                                    \
1424                                 default:                                   \
1425                                         code                                   \
1426                                 }                                          \
1427                         } /* end of while (true) */                    \
1428                         break;
1429
1430 #define ELSE(set_type)                                     \
1431                 ELSE_CODE(                                         \
1432                         lexer_token.type = set_type;                   \
1433                         return;                                        \
1434                 )
1435
1436 void lexer_next_preprocessing_token(void)
1437 {
1438         while (true) {
1439                 lexer_token.source_position = lexer_pos;
1440
1441                 switch (c) {
1442                 case ' ':
1443                 case '\t':
1444                         next_char();
1445                         break;
1446
1447                 MATCH_NEWLINE(
1448                         lexer_token.type = '\n';
1449                         return;
1450                 )
1451
1452                 SYMBOL_CHARS
1453                         parse_symbol();
1454                         /* might be a wide string ( L"string" ) */
1455                         if (lexer_token.symbol == symbol_L) {
1456                                 switch (c) {
1457                                         case '"':  parse_wide_string_literal();     break;
1458                                         case '\'': parse_wide_character_constant(); break;
1459                                 }
1460                         }
1461                         return;
1462
1463                 DIGITS
1464                         parse_number();
1465                         return;
1466
1467                 case '"':
1468                         parse_string_literal();
1469                         return;
1470
1471                 case '\'':
1472                         parse_character_constant();
1473                         return;
1474
1475                 case '.':
1476                         MAYBE_PROLOG
1477                                 DIGITS
1478                                         put_back(c);
1479                                         c = '.';
1480                                         parse_number();
1481                                         return;
1482
1483                                 case '.':
1484                                         MAYBE_PROLOG
1485                                         MAYBE('.', T_DOTDOTDOT)
1486                                         ELSE_CODE(
1487                                                 put_back(c);
1488                                                 c = '.';
1489                                                 lexer_token.type = '.';
1490                                                 return;
1491                                         )
1492                         ELSE('.')
1493                 case '&':
1494                         MAYBE_PROLOG
1495                         MAYBE('&', T_ANDAND)
1496                         MAYBE('=', T_ANDEQUAL)
1497                         ELSE('&')
1498                 case '*':
1499                         MAYBE_PROLOG
1500                         MAYBE('=', T_ASTERISKEQUAL)
1501                         ELSE('*')
1502                 case '+':
1503                         MAYBE_PROLOG
1504                         MAYBE('+', T_PLUSPLUS)
1505                         MAYBE('=', T_PLUSEQUAL)
1506                         ELSE('+')
1507                 case '-':
1508                         MAYBE_PROLOG
1509                         MAYBE('>', T_MINUSGREATER)
1510                         MAYBE('-', T_MINUSMINUS)
1511                         MAYBE('=', T_MINUSEQUAL)
1512                         ELSE('-')
1513                 case '!':
1514                         MAYBE_PROLOG
1515                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1516                         ELSE('!')
1517                 case '/':
1518                         MAYBE_PROLOG
1519                         MAYBE('=', T_SLASHEQUAL)
1520                                 case '*':
1521                                         next_char();
1522                                         skip_multiline_comment();
1523                                         lexer_next_preprocessing_token();
1524                                         return;
1525                                 case '/':
1526                                         next_char();
1527                                         skip_line_comment();
1528                                         lexer_next_preprocessing_token();
1529                                         return;
1530                         ELSE('/')
1531                 case '%':
1532                         MAYBE_PROLOG
1533                         MAYBE('>', '}')
1534                         MAYBE('=', T_PERCENTEQUAL)
1535                                 case ':':
1536                                         MAYBE_PROLOG
1537                                                 case '%':
1538                                                         MAYBE_PROLOG
1539                                                         MAYBE(':', T_HASHHASH)
1540                                                         ELSE_CODE(
1541                                                                 put_back(c);
1542                                                                 c = '%';
1543                                                                 lexer_token.type = '#';
1544                                                                 return;
1545                                                         )
1546                                         ELSE('#')
1547                         ELSE('%')
1548                 case '<':
1549                         MAYBE_PROLOG
1550                         MAYBE(':', '[')
1551                         MAYBE('%', '{')
1552                         MAYBE('=', T_LESSEQUAL)
1553                                 case '<':
1554                                         MAYBE_PROLOG
1555                                         MAYBE('=', T_LESSLESSEQUAL)
1556                                         ELSE(T_LESSLESS)
1557                         ELSE('<')
1558                 case '>':
1559                         MAYBE_PROLOG
1560                         MAYBE('=', T_GREATEREQUAL)
1561                                 case '>':
1562                                         MAYBE_PROLOG
1563                                         MAYBE('=', T_GREATERGREATEREQUAL)
1564                                         ELSE(T_GREATERGREATER)
1565                         ELSE('>')
1566                 case '^':
1567                         MAYBE_PROLOG
1568                         MAYBE('=', T_CARETEQUAL)
1569                         ELSE('^')
1570                 case '|':
1571                         MAYBE_PROLOG
1572                         MAYBE('=', T_PIPEEQUAL)
1573                         MAYBE('|', T_PIPEPIPE)
1574                         ELSE('|')
1575                 case ':':
1576                         MAYBE_PROLOG
1577                         MAYBE('>', ']')
1578                         MAYBE_MODE(':', T_COLONCOLON, _CXX)
1579                         ELSE(':')
1580                 case '=':
1581                         MAYBE_PROLOG
1582                         MAYBE('=', T_EQUALEQUAL)
1583                         ELSE('=')
1584                 case '#':
1585                         MAYBE_PROLOG
1586                         MAYBE('#', T_HASHHASH)
1587                         ELSE('#')
1588
1589                 case '?':
1590                 case '[':
1591                 case ']':
1592                 case '(':
1593                 case ')':
1594                 case '{':
1595                 case '}':
1596                 case '~':
1597                 case ';':
1598                 case ',':
1599                 case '\\':
1600                         lexer_token.type = c;
1601                         next_char();
1602                         return;
1603
1604                 case EOF:
1605                         lexer_token.type = T_EOF;
1606                         return;
1607
1608                 default:
1609 dollar_sign:
1610                         errorf(&lexer_pos, "unknown character '%c' found", c);
1611                         next_char();
1612                         lexer_token.type = T_ERROR;
1613                         return;
1614                 }
1615         }
1616 }
1617
1618 void lexer_next_token(void)
1619 {
1620         lexer_next_preprocessing_token();
1621
1622         while (lexer_token.type == '\n') {
1623 newline_found:
1624                 lexer_next_preprocessing_token();
1625         }
1626
1627         if (lexer_token.type == '#') {
1628                 parse_preprocessor_directive();
1629                 goto newline_found;
1630         }
1631 }
1632
1633 void init_lexer(void)
1634 {
1635         strset_init(&stringset);
1636         symbol_L = symbol_table_insert("L");
1637 }
1638
1639 void lexer_open_stream(FILE *stream, const char *input_name)
1640 {
1641         input                = stream;
1642         lexer_pos.lineno     = 0;
1643         lexer_pos.colno      = 0;
1644         lexer_pos.input_name = input_name;
1645
1646         bufpos = NULL;
1647         bufend = NULL;
1648
1649         /* place a virtual \n at the beginning so the lexer knows that we're
1650          * at the beginning of a line */
1651         c = '\n';
1652 }
1653
1654 void lexer_open_buffer(const char *buffer, size_t len, const char *input_name)
1655 {
1656         input                = NULL;
1657         lexer_pos.lineno     = 0;
1658         lexer_pos.colno      = 0;
1659         lexer_pos.input_name = input_name;
1660
1661 #if 0 // TODO
1662         bufpos = buffer;
1663         bufend = buffer + len;
1664
1665         /* place a virtual \n at the beginning so the lexer knows that we're
1666          * at the beginning of a line */
1667         c = '\n';
1668 #else
1669         (void)buffer;
1670         (void)len;
1671         panic("builtin lexing not done yet");
1672 #endif
1673 }
1674
1675 void exit_lexer(void)
1676 {
1677         strset_destroy(&stringset);
1678 }
1679
1680 static __attribute__((unused))
1681 void dbg_pos(const source_position_t source_position)
1682 {
1683         fprintf(stdout, "%s:%u:%u\n", source_position.input_name,
1684                 source_position.lineno, source_position.colno);
1685         fflush(stdout);
1686 }