nsz Git - cparser/blob - lexer.c

   1 /*
   2  * This file is part of cparser.
   3  * Copyright (C) 2007-2009 Matthias Braun <matze@braunis.de>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; either version 2
   8  * of the License, or (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  18  * 02111-1307, USA.
  19  */
  20 #include <config.h>
  21
  22 #include "diagnostic.h"
  23 #include "lexer.h"
  24 #include "symbol_t.h"
  25 #include "token_t.h"
  26 #include "symbol_table_t.h"
  27 #include "adt/error.h"
  28 #include "adt/strset.h"
  29 #include "adt/util.h"
  30 #include "types.h"
  31 #include "type_t.h"
  32 #include "target_architecture.h"
  33 #include "parser.h"
  34 #include "warning.h"
  35 #include "lang_features.h"
  36
  37 #include <assert.h>
  38 #include <errno.h>
  39 #include <string.h>
  40 #include <stdbool.h>
  41 #include <ctype.h>
  42
  43 #ifndef _WIN32
  44 #include <strings.h>
  45 #endif
  46
  47 //#define DEBUG_CHARS
  48 #define MAX_PUTBACK 3
  49 #define BUF_SIZE    1024
  50
  51 #if defined(_WIN32) || defined(__CYGWIN__)
  52 /* No strtold on windows and no replacement yet */
  53 #define strtold(s, e) strtod(s, e)
  54 #endif
  55
  56 static utf32        c;
  57 token_t             lexer_token;
  58 symbol_t           *symbol_L;
  59 static FILE        *input;
  60 static utf32        buf[BUF_SIZE + MAX_PUTBACK];
  61 static const utf32 *bufend;
  62 static const utf32 *bufpos;
  63 static strset_t     stringset;
  64 bool                allow_dollar_in_symbol = true;
  65
  66 /**
  67  * Prints a parse error message at the current token.
  68  *
  69  * @param msg   the error message
  70  */
  71 static void parse_error(const char *msg)
  72 {
  73         errorf(&lexer_token.source_position, "%s", msg);
  74 }
  75
  76 /**
  77  * Prints an internal error message at the current token.
  78  *
  79  * @param msg   the error message
  80  */
  81 static NORETURN internal_error(const char *msg)
  82 {
  83         internal_errorf(&lexer_token.source_position, "%s", msg);
  84 }
  85
  86 static size_t read_block(unsigned char *const read_buf, size_t const n)
  87 {
  88         size_t const s = fread(read_buf, 1, n, input);
  89         if (s == 0) {
  90                 if (ferror(input))
  91                         parse_error("read from input failed");
  92                 buf[MAX_PUTBACK] = EOF;
  93                 bufpos           = buf + MAX_PUTBACK;
  94                 bufend           = buf + MAX_PUTBACK + 1;
  95         }
  96         return s;
  97 }
  98
  99 static void decode_iso_8859_1(void)
 100 {
 101         unsigned char read_buf[BUF_SIZE];
 102         size_t const s = read_block(read_buf, sizeof(read_buf));
 103         if (s == 0)
 104                 return;
 105
 106         unsigned char const *src = read_buf;
 107         unsigned char const *end = read_buf + s;
 108         utf32               *dst = buf + MAX_PUTBACK;
 109         while (src != end)
 110                 *dst++ = *src++;
 111
 112         bufpos = buf + MAX_PUTBACK;
 113         bufend = dst;
 114 }
 115
 116 static void decode_iso_8859_15(void)
 117 {
 118         unsigned char read_buf[BUF_SIZE];
 119         size_t const s = read_block(read_buf, sizeof(read_buf));
 120         if (s == 0)
 121                 return;
 122
 123         unsigned char const *src = read_buf;
 124         unsigned char const *end = read_buf + s;
 125         utf32               *dst = buf + MAX_PUTBACK;
 126         while (src != end) {
 127                 utf32 tc = *src++;
 128                 switch (tc) {
 129                         case 0xA4: tc = 0x20AC; break; // €
 130                         case 0xA6: tc = 0x0160; break; // Š
 131                         case 0xA8: tc = 0x0161; break; // š
 132                         case 0xB4: tc = 0x017D; break; // Ž
 133                         case 0xB8: tc = 0x017E; break; // ž
 134                         case 0xBC: tc = 0x0152; break; // Œ
 135                         case 0xBD: tc = 0x0153; break; // œ
 136                         case 0xBE: tc = 0x0178; break; // Ÿ
 137                 }
 138                 *dst++ = tc;
 139         }
 140
 141         bufpos = buf + MAX_PUTBACK;
 142         bufend = dst;
 143 }
 144
 145 static void decode_utf8(void)
 146 {
 147         static utf32  part_decoded_min_code;
 148         static utf32  part_decoded_char;
 149         static size_t part_decoded_rest_len;
 150
 151         do {
 152                 unsigned char read_buf[BUF_SIZE];
 153                 size_t const s = read_block(read_buf, sizeof(read_buf));
 154                 if (s == 0) {
 155                         if (part_decoded_rest_len > 0)
 156                                 parse_error("incomplete input char at end of input");
 157                         return;
 158                 }
 159
 160                 unsigned char const *src = read_buf;
 161                 unsigned char const *end = read_buf + s;
 162                 utf32               *dst = buf + MAX_PUTBACK;
 163                 utf32                decoded;
 164                 utf32                min_code;
 165
 166                 if (part_decoded_rest_len != 0) {
 167                         min_code              = part_decoded_min_code;
 168                         decoded               = part_decoded_char;
 169                         size_t const rest_len = part_decoded_rest_len;
 170                         part_decoded_rest_len = 0;
 171                         switch (rest_len) {
 172                                 case 4:  goto realign;
 173                                 case 3:  goto three_more;
 174                                 case 2:  goto two_more;
 175                                 default: goto one_more;
 176                         }
 177                 }
 178
 179                 while (src != end) {
 180                         if ((*src & 0x80) == 0) {
 181                                 decoded = *src++;
 182                         } else if ((*src & 0xE0) == 0xC0) {
 183                                 min_code = 0x80;
 184                                 decoded  = *src++ & 0x1F;
 185 one_more:
 186                                 if (src == end) {
 187                                         part_decoded_min_code = min_code;
 188                                         part_decoded_char     = decoded;
 189                                         part_decoded_rest_len = 1;
 190                                         break;
 191                                 }
 192                                 if ((*src & 0xC0) == 0x80) {
 193                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 194                                 } else {
 195                                         goto invalid_char;
 196                                 }
 197                                 if (decoded < min_code                      ||
 198                                                 decoded > 0x10FFFF                      ||
 199                                                 (0xD800 <= decoded && decoded < 0xE000) || // high/low surrogates
 200                                                 (0xFDD0 <= decoded && decoded < 0xFDF0) || // noncharacters
 201                                                 (decoded & 0xFFFE) == 0xFFFE) {            // noncharacters
 202                                         parse_error("invalid byte sequence in input");
 203                                 }
 204                         } else if ((*src & 0xF0) == 0xE0) {
 205                                 min_code = 0x800;
 206                                 decoded  = *src++ & 0x0F;
 207 two_more:
 208                                 if (src == end) {
 209                                         part_decoded_min_code = min_code;
 210                                         part_decoded_char     = decoded;
 211                                         part_decoded_rest_len = 2;
 212                                         break;
 213                                 }
 214                                 if ((*src & 0xC0) == 0x80) {
 215                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 216                                 } else {
 217                                         goto invalid_char;
 218                                 }
 219                                 goto one_more;
 220                         } else if ((*src & 0xF8) == 0xF0) {
 221                                 min_code = 0x10000;
 222                                 decoded  = *src++ & 0x07;
 223 three_more:
 224                                 if (src == end) {
 225                                         part_decoded_min_code = min_code;
 226                                         part_decoded_char     = decoded;
 227                                         part_decoded_rest_len = 3;
 228                                         break;
 229                                 }
 230                                 if ((*src & 0xC0) == 0x80) {
 231                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 232                                 } else {
 233                                         goto invalid_char;
 234                                 }
 235                                 goto two_more;
 236                         } else {
 237 invalid_char:
 238                                 parse_error("invalid byte sequence in input");
 239 realign:
 240                                 do {
 241                                         ++src;
 242                                         if (src == end) {
 243                                                 part_decoded_rest_len = 4;
 244                                                 break;
 245                                         }
 246                                 } while ((*src & 0xC0) == 0x80 || (*src & 0xF8) == 0xF8);
 247                                 continue;
 248                         }
 249                         *dst++ = decoded;
 250                 }
 251
 252                 bufpos = buf + MAX_PUTBACK;
 253                 bufend = dst;
 254         } while (bufpos == bufend);
 255 }
 256
 257 static void decode_windows_1252(void)
 258 {
 259         unsigned char read_buf[BUF_SIZE];
 260         size_t const s = read_block(read_buf, sizeof(read_buf));
 261         if (s == 0)
 262                 return;
 263
 264         unsigned char const *src = read_buf;
 265         unsigned char const *end = read_buf + s;
 266         utf32               *dst = buf + MAX_PUTBACK;
 267         while (src != end) {
 268                 utf32 tc = *src++;
 269                 switch (tc) {
 270                         case 0x80: tc = 0x20AC; break; // €
 271                         case 0x82: tc = 0x201A; break; // ‚
 272                         case 0x83: tc = 0x0192; break; // ƒ
 273                         case 0x84: tc = 0x201E; break; // „
 274                         case 0x85: tc = 0x2026; break; // …
 275                         case 0x86: tc = 0x2020; break; // †
 276                         case 0x87: tc = 0x2021; break; // ‡
 277                         case 0x88: tc = 0x02C6; break; // ˆ
 278                         case 0x89: tc = 0x2030; break; // ‰
 279                         case 0x8A: tc = 0x0160; break; // Š
 280                         case 0x8B: tc = 0x2039; break; // ‹
 281                         case 0x8C: tc = 0x0152; break; // Œ
 282                         case 0x8E: tc = 0x017D; break; // Ž
 283                         case 0x91: tc = 0x2018; break; // ‘
 284                         case 0x92: tc = 0x2019; break; // ’
 285                         case 0x93: tc = 0x201C; break; // “
 286                         case 0x94: tc = 0x201D; break; // ”
 287                         case 0x95: tc = 0x2022; break; // •
 288                         case 0x96: tc = 0x2013; break; // –
 289                         case 0x97: tc = 0x2014; break; // —
 290                         case 0x98: tc = 0x02DC; break; // ˜
 291                         case 0x99: tc = 0x2122; break; // ™
 292                         case 0x9A: tc = 0x0161; break; // š
 293                         case 0x9B: tc = 0x203A; break; // ›
 294                         case 0x9C: tc = 0x0153; break; // œ
 295                         case 0x9E: tc = 0x017E; break; // ž
 296                         case 0x9F: tc = 0x0178; break; // Ÿ
 297                 }
 298                 *dst++ = tc;
 299         }
 300
 301         bufpos = buf + MAX_PUTBACK;
 302         bufend = dst;
 303 }
 304
 305 typedef void (*decoder_t)(void);
 306
 307 static decoder_t decoder = decode_utf8;
 308
 309 typedef struct named_decoder_t {
 310         char const *name;
 311         decoder_t   decoder;
 312 } named_decoder_t;
 313
 314 static named_decoder_t const decoders[] = {
 315         { "CP819",           decode_iso_8859_1   }, // offical alias
 316         { "IBM819",          decode_iso_8859_1   }, // offical alias
 317         { "ISO-8859-1",      decode_iso_8859_1   }, // offical alias
 318         { "ISO-8859-15",     decode_iso_8859_15  }, // offical name
 319         { "ISO8859-1",       decode_iso_8859_1   },
 320         { "ISO8859-15",      decode_iso_8859_15  },
 321         { "ISO_8859-1",      decode_iso_8859_1   }, // offical alias
 322         { "ISO_8859-15",     decode_iso_8859_15  }, // offical alias
 323         { "ISO_8859-1:1987", decode_iso_8859_1   }, // offical name
 324         { "Latin-9",         decode_iso_8859_15  }, // offical alias
 325         { "UTF-8",           decode_utf8         }, // offical name
 326         { "csISOLatin1",     decode_iso_8859_1   }, // offical alias
 327         { "cp1252",          decode_windows_1252 },
 328         { "iso-ir-100",      decode_iso_8859_1   }, // offical alias
 329         { "l1",              decode_iso_8859_1   }, // offical alias
 330         { "latin1",          decode_iso_8859_1   }, // offical alias
 331         { "windows-1252",    decode_windows_1252 }, // official name
 332
 333         { NULL,              NULL                }
 334 };
 335
 336 /** strcasecmp is not part of C99 so we need our own implementation here */
 337 static int my_strcasecmp(const char *s1, const char *s2)
 338 {
 339         for ( ; *s1 != 0; ++s1, ++s2) {
 340                 if (tolower(*s1) != tolower(*s2))
 341                         break;
 342         }
 343         return (unsigned char)*s1 - (unsigned char)*s2;
 344 }
 345
 346 void select_input_encoding(char const* const encoding)
 347 {
 348         for (named_decoder_t const *i = decoders; i->name != NULL; ++i) {
 349                 if (my_strcasecmp(encoding, i->name) != 0)
 350                         continue;
 351                 decoder = i->decoder;
 352                 return;
 353         }
 354         fprintf(stderr, "error: input encoding \"%s\" not supported\n", encoding);
 355 }
 356
 357 static inline void next_real_char(void)
 358 {
 359         assert(bufpos <= bufend);
 360         if (bufpos >= bufend) {
 361                 if (input == NULL) {
 362                         c = EOF;
 363                         return;
 364                 }
 365                 decoder();
 366         }
 367         c = *bufpos++;
 368 }
 369
 370 /**
 371  * Put a character back into the buffer.
 372  *
 373  * @param pc  the character to put back
 374  */
 375 static inline void put_back(utf32 const pc)
 376 {
 377         assert(bufpos > buf);
 378         *(--bufpos - buf + buf) = pc;
 379
 380 #ifdef DEBUG_CHARS
 381         printf("putback '%lc'\n", pc);
 382 #endif
 383 }
 384
 385 static inline void next_char(void);
 386
 387 #define MATCH_NEWLINE(code)                   \
 388         case '\r':                                \
 389                 next_char();                          \
 390                 if (c == '\n') {                      \
 391                         next_char();                      \
 392                 }                                     \
 393                 lexer_token.source_position.linenr++; \
 394                 code                                  \
 395         case '\n':                                \
 396                 next_char();                          \
 397                 lexer_token.source_position.linenr++; \
 398                 code
 399
 400 #define eat(c_type)  do { assert(c == c_type); next_char(); } while (0)
 401
 402 static void maybe_concat_lines(void)
 403 {
 404         eat('\\');
 405
 406         switch (c) {
 407         MATCH_NEWLINE(return;)
 408
 409         default:
 410                 break;
 411         }
 412
 413         put_back(c);
 414         c = '\\';
 415 }
 416
 417 /**
 418  * Set c to the next input character, ie.
 419  * after expanding trigraphs.
 420  */
 421 static inline void next_char(void)
 422 {
 423         next_real_char();
 424
 425         /* filter trigraphs */
 426         if (UNLIKELY(c == '\\')) {
 427                 maybe_concat_lines();
 428                 goto end_of_next_char;
 429         }
 430
 431         if (LIKELY(c != '?'))
 432                 goto end_of_next_char;
 433
 434         next_real_char();
 435         if (LIKELY(c != '?')) {
 436                 put_back(c);
 437                 c = '?';
 438                 goto end_of_next_char;
 439         }
 440
 441         next_real_char();
 442         switch (c) {
 443         case '=': c = '#'; break;
 444         case '(': c = '['; break;
 445         case '/': c = '\\'; maybe_concat_lines(); break;
 446         case ')': c = ']'; break;
 447         case '\'': c = '^'; break;
 448         case '<': c = '{'; break;
 449         case '!': c = '|'; break;
 450         case '>': c = '}'; break;
 451         case '-': c = '~'; break;
 452         default:
 453                 put_back(c);
 454                 put_back('?');
 455                 c = '?';
 456                 break;
 457         }
 458
 459 end_of_next_char:;
 460 #ifdef DEBUG_CHARS
 461         printf("nchar '%c'\n", c);
 462 #endif
 463 }
 464
 465 #define SYMBOL_CHARS  \
 466         case '$': if (!allow_dollar_in_symbol) goto dollar_sign; \
 467         case 'a':         \
 468         case 'b':         \
 469         case 'c':         \
 470         case 'd':         \
 471         case 'e':         \
 472         case 'f':         \
 473         case 'g':         \
 474         case 'h':         \
 475         case 'i':         \
 476         case 'j':         \
 477         case 'k':         \
 478         case 'l':         \
 479         case 'm':         \
 480         case 'n':         \
 481         case 'o':         \
 482         case 'p':         \
 483         case 'q':         \
 484         case 'r':         \
 485         case 's':         \
 486         case 't':         \
 487         case 'u':         \
 488         case 'v':         \
 489         case 'w':         \
 490         case 'x':         \
 491         case 'y':         \
 492         case 'z':         \
 493         case 'A':         \
 494         case 'B':         \
 495         case 'C':         \
 496         case 'D':         \
 497         case 'E':         \
 498         case 'F':         \
 499         case 'G':         \
 500         case 'H':         \
 501         case 'I':         \
 502         case 'J':         \
 503         case 'K':         \
 504         case 'L':         \
 505         case 'M':         \
 506         case 'N':         \
 507         case 'O':         \
 508         case 'P':         \
 509         case 'Q':         \
 510         case 'R':         \
 511         case 'S':         \
 512         case 'T':         \
 513         case 'U':         \
 514         case 'V':         \
 515         case 'W':         \
 516         case 'X':         \
 517         case 'Y':         \
 518         case 'Z':         \
 519         case '_':
 520
 521 #define DIGITS        \
 522         case '0':         \
 523         case '1':         \
 524         case '2':         \
 525         case '3':         \
 526         case '4':         \
 527         case '5':         \
 528         case '6':         \
 529         case '7':         \
 530         case '8':         \
 531         case '9':
 532
 533 /**
 534  * Read a symbol from the input and build
 535  * the lexer_token.
 536  */
 537 static void parse_symbol(void)
 538 {
 539         obstack_1grow(&symbol_obstack, (char) c);
 540         next_char();
 541
 542         while (true) {
 543                 switch (c) {
 544                 DIGITS
 545                 SYMBOL_CHARS
 546                         obstack_1grow(&symbol_obstack, (char) c);
 547                         next_char();
 548                         break;
 549
 550                 default:
 551 dollar_sign:
 552                         goto end_symbol;
 553                 }
 554         }
 555
 556 end_symbol:
 557         obstack_1grow(&symbol_obstack, '\0');
 558
 559         char     *string = obstack_finish(&symbol_obstack);
 560         symbol_t *symbol = symbol_table_insert(string);
 561
 562         lexer_token.type   = symbol->ID;
 563         lexer_token.symbol = symbol;
 564
 565         if (symbol->string != string) {
 566                 obstack_free(&symbol_obstack, string);
 567         }
 568 }
 569
 570 /**
 571  * parse suffixes like 'LU' or 'f' after numbers
 572  */
 573 static void parse_number_suffix(void)
 574 {
 575         assert(obstack_object_size(&symbol_obstack) == 0);
 576         while (true) {
 577                 switch (c) {
 578                 SYMBOL_CHARS
 579                         obstack_1grow(&symbol_obstack, (char) c);
 580                         next_char();
 581                         break;
 582                 default:
 583                 dollar_sign:
 584                         goto finish_suffix;
 585                 }
 586         }
 587 finish_suffix:
 588         if (obstack_object_size(&symbol_obstack) == 0) {
 589                 lexer_token.symbol = NULL;
 590                 return;
 591         }
 592
 593         obstack_1grow(&symbol_obstack, '\0');
 594         char     *string = obstack_finish(&symbol_obstack);
 595         symbol_t *symbol = symbol_table_insert(string);
 596
 597         if (symbol->string != string) {
 598                 obstack_free(&symbol_obstack, string);
 599         }
 600         lexer_token.symbol = symbol;
 601 }
 602
 603 static string_t identify_string(char *string, size_t len)
 604 {
 605         /* TODO hash */
 606 #if 0
 607         const char *result = strset_insert(&stringset, concat);
 608         if (result != concat) {
 609                 obstack_free(&symbol_obstack, concat);
 610         }
 611 #else
 612         const char *result = string;
 613 #endif
 614         return (string_t) {result, len};
 615 }
 616
 617 /**
 618  * Parses a hex number including hex floats and set the
 619  * lexer_token.
 620  */
 621 static void parse_number_hex(void)
 622 {
 623         bool is_float   = false;
 624         bool has_digits = false;
 625
 626         assert(obstack_object_size(&symbol_obstack) == 0);
 627         while (isxdigit(c)) {
 628                 has_digits = true;
 629                 obstack_1grow(&symbol_obstack, (char) c);
 630                 next_char();
 631         }
 632
 633         if (c == '.') {
 634                 is_float = true;
 635                 obstack_1grow(&symbol_obstack, (char) c);
 636                 next_char();
 637
 638                 while (isxdigit(c)) {
 639                         has_digits = true;
 640                         obstack_1grow(&symbol_obstack, (char) c);
 641                         next_char();
 642                 }
 643         }
 644         if (c == 'p' || c == 'P') {
 645                 is_float = true;
 646                 obstack_1grow(&symbol_obstack, (char) c);
 647                 next_char();
 648
 649                 if (c == '-' || c == '+') {
 650                         obstack_1grow(&symbol_obstack, (char) c);
 651                         next_char();
 652                 }
 653
 654                 while (isxdigit(c)) {
 655                         obstack_1grow(&symbol_obstack, (char) c);
 656                         next_char();
 657                 }
 658         } else if (is_float) {
 659                 errorf(&lexer_token.source_position,
 660                        "hexadecimal floatingpoint constant requires an exponent");
 661         }
 662         obstack_1grow(&symbol_obstack, '\0');
 663
 664         size_t  size   = obstack_object_size(&symbol_obstack) - 1;
 665         char   *string = obstack_finish(&symbol_obstack);
 666         lexer_token.literal = identify_string(string, size);
 667
 668         lexer_token.type    =
 669                 is_float ? T_FLOATINGPOINT_HEXADECIMAL : T_INTEGER_HEXADECIMAL;
 670
 671         if (!has_digits) {
 672                 errorf(&lexer_token.source_position, "invalid number literal '0x%S'",
 673                        &lexer_token.literal);
 674                 lexer_token.literal.begin = "0";
 675                 lexer_token.literal.size  = 1;
 676         }
 677
 678         parse_number_suffix();
 679 }
 680
 681 /**
 682  * Returns true if the given char is a octal digit.
 683  *
 684  * @param char  the character to check
 685  */
 686 static bool is_octal_digit(utf32 chr)
 687 {
 688         switch (chr) {
 689         case '0':
 690         case '1':
 691         case '2':
 692         case '3':
 693         case '4':
 694         case '5':
 695         case '6':
 696         case '7':
 697                 return true;
 698         default:
 699                 return false;
 700         }
 701 }
 702
 703 /**
 704  * Parses a number and sets the lexer_token.
 705  */
 706 static void parse_number(void)
 707 {
 708         bool is_float   = false;
 709         bool has_digits = false;
 710
 711         assert(obstack_object_size(&symbol_obstack) == 0);
 712         if (c == '0') {
 713                 next_char();
 714                 if (c == 'x' || c == 'X') {
 715                         next_char();
 716                         parse_number_hex();
 717                         return;
 718                 } else {
 719                         has_digits = true;
 720                 }
 721                 obstack_1grow(&symbol_obstack, '0');
 722         }
 723
 724         while (isdigit(c)) {
 725                 has_digits = true;
 726                 obstack_1grow(&symbol_obstack, (char) c);
 727                 next_char();
 728         }
 729
 730         if (c == '.') {
 731                 is_float = true;
 732                 obstack_1grow(&symbol_obstack, '.');
 733                 next_char();
 734
 735                 while (isdigit(c)) {
 736                         has_digits = true;
 737                         obstack_1grow(&symbol_obstack, (char) c);
 738                         next_char();
 739                 }
 740         }
 741         if (c == 'e' || c == 'E') {
 742                 is_float = true;
 743                 obstack_1grow(&symbol_obstack, 'e');
 744                 next_char();
 745
 746                 if (c == '-' || c == '+') {
 747                         obstack_1grow(&symbol_obstack, (char) c);
 748                         next_char();
 749                 }
 750
 751                 while (isdigit(c)) {
 752                         obstack_1grow(&symbol_obstack, (char) c);
 753                         next_char();
 754                 }
 755         }
 756
 757         obstack_1grow(&symbol_obstack, '\0');
 758         size_t  size   = obstack_object_size(&symbol_obstack) - 1;
 759         char   *string = obstack_finish(&symbol_obstack);
 760         lexer_token.literal = identify_string(string, size);
 761
 762         /* is it an octal number? */
 763         if (is_float) {
 764                 lexer_token.type = T_FLOATINGPOINT;
 765         } else if (string[0] == '0') {
 766                 lexer_token.type = T_INTEGER_OCTAL;
 767
 768                 /* check for invalid octal digits */
 769                 for (size_t i= 0; i < size; ++i) {
 770                         char t = string[i];
 771                         if (t == '8' || t == '9')
 772                                 errorf(&lexer_token.source_position,
 773                                        "invalid digit '%c' in octal number", t);
 774                 }
 775         } else {
 776                 lexer_token.type = T_INTEGER;
 777         }
 778
 779         if (!has_digits) {
 780                 errorf(&lexer_token.source_position, "invalid number literal '%S'",
 781                        &lexer_token.literal);
 782         }
 783
 784         parse_number_suffix();
 785 }
 786
 787 /**
 788  * Returns the value of a digit.
 789  * The only portable way to do it ...
 790  */
 791 static int digit_value(utf32 const digit)
 792 {
 793         switch (digit) {
 794         case '0': return 0;
 795         case '1': return 1;
 796         case '2': return 2;
 797         case '3': return 3;
 798         case '4': return 4;
 799         case '5': return 5;
 800         case '6': return 6;
 801         case '7': return 7;
 802         case '8': return 8;
 803         case '9': return 9;
 804         case 'a':
 805         case 'A': return 10;
 806         case 'b':
 807         case 'B': return 11;
 808         case 'c':
 809         case 'C': return 12;
 810         case 'd':
 811         case 'D': return 13;
 812         case 'e':
 813         case 'E': return 14;
 814         case 'f':
 815         case 'F': return 15;
 816         default:
 817                 internal_error("wrong character given");
 818         }
 819 }
 820
 821 /**
 822  * Parses an octal character sequence.
 823  *
 824  * @param first_digit  the already read first digit
 825  */
 826 static utf32 parse_octal_sequence(utf32 const first_digit)
 827 {
 828         assert(is_octal_digit(first_digit));
 829         utf32 value = digit_value(first_digit);
 830         if (!is_octal_digit(c)) return value;
 831         value = 8 * value + digit_value(c);
 832         next_char();
 833         if (!is_octal_digit(c)) return value;
 834         value = 8 * value + digit_value(c);
 835         next_char();
 836         return value;
 837 }
 838
 839 /**
 840  * Parses a hex character sequence.
 841  */
 842 static utf32 parse_hex_sequence(void)
 843 {
 844         utf32 value = 0;
 845         while (isxdigit(c)) {
 846                 value = 16 * value + digit_value(c);
 847                 next_char();
 848         }
 849         return value;
 850 }
 851
 852 /**
 853  * Parse an escape sequence.
 854  */
 855 static utf32 parse_escape_sequence(void)
 856 {
 857         eat('\\');
 858
 859         utf32 const ec = c;
 860         next_char();
 861
 862         switch (ec) {
 863         case '"':  return '"';
 864         case '\'': return '\'';
 865         case '\\': return '\\';
 866         case '?': return '\?';
 867         case 'a': return '\a';
 868         case 'b': return '\b';
 869         case 'f': return '\f';
 870         case 'n': return '\n';
 871         case 'r': return '\r';
 872         case 't': return '\t';
 873         case 'v': return '\v';
 874         case 'x':
 875                 return parse_hex_sequence();
 876         case '0':
 877         case '1':
 878         case '2':
 879         case '3':
 880         case '4':
 881         case '5':
 882         case '6':
 883         case '7':
 884                 return parse_octal_sequence(ec);
 885         case EOF:
 886                 parse_error("reached end of file while parsing escape sequence");
 887                 return EOF;
 888         /* \E is not documented, but handled, by GCC.  It is acceptable according
 889          * to §6.11.4, whereas \e is not. */
 890         case 'E':
 891         case 'e':
 892                 if (c_mode & _GNUC)
 893                         return 27;   /* hopefully 27 is ALWAYS the code for ESCAPE */
 894                 break;
 895         case 'u':
 896         case 'U':
 897                 parse_error("universal character parsing not implemented yet");
 898                 return EOF;
 899         default:
 900                 break;
 901         }
 902         /* §6.4.4.4:8 footnote 64 */
 903         parse_error("unknown escape sequence");
 904         return EOF;
 905 }
 906
 907 /**
 908  * Concatenate two strings.
 909  */
 910 string_t concat_strings(const string_t *const s1, const string_t *const s2)
 911 {
 912         const size_t len1 = s1->size - 1;
 913         const size_t len2 = s2->size - 1;
 914
 915         char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
 916         memcpy(concat, s1->begin, len1);
 917         memcpy(concat + len1, s2->begin, len2 + 1);
 918
 919         return identify_string(concat, len1 + len2 + 1);
 920 }
 921
 922 string_t make_string(const char *string)
 923 {
 924         size_t      len   = strlen(string) + 1;
 925         char *const space = obstack_alloc(&symbol_obstack, len);
 926         memcpy(space, string, len);
 927
 928         return identify_string(space, len);
 929 }
 930
 931 static void grow_symbol(utf32 const tc)
 932 {
 933         struct obstack *const o  = &symbol_obstack;
 934         if (tc < 0x80U) {
 935                 obstack_1grow(o, tc);
 936         } else if (tc < 0x800) {
 937                 obstack_1grow(o, 0xC0 | (tc >> 6));
 938                 obstack_1grow(o, 0x80 | (tc & 0x3F));
 939         } else if (tc < 0x10000) {
 940                 obstack_1grow(o, 0xE0 | ( tc >> 12));
 941                 obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
 942                 obstack_1grow(o, 0x80 | ( tc        & 0x3F));
 943         } else {
 944                 obstack_1grow(o, 0xF0 | ( tc >> 18));
 945                 obstack_1grow(o, 0x80 | ((tc >> 12) & 0x3F));
 946                 obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
 947                 obstack_1grow(o, 0x80 | ( tc        & 0x3F));
 948         }
 949 }
 950
 951 /**
 952  * Parse a string literal and set lexer_token.
 953  */
 954 static void parse_string_literal(void)
 955 {
 956         const unsigned start_linenr = lexer_token.source_position.linenr;
 957
 958         eat('"');
 959
 960         while (true) {
 961                 switch (c) {
 962                 case '\\': {
 963                         utf32 const tc = parse_escape_sequence();
 964                         if (tc >= 0x100) {
 965                                 warningf(&lexer_token.source_position,
 966                                                 "escape sequence out of range");
 967                         }
 968                         obstack_1grow(&symbol_obstack, tc);
 969                         break;
 970                 }
 971
 972                 case EOF: {
 973                         source_position_t source_position;
 974                         source_position.input_name = lexer_token.source_position.input_name;
 975                         source_position.linenr     = start_linenr;
 976                         errorf(&source_position, "string has no end");
 977                         lexer_token.type = T_ERROR;
 978                         return;
 979                 }
 980
 981                 case '"':
 982                         next_char();
 983                         goto end_of_string;
 984
 985                 default:
 986                         grow_symbol(c);
 987                         next_char();
 988                         break;
 989                 }
 990         }
 991
 992 end_of_string:
 993
 994         /* TODO: concatenate multiple strings separated by whitespace... */
 995
 996         /* add finishing 0 to the string */
 997         obstack_1grow(&symbol_obstack, '\0');
 998         const size_t  size   = (size_t)obstack_object_size(&symbol_obstack);
 999         char         *string = obstack_finish(&symbol_obstack);
1000
1001         lexer_token.type    = T_STRING_LITERAL;
1002         lexer_token.literal = identify_string(string, size);
1003 }
1004
1005 /**
1006  * Parse a wide character constant and set lexer_token.
1007  */
1008 static void parse_wide_character_constant(void)
1009 {
1010         const unsigned start_linenr = lexer_token.source_position.linenr;
1011
1012         eat('\'');
1013
1014         while (true) {
1015                 switch (c) {
1016                 case '\\': {
1017                         const utf32 tc = parse_escape_sequence();
1018                         grow_symbol(tc);
1019                         break;
1020                 }
1021
1022                 MATCH_NEWLINE(
1023                         parse_error("newline while parsing character constant");
1024                         break;
1025                 )
1026
1027                 case '\'':
1028                         next_char();
1029                         goto end_of_wide_char_constant;
1030
1031                 case EOF: {
1032                         source_position_t source_position = lexer_token.source_position;
1033                         source_position.linenr = start_linenr;
1034                         errorf(&source_position, "EOF while parsing character constant");
1035                         lexer_token.type = T_ERROR;
1036                         return;
1037                 }
1038
1039                 default:
1040                         grow_symbol(c);
1041                         next_char();
1042                         break;
1043                 }
1044         }
1045
1046 end_of_wide_char_constant:;
1047         obstack_1grow(&symbol_obstack, '\0');
1048         size_t  size   = (size_t) obstack_object_size(&symbol_obstack) - 1;
1049         char   *string = obstack_finish(&symbol_obstack);
1050
1051         lexer_token.type     = T_WIDE_CHARACTER_CONSTANT;
1052         lexer_token.literal  = identify_string(string, size);
1053 }
1054
1055 /**
1056  * Parse a wide string literal and set lexer_token.
1057  */
1058 static void parse_wide_string_literal(void)
1059 {
1060         parse_string_literal();
1061         if (lexer_token.type == T_STRING_LITERAL)
1062                 lexer_token.type = T_WIDE_STRING_LITERAL;
1063 }
1064
1065 /**
1066  * Parse a character constant and set lexer_token.
1067  */
1068 static void parse_character_constant(void)
1069 {
1070         const unsigned start_linenr = lexer_token.source_position.linenr;
1071
1072         eat('\'');
1073
1074         while (true) {
1075                 switch (c) {
1076                 case '\\': {
1077                         utf32 const tc = parse_escape_sequence();
1078                         if (tc >= 0x100) {
1079                                 warningf(&lexer_token.source_position,
1080                                                 "escape sequence out of range");
1081                         }
1082                         obstack_1grow(&symbol_obstack, tc);
1083                         break;
1084                 }
1085
1086                 MATCH_NEWLINE(
1087                         parse_error("newline while parsing character constant");
1088                         break;
1089                 )
1090
1091                 case '\'':
1092                         next_char();
1093                         goto end_of_char_constant;
1094
1095                 case EOF: {
1096                         source_position_t source_position;
1097                         source_position.input_name = lexer_token.source_position.input_name;
1098                         source_position.linenr     = start_linenr;
1099                         errorf(&source_position, "EOF while parsing character constant");
1100                         lexer_token.type = T_ERROR;
1101                         return;
1102                 }
1103
1104                 default:
1105                         grow_symbol(c);
1106                         next_char();
1107                         break;
1108
1109                 }
1110         }
1111
1112 end_of_char_constant:;
1113         obstack_1grow(&symbol_obstack, '\0');
1114         const size_t        size   = (size_t)obstack_object_size(&symbol_obstack)-1;
1115         char         *const string = obstack_finish(&symbol_obstack);
1116
1117         lexer_token.type    = T_CHARACTER_CONSTANT;
1118         lexer_token.literal = identify_string(string, size);
1119 }
1120
1121 /**
1122  * Skip a multiline comment.
1123  */
1124 static void skip_multiline_comment(void)
1125 {
1126         unsigned start_linenr = lexer_token.source_position.linenr;
1127
1128         while (true) {
1129                 switch (c) {
1130                 case '/':
1131                         next_char();
1132                         if (c == '*') {
1133                                 /* nested comment, warn here */
1134                                 if (warning.comment) {
1135                                         warningf(&lexer_token.source_position, "'/*' within comment");
1136                                 }
1137                         }
1138                         break;
1139                 case '*':
1140                         next_char();
1141                         if (c == '/') {
1142                                 next_char();
1143                                 return;
1144                         }
1145                         break;
1146
1147                 MATCH_NEWLINE(break;)
1148
1149                 case EOF: {
1150                         source_position_t source_position;
1151                         source_position.input_name = lexer_token.source_position.input_name;
1152                         source_position.linenr     = start_linenr;
1153                         errorf(&source_position, "at end of file while looking for comment end");
1154                         return;
1155                 }
1156
1157                 default:
1158                         next_char();
1159                         break;
1160                 }
1161         }
1162 }
1163
1164 /**
1165  * Skip a single line comment.
1166  */
1167 static void skip_line_comment(void)
1168 {
1169         while (true) {
1170                 switch (c) {
1171                 case EOF:
1172                         return;
1173
1174                 case '\n':
1175                 case '\r':
1176                         return;
1177
1178                 case '\\':
1179                         next_char();
1180                         if (c == '\n' || c == '\r') {
1181                                 if (warning.comment)
1182                                         warningf(&lexer_token.source_position, "multi-line comment");
1183                                 return;
1184                         }
1185                         break;
1186
1187                 default:
1188                         next_char();
1189                         break;
1190                 }
1191         }
1192 }
1193
1194 /** The current preprocessor token. */
1195 static token_t pp_token;
1196
1197 /**
1198  * Read the next preprocessor token.
1199  */
1200 static inline void next_pp_token(void)
1201 {
1202         lexer_next_preprocessing_token();
1203         pp_token = lexer_token;
1204 }
1205
1206 /**
1207  * Eat all preprocessor tokens until newline.
1208  */
1209 static void eat_until_newline(void)
1210 {
1211         while (pp_token.type != '\n' && pp_token.type != T_EOF) {
1212                 next_pp_token();
1213         }
1214 }
1215
1216 /**
1217  * Handle the define directive.
1218  */
1219 static void define_directive(void)
1220 {
1221         lexer_next_preprocessing_token();
1222         if (lexer_token.type != T_IDENTIFIER) {
1223                 parse_error("expected identifier after #define\n");
1224                 eat_until_newline();
1225         }
1226 }
1227
1228 /**
1229  * Handle the ifdef directive.
1230  */
1231 static void ifdef_directive(int is_ifndef)
1232 {
1233         (void) is_ifndef;
1234         lexer_next_preprocessing_token();
1235         //expect_identifier();
1236         //extect_newline();
1237 }
1238
1239 /**
1240  * Handle the endif directive.
1241  */
1242 static void endif_directive(void)
1243 {
1244         //expect_newline();
1245 }
1246
1247 /**
1248  * Parse the line directive.
1249  */
1250 static void parse_line_directive(void)
1251 {
1252         if (pp_token.type != T_INTEGER) {
1253                 parse_error("expected integer");
1254         } else {
1255                 /* use offset -1 as this is about the next line */
1256                 lexer_token.source_position.linenr = atoi(pp_token.literal.begin) - 1;
1257                 next_pp_token();
1258         }
1259         if (pp_token.type == T_STRING_LITERAL) {
1260                 lexer_token.source_position.input_name = pp_token.literal.begin;
1261                 next_pp_token();
1262         }
1263
1264         eat_until_newline();
1265 }
1266
1267 /**
1268  * STDC pragmas.
1269  */
1270 typedef enum stdc_pragma_kind_t {
1271         STDC_UNKNOWN,
1272         STDC_FP_CONTRACT,
1273         STDC_FENV_ACCESS,
1274         STDC_CX_LIMITED_RANGE
1275 } stdc_pragma_kind_t;
1276
1277 /**
1278  * STDC pragma values.
1279  */
1280 typedef enum stdc_pragma_value_kind_t {
1281         STDC_VALUE_UNKNOWN,
1282         STDC_VALUE_ON,
1283         STDC_VALUE_OFF,
1284         STDC_VALUE_DEFAULT
1285 } stdc_pragma_value_kind_t;
1286
1287 /**
1288  * Parse a pragma directive.
1289  */
1290 static void parse_pragma(void)
1291 {
1292         bool unknown_pragma = true;
1293
1294         next_pp_token();
1295         if (pp_token.symbol->pp_ID == TP_STDC) {
1296                 stdc_pragma_kind_t kind = STDC_UNKNOWN;
1297                 /* a STDC pragma */
1298                 if (c_mode & _C99) {
1299                         next_pp_token();
1300
1301                         switch (pp_token.symbol->pp_ID) {
1302                         case TP_FP_CONTRACT:
1303                                 kind = STDC_FP_CONTRACT;
1304                                 break;
1305                         case TP_FENV_ACCESS:
1306                                 kind = STDC_FENV_ACCESS;
1307                                 break;
1308                         case TP_CX_LIMITED_RANGE:
1309                                 kind = STDC_CX_LIMITED_RANGE;
1310                                 break;
1311                         default:
1312                                 break;
1313                         }
1314                         if (kind != STDC_UNKNOWN) {
1315                                 stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN;
1316                                 next_pp_token();
1317                                 switch (pp_token.symbol->pp_ID) {
1318                                 case TP_ON:
1319                                         value = STDC_VALUE_ON;
1320                                         break;
1321                                 case TP_OFF:
1322                                         value = STDC_VALUE_OFF;
1323                                         break;
1324                                 case TP_DEFAULT:
1325                                         value = STDC_VALUE_DEFAULT;
1326                                         break;
1327                                 default:
1328                                         break;
1329                                 }
1330                                 if (value != STDC_VALUE_UNKNOWN) {
1331                                         unknown_pragma = false;
1332                                 } else {
1333                                         errorf(&pp_token.source_position, "bad STDC pragma argument");
1334                                 }
1335                         }
1336                 }
1337         } else {
1338                 unknown_pragma = true;
1339         }
1340         eat_until_newline();
1341         if (unknown_pragma && warning.unknown_pragmas) {
1342                 warningf(&pp_token.source_position, "encountered unknown #pragma");
1343         }
1344 }
1345
1346 /**
1347  * Parse a preprocessor non-null directive.
1348  */
1349 static void parse_preprocessor_identifier(void)
1350 {
1351         assert(pp_token.type == T_IDENTIFIER);
1352         symbol_t *symbol = pp_token.symbol;
1353
1354         switch (symbol->pp_ID) {
1355         case TP_include:
1356                 printf("include - enable header name parsing!\n");
1357                 break;
1358         case TP_define:
1359                 define_directive();
1360                 break;
1361         case TP_ifdef:
1362                 ifdef_directive(0);
1363                 break;
1364         case TP_ifndef:
1365                 ifdef_directive(1);
1366                 break;
1367         case TP_endif:
1368                 endif_directive();
1369                 break;
1370         case TP_line:
1371                 next_pp_token();
1372                 parse_line_directive();
1373                 break;
1374         case TP_if:
1375         case TP_else:
1376         case TP_elif:
1377         case TP_undef:
1378         case TP_error:
1379                 /* TODO; output the rest of the line */
1380                 parse_error("#error directive: ");
1381                 break;
1382         case TP_pragma:
1383                 parse_pragma();
1384                 break;
1385         }
1386 }
1387
1388 /**
1389  * Parse a preprocessor directive.
1390  */
1391 static void parse_preprocessor_directive(void)
1392 {
1393         next_pp_token();
1394
1395         switch (pp_token.type) {
1396         case T_IDENTIFIER:
1397                 parse_preprocessor_identifier();
1398                 break;
1399         case T_INTEGER:
1400                 parse_line_directive();
1401                 break;
1402         case '\n':
1403                 /* NULL directive, see §6.10.7 */
1404                 break;
1405         default:
1406                 parse_error("invalid preprocessor directive");
1407                 eat_until_newline();
1408                 break;
1409         }
1410 }
1411
1412 #define MAYBE_PROLOG                                       \
1413                         next_char();                                   \
1414                         while (true) {                                 \
1415                                 switch (c) {
1416
1417 #define MAYBE(ch, set_type)                                \
1418                                 case ch:                                   \
1419                                         next_char();                           \
1420                                         lexer_token.type = set_type;           \
1421                                         return;
1422
1423 /* must use this as last thing */
1424 #define MAYBE_MODE(ch, set_type, mode)                     \
1425                                 case ch:                                   \
1426                                         if (c_mode & mode) {                   \
1427                                                 next_char();                       \
1428                                                 lexer_token.type = set_type;       \
1429                                                 return;                            \
1430                                         }                                      \
1431                                         /* fallthrough */
1432
1433 #define ELSE_CODE(code)                                    \
1434                                 default:                                   \
1435                                         code                                   \
1436                                 }                                          \
1437                         } /* end of while (true) */                    \
1438                         break;
1439
1440 #define ELSE(set_type)                                     \
1441                 ELSE_CODE(                                         \
1442                         lexer_token.type = set_type;                   \
1443                         return;                                        \
1444                 )
1445
1446 void lexer_next_preprocessing_token(void)
1447 {
1448         while (true) {
1449                 switch (c) {
1450                 case ' ':
1451                 case '\t':
1452                         next_char();
1453                         break;
1454
1455                 MATCH_NEWLINE(
1456                         lexer_token.type = '\n';
1457                         return;
1458                 )
1459
1460                 SYMBOL_CHARS
1461                         parse_symbol();
1462                         /* might be a wide string ( L"string" ) */
1463                         if (lexer_token.symbol == symbol_L) {
1464                                 switch (c) {
1465                                         case '"':  parse_wide_string_literal();     break;
1466                                         case '\'': parse_wide_character_constant(); break;
1467                                 }
1468                         }
1469                         return;
1470
1471                 DIGITS
1472                         parse_number();
1473                         return;
1474
1475                 case '"':
1476                         parse_string_literal();
1477                         return;
1478
1479                 case '\'':
1480                         parse_character_constant();
1481                         return;
1482
1483                 case '.':
1484                         MAYBE_PROLOG
1485                                 DIGITS
1486                                         put_back(c);
1487                                         c = '.';
1488                                         parse_number();
1489                                         return;
1490
1491                                 case '.':
1492                                         MAYBE_PROLOG
1493                                         MAYBE('.', T_DOTDOTDOT)
1494                                         ELSE_CODE(
1495                                                 put_back(c);
1496                                                 c = '.';
1497                                                 lexer_token.type = '.';
1498                                                 return;
1499                                         )
1500                         ELSE('.')
1501                 case '&':
1502                         MAYBE_PROLOG
1503                         MAYBE('&', T_ANDAND)
1504                         MAYBE('=', T_ANDEQUAL)
1505                         ELSE('&')
1506                 case '*':
1507                         MAYBE_PROLOG
1508                         MAYBE('=', T_ASTERISKEQUAL)
1509                         ELSE('*')
1510                 case '+':
1511                         MAYBE_PROLOG
1512                         MAYBE('+', T_PLUSPLUS)
1513                         MAYBE('=', T_PLUSEQUAL)
1514                         ELSE('+')
1515                 case '-':
1516                         MAYBE_PROLOG
1517                         MAYBE('>', T_MINUSGREATER)
1518                         MAYBE('-', T_MINUSMINUS)
1519                         MAYBE('=', T_MINUSEQUAL)
1520                         ELSE('-')
1521                 case '!':
1522                         MAYBE_PROLOG
1523                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1524                         ELSE('!')
1525                 case '/':
1526                         MAYBE_PROLOG
1527                         MAYBE('=', T_SLASHEQUAL)
1528                                 case '*':
1529                                         next_char();
1530                                         skip_multiline_comment();
1531                                         lexer_next_preprocessing_token();
1532                                         return;
1533                                 case '/':
1534                                         next_char();
1535                                         skip_line_comment();
1536                                         lexer_next_preprocessing_token();
1537                                         return;
1538                         ELSE('/')
1539                 case '%':
1540                         MAYBE_PROLOG
1541                         MAYBE('>', '}')
1542                         MAYBE('=', T_PERCENTEQUAL)
1543                                 case ':':
1544                                         MAYBE_PROLOG
1545                                                 case '%':
1546                                                         MAYBE_PROLOG
1547                                                         MAYBE(':', T_HASHHASH)
1548                                                         ELSE_CODE(
1549                                                                 put_back(c);
1550                                                                 c = '%';
1551                                                                 lexer_token.type = '#';
1552                                                                 return;
1553                                                         )
1554                                         ELSE('#')
1555                         ELSE('%')
1556                 case '<':
1557                         MAYBE_PROLOG
1558                         MAYBE(':', '[')
1559                         MAYBE('%', '{')
1560                         MAYBE('=', T_LESSEQUAL)
1561                                 case '<':
1562                                         MAYBE_PROLOG
1563                                         MAYBE('=', T_LESSLESSEQUAL)
1564                                         ELSE(T_LESSLESS)
1565                         ELSE('<')
1566                 case '>':
1567                         MAYBE_PROLOG
1568                         MAYBE('=', T_GREATEREQUAL)
1569                                 case '>':
1570                                         MAYBE_PROLOG
1571                                         MAYBE('=', T_GREATERGREATEREQUAL)
1572                                         ELSE(T_GREATERGREATER)
1573                         ELSE('>')
1574                 case '^':
1575                         MAYBE_PROLOG
1576                         MAYBE('=', T_CARETEQUAL)
1577                         ELSE('^')
1578                 case '|':
1579                         MAYBE_PROLOG
1580                         MAYBE('=', T_PIPEEQUAL)
1581                         MAYBE('|', T_PIPEPIPE)
1582                         ELSE('|')
1583                 case ':':
1584                         MAYBE_PROLOG
1585                         MAYBE('>', ']')
1586                         MAYBE_MODE(':', T_COLONCOLON, _CXX)
1587                         ELSE(':')
1588                 case '=':
1589                         MAYBE_PROLOG
1590                         MAYBE('=', T_EQUALEQUAL)
1591                         ELSE('=')
1592                 case '#':
1593                         MAYBE_PROLOG
1594                         MAYBE('#', T_HASHHASH)
1595                         ELSE('#')
1596
1597                 case '?':
1598                 case '[':
1599                 case ']':
1600                 case '(':
1601                 case ')':
1602                 case '{':
1603                 case '}':
1604                 case '~':
1605                 case ';':
1606                 case ',':
1607                 case '\\':
1608                         lexer_token.type = c;
1609                         next_char();
1610                         return;
1611
1612                 case EOF:
1613                         lexer_token.type = T_EOF;
1614                         return;
1615
1616                 default:
1617 dollar_sign:
1618                         errorf(&lexer_token.source_position, "unknown character '%c' found", c);
1619                         next_char();
1620                         lexer_token.type = T_ERROR;
1621                         return;
1622                 }
1623         }
1624 }
1625
1626 void lexer_next_token(void)
1627 {
1628         lexer_next_preprocessing_token();
1629
1630         while (lexer_token.type == '\n') {
1631 newline_found:
1632                 lexer_next_preprocessing_token();
1633         }
1634
1635         if (lexer_token.type == '#') {
1636                 parse_preprocessor_directive();
1637                 goto newline_found;
1638         }
1639 }
1640
1641 void init_lexer(void)
1642 {
1643         strset_init(&stringset);
1644         symbol_L = symbol_table_insert("L");
1645 }
1646
1647 void lexer_open_stream(FILE *stream, const char *input_name)
1648 {
1649         input                                  = stream;
1650         lexer_token.source_position.linenr     = 0;
1651         lexer_token.source_position.input_name = input_name;
1652
1653         bufpos = NULL;
1654         bufend = NULL;
1655
1656         /* place a virtual \n at the beginning so the lexer knows that we're
1657          * at the beginning of a line */
1658         c = '\n';
1659 }
1660
1661 void lexer_open_buffer(const char *buffer, size_t len, const char *input_name)
1662 {
1663         input                                  = NULL;
1664         lexer_token.source_position.linenr     = 0;
1665         lexer_token.source_position.input_name = input_name;
1666
1667 #if 0 // TODO
1668         bufpos = buffer;
1669         bufend = buffer + len;
1670
1671         /* place a virtual \n at the beginning so the lexer knows that we're
1672          * at the beginning of a line */
1673         c = '\n';
1674 #else
1675         (void)buffer;
1676         (void)len;
1677         panic("builtin lexing not done yet");
1678 #endif
1679 }
1680
1681 void exit_lexer(void)
1682 {
1683         strset_destroy(&stringset);
1684 }
1685
1686 static __attribute__((unused))
1687 void dbg_pos(const source_position_t source_position)
1688 {
1689         fprintf(stdout, "%s:%u\n", source_position.input_name,
1690                 source_position.linenr);
1691         fflush(stdout);
1692 }