nsz Git - cparser/blob - lexer.c

   1 /*
   2  * This file is part of cparser.
   3  * Copyright (C) 2007-2009 Matthias Braun <matze@braunis.de>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; either version 2
   8  * of the License, or (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  18  * 02111-1307, USA.
  19  */
  20 #include <config.h>
  21
  22 #include "diagnostic.h"
  23 #include "lexer.h"
  24 #include "symbol_t.h"
  25 #include "token_t.h"
  26 #include "symbol_table_t.h"
  27 #include "adt/error.h"
  28 #include "adt/strset.h"
  29 #include "adt/util.h"
  30 #include "types.h"
  31 #include "type_t.h"
  32 #include "target_architecture.h"
  33 #include "parser.h"
  34 #include "warning.h"
  35 #include "lang_features.h"
  36
  37 #include <assert.h>
  38 #include <errno.h>
  39 #include <string.h>
  40 #include <stdbool.h>
  41 #include <ctype.h>
  42
  43 #ifndef _WIN32
  44 #include <strings.h>
  45 #endif
  46
  47 //#define DEBUG_CHARS
  48 #define MAX_PUTBACK 3
  49 #define BUF_SIZE    1024
  50
  51 #if defined(_WIN32) || defined(__CYGWIN__)
  52 /* No strtold on windows and no replacement yet */
  53 #define strtold(s, e) strtod(s, e)
  54 #endif
  55
  56 static utf32        c;
  57 token_t             lexer_token;
  58 symbol_t           *symbol_L;
  59 static FILE        *input;
  60 static utf32        buf[BUF_SIZE + MAX_PUTBACK];
  61 static const utf32 *bufend;
  62 static const utf32 *bufpos;
  63 static strset_t     stringset;
  64 bool                allow_dollar_in_symbol = true;
  65
  66 /**
  67  * Prints a parse error message at the current token.
  68  *
  69  * @param msg   the error message
  70  */
  71 static void parse_error(const char *msg)
  72 {
  73         errorf(&lexer_token.source_position, "%s", msg);
  74 }
  75
  76 /**
  77  * Prints an internal error message at the current token.
  78  *
  79  * @param msg   the error message
  80  */
  81 static NORETURN internal_error(const char *msg)
  82 {
  83         internal_errorf(&lexer_token.source_position, "%s", msg);
  84 }
  85
  86 static size_t read_block(unsigned char *const read_buf, size_t const n)
  87 {
  88         size_t const s = fread(read_buf, 1, n, input);
  89         if (s == 0) {
  90                 /* on OS/X ferror appears to return true on eof as well when running
  91                  * the application in gdb... */
  92                 if (!feof(input) && ferror(input))
  93                         parse_error("read from input failed");
  94                 buf[MAX_PUTBACK] = EOF;
  95                 bufpos           = buf + MAX_PUTBACK;
  96                 bufend           = buf + MAX_PUTBACK + 1;
  97         }
  98         return s;
  99 }
 100
 101 static void decode_iso_8859_1(void)
 102 {
 103         unsigned char read_buf[BUF_SIZE];
 104         size_t const s = read_block(read_buf, sizeof(read_buf));
 105         if (s == 0)
 106                 return;
 107
 108         unsigned char const *src = read_buf;
 109         unsigned char const *end = read_buf + s;
 110         utf32               *dst = buf + MAX_PUTBACK;
 111         while (src != end)
 112                 *dst++ = *src++;
 113
 114         bufpos = buf + MAX_PUTBACK;
 115         bufend = dst;
 116 }
 117
 118 static void decode_iso_8859_15(void)
 119 {
 120         unsigned char read_buf[BUF_SIZE];
 121         size_t const s = read_block(read_buf, sizeof(read_buf));
 122         if (s == 0)
 123                 return;
 124
 125         unsigned char const *src = read_buf;
 126         unsigned char const *end = read_buf + s;
 127         utf32               *dst = buf + MAX_PUTBACK;
 128         while (src != end) {
 129                 utf32 tc = *src++;
 130                 switch (tc) {
 131                         case 0xA4: tc = 0x20AC; break; // €
 132                         case 0xA6: tc = 0x0160; break; // Š
 133                         case 0xA8: tc = 0x0161; break; // š
 134                         case 0xB4: tc = 0x017D; break; // Ž
 135                         case 0xB8: tc = 0x017E; break; // ž
 136                         case 0xBC: tc = 0x0152; break; // Œ
 137                         case 0xBD: tc = 0x0153; break; // œ
 138                         case 0xBE: tc = 0x0178; break; // Ÿ
 139                 }
 140                 *dst++ = tc;
 141         }
 142
 143         bufpos = buf + MAX_PUTBACK;
 144         bufend = dst;
 145 }
 146
 147 static void decode_utf8(void)
 148 {
 149         static utf32  part_decoded_min_code;
 150         static utf32  part_decoded_char;
 151         static size_t part_decoded_rest_len;
 152
 153         do {
 154                 unsigned char read_buf[BUF_SIZE];
 155                 size_t const s = read_block(read_buf, sizeof(read_buf));
 156                 if (s == 0) {
 157                         if (part_decoded_rest_len > 0)
 158                                 parse_error("incomplete input char at end of input");
 159                         return;
 160                 }
 161
 162                 unsigned char const *src = read_buf;
 163                 unsigned char const *end = read_buf + s;
 164                 utf32               *dst = buf + MAX_PUTBACK;
 165                 utf32                decoded;
 166                 utf32                min_code;
 167
 168                 if (part_decoded_rest_len != 0) {
 169                         min_code              = part_decoded_min_code;
 170                         decoded               = part_decoded_char;
 171                         size_t const rest_len = part_decoded_rest_len;
 172                         part_decoded_rest_len = 0;
 173                         switch (rest_len) {
 174                                 case 4:  goto realign;
 175                                 case 3:  goto three_more;
 176                                 case 2:  goto two_more;
 177                                 default: goto one_more;
 178                         }
 179                 }
 180
 181                 while (src != end) {
 182                         if ((*src & 0x80) == 0) {
 183                                 decoded = *src++;
 184                         } else if ((*src & 0xE0) == 0xC0) {
 185                                 min_code = 0x80;
 186                                 decoded  = *src++ & 0x1F;
 187 one_more:
 188                                 if (src == end) {
 189                                         part_decoded_min_code = min_code;
 190                                         part_decoded_char     = decoded;
 191                                         part_decoded_rest_len = 1;
 192                                         break;
 193                                 }
 194                                 if ((*src & 0xC0) == 0x80) {
 195                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 196                                 } else {
 197                                         goto invalid_char;
 198                                 }
 199                                 if (decoded < min_code                      ||
 200                                                 decoded > 0x10FFFF                      ||
 201                                                 (0xD800 <= decoded && decoded < 0xE000) || // high/low surrogates
 202                                                 (0xFDD0 <= decoded && decoded < 0xFDF0) || // noncharacters
 203                                                 (decoded & 0xFFFE) == 0xFFFE) {            // noncharacters
 204                                         parse_error("invalid byte sequence in input");
 205                                 }
 206                         } else if ((*src & 0xF0) == 0xE0) {
 207                                 min_code = 0x800;
 208                                 decoded  = *src++ & 0x0F;
 209 two_more:
 210                                 if (src == end) {
 211                                         part_decoded_min_code = min_code;
 212                                         part_decoded_char     = decoded;
 213                                         part_decoded_rest_len = 2;
 214                                         break;
 215                                 }
 216                                 if ((*src & 0xC0) == 0x80) {
 217                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 218                                 } else {
 219                                         goto invalid_char;
 220                                 }
 221                                 goto one_more;
 222                         } else if ((*src & 0xF8) == 0xF0) {
 223                                 min_code = 0x10000;
 224                                 decoded  = *src++ & 0x07;
 225 three_more:
 226                                 if (src == end) {
 227                                         part_decoded_min_code = min_code;
 228                                         part_decoded_char     = decoded;
 229                                         part_decoded_rest_len = 3;
 230                                         break;
 231                                 }
 232                                 if ((*src & 0xC0) == 0x80) {
 233                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 234                                 } else {
 235                                         goto invalid_char;
 236                                 }
 237                                 goto two_more;
 238                         } else {
 239 invalid_char:
 240                                 parse_error("invalid byte sequence in input");
 241 realign:
 242                                 do {
 243                                         ++src;
 244                                         if (src == end) {
 245                                                 part_decoded_rest_len = 4;
 246                                                 break;
 247                                         }
 248                                 } while ((*src & 0xC0) == 0x80 || (*src & 0xF8) == 0xF8);
 249                                 continue;
 250                         }
 251                         *dst++ = decoded;
 252                 }
 253
 254                 bufpos = buf + MAX_PUTBACK;
 255                 bufend = dst;
 256         } while (bufpos == bufend);
 257 }
 258
 259 static void decode_windows_1252(void)
 260 {
 261         unsigned char read_buf[BUF_SIZE];
 262         size_t const s = read_block(read_buf, sizeof(read_buf));
 263         if (s == 0)
 264                 return;
 265
 266         unsigned char const *src = read_buf;
 267         unsigned char const *end = read_buf + s;
 268         utf32               *dst = buf + MAX_PUTBACK;
 269         while (src != end) {
 270                 utf32 tc = *src++;
 271                 switch (tc) {
 272                         case 0x80: tc = 0x20AC; break; // €
 273                         case 0x82: tc = 0x201A; break; // ‚
 274                         case 0x83: tc = 0x0192; break; // ƒ
 275                         case 0x84: tc = 0x201E; break; // „
 276                         case 0x85: tc = 0x2026; break; // …
 277                         case 0x86: tc = 0x2020; break; // †
 278                         case 0x87: tc = 0x2021; break; // ‡
 279                         case 0x88: tc = 0x02C6; break; // ˆ
 280                         case 0x89: tc = 0x2030; break; // ‰
 281                         case 0x8A: tc = 0x0160; break; // Š
 282                         case 0x8B: tc = 0x2039; break; // ‹
 283                         case 0x8C: tc = 0x0152; break; // Œ
 284                         case 0x8E: tc = 0x017D; break; // Ž
 285                         case 0x91: tc = 0x2018; break; // ‘
 286                         case 0x92: tc = 0x2019; break; // ’
 287                         case 0x93: tc = 0x201C; break; // “
 288                         case 0x94: tc = 0x201D; break; // ”
 289                         case 0x95: tc = 0x2022; break; // •
 290                         case 0x96: tc = 0x2013; break; // –
 291                         case 0x97: tc = 0x2014; break; // —
 292                         case 0x98: tc = 0x02DC; break; // ˜
 293                         case 0x99: tc = 0x2122; break; // ™
 294                         case 0x9A: tc = 0x0161; break; // š
 295                         case 0x9B: tc = 0x203A; break; // ›
 296                         case 0x9C: tc = 0x0153; break; // œ
 297                         case 0x9E: tc = 0x017E; break; // ž
 298                         case 0x9F: tc = 0x0178; break; // Ÿ
 299                 }
 300                 *dst++ = tc;
 301         }
 302
 303         bufpos = buf + MAX_PUTBACK;
 304         bufend = dst;
 305 }
 306
 307 typedef void (*decoder_t)(void);
 308
 309 static decoder_t decoder = decode_utf8;
 310
 311 typedef struct named_decoder_t {
 312         char const *name;
 313         decoder_t   decoder;
 314 } named_decoder_t;
 315
 316 static named_decoder_t const decoders[] = {
 317         { "CP819",           decode_iso_8859_1   }, // offical alias
 318         { "IBM819",          decode_iso_8859_1   }, // offical alias
 319         { "ISO-8859-1",      decode_iso_8859_1   }, // offical alias
 320         { "ISO-8859-15",     decode_iso_8859_15  }, // offical name
 321         { "ISO8859-1",       decode_iso_8859_1   },
 322         { "ISO8859-15",      decode_iso_8859_15  },
 323         { "ISO_8859-1",      decode_iso_8859_1   }, // offical alias
 324         { "ISO_8859-15",     decode_iso_8859_15  }, // offical alias
 325         { "ISO_8859-1:1987", decode_iso_8859_1   }, // offical name
 326         { "Latin-9",         decode_iso_8859_15  }, // offical alias
 327         { "UTF-8",           decode_utf8         }, // offical name
 328         { "csISOLatin1",     decode_iso_8859_1   }, // offical alias
 329         { "cp1252",          decode_windows_1252 },
 330         { "iso-ir-100",      decode_iso_8859_1   }, // offical alias
 331         { "l1",              decode_iso_8859_1   }, // offical alias
 332         { "latin1",          decode_iso_8859_1   }, // offical alias
 333         { "windows-1252",    decode_windows_1252 }, // official name
 334
 335         { NULL,              NULL                }
 336 };
 337
 338 /** strcasecmp is not part of C99 so we need our own implementation here */
 339 static int my_strcasecmp(const char *s1, const char *s2)
 340 {
 341         for ( ; *s1 != 0; ++s1, ++s2) {
 342                 if (tolower(*s1) != tolower(*s2))
 343                         break;
 344         }
 345         return (unsigned char)*s1 - (unsigned char)*s2;
 346 }
 347
 348 void select_input_encoding(char const* const encoding)
 349 {
 350         for (named_decoder_t const *i = decoders; i->name != NULL; ++i) {
 351                 if (my_strcasecmp(encoding, i->name) != 0)
 352                         continue;
 353                 decoder = i->decoder;
 354                 return;
 355         }
 356         fprintf(stderr, "error: input encoding \"%s\" not supported\n", encoding);
 357 }
 358
 359 static inline void next_real_char(void)
 360 {
 361         assert(bufpos <= bufend);
 362         if (bufpos >= bufend) {
 363                 if (input == NULL) {
 364                         c = EOF;
 365                         return;
 366                 }
 367                 decoder();
 368         }
 369         c = *bufpos++;
 370 }
 371
 372 /**
 373  * Put a character back into the buffer.
 374  *
 375  * @param pc  the character to put back
 376  */
 377 static inline void put_back(utf32 const pc)
 378 {
 379         assert(bufpos > buf);
 380         *(--bufpos - buf + buf) = pc;
 381
 382 #ifdef DEBUG_CHARS
 383         printf("putback '%lc'\n", pc);
 384 #endif
 385 }
 386
 387 static inline void next_char(void);
 388
 389 #define MATCH_NEWLINE(code)                   \
 390         case '\r':                                \
 391                 next_char();                          \
 392                 if (c == '\n') {                      \
 393                         next_char();                      \
 394                 }                                     \
 395                 lexer_token.source_position.linenr++; \
 396                 code                                  \
 397         case '\n':                                \
 398                 next_char();                          \
 399                 lexer_token.source_position.linenr++; \
 400                 code
 401
 402 #define eat(c_type)  do { assert(c == c_type); next_char(); } while (0)
 403
 404 static void maybe_concat_lines(void)
 405 {
 406         eat('\\');
 407
 408         switch (c) {
 409         MATCH_NEWLINE(return;)
 410
 411         default:
 412                 break;
 413         }
 414
 415         put_back(c);
 416         c = '\\';
 417 }
 418
 419 /**
 420  * Set c to the next input character, ie.
 421  * after expanding trigraphs.
 422  */
 423 static inline void next_char(void)
 424 {
 425         next_real_char();
 426
 427         /* filter trigraphs */
 428         if (UNLIKELY(c == '\\')) {
 429                 maybe_concat_lines();
 430                 goto end_of_next_char;
 431         }
 432
 433         if (LIKELY(c != '?'))
 434                 goto end_of_next_char;
 435
 436         next_real_char();
 437         if (LIKELY(c != '?')) {
 438                 put_back(c);
 439                 c = '?';
 440                 goto end_of_next_char;
 441         }
 442
 443         next_real_char();
 444         switch (c) {
 445         case '=': c = '#'; break;
 446         case '(': c = '['; break;
 447         case '/': c = '\\'; maybe_concat_lines(); break;
 448         case ')': c = ']'; break;
 449         case '\'': c = '^'; break;
 450         case '<': c = '{'; break;
 451         case '!': c = '|'; break;
 452         case '>': c = '}'; break;
 453         case '-': c = '~'; break;
 454         default:
 455                 put_back(c);
 456                 put_back('?');
 457                 c = '?';
 458                 break;
 459         }
 460
 461 end_of_next_char:;
 462 #ifdef DEBUG_CHARS
 463         printf("nchar '%c'\n", c);
 464 #endif
 465 }
 466
 467 #define SYMBOL_CHARS  \
 468         case '$': if (!allow_dollar_in_symbol) goto dollar_sign; \
 469         case 'a':         \
 470         case 'b':         \
 471         case 'c':         \
 472         case 'd':         \
 473         case 'e':         \
 474         case 'f':         \
 475         case 'g':         \
 476         case 'h':         \
 477         case 'i':         \
 478         case 'j':         \
 479         case 'k':         \
 480         case 'l':         \
 481         case 'm':         \
 482         case 'n':         \
 483         case 'o':         \
 484         case 'p':         \
 485         case 'q':         \
 486         case 'r':         \
 487         case 's':         \
 488         case 't':         \
 489         case 'u':         \
 490         case 'v':         \
 491         case 'w':         \
 492         case 'x':         \
 493         case 'y':         \
 494         case 'z':         \
 495         case 'A':         \
 496         case 'B':         \
 497         case 'C':         \
 498         case 'D':         \
 499         case 'E':         \
 500         case 'F':         \
 501         case 'G':         \
 502         case 'H':         \
 503         case 'I':         \
 504         case 'J':         \
 505         case 'K':         \
 506         case 'L':         \
 507         case 'M':         \
 508         case 'N':         \
 509         case 'O':         \
 510         case 'P':         \
 511         case 'Q':         \
 512         case 'R':         \
 513         case 'S':         \
 514         case 'T':         \
 515         case 'U':         \
 516         case 'V':         \
 517         case 'W':         \
 518         case 'X':         \
 519         case 'Y':         \
 520         case 'Z':         \
 521         case '_':
 522
 523 #define DIGITS        \
 524         case '0':         \
 525         case '1':         \
 526         case '2':         \
 527         case '3':         \
 528         case '4':         \
 529         case '5':         \
 530         case '6':         \
 531         case '7':         \
 532         case '8':         \
 533         case '9':
 534
 535 /**
 536  * Read a symbol from the input and build
 537  * the lexer_token.
 538  */
 539 static void parse_symbol(void)
 540 {
 541         obstack_1grow(&symbol_obstack, (char) c);
 542         next_char();
 543
 544         while (true) {
 545                 switch (c) {
 546                 DIGITS
 547                 SYMBOL_CHARS
 548                         obstack_1grow(&symbol_obstack, (char) c);
 549                         next_char();
 550                         break;
 551
 552                 default:
 553 dollar_sign:
 554                         goto end_symbol;
 555                 }
 556         }
 557
 558 end_symbol:
 559         obstack_1grow(&symbol_obstack, '\0');
 560
 561         char     *string = obstack_finish(&symbol_obstack);
 562         symbol_t *symbol = symbol_table_insert(string);
 563
 564         lexer_token.type   = symbol->ID;
 565         lexer_token.symbol = symbol;
 566
 567         if (symbol->string != string) {
 568                 obstack_free(&symbol_obstack, string);
 569         }
 570 }
 571
 572 /**
 573  * parse suffixes like 'LU' or 'f' after numbers
 574  */
 575 static void parse_number_suffix(void)
 576 {
 577         assert(obstack_object_size(&symbol_obstack) == 0);
 578         while (true) {
 579                 switch (c) {
 580                 SYMBOL_CHARS
 581                         obstack_1grow(&symbol_obstack, (char) c);
 582                         next_char();
 583                         break;
 584                 default:
 585                 dollar_sign:
 586                         goto finish_suffix;
 587                 }
 588         }
 589 finish_suffix:
 590         if (obstack_object_size(&symbol_obstack) == 0) {
 591                 lexer_token.symbol = NULL;
 592                 return;
 593         }
 594
 595         obstack_1grow(&symbol_obstack, '\0');
 596         char     *string = obstack_finish(&symbol_obstack);
 597         symbol_t *symbol = symbol_table_insert(string);
 598
 599         if (symbol->string != string) {
 600                 obstack_free(&symbol_obstack, string);
 601         }
 602         lexer_token.symbol = symbol;
 603 }
 604
 605 static string_t identify_string(char *string, size_t len)
 606 {
 607         /* TODO hash */
 608 #if 0
 609         const char *result = strset_insert(&stringset, concat);
 610         if (result != concat) {
 611                 obstack_free(&symbol_obstack, concat);
 612         }
 613 #else
 614         const char *result = string;
 615 #endif
 616         return (string_t) {result, len};
 617 }
 618
 619 /**
 620  * Parses a hex number including hex floats and set the
 621  * lexer_token.
 622  */
 623 static void parse_number_hex(void)
 624 {
 625         bool is_float   = false;
 626         bool has_digits = false;
 627
 628         assert(obstack_object_size(&symbol_obstack) == 0);
 629         while (isxdigit(c)) {
 630                 has_digits = true;
 631                 obstack_1grow(&symbol_obstack, (char) c);
 632                 next_char();
 633         }
 634
 635         if (c == '.') {
 636                 is_float = true;
 637                 obstack_1grow(&symbol_obstack, (char) c);
 638                 next_char();
 639
 640                 while (isxdigit(c)) {
 641                         has_digits = true;
 642                         obstack_1grow(&symbol_obstack, (char) c);
 643                         next_char();
 644                 }
 645         }
 646         if (c == 'p' || c == 'P') {
 647                 is_float = true;
 648                 obstack_1grow(&symbol_obstack, (char) c);
 649                 next_char();
 650
 651                 if (c == '-' || c == '+') {
 652                         obstack_1grow(&symbol_obstack, (char) c);
 653                         next_char();
 654                 }
 655
 656                 while (isxdigit(c)) {
 657                         obstack_1grow(&symbol_obstack, (char) c);
 658                         next_char();
 659                 }
 660         } else if (is_float) {
 661                 errorf(&lexer_token.source_position,
 662                        "hexadecimal floatingpoint constant requires an exponent");
 663         }
 664         obstack_1grow(&symbol_obstack, '\0');
 665
 666         size_t  size   = obstack_object_size(&symbol_obstack) - 1;
 667         char   *string = obstack_finish(&symbol_obstack);
 668         lexer_token.literal = identify_string(string, size);
 669
 670         lexer_token.type    =
 671                 is_float ? T_FLOATINGPOINT_HEXADECIMAL : T_INTEGER_HEXADECIMAL;
 672
 673         if (!has_digits) {
 674                 errorf(&lexer_token.source_position, "invalid number literal '0x%S'",
 675                        &lexer_token.literal);
 676                 lexer_token.literal.begin = "0";
 677                 lexer_token.literal.size  = 1;
 678         }
 679
 680         parse_number_suffix();
 681 }
 682
 683 /**
 684  * Returns true if the given char is a octal digit.
 685  *
 686  * @param char  the character to check
 687  */
 688 static bool is_octal_digit(utf32 chr)
 689 {
 690         switch (chr) {
 691         case '0':
 692         case '1':
 693         case '2':
 694         case '3':
 695         case '4':
 696         case '5':
 697         case '6':
 698         case '7':
 699                 return true;
 700         default:
 701                 return false;
 702         }
 703 }
 704
 705 /**
 706  * Parses a number and sets the lexer_token.
 707  */
 708 static void parse_number(void)
 709 {
 710         bool is_float   = false;
 711         bool has_digits = false;
 712
 713         assert(obstack_object_size(&symbol_obstack) == 0);
 714         if (c == '0') {
 715                 next_char();
 716                 if (c == 'x' || c == 'X') {
 717                         next_char();
 718                         parse_number_hex();
 719                         return;
 720                 } else {
 721                         has_digits = true;
 722                 }
 723                 obstack_1grow(&symbol_obstack, '0');
 724         }
 725
 726         while (isdigit(c)) {
 727                 has_digits = true;
 728                 obstack_1grow(&symbol_obstack, (char) c);
 729                 next_char();
 730         }
 731
 732         if (c == '.') {
 733                 is_float = true;
 734                 obstack_1grow(&symbol_obstack, '.');
 735                 next_char();
 736
 737                 while (isdigit(c)) {
 738                         has_digits = true;
 739                         obstack_1grow(&symbol_obstack, (char) c);
 740                         next_char();
 741                 }
 742         }
 743         if (c == 'e' || c == 'E') {
 744                 is_float = true;
 745                 obstack_1grow(&symbol_obstack, 'e');
 746                 next_char();
 747
 748                 if (c == '-' || c == '+') {
 749                         obstack_1grow(&symbol_obstack, (char) c);
 750                         next_char();
 751                 }
 752
 753                 while (isdigit(c)) {
 754                         obstack_1grow(&symbol_obstack, (char) c);
 755                         next_char();
 756                 }
 757         }
 758
 759         obstack_1grow(&symbol_obstack, '\0');
 760         size_t  size   = obstack_object_size(&symbol_obstack) - 1;
 761         char   *string = obstack_finish(&symbol_obstack);
 762         lexer_token.literal = identify_string(string, size);
 763
 764         /* is it an octal number? */
 765         if (is_float) {
 766                 lexer_token.type = T_FLOATINGPOINT;
 767         } else if (string[0] == '0') {
 768                 lexer_token.type = T_INTEGER_OCTAL;
 769
 770                 /* check for invalid octal digits */
 771                 for (size_t i= 0; i < size; ++i) {
 772                         char t = string[i];
 773                         if (t >= '8')
 774                                 errorf(&lexer_token.source_position,
 775                                        "invalid digit '%c' in octal number", t);
 776                 }
 777         } else {
 778                 lexer_token.type = T_INTEGER;
 779         }
 780
 781         if (!has_digits) {
 782                 errorf(&lexer_token.source_position, "invalid number literal '%S'",
 783                        &lexer_token.literal);
 784         }
 785
 786         parse_number_suffix();
 787 }
 788
 789 /**
 790  * Returns the value of a digit.
 791  * The only portable way to do it ...
 792  */
 793 static int digit_value(utf32 const digit)
 794 {
 795         switch (digit) {
 796         case '0': return 0;
 797         case '1': return 1;
 798         case '2': return 2;
 799         case '3': return 3;
 800         case '4': return 4;
 801         case '5': return 5;
 802         case '6': return 6;
 803         case '7': return 7;
 804         case '8': return 8;
 805         case '9': return 9;
 806         case 'a':
 807         case 'A': return 10;
 808         case 'b':
 809         case 'B': return 11;
 810         case 'c':
 811         case 'C': return 12;
 812         case 'd':
 813         case 'D': return 13;
 814         case 'e':
 815         case 'E': return 14;
 816         case 'f':
 817         case 'F': return 15;
 818         default:
 819                 internal_error("wrong character given");
 820         }
 821 }
 822
 823 /**
 824  * Parses an octal character sequence.
 825  *
 826  * @param first_digit  the already read first digit
 827  */
 828 static utf32 parse_octal_sequence(utf32 const first_digit)
 829 {
 830         assert(is_octal_digit(first_digit));
 831         utf32 value = digit_value(first_digit);
 832         if (!is_octal_digit(c)) return value;
 833         value = 8 * value + digit_value(c);
 834         next_char();
 835         if (!is_octal_digit(c)) return value;
 836         value = 8 * value + digit_value(c);
 837         next_char();
 838         return value;
 839 }
 840
 841 /**
 842  * Parses a hex character sequence.
 843  */
 844 static utf32 parse_hex_sequence(void)
 845 {
 846         utf32 value = 0;
 847         while (isxdigit(c)) {
 848                 value = 16 * value + digit_value(c);
 849                 next_char();
 850         }
 851         return value;
 852 }
 853
 854 /**
 855  * Parse an escape sequence.
 856  */
 857 static utf32 parse_escape_sequence(void)
 858 {
 859         eat('\\');
 860
 861         utf32 const ec = c;
 862         next_char();
 863
 864         switch (ec) {
 865         case '"':  return '"';
 866         case '\'': return '\'';
 867         case '\\': return '\\';
 868         case '?': return '\?';
 869         case 'a': return '\a';
 870         case 'b': return '\b';
 871         case 'f': return '\f';
 872         case 'n': return '\n';
 873         case 'r': return '\r';
 874         case 't': return '\t';
 875         case 'v': return '\v';
 876         case 'x':
 877                 return parse_hex_sequence();
 878         case '0':
 879         case '1':
 880         case '2':
 881         case '3':
 882         case '4':
 883         case '5':
 884         case '6':
 885         case '7':
 886                 return parse_octal_sequence(ec);
 887         case EOF:
 888                 parse_error("reached end of file while parsing escape sequence");
 889                 return EOF;
 890         /* \E is not documented, but handled, by GCC.  It is acceptable according
 891          * to §6.11.4, whereas \e is not. */
 892         case 'E':
 893         case 'e':
 894                 if (c_mode & _GNUC)
 895                         return 27;   /* hopefully 27 is ALWAYS the code for ESCAPE */
 896                 break;
 897         case 'u':
 898         case 'U':
 899                 parse_error("universal character parsing not implemented yet");
 900                 return EOF;
 901         default:
 902                 break;
 903         }
 904         /* §6.4.4.4:8 footnote 64 */
 905         parse_error("unknown escape sequence");
 906         return EOF;
 907 }
 908
 909 /**
 910  * Concatenate two strings.
 911  */
 912 string_t concat_strings(const string_t *const s1, const string_t *const s2)
 913 {
 914         const size_t len1 = s1->size - 1;
 915         const size_t len2 = s2->size - 1;
 916
 917         char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
 918         memcpy(concat, s1->begin, len1);
 919         memcpy(concat + len1, s2->begin, len2 + 1);
 920
 921         return identify_string(concat, len1 + len2 + 1);
 922 }
 923
 924 string_t make_string(const char *string)
 925 {
 926         size_t      len   = strlen(string) + 1;
 927         char *const space = obstack_alloc(&symbol_obstack, len);
 928         memcpy(space, string, len);
 929
 930         return identify_string(space, len);
 931 }
 932
 933 static void grow_symbol(utf32 const tc)
 934 {
 935         struct obstack *const o  = &symbol_obstack;
 936         if (tc < 0x80U) {
 937                 obstack_1grow(o, tc);
 938         } else if (tc < 0x800) {
 939                 obstack_1grow(o, 0xC0 | (tc >> 6));
 940                 obstack_1grow(o, 0x80 | (tc & 0x3F));
 941         } else if (tc < 0x10000) {
 942                 obstack_1grow(o, 0xE0 | ( tc >> 12));
 943                 obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
 944                 obstack_1grow(o, 0x80 | ( tc        & 0x3F));
 945         } else {
 946                 obstack_1grow(o, 0xF0 | ( tc >> 18));
 947                 obstack_1grow(o, 0x80 | ((tc >> 12) & 0x3F));
 948                 obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
 949                 obstack_1grow(o, 0x80 | ( tc        & 0x3F));
 950         }
 951 }
 952
 953 /**
 954  * Parse a string literal and set lexer_token.
 955  */
 956 static void parse_string_literal(void)
 957 {
 958         const unsigned start_linenr = lexer_token.source_position.linenr;
 959
 960         eat('"');
 961
 962         while (true) {
 963                 switch (c) {
 964                 case '\\': {
 965                         utf32 const tc = parse_escape_sequence();
 966                         if (tc >= 0x100) {
 967                                 warningf(&lexer_token.source_position,
 968                                                 "escape sequence out of range");
 969                         }
 970                         obstack_1grow(&symbol_obstack, tc);
 971                         break;
 972                 }
 973
 974                 case EOF: {
 975                         source_position_t source_position;
 976                         source_position.input_name = lexer_token.source_position.input_name;
 977                         source_position.linenr     = start_linenr;
 978                         errorf(&source_position, "string has no end");
 979                         lexer_token.type = T_ERROR;
 980                         return;
 981                 }
 982
 983                 case '"':
 984                         next_char();
 985                         goto end_of_string;
 986
 987                 default:
 988                         grow_symbol(c);
 989                         next_char();
 990                         break;
 991                 }
 992         }
 993
 994 end_of_string:
 995
 996         /* TODO: concatenate multiple strings separated by whitespace... */
 997
 998         /* add finishing 0 to the string */
 999         obstack_1grow(&symbol_obstack, '\0');
1000         const size_t  size   = (size_t)obstack_object_size(&symbol_obstack);
1001         char         *string = obstack_finish(&symbol_obstack);
1002
1003         lexer_token.type    = T_STRING_LITERAL;
1004         lexer_token.literal = identify_string(string, size);
1005 }
1006
1007 /**
1008  * Parse a wide character constant and set lexer_token.
1009  */
1010 static void parse_wide_character_constant(void)
1011 {
1012         const unsigned start_linenr = lexer_token.source_position.linenr;
1013
1014         eat('\'');
1015
1016         while (true) {
1017                 switch (c) {
1018                 case '\\': {
1019                         const utf32 tc = parse_escape_sequence();
1020                         grow_symbol(tc);
1021                         break;
1022                 }
1023
1024                 MATCH_NEWLINE(
1025                         parse_error("newline while parsing character constant");
1026                         break;
1027                 )
1028
1029                 case '\'':
1030                         next_char();
1031                         goto end_of_wide_char_constant;
1032
1033                 case EOF: {
1034                         source_position_t source_position = lexer_token.source_position;
1035                         source_position.linenr = start_linenr;
1036                         errorf(&source_position, "EOF while parsing character constant");
1037                         lexer_token.type = T_ERROR;
1038                         return;
1039                 }
1040
1041                 default:
1042                         grow_symbol(c);
1043                         next_char();
1044                         break;
1045                 }
1046         }
1047
1048 end_of_wide_char_constant:;
1049         obstack_1grow(&symbol_obstack, '\0');
1050         size_t  size   = (size_t) obstack_object_size(&symbol_obstack) - 1;
1051         char   *string = obstack_finish(&symbol_obstack);
1052
1053         lexer_token.type     = T_WIDE_CHARACTER_CONSTANT;
1054         lexer_token.literal  = identify_string(string, size);
1055 }
1056
1057 /**
1058  * Parse a wide string literal and set lexer_token.
1059  */
1060 static void parse_wide_string_literal(void)
1061 {
1062         parse_string_literal();
1063         if (lexer_token.type == T_STRING_LITERAL)
1064                 lexer_token.type = T_WIDE_STRING_LITERAL;
1065 }
1066
1067 /**
1068  * Parse a character constant and set lexer_token.
1069  */
1070 static void parse_character_constant(void)
1071 {
1072         const unsigned start_linenr = lexer_token.source_position.linenr;
1073
1074         eat('\'');
1075
1076         while (true) {
1077                 switch (c) {
1078                 case '\\': {
1079                         utf32 const tc = parse_escape_sequence();
1080                         if (tc >= 0x100) {
1081                                 warningf(&lexer_token.source_position,
1082                                                 "escape sequence out of range");
1083                         }
1084                         obstack_1grow(&symbol_obstack, tc);
1085                         break;
1086                 }
1087
1088                 MATCH_NEWLINE(
1089                         parse_error("newline while parsing character constant");
1090                         break;
1091                 )
1092
1093                 case '\'':
1094                         next_char();
1095                         goto end_of_char_constant;
1096
1097                 case EOF: {
1098                         source_position_t source_position;
1099                         source_position.input_name = lexer_token.source_position.input_name;
1100                         source_position.linenr     = start_linenr;
1101                         errorf(&source_position, "EOF while parsing character constant");
1102                         lexer_token.type = T_ERROR;
1103                         return;
1104                 }
1105
1106                 default:
1107                         grow_symbol(c);
1108                         next_char();
1109                         break;
1110
1111                 }
1112         }
1113
1114 end_of_char_constant:;
1115         obstack_1grow(&symbol_obstack, '\0');
1116         const size_t        size   = (size_t)obstack_object_size(&symbol_obstack)-1;
1117         char         *const string = obstack_finish(&symbol_obstack);
1118
1119         lexer_token.type    = T_CHARACTER_CONSTANT;
1120         lexer_token.literal = identify_string(string, size);
1121 }
1122
1123 /**
1124  * Skip a multiline comment.
1125  */
1126 static void skip_multiline_comment(void)
1127 {
1128         unsigned start_linenr = lexer_token.source_position.linenr;
1129
1130         while (true) {
1131                 switch (c) {
1132                 case '/':
1133                         next_char();
1134                         if (c == '*') {
1135                                 /* nested comment, warn here */
1136                                 if (warning.comment) {
1137                                         warningf(&lexer_token.source_position, "'/*' within comment");
1138                                 }
1139                         }
1140                         break;
1141                 case '*':
1142                         next_char();
1143                         if (c == '/') {
1144                                 next_char();
1145                                 return;
1146                         }
1147                         break;
1148
1149                 MATCH_NEWLINE(break;)
1150
1151                 case EOF: {
1152                         source_position_t source_position;
1153                         source_position.input_name = lexer_token.source_position.input_name;
1154                         source_position.linenr     = start_linenr;
1155                         errorf(&source_position, "at end of file while looking for comment end");
1156                         return;
1157                 }
1158
1159                 default:
1160                         next_char();
1161                         break;
1162                 }
1163         }
1164 }
1165
1166 /**
1167  * Skip a single line comment.
1168  */
1169 static void skip_line_comment(void)
1170 {
1171         while (true) {
1172                 switch (c) {
1173                 case EOF:
1174                         return;
1175
1176                 case '\n':
1177                 case '\r':
1178                         return;
1179
1180                 case '\\':
1181                         next_char();
1182                         if (c == '\n' || c == '\r') {
1183                                 if (warning.comment)
1184                                         warningf(&lexer_token.source_position, "multi-line comment");
1185                                 return;
1186                         }
1187                         break;
1188
1189                 default:
1190                         next_char();
1191                         break;
1192                 }
1193         }
1194 }
1195
1196 /** The current preprocessor token. */
1197 static token_t pp_token;
1198
1199 /**
1200  * Read the next preprocessor token.
1201  */
1202 static inline void next_pp_token(void)
1203 {
1204         lexer_next_preprocessing_token();
1205         pp_token = lexer_token;
1206 }
1207
1208 /**
1209  * Eat all preprocessor tokens until newline.
1210  */
1211 static void eat_until_newline(void)
1212 {
1213         while (pp_token.type != '\n' && pp_token.type != T_EOF) {
1214                 next_pp_token();
1215         }
1216 }
1217
1218 /**
1219  * Handle the define directive.
1220  */
1221 static void define_directive(void)
1222 {
1223         lexer_next_preprocessing_token();
1224         if (lexer_token.type != T_IDENTIFIER) {
1225                 parse_error("expected identifier after #define\n");
1226                 eat_until_newline();
1227         }
1228 }
1229
1230 /**
1231  * Handle the ifdef directive.
1232  */
1233 static void ifdef_directive(int is_ifndef)
1234 {
1235         (void) is_ifndef;
1236         lexer_next_preprocessing_token();
1237         //expect_identifier();
1238         //extect_newline();
1239 }
1240
1241 /**
1242  * Handle the endif directive.
1243  */
1244 static void endif_directive(void)
1245 {
1246         //expect_newline();
1247 }
1248
1249 /**
1250  * Parse the line directive.
1251  */
1252 static void parse_line_directive(void)
1253 {
1254         if (pp_token.type != T_INTEGER) {
1255                 parse_error("expected integer");
1256         } else {
1257                 /* use offset -1 as this is about the next line */
1258                 lexer_token.source_position.linenr = atoi(pp_token.literal.begin) - 1;
1259                 next_pp_token();
1260         }
1261         if (pp_token.type == T_STRING_LITERAL) {
1262                 lexer_token.source_position.input_name = pp_token.literal.begin;
1263                 next_pp_token();
1264         }
1265
1266         eat_until_newline();
1267 }
1268
1269 /**
1270  * STDC pragmas.
1271  */
1272 typedef enum stdc_pragma_kind_t {
1273         STDC_UNKNOWN,
1274         STDC_FP_CONTRACT,
1275         STDC_FENV_ACCESS,
1276         STDC_CX_LIMITED_RANGE
1277 } stdc_pragma_kind_t;
1278
1279 /**
1280  * STDC pragma values.
1281  */
1282 typedef enum stdc_pragma_value_kind_t {
1283         STDC_VALUE_UNKNOWN,
1284         STDC_VALUE_ON,
1285         STDC_VALUE_OFF,
1286         STDC_VALUE_DEFAULT
1287 } stdc_pragma_value_kind_t;
1288
1289 /**
1290  * Parse a pragma directive.
1291  */
1292 static void parse_pragma(void)
1293 {
1294         bool unknown_pragma = true;
1295
1296         next_pp_token();
1297         if (pp_token.symbol->pp_ID == TP_STDC) {
1298                 stdc_pragma_kind_t kind = STDC_UNKNOWN;
1299                 /* a STDC pragma */
1300                 if (c_mode & _C99) {
1301                         next_pp_token();
1302
1303                         switch (pp_token.symbol->pp_ID) {
1304                         case TP_FP_CONTRACT:
1305                                 kind = STDC_FP_CONTRACT;
1306                                 break;
1307                         case TP_FENV_ACCESS:
1308                                 kind = STDC_FENV_ACCESS;
1309                                 break;
1310                         case TP_CX_LIMITED_RANGE:
1311                                 kind = STDC_CX_LIMITED_RANGE;
1312                                 break;
1313                         default:
1314                                 break;
1315                         }
1316                         if (kind != STDC_UNKNOWN) {
1317                                 stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN;
1318                                 next_pp_token();
1319                                 switch (pp_token.symbol->pp_ID) {
1320                                 case TP_ON:
1321                                         value = STDC_VALUE_ON;
1322                                         break;
1323                                 case TP_OFF:
1324                                         value = STDC_VALUE_OFF;
1325                                         break;
1326                                 case TP_DEFAULT:
1327                                         value = STDC_VALUE_DEFAULT;
1328                                         break;
1329                                 default:
1330                                         break;
1331                                 }
1332                                 if (value != STDC_VALUE_UNKNOWN) {
1333                                         unknown_pragma = false;
1334                                 } else {
1335                                         errorf(&pp_token.source_position, "bad STDC pragma argument");
1336                                 }
1337                         }
1338                 }
1339         } else {
1340                 unknown_pragma = true;
1341         }
1342         eat_until_newline();
1343         if (unknown_pragma && warning.unknown_pragmas) {
1344                 warningf(&pp_token.source_position, "encountered unknown #pragma");
1345         }
1346 }
1347
1348 /**
1349  * Parse a preprocessor non-null directive.
1350  */
1351 static void parse_preprocessor_identifier(void)
1352 {
1353         assert(pp_token.type == T_IDENTIFIER);
1354         symbol_t *symbol = pp_token.symbol;
1355
1356         switch (symbol->pp_ID) {
1357         case TP_include:
1358                 printf("include - enable header name parsing!\n");
1359                 break;
1360         case TP_define:
1361                 define_directive();
1362                 break;
1363         case TP_ifdef:
1364                 ifdef_directive(0);
1365                 break;
1366         case TP_ifndef:
1367                 ifdef_directive(1);
1368                 break;
1369         case TP_endif:
1370                 endif_directive();
1371                 break;
1372         case TP_line:
1373                 next_pp_token();
1374                 parse_line_directive();
1375                 break;
1376         case TP_if:
1377         case TP_else:
1378         case TP_elif:
1379         case TP_undef:
1380         case TP_error:
1381                 /* TODO; output the rest of the line */
1382                 parse_error("#error directive: ");
1383                 break;
1384         case TP_pragma:
1385                 parse_pragma();
1386                 break;
1387         }
1388 }
1389
1390 /**
1391  * Parse a preprocessor directive.
1392  */
1393 static void parse_preprocessor_directive(void)
1394 {
1395         next_pp_token();
1396
1397         switch (pp_token.type) {
1398         case T_IDENTIFIER:
1399                 parse_preprocessor_identifier();
1400                 break;
1401         case T_INTEGER:
1402                 parse_line_directive();
1403                 break;
1404         case '\n':
1405                 /* NULL directive, see §6.10.7 */
1406                 break;
1407         default:
1408                 parse_error("invalid preprocessor directive");
1409                 eat_until_newline();
1410                 break;
1411         }
1412 }
1413
1414 #define MAYBE_PROLOG                                       \
1415                         next_char();                                   \
1416                         while (true) {                                 \
1417                                 switch (c) {
1418
1419 #define MAYBE(ch, set_type)                                \
1420                                 case ch:                                   \
1421                                         next_char();                           \
1422                                         lexer_token.type = set_type;           \
1423                                         return;
1424
1425 /* must use this as last thing */
1426 #define MAYBE_MODE(ch, set_type, mode)                     \
1427                                 case ch:                                   \
1428                                         if (c_mode & mode) {                   \
1429                                                 next_char();                       \
1430                                                 lexer_token.type = set_type;       \
1431                                                 return;                            \
1432                                         }                                      \
1433                                         /* fallthrough */
1434
1435 #define ELSE_CODE(code)                                    \
1436                                 default:                                   \
1437                                         code                                   \
1438                                 }                                          \
1439                         } /* end of while (true) */                    \
1440                         break;
1441
1442 #define ELSE(set_type)                                     \
1443                 ELSE_CODE(                                         \
1444                         lexer_token.type = set_type;                   \
1445                         return;                                        \
1446                 )
1447
1448 void lexer_next_preprocessing_token(void)
1449 {
1450         while (true) {
1451                 switch (c) {
1452                 case ' ':
1453                 case '\t':
1454                         next_char();
1455                         break;
1456
1457                 MATCH_NEWLINE(
1458                         lexer_token.type = '\n';
1459                         return;
1460                 )
1461
1462                 SYMBOL_CHARS
1463                         parse_symbol();
1464                         /* might be a wide string ( L"string" ) */
1465                         if (lexer_token.symbol == symbol_L) {
1466                                 switch (c) {
1467                                         case '"':  parse_wide_string_literal();     break;
1468                                         case '\'': parse_wide_character_constant(); break;
1469                                 }
1470                         }
1471                         return;
1472
1473                 DIGITS
1474                         parse_number();
1475                         return;
1476
1477                 case '"':
1478                         parse_string_literal();
1479                         return;
1480
1481                 case '\'':
1482                         parse_character_constant();
1483                         return;
1484
1485                 case '.':
1486                         MAYBE_PROLOG
1487                                 DIGITS
1488                                         put_back(c);
1489                                         c = '.';
1490                                         parse_number();
1491                                         return;
1492
1493                                 case '.':
1494                                         MAYBE_PROLOG
1495                                         MAYBE('.', T_DOTDOTDOT)
1496                                         ELSE_CODE(
1497                                                 put_back(c);
1498                                                 c = '.';
1499                                                 lexer_token.type = '.';
1500                                                 return;
1501                                         )
1502                         ELSE('.')
1503                 case '&':
1504                         MAYBE_PROLOG
1505                         MAYBE('&', T_ANDAND)
1506                         MAYBE('=', T_ANDEQUAL)
1507                         ELSE('&')
1508                 case '*':
1509                         MAYBE_PROLOG
1510                         MAYBE('=', T_ASTERISKEQUAL)
1511                         ELSE('*')
1512                 case '+':
1513                         MAYBE_PROLOG
1514                         MAYBE('+', T_PLUSPLUS)
1515                         MAYBE('=', T_PLUSEQUAL)
1516                         ELSE('+')
1517                 case '-':
1518                         MAYBE_PROLOG
1519                         MAYBE('>', T_MINUSGREATER)
1520                         MAYBE('-', T_MINUSMINUS)
1521                         MAYBE('=', T_MINUSEQUAL)
1522                         ELSE('-')
1523                 case '!':
1524                         MAYBE_PROLOG
1525                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1526                         ELSE('!')
1527                 case '/':
1528                         MAYBE_PROLOG
1529                         MAYBE('=', T_SLASHEQUAL)
1530                                 case '*':
1531                                         next_char();
1532                                         skip_multiline_comment();
1533                                         lexer_next_preprocessing_token();
1534                                         return;
1535                                 case '/':
1536                                         next_char();
1537                                         skip_line_comment();
1538                                         lexer_next_preprocessing_token();
1539                                         return;
1540                         ELSE('/')
1541                 case '%':
1542                         MAYBE_PROLOG
1543                         MAYBE('>', '}')
1544                         MAYBE('=', T_PERCENTEQUAL)
1545                                 case ':':
1546                                         MAYBE_PROLOG
1547                                                 case '%':
1548                                                         MAYBE_PROLOG
1549                                                         MAYBE(':', T_HASHHASH)
1550                                                         ELSE_CODE(
1551                                                                 put_back(c);
1552                                                                 c = '%';
1553                                                                 lexer_token.type = '#';
1554                                                                 return;
1555                                                         )
1556                                         ELSE('#')
1557                         ELSE('%')
1558                 case '<':
1559                         MAYBE_PROLOG
1560                         MAYBE(':', '[')
1561                         MAYBE('%', '{')
1562                         MAYBE('=', T_LESSEQUAL)
1563                                 case '<':
1564                                         MAYBE_PROLOG
1565                                         MAYBE('=', T_LESSLESSEQUAL)
1566                                         ELSE(T_LESSLESS)
1567                         ELSE('<')
1568                 case '>':
1569                         MAYBE_PROLOG
1570                         MAYBE('=', T_GREATEREQUAL)
1571                                 case '>':
1572                                         MAYBE_PROLOG
1573                                         MAYBE('=', T_GREATERGREATEREQUAL)
1574                                         ELSE(T_GREATERGREATER)
1575                         ELSE('>')
1576                 case '^':
1577                         MAYBE_PROLOG
1578                         MAYBE('=', T_CARETEQUAL)
1579                         ELSE('^')
1580                 case '|':
1581                         MAYBE_PROLOG
1582                         MAYBE('=', T_PIPEEQUAL)
1583                         MAYBE('|', T_PIPEPIPE)
1584                         ELSE('|')
1585                 case ':':
1586                         MAYBE_PROLOG
1587                         MAYBE('>', ']')
1588                         MAYBE_MODE(':', T_COLONCOLON, _CXX)
1589                         ELSE(':')
1590                 case '=':
1591                         MAYBE_PROLOG
1592                         MAYBE('=', T_EQUALEQUAL)
1593                         ELSE('=')
1594                 case '#':
1595                         MAYBE_PROLOG
1596                         MAYBE('#', T_HASHHASH)
1597                         ELSE('#')
1598
1599                 case '?':
1600                 case '[':
1601                 case ']':
1602                 case '(':
1603                 case ')':
1604                 case '{':
1605                 case '}':
1606                 case '~':
1607                 case ';':
1608                 case ',':
1609                 case '\\':
1610                         lexer_token.type = c;
1611                         next_char();
1612                         return;
1613
1614                 case EOF:
1615                         lexer_token.type = T_EOF;
1616                         return;
1617
1618                 default:
1619 dollar_sign:
1620                         errorf(&lexer_token.source_position, "unknown character '%c' found", c);
1621                         next_char();
1622                         lexer_token.type = T_ERROR;
1623                         return;
1624                 }
1625         }
1626 }
1627
1628 void lexer_next_token(void)
1629 {
1630         lexer_next_preprocessing_token();
1631
1632         while (lexer_token.type == '\n') {
1633 newline_found:
1634                 lexer_next_preprocessing_token();
1635         }
1636
1637         if (lexer_token.type == '#') {
1638                 parse_preprocessor_directive();
1639                 goto newline_found;
1640         }
1641 }
1642
1643 void init_lexer(void)
1644 {
1645         strset_init(&stringset);
1646         symbol_L = symbol_table_insert("L");
1647 }
1648
1649 void lexer_open_stream(FILE *stream, const char *input_name)
1650 {
1651         input                                  = stream;
1652         lexer_token.source_position.linenr     = 0;
1653         lexer_token.source_position.input_name = input_name;
1654
1655         bufpos = NULL;
1656         bufend = NULL;
1657
1658         /* place a virtual \n at the beginning so the lexer knows that we're
1659          * at the beginning of a line */
1660         c = '\n';
1661 }
1662
1663 void lexer_open_buffer(const char *buffer, size_t len, const char *input_name)
1664 {
1665         input                                  = NULL;
1666         lexer_token.source_position.linenr     = 0;
1667         lexer_token.source_position.input_name = input_name;
1668
1669 #if 0 // TODO
1670         bufpos = buffer;
1671         bufend = buffer + len;
1672
1673         /* place a virtual \n at the beginning so the lexer knows that we're
1674          * at the beginning of a line */
1675         c = '\n';
1676 #else
1677         (void)buffer;
1678         (void)len;
1679         panic("builtin lexing not done yet");
1680 #endif
1681 }
1682
1683 void exit_lexer(void)
1684 {
1685         strset_destroy(&stringset);
1686 }
1687
1688 static __attribute__((unused))
1689 void dbg_pos(const source_position_t source_position)
1690 {
1691         fprintf(stdout, "%s:%u\n", source_position.input_name,
1692                 source_position.linenr);
1693         fflush(stdout);
1694 }