nsz Git - cparser/blob - lexer.c

   1 /*
   2  * This file is part of cparser.
   3  * Copyright (C) 2007-2009 Matthias Braun <matze@braunis.de>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; either version 2
   8  * of the License, or (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  18  * 02111-1307, USA.
  19  */
  20 #include <config.h>
  21
  22 #include "diagnostic.h"
  23 #include "lexer.h"
  24 #include "symbol_t.h"
  25 #include "token_t.h"
  26 #include "symbol_table_t.h"
  27 #include "adt/error.h"
  28 #include "adt/strset.h"
  29 #include "adt/util.h"
  30 #include "types.h"
  31 #include "type_t.h"
  32 #include "target_architecture.h"
  33 #include "parser.h"
  34 #include "warning.h"
  35 #include "lang_features.h"
  36
  37 #include <assert.h>
  38 #include <errno.h>
  39 #include <string.h>
  40 #include <stdbool.h>
  41 #include <ctype.h>
  42
  43 #ifndef _WIN32
  44 #include <strings.h>
  45 #endif
  46
  47 //#define DEBUG_CHARS
  48 #define MAX_PUTBACK 3
  49 #define BUF_SIZE    1024
  50
  51 #if defined(_WIN32) || defined(__CYGWIN__)
  52 /* No strtold on windows and no replacement yet */
  53 #define strtold(s, e) strtod(s, e)
  54 #endif
  55
  56 static utf32        c;
  57 token_t             lexer_token;
  58 symbol_t           *symbol_L;
  59 static FILE        *input;
  60 static utf32        buf[BUF_SIZE + MAX_PUTBACK];
  61 static const utf32 *bufend;
  62 static const utf32 *bufpos;
  63 static strset_t     stringset;
  64 bool                allow_dollar_in_symbol = true;
  65
  66 /**
  67  * Prints a parse error message at the current token.
  68  *
  69  * @param msg   the error message
  70  */
  71 static void parse_error(const char *msg)
  72 {
  73         errorf(&lexer_token.source_position, "%s", msg);
  74 }
  75
  76 /**
  77  * Prints an internal error message at the current token.
  78  *
  79  * @param msg   the error message
  80  */
  81 static NORETURN internal_error(const char *msg)
  82 {
  83         internal_errorf(&lexer_token.source_position, "%s", msg);
  84 }
  85
  86 static size_t read_block(unsigned char *const read_buf, size_t const n)
  87 {
  88         size_t const s = fread(read_buf, 1, n, input);
  89         if (s == 0) {
  90                 /* on OS/X ferror appears to return true on eof as well when running
  91                  * the application in gdb... */
  92                 if (!feof(input) && ferror(input))
  93                         parse_error("read from input failed");
  94                 buf[MAX_PUTBACK] = EOF;
  95                 bufpos           = buf + MAX_PUTBACK;
  96                 bufend           = buf + MAX_PUTBACK + 1;
  97         }
  98         return s;
  99 }
 100
 101 static void decode_iso_8859_1(void)
 102 {
 103         unsigned char read_buf[BUF_SIZE];
 104         size_t const s = read_block(read_buf, sizeof(read_buf));
 105         if (s == 0)
 106                 return;
 107
 108         unsigned char const *src = read_buf;
 109         unsigned char const *end = read_buf + s;
 110         utf32               *dst = buf + MAX_PUTBACK;
 111         while (src != end)
 112                 *dst++ = *src++;
 113
 114         bufpos = buf + MAX_PUTBACK;
 115         bufend = dst;
 116 }
 117
 118 static void decode_iso_8859_15(void)
 119 {
 120         unsigned char read_buf[BUF_SIZE];
 121         size_t const s = read_block(read_buf, sizeof(read_buf));
 122         if (s == 0)
 123                 return;
 124
 125         unsigned char const *src = read_buf;
 126         unsigned char const *end = read_buf + s;
 127         utf32               *dst = buf + MAX_PUTBACK;
 128         while (src != end) {
 129                 utf32 tc = *src++;
 130                 switch (tc) {
 131                         case 0xA4: tc = 0x20AC; break; // €
 132                         case 0xA6: tc = 0x0160; break; // Š
 133                         case 0xA8: tc = 0x0161; break; // š
 134                         case 0xB4: tc = 0x017D; break; // Ž
 135                         case 0xB8: tc = 0x017E; break; // ž
 136                         case 0xBC: tc = 0x0152; break; // Œ
 137                         case 0xBD: tc = 0x0153; break; // œ
 138                         case 0xBE: tc = 0x0178; break; // Ÿ
 139                 }
 140                 *dst++ = tc;
 141         }
 142
 143         bufpos = buf + MAX_PUTBACK;
 144         bufend = dst;
 145 }
 146
 147 static void decode_utf8(void)
 148 {
 149         static utf32  part_decoded_min_code;
 150         static utf32  part_decoded_char;
 151         static size_t part_decoded_rest_len;
 152
 153         do {
 154                 unsigned char read_buf[BUF_SIZE];
 155                 size_t const s = read_block(read_buf, sizeof(read_buf));
 156                 if (s == 0) {
 157                         if (part_decoded_rest_len > 0)
 158                                 parse_error("incomplete input char at end of input");
 159                         return;
 160                 }
 161
 162                 unsigned char const *src = read_buf;
 163                 unsigned char const *end = read_buf + s;
 164                 utf32               *dst = buf + MAX_PUTBACK;
 165                 utf32                decoded;
 166                 utf32                min_code;
 167
 168                 if (part_decoded_rest_len != 0) {
 169                         min_code              = part_decoded_min_code;
 170                         decoded               = part_decoded_char;
 171                         size_t const rest_len = part_decoded_rest_len;
 172                         part_decoded_rest_len = 0;
 173                         switch (rest_len) {
 174                                 case 4:  goto realign;
 175                                 case 3:  goto three_more;
 176                                 case 2:  goto two_more;
 177                                 default: goto one_more;
 178                         }
 179                 }
 180
 181                 while (src != end) {
 182                         if ((*src & 0x80) == 0) {
 183                                 decoded = *src++;
 184                         } else if ((*src & 0xE0) == 0xC0) {
 185                                 min_code = 0x80;
 186                                 decoded  = *src++ & 0x1F;
 187 one_more:
 188                                 if (src == end) {
 189                                         part_decoded_min_code = min_code;
 190                                         part_decoded_char     = decoded;
 191                                         part_decoded_rest_len = 1;
 192                                         break;
 193                                 }
 194                                 if ((*src & 0xC0) == 0x80) {
 195                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 196                                 } else {
 197                                         goto invalid_char;
 198                                 }
 199                                 if (decoded < min_code                      ||
 200                                                 decoded > 0x10FFFF                      ||
 201                                                 (0xD800 <= decoded && decoded < 0xE000) || // high/low surrogates
 202                                                 (0xFDD0 <= decoded && decoded < 0xFDF0) || // noncharacters
 203                                                 (decoded & 0xFFFE) == 0xFFFE) {            // noncharacters
 204                                         parse_error("invalid byte sequence in input");
 205                                 }
 206                         } else if ((*src & 0xF0) == 0xE0) {
 207                                 min_code = 0x800;
 208                                 decoded  = *src++ & 0x0F;
 209 two_more:
 210                                 if (src == end) {
 211                                         part_decoded_min_code = min_code;
 212                                         part_decoded_char     = decoded;
 213                                         part_decoded_rest_len = 2;
 214                                         break;
 215                                 }
 216                                 if ((*src & 0xC0) == 0x80) {
 217                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 218                                 } else {
 219                                         goto invalid_char;
 220                                 }
 221                                 goto one_more;
 222                         } else if ((*src & 0xF8) == 0xF0) {
 223                                 min_code = 0x10000;
 224                                 decoded  = *src++ & 0x07;
 225 three_more:
 226                                 if (src == end) {
 227                                         part_decoded_min_code = min_code;
 228                                         part_decoded_char     = decoded;
 229                                         part_decoded_rest_len = 3;
 230                                         break;
 231                                 }
 232                                 if ((*src & 0xC0) == 0x80) {
 233                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 234                                 } else {
 235                                         goto invalid_char;
 236                                 }
 237                                 goto two_more;
 238                         } else {
 239 invalid_char:
 240                                 parse_error("invalid byte sequence in input");
 241 realign:
 242                                 do {
 243                                         ++src;
 244                                         if (src == end) {
 245                                                 part_decoded_rest_len = 4;
 246                                                 break;
 247                                         }
 248                                 } while ((*src & 0xC0) == 0x80 || (*src & 0xF8) == 0xF8);
 249                                 continue;
 250                         }
 251                         *dst++ = decoded;
 252                 }
 253
 254                 bufpos = buf + MAX_PUTBACK;
 255                 bufend = dst;
 256         } while (bufpos == bufend);
 257 }
 258
 259 static void decode_windows_1252(void)
 260 {
 261         unsigned char read_buf[BUF_SIZE];
 262         size_t const s = read_block(read_buf, sizeof(read_buf));
 263         if (s == 0)
 264                 return;
 265
 266         unsigned char const *src = read_buf;
 267         unsigned char const *end = read_buf + s;
 268         utf32               *dst = buf + MAX_PUTBACK;
 269         while (src != end) {
 270                 utf32 tc = *src++;
 271                 switch (tc) {
 272                         case 0x80: tc = 0x20AC; break; // €
 273                         case 0x82: tc = 0x201A; break; // ‚
 274                         case 0x83: tc = 0x0192; break; // ƒ
 275                         case 0x84: tc = 0x201E; break; // „
 276                         case 0x85: tc = 0x2026; break; // …
 277                         case 0x86: tc = 0x2020; break; // †
 278                         case 0x87: tc = 0x2021; break; // ‡
 279                         case 0x88: tc = 0x02C6; break; // ˆ
 280                         case 0x89: tc = 0x2030; break; // ‰
 281                         case 0x8A: tc = 0x0160; break; // Š
 282                         case 0x8B: tc = 0x2039; break; // ‹
 283                         case 0x8C: tc = 0x0152; break; // Œ
 284                         case 0x8E: tc = 0x017D; break; // Ž
 285                         case 0x91: tc = 0x2018; break; // ‘
 286                         case 0x92: tc = 0x2019; break; // ’
 287                         case 0x93: tc = 0x201C; break; // “
 288                         case 0x94: tc = 0x201D; break; // ”
 289                         case 0x95: tc = 0x2022; break; // •
 290                         case 0x96: tc = 0x2013; break; // –
 291                         case 0x97: tc = 0x2014; break; // —
 292                         case 0x98: tc = 0x02DC; break; // ˜
 293                         case 0x99: tc = 0x2122; break; // ™
 294                         case 0x9A: tc = 0x0161; break; // š
 295                         case 0x9B: tc = 0x203A; break; // ›
 296                         case 0x9C: tc = 0x0153; break; // œ
 297                         case 0x9E: tc = 0x017E; break; // ž
 298                         case 0x9F: tc = 0x0178; break; // Ÿ
 299                 }
 300                 *dst++ = tc;
 301         }
 302
 303         bufpos = buf + MAX_PUTBACK;
 304         bufend = dst;
 305 }
 306
 307 typedef void (*decoder_t)(void);
 308
 309 static decoder_t decoder = decode_utf8;
 310
 311 typedef struct named_decoder_t {
 312         char const *name;
 313         decoder_t   decoder;
 314 } named_decoder_t;
 315
 316 static named_decoder_t const decoders[] = {
 317         { "CP819",           decode_iso_8859_1   }, // offical alias
 318         { "IBM819",          decode_iso_8859_1   }, // offical alias
 319         { "ISO-8859-1",      decode_iso_8859_1   }, // offical alias
 320         { "ISO-8859-15",     decode_iso_8859_15  }, // offical name
 321         { "ISO8859-1",       decode_iso_8859_1   },
 322         { "ISO8859-15",      decode_iso_8859_15  },
 323         { "ISO_8859-1",      decode_iso_8859_1   }, // offical alias
 324         { "ISO_8859-15",     decode_iso_8859_15  }, // offical alias
 325         { "ISO_8859-1:1987", decode_iso_8859_1   }, // offical name
 326         { "Latin-9",         decode_iso_8859_15  }, // offical alias
 327         { "UTF-8",           decode_utf8         }, // offical name
 328         { "csISOLatin1",     decode_iso_8859_1   }, // offical alias
 329         { "cp1252",          decode_windows_1252 },
 330         { "iso-ir-100",      decode_iso_8859_1   }, // offical alias
 331         { "l1",              decode_iso_8859_1   }, // offical alias
 332         { "latin1",          decode_iso_8859_1   }, // offical alias
 333         { "windows-1252",    decode_windows_1252 }, // official name
 334
 335         { NULL,              NULL                }
 336 };
 337
 338 /** strcasecmp is not part of C99 so we need our own implementation here */
 339 static int my_strcasecmp(const char *s1, const char *s2)
 340 {
 341         for ( ; *s1 != 0; ++s1, ++s2) {
 342                 if (tolower(*s1) != tolower(*s2))
 343                         break;
 344         }
 345         return (unsigned char)*s1 - (unsigned char)*s2;
 346 }
 347
 348 void select_input_encoding(char const* const encoding)
 349 {
 350         for (named_decoder_t const *i = decoders; i->name != NULL; ++i) {
 351                 if (my_strcasecmp(encoding, i->name) != 0)
 352                         continue;
 353                 decoder = i->decoder;
 354                 return;
 355         }
 356         fprintf(stderr, "error: input encoding \"%s\" not supported\n", encoding);
 357 }
 358
 359 static inline void next_real_char(void)
 360 {
 361         assert(bufpos <= bufend);
 362         if (bufpos >= bufend) {
 363                 if (input == NULL) {
 364                         c = EOF;
 365                         return;
 366                 }
 367                 decoder();
 368         }
 369         c = *bufpos++;
 370 }
 371
 372 /**
 373  * Put a character back into the buffer.
 374  *
 375  * @param pc  the character to put back
 376  */
 377 static inline void put_back(utf32 const pc)
 378 {
 379         assert(bufpos > buf);
 380         *(--bufpos - buf + buf) = pc;
 381
 382 #ifdef DEBUG_CHARS
 383         printf("putback '%lc'\n", pc);
 384 #endif
 385 }
 386
 387 static inline void next_char(void);
 388
 389 #define MATCH_NEWLINE(code)                   \
 390         case '\r':                                \
 391                 next_char();                          \
 392                 if (c == '\n') {                      \
 393                         next_char();                      \
 394                 }                                     \
 395                 lexer_token.source_position.linenr++; \
 396                 code                                  \
 397         case '\n':                                \
 398                 next_char();                          \
 399                 lexer_token.source_position.linenr++; \
 400                 code
 401
 402 #define eat(c_type)  do { assert(c == c_type); next_char(); } while (0)
 403
 404 static void maybe_concat_lines(void)
 405 {
 406         eat('\\');
 407
 408         switch (c) {
 409         MATCH_NEWLINE(return;)
 410
 411         default:
 412                 break;
 413         }
 414
 415         put_back(c);
 416         c = '\\';
 417 }
 418
 419 /**
 420  * Set c to the next input character, ie.
 421  * after expanding trigraphs.
 422  */
 423 static inline void next_char(void)
 424 {
 425         next_real_char();
 426
 427         /* filter trigraphs */
 428         if (UNLIKELY(c == '\\')) {
 429                 maybe_concat_lines();
 430                 goto end_of_next_char;
 431         }
 432
 433         if (LIKELY(c != '?'))
 434                 goto end_of_next_char;
 435
 436         next_real_char();
 437         if (LIKELY(c != '?')) {
 438                 put_back(c);
 439                 c = '?';
 440                 goto end_of_next_char;
 441         }
 442
 443         next_real_char();
 444         switch (c) {
 445         case '=': c = '#'; break;
 446         case '(': c = '['; break;
 447         case '/': c = '\\'; maybe_concat_lines(); break;
 448         case ')': c = ']'; break;
 449         case '\'': c = '^'; break;
 450         case '<': c = '{'; break;
 451         case '!': c = '|'; break;
 452         case '>': c = '}'; break;
 453         case '-': c = '~'; break;
 454         default:
 455                 put_back(c);
 456                 put_back('?');
 457                 c = '?';
 458                 break;
 459         }
 460
 461 end_of_next_char:;
 462 #ifdef DEBUG_CHARS
 463         printf("nchar '%c'\n", c);
 464 #endif
 465 }
 466
 467 #define SYMBOL_CHARS  \
 468         case '$': if (!allow_dollar_in_symbol) goto dollar_sign; \
 469         case 'a':         \
 470         case 'b':         \
 471         case 'c':         \
 472         case 'd':         \
 473         case 'e':         \
 474         case 'f':         \
 475         case 'g':         \
 476         case 'h':         \
 477         case 'i':         \
 478         case 'j':         \
 479         case 'k':         \
 480         case 'l':         \
 481         case 'm':         \
 482         case 'n':         \
 483         case 'o':         \
 484         case 'p':         \
 485         case 'q':         \
 486         case 'r':         \
 487         case 's':         \
 488         case 't':         \
 489         case 'u':         \
 490         case 'v':         \
 491         case 'w':         \
 492         case 'x':         \
 493         case 'y':         \
 494         case 'z':         \
 495         case 'A':         \
 496         case 'B':         \
 497         case 'C':         \
 498         case 'D':         \
 499         case 'E':         \
 500         case 'F':         \
 501         case 'G':         \
 502         case 'H':         \
 503         case 'I':         \
 504         case 'J':         \
 505         case 'K':         \
 506         case 'L':         \
 507         case 'M':         \
 508         case 'N':         \
 509         case 'O':         \
 510         case 'P':         \
 511         case 'Q':         \
 512         case 'R':         \
 513         case 'S':         \
 514         case 'T':         \
 515         case 'U':         \
 516         case 'V':         \
 517         case 'W':         \
 518         case 'X':         \
 519         case 'Y':         \
 520         case 'Z':         \
 521         case '_':
 522
 523 #define DIGITS        \
 524         case '0':         \
 525         case '1':         \
 526         case '2':         \
 527         case '3':         \
 528         case '4':         \
 529         case '5':         \
 530         case '6':         \
 531         case '7':         \
 532         case '8':         \
 533         case '9':
 534
 535 /**
 536  * Read a symbol from the input and build
 537  * the lexer_token.
 538  */
 539 static void parse_symbol(void)
 540 {
 541         obstack_1grow(&symbol_obstack, (char) c);
 542         next_char();
 543
 544         while (true) {
 545                 switch (c) {
 546                 DIGITS
 547                 SYMBOL_CHARS
 548                         obstack_1grow(&symbol_obstack, (char) c);
 549                         next_char();
 550                         break;
 551
 552                 default:
 553 dollar_sign:
 554                         goto end_symbol;
 555                 }
 556         }
 557
 558 end_symbol:
 559         obstack_1grow(&symbol_obstack, '\0');
 560
 561         char     *string = obstack_finish(&symbol_obstack);
 562         symbol_t *symbol = symbol_table_insert(string);
 563
 564         lexer_token.type   = symbol->ID;
 565         lexer_token.symbol = symbol;
 566
 567         if (symbol->string != string) {
 568                 obstack_free(&symbol_obstack, string);
 569         }
 570 }
 571
 572 /**
 573  * parse suffixes like 'LU' or 'f' after numbers
 574  */
 575 static void parse_number_suffix(void)
 576 {
 577         assert(obstack_object_size(&symbol_obstack) == 0);
 578         while (true) {
 579                 switch (c) {
 580                 SYMBOL_CHARS
 581                         obstack_1grow(&symbol_obstack, (char) c);
 582                         next_char();
 583                         break;
 584                 default:
 585                 dollar_sign:
 586                         goto finish_suffix;
 587                 }
 588         }
 589 finish_suffix:
 590         if (obstack_object_size(&symbol_obstack) == 0) {
 591                 lexer_token.symbol = NULL;
 592                 return;
 593         }
 594
 595         obstack_1grow(&symbol_obstack, '\0');
 596         char     *string = obstack_finish(&symbol_obstack);
 597         symbol_t *symbol = symbol_table_insert(string);
 598
 599         if (symbol->string != string) {
 600                 obstack_free(&symbol_obstack, string);
 601         }
 602         lexer_token.symbol = symbol;
 603 }
 604
 605 static string_t identify_string(char *string, size_t len)
 606 {
 607         /* TODO hash */
 608 #if 0
 609         const char *result = strset_insert(&stringset, concat);
 610         if (result != concat) {
 611                 obstack_free(&symbol_obstack, concat);
 612         }
 613 #else
 614         const char *result = string;
 615 #endif
 616         return (string_t) {result, len};
 617 }
 618
 619 /**
 620  * Parses a hex number including hex floats and set the
 621  * lexer_token.
 622  */
 623 static void parse_number_hex(void)
 624 {
 625         bool is_float   = false;
 626         bool has_digits = false;
 627
 628         assert(obstack_object_size(&symbol_obstack) == 0);
 629         while (isxdigit(c)) {
 630                 has_digits = true;
 631                 obstack_1grow(&symbol_obstack, (char) c);
 632                 next_char();
 633         }
 634
 635         if (c == '.') {
 636                 is_float = true;
 637                 obstack_1grow(&symbol_obstack, (char) c);
 638                 next_char();
 639
 640                 while (isxdigit(c)) {
 641                         has_digits = true;
 642                         obstack_1grow(&symbol_obstack, (char) c);
 643                         next_char();
 644                 }
 645         }
 646         if (c == 'p' || c == 'P') {
 647                 is_float = true;
 648                 obstack_1grow(&symbol_obstack, (char) c);
 649                 next_char();
 650
 651                 if (c == '-' || c == '+') {
 652                         obstack_1grow(&symbol_obstack, (char) c);
 653                         next_char();
 654                 }
 655
 656                 while (isxdigit(c)) {
 657                         obstack_1grow(&symbol_obstack, (char) c);
 658                         next_char();
 659                 }
 660         } else if (is_float) {
 661                 errorf(&lexer_token.source_position,
 662                        "hexadecimal floatingpoint constant requires an exponent");
 663         }
 664         obstack_1grow(&symbol_obstack, '\0');
 665
 666         size_t  size   = obstack_object_size(&symbol_obstack) - 1;
 667         char   *string = obstack_finish(&symbol_obstack);
 668         lexer_token.literal = identify_string(string, size);
 669
 670         lexer_token.type    =
 671                 is_float ? T_FLOATINGPOINT_HEXADECIMAL : T_INTEGER_HEXADECIMAL;
 672
 673         if (!has_digits) {
 674                 errorf(&lexer_token.source_position, "invalid number literal '0x%S'",
 675                        &lexer_token.literal);
 676                 lexer_token.literal.begin = "0";
 677                 lexer_token.literal.size  = 1;
 678         }
 679
 680         parse_number_suffix();
 681 }
 682
 683 /**
 684  * Returns true if the given char is a octal digit.
 685  *
 686  * @param char  the character to check
 687  */
 688 static bool is_octal_digit(utf32 chr)
 689 {
 690         switch (chr) {
 691         case '0':
 692         case '1':
 693         case '2':
 694         case '3':
 695         case '4':
 696         case '5':
 697         case '6':
 698         case '7':
 699                 return true;
 700         default:
 701                 return false;
 702         }
 703 }
 704
 705 /**
 706  * Parses a number and sets the lexer_token.
 707  */
 708 static void parse_number(void)
 709 {
 710         bool is_float   = false;
 711         bool has_digits = false;
 712
 713         assert(obstack_object_size(&symbol_obstack) == 0);
 714         if (c == '0') {
 715                 next_char();
 716                 if (c == 'x' || c == 'X') {
 717                         next_char();
 718                         parse_number_hex();
 719                         return;
 720                 } else {
 721                         has_digits = true;
 722                 }
 723                 obstack_1grow(&symbol_obstack, '0');
 724         }
 725
 726         while (isdigit(c)) {
 727                 has_digits = true;
 728                 obstack_1grow(&symbol_obstack, (char) c);
 729                 next_char();
 730         }
 731
 732         if (c == '.') {
 733                 is_float = true;
 734                 obstack_1grow(&symbol_obstack, '.');
 735                 next_char();
 736
 737                 while (isdigit(c)) {
 738                         has_digits = true;
 739                         obstack_1grow(&symbol_obstack, (char) c);
 740                         next_char();
 741                 }
 742         }
 743         if (c == 'e' || c == 'E') {
 744                 is_float = true;
 745                 obstack_1grow(&symbol_obstack, 'e');
 746                 next_char();
 747
 748                 if (c == '-' || c == '+') {
 749                         obstack_1grow(&symbol_obstack, (char) c);
 750                         next_char();
 751                 }
 752
 753                 while (isdigit(c)) {
 754                         obstack_1grow(&symbol_obstack, (char) c);
 755                         next_char();
 756                 }
 757         }
 758
 759         obstack_1grow(&symbol_obstack, '\0');
 760         size_t  size   = obstack_object_size(&symbol_obstack) - 1;
 761         char   *string = obstack_finish(&symbol_obstack);
 762         lexer_token.literal = identify_string(string, size);
 763
 764         /* is it an octal number? */
 765         if (is_float) {
 766                 lexer_token.type = T_FLOATINGPOINT;
 767         } else if (string[0] == '0') {
 768                 lexer_token.type = T_INTEGER_OCTAL;
 769
 770                 /* check for invalid octal digits */
 771                 for (size_t i= 0; i < size; ++i) {
 772                         char t = string[i];
 773                         if (t >= '8')
 774                                 errorf(&lexer_token.source_position,
 775                                        "invalid digit '%c' in octal number", t);
 776                 }
 777         } else {
 778                 lexer_token.type = T_INTEGER;
 779         }
 780
 781         if (!has_digits) {
 782                 errorf(&lexer_token.source_position, "invalid number literal '%S'",
 783                        &lexer_token.literal);
 784         }
 785
 786         parse_number_suffix();
 787 }
 788
 789 /**
 790  * Returns the value of a digit.
 791  * The only portable way to do it ...
 792  */
 793 static int digit_value(utf32 const digit)
 794 {
 795         switch (digit) {
 796         case '0': return 0;
 797         case '1': return 1;
 798         case '2': return 2;
 799         case '3': return 3;
 800         case '4': return 4;
 801         case '5': return 5;
 802         case '6': return 6;
 803         case '7': return 7;
 804         case '8': return 8;
 805         case '9': return 9;
 806         case 'a':
 807         case 'A': return 10;
 808         case 'b':
 809         case 'B': return 11;
 810         case 'c':
 811         case 'C': return 12;
 812         case 'd':
 813         case 'D': return 13;
 814         case 'e':
 815         case 'E': return 14;
 816         case 'f':
 817         case 'F': return 15;
 818         default:
 819                 internal_error("wrong character given");
 820         }
 821 }
 822
 823 /**
 824  * Parses an octal character sequence.
 825  *
 826  * @param first_digit  the already read first digit
 827  */
 828 static utf32 parse_octal_sequence(utf32 const first_digit)
 829 {
 830         assert(is_octal_digit(first_digit));
 831         utf32 value = digit_value(first_digit);
 832         if (!is_octal_digit(c)) return value;
 833         value = 8 * value + digit_value(c);
 834         next_char();
 835         if (!is_octal_digit(c)) return value;
 836         value = 8 * value + digit_value(c);
 837         next_char();
 838         return value;
 839 }
 840
 841 /**
 842  * Parses a hex character sequence.
 843  */
 844 static utf32 parse_hex_sequence(void)
 845 {
 846         utf32 value = 0;
 847         while (isxdigit(c)) {
 848                 value = 16 * value + digit_value(c);
 849                 next_char();
 850         }
 851         return value;
 852 }
 853
 854 /**
 855  * Parse an escape sequence.
 856  */
 857 static utf32 parse_escape_sequence(void)
 858 {
 859         eat('\\');
 860
 861         utf32 const ec = c;
 862         next_char();
 863
 864         switch (ec) {
 865         case '"':  return '"';
 866         case '\'': return '\'';
 867         case '\\': return '\\';
 868         case '?': return '\?';
 869         case 'a': return '\a';
 870         case 'b': return '\b';
 871         case 'f': return '\f';
 872         case 'n': return '\n';
 873         case 'r': return '\r';
 874         case 't': return '\t';
 875         case 'v': return '\v';
 876         case 'x':
 877                 return parse_hex_sequence();
 878         case '0':
 879         case '1':
 880         case '2':
 881         case '3':
 882         case '4':
 883         case '5':
 884         case '6':
 885         case '7':
 886                 return parse_octal_sequence(ec);
 887         case EOF:
 888                 parse_error("reached end of file while parsing escape sequence");
 889                 return EOF;
 890         /* \E is not documented, but handled, by GCC.  It is acceptable according
 891          * to §6.11.4, whereas \e is not. */
 892         case 'E':
 893         case 'e':
 894                 if (c_mode & _GNUC)
 895                         return 27;   /* hopefully 27 is ALWAYS the code for ESCAPE */
 896                 break;
 897         case 'u':
 898         case 'U':
 899                 parse_error("universal character parsing not implemented yet");
 900                 return EOF;
 901         default:
 902                 break;
 903         }
 904         /* §6.4.4.4:8 footnote 64 */
 905         parse_error("unknown escape sequence");
 906         return EOF;
 907 }
 908
 909 /**
 910  * Concatenate two strings.
 911  */
 912 string_t concat_strings(const string_t *const s1, const string_t *const s2)
 913 {
 914         const size_t len1 = s1->size - 1;
 915         const size_t len2 = s2->size - 1;
 916
 917         char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
 918         memcpy(concat, s1->begin, len1);
 919         memcpy(concat + len1, s2->begin, len2 + 1);
 920
 921         return identify_string(concat, len1 + len2 + 1);
 922 }
 923
 924 string_t make_string(const char *string)
 925 {
 926         size_t      len   = strlen(string) + 1;
 927         char *const space = obstack_alloc(&symbol_obstack, len);
 928         memcpy(space, string, len);
 929
 930         return identify_string(space, len);
 931 }
 932
 933 static void grow_symbol(utf32 const tc)
 934 {
 935         struct obstack *const o  = &symbol_obstack;
 936         if (tc < 0x80U) {
 937                 obstack_1grow(o, tc);
 938         } else if (tc < 0x800) {
 939                 obstack_1grow(o, 0xC0 | (tc >> 6));
 940                 obstack_1grow(o, 0x80 | (tc & 0x3F));
 941         } else if (tc < 0x10000) {
 942                 obstack_1grow(o, 0xE0 | ( tc >> 12));
 943                 obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
 944                 obstack_1grow(o, 0x80 | ( tc        & 0x3F));
 945         } else {
 946                 obstack_1grow(o, 0xF0 | ( tc >> 18));
 947                 obstack_1grow(o, 0x80 | ((tc >> 12) & 0x3F));
 948                 obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
 949                 obstack_1grow(o, 0x80 | ( tc        & 0x3F));
 950         }
 951 }
 952
 953 /**
 954  * Parse a string literal and set lexer_token.
 955  */
 956 static void parse_string_literal(void)
 957 {
 958         const unsigned start_linenr = lexer_token.source_position.linenr;
 959
 960         eat('"');
 961
 962         while (true) {
 963                 switch (c) {
 964                 case '\\': {
 965                         utf32 const tc = parse_escape_sequence();
 966                         if (tc >= 0x100) {
 967                                 warningf(&lexer_token.source_position,
 968                                                 "escape sequence out of range");
 969                         }
 970                         obstack_1grow(&symbol_obstack, tc);
 971                         break;
 972                 }
 973
 974                 case EOF: {
 975                         source_position_t source_position;
 976                         source_position.input_name = lexer_token.source_position.input_name;
 977                         source_position.linenr     = start_linenr;
 978                         errorf(&source_position, "string has no end");
 979                         lexer_token.type = T_ERROR;
 980                         return;
 981                 }
 982
 983                 case '"':
 984                         next_char();
 985                         goto end_of_string;
 986
 987                 default:
 988                         grow_symbol(c);
 989                         next_char();
 990                         break;
 991                 }
 992         }
 993
 994 end_of_string:
 995
 996         /* TODO: concatenate multiple strings separated by whitespace... */
 997
 998         /* add finishing 0 to the string */
 999         obstack_1grow(&symbol_obstack, '\0');
1000         const size_t  size   = (size_t)obstack_object_size(&symbol_obstack);
1001         char         *string = obstack_finish(&symbol_obstack);
1002
1003         lexer_token.type    = T_STRING_LITERAL;
1004         lexer_token.literal = identify_string(string, size);
1005 }
1006
1007 /**
1008  * Parse a wide character constant and set lexer_token.
1009  */
1010 static void parse_wide_character_constant(void)
1011 {
1012         const unsigned start_linenr = lexer_token.source_position.linenr;
1013
1014         eat('\'');
1015
1016         while (true) {
1017                 switch (c) {
1018                 case '\\': {
1019                         const utf32 tc = parse_escape_sequence();
1020                         grow_symbol(tc);
1021                         break;
1022                 }
1023
1024                 MATCH_NEWLINE(
1025                         parse_error("newline while parsing character constant");
1026                         break;
1027                 )
1028
1029                 case '\'':
1030                         next_char();
1031                         goto end_of_wide_char_constant;
1032
1033                 case EOF: {
1034                         source_position_t source_position = lexer_token.source_position;
1035                         source_position.linenr = start_linenr;
1036                         errorf(&source_position, "EOF while parsing character constant");
1037                         lexer_token.type = T_ERROR;
1038                         return;
1039                 }
1040
1041                 default:
1042                         grow_symbol(c);
1043                         next_char();
1044                         break;
1045                 }
1046         }
1047
1048 end_of_wide_char_constant:;
1049         obstack_1grow(&symbol_obstack, '\0');
1050         size_t  size   = (size_t) obstack_object_size(&symbol_obstack) - 1;
1051         char   *string = obstack_finish(&symbol_obstack);
1052
1053         lexer_token.type     = T_WIDE_CHARACTER_CONSTANT;
1054         lexer_token.literal  = identify_string(string, size);
1055
1056         if (size == 0) {
1057                 parse_error("empty character constant");
1058         }
1059 }
1060
1061 /**
1062  * Parse a wide string literal and set lexer_token.
1063  */
1064 static void parse_wide_string_literal(void)
1065 {
1066         parse_string_literal();
1067         if (lexer_token.type == T_STRING_LITERAL)
1068                 lexer_token.type = T_WIDE_STRING_LITERAL;
1069 }
1070
1071 /**
1072  * Parse a character constant and set lexer_token.
1073  */
1074 static void parse_character_constant(void)
1075 {
1076         const unsigned start_linenr = lexer_token.source_position.linenr;
1077
1078         eat('\'');
1079
1080         while (true) {
1081                 switch (c) {
1082                 case '\\': {
1083                         utf32 const tc = parse_escape_sequence();
1084                         if (tc >= 0x100) {
1085                                 warningf(&lexer_token.source_position,
1086                                                 "escape sequence out of range");
1087                         }
1088                         obstack_1grow(&symbol_obstack, tc);
1089                         break;
1090                 }
1091
1092                 MATCH_NEWLINE(
1093                         parse_error("newline while parsing character constant");
1094                         break;
1095                 )
1096
1097                 case '\'':
1098                         next_char();
1099                         goto end_of_char_constant;
1100
1101                 case EOF: {
1102                         source_position_t source_position;
1103                         source_position.input_name = lexer_token.source_position.input_name;
1104                         source_position.linenr     = start_linenr;
1105                         errorf(&source_position, "EOF while parsing character constant");
1106                         lexer_token.type = T_ERROR;
1107                         return;
1108                 }
1109
1110                 default:
1111                         grow_symbol(c);
1112                         next_char();
1113                         break;
1114
1115                 }
1116         }
1117
1118 end_of_char_constant:;
1119         obstack_1grow(&symbol_obstack, '\0');
1120         const size_t        size   = (size_t)obstack_object_size(&symbol_obstack)-1;
1121         char         *const string = obstack_finish(&symbol_obstack);
1122
1123         lexer_token.type    = T_CHARACTER_CONSTANT;
1124         lexer_token.literal = identify_string(string, size);
1125
1126         if (size == 0) {
1127                 parse_error("empty character constant");
1128         }
1129 }
1130
1131 /**
1132  * Skip a multiline comment.
1133  */
1134 static void skip_multiline_comment(void)
1135 {
1136         unsigned start_linenr = lexer_token.source_position.linenr;
1137
1138         while (true) {
1139                 switch (c) {
1140                 case '/':
1141                         next_char();
1142                         if (c == '*') {
1143                                 /* nested comment, warn here */
1144                                 if (warning.comment) {
1145                                         warningf(&lexer_token.source_position, "'/*' within comment");
1146                                 }
1147                         }
1148                         break;
1149                 case '*':
1150                         next_char();
1151                         if (c == '/') {
1152                                 next_char();
1153                                 return;
1154                         }
1155                         break;
1156
1157                 MATCH_NEWLINE(break;)
1158
1159                 case EOF: {
1160                         source_position_t source_position;
1161                         source_position.input_name = lexer_token.source_position.input_name;
1162                         source_position.linenr     = start_linenr;
1163                         errorf(&source_position, "at end of file while looking for comment end");
1164                         return;
1165                 }
1166
1167                 default:
1168                         next_char();
1169                         break;
1170                 }
1171         }
1172 }
1173
1174 /**
1175  * Skip a single line comment.
1176  */
1177 static void skip_line_comment(void)
1178 {
1179         while (true) {
1180                 switch (c) {
1181                 case EOF:
1182                         return;
1183
1184                 case '\n':
1185                 case '\r':
1186                         return;
1187
1188                 case '\\':
1189                         next_char();
1190                         if (c == '\n' || c == '\r') {
1191                                 if (warning.comment)
1192                                         warningf(&lexer_token.source_position, "multi-line comment");
1193                                 return;
1194                         }
1195                         break;
1196
1197                 default:
1198                         next_char();
1199                         break;
1200                 }
1201         }
1202 }
1203
1204 /** The current preprocessor token. */
1205 static token_t pp_token;
1206
1207 /**
1208  * Read the next preprocessor token.
1209  */
1210 static inline void next_pp_token(void)
1211 {
1212         lexer_next_preprocessing_token();
1213         pp_token = lexer_token;
1214 }
1215
1216 /**
1217  * Eat all preprocessor tokens until newline.
1218  */
1219 static void eat_until_newline(void)
1220 {
1221         while (pp_token.type != '\n' && pp_token.type != T_EOF) {
1222                 next_pp_token();
1223         }
1224 }
1225
1226 /**
1227  * Handle the define directive.
1228  */
1229 static void define_directive(void)
1230 {
1231         lexer_next_preprocessing_token();
1232         if (lexer_token.type != T_IDENTIFIER) {
1233                 parse_error("expected identifier after #define\n");
1234                 eat_until_newline();
1235         }
1236 }
1237
1238 /**
1239  * Handle the ifdef directive.
1240  */
1241 static void ifdef_directive(int is_ifndef)
1242 {
1243         (void) is_ifndef;
1244         lexer_next_preprocessing_token();
1245         //expect_identifier();
1246         //extect_newline();
1247 }
1248
1249 /**
1250  * Handle the endif directive.
1251  */
1252 static void endif_directive(void)
1253 {
1254         //expect_newline();
1255 }
1256
1257 /**
1258  * Parse the line directive.
1259  */
1260 static void parse_line_directive(void)
1261 {
1262         if (pp_token.type != T_INTEGER) {
1263                 parse_error("expected integer");
1264         } else {
1265                 /* use offset -1 as this is about the next line */
1266                 lexer_token.source_position.linenr = atoi(pp_token.literal.begin) - 1;
1267                 next_pp_token();
1268         }
1269         if (pp_token.type == T_STRING_LITERAL) {
1270                 lexer_token.source_position.input_name = pp_token.literal.begin;
1271                 next_pp_token();
1272         }
1273
1274         eat_until_newline();
1275 }
1276
1277 /**
1278  * STDC pragmas.
1279  */
1280 typedef enum stdc_pragma_kind_t {
1281         STDC_UNKNOWN,
1282         STDC_FP_CONTRACT,
1283         STDC_FENV_ACCESS,
1284         STDC_CX_LIMITED_RANGE
1285 } stdc_pragma_kind_t;
1286
1287 /**
1288  * STDC pragma values.
1289  */
1290 typedef enum stdc_pragma_value_kind_t {
1291         STDC_VALUE_UNKNOWN,
1292         STDC_VALUE_ON,
1293         STDC_VALUE_OFF,
1294         STDC_VALUE_DEFAULT
1295 } stdc_pragma_value_kind_t;
1296
1297 /**
1298  * Parse a pragma directive.
1299  */
1300 static void parse_pragma(void)
1301 {
1302         bool unknown_pragma = true;
1303
1304         next_pp_token();
1305         if (pp_token.symbol->pp_ID == TP_STDC) {
1306                 stdc_pragma_kind_t kind = STDC_UNKNOWN;
1307                 /* a STDC pragma */
1308                 if (c_mode & _C99) {
1309                         next_pp_token();
1310
1311                         switch (pp_token.symbol->pp_ID) {
1312                         case TP_FP_CONTRACT:
1313                                 kind = STDC_FP_CONTRACT;
1314                                 break;
1315                         case TP_FENV_ACCESS:
1316                                 kind = STDC_FENV_ACCESS;
1317                                 break;
1318                         case TP_CX_LIMITED_RANGE:
1319                                 kind = STDC_CX_LIMITED_RANGE;
1320                                 break;
1321                         default:
1322                                 break;
1323                         }
1324                         if (kind != STDC_UNKNOWN) {
1325                                 stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN;
1326                                 next_pp_token();
1327                                 switch (pp_token.symbol->pp_ID) {
1328                                 case TP_ON:
1329                                         value = STDC_VALUE_ON;
1330                                         break;
1331                                 case TP_OFF:
1332                                         value = STDC_VALUE_OFF;
1333                                         break;
1334                                 case TP_DEFAULT:
1335                                         value = STDC_VALUE_DEFAULT;
1336                                         break;
1337                                 default:
1338                                         break;
1339                                 }
1340                                 if (value != STDC_VALUE_UNKNOWN) {
1341                                         unknown_pragma = false;
1342                                 } else {
1343                                         errorf(&pp_token.source_position, "bad STDC pragma argument");
1344                                 }
1345                         }
1346                 }
1347         } else {
1348                 unknown_pragma = true;
1349         }
1350         eat_until_newline();
1351         if (unknown_pragma && warning.unknown_pragmas) {
1352                 warningf(&pp_token.source_position, "encountered unknown #pragma");
1353         }
1354 }
1355
1356 /**
1357  * Parse a preprocessor non-null directive.
1358  */
1359 static void parse_preprocessor_identifier(void)
1360 {
1361         assert(pp_token.type == T_IDENTIFIER);
1362         symbol_t *symbol = pp_token.symbol;
1363
1364         switch (symbol->pp_ID) {
1365         case TP_include:
1366                 printf("include - enable header name parsing!\n");
1367                 break;
1368         case TP_define:
1369                 define_directive();
1370                 break;
1371         case TP_ifdef:
1372                 ifdef_directive(0);
1373                 break;
1374         case TP_ifndef:
1375                 ifdef_directive(1);
1376                 break;
1377         case TP_endif:
1378                 endif_directive();
1379                 break;
1380         case TP_line:
1381                 next_pp_token();
1382                 parse_line_directive();
1383                 break;
1384         case TP_if:
1385         case TP_else:
1386         case TP_elif:
1387         case TP_undef:
1388         case TP_error:
1389                 /* TODO; output the rest of the line */
1390                 parse_error("#error directive: ");
1391                 break;
1392         case TP_pragma:
1393                 parse_pragma();
1394                 break;
1395         }
1396 }
1397
1398 /**
1399  * Parse a preprocessor directive.
1400  */
1401 static void parse_preprocessor_directive(void)
1402 {
1403         next_pp_token();
1404
1405         switch (pp_token.type) {
1406         case T_IDENTIFIER:
1407                 parse_preprocessor_identifier();
1408                 break;
1409         case T_INTEGER:
1410                 parse_line_directive();
1411                 break;
1412         case '\n':
1413                 /* NULL directive, see §6.10.7 */
1414                 break;
1415         default:
1416                 parse_error("invalid preprocessor directive");
1417                 eat_until_newline();
1418                 break;
1419         }
1420 }
1421
1422 #define MAYBE_PROLOG                                       \
1423                         next_char();                                   \
1424                         while (true) {                                 \
1425                                 switch (c) {
1426
1427 #define MAYBE(ch, set_type)                                \
1428                                 case ch:                                   \
1429                                         next_char();                           \
1430                                         lexer_token.type = set_type;           \
1431                                         return;
1432
1433 /* must use this as last thing */
1434 #define MAYBE_MODE(ch, set_type, mode)                     \
1435                                 case ch:                                   \
1436                                         if (c_mode & mode) {                   \
1437                                                 next_char();                       \
1438                                                 lexer_token.type = set_type;       \
1439                                                 return;                            \
1440                                         }                                      \
1441                                         /* fallthrough */
1442
1443 #define ELSE_CODE(code)                                    \
1444                                 default:                                   \
1445                                         code                                   \
1446                                 }                                          \
1447                         } /* end of while (true) */                    \
1448                         break;
1449
1450 #define ELSE(set_type)                                     \
1451                 ELSE_CODE(                                         \
1452                         lexer_token.type = set_type;                   \
1453                         return;                                        \
1454                 )
1455
1456 void lexer_next_preprocessing_token(void)
1457 {
1458         while (true) {
1459                 switch (c) {
1460                 case ' ':
1461                 case '\t':
1462                         next_char();
1463                         break;
1464
1465                 MATCH_NEWLINE(
1466                         lexer_token.type = '\n';
1467                         return;
1468                 )
1469
1470                 SYMBOL_CHARS
1471                         parse_symbol();
1472                         /* might be a wide string ( L"string" ) */
1473                         if (lexer_token.symbol == symbol_L) {
1474                                 switch (c) {
1475                                         case '"':  parse_wide_string_literal();     break;
1476                                         case '\'': parse_wide_character_constant(); break;
1477                                 }
1478                         }
1479                         return;
1480
1481                 DIGITS
1482                         parse_number();
1483                         return;
1484
1485                 case '"':
1486                         parse_string_literal();
1487                         return;
1488
1489                 case '\'':
1490                         parse_character_constant();
1491                         return;
1492
1493                 case '.':
1494                         MAYBE_PROLOG
1495                                 DIGITS
1496                                         put_back(c);
1497                                         c = '.';
1498                                         parse_number();
1499                                         return;
1500
1501                                 case '.':
1502                                         MAYBE_PROLOG
1503                                         MAYBE('.', T_DOTDOTDOT)
1504                                         ELSE_CODE(
1505                                                 put_back(c);
1506                                                 c = '.';
1507                                                 lexer_token.type = '.';
1508                                                 return;
1509                                         )
1510                         ELSE('.')
1511                 case '&':
1512                         MAYBE_PROLOG
1513                         MAYBE('&', T_ANDAND)
1514                         MAYBE('=', T_ANDEQUAL)
1515                         ELSE('&')
1516                 case '*':
1517                         MAYBE_PROLOG
1518                         MAYBE('=', T_ASTERISKEQUAL)
1519                         ELSE('*')
1520                 case '+':
1521                         MAYBE_PROLOG
1522                         MAYBE('+', T_PLUSPLUS)
1523                         MAYBE('=', T_PLUSEQUAL)
1524                         ELSE('+')
1525                 case '-':
1526                         MAYBE_PROLOG
1527                         MAYBE('>', T_MINUSGREATER)
1528                         MAYBE('-', T_MINUSMINUS)
1529                         MAYBE('=', T_MINUSEQUAL)
1530                         ELSE('-')
1531                 case '!':
1532                         MAYBE_PROLOG
1533                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1534                         ELSE('!')
1535                 case '/':
1536                         MAYBE_PROLOG
1537                         MAYBE('=', T_SLASHEQUAL)
1538                                 case '*':
1539                                         next_char();
1540                                         skip_multiline_comment();
1541                                         lexer_next_preprocessing_token();
1542                                         return;
1543                                 case '/':
1544                                         next_char();
1545                                         skip_line_comment();
1546                                         lexer_next_preprocessing_token();
1547                                         return;
1548                         ELSE('/')
1549                 case '%':
1550                         MAYBE_PROLOG
1551                         MAYBE('>', '}')
1552                         MAYBE('=', T_PERCENTEQUAL)
1553                                 case ':':
1554                                         MAYBE_PROLOG
1555                                                 case '%':
1556                                                         MAYBE_PROLOG
1557                                                         MAYBE(':', T_HASHHASH)
1558                                                         ELSE_CODE(
1559                                                                 put_back(c);
1560                                                                 c = '%';
1561                                                                 lexer_token.type = '#';
1562                                                                 return;
1563                                                         )
1564                                         ELSE('#')
1565                         ELSE('%')
1566                 case '<':
1567                         MAYBE_PROLOG
1568                         MAYBE(':', '[')
1569                         MAYBE('%', '{')
1570                         MAYBE('=', T_LESSEQUAL)
1571                                 case '<':
1572                                         MAYBE_PROLOG
1573                                         MAYBE('=', T_LESSLESSEQUAL)
1574                                         ELSE(T_LESSLESS)
1575                         ELSE('<')
1576                 case '>':
1577                         MAYBE_PROLOG
1578                         MAYBE('=', T_GREATEREQUAL)
1579                                 case '>':
1580                                         MAYBE_PROLOG
1581                                         MAYBE('=', T_GREATERGREATEREQUAL)
1582                                         ELSE(T_GREATERGREATER)
1583                         ELSE('>')
1584                 case '^':
1585                         MAYBE_PROLOG
1586                         MAYBE('=', T_CARETEQUAL)
1587                         ELSE('^')
1588                 case '|':
1589                         MAYBE_PROLOG
1590                         MAYBE('=', T_PIPEEQUAL)
1591                         MAYBE('|', T_PIPEPIPE)
1592                         ELSE('|')
1593                 case ':':
1594                         MAYBE_PROLOG
1595                         MAYBE('>', ']')
1596                         MAYBE_MODE(':', T_COLONCOLON, _CXX)
1597                         ELSE(':')
1598                 case '=':
1599                         MAYBE_PROLOG
1600                         MAYBE('=', T_EQUALEQUAL)
1601                         ELSE('=')
1602                 case '#':
1603                         MAYBE_PROLOG
1604                         MAYBE('#', T_HASHHASH)
1605                         ELSE('#')
1606
1607                 case '?':
1608                 case '[':
1609                 case ']':
1610                 case '(':
1611                 case ')':
1612                 case '{':
1613                 case '}':
1614                 case '~':
1615                 case ';':
1616                 case ',':
1617                 case '\\':
1618                         lexer_token.type = c;
1619                         next_char();
1620                         return;
1621
1622                 case EOF:
1623                         lexer_token.type = T_EOF;
1624                         return;
1625
1626                 default:
1627 dollar_sign:
1628                         errorf(&lexer_token.source_position, "unknown character '%c' found", c);
1629                         next_char();
1630                         lexer_token.type = T_ERROR;
1631                         return;
1632                 }
1633         }
1634 }
1635
1636 void lexer_next_token(void)
1637 {
1638         lexer_next_preprocessing_token();
1639
1640         while (lexer_token.type == '\n') {
1641 newline_found:
1642                 lexer_next_preprocessing_token();
1643         }
1644
1645         if (lexer_token.type == '#') {
1646                 parse_preprocessor_directive();
1647                 goto newline_found;
1648         }
1649 }
1650
1651 void init_lexer(void)
1652 {
1653         strset_init(&stringset);
1654         symbol_L = symbol_table_insert("L");
1655 }
1656
1657 void lexer_open_stream(FILE *stream, const char *input_name)
1658 {
1659         input                                  = stream;
1660         lexer_token.source_position.linenr     = 0;
1661         lexer_token.source_position.input_name = input_name;
1662
1663         bufpos = NULL;
1664         bufend = NULL;
1665
1666         /* place a virtual \n at the beginning so the lexer knows that we're
1667          * at the beginning of a line */
1668         c = '\n';
1669 }
1670
1671 void lexer_open_buffer(const char *buffer, size_t len, const char *input_name)
1672 {
1673         input                                  = NULL;
1674         lexer_token.source_position.linenr     = 0;
1675         lexer_token.source_position.input_name = input_name;
1676
1677 #if 0 // TODO
1678         bufpos = buffer;
1679         bufend = buffer + len;
1680
1681         /* place a virtual \n at the beginning so the lexer knows that we're
1682          * at the beginning of a line */
1683         c = '\n';
1684 #else
1685         (void)buffer;
1686         (void)len;
1687         panic("builtin lexing not done yet");
1688 #endif
1689 }
1690
1691 void exit_lexer(void)
1692 {
1693         strset_destroy(&stringset);
1694 }
1695
1696 static __attribute__((unused))
1697 void dbg_pos(const source_position_t source_position)
1698 {
1699         fprintf(stdout, "%s:%u\n", source_position.input_name,
1700                 source_position.linenr);
1701         fflush(stdout);
1702 }