nsz Git - cparser/blob - lexer.c

   1 /*
   2  * This file is part of cparser.
   3  * Copyright (C) 2007-2009 Matthias Braun <matze@braunis.de>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; either version 2
   8  * of the License, or (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  18  * 02111-1307, USA.
  19  */
  20 #include <config.h>
  21
  22 #include "diagnostic.h"
  23 #include "lexer.h"
  24 #include "symbol_t.h"
  25 #include "token_t.h"
  26 #include "symbol_table_t.h"
  27 #include "adt/error.h"
  28 #include "adt/strset.h"
  29 #include "adt/util.h"
  30 #include "types.h"
  31 #include "type_t.h"
  32 #include "target_architecture.h"
  33 #include "parser.h"
  34 #include "warning.h"
  35 #include "lang_features.h"
  36
  37 #include <assert.h>
  38 #include <errno.h>
  39 #include <string.h>
  40 #include <stdbool.h>
  41 #include <ctype.h>
  42
  43 #ifndef _WIN32
  44 #include <strings.h>
  45 #endif
  46
  47 //#define DEBUG_CHARS
  48 #define MAX_PUTBACK 3
  49 #define BUF_SIZE    1024
  50
  51 #if defined(_WIN32) || defined(__CYGWIN__)
  52 /* No strtold on windows and no replacement yet */
  53 #define strtold(s, e) strtod(s, e)
  54 #endif
  55
  56 static utf32        c;
  57 token_t             lexer_token;
  58 symbol_t           *symbol_L;
  59 static FILE        *input;
  60 static utf32        buf[BUF_SIZE + MAX_PUTBACK];
  61 static const utf32 *bufend;
  62 static const utf32 *bufpos;
  63 static strset_t     stringset;
  64 bool                allow_dollar_in_symbol = true;
  65
  66 /**
  67  * Prints a parse error message at the current token.
  68  *
  69  * @param msg   the error message
  70  */
  71 static void parse_error(const char *msg)
  72 {
  73         errorf(&lexer_token.source_position, "%s", msg);
  74 }
  75
  76 /**
  77  * Prints an internal error message at the current token.
  78  *
  79  * @param msg   the error message
  80  */
  81 static NORETURN internal_error(const char *msg)
  82 {
  83         internal_errorf(&lexer_token.source_position, "%s", msg);
  84 }
  85
  86 static size_t read_block(unsigned char *const read_buf, size_t const n)
  87 {
  88         size_t const s = fread(read_buf, 1, n, input);
  89         if (s == 0) {
  90                 if (ferror(input))
  91                         parse_error("read from input failed");
  92                 buf[MAX_PUTBACK] = EOF;
  93                 bufpos           = buf + MAX_PUTBACK;
  94                 bufend           = buf + MAX_PUTBACK + 1;
  95         }
  96         return s;
  97 }
  98
  99 static void decode_iso_8859_1(void)
 100 {
 101         unsigned char read_buf[BUF_SIZE];
 102         size_t const s = read_block(read_buf, sizeof(read_buf));
 103         if (s == 0)
 104                 return;
 105
 106         unsigned char const *src = read_buf;
 107         unsigned char const *end = read_buf + s;
 108         utf32               *dst = buf + MAX_PUTBACK;
 109         while (src != end)
 110                 *dst++ = *src++;
 111
 112         bufpos = buf + MAX_PUTBACK;
 113         bufend = dst;
 114 }
 115
 116 static void decode_iso_8859_15(void)
 117 {
 118         unsigned char read_buf[BUF_SIZE];
 119         size_t const s = read_block(read_buf, sizeof(read_buf));
 120         if (s == 0)
 121                 return;
 122
 123         unsigned char const *src = read_buf;
 124         unsigned char const *end = read_buf + s;
 125         utf32               *dst = buf + MAX_PUTBACK;
 126         while (src != end) {
 127                 utf32 tc = *src++;
 128                 switch (tc) {
 129                         case 0xA4: tc = 0x20AC; break; // €
 130                         case 0xA6: tc = 0x0160; break; // Š
 131                         case 0xA8: tc = 0x0161; break; // š
 132                         case 0xB4: tc = 0x017D; break; // Ž
 133                         case 0xB8: tc = 0x017E; break; // ž
 134                         case 0xBC: tc = 0x0152; break; // Œ
 135                         case 0xBD: tc = 0x0153; break; // œ
 136                         case 0xBE: tc = 0x0178; break; // Ÿ
 137                 }
 138                 *dst++ = tc;
 139         }
 140
 141         bufpos = buf + MAX_PUTBACK;
 142         bufend = dst;
 143 }
 144
 145 static void decode_utf8(void)
 146 {
 147         static utf32  part_decoded_min_code;
 148         static utf32  part_decoded_char;
 149         static size_t part_decoded_rest_len;
 150
 151         do {
 152                 unsigned char read_buf[BUF_SIZE];
 153                 size_t const s = read_block(read_buf, sizeof(read_buf));
 154                 if (s == 0) {
 155                         if (part_decoded_rest_len > 0)
 156                                 parse_error("incomplete input char at end of input");
 157                         return;
 158                 }
 159
 160                 unsigned char const *src = read_buf;
 161                 unsigned char const *end = read_buf + s;
 162                 utf32               *dst = buf + MAX_PUTBACK;
 163                 utf32                decoded;
 164                 utf32                min_code;
 165
 166                 if (part_decoded_rest_len != 0) {
 167                         min_code              = part_decoded_min_code;
 168                         decoded               = part_decoded_char;
 169                         size_t const rest_len = part_decoded_rest_len;
 170                         part_decoded_rest_len = 0;
 171                         switch (rest_len) {
 172                                 case 4:  goto realign;
 173                                 case 3:  goto three_more;
 174                                 case 2:  goto two_more;
 175                                 default: goto one_more;
 176                         }
 177                 }
 178
 179                 while (src != end) {
 180                         if ((*src & 0x80) == 0) {
 181                                 decoded = *src++;
 182                         } else if ((*src & 0xE0) == 0xC0) {
 183                                 min_code = 0x80;
 184                                 decoded  = *src++ & 0x1F;
 185 one_more:
 186                                 if (src == end) {
 187                                         part_decoded_min_code = min_code;
 188                                         part_decoded_char     = decoded;
 189                                         part_decoded_rest_len = 1;
 190                                         break;
 191                                 }
 192                                 if ((*src & 0xC0) == 0x80) {
 193                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 194                                 } else {
 195                                         goto invalid_char;
 196                                 }
 197                                 if (decoded < min_code                      ||
 198                                                 decoded > 0x10FFFF                      ||
 199                                                 (0xD800 <= decoded && decoded < 0xE000) || // high/low surrogates
 200                                                 (0xFDD0 <= decoded && decoded < 0xFDF0) || // noncharacters
 201                                                 (decoded & 0xFFFE) == 0xFFFE) {            // noncharacters
 202                                         parse_error("invalid byte sequence in input");
 203                                 }
 204                         } else if ((*src & 0xF0) == 0xE0) {
 205                                 min_code = 0x800;
 206                                 decoded  = *src++ & 0x0F;
 207 two_more:
 208                                 if (src == end) {
 209                                         part_decoded_min_code = min_code;
 210                                         part_decoded_char     = decoded;
 211                                         part_decoded_rest_len = 2;
 212                                         break;
 213                                 }
 214                                 if ((*src & 0xC0) == 0x80) {
 215                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 216                                 } else {
 217                                         goto invalid_char;
 218                                 }
 219                                 goto one_more;
 220                         } else if ((*src & 0xF8) == 0xF0) {
 221                                 min_code = 0x10000;
 222                                 decoded  = *src++ & 0x07;
 223 three_more:
 224                                 if (src == end) {
 225                                         part_decoded_min_code = min_code;
 226                                         part_decoded_char     = decoded;
 227                                         part_decoded_rest_len = 3;
 228                                         break;
 229                                 }
 230                                 if ((*src & 0xC0) == 0x80) {
 231                                         decoded = (decoded << 6) | (*src++ & 0x3F);
 232                                 } else {
 233                                         goto invalid_char;
 234                                 }
 235                                 goto two_more;
 236                         } else {
 237 invalid_char:
 238                                 parse_error("invalid byte sequence in input");
 239 realign:
 240                                 do {
 241                                         ++src;
 242                                         if (src == end) {
 243                                                 part_decoded_rest_len = 4;
 244                                                 break;
 245                                         }
 246                                 } while ((*src & 0xC0) == 0x80 || (*src & 0xF8) == 0xF8);
 247                                 continue;
 248                         }
 249                         *dst++ = decoded;
 250                 }
 251
 252                 bufpos = buf + MAX_PUTBACK;
 253                 bufend = dst;
 254         } while (bufpos == bufend);
 255 }
 256
 257 static void decode_windows_1252(void)
 258 {
 259         unsigned char read_buf[BUF_SIZE];
 260         size_t const s = read_block(read_buf, sizeof(read_buf));
 261         if (s == 0)
 262                 return;
 263
 264         unsigned char const *src = read_buf;
 265         unsigned char const *end = read_buf + s;
 266         utf32               *dst = buf + MAX_PUTBACK;
 267         while (src != end) {
 268                 utf32 tc = *src++;
 269                 switch (tc) {
 270                         case 0x80: tc = 0x20AC; break; // €
 271                         case 0x82: tc = 0x201A; break; // ‚
 272                         case 0x83: tc = 0x0192; break; // ƒ
 273                         case 0x84: tc = 0x201E; break; // „
 274                         case 0x85: tc = 0x2026; break; // …
 275                         case 0x86: tc = 0x2020; break; // †
 276                         case 0x87: tc = 0x2021; break; // ‡
 277                         case 0x88: tc = 0x02C6; break; // ˆ
 278                         case 0x89: tc = 0x2030; break; // ‰
 279                         case 0x8A: tc = 0x0160; break; // Š
 280                         case 0x8B: tc = 0x2039; break; // ‹
 281                         case 0x8C: tc = 0x0152; break; // Œ
 282                         case 0x8E: tc = 0x017D; break; // Ž
 283                         case 0x91: tc = 0x2018; break; // ‘
 284                         case 0x92: tc = 0x2019; break; // ’
 285                         case 0x93: tc = 0x201C; break; // “
 286                         case 0x94: tc = 0x201D; break; // ”
 287                         case 0x95: tc = 0x2022; break; // •
 288                         case 0x96: tc = 0x2013; break; // –
 289                         case 0x97: tc = 0x2014; break; // —
 290                         case 0x98: tc = 0x02DC; break; // ˜
 291                         case 0x99: tc = 0x2122; break; // ™
 292                         case 0x9A: tc = 0x0161; break; // š
 293                         case 0x9B: tc = 0x203A; break; // ›
 294                         case 0x9C: tc = 0x0153; break; // œ
 295                         case 0x9E: tc = 0x017E; break; // ž
 296                         case 0x9F: tc = 0x0178; break; // Ÿ
 297                 }
 298                 *dst++ = tc;
 299         }
 300
 301         bufpos = buf + MAX_PUTBACK;
 302         bufend = dst;
 303 }
 304
 305 typedef void (*decoder_t)(void);
 306
 307 static decoder_t decoder = decode_utf8;
 308
 309 typedef struct named_decoder_t {
 310         char const *name;
 311         decoder_t   decoder;
 312 } named_decoder_t;
 313
 314 static named_decoder_t const decoders[] = {
 315         { "CP819",           decode_iso_8859_1   }, // offical alias
 316         { "IBM819",          decode_iso_8859_1   }, // offical alias
 317         { "ISO-8859-1",      decode_iso_8859_1   }, // offical alias
 318         { "ISO-8859-15",     decode_iso_8859_15  }, // offical name
 319         { "ISO8859-1",       decode_iso_8859_1   },
 320         { "ISO8859-15",      decode_iso_8859_15  },
 321         { "ISO_8859-1",      decode_iso_8859_1   }, // offical alias
 322         { "ISO_8859-15",     decode_iso_8859_15  }, // offical alias
 323         { "ISO_8859-1:1987", decode_iso_8859_1   }, // offical name
 324         { "Latin-9",         decode_iso_8859_15  }, // offical alias
 325         { "UTF-8",           decode_utf8         }, // offical name
 326         { "csISOLatin1",     decode_iso_8859_1   }, // offical alias
 327         { "cp1252",          decode_windows_1252 },
 328         { "iso-ir-100",      decode_iso_8859_1   }, // offical alias
 329         { "l1",              decode_iso_8859_1   }, // offical alias
 330         { "latin1",          decode_iso_8859_1   }, // offical alias
 331         { "windows-1252",    decode_windows_1252 }, // official name
 332
 333         { NULL,              NULL                }
 334 };
 335
 336 void select_input_encoding(char const* const encoding)
 337 {
 338         for (named_decoder_t const *i = decoders; i->name != NULL; ++i) {
 339                 if (strcasecmp(encoding, i->name) != 0)
 340                         continue;
 341                 decoder = i->decoder;
 342                 return;
 343         }
 344         fprintf(stderr, "error: input encoding \"%s\" not supported\n", encoding);
 345 }
 346
 347 static inline void next_real_char(void)
 348 {
 349         assert(bufpos <= bufend);
 350         if (bufpos >= bufend) {
 351                 if (input == NULL) {
 352                         c = EOF;
 353                         return;
 354                 }
 355                 decoder();
 356         }
 357         c = *bufpos++;
 358 }
 359
 360 /**
 361  * Put a character back into the buffer.
 362  *
 363  * @param pc  the character to put back
 364  */
 365 static inline void put_back(utf32 const pc)
 366 {
 367         assert(bufpos > buf);
 368         *(--bufpos - buf + buf) = pc;
 369
 370 #ifdef DEBUG_CHARS
 371         printf("putback '%lc'\n", pc);
 372 #endif
 373 }
 374
 375 static inline void next_char(void);
 376
 377 #define MATCH_NEWLINE(code)                   \
 378         case '\r':                                \
 379                 next_char();                          \
 380                 if (c == '\n') {                      \
 381                         next_char();                      \
 382                 }                                     \
 383                 lexer_token.source_position.linenr++; \
 384                 code                                  \
 385         case '\n':                                \
 386                 next_char();                          \
 387                 lexer_token.source_position.linenr++; \
 388                 code
 389
 390 #define eat(c_type)  do { assert(c == c_type); next_char(); } while (0)
 391
 392 static void maybe_concat_lines(void)
 393 {
 394         eat('\\');
 395
 396         switch (c) {
 397         MATCH_NEWLINE(return;)
 398
 399         default:
 400                 break;
 401         }
 402
 403         put_back(c);
 404         c = '\\';
 405 }
 406
 407 /**
 408  * Set c to the next input character, ie.
 409  * after expanding trigraphs.
 410  */
 411 static inline void next_char(void)
 412 {
 413         next_real_char();
 414
 415         /* filter trigraphs */
 416         if (UNLIKELY(c == '\\')) {
 417                 maybe_concat_lines();
 418                 goto end_of_next_char;
 419         }
 420
 421         if (LIKELY(c != '?'))
 422                 goto end_of_next_char;
 423
 424         next_real_char();
 425         if (LIKELY(c != '?')) {
 426                 put_back(c);
 427                 c = '?';
 428                 goto end_of_next_char;
 429         }
 430
 431         next_real_char();
 432         switch (c) {
 433         case '=': c = '#'; break;
 434         case '(': c = '['; break;
 435         case '/': c = '\\'; maybe_concat_lines(); break;
 436         case ')': c = ']'; break;
 437         case '\'': c = '^'; break;
 438         case '<': c = '{'; break;
 439         case '!': c = '|'; break;
 440         case '>': c = '}'; break;
 441         case '-': c = '~'; break;
 442         default:
 443                 put_back(c);
 444                 put_back('?');
 445                 c = '?';
 446                 break;
 447         }
 448
 449 end_of_next_char:;
 450 #ifdef DEBUG_CHARS
 451         printf("nchar '%c'\n", c);
 452 #endif
 453 }
 454
 455 #define SYMBOL_CHARS  \
 456         case '$': if (!allow_dollar_in_symbol) goto dollar_sign; \
 457         case 'a':         \
 458         case 'b':         \
 459         case 'c':         \
 460         case 'd':         \
 461         case 'e':         \
 462         case 'f':         \
 463         case 'g':         \
 464         case 'h':         \
 465         case 'i':         \
 466         case 'j':         \
 467         case 'k':         \
 468         case 'l':         \
 469         case 'm':         \
 470         case 'n':         \
 471         case 'o':         \
 472         case 'p':         \
 473         case 'q':         \
 474         case 'r':         \
 475         case 's':         \
 476         case 't':         \
 477         case 'u':         \
 478         case 'v':         \
 479         case 'w':         \
 480         case 'x':         \
 481         case 'y':         \
 482         case 'z':         \
 483         case 'A':         \
 484         case 'B':         \
 485         case 'C':         \
 486         case 'D':         \
 487         case 'E':         \
 488         case 'F':         \
 489         case 'G':         \
 490         case 'H':         \
 491         case 'I':         \
 492         case 'J':         \
 493         case 'K':         \
 494         case 'L':         \
 495         case 'M':         \
 496         case 'N':         \
 497         case 'O':         \
 498         case 'P':         \
 499         case 'Q':         \
 500         case 'R':         \
 501         case 'S':         \
 502         case 'T':         \
 503         case 'U':         \
 504         case 'V':         \
 505         case 'W':         \
 506         case 'X':         \
 507         case 'Y':         \
 508         case 'Z':         \
 509         case '_':
 510
 511 #define DIGITS        \
 512         case '0':         \
 513         case '1':         \
 514         case '2':         \
 515         case '3':         \
 516         case '4':         \
 517         case '5':         \
 518         case '6':         \
 519         case '7':         \
 520         case '8':         \
 521         case '9':
 522
 523 /**
 524  * Read a symbol from the input and build
 525  * the lexer_token.
 526  */
 527 static void parse_symbol(void)
 528 {
 529         obstack_1grow(&symbol_obstack, (char) c);
 530         next_char();
 531
 532         while (true) {
 533                 switch (c) {
 534                 DIGITS
 535                 SYMBOL_CHARS
 536                         obstack_1grow(&symbol_obstack, (char) c);
 537                         next_char();
 538                         break;
 539
 540                 default:
 541 dollar_sign:
 542                         goto end_symbol;
 543                 }
 544         }
 545
 546 end_symbol:
 547         obstack_1grow(&symbol_obstack, '\0');
 548
 549         char     *string = obstack_finish(&symbol_obstack);
 550         symbol_t *symbol = symbol_table_insert(string);
 551
 552         lexer_token.type   = symbol->ID;
 553         lexer_token.symbol = symbol;
 554
 555         if (symbol->string != string) {
 556                 obstack_free(&symbol_obstack, string);
 557         }
 558 }
 559
 560 /**
 561  * parse suffixes like 'LU' or 'f' after numbers
 562  */
 563 static void parse_number_suffix(void)
 564 {
 565         assert(obstack_object_size(&symbol_obstack) == 0);
 566         while (true) {
 567                 switch (c) {
 568                 SYMBOL_CHARS
 569                         obstack_1grow(&symbol_obstack, (char) c);
 570                         next_char();
 571                         break;
 572                 default:
 573                 dollar_sign:
 574                         goto finish_suffix;
 575                 }
 576         }
 577 finish_suffix:
 578         if (obstack_object_size(&symbol_obstack) == 0) {
 579                 lexer_token.symbol = NULL;
 580                 return;
 581         }
 582
 583         obstack_1grow(&symbol_obstack, '\0');
 584         char     *string = obstack_finish(&symbol_obstack);
 585         symbol_t *symbol = symbol_table_insert(string);
 586
 587         if (symbol->string != string) {
 588                 obstack_free(&symbol_obstack, string);
 589         }
 590         lexer_token.symbol = symbol;
 591 }
 592
 593 static string_t identify_string(char *string, size_t len)
 594 {
 595         /* TODO hash */
 596 #if 0
 597         const char *result = strset_insert(&stringset, concat);
 598         if (result != concat) {
 599                 obstack_free(&symbol_obstack, concat);
 600         }
 601 #else
 602         const char *result = string;
 603 #endif
 604         return (string_t) {result, len};
 605 }
 606
 607 /**
 608  * Parses a hex number including hex floats and set the
 609  * lexer_token.
 610  */
 611 static void parse_number_hex(void)
 612 {
 613         bool is_float   = false;
 614         bool has_digits = false;
 615
 616         assert(obstack_object_size(&symbol_obstack) == 0);
 617         while (isxdigit(c)) {
 618                 has_digits = true;
 619                 obstack_1grow(&symbol_obstack, (char) c);
 620                 next_char();
 621         }
 622
 623         if (c == '.') {
 624                 is_float = true;
 625                 obstack_1grow(&symbol_obstack, (char) c);
 626                 next_char();
 627
 628                 while (isxdigit(c)) {
 629                         has_digits = true;
 630                         obstack_1grow(&symbol_obstack, (char) c);
 631                         next_char();
 632                 }
 633         }
 634         if (c == 'p' || c == 'P') {
 635                 is_float = true;
 636                 obstack_1grow(&symbol_obstack, (char) c);
 637                 next_char();
 638
 639                 if (c == '-' || c == '+') {
 640                         obstack_1grow(&symbol_obstack, (char) c);
 641                         next_char();
 642                 }
 643
 644                 while (isxdigit(c)) {
 645                         obstack_1grow(&symbol_obstack, (char) c);
 646                         next_char();
 647                 }
 648         } else if (is_float) {
 649                 errorf(&lexer_token.source_position,
 650                        "hexadecimal floatingpoint constant requires an exponent");
 651         }
 652
 653         size_t  size   = obstack_object_size(&symbol_obstack);
 654         char   *string = obstack_finish(&symbol_obstack);
 655         lexer_token.literal = identify_string(string, size);
 656
 657         lexer_token.type    =
 658                 is_float ? T_FLOATINGPOINT_HEXADECIMAL : T_INTEGER_HEXADECIMAL;
 659
 660         if (!has_digits) {
 661                 errorf(&lexer_token.source_position, "invalid number literal '0x%S'",
 662                        &lexer_token.literal);
 663                 lexer_token.literal.begin = "0";
 664                 lexer_token.literal.size  = 1;
 665         }
 666
 667         parse_number_suffix();
 668 }
 669
 670 /**
 671  * Returns true if the given char is a octal digit.
 672  *
 673  * @param char  the character to check
 674  */
 675 static bool is_octal_digit(utf32 chr)
 676 {
 677         switch (chr) {
 678         case '0':
 679         case '1':
 680         case '2':
 681         case '3':
 682         case '4':
 683         case '5':
 684         case '6':
 685         case '7':
 686                 return true;
 687         default:
 688                 return false;
 689         }
 690 }
 691
 692 /**
 693  * Parses a number and sets the lexer_token.
 694  */
 695 static void parse_number(void)
 696 {
 697         bool is_float   = false;
 698         bool has_digits = false;
 699
 700         assert(obstack_object_size(&symbol_obstack) == 0);
 701         if (c == '0') {
 702                 next_char();
 703                 if (c == 'x' || c == 'X') {
 704                         next_char();
 705                         parse_number_hex();
 706                         return;
 707                 } else {
 708                         has_digits = true;
 709                 }
 710                 obstack_1grow(&symbol_obstack, '0');
 711         }
 712
 713         while (isdigit(c)) {
 714                 has_digits = true;
 715                 obstack_1grow(&symbol_obstack, (char) c);
 716                 next_char();
 717         }
 718
 719         if (c == '.') {
 720                 is_float = true;
 721                 obstack_1grow(&symbol_obstack, '.');
 722                 next_char();
 723
 724                 while (isdigit(c)) {
 725                         has_digits = true;
 726                         obstack_1grow(&symbol_obstack, (char) c);
 727                         next_char();
 728                 }
 729         }
 730         if (c == 'e' || c == 'E') {
 731                 is_float = true;
 732                 obstack_1grow(&symbol_obstack, 'e');
 733                 next_char();
 734
 735                 if (c == '-' || c == '+') {
 736                         obstack_1grow(&symbol_obstack, (char) c);
 737                         next_char();
 738                 }
 739
 740                 while (isdigit(c)) {
 741                         obstack_1grow(&symbol_obstack, (char) c);
 742                         next_char();
 743                 }
 744         }
 745
 746         size_t  size   = obstack_object_size(&symbol_obstack);
 747         char   *string = obstack_finish(&symbol_obstack);
 748         lexer_token.literal = identify_string(string, size);
 749
 750         /* is it an octal number? */
 751         if (is_float) {
 752                 lexer_token.type = T_FLOATINGPOINT;
 753         } else if (string[0] == '0') {
 754                 lexer_token.type = T_INTEGER_OCTAL;
 755
 756                 /* check for invalid octal digits */
 757                 for (size_t i= 0; i < size; ++i) {
 758                         char t = string[i];
 759                         if (t == '8' || t == '9')
 760                                 errorf(&lexer_token.source_position,
 761                                        "invalid digit '%c' in octal number", t);
 762                 }
 763         } else {
 764                 lexer_token.type = T_INTEGER;
 765         }
 766
 767         if (!has_digits) {
 768                 errorf(&lexer_token.source_position, "invalid number literal '%S'",
 769                        &lexer_token.literal);
 770         }
 771
 772         parse_number_suffix();
 773 }
 774
 775 /**
 776  * Returns the value of a digit.
 777  * The only portable way to do it ...
 778  */
 779 static int digit_value(utf32 const digit)
 780 {
 781         switch (digit) {
 782         case '0': return 0;
 783         case '1': return 1;
 784         case '2': return 2;
 785         case '3': return 3;
 786         case '4': return 4;
 787         case '5': return 5;
 788         case '6': return 6;
 789         case '7': return 7;
 790         case '8': return 8;
 791         case '9': return 9;
 792         case 'a':
 793         case 'A': return 10;
 794         case 'b':
 795         case 'B': return 11;
 796         case 'c':
 797         case 'C': return 12;
 798         case 'd':
 799         case 'D': return 13;
 800         case 'e':
 801         case 'E': return 14;
 802         case 'f':
 803         case 'F': return 15;
 804         default:
 805                 internal_error("wrong character given");
 806         }
 807 }
 808
 809 /**
 810  * Parses an octal character sequence.
 811  *
 812  * @param first_digit  the already read first digit
 813  */
 814 static utf32 parse_octal_sequence(utf32 const first_digit)
 815 {
 816         assert(is_octal_digit(first_digit));
 817         utf32 value = digit_value(first_digit);
 818         if (!is_octal_digit(c)) return value;
 819         value = 8 * value + digit_value(c);
 820         next_char();
 821         if (!is_octal_digit(c)) return value;
 822         value = 8 * value + digit_value(c);
 823         next_char();
 824         return value;
 825 }
 826
 827 /**
 828  * Parses a hex character sequence.
 829  */
 830 static utf32 parse_hex_sequence(void)
 831 {
 832         utf32 value = 0;
 833         while (isxdigit(c)) {
 834                 value = 16 * value + digit_value(c);
 835                 next_char();
 836         }
 837         return value;
 838 }
 839
 840 /**
 841  * Parse an escape sequence.
 842  */
 843 static utf32 parse_escape_sequence(void)
 844 {
 845         eat('\\');
 846
 847         utf32 const ec = c;
 848         next_char();
 849
 850         switch (ec) {
 851         case '"':  return '"';
 852         case '\'': return '\'';
 853         case '\\': return '\\';
 854         case '?': return '\?';
 855         case 'a': return '\a';
 856         case 'b': return '\b';
 857         case 'f': return '\f';
 858         case 'n': return '\n';
 859         case 'r': return '\r';
 860         case 't': return '\t';
 861         case 'v': return '\v';
 862         case 'x':
 863                 return parse_hex_sequence();
 864         case '0':
 865         case '1':
 866         case '2':
 867         case '3':
 868         case '4':
 869         case '5':
 870         case '6':
 871         case '7':
 872                 return parse_octal_sequence(ec);
 873         case EOF:
 874                 parse_error("reached end of file while parsing escape sequence");
 875                 return EOF;
 876         /* \E is not documented, but handled, by GCC.  It is acceptable according
 877          * to §6.11.4, whereas \e is not. */
 878         case 'E':
 879         case 'e':
 880                 if (c_mode & _GNUC)
 881                         return 27;   /* hopefully 27 is ALWAYS the code for ESCAPE */
 882                 break;
 883         case 'u':
 884         case 'U':
 885                 parse_error("universal character parsing not implemented yet");
 886                 return EOF;
 887         default:
 888                 break;
 889         }
 890         /* §6.4.4.4:8 footnote 64 */
 891         parse_error("unknown escape sequence");
 892         return EOF;
 893 }
 894
 895 /**
 896  * Concatenate two strings.
 897  */
 898 string_t concat_strings(const string_t *const s1, const string_t *const s2)
 899 {
 900         const size_t len1 = s1->size - 1;
 901         const size_t len2 = s2->size - 1;
 902
 903         char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
 904         memcpy(concat, s1->begin, len1);
 905         memcpy(concat + len1, s2->begin, len2 + 1);
 906
 907         return identify_string(concat, len1 + len2 + 1);
 908 }
 909
 910 string_t make_string(const char *string)
 911 {
 912         size_t      len   = strlen(string) + 1;
 913         char *const space = obstack_alloc(&symbol_obstack, len);
 914         memcpy(space, string, len);
 915
 916         return identify_string(space, len);
 917 }
 918
 919 static void grow_symbol(utf32 const tc)
 920 {
 921         struct obstack *const o  = &symbol_obstack;
 922         if (tc < 0x80U) {
 923                 obstack_1grow(o, tc);
 924         } else if (tc < 0x800) {
 925                 obstack_1grow(o, 0xC0 | (tc >> 6));
 926                 obstack_1grow(o, 0x80 | (tc & 0x3F));
 927         } else if (tc < 0x10000) {
 928                 obstack_1grow(o, 0xE0 | ( tc >> 12));
 929                 obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
 930                 obstack_1grow(o, 0x80 | ( tc        & 0x3F));
 931         } else {
 932                 obstack_1grow(o, 0xF0 | ( tc >> 18));
 933                 obstack_1grow(o, 0x80 | ((tc >> 12) & 0x3F));
 934                 obstack_1grow(o, 0x80 | ((tc >>  6) & 0x3F));
 935                 obstack_1grow(o, 0x80 | ( tc        & 0x3F));
 936         }
 937 }
 938
 939 /**
 940  * Parse a string literal and set lexer_token.
 941  */
 942 static void parse_string_literal(void)
 943 {
 944         const unsigned start_linenr = lexer_token.source_position.linenr;
 945
 946         eat('"');
 947
 948         while (true) {
 949                 switch (c) {
 950                 case '\\': {
 951                         utf32 const tc = parse_escape_sequence();
 952                         if (tc >= 0x100) {
 953                                 warningf(&lexer_token.source_position,
 954                                                 "escape sequence out of range");
 955                         }
 956                         obstack_1grow(&symbol_obstack, tc);
 957                         break;
 958                 }
 959
 960                 case EOF: {
 961                         source_position_t source_position;
 962                         source_position.input_name = lexer_token.source_position.input_name;
 963                         source_position.linenr     = start_linenr;
 964                         errorf(&source_position, "string has no end");
 965                         lexer_token.type = T_ERROR;
 966                         return;
 967                 }
 968
 969                 case '"':
 970                         next_char();
 971                         goto end_of_string;
 972
 973                 default:
 974                         grow_symbol(c);
 975                         next_char();
 976                         break;
 977                 }
 978         }
 979
 980 end_of_string:
 981
 982         /* TODO: concatenate multiple strings separated by whitespace... */
 983
 984         /* add finishing 0 to the string */
 985         obstack_1grow(&symbol_obstack, '\0');
 986         const size_t  size   = (size_t)obstack_object_size(&symbol_obstack);
 987         char         *string = obstack_finish(&symbol_obstack);
 988
 989         lexer_token.type    = T_STRING_LITERAL;
 990         lexer_token.literal = identify_string(string, size);
 991 }
 992
 993 /**
 994  * Parse a wide character constant and set lexer_token.
 995  */
 996 static void parse_wide_character_constant(void)
 997 {
 998         const unsigned start_linenr = lexer_token.source_position.linenr;
 999
1000         eat('\'');
1001
1002         while (true) {
1003                 switch (c) {
1004                 case '\\': {
1005                         const utf32 tc = parse_escape_sequence();
1006                         grow_symbol(tc);
1007                         break;
1008                 }
1009
1010                 MATCH_NEWLINE(
1011                         parse_error("newline while parsing character constant");
1012                         break;
1013                 )
1014
1015                 case '\'':
1016                         next_char();
1017                         goto end_of_wide_char_constant;
1018
1019                 case EOF: {
1020                         source_position_t source_position = lexer_token.source_position;
1021                         source_position.linenr = start_linenr;
1022                         errorf(&source_position, "EOF while parsing character constant");
1023                         lexer_token.type = T_ERROR;
1024                         return;
1025                 }
1026
1027                 default:
1028                         grow_symbol(c);
1029                         next_char();
1030                         break;
1031                 }
1032         }
1033
1034 end_of_wide_char_constant:;
1035         size_t  size   = (size_t) obstack_object_size(&symbol_obstack);
1036         char   *string = obstack_finish(&symbol_obstack);
1037
1038         lexer_token.type     = T_WIDE_CHARACTER_CONSTANT;
1039         lexer_token.literal  = identify_string(string, size);
1040 }
1041
1042 /**
1043  * Parse a wide string literal and set lexer_token.
1044  */
1045 static void parse_wide_string_literal(void)
1046 {
1047         parse_string_literal();
1048         if (lexer_token.type == T_STRING_LITERAL)
1049                 lexer_token.type = T_WIDE_STRING_LITERAL;
1050 }
1051
1052 /**
1053  * Parse a character constant and set lexer_token.
1054  */
1055 static void parse_character_constant(void)
1056 {
1057         const unsigned start_linenr = lexer_token.source_position.linenr;
1058
1059         eat('\'');
1060
1061         while (true) {
1062                 switch (c) {
1063                 case '\\': {
1064                         utf32 const tc = parse_escape_sequence();
1065                         if (tc >= 0x100) {
1066                                 warningf(&lexer_token.source_position,
1067                                                 "escape sequence out of range");
1068                         }
1069                         obstack_1grow(&symbol_obstack, tc);
1070                         break;
1071                 }
1072
1073                 MATCH_NEWLINE(
1074                         parse_error("newline while parsing character constant");
1075                         break;
1076                 )
1077
1078                 case '\'':
1079                         next_char();
1080                         goto end_of_char_constant;
1081
1082                 case EOF: {
1083                         source_position_t source_position;
1084                         source_position.input_name = lexer_token.source_position.input_name;
1085                         source_position.linenr     = start_linenr;
1086                         errorf(&source_position, "EOF while parsing character constant");
1087                         lexer_token.type = T_ERROR;
1088                         return;
1089                 }
1090
1091                 default:
1092                         grow_symbol(c);
1093                         next_char();
1094                         break;
1095
1096                 }
1097         }
1098
1099 end_of_char_constant:;
1100         const size_t        size   = (size_t)obstack_object_size(&symbol_obstack);
1101         char         *const string = obstack_finish(&symbol_obstack);
1102
1103         lexer_token.type    = T_CHARACTER_CONSTANT;
1104         lexer_token.literal = identify_string(string, size);
1105 }
1106
1107 /**
1108  * Skip a multiline comment.
1109  */
1110 static void skip_multiline_comment(void)
1111 {
1112         unsigned start_linenr = lexer_token.source_position.linenr;
1113
1114         while (true) {
1115                 switch (c) {
1116                 case '/':
1117                         next_char();
1118                         if (c == '*') {
1119                                 /* nested comment, warn here */
1120                                 if (warning.comment) {
1121                                         warningf(&lexer_token.source_position, "'/*' within comment");
1122                                 }
1123                         }
1124                         break;
1125                 case '*':
1126                         next_char();
1127                         if (c == '/') {
1128                                 next_char();
1129                                 return;
1130                         }
1131                         break;
1132
1133                 MATCH_NEWLINE(break;)
1134
1135                 case EOF: {
1136                         source_position_t source_position;
1137                         source_position.input_name = lexer_token.source_position.input_name;
1138                         source_position.linenr     = start_linenr;
1139                         errorf(&source_position, "at end of file while looking for comment end");
1140                         return;
1141                 }
1142
1143                 default:
1144                         next_char();
1145                         break;
1146                 }
1147         }
1148 }
1149
1150 /**
1151  * Skip a single line comment.
1152  */
1153 static void skip_line_comment(void)
1154 {
1155         while (true) {
1156                 switch (c) {
1157                 case EOF:
1158                         return;
1159
1160                 case '\n':
1161                 case '\r':
1162                         return;
1163
1164                 case '\\':
1165                         next_char();
1166                         if (c == '\n' || c == '\r') {
1167                                 if (warning.comment)
1168                                         warningf(&lexer_token.source_position, "multi-line comment");
1169                                 return;
1170                         }
1171                         break;
1172
1173                 default:
1174                         next_char();
1175                         break;
1176                 }
1177         }
1178 }
1179
1180 /** The current preprocessor token. */
1181 static token_t pp_token;
1182
1183 /**
1184  * Read the next preprocessor token.
1185  */
1186 static inline void next_pp_token(void)
1187 {
1188         lexer_next_preprocessing_token();
1189         pp_token = lexer_token;
1190 }
1191
1192 /**
1193  * Eat all preprocessor tokens until newline.
1194  */
1195 static void eat_until_newline(void)
1196 {
1197         while (pp_token.type != '\n' && pp_token.type != T_EOF) {
1198                 next_pp_token();
1199         }
1200 }
1201
1202 /**
1203  * Handle the define directive.
1204  */
1205 static void define_directive(void)
1206 {
1207         lexer_next_preprocessing_token();
1208         if (lexer_token.type != T_IDENTIFIER) {
1209                 parse_error("expected identifier after #define\n");
1210                 eat_until_newline();
1211         }
1212 }
1213
1214 /**
1215  * Handle the ifdef directive.
1216  */
1217 static void ifdef_directive(int is_ifndef)
1218 {
1219         (void) is_ifndef;
1220         lexer_next_preprocessing_token();
1221         //expect_identifier();
1222         //extect_newline();
1223 }
1224
1225 /**
1226  * Handle the endif directive.
1227  */
1228 static void endif_directive(void)
1229 {
1230         //expect_newline();
1231 }
1232
1233 /**
1234  * Parse the line directive.
1235  */
1236 static void parse_line_directive(void)
1237 {
1238         if (pp_token.type != T_INTEGER) {
1239                 parse_error("expected integer");
1240         } else {
1241                 lexer_token.source_position.linenr = atoi(pp_token.literal.begin);
1242                 next_pp_token();
1243         }
1244         if (pp_token.type == T_STRING_LITERAL) {
1245                 lexer_token.source_position.input_name = pp_token.literal.begin;
1246                 next_pp_token();
1247         }
1248
1249         eat_until_newline();
1250 }
1251
1252 /**
1253  * STDC pragmas.
1254  */
1255 typedef enum stdc_pragma_kind_t {
1256         STDC_UNKNOWN,
1257         STDC_FP_CONTRACT,
1258         STDC_FENV_ACCESS,
1259         STDC_CX_LIMITED_RANGE
1260 } stdc_pragma_kind_t;
1261
1262 /**
1263  * STDC pragma values.
1264  */
1265 typedef enum stdc_pragma_value_kind_t {
1266         STDC_VALUE_UNKNOWN,
1267         STDC_VALUE_ON,
1268         STDC_VALUE_OFF,
1269         STDC_VALUE_DEFAULT
1270 } stdc_pragma_value_kind_t;
1271
1272 /**
1273  * Parse a pragma directive.
1274  */
1275 static void parse_pragma(void)
1276 {
1277         bool unknown_pragma = true;
1278
1279         next_pp_token();
1280         if (pp_token.symbol->pp_ID == TP_STDC) {
1281                 stdc_pragma_kind_t kind = STDC_UNKNOWN;
1282                 /* a STDC pragma */
1283                 if (c_mode & _C99) {
1284                         next_pp_token();
1285
1286                         switch (pp_token.symbol->pp_ID) {
1287                         case TP_FP_CONTRACT:
1288                                 kind = STDC_FP_CONTRACT;
1289                                 break;
1290                         case TP_FENV_ACCESS:
1291                                 kind = STDC_FENV_ACCESS;
1292                                 break;
1293                         case TP_CX_LIMITED_RANGE:
1294                                 kind = STDC_CX_LIMITED_RANGE;
1295                                 break;
1296                         default:
1297                                 break;
1298                         }
1299                         if (kind != STDC_UNKNOWN) {
1300                                 stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN;
1301                                 next_pp_token();
1302                                 switch (pp_token.symbol->pp_ID) {
1303                                 case TP_ON:
1304                                         value = STDC_VALUE_ON;
1305                                         break;
1306                                 case TP_OFF:
1307                                         value = STDC_VALUE_OFF;
1308                                         break;
1309                                 case TP_DEFAULT:
1310                                         value = STDC_VALUE_DEFAULT;
1311                                         break;
1312                                 default:
1313                                         break;
1314                                 }
1315                                 if (value != STDC_VALUE_UNKNOWN) {
1316                                         unknown_pragma = false;
1317                                 } else {
1318                                         errorf(&pp_token.source_position, "bad STDC pragma argument");
1319                                 }
1320                         }
1321                 }
1322         } else {
1323                 unknown_pragma = true;
1324         }
1325         eat_until_newline();
1326         if (unknown_pragma && warning.unknown_pragmas) {
1327                 warningf(&pp_token.source_position, "encountered unknown #pragma");
1328         }
1329 }
1330
1331 /**
1332  * Parse a preprocessor non-null directive.
1333  */
1334 static void parse_preprocessor_identifier(void)
1335 {
1336         assert(pp_token.type == T_IDENTIFIER);
1337         symbol_t *symbol = pp_token.symbol;
1338
1339         switch (symbol->pp_ID) {
1340         case TP_include:
1341                 printf("include - enable header name parsing!\n");
1342                 break;
1343         case TP_define:
1344                 define_directive();
1345                 break;
1346         case TP_ifdef:
1347                 ifdef_directive(0);
1348                 break;
1349         case TP_ifndef:
1350                 ifdef_directive(1);
1351                 break;
1352         case TP_endif:
1353                 endif_directive();
1354                 break;
1355         case TP_line:
1356                 next_pp_token();
1357                 parse_line_directive();
1358                 break;
1359         case TP_if:
1360         case TP_else:
1361         case TP_elif:
1362         case TP_undef:
1363         case TP_error:
1364                 /* TODO; output the rest of the line */
1365                 parse_error("#error directive: ");
1366                 break;
1367         case TP_pragma:
1368                 parse_pragma();
1369                 break;
1370         }
1371 }
1372
1373 /**
1374  * Parse a preprocessor directive.
1375  */
1376 static void parse_preprocessor_directive(void)
1377 {
1378         next_pp_token();
1379
1380         switch (pp_token.type) {
1381         case T_IDENTIFIER:
1382                 parse_preprocessor_identifier();
1383                 break;
1384         case T_INTEGER:
1385                 parse_line_directive();
1386                 break;
1387         case '\n':
1388                 /* NULL directive, see §6.10.7 */
1389                 break;
1390         default:
1391                 parse_error("invalid preprocessor directive");
1392                 eat_until_newline();
1393                 break;
1394         }
1395 }
1396
1397 #define MAYBE_PROLOG                                       \
1398                         next_char();                                   \
1399                         while (true) {                                 \
1400                                 switch (c) {
1401
1402 #define MAYBE(ch, set_type)                                \
1403                                 case ch:                                   \
1404                                         next_char();                           \
1405                                         lexer_token.type = set_type;           \
1406                                         return;
1407
1408 /* must use this as last thing */
1409 #define MAYBE_MODE(ch, set_type, mode)                     \
1410                                 case ch:                                   \
1411                                         if (c_mode & mode) {                   \
1412                                                 next_char();                       \
1413                                                 lexer_token.type = set_type;       \
1414                                                 return;                            \
1415                                         }                                      \
1416                                         /* fallthrough */
1417
1418 #define ELSE_CODE(code)                                    \
1419                                 default:                                   \
1420                                         code                                   \
1421                                 }                                          \
1422                         } /* end of while (true) */                    \
1423                         break;
1424
1425 #define ELSE(set_type)                                     \
1426                 ELSE_CODE(                                         \
1427                         lexer_token.type = set_type;                   \
1428                         return;                                        \
1429                 )
1430
1431 void lexer_next_preprocessing_token(void)
1432 {
1433         while (true) {
1434                 switch (c) {
1435                 case ' ':
1436                 case '\t':
1437                         next_char();
1438                         break;
1439
1440                 MATCH_NEWLINE(
1441                         lexer_token.type = '\n';
1442                         return;
1443                 )
1444
1445                 SYMBOL_CHARS
1446                         parse_symbol();
1447                         /* might be a wide string ( L"string" ) */
1448                         if (lexer_token.symbol == symbol_L) {
1449                                 switch (c) {
1450                                         case '"':  parse_wide_string_literal();     break;
1451                                         case '\'': parse_wide_character_constant(); break;
1452                                 }
1453                         }
1454                         return;
1455
1456                 DIGITS
1457                         parse_number();
1458                         return;
1459
1460                 case '"':
1461                         parse_string_literal();
1462                         return;
1463
1464                 case '\'':
1465                         parse_character_constant();
1466                         return;
1467
1468                 case '.':
1469                         MAYBE_PROLOG
1470                                 DIGITS
1471                                         put_back(c);
1472                                         c = '.';
1473                                         parse_number();
1474                                         return;
1475
1476                                 case '.':
1477                                         MAYBE_PROLOG
1478                                         MAYBE('.', T_DOTDOTDOT)
1479                                         ELSE_CODE(
1480                                                 put_back(c);
1481                                                 c = '.';
1482                                                 lexer_token.type = '.';
1483                                                 return;
1484                                         )
1485                         ELSE('.')
1486                 case '&':
1487                         MAYBE_PROLOG
1488                         MAYBE('&', T_ANDAND)
1489                         MAYBE('=', T_ANDEQUAL)
1490                         ELSE('&')
1491                 case '*':
1492                         MAYBE_PROLOG
1493                         MAYBE('=', T_ASTERISKEQUAL)
1494                         ELSE('*')
1495                 case '+':
1496                         MAYBE_PROLOG
1497                         MAYBE('+', T_PLUSPLUS)
1498                         MAYBE('=', T_PLUSEQUAL)
1499                         ELSE('+')
1500                 case '-':
1501                         MAYBE_PROLOG
1502                         MAYBE('>', T_MINUSGREATER)
1503                         MAYBE('-', T_MINUSMINUS)
1504                         MAYBE('=', T_MINUSEQUAL)
1505                         ELSE('-')
1506                 case '!':
1507                         MAYBE_PROLOG
1508                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1509                         ELSE('!')
1510                 case '/':
1511                         MAYBE_PROLOG
1512                         MAYBE('=', T_SLASHEQUAL)
1513                                 case '*':
1514                                         next_char();
1515                                         skip_multiline_comment();
1516                                         lexer_next_preprocessing_token();
1517                                         return;
1518                                 case '/':
1519                                         next_char();
1520                                         skip_line_comment();
1521                                         lexer_next_preprocessing_token();
1522                                         return;
1523                         ELSE('/')
1524                 case '%':
1525                         MAYBE_PROLOG
1526                         MAYBE('>', '}')
1527                         MAYBE('=', T_PERCENTEQUAL)
1528                                 case ':':
1529                                         MAYBE_PROLOG
1530                                                 case '%':
1531                                                         MAYBE_PROLOG
1532                                                         MAYBE(':', T_HASHHASH)
1533                                                         ELSE_CODE(
1534                                                                 put_back(c);
1535                                                                 c = '%';
1536                                                                 lexer_token.type = '#';
1537                                                                 return;
1538                                                         )
1539                                         ELSE('#')
1540                         ELSE('%')
1541                 case '<':
1542                         MAYBE_PROLOG
1543                         MAYBE(':', '[')
1544                         MAYBE('%', '{')
1545                         MAYBE('=', T_LESSEQUAL)
1546                                 case '<':
1547                                         MAYBE_PROLOG
1548                                         MAYBE('=', T_LESSLESSEQUAL)
1549                                         ELSE(T_LESSLESS)
1550                         ELSE('<')
1551                 case '>':
1552                         MAYBE_PROLOG
1553                         MAYBE('=', T_GREATEREQUAL)
1554                                 case '>':
1555                                         MAYBE_PROLOG
1556                                         MAYBE('=', T_GREATERGREATEREQUAL)
1557                                         ELSE(T_GREATERGREATER)
1558                         ELSE('>')
1559                 case '^':
1560                         MAYBE_PROLOG
1561                         MAYBE('=', T_CARETEQUAL)
1562                         ELSE('^')
1563                 case '|':
1564                         MAYBE_PROLOG
1565                         MAYBE('=', T_PIPEEQUAL)
1566                         MAYBE('|', T_PIPEPIPE)
1567                         ELSE('|')
1568                 case ':':
1569                         MAYBE_PROLOG
1570                         MAYBE('>', ']')
1571                         MAYBE_MODE(':', T_COLONCOLON, _CXX)
1572                         ELSE(':')
1573                 case '=':
1574                         MAYBE_PROLOG
1575                         MAYBE('=', T_EQUALEQUAL)
1576                         ELSE('=')
1577                 case '#':
1578                         MAYBE_PROLOG
1579                         MAYBE('#', T_HASHHASH)
1580                         ELSE('#')
1581
1582                 case '?':
1583                 case '[':
1584                 case ']':
1585                 case '(':
1586                 case ')':
1587                 case '{':
1588                 case '}':
1589                 case '~':
1590                 case ';':
1591                 case ',':
1592                 case '\\':
1593                         lexer_token.type = c;
1594                         next_char();
1595                         return;
1596
1597                 case EOF:
1598                         lexer_token.type = T_EOF;
1599                         return;
1600
1601                 default:
1602 dollar_sign:
1603                         errorf(&lexer_token.source_position, "unknown character '%c' found", c);
1604                         next_char();
1605                         lexer_token.type = T_ERROR;
1606                         return;
1607                 }
1608         }
1609 }
1610
1611 void lexer_next_token(void)
1612 {
1613         lexer_next_preprocessing_token();
1614
1615         while (lexer_token.type == '\n') {
1616 newline_found:
1617                 lexer_next_preprocessing_token();
1618         }
1619
1620         if (lexer_token.type == '#') {
1621                 parse_preprocessor_directive();
1622                 goto newline_found;
1623         }
1624 }
1625
1626 void init_lexer(void)
1627 {
1628         strset_init(&stringset);
1629         symbol_L = symbol_table_insert("L");
1630 }
1631
1632 void lexer_open_stream(FILE *stream, const char *input_name)
1633 {
1634         input                                  = stream;
1635         lexer_token.source_position.linenr     = 0;
1636         lexer_token.source_position.input_name = input_name;
1637
1638         bufpos = NULL;
1639         bufend = NULL;
1640
1641         /* place a virtual \n at the beginning so the lexer knows that we're
1642          * at the beginning of a line */
1643         c = '\n';
1644 }
1645
1646 void lexer_open_buffer(const char *buffer, size_t len, const char *input_name)
1647 {
1648         input                                  = NULL;
1649         lexer_token.source_position.linenr     = 0;
1650         lexer_token.source_position.input_name = input_name;
1651
1652 #if 0 // TODO
1653         bufpos = buffer;
1654         bufend = buffer + len;
1655 #else
1656         (void)buffer;
1657         (void)len;
1658         panic("builtin lexing not done yet");
1659 #endif
1660
1661         /* place a virtual \n at the beginning so the lexer knows that we're
1662          * at the beginning of a line */
1663         c = '\n';
1664 }
1665
1666 void exit_lexer(void)
1667 {
1668         strset_destroy(&stringset);
1669 }
1670
1671 static __attribute__((unused))
1672 void dbg_pos(const source_position_t source_position)
1673 {
1674         fprintf(stdout, "%s:%u\n", source_position.input_name,
1675                 source_position.linenr);
1676         fflush(stdout);
1677 }