nsz Git - cparser/blob - lexer.c

   1 /*
   2  * This file is part of cparser.
   3  * Copyright (C) 2007-2008 Matthias Braun <matze@braunis.de>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; either version 2
   8  * of the License, or (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  18  * 02111-1307, USA.
  19  */
  20 #include <config.h>
  21
  22 #include "diagnostic.h"
  23 #include "lexer.h"
  24 #include "symbol_t.h"
  25 #include "token_t.h"
  26 #include "symbol_table_t.h"
  27 #include "adt/error.h"
  28 #include "adt/strset.h"
  29 #include "adt/util.h"
  30 #include "types.h"
  31 #include "type_t.h"
  32 #include "target_architecture.h"
  33 #include "parser.h"
  34 #include "warning.h"
  35 #include "lang_features.h"
  36
  37 #include <assert.h>
  38 #include <errno.h>
  39 #include <string.h>
  40 #include <stdbool.h>
  41 #include <ctype.h>
  42
  43 //#define DEBUG_CHARS
  44 #define MAX_PUTBACK 3
  45
  46 #if defined(_WIN32) || defined(__CYGWIN__)
  47 /* No strtold on windows and no replacement yet */
  48 #define strtold(s, e) strtod(s, e)
  49 #endif
  50
  51 static int         c;
  52 token_t            lexer_token;
  53 symbol_t          *symbol_L;
  54 static FILE       *input;
  55 static char        buf[1024 + MAX_PUTBACK];
  56 static const char *bufend;
  57 static const char *bufpos;
  58 static strset_t    stringset;
  59 bool               allow_dollar_in_symbol = true;
  60
  61 /**
  62  * Prints a parse error message at the current token.
  63  *
  64  * @param msg   the error message
  65  */
  66 static void parse_error(const char *msg)
  67 {
  68         errorf(&lexer_token.source_position, "%s", msg);
  69 }
  70
  71 /**
  72  * Prints an internal error message at the current token.
  73  *
  74  * @param msg   the error message
  75  */
  76 static NORETURN internal_error(const char *msg)
  77 {
  78         internal_errorf(&lexer_token.source_position, "%s", msg);
  79 }
  80
  81 static inline void next_real_char(void)
  82 {
  83         assert(bufpos <= bufend);
  84         if (bufpos >= bufend) {
  85                 if (input == NULL) {
  86                         c = EOF;
  87                         return;
  88                 }
  89
  90                 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
  91                                  input);
  92                 if(s == 0) {
  93                         c = EOF;
  94                         return;
  95                 }
  96                 bufpos = buf + MAX_PUTBACK;
  97                 bufend = buf + MAX_PUTBACK + s;
  98         }
  99         c = (unsigned char)*bufpos++;
 100 }
 101
 102 /**
 103  * Put a character back into the buffer.
 104  *
 105  * @param pc  the character to put back
 106  */
 107 static inline void put_back(int pc)
 108 {
 109         assert(bufpos > buf);
 110         *(--bufpos - buf + buf) = (char) pc;
 111
 112 #ifdef DEBUG_CHARS
 113         printf("putback '%c'\n", pc);
 114 #endif
 115 }
 116
 117 static inline void next_char(void);
 118
 119 #define MATCH_NEWLINE(code)                   \
 120         case '\r':                                \
 121                 next_char();                          \
 122                 if(c == '\n') {                       \
 123                         next_char();                      \
 124                 }                                     \
 125                 lexer_token.source_position.linenr++; \
 126                 code                                  \
 127         case '\n':                                \
 128                 next_char();                          \
 129                 lexer_token.source_position.linenr++; \
 130                 code
 131
 132 #define eat(c_type)  do { assert(c == c_type); next_char(); } while(0)
 133
 134 static void maybe_concat_lines(void)
 135 {
 136         eat('\\');
 137
 138         switch(c) {
 139         MATCH_NEWLINE(return;)
 140
 141         default:
 142                 break;
 143         }
 144
 145         put_back(c);
 146         c = '\\';
 147 }
 148
 149 /**
 150  * Set c to the next input character, ie.
 151  * after expanding trigraphs.
 152  */
 153 static inline void next_char(void)
 154 {
 155         next_real_char();
 156
 157         /* filter trigraphs */
 158         if(UNLIKELY(c == '\\')) {
 159                 maybe_concat_lines();
 160                 goto end_of_next_char;
 161         }
 162
 163         if(LIKELY(c != '?'))
 164                 goto end_of_next_char;
 165
 166         next_real_char();
 167         if(LIKELY(c != '?')) {
 168                 put_back(c);
 169                 c = '?';
 170                 goto end_of_next_char;
 171         }
 172
 173         next_real_char();
 174         switch(c) {
 175         case '=': c = '#'; break;
 176         case '(': c = '['; break;
 177         case '/': c = '\\'; maybe_concat_lines(); break;
 178         case ')': c = ']'; break;
 179         case '\'': c = '^'; break;
 180         case '<': c = '{'; break;
 181         case '!': c = '|'; break;
 182         case '>': c = '}'; break;
 183         case '-': c = '~'; break;
 184         default:
 185                 put_back(c);
 186                 put_back('?');
 187                 c = '?';
 188                 break;
 189         }
 190
 191 end_of_next_char:;
 192 #ifdef DEBUG_CHARS
 193         printf("nchar '%c'\n", c);
 194 #endif
 195 }
 196
 197 #define SYMBOL_CHARS  \
 198         case '$': if (!allow_dollar_in_symbol) goto dollar_sign; \
 199         case 'a':         \
 200         case 'b':         \
 201         case 'c':         \
 202         case 'd':         \
 203         case 'e':         \
 204         case 'f':         \
 205         case 'g':         \
 206         case 'h':         \
 207         case 'i':         \
 208         case 'j':         \
 209         case 'k':         \
 210         case 'l':         \
 211         case 'm':         \
 212         case 'n':         \
 213         case 'o':         \
 214         case 'p':         \
 215         case 'q':         \
 216         case 'r':         \
 217         case 's':         \
 218         case 't':         \
 219         case 'u':         \
 220         case 'v':         \
 221         case 'w':         \
 222         case 'x':         \
 223         case 'y':         \
 224         case 'z':         \
 225         case 'A':         \
 226         case 'B':         \
 227         case 'C':         \
 228         case 'D':         \
 229         case 'E':         \
 230         case 'F':         \
 231         case 'G':         \
 232         case 'H':         \
 233         case 'I':         \
 234         case 'J':         \
 235         case 'K':         \
 236         case 'L':         \
 237         case 'M':         \
 238         case 'N':         \
 239         case 'O':         \
 240         case 'P':         \
 241         case 'Q':         \
 242         case 'R':         \
 243         case 'S':         \
 244         case 'T':         \
 245         case 'U':         \
 246         case 'V':         \
 247         case 'W':         \
 248         case 'X':         \
 249         case 'Y':         \
 250         case 'Z':         \
 251         case '_':
 252
 253 #define DIGITS        \
 254         case '0':         \
 255         case '1':         \
 256         case '2':         \
 257         case '3':         \
 258         case '4':         \
 259         case '5':         \
 260         case '6':         \
 261         case '7':         \
 262         case '8':         \
 263         case '9':
 264
 265 /**
 266  * Read a symbol from the input and build
 267  * the lexer_token.
 268  */
 269 static void parse_symbol(void)
 270 {
 271         symbol_t *symbol;
 272         char     *string;
 273
 274         obstack_1grow(&symbol_obstack, (char) c);
 275         next_char();
 276
 277         while(1) {
 278                 switch(c) {
 279                 DIGITS
 280                 SYMBOL_CHARS
 281                         obstack_1grow(&symbol_obstack, (char) c);
 282                         next_char();
 283                         break;
 284
 285                 default:
 286 dollar_sign:
 287                         goto end_symbol;
 288                 }
 289         }
 290
 291 end_symbol:
 292         obstack_1grow(&symbol_obstack, '\0');
 293
 294         string = obstack_finish(&symbol_obstack);
 295         symbol = symbol_table_insert(string);
 296
 297         lexer_token.type     = symbol->ID;
 298         lexer_token.v.symbol = symbol;
 299
 300         if(symbol->string != string) {
 301                 obstack_free(&symbol_obstack, string);
 302         }
 303 }
 304
 305 static void parse_integer_suffix(bool is_oct_hex)
 306 {
 307         bool is_unsigned     = false;
 308         bool min_long        = false;
 309         bool min_longlong    = false;
 310         bool not_traditional = false;
 311         int  pos             = 0;
 312         char suffix[4];
 313
 314         if (c == 'U' || c == 'u') {
 315                 not_traditional = true;
 316                 suffix[pos++]   = toupper(c);
 317                 is_unsigned     = true;
 318                 next_char();
 319                 if (c == 'L' || c == 'l') {
 320                         suffix[pos++] = toupper(c);
 321                         min_long = true;
 322                         next_char();
 323                         if (c == 'L' || c == 'l') {
 324                                 suffix[pos++] = toupper(c);
 325                                 min_longlong = true;
 326                                 next_char();
 327                         }
 328                 }
 329         } else if (c == 'l' || c == 'L') {
 330                 suffix[pos++] = toupper(c);
 331                 min_long = true;
 332                 next_char();
 333                 if (c == 'l' || c == 'L') {
 334                         not_traditional = true;
 335                         suffix[pos++]   = toupper(c);
 336                         min_longlong    = true;
 337                         next_char();
 338                         if (c == 'u' || c == 'U') {
 339                                 suffix[pos++] = toupper(c);
 340                                 is_unsigned   = true;
 341                                 next_char();
 342                         }
 343                 } else if (c == 'u' || c == 'U') {
 344                         not_traditional = true;
 345                         suffix[pos++]   = toupper(c);
 346                         is_unsigned     = true;
 347                         next_char();
 348                         lexer_token.datatype = type_unsigned_long;
 349                 }
 350         }
 351
 352         if (warning.traditional && not_traditional) {
 353                 suffix[pos] = '\0';
 354                 warningf(&lexer_token.source_position,
 355                         "traditional C rejects the '%s' suffix", suffix);
 356         }
 357         if (!is_unsigned) {
 358                 long long v = lexer_token.v.intvalue;
 359                 if (!min_long) {
 360                         if (v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
 361                                 lexer_token.datatype = type_int;
 362                                 return;
 363                         } else if (is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
 364                                 lexer_token.datatype = type_unsigned_int;
 365                                 return;
 366                         }
 367                 }
 368                 if (!min_longlong) {
 369                         if (v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
 370                                 lexer_token.datatype = type_long;
 371                                 return;
 372                         } else if (is_oct_hex && v >= 0 && (unsigned long long)v <= (unsigned long long)TARGET_ULONG_MAX) {
 373                                 lexer_token.datatype = type_unsigned_long;
 374                                 return;
 375                         }
 376                 }
 377                 unsigned long long uv = (unsigned long long) v;
 378                 if (is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
 379                         lexer_token.datatype = type_unsigned_long_long;
 380                         return;
 381                 }
 382
 383                 lexer_token.datatype = type_long_long;
 384         } else {
 385                 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
 386                 if (!min_long && v <= TARGET_UINT_MAX) {
 387                         lexer_token.datatype = type_unsigned_int;
 388                         return;
 389                 }
 390                 if (!min_longlong && v <= TARGET_ULONG_MAX) {
 391                         lexer_token.datatype = type_unsigned_long;
 392                         return;
 393                 }
 394                 lexer_token.datatype = type_unsigned_long_long;
 395         }
 396 }
 397
 398 static void parse_floating_suffix(void)
 399 {
 400         switch(c) {
 401         /* TODO: do something useful with the suffixes... */
 402         case 'f':
 403         case 'F':
 404                 if (warning.traditional) {
 405                         warningf(&lexer_token.source_position,
 406                                 "traditional C rejects the 'F' suffix");
 407                 }
 408                 next_char();
 409                 lexer_token.datatype = type_float;
 410                 break;
 411         case 'l':
 412         case 'L':
 413                 if (warning.traditional) {
 414                         warningf(&lexer_token.source_position,
 415                                 "traditional C rejects the 'F' suffix");
 416                 }
 417                 next_char();
 418                 lexer_token.datatype = type_long_double;
 419                 break;
 420         default:
 421                 lexer_token.datatype = type_double;
 422                 break;
 423         }
 424 }
 425
 426 /**
 427  * A replacement for strtoull. Only those parts needed for
 428  * our parser are implemented.
 429  */
 430 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
 431         unsigned long long v = 0;
 432
 433         switch (base) {
 434         case 16:
 435                 for (;; ++s) {
 436                         /* check for overrun */
 437                         if (v >= 0x1000000000000000ULL)
 438                                 break;
 439                         switch (tolower(*s)) {
 440                         case '0': v <<= 4; break;
 441                         case '1': v <<= 4; v |= 0x1; break;
 442                         case '2': v <<= 4; v |= 0x2; break;
 443                         case '3': v <<= 4; v |= 0x3; break;
 444                         case '4': v <<= 4; v |= 0x4; break;
 445                         case '5': v <<= 4; v |= 0x5; break;
 446                         case '6': v <<= 4; v |= 0x6; break;
 447                         case '7': v <<= 4; v |= 0x7; break;
 448                         case '8': v <<= 4; v |= 0x8; break;
 449                         case '9': v <<= 4; v |= 0x9; break;
 450                         case 'a': v <<= 4; v |= 0xa; break;
 451                         case 'b': v <<= 4; v |= 0xb; break;
 452                         case 'c': v <<= 4; v |= 0xc; break;
 453                         case 'd': v <<= 4; v |= 0xd; break;
 454                         case 'e': v <<= 4; v |= 0xe; break;
 455                         case 'f': v <<= 4; v |= 0xf; break;
 456                         default:
 457                                 goto end;
 458                         }
 459                 }
 460                 break;
 461         case 8:
 462                 for (;; ++s) {
 463                         /* check for overrun */
 464                         if (v >= 0x2000000000000000ULL)
 465                                 break;
 466                         switch (tolower(*s)) {
 467                         case '0': v <<= 3; break;
 468                         case '1': v <<= 3; v |= 1; break;
 469                         case '2': v <<= 3; v |= 2; break;
 470                         case '3': v <<= 3; v |= 3; break;
 471                         case '4': v <<= 3; v |= 4; break;
 472                         case '5': v <<= 3; v |= 5; break;
 473                         case '6': v <<= 3; v |= 6; break;
 474                         case '7': v <<= 3; v |= 7; break;
 475                         default:
 476                                 goto end;
 477                         }
 478                 }
 479                 break;
 480         case 10:
 481                 for (;; ++s) {
 482                         /* check for overrun */
 483                         if (v > 0x1999999999999999ULL)
 484                                 break;
 485                         switch (tolower(*s)) {
 486                         case '0': v *= 10; break;
 487                         case '1': v *= 10; v += 1; break;
 488                         case '2': v *= 10; v += 2; break;
 489                         case '3': v *= 10; v += 3; break;
 490                         case '4': v *= 10; v += 4; break;
 491                         case '5': v *= 10; v += 5; break;
 492                         case '6': v *= 10; v += 6; break;
 493                         case '7': v *= 10; v += 7; break;
 494                         case '8': v *= 10; v += 8; break;
 495                         case '9': v *= 10; v += 9; break;
 496                         default:
 497                                 goto end;
 498                         }
 499                 }
 500                 break;
 501         default:
 502                 assert(0);
 503                 break;
 504         }
 505 end:
 506         *endptr = s;
 507         return v;
 508 }
 509
 510 /**
 511  * Parses a hex number including hex floats and set the
 512  * lexer_token.
 513  */
 514 static void parse_number_hex(void)
 515 {
 516         bool is_float = false;
 517         assert(c == 'x' || c == 'X');
 518         next_char();
 519
 520         obstack_1grow(&symbol_obstack, '0');
 521         obstack_1grow(&symbol_obstack, 'x');
 522
 523         while(isxdigit(c)) {
 524                 obstack_1grow(&symbol_obstack, (char) c);
 525                 next_char();
 526         }
 527
 528         if (c == '.') {
 529                 obstack_1grow(&symbol_obstack, (char) c);
 530                 next_char();
 531
 532                 while (isxdigit(c)) {
 533                         obstack_1grow(&symbol_obstack, (char) c);
 534                         next_char();
 535                 }
 536                 is_float = true;
 537         }
 538         if (c == 'p' || c == 'P') {
 539                 obstack_1grow(&symbol_obstack, (char) c);
 540                 next_char();
 541
 542                 if (c == '-' || c == '+') {
 543                         obstack_1grow(&symbol_obstack, (char) c);
 544                         next_char();
 545                 }
 546
 547                 while (isxdigit(c)) {
 548                         obstack_1grow(&symbol_obstack, (char) c);
 549                         next_char();
 550                 }
 551                 is_float = true;
 552         }
 553
 554         obstack_1grow(&symbol_obstack, '\0');
 555         char *string = obstack_finish(&symbol_obstack);
 556         if(*string == '\0') {
 557                 parse_error("invalid hex number");
 558                 lexer_token.type = T_ERROR;
 559                 obstack_free(&symbol_obstack, string);
 560                 return;
 561         }
 562
 563         if (is_float) {
 564                 char *endptr;
 565                 lexer_token.type         = T_FLOATINGPOINT;
 566                 lexer_token.v.floatvalue = strtold(string, &endptr);
 567
 568                 if(*endptr != '\0') {
 569                         parse_error("invalid hex float literal");
 570                 }
 571
 572                 parse_floating_suffix();
 573         } else {
 574                 const char *endptr;
 575                 lexer_token.type       = T_INTEGER;
 576                 lexer_token.v.intvalue = parse_int_string(string + 2, &endptr, 16);
 577                 if(*endptr != '\0') {
 578                         parse_error("hex number literal too long");
 579                 }
 580                 parse_integer_suffix(true);
 581         }
 582
 583         obstack_free(&symbol_obstack, string);
 584 }
 585
 586 /**
 587  * Returns true if the given char is a octal digit.
 588  *
 589  * @param char  the character to check
 590  */
 591 static inline bool is_octal_digit(int chr)
 592 {
 593         switch(chr) {
 594         case '0':
 595         case '1':
 596         case '2':
 597         case '3':
 598         case '4':
 599         case '5':
 600         case '6':
 601         case '7':
 602                 return true;
 603         default:
 604                 return false;
 605         }
 606 }
 607
 608 /**
 609  * Parses a octal number and set the lexer_token.
 610  */
 611 static void parse_number_oct(void)
 612 {
 613         while(is_octal_digit(c)) {
 614                 obstack_1grow(&symbol_obstack, (char) c);
 615                 next_char();
 616         }
 617         obstack_1grow(&symbol_obstack, '\0');
 618         char *string = obstack_finish(&symbol_obstack);
 619
 620         const char *endptr;
 621         lexer_token.type       = T_INTEGER;
 622         lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
 623         if(*endptr != '\0') {
 624                 parse_error("octal number literal too long");
 625         }
 626
 627         obstack_free(&symbol_obstack, string);
 628         parse_integer_suffix(true);
 629 }
 630
 631 /**
 632  * Parses a decimal including float number and set the
 633  * lexer_token.
 634  */
 635 static void parse_number_dec(void)
 636 {
 637         bool is_float = false;
 638         while (isdigit(c)) {
 639                 obstack_1grow(&symbol_obstack, (char) c);
 640                 next_char();
 641         }
 642
 643         if (c == '.') {
 644                 obstack_1grow(&symbol_obstack, '.');
 645                 next_char();
 646
 647                 while (isdigit(c)) {
 648                         obstack_1grow(&symbol_obstack, (char) c);
 649                         next_char();
 650                 }
 651                 is_float = true;
 652         }
 653         if(c == 'e' || c == 'E') {
 654                 obstack_1grow(&symbol_obstack, (char) c);
 655                 next_char();
 656
 657                 if(c == '-' || c == '+') {
 658                         obstack_1grow(&symbol_obstack, (char) c);
 659                         next_char();
 660                 }
 661
 662                 while(isdigit(c)) {
 663                         obstack_1grow(&symbol_obstack, (char) c);
 664                         next_char();
 665                 }
 666                 is_float = true;
 667         }
 668
 669         obstack_1grow(&symbol_obstack, '\0');
 670         char *string = obstack_finish(&symbol_obstack);
 671
 672         if(is_float) {
 673                 char *endptr;
 674                 lexer_token.type         = T_FLOATINGPOINT;
 675                 lexer_token.v.floatvalue = strtold(string, &endptr);
 676
 677                 if(*endptr != '\0') {
 678                         parse_error("invalid number literal");
 679                 }
 680
 681                 parse_floating_suffix();
 682         } else {
 683                 const char *endptr;
 684                 lexer_token.type       = T_INTEGER;
 685                 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
 686
 687                 if(*endptr != '\0') {
 688                         parse_error("invalid number literal");
 689                 }
 690
 691                 parse_integer_suffix(false);
 692         }
 693         obstack_free(&symbol_obstack, string);
 694 }
 695
 696 /**
 697  * Parses a number and sets the lexer_token.
 698  */
 699 static void parse_number(void)
 700 {
 701         if (c == '0') {
 702                 next_char();
 703                 switch (c) {
 704                         case 'X':
 705                         case 'x':
 706                                 parse_number_hex();
 707                                 break;
 708                         case '0':
 709                         case '1':
 710                         case '2':
 711                         case '3':
 712                         case '4':
 713                         case '5':
 714                         case '6':
 715                         case '7':
 716                                 parse_number_oct();
 717                                 break;
 718                         case '8':
 719                         case '9':
 720                                 next_char();
 721                                 parse_error("invalid octal number");
 722                                 lexer_token.type = T_ERROR;
 723                                 return;
 724                         case '.':
 725                         case 'e':
 726                         case 'E':
 727                         default:
 728                                 obstack_1grow(&symbol_obstack, '0');
 729                                 parse_number_dec();
 730                                 return;
 731                 }
 732         } else {
 733                 parse_number_dec();
 734         }
 735 }
 736
 737 /**
 738  * Returns the value of a digit.
 739  * The only portable way to do it ...
 740  */
 741 static int digit_value(int digit) {
 742         switch (digit) {
 743         case '0': return 0;
 744         case '1': return 1;
 745         case '2': return 2;
 746         case '3': return 3;
 747         case '4': return 4;
 748         case '5': return 5;
 749         case '6': return 6;
 750         case '7': return 7;
 751         case '8': return 8;
 752         case '9': return 9;
 753         case 'a':
 754         case 'A': return 10;
 755         case 'b':
 756         case 'B': return 11;
 757         case 'c':
 758         case 'C': return 12;
 759         case 'd':
 760         case 'D': return 13;
 761         case 'e':
 762         case 'E': return 14;
 763         case 'f':
 764         case 'F': return 15;
 765         default:
 766                 internal_error("wrong character given");
 767         }
 768 }
 769
 770 /**
 771  * Parses an octal character sequence.
 772  *
 773  * @param first_digit  the already read first digit
 774  */
 775 static int parse_octal_sequence(const int first_digit)
 776 {
 777         assert(is_octal_digit(first_digit));
 778         int value = digit_value(first_digit);
 779         if (!is_octal_digit(c)) return value;
 780         value = 8 * value + digit_value(c);
 781         next_char();
 782         if (!is_octal_digit(c)) return value;
 783         value = 8 * value + digit_value(c);
 784         next_char();
 785
 786         if(char_is_signed) {
 787                 return (signed char) value;
 788         } else {
 789                 return (unsigned char) value;
 790         }
 791 }
 792
 793 /**
 794  * Parses a hex character sequence.
 795  */
 796 static int parse_hex_sequence(void)
 797 {
 798         int value = 0;
 799         while(isxdigit(c)) {
 800                 value = 16 * value + digit_value(c);
 801                 next_char();
 802         }
 803
 804         if(char_is_signed) {
 805                 return (signed char) value;
 806         } else {
 807                 return (unsigned char) value;
 808         }
 809 }
 810
 811 /**
 812  * Parse an escape sequence.
 813  */
 814 static int parse_escape_sequence(void)
 815 {
 816         eat('\\');
 817
 818         int ec = c;
 819         next_char();
 820
 821         switch (ec) {
 822         case '"':  return '"';
 823         case '\'': return '\'';
 824         case '\\': return '\\';
 825         case '?': return '\?';
 826         case 'a': return '\a';
 827         case 'b': return '\b';
 828         case 'f': return '\f';
 829         case 'n': return '\n';
 830         case 'r': return '\r';
 831         case 't': return '\t';
 832         case 'v': return '\v';
 833         case 'x':
 834                 return parse_hex_sequence();
 835         case '0':
 836         case '1':
 837         case '2':
 838         case '3':
 839         case '4':
 840         case '5':
 841         case '6':
 842         case '7':
 843                 return parse_octal_sequence(ec);
 844         case EOF:
 845                 parse_error("reached end of file while parsing escape sequence");
 846                 return EOF;
 847         /* \E is not documented, but handled, by GCC.  It is acceptable according
 848          * to §6.11.4, whereas \e is not. */
 849         case 'E':
 850         case 'e':
 851                 if (c_mode & _GNUC)
 852                         return 27;   /* hopefully 27 is ALWAYS the code for ESCAPE */
 853                 /* FALLTHROUGH */
 854         default:
 855                 /* §6.4.4.4:8 footnote 64 */
 856                 parse_error("unknown escape sequence");
 857                 return EOF;
 858         }
 859 }
 860
 861 /**
 862  * Concatenate two strings.
 863  */
 864 string_t concat_strings(const string_t *const s1, const string_t *const s2)
 865 {
 866         const size_t len1 = s1->size - 1;
 867         const size_t len2 = s2->size - 1;
 868
 869         char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
 870         memcpy(concat, s1->begin, len1);
 871         memcpy(concat + len1, s2->begin, len2 + 1);
 872
 873         if (warning.traditional) {
 874                 warningf(&lexer_token.source_position,
 875                         "traditional C rejects string constant concatenation");
 876         }
 877 #if 0 /* TODO hash */
 878         const char *result = strset_insert(&stringset, concat);
 879         if(result != concat) {
 880                 obstack_free(&symbol_obstack, concat);
 881         }
 882
 883         return result;
 884 #else
 885         return (string_t){ concat, len1 + len2 + 1 };
 886 #endif
 887 }
 888
 889 /**
 890  * Concatenate a string and a wide string.
 891  */
 892 wide_string_t concat_string_wide_string(const string_t *const s1, const wide_string_t *const s2)
 893 {
 894         const size_t len1 = s1->size - 1;
 895         const size_t len2 = s2->size - 1;
 896
 897         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 898         const char *const src = s1->begin;
 899         for (size_t i = 0; i != len1; ++i) {
 900                 concat[i] = src[i];
 901         }
 902         memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
 903         if (warning.traditional) {
 904                 warningf(&lexer_token.source_position,
 905                         "traditional C rejects string constant concatenation");
 906         }
 907
 908         return (wide_string_t){ concat, len1 + len2 + 1 };
 909 }
 910
 911 /**
 912  * Concatenate two wide strings.
 913  */
 914 wide_string_t concat_wide_strings(const wide_string_t *const s1, const wide_string_t *const s2)
 915 {
 916         const size_t len1 = s1->size - 1;
 917         const size_t len2 = s2->size - 1;
 918
 919         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 920         memcpy(concat,        s1->begin, len1       * sizeof(*concat));
 921         memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
 922         if (warning.traditional) {
 923                 warningf(&lexer_token.source_position,
 924                         "traditional C rejects string constant concatenation");
 925         }
 926
 927         return (wide_string_t){ concat, len1 + len2 + 1 };
 928 }
 929
 930 /**
 931  * Concatenate a wide string and a string.
 932  */
 933 wide_string_t concat_wide_string_string(const wide_string_t *const s1, const string_t *const s2)
 934 {
 935         const size_t len1 = s1->size - 1;
 936         const size_t len2 = s2->size - 1;
 937
 938         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 939         memcpy(concat, s1->begin, len1 * sizeof(*concat));
 940         const char  *const src = s2->begin;
 941         wchar_rep_t *const dst = concat + len1;
 942         for (size_t i = 0; i != len2 + 1; ++i) {
 943                 dst[i] = src[i];
 944         }
 945         if (warning.traditional) {
 946                 warningf(&lexer_token.source_position,
 947                         "traditional C rejects string constant concatenation");
 948         }
 949
 950         return (wide_string_t){ concat, len1 + len2 + 1 };
 951 }
 952
 953 /**
 954  * Parse a string literal and set lexer_token.
 955  */
 956 static void parse_string_literal(void)
 957 {
 958         const unsigned start_linenr = lexer_token.source_position.linenr;
 959
 960         eat('"');
 961
 962         int tc;
 963         while(1) {
 964                 switch(c) {
 965                 case '\\':
 966                         tc = parse_escape_sequence();
 967                         obstack_1grow(&symbol_obstack, (char) tc);
 968                         break;
 969
 970                 case EOF: {
 971                         source_position_t source_position;
 972                         source_position.input_name = lexer_token.source_position.input_name;
 973                         source_position.linenr     = start_linenr;
 974                         errorf(&source_position, "string has no end");
 975                         lexer_token.type = T_ERROR;
 976                         return;
 977                 }
 978
 979                 case '"':
 980                         next_char();
 981                         goto end_of_string;
 982
 983                 default:
 984                         obstack_1grow(&symbol_obstack, (char) c);
 985                         next_char();
 986                         break;
 987                 }
 988         }
 989
 990 end_of_string:
 991
 992         /* TODO: concatenate multiple strings separated by whitespace... */
 993
 994         /* add finishing 0 to the string */
 995         obstack_1grow(&symbol_obstack, '\0');
 996         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
 997         const char *const string = obstack_finish(&symbol_obstack);
 998
 999 #if 0 /* TODO hash */
1000         /* check if there is already a copy of the string */
1001         result = strset_insert(&stringset, string);
1002         if(result != string) {
1003                 obstack_free(&symbol_obstack, string);
1004         }
1005 #else
1006         const char *const result = string;
1007 #endif
1008
1009         lexer_token.type           = T_STRING_LITERAL;
1010         lexer_token.v.string.begin = result;
1011         lexer_token.v.string.size  = size;
1012 }
1013
1014 /**
1015  * Parse a wide character constant and set lexer_token.
1016  */
1017 static void parse_wide_character_constant(void)
1018 {
1019         const unsigned start_linenr = lexer_token.source_position.linenr;
1020
1021         eat('\'');
1022
1023         while(1) {
1024                 switch(c) {
1025                 case '\\': {
1026                         wchar_rep_t tc = parse_escape_sequence();
1027                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1028                         break;
1029                 }
1030
1031                 MATCH_NEWLINE(
1032                         parse_error("newline while parsing character constant");
1033                         break;
1034                 )
1035
1036                 case '\'':
1037                         next_char();
1038                         goto end_of_wide_char_constant;
1039
1040                 case EOF: {
1041                         source_position_t source_position = lexer_token.source_position;
1042                         source_position.linenr = start_linenr;
1043                         errorf(&source_position, "EOF while parsing character constant");
1044                         lexer_token.type = T_ERROR;
1045                         return;
1046                 }
1047
1048                 default: {
1049                         wchar_rep_t tc = (wchar_rep_t) c;
1050                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1051                         next_char();
1052                         break;
1053                 }
1054                 }
1055         }
1056
1057 end_of_wide_char_constant:;
1058         size_t             size   = (size_t) obstack_object_size(&symbol_obstack);
1059         assert(size % sizeof(wchar_rep_t) == 0);
1060         size /= sizeof(wchar_rep_t);
1061
1062         const wchar_rep_t *string = obstack_finish(&symbol_obstack);
1063
1064         lexer_token.type                = T_WIDE_CHARACTER_CONSTANT;
1065         lexer_token.v.wide_string.begin = string;
1066         lexer_token.v.wide_string.size  = size;
1067         lexer_token.datatype            = type_wchar_t;
1068 }
1069
1070 /**
1071  * Parse a wide string literal and set lexer_token.
1072  */
1073 static void parse_wide_string_literal(void)
1074 {
1075         const unsigned start_linenr = lexer_token.source_position.linenr;
1076
1077         assert(c == '"');
1078         next_char();
1079
1080         while(1) {
1081                 switch(c) {
1082                 case '\\': {
1083                         wchar_rep_t tc = parse_escape_sequence();
1084                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1085                         break;
1086                 }
1087
1088                 case EOF: {
1089                         source_position_t source_position;
1090                         source_position.input_name = lexer_token.source_position.input_name;
1091                         source_position.linenr     = start_linenr;
1092                         errorf(&source_position, "string has no end");
1093                         lexer_token.type = T_ERROR;
1094                         return;
1095                 }
1096
1097                 case '"':
1098                         next_char();
1099                         goto end_of_string;
1100
1101                 default: {
1102                         wchar_rep_t tc = c;
1103                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1104                         next_char();
1105                         break;
1106                 }
1107                 }
1108         }
1109
1110 end_of_string:;
1111
1112         /* TODO: concatenate multiple strings separated by whitespace... */
1113
1114         /* add finishing 0 to the string */
1115         wchar_rep_t nul = L'\0';
1116         obstack_grow(&symbol_obstack, &nul, sizeof(nul));
1117         const size_t             size   = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
1118         const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
1119
1120 #if 0 /* TODO hash */
1121         /* check if there is already a copy of the string */
1122         const wchar_rep_t *const result = strset_insert(&stringset, string);
1123         if(result != string) {
1124                 obstack_free(&symbol_obstack, string);
1125         }
1126 #else
1127         const wchar_rep_t *const result = string;
1128 #endif
1129
1130         lexer_token.type                = T_WIDE_STRING_LITERAL;
1131         lexer_token.v.wide_string.begin = result;
1132         lexer_token.v.wide_string.size  = size;
1133 }
1134
1135 /**
1136  * Parse a character constant and set lexer_token.
1137  */
1138 static void parse_character_constant(void)
1139 {
1140         const unsigned start_linenr = lexer_token.source_position.linenr;
1141
1142         eat('\'');
1143
1144         while(1) {
1145                 switch(c) {
1146                 case '\\': {
1147                         int tc = parse_escape_sequence();
1148                         obstack_1grow(&symbol_obstack, (char) tc);
1149                         break;
1150                 }
1151
1152                 MATCH_NEWLINE(
1153                         parse_error("newline while parsing character constant");
1154                         break;
1155                 )
1156
1157                 case '\'':
1158                         next_char();
1159                         goto end_of_char_constant;
1160
1161                 case EOF: {
1162                         source_position_t source_position;
1163                         source_position.input_name = lexer_token.source_position.input_name;
1164                         source_position.linenr     = start_linenr;
1165                         errorf(&source_position, "EOF while parsing character constant");
1166                         lexer_token.type = T_ERROR;
1167                         return;
1168                 }
1169
1170                 default:
1171                         obstack_1grow(&symbol_obstack, (char) c);
1172                         next_char();
1173                         break;
1174
1175                 }
1176         }
1177
1178 end_of_char_constant:;
1179         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
1180         const char *const string = obstack_finish(&symbol_obstack);
1181
1182         lexer_token.type           = T_CHARACTER_CONSTANT;
1183         lexer_token.v.string.begin = string;
1184         lexer_token.v.string.size  = size;
1185         lexer_token.datatype       = c_mode & _CXX && size == 1 ? type_char : type_int;
1186 }
1187
1188 /**
1189  * Skip a multiline comment.
1190  */
1191 static void skip_multiline_comment(void)
1192 {
1193         unsigned start_linenr = lexer_token.source_position.linenr;
1194
1195         while(1) {
1196                 switch(c) {
1197                 case '/':
1198                         next_char();
1199                         if (c == '*') {
1200                                 /* nested comment, warn here */
1201                                 if (warning.comment) {
1202                                         warningf(&lexer_token.source_position, "'/*' within comment");
1203                                 }
1204                         }
1205                         break;
1206                 case '*':
1207                         next_char();
1208                         if(c == '/') {
1209                                 next_char();
1210                                 return;
1211                         }
1212                         break;
1213
1214                 MATCH_NEWLINE(break;)
1215
1216                 case EOF: {
1217                         source_position_t source_position;
1218                         source_position.input_name = lexer_token.source_position.input_name;
1219                         source_position.linenr     = start_linenr;
1220                         errorf(&source_position, "at end of file while looking for comment end");
1221                         return;
1222                 }
1223
1224                 default:
1225                         next_char();
1226                         break;
1227                 }
1228         }
1229 }
1230
1231 /**
1232  * Skip a single line comment.
1233  */
1234 static void skip_line_comment(void)
1235 {
1236         while(1) {
1237                 switch(c) {
1238                 case EOF:
1239                         return;
1240
1241                 case '\n':
1242                 case '\r':
1243                         return;
1244
1245                 case '\\':
1246                         next_char();
1247                         if (c == '\n' || c == '\r') {
1248                                 if (warning.comment)
1249                                         warningf(&lexer_token.source_position, "multi-line comment");
1250                                 return;
1251                         }
1252                         break;
1253
1254                 default:
1255                         next_char();
1256                         break;
1257                 }
1258         }
1259 }
1260
1261 /** The current preprocessor token. */
1262 static token_t pp_token;
1263
1264 /**
1265  * Read the next preprocessor token.
1266  */
1267 static inline void next_pp_token(void)
1268 {
1269         lexer_next_preprocessing_token();
1270         pp_token = lexer_token;
1271 }
1272
1273 /**
1274  * Eat all preprocessor tokens until newline.
1275  */
1276 static void eat_until_newline(void)
1277 {
1278         while(pp_token.type != '\n' && pp_token.type != T_EOF) {
1279                 next_pp_token();
1280         }
1281 }
1282
1283 /**
1284  * Handle the define directive.
1285  */
1286 static void define_directive(void)
1287 {
1288         lexer_next_preprocessing_token();
1289         if(lexer_token.type != T_IDENTIFIER) {
1290                 parse_error("expected identifier after #define\n");
1291                 eat_until_newline();
1292         }
1293 }
1294
1295 /**
1296  * Handle the ifdef directive.
1297  */
1298 static void ifdef_directive(int is_ifndef)
1299 {
1300         (void) is_ifndef;
1301         lexer_next_preprocessing_token();
1302         //expect_identifier();
1303         //extect_newline();
1304 }
1305
1306 /**
1307  * Handle the endif directive.
1308  */
1309 static void endif_directive(void)
1310 {
1311         //expect_newline();
1312 }
1313
1314 /**
1315  * Parse the line directive.
1316  */
1317 static void parse_line_directive(void)
1318 {
1319         if(pp_token.type != T_INTEGER) {
1320                 parse_error("expected integer");
1321         } else {
1322                 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
1323                 next_pp_token();
1324         }
1325         if(pp_token.type == T_STRING_LITERAL) {
1326                 lexer_token.source_position.input_name = pp_token.v.string.begin;
1327                 next_pp_token();
1328         }
1329
1330         eat_until_newline();
1331 }
1332
1333 /**
1334  * STDC pragmas.
1335  */
1336 typedef enum stdc_pragma_kind_t {
1337         STDC_UNKNOWN,
1338         STDC_FP_CONTRACT,
1339         STDC_FENV_ACCESS,
1340         STDC_CX_LIMITED_RANGE
1341 } stdc_pragma_kind_t;
1342
1343 /**
1344  * STDC pragma values.
1345  */
1346 typedef enum stdc_pragma_value_kind_t {
1347         STDC_VALUE_UNKNOWN,
1348         STDC_VALUE_ON,
1349         STDC_VALUE_OFF,
1350         STDC_VALUE_DEFAULT
1351 } stdc_pragma_value_kind_t;
1352
1353 /**
1354  * Parse a pragma directive.
1355  */
1356 static void parse_pragma(void) {
1357         bool unknown_pragma = true;
1358
1359         next_pp_token();
1360         if (pp_token.v.symbol->pp_ID == TP_STDC) {
1361                 stdc_pragma_kind_t kind = STDC_UNKNOWN;
1362                 /* a STDC pragma */
1363                 if (c_mode & _C99) {
1364                         next_pp_token();
1365
1366                         switch (pp_token.v.symbol->pp_ID) {
1367                         case TP_FP_CONTRACT:
1368                                 kind = STDC_FP_CONTRACT;
1369                                 break;
1370                         case TP_FENV_ACCESS:
1371                                 kind = STDC_FENV_ACCESS;
1372                                 break;
1373                         case TP_CX_LIMITED_RANGE:
1374                                 kind = STDC_CX_LIMITED_RANGE;
1375                                 break;
1376                         default:
1377                                 break;
1378                         }
1379                         if (kind != STDC_UNKNOWN) {
1380                                 stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN;
1381                                 next_pp_token();
1382                                 switch (pp_token.v.symbol->pp_ID) {
1383                                 case TP_ON:
1384                                         value = STDC_VALUE_ON;
1385                                         break;
1386                                 case TP_OFF:
1387                                         value = STDC_VALUE_OFF;
1388                                         break;
1389                                 case TP_DEFAULT:
1390                                         value = STDC_VALUE_DEFAULT;
1391                                         break;
1392                                 default:
1393                                         break;
1394                                 }
1395                                 if (value != STDC_VALUE_UNKNOWN) {
1396                                         unknown_pragma = false;
1397                                 } else {
1398                                         errorf(&pp_token.source_position, "bad STDC pragma argument");
1399                                 }
1400                         }
1401                 }
1402         } else {
1403                 unknown_pragma = true;
1404         }
1405         eat_until_newline();
1406         if (unknown_pragma && warning.unknown_pragmas) {
1407                 warningf(&pp_token.source_position, "encountered unknown #pragma");
1408         }
1409 }
1410
1411 /**
1412  * Parse a preprocessor non-null directive.
1413  */
1414 static void parse_preprocessor_identifier(void)
1415 {
1416         assert(pp_token.type == T_IDENTIFIER);
1417         symbol_t *symbol = pp_token.v.symbol;
1418
1419         switch(symbol->pp_ID) {
1420         case TP_include:
1421                 printf("include - enable header name parsing!\n");
1422                 break;
1423         case TP_define:
1424                 define_directive();
1425                 break;
1426         case TP_ifdef:
1427                 ifdef_directive(0);
1428                 break;
1429         case TP_ifndef:
1430                 ifdef_directive(1);
1431                 break;
1432         case TP_endif:
1433                 endif_directive();
1434                 break;
1435         case TP_line:
1436                 next_pp_token();
1437                 parse_line_directive();
1438                 break;
1439         case TP_if:
1440         case TP_else:
1441         case TP_elif:
1442         case TP_undef:
1443         case TP_error:
1444                 /* TODO; output the rest of the line */
1445                 parse_error("#error directive: ");
1446                 break;
1447         case TP_pragma:
1448                 parse_pragma();
1449                 break;
1450         }
1451 }
1452
1453 /**
1454  * Parse a preprocessor directive.
1455  */
1456 static void parse_preprocessor_directive(void)
1457 {
1458         next_pp_token();
1459
1460         switch(pp_token.type) {
1461         case T_IDENTIFIER:
1462                 parse_preprocessor_identifier();
1463                 break;
1464         case T_INTEGER:
1465                 parse_line_directive();
1466                 break;
1467         case '\n':
1468                 /* NULL directive, see § 6.10.7 */
1469                 break;
1470         default:
1471                 parse_error("invalid preprocessor directive");
1472                 eat_until_newline();
1473                 break;
1474         }
1475 }
1476
1477 #define MAYBE_PROLOG                                       \
1478                         next_char();                                   \
1479                         while(1) {                                     \
1480                                 switch(c) {
1481
1482 #define MAYBE(ch, set_type)                                \
1483                                 case ch:                                   \
1484                                         next_char();                           \
1485                                         lexer_token.type = set_type;           \
1486                                         return;
1487
1488 #define ELSE_CODE(code)                                    \
1489                                 default:                                   \
1490                                         code                                   \
1491                                 }                                          \
1492                         } /* end of while(1) */                        \
1493                         break;
1494
1495 #define ELSE(set_type)                                     \
1496                 ELSE_CODE(                                         \
1497                         lexer_token.type = set_type;                   \
1498                         return;                                        \
1499                 )
1500
1501 void lexer_next_preprocessing_token(void)
1502 {
1503         while(1) {
1504                 switch(c) {
1505                 case ' ':
1506                 case '\t':
1507                         next_char();
1508                         break;
1509
1510                 MATCH_NEWLINE(
1511                         lexer_token.type = '\n';
1512                         return;
1513                 )
1514
1515                 SYMBOL_CHARS
1516                         parse_symbol();
1517                         /* might be a wide string ( L"string" ) */
1518                         if(lexer_token.type == T_IDENTIFIER &&
1519                             lexer_token.v.symbol == symbol_L) {
1520                             if(c == '"') {
1521                                         parse_wide_string_literal();
1522                                 } else if(c == '\'') {
1523                                         parse_wide_character_constant();
1524                                 }
1525                         }
1526                         return;
1527
1528                 DIGITS
1529                         parse_number();
1530                         return;
1531
1532                 case '"':
1533                         parse_string_literal();
1534                         return;
1535
1536                 case '\'':
1537                         parse_character_constant();
1538                         return;
1539
1540                 case '.':
1541                         MAYBE_PROLOG
1542                                 DIGITS
1543                                         put_back(c);
1544                                         c = '.';
1545                                         parse_number_dec();
1546                                         return;
1547
1548                                 case '.':
1549                                         MAYBE_PROLOG
1550                                         MAYBE('.', T_DOTDOTDOT)
1551                                         ELSE_CODE(
1552                                                 put_back(c);
1553                                                 c = '.';
1554                                                 lexer_token.type = '.';
1555                                                 return;
1556                                         )
1557                         ELSE('.')
1558                 case '&':
1559                         MAYBE_PROLOG
1560                         MAYBE('&', T_ANDAND)
1561                         MAYBE('=', T_ANDEQUAL)
1562                         ELSE('&')
1563                 case '*':
1564                         MAYBE_PROLOG
1565                         MAYBE('=', T_ASTERISKEQUAL)
1566                         ELSE('*')
1567                 case '+':
1568                         MAYBE_PROLOG
1569                         MAYBE('+', T_PLUSPLUS)
1570                         MAYBE('=', T_PLUSEQUAL)
1571                         ELSE('+')
1572                 case '-':
1573                         MAYBE_PROLOG
1574                         MAYBE('>', T_MINUSGREATER)
1575                         MAYBE('-', T_MINUSMINUS)
1576                         MAYBE('=', T_MINUSEQUAL)
1577                         ELSE('-')
1578                 case '!':
1579                         MAYBE_PROLOG
1580                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1581                         ELSE('!')
1582                 case '/':
1583                         MAYBE_PROLOG
1584                         MAYBE('=', T_SLASHEQUAL)
1585                                 case '*':
1586                                         next_char();
1587                                         skip_multiline_comment();
1588                                         lexer_next_preprocessing_token();
1589                                         return;
1590                                 case '/':
1591                                         next_char();
1592                                         skip_line_comment();
1593                                         lexer_next_preprocessing_token();
1594                                         return;
1595                         ELSE('/')
1596                 case '%':
1597                         MAYBE_PROLOG
1598                         MAYBE('>', '}')
1599                         MAYBE('=', T_PERCENTEQUAL)
1600                                 case ':':
1601                                         MAYBE_PROLOG
1602                                                 case '%':
1603                                                         MAYBE_PROLOG
1604                                                         MAYBE(':', T_HASHHASH)
1605                                                         ELSE_CODE(
1606                                                                 put_back(c);
1607                                                                 c = '%';
1608                                                                 lexer_token.type = '#';
1609                                                                 return;
1610                                                         )
1611                                         ELSE('#')
1612                         ELSE('%')
1613                 case '<':
1614                         MAYBE_PROLOG
1615                         MAYBE(':', '[')
1616                         MAYBE('%', '{')
1617                         MAYBE('=', T_LESSEQUAL)
1618                                 case '<':
1619                                         MAYBE_PROLOG
1620                                         MAYBE('=', T_LESSLESSEQUAL)
1621                                         ELSE(T_LESSLESS)
1622                         ELSE('<')
1623                 case '>':
1624                         MAYBE_PROLOG
1625                         MAYBE('=', T_GREATEREQUAL)
1626                                 case '>':
1627                                         MAYBE_PROLOG
1628                                         MAYBE('=', T_GREATERGREATEREQUAL)
1629                                         ELSE(T_GREATERGREATER)
1630                         ELSE('>')
1631                 case '^':
1632                         MAYBE_PROLOG
1633                         MAYBE('=', T_CARETEQUAL)
1634                         ELSE('^')
1635                 case '|':
1636                         MAYBE_PROLOG
1637                         MAYBE('=', T_PIPEEQUAL)
1638                         MAYBE('|', T_PIPEPIPE)
1639                         ELSE('|')
1640                 case ':':
1641                         MAYBE_PROLOG
1642                         MAYBE('>', ']')
1643                         ELSE(':')
1644                 case '=':
1645                         MAYBE_PROLOG
1646                         MAYBE('=', T_EQUALEQUAL)
1647                         ELSE('=')
1648                 case '#':
1649                         MAYBE_PROLOG
1650                         MAYBE('#', T_HASHHASH)
1651                         ELSE('#')
1652
1653                 case '?':
1654                 case '[':
1655                 case ']':
1656                 case '(':
1657                 case ')':
1658                 case '{':
1659                 case '}':
1660                 case '~':
1661                 case ';':
1662                 case ',':
1663                 case '\\':
1664                         lexer_token.type = c;
1665                         next_char();
1666                         return;
1667
1668                 case EOF:
1669                         lexer_token.type = T_EOF;
1670                         return;
1671
1672                 default:
1673 dollar_sign:
1674                         errorf(&lexer_token.source_position, "unknown character '%c' found", c);
1675                         next_char();
1676                         lexer_token.type = T_ERROR;
1677                         return;
1678                 }
1679         }
1680 }
1681
1682 void lexer_next_token(void)
1683 {
1684         lexer_next_preprocessing_token();
1685
1686         while (lexer_token.type == '\n') {
1687 newline_found:
1688                 lexer_next_preprocessing_token();
1689         }
1690
1691         if (lexer_token.type == '#') {
1692                 parse_preprocessor_directive();
1693                 goto newline_found;
1694         }
1695 }
1696
1697 void init_lexer(void)
1698 {
1699         strset_init(&stringset);
1700         symbol_L = symbol_table_insert("L");
1701 }
1702
1703 void lexer_open_stream(FILE *stream, const char *input_name)
1704 {
1705         input                                  = stream;
1706         lexer_token.source_position.linenr     = 0;
1707         lexer_token.source_position.input_name = input_name;
1708
1709         bufpos = NULL;
1710         bufend = NULL;
1711
1712         /* place a virtual \n at the beginning so the lexer knows that we're
1713          * at the beginning of a line */
1714         c = '\n';
1715 }
1716
1717 void lexer_open_buffer(const char *buffer, size_t len, const char *input_name)
1718 {
1719         input                                  = NULL;
1720         lexer_token.source_position.linenr     = 0;
1721         lexer_token.source_position.input_name = input_name;
1722
1723         bufpos = buffer;
1724         bufend = buffer + len;
1725
1726         /* place a virtual \n at the beginning so the lexer knows that we're
1727          * at the beginning of a line */
1728         c = '\n';
1729 }
1730
1731 void exit_lexer(void)
1732 {
1733         strset_destroy(&stringset);
1734 }
1735
1736 static __attribute__((unused))
1737 void dbg_pos(const source_position_t source_position)
1738 {
1739         fprintf(stdout, "%s:%u\n", source_position.input_name,
1740                 source_position.linenr);
1741         fflush(stdout);
1742 }