nsz Git - cparser/blob - lexer.c

   1 /*
   2  * This file is part of cparser.
   3  * Copyright (C) 2007-2008 Matthias Braun <matze@braunis.de>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; either version 2
   8  * of the License, or (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  18  * 02111-1307, USA.
  19  */
  20 #include <config.h>
  21
  22 #include "diagnostic.h"
  23 #include "lexer.h"
  24 #include "symbol_t.h"
  25 #include "token_t.h"
  26 #include "symbol_table_t.h"
  27 #include "adt/error.h"
  28 #include "adt/strset.h"
  29 #include "adt/util.h"
  30 #include "types.h"
  31 #include "type_t.h"
  32 #include "target_architecture.h"
  33 #include "parser.h"
  34 #include "warning.h"
  35 #include "lang_features.h"
  36
  37 #include <assert.h>
  38 #include <errno.h>
  39 #include <string.h>
  40 #include <stdbool.h>
  41 #include <ctype.h>
  42
  43 //#define DEBUG_CHARS
  44 #define MAX_PUTBACK 3
  45
  46 #ifdef _WIN32
  47 /* No strtold on windows and no replacement yet */
  48 #define strtold(s, e) strtod(s, e)
  49 #endif
  50
  51 static int         c;
  52 token_t            lexer_token;
  53 symbol_t          *symbol_L;
  54 static FILE       *input;
  55 static char        buf[1024 + MAX_PUTBACK];
  56 static const char *bufend;
  57 static const char *bufpos;
  58 static strset_t    stringset;
  59 bool               allow_dollar_in_symbol = true;
  60
  61 /**
  62  * Prints a parse error message at the current token.
  63  *
  64  * @param msg   the error message
  65  */
  66 static void parse_error(const char *msg)
  67 {
  68         errorf(&lexer_token.source_position,  "%s", msg);
  69 }
  70
  71 /**
  72  * Prints an internal error message at the current token.
  73  *
  74  * @param msg   the error message
  75  */
  76 static NORETURN internal_error(const char *msg)
  77 {
  78         internal_errorf(&lexer_token.source_position,  "%s", msg);
  79 }
  80
  81 static inline void next_real_char(void)
  82 {
  83         assert(bufpos <= bufend);
  84         if (bufpos >= bufend) {
  85                 if (input == NULL) {
  86                         c = EOF;
  87                         return;
  88                 }
  89
  90                 size_t s = fread(buf + MAX_PUTBACK, 1, sizeof(buf) - MAX_PUTBACK,
  91                                  input);
  92                 if(s == 0) {
  93                         c = EOF;
  94                         return;
  95                 }
  96                 bufpos = buf + MAX_PUTBACK;
  97                 bufend = buf + MAX_PUTBACK + s;
  98         }
  99         c = *bufpos++;
 100 }
 101
 102 /**
 103  * Put a character back into the buffer.
 104  *
 105  * @param pc  the character to put back
 106  */
 107 static inline void put_back(int pc)
 108 {
 109         assert(bufpos > buf);
 110         *(--bufpos - buf + buf) = (char) pc;
 111
 112 #ifdef DEBUG_CHARS
 113         printf("putback '%c'\n", pc);
 114 #endif
 115 }
 116
 117 static inline void next_char(void);
 118
 119 #define MATCH_NEWLINE(code)                   \
 120         case '\r':                                \
 121                 next_char();                          \
 122                 if(c == '\n') {                       \
 123                         next_char();                      \
 124                 }                                     \
 125                 lexer_token.source_position.linenr++; \
 126                 code                                  \
 127         case '\n':                                \
 128                 next_char();                          \
 129                 lexer_token.source_position.linenr++; \
 130                 code
 131
 132 #define eat(c_type)  do { assert(c == c_type); next_char(); } while(0)
 133
 134 static void maybe_concat_lines(void)
 135 {
 136         eat('\\');
 137
 138         switch(c) {
 139         MATCH_NEWLINE(return;)
 140
 141         default:
 142                 break;
 143         }
 144
 145         put_back(c);
 146         c = '\\';
 147 }
 148
 149 /**
 150  * Set c to the next input character, ie.
 151  * after expanding trigraphs.
 152  */
 153 static inline void next_char(void)
 154 {
 155         next_real_char();
 156
 157         /* filter trigraphs */
 158         if(UNLIKELY(c == '\\')) {
 159                 maybe_concat_lines();
 160                 goto end_of_next_char;
 161         }
 162
 163         if(LIKELY(c != '?'))
 164                 goto end_of_next_char;
 165
 166         next_real_char();
 167         if(LIKELY(c != '?')) {
 168                 put_back(c);
 169                 c = '?';
 170                 goto end_of_next_char;
 171         }
 172
 173         next_real_char();
 174         switch(c) {
 175         case '=': c = '#'; break;
 176         case '(': c = '['; break;
 177         case '/': c = '\\'; maybe_concat_lines(); break;
 178         case ')': c = ']'; break;
 179         case '\'': c = '^'; break;
 180         case '<': c = '{'; break;
 181         case '!': c = '|'; break;
 182         case '>': c = '}'; break;
 183         case '-': c = '~'; break;
 184         default:
 185                 put_back(c);
 186                 put_back('?');
 187                 c = '?';
 188                 break;
 189         }
 190
 191 end_of_next_char:;
 192 #ifdef DEBUG_CHARS
 193         printf("nchar '%c'\n", c);
 194 #endif
 195 }
 196
 197 #define SYMBOL_CHARS  \
 198         case '$': if (!allow_dollar_in_symbol) goto dollar_sign; \
 199         case 'a':         \
 200         case 'b':         \
 201         case 'c':         \
 202         case 'd':         \
 203         case 'e':         \
 204         case 'f':         \
 205         case 'g':         \
 206         case 'h':         \
 207         case 'i':         \
 208         case 'j':         \
 209         case 'k':         \
 210         case 'l':         \
 211         case 'm':         \
 212         case 'n':         \
 213         case 'o':         \
 214         case 'p':         \
 215         case 'q':         \
 216         case 'r':         \
 217         case 's':         \
 218         case 't':         \
 219         case 'u':         \
 220         case 'v':         \
 221         case 'w':         \
 222         case 'x':         \
 223         case 'y':         \
 224         case 'z':         \
 225         case 'A':         \
 226         case 'B':         \
 227         case 'C':         \
 228         case 'D':         \
 229         case 'E':         \
 230         case 'F':         \
 231         case 'G':         \
 232         case 'H':         \
 233         case 'I':         \
 234         case 'J':         \
 235         case 'K':         \
 236         case 'L':         \
 237         case 'M':         \
 238         case 'N':         \
 239         case 'O':         \
 240         case 'P':         \
 241         case 'Q':         \
 242         case 'R':         \
 243         case 'S':         \
 244         case 'T':         \
 245         case 'U':         \
 246         case 'V':         \
 247         case 'W':         \
 248         case 'X':         \
 249         case 'Y':         \
 250         case 'Z':         \
 251         case '_':
 252
 253 #define DIGITS        \
 254         case '0':         \
 255         case '1':         \
 256         case '2':         \
 257         case '3':         \
 258         case '4':         \
 259         case '5':         \
 260         case '6':         \
 261         case '7':         \
 262         case '8':         \
 263         case '9':
 264
 265 /**
 266  * Read a symbol from the input and build
 267  * the lexer_token.
 268  */
 269 static void parse_symbol(void)
 270 {
 271         symbol_t *symbol;
 272         char     *string;
 273
 274         obstack_1grow(&symbol_obstack, (char) c);
 275         next_char();
 276
 277         while(1) {
 278                 switch(c) {
 279                 DIGITS
 280                 SYMBOL_CHARS
 281                         obstack_1grow(&symbol_obstack, (char) c);
 282                         next_char();
 283                         break;
 284
 285                 default:
 286 dollar_sign:
 287                         goto end_symbol;
 288                 }
 289         }
 290
 291 end_symbol:
 292         obstack_1grow(&symbol_obstack, '\0');
 293
 294         string = obstack_finish(&symbol_obstack);
 295         symbol = symbol_table_insert(string);
 296
 297         lexer_token.type     = symbol->ID;
 298         lexer_token.v.symbol = symbol;
 299
 300         if(symbol->string != string) {
 301                 obstack_free(&symbol_obstack, string);
 302         }
 303 }
 304
 305 static void parse_integer_suffix(bool is_oct_hex)
 306 {
 307         bool is_unsigned  = false;
 308         bool min_long     = false;
 309         bool min_longlong = false;
 310
 311         if(c == 'U' || c == 'u') {
 312                 is_unsigned = true;
 313                 next_char();
 314                 if(c == 'L' || c == 'l') {
 315                         min_long = true;
 316                         next_char();
 317                         if(c == 'L' || c == 'l') {
 318                                 min_longlong = true;
 319                                 next_char();
 320                         }
 321                 }
 322         } else if(c == 'l' || c == 'L') {
 323                 min_long = true;
 324                 next_char();
 325                 if(c == 'l' || c == 'L') {
 326                         min_longlong = true;
 327                         next_char();
 328                         if(c == 'u' || c == 'U') {
 329                                 is_unsigned = true;
 330                                 next_char();
 331                         }
 332                 } else if(c == 'u' || c == 'U') {
 333                         is_unsigned = true;
 334                         next_char();
 335                         lexer_token.datatype = type_unsigned_long;
 336                 }
 337         }
 338
 339         if(!is_unsigned) {
 340                 long long v = lexer_token.v.intvalue;
 341                 if(!min_long) {
 342                         if(v >= TARGET_INT_MIN && v <= TARGET_INT_MAX) {
 343                                 lexer_token.datatype = type_int;
 344                                 return;
 345                         } else if(is_oct_hex && v >= 0 && v <= TARGET_UINT_MAX) {
 346                                 lexer_token.datatype = type_unsigned_int;
 347                                 return;
 348                         }
 349                 }
 350                 if(!min_longlong) {
 351                         if(v >= TARGET_LONG_MIN && v <= TARGET_LONG_MAX) {
 352                                 lexer_token.datatype = type_long;
 353                                 return;
 354                         } else if(is_oct_hex && v >= 0 && (unsigned long long)v <= (unsigned long long)TARGET_ULONG_MAX) {
 355                                 lexer_token.datatype = type_unsigned_long;
 356                                 return;
 357                         }
 358                 }
 359                 unsigned long long uv = (unsigned long long) v;
 360                 if(is_oct_hex && uv > (unsigned long long) TARGET_LONGLONG_MAX) {
 361                         lexer_token.datatype = type_unsigned_long_long;
 362                         return;
 363                 }
 364
 365                 lexer_token.datatype = type_long_long;
 366         } else {
 367                 unsigned long long v = (unsigned long long) lexer_token.v.intvalue;
 368                 if(!min_long && v <= TARGET_UINT_MAX) {
 369                         lexer_token.datatype = type_unsigned_int;
 370                         return;
 371                 }
 372                 if(!min_longlong && v <= TARGET_ULONG_MAX) {
 373                         lexer_token.datatype = type_unsigned_long;
 374                         return;
 375                 }
 376                 lexer_token.datatype = type_unsigned_long_long;
 377         }
 378 }
 379
 380 static void parse_floating_suffix(void)
 381 {
 382         switch(c) {
 383         /* TODO: do something useful with the suffixes... */
 384         case 'f':
 385         case 'F':
 386                 next_char();
 387                 lexer_token.datatype = type_float;
 388                 break;
 389         case 'l':
 390         case 'L':
 391                 next_char();
 392                 lexer_token.datatype = type_long_double;
 393                 break;
 394         default:
 395                 lexer_token.datatype = type_double;
 396                 break;
 397         }
 398 }
 399
 400 /**
 401  * A replacement for strtoull. Only those parts needed for
 402  * our parser are implemented.
 403  */
 404 static unsigned long long parse_int_string(const char *s, const char **endptr, int base) {
 405         unsigned long long v = 0;
 406
 407         switch (base) {
 408         case 16:
 409                 for (;; ++s) {
 410                         /* check for overrun */
 411                         if (v >= 0x1000000000000000ULL)
 412                                 break;
 413                         switch (tolower(*s)) {
 414                         case '0': v <<= 4; break;
 415                         case '1': v <<= 4; v |= 0x1; break;
 416                         case '2': v <<= 4; v |= 0x2; break;
 417                         case '3': v <<= 4; v |= 0x3; break;
 418                         case '4': v <<= 4; v |= 0x4; break;
 419                         case '5': v <<= 4; v |= 0x5; break;
 420                         case '6': v <<= 4; v |= 0x6; break;
 421                         case '7': v <<= 4; v |= 0x7; break;
 422                         case '8': v <<= 4; v |= 0x8; break;
 423                         case '9': v <<= 4; v |= 0x9; break;
 424                         case 'a': v <<= 4; v |= 0xa; break;
 425                         case 'b': v <<= 4; v |= 0xb; break;
 426                         case 'c': v <<= 4; v |= 0xc; break;
 427                         case 'd': v <<= 4; v |= 0xd; break;
 428                         case 'e': v <<= 4; v |= 0xe; break;
 429                         case 'f': v <<= 4; v |= 0xf; break;
 430                         default:
 431                                 goto end;
 432                         }
 433                 }
 434                 break;
 435         case 8:
 436                 for (;; ++s) {
 437                         /* check for overrun */
 438                         if (v >= 0x2000000000000000ULL)
 439                                 break;
 440                         switch (tolower(*s)) {
 441                         case '0': v <<= 3; break;
 442                         case '1': v <<= 3; v |= 1; break;
 443                         case '2': v <<= 3; v |= 2; break;
 444                         case '3': v <<= 3; v |= 3; break;
 445                         case '4': v <<= 3; v |= 4; break;
 446                         case '5': v <<= 3; v |= 5; break;
 447                         case '6': v <<= 3; v |= 6; break;
 448                         case '7': v <<= 3; v |= 7; break;
 449                         default:
 450                                 goto end;
 451                         }
 452                 }
 453                 break;
 454         case 10:
 455                 for (;; ++s) {
 456                         /* check for overrun */
 457                         if (v > 0x1999999999999999ULL)
 458                                 break;
 459                         switch (tolower(*s)) {
 460                         case '0': v *= 10; break;
 461                         case '1': v *= 10; v += 1; break;
 462                         case '2': v *= 10; v += 2; break;
 463                         case '3': v *= 10; v += 3; break;
 464                         case '4': v *= 10; v += 4; break;
 465                         case '5': v *= 10; v += 5; break;
 466                         case '6': v *= 10; v += 6; break;
 467                         case '7': v *= 10; v += 7; break;
 468                         case '8': v *= 10; v += 8; break;
 469                         case '9': v *= 10; v += 9; break;
 470                         default:
 471                                 goto end;
 472                         }
 473                 }
 474                 break;
 475         default:
 476                 assert(0);
 477                 break;
 478         }
 479 end:
 480         *endptr = s;
 481         return v;
 482 }
 483
 484 /**
 485  * Parses a hex number including hex floats and set the
 486  * lexer_token.
 487  */
 488 static void parse_number_hex(void)
 489 {
 490         bool is_float = false;
 491         assert(c == 'x' || c == 'X');
 492         next_char();
 493
 494         obstack_1grow(&symbol_obstack, '0');
 495         obstack_1grow(&symbol_obstack, 'x');
 496
 497         while(isxdigit(c)) {
 498                 obstack_1grow(&symbol_obstack, (char) c);
 499                 next_char();
 500         }
 501
 502         if (c == '.') {
 503                 obstack_1grow(&symbol_obstack, (char) c);
 504                 next_char();
 505
 506                 while (isxdigit(c)) {
 507                         obstack_1grow(&symbol_obstack, (char) c);
 508                         next_char();
 509                 }
 510                 is_float = true;
 511         }
 512         if (c == 'p' || c == 'P') {
 513                 obstack_1grow(&symbol_obstack, (char) c);
 514                 next_char();
 515
 516                 if (c == '-' || c == '+') {
 517                         obstack_1grow(&symbol_obstack, (char) c);
 518                         next_char();
 519                 }
 520
 521                 while (isxdigit(c)) {
 522                         obstack_1grow(&symbol_obstack, (char) c);
 523                         next_char();
 524                 }
 525                 is_float = true;
 526         }
 527
 528         obstack_1grow(&symbol_obstack, '\0');
 529         char *string = obstack_finish(&symbol_obstack);
 530         if(*string == '\0') {
 531                 parse_error("invalid hex number");
 532                 lexer_token.type = T_ERROR;
 533                 obstack_free(&symbol_obstack, string);
 534                 return;
 535         }
 536
 537         if (is_float) {
 538                 char *endptr;
 539                 lexer_token.type         = T_FLOATINGPOINT;
 540                 lexer_token.v.floatvalue = strtold(string, &endptr);
 541
 542                 if(*endptr != '\0') {
 543                         parse_error("invalid hex float literal");
 544                 }
 545
 546                 parse_floating_suffix();
 547         } else {
 548                 const char *endptr;
 549                 lexer_token.type       = T_INTEGER;
 550                 lexer_token.v.intvalue = parse_int_string(string + 2, &endptr, 16);
 551                 if(*endptr != '\0') {
 552                         parse_error("hex number literal too long");
 553                 }
 554                 parse_integer_suffix(true);
 555         }
 556
 557         obstack_free(&symbol_obstack, string);
 558 }
 559
 560 /**
 561  * Returns true if the given char is a octal digit.
 562  *
 563  * @param char  the character to check
 564  */
 565 static inline bool is_octal_digit(int chr)
 566 {
 567         switch(chr) {
 568         case '0':
 569         case '1':
 570         case '2':
 571         case '3':
 572         case '4':
 573         case '5':
 574         case '6':
 575         case '7':
 576                 return true;
 577         default:
 578                 return false;
 579         }
 580 }
 581
 582 /**
 583  * Parses a octal number and set the lexer_token.
 584  */
 585 static void parse_number_oct(void)
 586 {
 587         while(is_octal_digit(c)) {
 588                 obstack_1grow(&symbol_obstack, (char) c);
 589                 next_char();
 590         }
 591         obstack_1grow(&symbol_obstack, '\0');
 592         char *string = obstack_finish(&symbol_obstack);
 593
 594         const char *endptr;
 595         lexer_token.type       = T_INTEGER;
 596         lexer_token.v.intvalue = parse_int_string(string, &endptr, 8);
 597         if(*endptr != '\0') {
 598                 parse_error("octal number literal too long");
 599         }
 600
 601         obstack_free(&symbol_obstack, string);
 602         parse_integer_suffix(true);
 603 }
 604
 605 /**
 606  * Parses a decimal including float number and set the
 607  * lexer_token.
 608  */
 609 static void parse_number_dec(void)
 610 {
 611         bool is_float = false;
 612         while (isdigit(c)) {
 613                 obstack_1grow(&symbol_obstack, (char) c);
 614                 next_char();
 615         }
 616
 617         if (c == '.') {
 618                 obstack_1grow(&symbol_obstack, '.');
 619                 next_char();
 620
 621                 while (isdigit(c)) {
 622                         obstack_1grow(&symbol_obstack, (char) c);
 623                         next_char();
 624                 }
 625                 is_float = true;
 626         }
 627         if(c == 'e' || c == 'E') {
 628                 obstack_1grow(&symbol_obstack, (char) c);
 629                 next_char();
 630
 631                 if(c == '-' || c == '+') {
 632                         obstack_1grow(&symbol_obstack, (char) c);
 633                         next_char();
 634                 }
 635
 636                 while(isdigit(c)) {
 637                         obstack_1grow(&symbol_obstack, (char) c);
 638                         next_char();
 639                 }
 640                 is_float = true;
 641         }
 642
 643         obstack_1grow(&symbol_obstack, '\0');
 644         char *string = obstack_finish(&symbol_obstack);
 645
 646         if(is_float) {
 647                 char *endptr;
 648                 lexer_token.type         = T_FLOATINGPOINT;
 649                 lexer_token.v.floatvalue = strtold(string, &endptr);
 650
 651                 if(*endptr != '\0') {
 652                         parse_error("invalid number literal");
 653                 }
 654
 655                 parse_floating_suffix();
 656         } else {
 657                 const char *endptr;
 658                 lexer_token.type       = T_INTEGER;
 659                 lexer_token.v.intvalue = parse_int_string(string, &endptr, 10);
 660
 661                 if(*endptr != '\0') {
 662                         parse_error("invalid number literal");
 663                 }
 664
 665                 parse_integer_suffix(false);
 666         }
 667         obstack_free(&symbol_obstack, string);
 668 }
 669
 670 /**
 671  * Parses a number and sets the lexer_token.
 672  */
 673 static void parse_number(void)
 674 {
 675         if (c == '0') {
 676                 next_char();
 677                 switch (c) {
 678                         case 'X':
 679                         case 'x':
 680                                 parse_number_hex();
 681                                 break;
 682                         case '0':
 683                         case '1':
 684                         case '2':
 685                         case '3':
 686                         case '4':
 687                         case '5':
 688                         case '6':
 689                         case '7':
 690                                 parse_number_oct();
 691                                 break;
 692                         case '8':
 693                         case '9':
 694                                 next_char();
 695                                 parse_error("invalid octal number");
 696                                 lexer_token.type = T_ERROR;
 697                                 return;
 698                         case '.':
 699                         case 'e':
 700                         case 'E':
 701                         default:
 702                                 obstack_1grow(&symbol_obstack, '0');
 703                                 parse_number_dec();
 704                                 return;
 705                 }
 706         } else {
 707                 parse_number_dec();
 708         }
 709 }
 710
 711 /**
 712  * Returns the value of a digit.
 713  * The only portable way to do it ...
 714  */
 715 static int digit_value(int digit) {
 716         switch (digit) {
 717         case '0': return 0;
 718         case '1': return 1;
 719         case '2': return 2;
 720         case '3': return 3;
 721         case '4': return 4;
 722         case '5': return 5;
 723         case '6': return 6;
 724         case '7': return 7;
 725         case '8': return 8;
 726         case '9': return 9;
 727         case 'a':
 728         case 'A': return 10;
 729         case 'b':
 730         case 'B': return 11;
 731         case 'c':
 732         case 'C': return 12;
 733         case 'd':
 734         case 'D': return 13;
 735         case 'e':
 736         case 'E': return 14;
 737         case 'f':
 738         case 'F': return 15;
 739         default:
 740                 internal_error("wrong character given");
 741         }
 742 }
 743
 744 /**
 745  * Parses an octal character sequence.
 746  *
 747  * @param first_digit  the already read first digit
 748  */
 749 static int parse_octal_sequence(const int first_digit)
 750 {
 751         assert(is_octal_digit(first_digit));
 752         int value = digit_value(first_digit);
 753         if (!is_octal_digit(c)) return value;
 754         value = 8 * value + digit_value(c);
 755         next_char();
 756         if (!is_octal_digit(c)) return value;
 757         value = 8 * value + digit_value(c);
 758         next_char();
 759
 760         if(char_is_signed) {
 761                 return (signed char) value;
 762         } else {
 763                 return (unsigned char) value;
 764         }
 765 }
 766
 767 /**
 768  * Parses a hex character sequence.
 769  */
 770 static int parse_hex_sequence(void)
 771 {
 772         int value = 0;
 773         while(isxdigit(c)) {
 774                 value = 16 * value + digit_value(c);
 775                 next_char();
 776         }
 777
 778         if(char_is_signed) {
 779                 return (signed char) value;
 780         } else {
 781                 return (unsigned char) value;
 782         }
 783 }
 784
 785 /**
 786  * Parse an escape sequence.
 787  */
 788 static int parse_escape_sequence(void)
 789 {
 790         eat('\\');
 791
 792         int ec = c;
 793         next_char();
 794
 795         switch (ec) {
 796         case '"':  return '"';
 797         case '\'': return '\'';
 798         case '\\': return '\\';
 799         case '?': return '\?';
 800         case 'a': return '\a';
 801         case 'b': return '\b';
 802         case 'f': return '\f';
 803         case 'n': return '\n';
 804         case 'r': return '\r';
 805         case 't': return '\t';
 806         case 'v': return '\v';
 807         case 'x':
 808                 return parse_hex_sequence();
 809         case '0':
 810         case '1':
 811         case '2':
 812         case '3':
 813         case '4':
 814         case '5':
 815         case '6':
 816         case '7':
 817                 return parse_octal_sequence(ec);
 818         case EOF:
 819                 parse_error("reached end of file while parsing escape sequence");
 820                 return EOF;
 821         case 'e':
 822                 if (c_mode & _GNUC)
 823                         return 27;   /* hopefully 27 is ALWAYS the code for ESACAPE */
 824                 /*fallthrough*/
 825         default:
 826                 parse_error("unknown escape sequence");
 827                 return EOF;
 828         }
 829 }
 830
 831 /**
 832  * Concatenate two strings.
 833  */
 834 string_t concat_strings(const string_t *const s1, const string_t *const s2)
 835 {
 836         const size_t len1 = s1->size - 1;
 837         const size_t len2 = s2->size - 1;
 838
 839         char *const concat = obstack_alloc(&symbol_obstack, len1 + len2 + 1);
 840         memcpy(concat, s1->begin, len1);
 841         memcpy(concat + len1, s2->begin, len2 + 1);
 842
 843 #if 0 /* TODO hash */
 844         const char *result = strset_insert(&stringset, concat);
 845         if(result != concat) {
 846                 obstack_free(&symbol_obstack, concat);
 847         }
 848
 849         return result;
 850 #else
 851         return (string_t){ concat, len1 + len2 + 1 };
 852 #endif
 853 }
 854
 855 /**
 856  * Concatenate a string and a wide string.
 857  */
 858 wide_string_t concat_string_wide_string(const string_t *const s1, const wide_string_t *const s2)
 859 {
 860         const size_t len1 = s1->size - 1;
 861         const size_t len2 = s2->size - 1;
 862
 863         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 864         const char *const src = s1->begin;
 865         for (size_t i = 0; i != len1; ++i) {
 866                 concat[i] = src[i];
 867         }
 868         memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
 869
 870         return (wide_string_t){ concat, len1 + len2 + 1 };
 871 }
 872
 873 /**
 874  * Concatenate two wide strings.
 875  */
 876 wide_string_t concat_wide_strings(const wide_string_t *const s1, const wide_string_t *const s2)
 877 {
 878         const size_t len1 = s1->size - 1;
 879         const size_t len2 = s2->size - 1;
 880
 881         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 882         memcpy(concat,        s1->begin, len1       * sizeof(*concat));
 883         memcpy(concat + len1, s2->begin, (len2 + 1) * sizeof(*concat));
 884
 885         return (wide_string_t){ concat, len1 + len2 + 1 };
 886 }
 887
 888 /**
 889  * Concatenate a wide string and a string.
 890  */
 891 wide_string_t concat_wide_string_string(const wide_string_t *const s1, const string_t *const s2)
 892 {
 893         const size_t len1 = s1->size - 1;
 894         const size_t len2 = s2->size - 1;
 895
 896         wchar_rep_t *const concat = obstack_alloc(&symbol_obstack, (len1 + len2 + 1) * sizeof(*concat));
 897         memcpy(concat, s1->begin, len1 * sizeof(*concat));
 898         const char *const src = s2->begin;
 899         for (size_t i = 0; i != len2 + 1; ++i) {
 900                 concat[i] = src[i];
 901         }
 902
 903         return (wide_string_t){ concat, len1 + len2 + 1 };
 904 }
 905
 906 /**
 907  * Parse a string literal and set lexer_token.
 908  */
 909 static void parse_string_literal(void)
 910 {
 911         const unsigned start_linenr = lexer_token.source_position.linenr;
 912
 913         eat('"');
 914
 915         int tc;
 916         while(1) {
 917                 switch(c) {
 918                 case '\\':
 919                         tc = parse_escape_sequence();
 920                         obstack_1grow(&symbol_obstack, (char) tc);
 921                         break;
 922
 923                 case EOF: {
 924                         source_position_t source_position;
 925                         source_position.input_name = lexer_token.source_position.input_name;
 926                         source_position.linenr     = start_linenr;
 927                         errorf(&source_position, "string has no end");
 928                         lexer_token.type = T_ERROR;
 929                         return;
 930                 }
 931
 932                 case '"':
 933                         next_char();
 934                         goto end_of_string;
 935
 936                 default:
 937                         obstack_1grow(&symbol_obstack, (char) c);
 938                         next_char();
 939                         break;
 940                 }
 941         }
 942
 943 end_of_string:
 944
 945         /* TODO: concatenate multiple strings separated by whitespace... */
 946
 947         /* add finishing 0 to the string */
 948         obstack_1grow(&symbol_obstack, '\0');
 949         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
 950         const char *const string = obstack_finish(&symbol_obstack);
 951
 952 #if 0 /* TODO hash */
 953         /* check if there is already a copy of the string */
 954         result = strset_insert(&stringset, string);
 955         if(result != string) {
 956                 obstack_free(&symbol_obstack, string);
 957         }
 958 #else
 959         const char *const result = string;
 960 #endif
 961
 962         lexer_token.type           = T_STRING_LITERAL;
 963         lexer_token.v.string.begin = result;
 964         lexer_token.v.string.size  = size;
 965 }
 966
 967 /**
 968  * Parse a wide character constant and set lexer_token.
 969  */
 970 static void parse_wide_character_constant(void)
 971 {
 972         const unsigned start_linenr = lexer_token.source_position.linenr;
 973
 974         eat('\'');
 975
 976         while(1) {
 977                 switch(c) {
 978                 case '\\': {
 979                         wchar_rep_t tc = parse_escape_sequence();
 980                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
 981                         break;
 982                 }
 983
 984                 MATCH_NEWLINE(
 985                         parse_error("newline while parsing character constant");
 986                         break;
 987                 )
 988
 989                 case '\'':
 990                         next_char();
 991                         goto end_of_wide_char_constant;
 992
 993                 case EOF: {
 994                         source_position_t source_position = lexer_token.source_position;
 995                         source_position.linenr = start_linenr;
 996                         errorf(&source_position, "EOF while parsing character constant");
 997                         lexer_token.type = T_ERROR;
 998                         return;
 999                 }
1000
1001                 default: {
1002                         wchar_rep_t tc = (wchar_rep_t) c;
1003                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1004                         next_char();
1005                         break;
1006                 }
1007                 }
1008         }
1009
1010 end_of_wide_char_constant:;
1011         size_t             size   = (size_t) obstack_object_size(&symbol_obstack);
1012         assert(size % sizeof(wchar_rep_t) == 0);
1013         size /= sizeof(wchar_rep_t);
1014
1015         const wchar_rep_t *string = obstack_finish(&symbol_obstack);
1016
1017         lexer_token.type                = T_WIDE_CHARACTER_CONSTANT;
1018         lexer_token.v.wide_string.begin = string;
1019         lexer_token.v.wide_string.size  = size;
1020         lexer_token.datatype            = type_wchar_t;
1021 }
1022
1023 /**
1024  * Parse a wide string literal and set lexer_token.
1025  */
1026 static void parse_wide_string_literal(void)
1027 {
1028         const unsigned start_linenr = lexer_token.source_position.linenr;
1029
1030         assert(c == '"');
1031         next_char();
1032
1033         while(1) {
1034                 switch(c) {
1035                 case '\\': {
1036                         wchar_rep_t tc = parse_escape_sequence();
1037                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1038                         break;
1039                 }
1040
1041                 case EOF: {
1042                         source_position_t source_position;
1043                         source_position.input_name = lexer_token.source_position.input_name;
1044                         source_position.linenr     = start_linenr;
1045                         errorf(&source_position, "string has no end");
1046                         lexer_token.type = T_ERROR;
1047                         return;
1048                 }
1049
1050                 case '"':
1051                         next_char();
1052                         goto end_of_string;
1053
1054                 default: {
1055                         wchar_rep_t tc = c;
1056                         obstack_grow(&symbol_obstack, &tc, sizeof(tc));
1057                         next_char();
1058                         break;
1059                 }
1060                 }
1061         }
1062
1063 end_of_string:;
1064
1065         /* TODO: concatenate multiple strings separated by whitespace... */
1066
1067         /* add finishing 0 to the string */
1068         wchar_rep_t nul = L'\0';
1069         obstack_grow(&symbol_obstack, &nul, sizeof(nul));
1070         const size_t             size   = (size_t)obstack_object_size(&symbol_obstack) / sizeof(wchar_rep_t);
1071         const wchar_rep_t *const string = obstack_finish(&symbol_obstack);
1072
1073 #if 0 /* TODO hash */
1074         /* check if there is already a copy of the string */
1075         const wchar_rep_t *const result = strset_insert(&stringset, string);
1076         if(result != string) {
1077                 obstack_free(&symbol_obstack, string);
1078         }
1079 #else
1080         const wchar_rep_t *const result = string;
1081 #endif
1082
1083         lexer_token.type                = T_WIDE_STRING_LITERAL;
1084         lexer_token.v.wide_string.begin = result;
1085         lexer_token.v.wide_string.size  = size;
1086 }
1087
1088 /**
1089  * Parse a character constant and set lexer_token.
1090  */
1091 static void parse_character_constant(void)
1092 {
1093         const unsigned start_linenr = lexer_token.source_position.linenr;
1094
1095         eat('\'');
1096
1097         while(1) {
1098                 switch(c) {
1099                 case '\\': {
1100                         int tc = parse_escape_sequence();
1101                         obstack_1grow(&symbol_obstack, (char) tc);
1102                         break;
1103                 }
1104
1105                 MATCH_NEWLINE(
1106                         parse_error("newline while parsing character constant");
1107                         break;
1108                 )
1109
1110                 case '\'':
1111                         next_char();
1112                         goto end_of_char_constant;
1113
1114                 case EOF: {
1115                         source_position_t source_position;
1116                         source_position.input_name = lexer_token.source_position.input_name;
1117                         source_position.linenr     = start_linenr;
1118                         errorf(&source_position, "EOF while parsing character constant");
1119                         lexer_token.type = T_ERROR;
1120                         return;
1121                 }
1122
1123                 default:
1124                         obstack_1grow(&symbol_obstack, (char) c);
1125                         next_char();
1126                         break;
1127
1128                 }
1129         }
1130
1131 end_of_char_constant:;
1132         const size_t      size   = (size_t)obstack_object_size(&symbol_obstack);
1133         const char *const string = obstack_finish(&symbol_obstack);
1134
1135         lexer_token.type           = T_CHARACTER_CONSTANT;
1136         lexer_token.v.string.begin = string;
1137         lexer_token.v.string.size  = size;
1138         lexer_token.datatype       = type_int;
1139 }
1140
1141 /**
1142  * Skip a multiline comment.
1143  */
1144 static void skip_multiline_comment(void)
1145 {
1146         unsigned start_linenr = lexer_token.source_position.linenr;
1147
1148         while(1) {
1149                 switch(c) {
1150                 case '/':
1151                         next_char();
1152                         if (c == '*') {
1153                                 /* nested comment, warn here */
1154                                 if (warning.comment) {
1155                                         warningf(&lexer_token.source_position, "'/*' within comment");
1156                                 }
1157                         }
1158                         break;
1159                 case '*':
1160                         next_char();
1161                         if(c == '/') {
1162                                 next_char();
1163                                 return;
1164                         }
1165                         break;
1166
1167                 MATCH_NEWLINE(break;)
1168
1169                 case EOF: {
1170                         source_position_t source_position;
1171                         source_position.input_name = lexer_token.source_position.input_name;
1172                         source_position.linenr     = start_linenr;
1173                         errorf(&source_position, "at end of file while looking for comment end");
1174                         return;
1175                 }
1176
1177                 default:
1178                         next_char();
1179                         break;
1180                 }
1181         }
1182 }
1183
1184 /**
1185  * Skip a single line comment.
1186  */
1187 static void skip_line_comment(void)
1188 {
1189         while(1) {
1190                 switch(c) {
1191                 case EOF:
1192                         return;
1193
1194                 case '\n':
1195                 case '\r':
1196                         return;
1197
1198                 case '\\':
1199                         next_char();
1200                         if (c == '\n' || c == '\r') {
1201                                 if (warning.comment)
1202                                         warningf(&lexer_token.source_position, "multi-line comment");
1203                                 return;
1204                         }
1205                         break;
1206
1207                 default:
1208                         next_char();
1209                         break;
1210                 }
1211         }
1212 }
1213
1214 /** The current preprocessor token. */
1215 static token_t pp_token;
1216
1217 /**
1218  * Read the next preprocessor token.
1219  */
1220 static inline void next_pp_token(void)
1221 {
1222         lexer_next_preprocessing_token();
1223         pp_token = lexer_token;
1224 }
1225
1226 /**
1227  * Eat all preprocessor tokens until newline.
1228  */
1229 static void eat_until_newline(void)
1230 {
1231         while(pp_token.type != '\n' && pp_token.type != T_EOF) {
1232                 next_pp_token();
1233         }
1234 }
1235
1236 /**
1237  * Handle the define directive.
1238  */
1239 static void define_directive(void)
1240 {
1241         lexer_next_preprocessing_token();
1242         if(lexer_token.type != T_IDENTIFIER) {
1243                 parse_error("expected identifier after #define\n");
1244                 eat_until_newline();
1245         }
1246 }
1247
1248 /**
1249  * Handle the ifdef directive.
1250  */
1251 static void ifdef_directive(int is_ifndef)
1252 {
1253         (void) is_ifndef;
1254         lexer_next_preprocessing_token();
1255         //expect_identifier();
1256         //extect_newline();
1257 }
1258
1259 /**
1260  * Handle the endif directive.
1261  */
1262 static void endif_directive(void)
1263 {
1264         //expect_newline();
1265 }
1266
1267 /**
1268  * Parse the line directive.
1269  */
1270 static void parse_line_directive(void)
1271 {
1272         if(pp_token.type != T_INTEGER) {
1273                 parse_error("expected integer");
1274         } else {
1275                 lexer_token.source_position.linenr = (unsigned int)(pp_token.v.intvalue - 1);
1276                 next_pp_token();
1277         }
1278         if(pp_token.type == T_STRING_LITERAL) {
1279                 lexer_token.source_position.input_name = pp_token.v.string.begin;
1280                 next_pp_token();
1281         }
1282
1283         eat_until_newline();
1284 }
1285
1286 /**
1287  * STDC pragmas.
1288  */
1289 typedef enum stdc_pragma_kind_t {
1290         STDC_UNKNOWN,
1291         STDC_FP_CONTRACT,
1292         STDC_FENV_ACCESS,
1293         STDC_CX_LIMITED_RANGE
1294 } stdc_pragma_kind_t;
1295
1296 /**
1297  * STDC pragma values.
1298  */
1299 typedef enum stdc_pragma_value_kind_t {
1300         STDC_VALUE_UNKNOWN,
1301         STDC_VALUE_ON,
1302         STDC_VALUE_OFF,
1303         STDC_VALUE_DEFAULT
1304 } stdc_pragma_value_kind_t;
1305
1306 /**
1307  * Parse a pragma directive.
1308  */
1309 static void parse_pragma(void) {
1310         bool unknown_pragma = true;
1311
1312         next_pp_token();
1313         if (pp_token.v.symbol->pp_ID == TP_STDC) {
1314                 stdc_pragma_kind_t kind = STDC_UNKNOWN;
1315                 /* a STDC pragma */
1316                 if (c_mode & _C99) {
1317                         next_pp_token();
1318
1319                         switch (pp_token.v.symbol->pp_ID) {
1320                         case TP_FP_CONTRACT:
1321                                 kind = STDC_FP_CONTRACT;
1322                                 break;
1323                         case TP_FENV_ACCESS:
1324                                 kind = STDC_FENV_ACCESS;
1325                                 break;
1326                         case TP_CX_LIMITED_RANGE:
1327                                 kind = STDC_CX_LIMITED_RANGE;
1328                                 break;
1329                         default:
1330                                 break;
1331                         }
1332                         if (kind != STDC_UNKNOWN) {
1333                                 stdc_pragma_value_kind_t value = STDC_VALUE_UNKNOWN;
1334                                 next_pp_token();
1335                                 switch (pp_token.v.symbol->pp_ID) {
1336                                 case TP_ON:
1337                                         value = STDC_VALUE_ON;
1338                                         break;
1339                                 case TP_OFF:
1340                                         value = STDC_VALUE_OFF;
1341                                         break;
1342                                 case TP_DEFAULT:
1343                                         value = STDC_VALUE_DEFAULT;
1344                                         break;
1345                                 default:
1346                                         break;
1347                                 }
1348                                 if (value != STDC_VALUE_UNKNOWN) {
1349                                         unknown_pragma = false;
1350                                 } else {
1351                                         errorf(&pp_token.source_position, "bad STDC pragma argument");
1352                                 }
1353                         }
1354                 }
1355         } else {
1356                 unknown_pragma = true;
1357         }
1358         eat_until_newline();
1359         if (unknown_pragma && warning.unknown_pragmas) {
1360                 warningf(&pp_token.source_position, "encountered unknown #pragma");
1361         }
1362 }
1363
1364 /**
1365  * Parse a preprocessor non-null directive.
1366  */
1367 static void parse_preprocessor_identifier(void)
1368 {
1369         assert(pp_token.type == T_IDENTIFIER);
1370         symbol_t *symbol = pp_token.v.symbol;
1371
1372         switch(symbol->pp_ID) {
1373         case TP_include:
1374                 printf("include - enable header name parsing!\n");
1375                 break;
1376         case TP_define:
1377                 define_directive();
1378                 break;
1379         case TP_ifdef:
1380                 ifdef_directive(0);
1381                 break;
1382         case TP_ifndef:
1383                 ifdef_directive(1);
1384                 break;
1385         case TP_endif:
1386                 endif_directive();
1387                 break;
1388         case TP_line:
1389                 next_pp_token();
1390                 parse_line_directive();
1391                 break;
1392         case TP_if:
1393         case TP_else:
1394         case TP_elif:
1395         case TP_undef:
1396         case TP_error:
1397                 /* TODO; output the rest of the line */
1398                 parse_error("#error directive: ");
1399                 break;
1400         case TP_pragma:
1401                 parse_pragma();
1402                 break;
1403         }
1404 }
1405
1406 /**
1407  * Parse a preprocessor directive.
1408  */
1409 static void parse_preprocessor_directive(void)
1410 {
1411         next_pp_token();
1412
1413         switch(pp_token.type) {
1414         case T_IDENTIFIER:
1415                 parse_preprocessor_identifier();
1416                 break;
1417         case T_INTEGER:
1418                 parse_line_directive();
1419                 break;
1420         case '\n':
1421                 /* NULL directive, see § 6.10.7 */
1422                 break;
1423         default:
1424                 parse_error("invalid preprocessor directive");
1425                 eat_until_newline();
1426                 break;
1427         }
1428 }
1429
1430 #define MAYBE_PROLOG                                       \
1431                         next_char();                                   \
1432                         while(1) {                                     \
1433                                 switch(c) {
1434
1435 #define MAYBE(ch, set_type)                                \
1436                                 case ch:                                   \
1437                                         next_char();                           \
1438                                         lexer_token.type = set_type;           \
1439                                         return;
1440
1441 #define ELSE_CODE(code)                                    \
1442                                 default:                                   \
1443                                         code                                   \
1444                                 }                                          \
1445                         } /* end of while(1) */                        \
1446                         break;
1447
1448 #define ELSE(set_type)                                     \
1449                 ELSE_CODE(                                         \
1450                         lexer_token.type = set_type;                   \
1451                         return;                                        \
1452                 )
1453
1454 void lexer_next_preprocessing_token(void)
1455 {
1456         while(1) {
1457                 switch(c) {
1458                 case ' ':
1459                 case '\t':
1460                         next_char();
1461                         break;
1462
1463                 MATCH_NEWLINE(
1464                         lexer_token.type = '\n';
1465                         return;
1466                 )
1467
1468                 SYMBOL_CHARS
1469                         parse_symbol();
1470                         /* might be a wide string ( L"string" ) */
1471                         if(lexer_token.type == T_IDENTIFIER &&
1472                             lexer_token.v.symbol == symbol_L) {
1473                             if(c == '"') {
1474                                         parse_wide_string_literal();
1475                                 } else if(c == '\'') {
1476                                         parse_wide_character_constant();
1477                                 }
1478                         }
1479                         return;
1480
1481                 DIGITS
1482                         parse_number();
1483                         return;
1484
1485                 case '"':
1486                         parse_string_literal();
1487                         return;
1488
1489                 case '\'':
1490                         parse_character_constant();
1491                         return;
1492
1493                 case '.':
1494                         MAYBE_PROLOG
1495                                 DIGITS
1496                                         put_back(c);
1497                                         c = '.';
1498                                         parse_number_dec();
1499                                         return;
1500
1501                                 case '.':
1502                                         MAYBE_PROLOG
1503                                         MAYBE('.', T_DOTDOTDOT)
1504                                         ELSE_CODE(
1505                                                 put_back(c);
1506                                                 c = '.';
1507                                                 lexer_token.type = '.';
1508                                                 return;
1509                                         )
1510                         ELSE('.')
1511                 case '&':
1512                         MAYBE_PROLOG
1513                         MAYBE('&', T_ANDAND)
1514                         MAYBE('=', T_ANDEQUAL)
1515                         ELSE('&')
1516                 case '*':
1517                         MAYBE_PROLOG
1518                         MAYBE('=', T_ASTERISKEQUAL)
1519                         ELSE('*')
1520                 case '+':
1521                         MAYBE_PROLOG
1522                         MAYBE('+', T_PLUSPLUS)
1523                         MAYBE('=', T_PLUSEQUAL)
1524                         ELSE('+')
1525                 case '-':
1526                         MAYBE_PROLOG
1527                         MAYBE('>', T_MINUSGREATER)
1528                         MAYBE('-', T_MINUSMINUS)
1529                         MAYBE('=', T_MINUSEQUAL)
1530                         ELSE('-')
1531                 case '!':
1532                         MAYBE_PROLOG
1533                         MAYBE('=', T_EXCLAMATIONMARKEQUAL)
1534                         ELSE('!')
1535                 case '/':
1536                         MAYBE_PROLOG
1537                         MAYBE('=', T_SLASHEQUAL)
1538                                 case '*':
1539                                         next_char();
1540                                         skip_multiline_comment();
1541                                         lexer_next_preprocessing_token();
1542                                         return;
1543                                 case '/':
1544                                         next_char();
1545                                         skip_line_comment();
1546                                         lexer_next_preprocessing_token();
1547                                         return;
1548                         ELSE('/')
1549                 case '%':
1550                         MAYBE_PROLOG
1551                         MAYBE('>', '}')
1552                         MAYBE('=', T_PERCENTEQUAL)
1553                                 case ':':
1554                                         MAYBE_PROLOG
1555                                                 case '%':
1556                                                         MAYBE_PROLOG
1557                                                         MAYBE(':', T_HASHHASH)
1558                                                         ELSE_CODE(
1559                                                                 put_back(c);
1560                                                                 c = '%';
1561                                                                 lexer_token.type = '#';
1562                                                                 return;
1563                                                         )
1564                                         ELSE('#')
1565                         ELSE('%')
1566                 case '<':
1567                         MAYBE_PROLOG
1568                         MAYBE(':', '[')
1569                         MAYBE('%', '{')
1570                         MAYBE('=', T_LESSEQUAL)
1571                                 case '<':
1572                                         MAYBE_PROLOG
1573                                         MAYBE('=', T_LESSLESSEQUAL)
1574                                         ELSE(T_LESSLESS)
1575                         ELSE('<')
1576                 case '>':
1577                         MAYBE_PROLOG
1578                         MAYBE('=', T_GREATEREQUAL)
1579                                 case '>':
1580                                         MAYBE_PROLOG
1581                                         MAYBE('=', T_GREATERGREATEREQUAL)
1582                                         ELSE(T_GREATERGREATER)
1583                         ELSE('>')
1584                 case '^':
1585                         MAYBE_PROLOG
1586                         MAYBE('=', T_CARETEQUAL)
1587                         ELSE('^')
1588                 case '|':
1589                         MAYBE_PROLOG
1590                         MAYBE('=', T_PIPEEQUAL)
1591                         MAYBE('|', T_PIPEPIPE)
1592                         ELSE('|')
1593                 case ':':
1594                         MAYBE_PROLOG
1595                         MAYBE('>', ']')
1596                         ELSE(':')
1597                 case '=':
1598                         MAYBE_PROLOG
1599                         MAYBE('=', T_EQUALEQUAL)
1600                         ELSE('=')
1601                 case '#':
1602                         MAYBE_PROLOG
1603                         MAYBE('#', T_HASHHASH)
1604                         ELSE('#')
1605
1606                 case '?':
1607                 case '[':
1608                 case ']':
1609                 case '(':
1610                 case ')':
1611                 case '{':
1612                 case '}':
1613                 case '~':
1614                 case ';':
1615                 case ',':
1616                 case '\\':
1617                         lexer_token.type = c;
1618                         next_char();
1619                         return;
1620
1621                 case EOF:
1622                         lexer_token.type = T_EOF;
1623                         return;
1624
1625                 default:
1626 dollar_sign:
1627                         errorf(&lexer_token.source_position, "unknown character '%c' found", c);
1628                         next_char();
1629                         lexer_token.type = T_ERROR;
1630                         return;
1631                 }
1632         }
1633 }
1634
1635 void lexer_next_token(void)
1636 {
1637         lexer_next_preprocessing_token();
1638
1639         while (lexer_token.type == '\n') {
1640 newline_found:
1641                 lexer_next_preprocessing_token();
1642         }
1643
1644         if (lexer_token.type == '#') {
1645                 parse_preprocessor_directive();
1646                 goto newline_found;
1647         }
1648 }
1649
1650 void init_lexer(void)
1651 {
1652         strset_init(&stringset);
1653         symbol_L = symbol_table_insert("L");
1654 }
1655
1656 void lexer_open_stream(FILE *stream, const char *input_name)
1657 {
1658         input                                  = stream;
1659         lexer_token.source_position.linenr     = 0;
1660         lexer_token.source_position.input_name = input_name;
1661
1662         bufpos = NULL;
1663         bufend = NULL;
1664
1665         /* place a virtual \n at the beginning so the lexer knows that we're
1666          * at the beginning of a line */
1667         c = '\n';
1668 }
1669
1670 void lexer_open_buffer(const char *buffer, size_t len, const char *input_name)
1671 {
1672         input                                  = NULL;
1673         lexer_token.source_position.linenr     = 0;
1674         lexer_token.source_position.input_name = input_name;
1675
1676         bufpos = buffer;
1677         bufend = buffer + len;
1678
1679         /* place a virtual \n at the beginning so the lexer knows that we're
1680          * at the beginning of a line */
1681         c = '\n';
1682 }
1683
1684 void exit_lexer(void)
1685 {
1686         strset_destroy(&stringset);
1687 }
1688
1689 static __attribute__((unused))
1690 void dbg_pos(const source_position_t source_position)
1691 {
1692         fprintf(stdout, "%s:%u\n", source_position.input_name,
1693                 source_position.linenr);
1694         fflush(stdout);
1695 }